indexx.dev / tweets2bsky
forked from j4ck.xyz/tweets2bsky
A simple tool which lets you scrape twitter accounts and crosspost them to bluesky accounts! Comes with a CLI and a webapp for managing profiles! Works with images/videos/link embeds/threads.
fork atom

feat: improve image quality and add retry logic

- Download Twitter images at original quality with :orig suffix
- Fall back to standard quality if high-res download fails
- Increase JPEG quality from 85% to 92% for better quality
- Preserve PNG transparency with max compression level
- Add 10 second fixed pacing delay between posts
- Add logic tests for image processing and retry behavior

jack 4d3c5eab ddfed152

+362 -37
unified split
+80 -37
src/index.ts
··· 291 291 url, 292 292 method: 'GET', 293 293 responseType: 'arraybuffer', 294 + timeout: 30000, 294 295 }); 295 296 return { 296 297 buffer: Buffer.from(response.data as ArrayBuffer), ··· 301 302 async function uploadToBluesky(agent: BskyAgent, buffer: Buffer, mimeType: string): Promise<BlobRef> { 302 303 let finalBuffer = buffer; 303 304 let finalMimeType = mimeType; 304 - const MAX_SIZE = 950 * 1024; // 950KB safety margin 305 + const MAX_SIZE = 950 * 1024; 306 + 307 + const isPng = mimeType === 'image/png'; 308 + const isJpeg = mimeType === 'image/jpeg' || mimeType === 'image/jpg'; 309 + const isWebp = mimeType === 'image/webp'; 310 + const isGif = mimeType === 'image/gif'; 311 + const isAnimation = isGif || isWebp; 305 312 306 - if (buffer.length > MAX_SIZE && (mimeType.startsWith('image/') || mimeType === 'application/octet-stream')) { 307 - console.log(`[UPLOAD] ⚖️ Image too large (${(buffer.length / 1024).toFixed(2)} KB). Compressing...`); 313 + if ((buffer.length > MAX_SIZE && (mimeType.startsWith('image/') || mimeType === 'application/octet-stream')) || (isPng && buffer.length > MAX_SIZE)) { 314 + console.log(`[UPLOAD] ⚖️ Image too large (${(buffer.length / 1024).toFixed(2)} KB). Optimizing...`); 308 315 try { 309 - const image = sharp(buffer); 316 + let image = sharp(buffer); 310 317 const metadata = await image.metadata(); 311 - 312 - let pipeline = image; 313 - // If it's a very large resolution, downscale it slightly 314 - if (metadata.width && metadata.width > 2000) { 315 - pipeline = pipeline.resize(2000, undefined, { withoutEnlargement: true }); 318 + 319 + if (isAnimation) { 320 + console.log(`[UPLOAD] 🖼️ Preserving animation format.`); 321 + if (isWebp && buffer.length > MAX_SIZE) { 322 + image = image.webp({ quality: 90, effort: 6 }); 323 + } 324 + } else { 325 + if (metadata.width && metadata.width > 2000) { 326 + image = image.resize(2000, undefined, { withoutEnlargement: true }); 327 + } 328 + 329 + if (isPng) { 330 + if (metadata.hasAlpha) { 331 + image = image.png({ compressionLevel: 9, adaptiveFiltering: true }); 332 + } else { 333 + image = image.jpeg({ quality: 92, mozjpeg: true }); 334 + finalMimeType = 'image/jpeg'; 335 + } 336 + } else if (isJpeg) { 337 + if (buffer.length > MAX_SIZE) { 338 + image = image.jpeg({ quality: 92, mozjpeg: true }); 339 + } 340 + } else { 341 + image = image.jpeg({ quality: 92, mozjpeg: true }); 342 + finalMimeType = 'image/jpeg'; 343 + } 316 344 } 317 345 318 - finalBuffer = await pipeline 319 - .jpeg({ quality: 85, mozjpeg: true }) 320 - .toBuffer(); 321 - finalMimeType = 'image/jpeg'; 322 - 323 - console.log(`[UPLOAD] ✅ Compressed to ${(finalBuffer.length / 1024).toFixed(2)} KB`); 324 - 325 - // If still too large, aggressive compression 326 - if (finalBuffer.length > MAX_SIZE) { 327 - finalBuffer = await sharp(buffer) 328 - .resize(1200, undefined, { withoutEnlargement: true }) 329 - .jpeg({ quality: 70 }) 330 - .toBuffer(); 331 - console.log(`[UPLOAD] ⚠️ Required aggressive compression: ${(finalBuffer.length / 1024).toFixed(2)} KB`); 346 + finalBuffer = await image.toBuffer(); 347 + console.log(`[UPLOAD] ✅ Optimized to ${(finalBuffer.length / 1024).toFixed(2)} KB`); 348 + 349 + if (finalBuffer.length > MAX_SIZE && !isAnimation) { 350 + console.log(`[UPLOAD] ⚠️ Still large, trying higher compression...`); 351 + const pipeline = sharp(buffer); 352 + const md = await pipeline.metadata(); 353 + 354 + if (md.width && md.width > 1600) { 355 + pipeline.resize(1600, undefined, { withoutEnlargement: true }); 356 + } 357 + 358 + if (mimeType === 'image/png' && md.hasAlpha) { 359 + finalBuffer = await pipeline.png({ compressionLevel: 9 }).toBuffer(); 360 + finalMimeType = 'image/png'; 361 + } else { 362 + finalBuffer = await pipeline.jpeg({ quality: 85, mozjpeg: true }).toBuffer(); 363 + finalMimeType = 'image/jpeg'; 364 + } 365 + console.log(`[UPLOAD] ✅ Further compressed to ${(finalBuffer.length / 1024).toFixed(2)} KB`); 332 366 } 333 367 } catch (err) { 334 - console.warn(`[UPLOAD] ⚠️ Compression failed, attempting original upload:`, (err as Error).message); 368 + console.warn(`[UPLOAD] ⚠️ Optimization failed, attempting original upload:`, (err as Error).message); 369 + finalBuffer = buffer; 370 + finalMimeType = mimeType; 335 371 } 336 372 } 337 373 ··· 535 571 console.error(`[VIDEO] ❌ Error in uploadVideoToBluesky:`, (err as Error).message); 536 572 throw err; 537 573 } 538 - } 539 - 540 - function getRandomDelay(min = 1000, max = 4000): number { 541 - return Math.floor(Math.random() * (max - min + 1) + min); 542 574 } 543 575 544 576 function splitText(text: string, limit = 300): string[] { ··· 737 769 const url = media.media_url_https; 738 770 if (!url) continue; 739 771 try { 740 - console.log(`[${twitterUsername}] 📥 Downloading image: ${url}`); 741 - updateAppStatus({ message: `Downloading image: ${path.basename(url)}` }); 742 - const { buffer, mimeType } = await downloadMedia(url); 772 + const highQualityUrl = url.includes('?') ? url.replace('?', ':orig?') : url + ':orig'; 773 + console.log(`[${twitterUsername}] 📥 Downloading image (high quality): ${path.basename(highQualityUrl)}`); 774 + updateAppStatus({ message: `Downloading high quality image...` }); 775 + const { buffer, mimeType } = await downloadMedia(highQualityUrl); 743 776 console.log(`[${twitterUsername}] 📤 Uploading image to Bluesky...`); 744 777 updateAppStatus({ message: `Uploading image to Bluesky...` }); 745 778 const blob = await uploadToBluesky(agent, buffer, mimeType); 746 779 images.push({ alt: media.ext_alt_text || 'Image from Twitter', image: blob, aspectRatio }); 747 780 console.log(`[${twitterUsername}] ✅ Image uploaded.`); 748 781 } catch (err) { 749 - console.error(`[${twitterUsername}] ❌ Failed to upload image ${url}:`, (err as Error).message); 782 + console.error(`[${twitterUsername}] ❌ High quality upload failed:`, (err as Error).message); 783 + try { 784 + console.log(`[${twitterUsername}] 🔄 Retrying with standard quality...`); 785 + updateAppStatus({ message: `Retrying with standard quality...` }); 786 + const { buffer, mimeType } = await downloadMedia(url); 787 + const blob = await uploadToBluesky(agent, buffer, mimeType); 788 + images.push({ alt: media.ext_alt_text || 'Image from Twitter', image: blob, aspectRatio }); 789 + console.log(`[${twitterUsername}] ✅ Image uploaded on retry.`); 790 + } catch (retryErr) { 791 + console.error(`[${twitterUsername}] ❌ Retry also failed:`, (retryErr as Error).message); 792 + } 750 793 } 751 794 } else if (media.type === 'video' || media.type === 'animated_gif') { 752 795 const variants = media.video_info?.variants || []; ··· 915 958 console.log(`[${twitterUsername}] ✅ Chunk ${i + 1} posted successfully.`); 916 959 917 960 if (chunks.length > 1) { 918 - await new Promise((r) => setTimeout(r, 2000)); 961 + await new Promise((r) => setTimeout(r, 3000)); 919 962 } 920 963 } catch (err) { 921 964 console.error(`[${twitterUsername}] ❌ Failed to post ${tweetId} (chunk ${i + 1}):`, err); ··· 923 966 } 924 967 } 925 968 926 - const wait = getRandomDelay(2000, 5000); 927 - console.log(`[${twitterUsername}] 😴 Pacing: Waiting ${wait}ms before next tweet.`); 928 - updateAppStatus({ state: 'pacing', message: `Pacing: Waiting ${Math.round(wait/1000)}s...` }); 969 + const wait = 10000; 970 + console.log(`[${twitterUsername}] 😴 Pacing: Waiting ${wait / 1000}s before next tweet.`); 971 + updateAppStatus({ state: 'pacing', message: `Pacing: Waiting ${wait / 1000}s...` }); 929 972 await new Promise((r) => setTimeout(r, wait)); 930 973 } 931 974 } ··· 1055 1098 1056 1099 if (limit && allFoundTweets.length >= limit) break; 1057 1100 1058 - await new Promise((r) => setTimeout(r, 2000)); 1101 + await new Promise((r) => setTimeout(r, 5000)); 1059 1102 } 1060 1103 1061 1104 console.log(`Fetch complete. Found ${allFoundTweets.length} new tweets to import.`);
+282
src/run-tests.js
··· 1 + #!/usr/bin/env node 2 + 3 + let testsPassed = 0; 4 + let testsFailed = 0; 5 + 6 + function assert(condition, message) { 7 + if (condition) { 8 + console.log(` ✓ ${message}`); 9 + testsPassed++; 10 + } else { 11 + console.log(` ✗ ${message}`); 12 + testsFailed++; 13 + } 14 + } 15 + 16 + console.log('Running logic tests...\n'); 17 + 18 + // Test 1: Twitter URL Manipulation 19 + console.log('Test 1: Twitter URL Manipulation (High Quality Download)'); 20 + { 21 + const url1 = 'https://pbs.twimg.com/media/ABC123.jpg'; 22 + const highQuality1 = url1.includes('?') ? url1.replace('?', ':orig?') : url1 + ':orig'; 23 + assert(highQuality1 === 'https://pbs.twimg.com/media/ABC123.jpg:orig', 'Should append :orig to plain URLs'); 24 + 25 + const url2 = 'https://pbs.twimg.com/media/ABC123.jpg?format=jpg&name=small'; 26 + const highQuality2 = url2.includes('?') ? url2.replace('?', ':orig?') : url2 + ':orig'; 27 + assert(highQuality2 === 'https://pbs.twimg.com/media/ABC123.jpg:orig?format=jpg&name=small', 'Should replace ? with :orig? for query URLs'); 28 + 29 + const url3 = 'https://pbs.twimg.com/media/DEF456.png?name=large'; 30 + const highQuality3 = url3.includes('?') ? url3.replace('?', ':orig?') : url3 + ':orig'; 31 + assert(highQuality3 === 'https://pbs.twimg.com/media/DEF456.png:orig?name=large', 'Should work with PNGs too'); 32 + console.log(); 33 + } 34 + 35 + // Test 2: Text Splitting Logic 36 + console.log('Test 2: Text Splitting Logic'); 37 + { 38 + function splitText(text, limit = 300) { 39 + if (text.length <= limit) return [text]; 40 + const chunks = []; 41 + let remaining = text; 42 + while (remaining.length > 0) { 43 + if (remaining.length <= limit) { 44 + chunks.push(remaining); 45 + break; 46 + } 47 + let splitIndex = remaining.lastIndexOf('\n\n', limit); 48 + if (splitIndex === -1) { 49 + splitIndex = remaining.lastIndexOf('. ', limit); 50 + if (splitIndex === -1) { 51 + splitIndex = remaining.lastIndexOf(' ', limit); 52 + if (splitIndex === -1) { 53 + splitIndex = limit; 54 + } 55 + } else { 56 + splitIndex += 1; 57 + } 58 + } 59 + chunks.push(remaining.substring(0, splitIndex).trim()); 60 + remaining = remaining.substring(splitIndex).trim(); 61 + } 62 + return chunks; 63 + } 64 + 65 + const text1 = 'Hello world'; 66 + const result1 = splitText(text1, 300); 67 + assert(result1.length === 1, 'Short text should not be split'); 68 + assert(result1[0] === 'Hello world', 'Content should be preserved'); 69 + 70 + const text2 = 'First paragraph.\n\nSecond paragraph.\n\nThird paragraph.'; 71 + const result2 = splitText(text2, 50); 72 + assert(result2.length >= 2, `Should split at paragraph breaks (got ${result2.length} chunks)`); 73 + const allHaveContent = result2.every(c => c.length > 0); 74 + assert(allHaveContent, 'All chunks have content'); 75 + console.log(); 76 + } 77 + 78 + // Test 3: MIME Type Detection 79 + console.log('Test 3: MIME Type Detection'); 80 + { 81 + const isPng = (mimeType) => mimeType === 'image/png'; 82 + const isJpeg = (mimeType) => mimeType === 'image/jpeg' || mimeType === 'image/jpg'; 83 + const isWebp = (mimeType) => mimeType === 'image/webp'; 84 + const isGif = (mimeType) => mimeType === 'image/gif'; 85 + const isAnimation = (mimeType) => isGif(mimeType) || isWebp(mimeType); 86 + 87 + assert(isPng('image/png') === true, 'PNG detection works'); 88 + assert(isPng('image/jpeg') === false, 'JPEG is not PNG'); 89 + assert(isJpeg('image/jpeg') === true, 'JPEG detection works'); 90 + assert(isJpeg('image/jpg') === true, 'JPEG with JPG extension detection works'); 91 + assert(isWebp('image/webp') === true, 'WebP detection works'); 92 + assert(isGif('image/gif') === true, 'GIF detection works'); 93 + assert(isAnimation('image/webp') === true, 'WebP is animation'); 94 + assert(isAnimation('image/gif') === true, 'GIF is animation'); 95 + assert(isAnimation('image/jpeg') === false, 'JPEG is not animation'); 96 + console.log(); 97 + } 98 + 99 + // Test 4: Aspect Ratio Calculation 100 + console.log('Test 4: Aspect Ratio Calculation'); 101 + { 102 + const sizes = { 103 + large: { w: 1200, h: 800 }, 104 + medium: { w: 600, h: 400 }, 105 + small: { w: 300, h: 200 } 106 + }; 107 + 108 + const getAspectRatio = (mediaSizes, originalInfo) => { 109 + if (mediaSizes?.large) { 110 + return { width: mediaSizes.large.w, height: mediaSizes.large.h }; 111 + } else if (originalInfo) { 112 + return { width: originalInfo.width, height: originalInfo.height }; 113 + } 114 + return undefined; 115 + }; 116 + 117 + const ratio1 = getAspectRatio(sizes, undefined); 118 + assert(ratio1.width === 1200 && ratio1.height === 800, 'Uses large size when available'); 119 + 120 + const ratio2 = getAspectRatio(undefined, { width: 1920, height: 1080 }); 121 + assert(ratio2.width === 1920 && ratio2.height === 1080, 'Falls back to original_info'); 122 + 123 + const ratio3 = getAspectRatio(undefined, undefined); 124 + assert(ratio3 === undefined, 'Returns undefined when no data'); 125 + console.log(); 126 + } 127 + 128 + // Test 5: Video Variant Sorting 129 + console.log('Test 5: Video Variant Sorting (Highest Quality First)'); 130 + { 131 + const variants = [ 132 + { content_type: 'video/mp4', url: 'low.mp4', bitrate: 500000 }, 133 + { content_type: 'video/mp4', url: 'high.mp4', bitrate: 2000000 }, 134 + { content_type: 'video/mp4', url: 'medium.mp4', bitrate: 1000000 }, 135 + { content_type: 'audio/mp4', url: 'audio.mp4', bitrate: 128000 } 136 + ]; 137 + 138 + const mp4s = variants 139 + .filter((v) => v.content_type === 'video/mp4') 140 + .sort((a, b) => (b.bitrate || 0) - (a.bitrate || 0)); 141 + 142 + assert(mp4s.length === 3, 'Should filter to only MP4 videos'); 143 + assert(mp4s[0].url === 'high.mp4', 'Highest bitrate first'); 144 + assert(mp4s[1].url === 'medium.mp4', 'Medium bitrate second'); 145 + assert(mp4s[2].url === 'low.mp4', 'Low bitrate last'); 146 + console.log(); 147 + } 148 + 149 + // Test 6: Size Formatting 150 + console.log('Test 6: Size Formatting'); 151 + { 152 + const formatSize = (bytes) => (bytes / 1024).toFixed(2) + ' KB'; 153 + const formatSizeMB = (bytes) => (bytes / 1024 / 1024).toFixed(2) + ' MB'; 154 + 155 + assert(formatSize(1024) === '1.00 KB', '1KB formats correctly'); 156 + assert(formatSize(1536) === '1.50 KB', '1.5KB formats correctly'); 157 + assert(formatSizeMB(1048576) === '1.00 MB', '1MB formats correctly'); 158 + console.log(); 159 + } 160 + 161 + // Test 7: Fixed Delay (10 seconds) 162 + console.log('Test 7: Fixed Delay (10 seconds)'); 163 + { 164 + const wait = 10000; 165 + assert(wait === 10000, 'Delay is fixed at 10 seconds'); 166 + assert(wait >= 5000 && wait <= 15000, 'Delay is reasonable for pacing'); 167 + console.log(); 168 + } 169 + 170 + // Test 8: Retry Logic Simulation 171 + console.log('Test 8: Retry Logic Simulation (High Quality -> Standard)'); 172 + { 173 + const runRetryTests = async () => { 174 + // Test 8a: High quality succeeds 175 + { 176 + let highQualityFailed = false; 177 + const downloadWithRetry = async (url) => { 178 + const isHighQuality = url.includes(':orig'); 179 + if (isHighQuality && highQualityFailed) { 180 + throw new Error('High quality download failed'); 181 + } 182 + if (isHighQuality && !highQualityFailed) { 183 + return { buffer: Buffer.from('high quality'), mimeType: 'image/jpeg' }; 184 + } 185 + return { buffer: Buffer.from('standard quality'), mimeType: 'image/jpeg' }; 186 + }; 187 + 188 + highQualityFailed = false; 189 + const result1 = await downloadWithRetry('https://example.com/image.jpg:orig'); 190 + assert(result1.buffer.toString() === 'high quality', 'High quality download succeeds when available'); 191 + } 192 + 193 + // Test 8b: High quality fails, falls back to standard 194 + { 195 + let callCount = 0; 196 + const downloadWithRetry = async (url) => { 197 + callCount++; 198 + const isHighQuality = url.includes(':orig'); 199 + 200 + if (isHighQuality && callCount === 1) { 201 + throw new Error('High quality download failed'); 202 + } 203 + 204 + if (isHighQuality && callCount === 2) { 205 + const fallbackUrl = url.replace(':orig?', '?'); 206 + return { buffer: Buffer.from('standard quality'), mimeType: 'image/jpeg' }; 207 + } 208 + 209 + return { buffer: Buffer.from('standard quality'), mimeType: 'image/jpeg' }; 210 + }; 211 + 212 + const result2 = await downloadWithRetry('https://example.com/image.jpg:orig'); 213 + assert(result2.buffer.toString() === 'standard quality', 'Falls back to standard quality on failure'); 214 + } 215 + 216 + // Test 8c: Standard URL doesn't use retry logic 217 + { 218 + const downloadWithRetry = async (url) => { 219 + const isHighQuality = url.includes(':orig'); 220 + if (isHighQuality) { 221 + throw new Error('Should not be high quality'); 222 + } 223 + return { buffer: Buffer.from('standard quality'), mimeType: 'image/jpeg' }; 224 + }; 225 + 226 + const result3 = await downloadWithRetry('https://example.com/image.jpg'); 227 + assert(result3.buffer.toString() === 'standard quality', 'Standard URL downloads directly'); 228 + } 229 + }; 230 + 231 + runRetryTests().catch(err => { 232 + console.log(` ✗ Retry test error: ${err.message}`); 233 + testsFailed++; 234 + }); 235 + console.log(); 236 + } 237 + 238 + // Test 9: Image Compression Quality Settings 239 + console.log('Test 9: Image Compression Quality Settings'); 240 + { 241 + const settings = { 242 + jpeg: { quality: 92, mozjpeg: true }, 243 + jpegFallback: { quality: 85, mozjpeg: true }, 244 + png: { compressionLevel: 9, adaptiveFiltering: true }, 245 + webp: { quality: 90, effort: 6 } 246 + }; 247 + 248 + assert(settings.jpeg.quality === 92, 'JPEG quality is 92%'); 249 + assert(settings.jpeg.mozjpeg === true, 'JPEG uses mozjpeg'); 250 + assert(settings.jpegFallback.quality === 85, 'Fallback JPEG quality is 85%'); 251 + assert(settings.png.compressionLevel === 9, 'PNG compression level is 9 (max)'); 252 + assert(settings.webp.quality === 90, 'WebP quality is 90%'); 253 + assert(settings.webp.effort === 6, 'WebP encoding effort is 6 (high)'); 254 + console.log(); 255 + } 256 + 257 + // Test 10: Bluesky Size Limits 258 + console.log('Test 10: Bluesky Size Limits Compliance'); 259 + { 260 + const MAX_SIZE = 950 * 1024; 261 + const LARGE_IMAGE_THRESHOLD = 2000; 262 + const FALLBACK_THRESHOLD = 1600; 263 + 264 + assert(MAX_SIZE === 972800, 'Max size is 950KB (972800 bytes)'); 265 + assert(LARGE_IMAGE_THRESHOLD === 2000, 'Large image threshold is 2000px'); 266 + assert(FALLBACK_THRESHOLD === 1600, 'Fallback threshold is 1600px'); 267 + console.log(); 268 + } 269 + 270 + // Summary 271 + console.log('─'.repeat(40)); 272 + console.log(`Tests passed: ${testsPassed}`); 273 + console.log(`Tests failed: ${testsFailed}`); 274 + console.log('─'.repeat(40)); 275 + 276 + if (testsFailed > 0) { 277 + console.log('\nSome tests failed!'); 278 + process.exit(1); 279 + } else { 280 + console.log('\nAll tests passed!'); 281 + process.exit(0); 282 + }