diff --git a/README.md b/README.md index 11a83d8..5be53ff 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,14 @@ Set these when Wasabi image storage is ready: Use a dedicated private bucket and access key for FlockPal images. Grant only the S3 permissions the app needs for that bucket. When `S3_PUBLIC_BASE_URL` is blank, FlockPal stores private object keys and returns short-lived signed URLs for bird photos. +Migrate existing Postgres-stored bird photos after deploying S3 image storage: + +```bash +docker compose -f docker-compose.prod.yml exec backend npm run migrate:bird-photos-to-s3 -- --apply +``` + +Run a dry run first by omitting `--apply`. Use `--limit=10` to migrate a small batch, and `--keep-data-url` if you want to leave the original inline image in Postgres during an initial verification pass. + Bucket settings recommendation: - Enable bucket versioning if you want rollback protection from accidental overwrites or deletes. Add a lifecycle policy once upload volume is known because every object version contributes to stored data. diff --git a/backend/package.json b/backend/package.json index 2e7023f..7adcb12 100644 --- a/backend/package.json +++ b/backend/package.json @@ -8,6 +8,8 @@ "worker:dev": "tsx watch src/worker.ts", "build": "tsc", "test": "tsx --test src/**/*.test.ts", + "migrate:bird-photos-to-s3": "node dist/scripts/migrateBirdPhotosToS3.js", + "migrate:bird-photos-to-s3:dev": "tsx src/scripts/migrateBirdPhotosToS3.ts", "start": "node dist/app.js", "worker": "node dist/worker.js" }, diff --git a/backend/src/scripts/migrateBirdPhotosToS3.ts b/backend/src/scripts/migrateBirdPhotosToS3.ts new file mode 100644 index 0000000..85fa41e --- /dev/null +++ b/backend/src/scripts/migrateBirdPhotosToS3.ts @@ -0,0 +1,134 @@ +import { db } from '../db/client.js'; +import { ensureSchema } from '../db/schema.js'; +import { buildBirdPhotoObjectKey, getImageExtensionFromContentType, getS3ImageStorageConfig } from '../storage/imageStorageConfig.js'; +import { deleteS3Object, putS3Object } from '../storage/s3Client.js'; + +type BirdPhotoMigrationRow = { + id: string; + workspace_id: number; + name: string; + photo_data_url: string; +}; + +const parseDataImage = (dataUrl: string) => { + const match = /^data:(image\/[a-zA-Z0-9.+-]+);base64,(.+)$/.exec(dataUrl); + if (!match) { + return null; + } + + return { + contentType: match[1], + content: Buffer.from(match[2], 'base64'), + }; +}; + +const getArgValue = (name: string) => { + const prefix = `${name}=`; + const match = process.argv.find((arg) => arg.startsWith(prefix)); + return match ? match.slice(prefix.length) : null; +}; + +const dryRun = !process.argv.includes('--apply'); +const keepDataUrl = process.argv.includes('--keep-data-url'); +const limitArg = getArgValue('--limit'); +const limit = limitArg ? Number(limitArg) : null; + +if (limit !== null && (!Number.isInteger(limit) || limit <= 0)) { + console.error('Invalid --limit value. Use a positive integer.'); + process.exit(2); +} + +const run = async () => { + await ensureSchema(); + + const s3Config = getS3ImageStorageConfig(); + + if (!s3Config) { + throw new Error('S3 image storage is not fully configured. Set IMAGE_STORAGE_PROVIDER=s3 and the S3_* environment variables.'); + } + + const result = await db.query( + `SELECT id, workspace_id, name, photo_data_url + FROM birds + WHERE photo_object_key IS NULL + AND photo_data_url LIKE 'data:image/%' + ORDER BY created_at ASC + ${limit ? 'LIMIT $1' : ''}`, + limit ? [limit] : undefined, + ); + + if (dryRun) { + console.log(`Dry run: ${result.rows.length} bird photo(s) would be migrated to bucket ${s3Config.bucket}.`); + console.log('Run with --apply to upload objects and update rows.'); + return; + } + + let migrated = 0; + let skipped = 0; + let failed = 0; + + for (const bird of result.rows) { + const parsedImage = parseDataImage(bird.photo_data_url); + + if (!parsedImage) { + skipped += 1; + console.warn(`Skipping bird ${bird.id} (${bird.name}): invalid data URL.`); + continue; + } + + const objectKey = buildBirdPhotoObjectKey({ + workspaceId: bird.workspace_id, + birdId: bird.id, + extension: getImageExtensionFromContentType(parsedImage.contentType), + }); + + try { + await putS3Object({ + config: s3Config, + objectKey, + content: parsedImage.content, + contentType: parsedImage.contentType, + }); + + const updateResult = await db.query( + `UPDATE birds + SET photo_object_key = $2, + photo_content_type = $3, + photo_updated_at = CURRENT_TIMESTAMP, + photo_data_url = CASE WHEN $4::boolean THEN photo_data_url ELSE NULL END + WHERE id = $1 + AND photo_object_key IS NULL + AND photo_data_url LIKE 'data:image/%'`, + [bird.id, objectKey, parsedImage.contentType, keepDataUrl], + ); + + if (updateResult.rowCount !== 1) { + await deleteS3Object({ config: s3Config, objectKey }); + skipped += 1; + console.warn(`Skipped bird ${bird.id} (${bird.name}): row changed before update.`); + continue; + } + + migrated += 1; + console.log(`Migrated bird ${bird.id} (${bird.name}) -> ${objectKey}`); + } catch (error) { + failed += 1; + console.error(`Failed to migrate bird ${bird.id} (${bird.name}):`, error); + } + } + + console.log(`Migration complete: migrated=${migrated}, skipped=${skipped}, failed=${failed}`); + + if (failed > 0) { + process.exitCode = 1; + } +}; + +run() + .catch((error) => { + console.error('Bird photo migration failed:', error); + process.exitCode = 1; + }) + .finally(async () => { + await db.close(); + });