diff --git a/README.md b/README.md index fa80dcc..11a83d8 100644 --- a/README.md +++ b/README.md @@ -122,10 +122,10 @@ Set these when Wasabi image storage is ready: - `S3_BUCKET=` - `S3_ACCESS_KEY_ID=` - `S3_SECRET_ACCESS_KEY=` -- `S3_PUBLIC_BASE_URL=` +- `S3_PUBLIC_BASE_URL=` - `S3_KEY_PREFIX=bird-photos` -Use a dedicated bucket and access key for FlockPal images. Grant only the S3 permissions the app needs for that bucket. +Use a dedicated private bucket and access key for FlockPal images. Grant only the S3 permissions the app needs for that bucket. When `S3_PUBLIC_BASE_URL` is blank, FlockPal stores private object keys and returns short-lived signed URLs for bird photos. Bucket settings recommendation: diff --git a/backend/src/app.ts b/backend/src/app.ts index d5c2e49..bf6b1a8 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -60,6 +60,13 @@ import { updateVetVisitForBird, } from './repositories/birdRepository.js'; import { createIntegrationTokenRecord, listIntegrationTokens, revokeIntegrationToken } from './repositories/integrationTokenRepository.js'; +import { + buildBirdPhotoObjectKey, + getImageExtensionFromContentType, + getImageStorageProvider, + getS3ImageStorageConfig, +} from './storage/imageStorageConfig.js'; +import { deleteS3Object, getSignedS3ObjectUrl, putS3Object } from './storage/s3Client.js'; import { cancelRescueVerificationRequest, claimWorkspaceInvites, @@ -160,6 +167,7 @@ const photoDataUrlSchema = z .string() .regex(/^data:image\/(?:png|jpeg|jpg|webp|gif);base64,[A-Za-z0-9+/=]+$/) .max(1_500_000); +const photoUrlSchema = z.string().trim().url().max(2000); const magicLinkRequestSchema = z.object({ name: z.string().trim().max(160).optional().or(z.literal('')), @@ -230,7 +238,7 @@ const birdSchema = z.object({ dateOfBirth: dateStringSchema.optional().or(z.literal('')), gotchaDay: dateStringSchema.optional().or(z.literal('')), chartColor: chartColorSchema.optional(), - photoDataUrl: photoDataUrlSchema.optional().or(z.literal('')), + photoDataUrl: z.union([photoDataUrlSchema, photoUrlSchema, z.literal('')]).optional(), notifyOnDob: z.boolean().optional(), notifyOnGotchaDay: z.boolean().optional(), }); @@ -471,6 +479,24 @@ const normalizeWorkspaceMember = (row: WorkspaceMemberRow) => ({ createdAt: row.created_at, }); +const getBirdPhotoUrl = (row: BirdRow) => { + if (!row.photo_object_key) { + return row.photo_data_url; + } + + const s3Config = getS3ImageStorageConfig(); + + if (!s3Config) { + return row.photo_data_url; + } + + return getSignedS3ObjectUrl({ + config: s3Config, + objectKey: row.photo_object_key, + expiresInSeconds: 15 * 60, + }); +}; + const normalizeBird = (row: BirdRow) => ({ id: row.id, workspaceId: row.workspace_id, @@ -481,7 +507,7 @@ const normalizeBird = (row: BirdRow) => ({ dateOfBirth: row.date_of_birth, gotchaDay: row.gotcha_day, chartColor: row.chart_color, - photoDataUrl: row.photo_data_url, + photoDataUrl: getBirdPhotoUrl(row), photoObjectKey: row.photo_object_key, photoContentType: row.photo_content_type, photoUpdatedAt: row.photo_updated_at, @@ -1021,6 +1047,107 @@ const parseDataImage = (dataUrl: string) => { }; }; +const isDataImageUrl = (value: string | null | undefined) => Boolean(value && value.startsWith('data:image/')); + +const resolveBirdPhotoStorage = async ({ + birdId, + workspaceId, + photoDataUrl, + existingBird, +}: { + birdId: string; + workspaceId: number; + photoDataUrl: string | null; + existingBird?: BirdRow | null; +}) => { + if (!photoDataUrl) { + return { + photoDataUrl: null, + photoObjectKey: null, + photoContentType: null, + photoUpdatedAt: null, + objectKeyToDelete: existingBird?.photo_object_key ?? null, + }; + } + + if (!isDataImageUrl(photoDataUrl)) { + if (existingBird?.photo_object_key) { + return { + photoDataUrl: null, + photoObjectKey: existingBird.photo_object_key, + photoContentType: existingBird.photo_content_type, + photoUpdatedAt: existingBird.photo_updated_at, + objectKeyToDelete: null, + }; + } + + return { + photoDataUrl, + photoObjectKey: null, + photoContentType: null, + photoUpdatedAt: null, + objectKeyToDelete: existingBird?.photo_object_key ?? null, + }; + } + + const parsedImage = parseDataImage(photoDataUrl); + + if (!parsedImage) { + throw new Error('Unable to process bird photo.'); + } + + if (getImageStorageProvider() !== 's3') { + return { + photoDataUrl, + photoObjectKey: null, + photoContentType: null, + photoUpdatedAt: null, + objectKeyToDelete: existingBird?.photo_object_key ?? null, + }; + } + + const s3Config = getS3ImageStorageConfig(); + + if (!s3Config) { + throw new Error('S3 image storage is enabled but not fully configured.'); + } + + const extension = getImageExtensionFromContentType(parsedImage.contentType); + const objectKey = buildBirdPhotoObjectKey({ workspaceId, birdId, extension }); + await putS3Object({ + config: s3Config, + objectKey, + content: parsedImage.content, + contentType: parsedImage.contentType, + }); + + return { + photoDataUrl: null, + photoObjectKey: objectKey, + photoContentType: parsedImage.contentType, + photoUpdatedAt: new Date().toISOString(), + objectKeyToDelete: existingBird?.photo_object_key ?? null, + }; +}; + +const deleteBirdPhotoObjectIfNeeded = async (objectKey: string | null) => { + if (!objectKey) { + return; + } + + const s3Config = getS3ImageStorageConfig(); + + if (!s3Config) { + return; + } + + try { + await deleteS3Object({ config: s3Config, objectKey }); + } catch (error) { + console.warn(`Unable to delete old bird photo object ${objectKey}:`, error); + } +}; + const getDefaultBirdPhotoAttachment = () => { const defaultPhotoPath = path.join(process.cwd(), 'assets', 'yoda-default.png'); @@ -2558,8 +2685,18 @@ app.post('/api/birds', requireAuth, requireWriteAccess, requireWorkspaceRole(['o return; } + let uploadedObjectKeyToCleanup: string | null = null; + try { + const birdId = crypto.randomUUID(); + const photoStorage = await resolveBirdPhotoStorage({ + birdId, + workspaceId: req.auth!.workspace.id, + photoDataUrl: emptyToNull(parsed.data.photoDataUrl), + }); + uploadedObjectKeyToCleanup = photoStorage.photoObjectKey; const bird = await createBird({ + birdId, workspaceId: req.auth!.workspace.id, name: parsed.data.name, tagId: normalizeBandId(parsed.data.tagId), @@ -2568,13 +2705,19 @@ app.post('/api/birds', requireAuth, requireWriteAccess, requireWorkspaceRole(['o dateOfBirth: emptyToNull(parsed.data.dateOfBirth), gotchaDay: emptyToNull(parsed.data.gotchaDay), chartColor: parsed.data.chartColor ?? '#cb3a35', - photoDataUrl: emptyToNull(parsed.data.photoDataUrl), + photoDataUrl: photoStorage.photoDataUrl, + photoObjectKey: photoStorage.photoObjectKey, + photoContentType: photoStorage.photoContentType, + photoUpdatedAt: photoStorage.photoUpdatedAt, notifyOnDob: parsed.data.notifyOnDob ?? false, notifyOnGotchaDay: parsed.data.notifyOnGotchaDay ?? false, }); + uploadedObjectKeyToCleanup = null; res.status(201).json({ bird: normalizeBird(bird!) }); } catch (error) { + await deleteBirdPhotoObjectIfNeeded(uploadedObjectKeyToCleanup); + if (typeof error === 'object' && error && 'code' in error && error.code === '23505') { res.status(409).json({ error: 'That band/tag ID is already in use in this flock.' }); return; @@ -2667,6 +2810,8 @@ app.put('/api/birds/:birdId', requireAuth, requireWriteAccess, requireWorkspaceR return; } + let uploadedObjectKeyToCleanup: string | null = null; + try { const existingBird = await getBirdById(req.params.birdId, req.auth!.workspace.id); @@ -2679,6 +2824,14 @@ app.put('/api/birds/:birdId', requireAuth, requireWriteAccess, requireWorkspaceR return; } + const photoStorage = await resolveBirdPhotoStorage({ + birdId: req.params.birdId, + workspaceId: req.auth!.workspace.id, + photoDataUrl: emptyToNull(parsed.data.photoDataUrl), + existingBird, + }); + uploadedObjectKeyToCleanup = + photoStorage.photoObjectKey && photoStorage.photoObjectKey !== existingBird.photo_object_key ? photoStorage.photoObjectKey : null; const bird = await updateBird({ birdId: req.params.birdId, workspaceId: req.auth!.workspace.id, @@ -2689,7 +2842,10 @@ app.put('/api/birds/:birdId', requireAuth, requireWriteAccess, requireWorkspaceR dateOfBirth: emptyToNull(parsed.data.dateOfBirth), gotchaDay: emptyToNull(parsed.data.gotchaDay), chartColor: parsed.data.chartColor ?? '#cb3a35', - photoDataUrl: emptyToNull(parsed.data.photoDataUrl), + photoDataUrl: photoStorage.photoDataUrl, + photoObjectKey: photoStorage.photoObjectKey, + photoContentType: photoStorage.photoContentType, + photoUpdatedAt: photoStorage.photoUpdatedAt, notifyOnDob: parsed.data.notifyOnDob ?? false, notifyOnGotchaDay: parsed.data.notifyOnGotchaDay ?? false, }); @@ -2699,8 +2855,12 @@ app.put('/api/birds/:birdId', requireAuth, requireWriteAccess, requireWorkspaceR return; } + uploadedObjectKeyToCleanup = null; + await deleteBirdPhotoObjectIfNeeded(photoStorage.objectKeyToDelete); res.json({ bird: normalizeBird(bird) }); } catch (error) { + await deleteBirdPhotoObjectIfNeeded(uploadedObjectKeyToCleanup); + if (typeof error === 'object' && error && 'code' in error && error.code === '23505') { res.status(409).json({ error: 'That band/tag ID is already in use in this flock.' }); return; @@ -2731,6 +2891,7 @@ app.delete('/api/birds/:birdId', requireAuth, requireWriteAccess, requireWorkspa } res.status(204).send(); + await deleteBirdPhotoObjectIfNeeded(bird.photo_object_key); } catch (error) { next(error); } diff --git a/backend/src/queues/birdMilestoneReminderQueue.ts b/backend/src/queues/birdMilestoneReminderQueue.ts index bcc6714..6f3ac49 100644 --- a/backend/src/queues/birdMilestoneReminderQueue.ts +++ b/backend/src/queues/birdMilestoneReminderQueue.ts @@ -41,7 +41,7 @@ export const enqueueBirdMilestoneReminderJob = (runDate: string): Promise { const result = await db.query( - `INSERT INTO birds (workspace_id, name, tag_id, species, gender, date_of_birth, gotcha_day, chart_color, photo_data_url, notify_on_dob, notify_on_gotcha_day) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + `INSERT INTO birds (id, workspace_id, name, tag_id, species, gender, date_of_birth, gotcha_day, chart_color, photo_data_url, photo_object_key, photo_content_type, photo_updated_at, notify_on_dob, notify_on_gotcha_day) + VALUES (COALESCE($1::uuid, gen_random_uuid()), $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, photo_object_key, photo_content_type, photo_updated_at, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, NULL::text AS latest_weight_grams, NULL::text AS latest_recorded_on`, - [workspaceId, name, tagId, species, gender, dateOfBirth, gotchaDay, chartColor, photoDataUrl, notifyOnDob, notifyOnGotchaDay], + [ + birdId ?? null, + workspaceId, + name, + tagId, + species, + gender, + dateOfBirth, + gotchaDay, + chartColor, + photoDataUrl, + photoObjectKey, + photoContentType, + photoUpdatedAt, + notifyOnDob, + notifyOnGotchaDay, + ], ); return result.rows[0] ?? null; @@ -298,6 +322,9 @@ export const updateBird = async ({ gotchaDay, chartColor, photoDataUrl, + photoObjectKey = null, + photoContentType = null, + photoUpdatedAt = null, notifyOnDob, notifyOnGotchaDay, }: { @@ -311,6 +338,9 @@ export const updateBird = async ({ gotchaDay: string | null; chartColor: string; photoDataUrl: string | null; + photoObjectKey?: string | null; + photoContentType?: string | null; + photoUpdatedAt?: string | null; notifyOnDob: boolean; notifyOnGotchaDay: boolean; }) => { @@ -324,10 +354,13 @@ export const updateBird = async ({ gotcha_day = $7, chart_color = $8, photo_data_url = $9, - notify_on_dob = $10, - notify_on_gotcha_day = $11 + photo_object_key = $10, + photo_content_type = $11, + photo_updated_at = $12, + notify_on_dob = $13, + notify_on_gotcha_day = $14 WHERE id = $1 - AND workspace_id = $12 + AND workspace_id = $15 AND memorialized_at IS NULL RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, photo_object_key, photo_content_type, photo_updated_at, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, ( @@ -344,7 +377,23 @@ export const updateBird = async ({ ORDER BY recorded_on DESC LIMIT 1 ) AS latest_recorded_on`, - [birdId, name, tagId, species, gender, dateOfBirth, gotchaDay, chartColor, photoDataUrl, notifyOnDob, notifyOnGotchaDay, workspaceId], + [ + birdId, + name, + tagId, + species, + gender, + dateOfBirth, + gotchaDay, + chartColor, + photoDataUrl, + photoObjectKey, + photoContentType, + photoUpdatedAt, + notifyOnDob, + notifyOnGotchaDay, + workspaceId, + ], ); return result.rows[0] ?? null; diff --git a/backend/src/storage/imageStorageConfig.ts b/backend/src/storage/imageStorageConfig.ts index 10e3f5e..8d47d57 100644 --- a/backend/src/storage/imageStorageConfig.ts +++ b/backend/src/storage/imageStorageConfig.ts @@ -65,3 +65,19 @@ export const buildBirdPhotoObjectKey = ({ return `${prefix}/workspace-${workspaceId}/${birdId}/${timestamp}.${safeExtension}`; }; + +export const getImageExtensionFromContentType = (contentType: string) => { + switch (contentType.toLowerCase()) { + case 'image/jpeg': + case 'image/jpg': + return 'jpg'; + case 'image/png': + return 'png'; + case 'image/webp': + return 'webp'; + case 'image/gif': + return 'gif'; + default: + return 'bin'; + } +}; diff --git a/backend/src/storage/s3Client.ts b/backend/src/storage/s3Client.ts new file mode 100644 index 0000000..ab584c6 --- /dev/null +++ b/backend/src/storage/s3Client.ts @@ -0,0 +1,161 @@ +import crypto from 'crypto'; + +import type { S3ImageStorageConfig } from './imageStorageConfig.js'; + +const awsDate = (date: Date) => date.toISOString().replace(/[:-]|\.\d{3}/g, ''); +const shortDate = (date: Date) => date.toISOString().slice(0, 10).replace(/-/g, ''); + +const hmac = (key: crypto.BinaryLike, value: string) => crypto.createHmac('sha256', key).update(value).digest(); +const sha256Hex = (value: crypto.BinaryLike) => crypto.createHash('sha256').update(value).digest('hex'); + +const encodeObjectKey = (key: string) => key.split('/').map(encodeURIComponent).join('/'); +const encodeQueryValue = (value: string) => encodeURIComponent(value).replace(/[!'()*]/g, (character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}`); + +const getSigningKey = (secretAccessKey: string, date: string, region: string) => { + const dateKey = hmac(`AWS4${secretAccessKey}`, date); + const regionKey = hmac(dateKey, region); + const serviceKey = hmac(regionKey, 's3'); + return hmac(serviceKey, 'aws4_request'); +}; + +const buildObjectUrl = (config: S3ImageStorageConfig, objectKey: string) => { + const endpoint = config.endpoint.replace(/\/+$/, ''); + return new URL(`${endpoint}/${encodeURIComponent(config.bucket)}/${encodeObjectKey(objectKey)}`); +}; + +const signS3Request = ({ + config, + method, + objectKey, + contentHash, + contentType, + now = new Date(), +}: { + config: S3ImageStorageConfig; + method: 'DELETE' | 'PUT'; + objectKey: string; + contentHash: string; + contentType?: string; + now?: Date; +}) => { + const url = buildObjectUrl(config, objectKey); + const amzDate = awsDate(now); + const date = shortDate(now); + const credentialScope = `${date}/${config.region}/s3/aws4_request`; + const headers: Record = { + host: url.host, + 'x-amz-content-sha256': contentHash, + 'x-amz-date': amzDate, + }; + + if (contentType) { + headers['content-type'] = contentType; + } + + const signedHeaders = Object.keys(headers).sort().join(';'); + const canonicalHeaders = Object.keys(headers) + .sort() + .map((key) => `${key}:${headers[key]}\n`) + .join(''); + const canonicalRequest = [method, url.pathname, url.search.slice(1), canonicalHeaders, signedHeaders, contentHash].join('\n'); + const stringToSign = ['AWS4-HMAC-SHA256', amzDate, credentialScope, sha256Hex(canonicalRequest)].join('\n'); + const signature = crypto.createHmac('sha256', getSigningKey(config.secretAccessKey, date, config.region)).update(stringToSign).digest('hex'); + + return { + url, + headers: { + ...headers, + Authorization: `AWS4-HMAC-SHA256 Credential=${config.accessKeyId}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}`, + }, + }; +}; + +export const getPublicObjectUrl = (config: S3ImageStorageConfig, objectKey: string) => { + if (config.publicBaseUrl) { + return `${config.publicBaseUrl.replace(/\/+$/, '')}/${encodeObjectKey(objectKey)}`; + } + + return buildObjectUrl(config, objectKey).toString(); +}; + +export const getSignedS3ObjectUrl = ({ + config, + objectKey, + expiresInSeconds = 900, + now = new Date(), +}: { + config: S3ImageStorageConfig; + objectKey: string; + expiresInSeconds?: number; + now?: Date; +}) => { + const url = buildObjectUrl(config, objectKey); + const amzDate = awsDate(now); + const date = shortDate(now); + const credentialScope = `${date}/${config.region}/s3/aws4_request`; + const credential = `${config.accessKeyId}/${credentialScope}`; + const queryParams: Record = { + 'X-Amz-Algorithm': 'AWS4-HMAC-SHA256', + 'X-Amz-Credential': credential, + 'X-Amz-Date': amzDate, + 'X-Amz-Expires': String(Math.min(Math.max(expiresInSeconds, 1), 604800)), + 'X-Amz-SignedHeaders': 'host', + }; + const canonicalQuery = Object.keys(queryParams) + .sort() + .map((key) => `${encodeQueryValue(key)}=${encodeQueryValue(queryParams[key])}`) + .join('&'); + const canonicalHeaders = `host:${url.host}\n`; + const canonicalRequest = ['GET', url.pathname, canonicalQuery, canonicalHeaders, 'host', 'UNSIGNED-PAYLOAD'].join('\n'); + const stringToSign = ['AWS4-HMAC-SHA256', amzDate, credentialScope, sha256Hex(canonicalRequest)].join('\n'); + const signature = crypto.createHmac('sha256', getSigningKey(config.secretAccessKey, date, config.region)).update(stringToSign).digest('hex'); + + url.search = `${canonicalQuery}&X-Amz-Signature=${signature}`; + return url.toString(); +}; + +export const putS3Object = async ({ + config, + objectKey, + content, + contentType, +}: { + config: S3ImageStorageConfig; + objectKey: string; + content: Buffer; + contentType: string; +}) => { + const contentHash = sha256Hex(content); + const signed = signS3Request({ config, method: 'PUT', objectKey, contentHash, contentType }); + const response = await fetch(signed.url, { + method: 'PUT', + headers: signed.headers, + body: content, + }); + + if (!response.ok) { + const errorText = await response.text().catch(() => ''); + throw new Error(`Wasabi upload failed with ${response.status}${errorText ? `: ${errorText.slice(0, 300)}` : ''}`); + } + + return getPublicObjectUrl(config, objectKey); +}; + +export const deleteS3Object = async ({ + config, + objectKey, +}: { + config: S3ImageStorageConfig; + objectKey: string; +}) => { + const signed = signS3Request({ config, method: 'DELETE', objectKey, contentHash: sha256Hex('') }); + const response = await fetch(signed.url, { + method: 'DELETE', + headers: signed.headers, + }); + + if (!response.ok && response.status !== 404) { + const errorText = await response.text().catch(() => ''); + throw new Error(`Wasabi delete failed with ${response.status}${errorText ? `: ${errorText.slice(0, 300)}` : ''}`); + } +}; diff --git a/scripts/backup-postgres.sh b/scripts/backup-postgres.sh index 7191590..341522a 100755 --- a/scripts/backup-postgres.sh +++ b/scripts/backup-postgres.sh @@ -1,4 +1,8 @@ #!/usr/bin/env bash +if [ -z "${BASH_VERSION:-}" ]; then + exec bash "$0" "$@" +fi + set -euo pipefail compose_file="${COMPOSE_FILE:-docker-compose.prod.yml}" diff --git a/scripts/restore-test-postgres.sh b/scripts/restore-test-postgres.sh index aa6e357..63dae2b 100755 --- a/scripts/restore-test-postgres.sh +++ b/scripts/restore-test-postgres.sh @@ -1,4 +1,8 @@ #!/usr/bin/env bash +if [ -z "${BASH_VERSION:-}" ]; then + exec bash "$0" "$@" +fi + set -euo pipefail if [[ $# -ne 1 ]]; then