From 1bb3002bafe77d786af5f76c2b24aac30386f680 Mon Sep 17 00:00:00 2001 From: blaisadmin Date: Sat, 2 May 2026 01:17:43 -0400 Subject: [PATCH] Adding Wasabi --- .env.example | 8 +++ README.md | 28 +++++++++ backend/src/app.ts | 3 + backend/src/db/schema.ts | 10 ++++ backend/src/repositories/birdRepository.ts | 13 +++-- backend/src/storage/imageStorageConfig.ts | 67 ++++++++++++++++++++++ backend/src/types.ts | 3 + docker-compose.prod.yml | 16 ++++++ docker-compose.yml | 16 ++++++ 9 files changed, 159 insertions(+), 5 deletions(-) create mode 100644 backend/src/storage/imageStorageConfig.ts diff --git a/.env.example b/.env.example index 3849fbf..cbbe7e0 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,14 @@ POSTGRES_DB=flockpal POSTGRES_USER=flockpal POSTGRES_PASSWORD=change_me_for_production REDIS_URL=redis://redis:6379 +IMAGE_STORAGE_PROVIDER=database +S3_ENDPOINT= +S3_REGION= +S3_BUCKET= +S3_ACCESS_KEY_ID= +S3_SECRET_ACCESS_KEY= +S3_PUBLIC_BASE_URL= +S3_KEY_PREFIX=bird-photos FRONTEND_URL=http://localhost:3000 BACKEND_URL=http://localhost:5000 VITE_API_BASE_URL=http://localhost:5000/api diff --git a/README.md b/README.md index 9e2092e..fa80dcc 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,12 @@ curl -H "Authorization: Bearer " https://your-host/api/metrics - `BACKEND_URL` - `VITE_API_BASE_URL` - `REDIS_URL` + - `IMAGE_STORAGE_PROVIDER` + - `S3_ENDPOINT` + - `S3_REGION` + - `S3_BUCKET` + - `S3_ACCESS_KEY_ID` + - `S3_SECRET_ACCESS_KEY` - `RESCUE_ONBOARDING_WEBHOOK_URL` 2. Build and start the production stack: @@ -104,6 +110,28 @@ Compose includes a Redis service at `redis://redis:6379` and passes that value t Scheduled milestone reminders are enqueued through Redis with a per-date job id, then processed by the worker. This keeps scheduled work out of API containers and prevents duplicate scheduled jobs when the API is scaled horizontally. Redis can also support later shared rate-limit state and short-lived cache entries. +## Image storage + +FlockPal currently keeps bird photos in Postgres as `photo_data_url`. The schema also has S3 object metadata columns so image storage can move to Wasabi/S3 without changing the bird record contract. + +Set these when Wasabi image storage is ready: + +- `IMAGE_STORAGE_PROVIDER=s3` +- `S3_ENDPOINT=https://s3..wasabisys.com` +- `S3_REGION=` +- `S3_BUCKET=` +- `S3_ACCESS_KEY_ID=` +- `S3_SECRET_ACCESS_KEY=` +- `S3_PUBLIC_BASE_URL=` +- `S3_KEY_PREFIX=bird-photos` + +Use a dedicated bucket and access key for FlockPal images. Grant only the S3 permissions the app needs for that bucket. + +Bucket settings recommendation: + +- Enable bucket versioning if you want rollback protection from accidental overwrites or deletes. Add a lifecycle policy once upload volume is known because every object version contributes to stored data. +- Do not enable Object Lock on the primary app image bucket unless there is a strict legal/compliance retention requirement. Object Lock must be enabled when creating the bucket, depends on versioning, and can make user-requested image deletion or replacement harder. + ## Worker process The API container does not run scheduled reminder loops. Background reminders run in the `worker` service so the API can be scaled horizontally without multiple API containers sending duplicate scheduled emails. diff --git a/backend/src/app.ts b/backend/src/app.ts index 86a1eb2..d5c2e49 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -482,6 +482,9 @@ const normalizeBird = (row: BirdRow) => ({ gotchaDay: row.gotcha_day, chartColor: row.chart_color, photoDataUrl: row.photo_data_url, + photoObjectKey: row.photo_object_key, + photoContentType: row.photo_content_type, + photoUpdatedAt: row.photo_updated_at, notifyOnDob: row.notify_on_dob, notifyOnGotchaDay: row.notify_on_gotcha_day, memorializedAt: row.memorialized_at, diff --git a/backend/src/db/schema.ts b/backend/src/db/schema.ts index e4bdfd6..05a41db 100644 --- a/backend/src/db/schema.ts +++ b/backend/src/db/schema.ts @@ -221,6 +221,9 @@ export const ensureSchema = async (database: DatabaseClient = db) => { gotcha_day DATE, chart_color VARCHAR(7) NOT NULL DEFAULT '#cb3a35', photo_data_url TEXT, + photo_object_key TEXT, + photo_content_type VARCHAR(80), + photo_updated_at TIMESTAMPTZ, notify_on_dob BOOLEAN NOT NULL DEFAULT FALSE, notify_on_gotcha_day BOOLEAN NOT NULL DEFAULT FALSE, memorialized_at TIMESTAMPTZ, @@ -237,6 +240,9 @@ export const ensureSchema = async (database: DatabaseClient = db) => { ADD COLUMN IF NOT EXISTS gotcha_day DATE, ADD COLUMN IF NOT EXISTS chart_color VARCHAR(7) NOT NULL DEFAULT '#cb3a35', ADD COLUMN IF NOT EXISTS photo_data_url TEXT, + ADD COLUMN IF NOT EXISTS photo_object_key TEXT, + ADD COLUMN IF NOT EXISTS photo_content_type VARCHAR(80), + ADD COLUMN IF NOT EXISTS photo_updated_at TIMESTAMPTZ, ADD COLUMN IF NOT EXISTS notify_on_dob BOOLEAN NOT NULL DEFAULT FALSE, ADD COLUMN IF NOT EXISTS notify_on_gotcha_day BOOLEAN NOT NULL DEFAULT FALSE, ADD COLUMN IF NOT EXISTS memorialized_at TIMESTAMPTZ, @@ -287,6 +293,10 @@ export const ensureSchema = async (database: DatabaseClient = db) => { AND LOWER(BTRIM(tag_id)) NOT IN ('unknown', 'not recorded', 'n/a', 'na', 'none') AND memorialized_at IS NULL; + CREATE INDEX IF NOT EXISTS idx_birds_photo_object_key + ON birds (photo_object_key) + WHERE photo_object_key IS NOT NULL; + CREATE TABLE IF NOT EXISTS pending_bird_transfers ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), bird_id UUID NOT NULL REFERENCES birds(id) ON DELETE CASCADE, diff --git a/backend/src/repositories/birdRepository.ts b/backend/src/repositories/birdRepository.ts index 400fcb7..d0e6414 100644 --- a/backend/src/repositories/birdRepository.ts +++ b/backend/src/repositories/birdRepository.ts @@ -25,6 +25,9 @@ const birdSelectFields = ` birds.gotcha_day::text, birds.chart_color, birds.photo_data_url, + birds.photo_object_key, + birds.photo_content_type, + birds.photo_updated_at, birds.notify_on_dob, birds.notify_on_gotcha_day, birds.memorialized_at, @@ -277,7 +280,7 @@ export const createBird = async ({ const result = await db.query( `INSERT INTO birds (workspace_id, name, tag_id, species, gender, date_of_birth, gotcha_day, chart_color, photo_data_url, notify_on_dob, notify_on_gotcha_day) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) - RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, NULL::text AS latest_weight_grams, NULL::text AS latest_recorded_on`, + RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, photo_object_key, photo_content_type, photo_updated_at, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, NULL::text AS latest_weight_grams, NULL::text AS latest_recorded_on`, [workspaceId, name, tagId, species, gender, dateOfBirth, gotchaDay, chartColor, photoDataUrl, notifyOnDob, notifyOnGotchaDay], ); @@ -326,7 +329,7 @@ export const updateBird = async ({ WHERE id = $1 AND workspace_id = $12 AND memorialized_at IS NULL - RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, + RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, photo_object_key, photo_content_type, photo_updated_at, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, ( SELECT weight_grams::text FROM weight_records @@ -369,7 +372,7 @@ export const memorializeBird = async ({ WHERE id = $1 AND workspace_id = $2 AND memorialized_at IS NULL - RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, + RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, photo_object_key, photo_content_type, photo_updated_at, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, ( SELECT weight_grams::text FROM weight_records @@ -405,7 +408,7 @@ export const updateMemorialReminderPreference = async ({ WHERE id = $1 AND workspace_id = $2 AND memorialized_at IS NOT NULL - RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, + RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, photo_object_key, photo_content_type, photo_updated_at, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, ( SELECT weight_grams::text FROM weight_records @@ -445,7 +448,7 @@ export const transferBirdToWorkspace = async (birdId: string, sourceWorkspaceId: WHERE id = $1 AND workspace_id = $2 AND memorialized_at IS NULL - RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, + RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, photo_object_key, photo_content_type, photo_updated_at, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, ( SELECT weight_grams::text FROM weight_records diff --git a/backend/src/storage/imageStorageConfig.ts b/backend/src/storage/imageStorageConfig.ts new file mode 100644 index 0000000..10e3f5e --- /dev/null +++ b/backend/src/storage/imageStorageConfig.ts @@ -0,0 +1,67 @@ +export type ImageStorageProvider = 'database' | 's3'; + +export type S3ImageStorageConfig = { + provider: 's3'; + endpoint: string; + region: string; + bucket: string; + accessKeyId: string; + secretAccessKey: string; + publicBaseUrl: string | null; + keyPrefix: string; +}; + +const trimOptional = (value: string | undefined) => { + const trimmed = value?.trim(); + return trimmed ? trimmed : null; +}; + +export const getImageStorageProvider = (): ImageStorageProvider => + process.env.IMAGE_STORAGE_PROVIDER === 's3' ? 's3' : 'database'; + +export const getS3ImageStorageConfig = (): S3ImageStorageConfig | null => { + if (getImageStorageProvider() !== 's3') { + return null; + } + + const endpoint = trimOptional(process.env.S3_ENDPOINT); + const region = trimOptional(process.env.S3_REGION); + const bucket = trimOptional(process.env.S3_BUCKET); + const accessKeyId = trimOptional(process.env.S3_ACCESS_KEY_ID); + const secretAccessKey = trimOptional(process.env.S3_SECRET_ACCESS_KEY); + + if (!endpoint || !region || !bucket || !accessKeyId || !secretAccessKey) { + return null; + } + + return { + provider: 's3', + endpoint, + region, + bucket, + accessKeyId, + secretAccessKey, + publicBaseUrl: trimOptional(process.env.S3_PUBLIC_BASE_URL), + keyPrefix: trimOptional(process.env.S3_KEY_PREFIX) ?? 'bird-photos', + }; +}; + +export const isS3ImageStorageConfigured = () => getS3ImageStorageConfig() !== null; + +export const buildBirdPhotoObjectKey = ({ + workspaceId, + birdId, + extension, + now = new Date(), +}: { + workspaceId: number; + birdId: string; + extension: string; + now?: Date; +}) => { + const prefix = trimOptional(process.env.S3_KEY_PREFIX) ?? 'bird-photos'; + const safeExtension = extension.replace(/^\./, '').toLowerCase() || 'bin'; + const timestamp = now.toISOString().replace(/[:.]/g, '-'); + + return `${prefix}/workspace-${workspaceId}/${birdId}/${timestamp}.${safeExtension}`; +}; diff --git a/backend/src/types.ts b/backend/src/types.ts index 72e9b2b..b27850f 100644 --- a/backend/src/types.ts +++ b/backend/src/types.ts @@ -103,6 +103,9 @@ export type BirdRow = { gotcha_day: string | null; chart_color: string; photo_data_url: string | null; + photo_object_key: string | null; + photo_content_type: string | null; + photo_updated_at: string | null; notify_on_dob: boolean; notify_on_gotcha_day: boolean; memorialized_at: string | null; diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index f12bb87..63ae0cc 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -43,6 +43,14 @@ services: POSTGRES_USER: ${POSTGRES_USER:-flockpal} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD for production} REDIS_URL: ${REDIS_URL:-redis://redis:6379} + IMAGE_STORAGE_PROVIDER: ${IMAGE_STORAGE_PROVIDER:-database} + S3_ENDPOINT: ${S3_ENDPOINT:-} + S3_REGION: ${S3_REGION:-} + S3_BUCKET: ${S3_BUCKET:-} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-} + S3_PUBLIC_BASE_URL: ${S3_PUBLIC_BASE_URL:-} + S3_KEY_PREFIX: ${S3_KEY_PREFIX:-bird-photos} FRONTEND_URL: ${FRONTEND_URL:?set FRONTEND_URL for production} BACKEND_URL: ${BACKEND_URL:?set BACKEND_URL for production} ADMIN_EMAILS: ${ADMIN_EMAILS:-} @@ -111,6 +119,14 @@ services: POSTGRES_USER: ${POSTGRES_USER:-flockpal} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD for production} REDIS_URL: ${REDIS_URL:-redis://redis:6379} + IMAGE_STORAGE_PROVIDER: ${IMAGE_STORAGE_PROVIDER:-database} + S3_ENDPOINT: ${S3_ENDPOINT:-} + S3_REGION: ${S3_REGION:-} + S3_BUCKET: ${S3_BUCKET:-} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-} + S3_PUBLIC_BASE_URL: ${S3_PUBLIC_BASE_URL:-} + S3_KEY_PREFIX: ${S3_KEY_PREFIX:-bird-photos} FRONTEND_URL: ${FRONTEND_URL:?set FRONTEND_URL for production} BACKEND_URL: ${BACKEND_URL:?set BACKEND_URL for production} ADMIN_EMAILS: ${ADMIN_EMAILS:-} diff --git a/docker-compose.yml b/docker-compose.yml index 409332c..1e8ce9c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,6 +41,14 @@ services: POSTGRES_USER: ${POSTGRES_USER:-flockpal} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-flockpal_dev_password} REDIS_URL: ${REDIS_URL:-redis://redis:6379} + IMAGE_STORAGE_PROVIDER: ${IMAGE_STORAGE_PROVIDER:-database} + S3_ENDPOINT: ${S3_ENDPOINT:-} + S3_REGION: ${S3_REGION:-} + S3_BUCKET: ${S3_BUCKET:-} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-} + S3_PUBLIC_BASE_URL: ${S3_PUBLIC_BASE_URL:-} + S3_KEY_PREFIX: ${S3_KEY_PREFIX:-bird-photos} FRONTEND_URL: ${FRONTEND_URL:-http://localhost:3000} BACKEND_URL: ${BACKEND_URL:-http://localhost:5000} ADMIN_EMAILS: ${ADMIN_EMAILS:-} @@ -104,6 +112,14 @@ services: POSTGRES_USER: ${POSTGRES_USER:-flockpal} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-flockpal_dev_password} REDIS_URL: ${REDIS_URL:-redis://redis:6379} + IMAGE_STORAGE_PROVIDER: ${IMAGE_STORAGE_PROVIDER:-database} + S3_ENDPOINT: ${S3_ENDPOINT:-} + S3_REGION: ${S3_REGION:-} + S3_BUCKET: ${S3_BUCKET:-} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-} + S3_PUBLIC_BASE_URL: ${S3_PUBLIC_BASE_URL:-} + S3_KEY_PREFIX: ${S3_KEY_PREFIX:-bird-photos} FRONTEND_URL: ${FRONTEND_URL:-http://localhost:3000} BACKEND_URL: ${BACKEND_URL:-http://localhost:5000} ADMIN_EMAILS: ${ADMIN_EMAILS:-}