From 673df353b91aaa7cb2f4983ec02ada870ec0bf95 Mon Sep 17 00:00:00 2001 From: blaisadmin Date: Sat, 2 May 2026 00:14:56 -0400 Subject: [PATCH 1/7] Added redis and worker services --- .env.example | 2 + .gitignore | 1 + README.md | 64 +++- backend/package-lock.json | 307 ++++++++++++++++++ backend/package.json | 5 +- backend/src/app.ts | 98 +++++- backend/src/db/schema.ts | 33 ++ .../src/queues/birdMilestoneReminderQueue.ts | 53 +++ backend/src/queues/redisConnection.ts | 19 ++ backend/src/worker.ts | 61 ++++ docker-compose.prod.yml | 53 +++ docker-compose.yml | 57 ++++ docs/API_REFERENCE.md | 55 ++++ scripts/backup-postgres.sh | 13 + scripts/restore-test-postgres.sh | 27 ++ 15 files changed, 833 insertions(+), 15 deletions(-) create mode 100644 backend/src/queues/birdMilestoneReminderQueue.ts create mode 100644 backend/src/queues/redisConnection.ts create mode 100644 backend/src/worker.ts create mode 100755 scripts/backup-postgres.sh create mode 100755 scripts/restore-test-postgres.sh diff --git a/.env.example b/.env.example index e7526ef..3849fbf 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,7 @@ POSTGRES_DB=flockpal POSTGRES_USER=flockpal POSTGRES_PASSWORD=change_me_for_production +REDIS_URL=redis://redis:6379 FRONTEND_URL=http://localhost:3000 BACKEND_URL=http://localhost:5000 VITE_API_BASE_URL=http://localhost:5000/api @@ -8,6 +9,7 @@ NODE_ENV=development TRUST_PROXY= ADMIN_EMAILS=corey@blaishome.online RESCUE_STATUS_NOTIFICATION_EMAIL=appadmin@flockpal.app +RESCUE_ONBOARDING_WEBHOOK_URL=https://n8n.blaishome.online/webhook/395cd538-5e0d-4e89-8070-9e66f571b7ee MILESTONE_REMINDERS_ENABLED=true MILESTONE_REMINDER_TIME_ZONE=America/New_York STRIPE_SECRET_KEY= diff --git a/.gitignore b/.gitignore index 2a81bc9..3574fd2 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ frontend/node_modules backend/dist frontend/dist data/ +backups/ .DS_Store diff --git a/README.md b/README.md index 13c6fb6..9e2092e 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ FlockPal is a Dockerized TypeScript app for tracking flock health with a clean, - Vet visit history with notes - Postgres-backed storage - React frontend and Express backend +- Separate worker process for scheduled reminders and background tasks +- Redis-backed background queue for scheduled reminders - Security-minded defaults like Helmet, CORS allow-listing, rate limiting, and input validation ## Planned next steps @@ -43,6 +45,40 @@ Full API documentation is available in [docs/API_REFERENCE.md](docs/API_REFERENC The default `docker-compose.yml` is development-only. It mounts source files, installs dev dependencies, and runs the backend and frontend in watch mode. +## Operations + +### Backups + +Create a compressed Postgres backup from the Docker Compose Postgres service: + +```bash +./scripts/backup-postgres.sh +``` + +By default this uses `docker-compose.prod.yml` and writes to `backups/postgres/`. Override with `COMPOSE_FILE` or `BACKUP_DIR`: + +```bash +COMPOSE_FILE=docker-compose.yml BACKUP_DIR=/srv/flockpal-backups ./scripts/backup-postgres.sh +``` + +Test a backup by restoring it into a temporary database in the same Postgres container: + +```bash +./scripts/restore-test-postgres.sh backups/postgres/flockpal-YYYYMMDDTHHMMSSZ.dump +``` + +The restore test creates a temporary database, runs `pg_restore`, verifies it can query `workspaces`, and then drops the temporary database. It does not overwrite the live database. + +### Metrics and logs + +The backend writes HTTP access logs through Morgan. Production uses the standard combined log format so Docker, Traefik, or a log shipper can collect it from container stdout. + +Admin users can fetch lightweight process/request metrics: + +```bash +curl -H "Authorization: Bearer " https://your-host/api/metrics +``` + ## Production 1. Set production values in your environment or `.env`, especially: @@ -50,6 +86,8 @@ The default `docker-compose.yml` is development-only. It mounts source files, in - `FRONTEND_URL` - `BACKEND_URL` - `VITE_API_BASE_URL` + - `REDIS_URL` + - `RESCUE_ONBOARDING_WEBHOOK_URL` 2. Build and start the production stack: ```bash @@ -57,7 +95,31 @@ docker compose -f docker-compose.prod.yml up --build -d ``` 3. The production backend runs the compiled Node app from `dist/app.js`. -4. The production frontend is built with Vite and served by Nginx on port `3000`. +4. The production worker runs `dist/worker.js` and owns scheduled reminder execution. +5. The production frontend is built with Vite and served by Nginx on port `3000`. + +## Redis + +Compose includes a Redis service at `redis://redis:6379` and passes that value through `REDIS_URL` to the backend and worker. Redis uses append-only persistence under `data/redis/`. + +Scheduled milestone reminders are enqueued through Redis with a per-date job id, then processed by the worker. This keeps scheduled work out of API containers and prevents duplicate scheduled jobs when the API is scaled horizontally. Redis can also support later shared rate-limit state and short-lived cache entries. + +## Worker process + +The API container does not run scheduled reminder loops. Background reminders run in the `worker` service so the API can be scaled horizontally without multiple API containers sending duplicate scheduled emails. + +Run the worker locally through Compose: + +```bash +docker compose up worker +``` + +Run the compiled worker directly from the backend package: + +```bash +npm run build +npm run worker +``` ## Auth and flock notes diff --git a/backend/package-lock.json b/backend/package-lock.json index f080828..5317c51 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -9,6 +9,7 @@ "version": "0.1.0", "dependencies": { "@types/nodemailer": "^8.0.0", + "bullmq": "^5.76.4", "cors": "2.8.5", "dotenv": "16.4.5", "express": "4.21.2", @@ -438,6 +439,90 @@ "node": ">=18" } }, + "node_modules/@ioredis/commands": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.5.1.tgz", + "integrity": "sha512-JH8ZL/ywcJyR9MmJ5BNqZllXNZQqQbnVZOqpPQqE1vHiFgAw4NHbvE0FOduNU8IX9babitBT46571OnPTT0Zcw==", + "license": "MIT" + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz", + "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz", + "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz", + "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz", + "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz", + "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz", + "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@types/body-parser": { "version": "1.19.6", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz", @@ -638,6 +723,23 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/bullmq": { + "version": "5.76.4", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.76.4.tgz", + "integrity": "sha512-hVAplia7zfN3BxSCgAoRInJnbemfLwJdQLqJy/txEX8UMSTAeg0saPFNGWIlzES/Ct5xQ20TUaik/XwS99DOMA==", + "license": "MIT", + "dependencies": { + "cron-parser": "4.9.0", + "ioredis": "5.10.1", + "msgpackr": "1.11.5", + "node-abort-controller": "3.1.1", + "semver": "7.7.4", + "tslib": "2.8.1" + }, + "engines": { + "node": ">=12.22.0" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -676,6 +778,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -725,6 +836,18 @@ "node": ">= 0.10" } }, + "node_modules/cron-parser": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", + "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", + "license": "MIT", + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -734,6 +857,15 @@ "ms": "2.0.0" } }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -753,6 +885,16 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=8" + } + }, "node_modules/dotenv": { "version": "16.4.5", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", @@ -1129,6 +1271,53 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "license": "ISC" }, + "node_modules/ioredis": { + "version": "5.10.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.10.1.tgz", + "integrity": "sha512-HuEDBTI70aYdx1v6U97SbNx9F1+svQKBDo30o0b9fw055LMepzpOOd0Ccg9Q6tbqmBSJaMuY0fB7yw9/vjBYCA==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "1.5.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/ioredis/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/ioredis/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -1138,6 +1327,27 @@ "node": ">= 0.10" } }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "license": "MIT" + }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", + "license": "MIT" + }, + "node_modules/luxon": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz", + "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -1241,6 +1451,37 @@ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "license": "MIT" }, + "node_modules/msgpackr": { + "version": "1.11.5", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.5.tgz", + "integrity": "sha512-UjkUHN0yqp9RWKy0Lplhh+wlpdt9oQBYgULZOiFhV3VclSF1JnSQWZ5r9gORQlNYaUKQoR8itv7g7z1xDDuACA==", + "license": "MIT", + "optionalDependencies": { + "msgpackr-extract": "^3.0.2" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz", + "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.2.2" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" + } + }, "node_modules/negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -1250,6 +1491,27 @@ "node": ">= 0.6" } }, + "node_modules/node-abort-controller": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", + "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==", + "license": "MIT" + }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", + "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", + "license": "MIT", + "optional": true, + "dependencies": { + "detect-libc": "^2.0.1" + }, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, "node_modules/nodemailer": { "version": "8.0.5", "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-8.0.5.tgz", @@ -1582,6 +1844,27 @@ "node": ">= 0.8" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/resolve-pkg-maps": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", @@ -1618,6 +1901,18 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "license": "MIT" }, + "node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/send": { "version": "0.19.0", "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", @@ -1759,6 +2054,12 @@ "node": ">= 10.x" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -1794,6 +2095,12 @@ "node": ">=0.6" } }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, "node_modules/tsx": { "version": "4.19.2", "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.19.2.tgz", diff --git a/backend/package.json b/backend/package.json index 3dfee60..2e7023f 100644 --- a/backend/package.json +++ b/backend/package.json @@ -5,12 +5,15 @@ "type": "module", "scripts": { "dev": "tsx watch src/app.ts", + "worker:dev": "tsx watch src/worker.ts", "build": "tsc", "test": "tsx --test src/**/*.test.ts", - "start": "node dist/app.js" + "start": "node dist/app.js", + "worker": "node dist/worker.js" }, "dependencies": { "@types/nodemailer": "^8.0.0", + "bullmq": "^5.76.4", "cors": "2.8.5", "dotenv": "16.4.5", "express": "4.21.2", diff --git a/backend/src/app.ts b/backend/src/app.ts index 2b031da..0bdc817 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -1,6 +1,7 @@ import crypto from 'crypto'; import { existsSync } from 'fs'; import path from 'path'; +import { fileURLToPath } from 'url'; import cors from 'cors'; import dotenv from 'dotenv'; import express, { type NextFunction, type Request, type Response } from 'express'; @@ -12,6 +13,7 @@ import Stripe from 'stripe'; import { z } from 'zod'; import { ensureSchema } from './db/schema.js'; +import { enqueueBirdMilestoneReminderJob, getBirdMilestoneReminderQueueCounts } from './queues/birdMilestoneReminderQueue.js'; import { consumeMagicLinkToken, consumeOAuthState, @@ -337,7 +339,8 @@ const smtpPass = process.env.SMTP_PASS?.trim() ?? ''; const smtpFromEmail = process.env.SMTP_FROM_EMAIL?.trim() ?? ''; const smtpFromName = process.env.SMTP_FROM_NAME?.trim() || 'FlockPal'; const rescueStatusNotificationEmail = process.env.RESCUE_STATUS_NOTIFICATION_EMAIL?.trim() || 'appadmin@flockpal.app'; -const rescueOnboardingWebhookUrl = 'https://n8n.blaishome.online/webhook/395cd538-5e0d-4e89-8070-9e66f571b7ee'; +const rescueOnboardingWebhookUrl = + process.env.RESCUE_ONBOARDING_WEBHOOK_URL?.trim() || 'https://n8n.blaishome.online/webhook/395cd538-5e0d-4e89-8070-9e66f571b7ee'; const stripeSecretKey = process.env.STRIPE_SECRET_KEY?.trim() ?? ''; const stripeWebhookSecret = process.env.STRIPE_WEBHOOK_SECRET?.trim() ?? ''; const withBillingRedirectState = (url: string, billingState: 'success' | 'cancelled' | 'portal') => { @@ -690,6 +693,40 @@ app.use(express.json({ limit: '2mb' })); app.use(express.urlencoded({ extended: false })); app.use(morgan(process.env.NODE_ENV === 'production' ? 'combined' : 'dev')); +const requestMetrics = { + startedAt: new Date().toISOString(), + totalRequests: 0, + totalErrors: 0, + inFlightRequests: 0, + totalDurationMs: 0, + byStatus: {} as Record, + byRoute: {} as Record, +}; + +app.use((req: Request, res: Response, next: NextFunction) => { + const startedAt = process.hrtime.bigint(); + requestMetrics.totalRequests += 1; + requestMetrics.inFlightRequests += 1; + + res.on('finish', () => { + const durationMs = Number(process.hrtime.bigint() - startedAt) / 1_000_000; + requestMetrics.inFlightRequests -= 1; + requestMetrics.totalDurationMs += durationMs; + + const statusBucket = `${Math.floor(res.statusCode / 100)}xx`; + requestMetrics.byStatus[statusBucket] = (requestMetrics.byStatus[statusBucket] ?? 0) + 1; + + if (res.statusCode >= 500) { + requestMetrics.totalErrors += 1; + } + + const routeKey = `${req.method} ${req.route?.path ?? req.path}`; + requestMetrics.byRoute[routeKey] = (requestMetrics.byRoute[routeKey] ?? 0) + 1; + }); + + next(); +}); + const normalizeWorkspaceMembershipList = async (userId: string) => (await listMembershipsForUser(userId)).map((row) => ({ membership: normalizeWorkspaceMember(row), @@ -1412,7 +1449,7 @@ const sendBirdMilestoneReminderNotification = async ({ return { delivered: true }; }; -const runBirdMilestoneReminders = async (runDate = getDateInTimeZone()) => { +export const runBirdMilestoneReminders = async (runDate = getDateInTimeZone()) => { const reminders = await listDueBirdMilestoneReminders(runDate); let sent = 0; let skipped = 0; @@ -1458,7 +1495,7 @@ const runBirdMilestoneReminders = async (runDate = getDateInTimeZone()) => { let lastMilestoneReminderRunDate = ''; -const startBirdMilestoneReminderScheduler = () => { +export const startBirdMilestoneReminderScheduler = () => { if (!milestoneRemindersEnabled) { console.log('Bird milestone reminders are disabled.'); return; @@ -1471,10 +1508,8 @@ const startBirdMilestoneReminderScheduler = () => { } lastMilestoneReminderRunDate = runDate; - const result = await runBirdMilestoneReminders(runDate); - console.log( - `Bird milestone reminders completed for ${result.runDate}: checked=${result.checked}, sent=${result.sent}, skipped=${result.skipped}, failed=${result.failed}`, - ); + const job = await enqueueBirdMilestoneReminderJob(runDate); + console.log(`Bird milestone reminder job queued for ${runDate}: id=${job.id ?? 'unknown'}`); }; setTimeout(() => { @@ -1614,6 +1649,40 @@ app.get('/api/health', (_req: Request, res: Response) => { res.json({ ok: true }); }); +app.get('/api/metrics', requireAuth, requireAdmin, async (_req: Request, res: Response, next: NextFunction) => { + const memoryUsage = process.memoryUsage(); + const averageDurationMs = requestMetrics.totalRequests > 0 ? requestMetrics.totalDurationMs / requestMetrics.totalRequests : 0; + + try { + const birdMilestoneReminderQueueCounts = await getBirdMilestoneReminderQueueCounts(); + + res.json({ + startedAt: requestMetrics.startedAt, + uptimeSeconds: Math.round(process.uptime()), + requests: { + total: requestMetrics.totalRequests, + inFlight: requestMetrics.inFlightRequests, + errors: requestMetrics.totalErrors, + averageDurationMs: Number(averageDurationMs.toFixed(2)), + byStatus: requestMetrics.byStatus, + byRoute: requestMetrics.byRoute, + }, + memory: { + rss: memoryUsage.rss, + heapTotal: memoryUsage.heapTotal, + heapUsed: memoryUsage.heapUsed, + external: memoryUsage.external, + arrayBuffers: memoryUsage.arrayBuffers, + }, + queues: { + birdMilestoneReminders: birdMilestoneReminderQueueCounts, + }, + }); + } catch (error) { + next(error); + } +}); + app.post('/api/lost-bird/report', lostBirdReportLimiter, async (req: Request, res: Response) => { const parsed = lostBirdReportSchema.safeParse(req.body); @@ -3073,15 +3142,18 @@ app.use((error: unknown, _req: Request, res: Response, _next: NextFunction) => { res.status(500).json({ error: error instanceof Error ? error.message : 'Internal server error' }); }); -const start = async () => { +export const startApiServer = async () => { await ensureSchema(); app.listen(port, () => { console.log(`FlockPal backend listening on port ${port}`); }); - startBirdMilestoneReminderScheduler(); }; -start().catch((error) => { - console.error('Failed to start backend', error); - process.exit(1); -}); +const currentModulePath = fileURLToPath(import.meta.url); + +if (process.argv[1] === currentModulePath) { + startApiServer().catch((error) => { + console.error('Failed to start backend', error); + process.exit(1); + }); +} diff --git a/backend/src/db/schema.ts b/backend/src/db/schema.ts index b16ed63..e4bdfd6 100644 --- a/backend/src/db/schema.ts +++ b/backend/src/db/schema.ts @@ -47,6 +47,9 @@ export const ensureSchema = async (database: DatabaseClient = db) => { ON workspaces (stripe_customer_id) WHERE stripe_customer_id IS NOT NULL; + CREATE INDEX IF NOT EXISTS idx_workspaces_rescue_status + ON workspaces (workspace_type, rescue_verification_status, created_at DESC); + UPDATE workspaces SET subscription_status = 'none' WHERE workspace_type = 'standard' @@ -119,6 +122,15 @@ export const ensureSchema = async (database: DatabaseClient = db) => { ON workspace_members (workspace_id, user_id) WHERE user_id IS NOT NULL; + CREATE INDEX IF NOT EXISTS idx_workspace_members_user_accepted + ON workspace_members (user_id, accepted_at, workspace_id) + WHERE user_id IS NOT NULL; + + CREATE INDEX IF NOT EXISTS idx_workspace_members_owner_email + ON workspace_members (LOWER(COALESCE(invite_email, email)), workspace_id) + WHERE role = 'owner' + AND accepted_at IS NOT NULL; + CREATE TABLE IF NOT EXISTS auth_sessions ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, @@ -128,6 +140,9 @@ export const ensureSchema = async (database: DatabaseClient = db) => { created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP ); + CREATE INDEX IF NOT EXISTS idx_auth_sessions_created_user + ON auth_sessions (created_at DESC, user_id); + CREATE TABLE IF NOT EXISTS integration_tokens ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, @@ -257,6 +272,21 @@ export const ensureSchema = async (database: DatabaseClient = db) => { AND BTRIM(tag_id) <> '' AND LOWER(BTRIM(tag_id)) NOT IN ('unknown', 'not recorded', 'n/a', 'na', 'none'); + CREATE INDEX IF NOT EXISTS idx_birds_workspace_active_name + ON birds (workspace_id, name) + WHERE memorialized_at IS NULL; + + CREATE INDEX IF NOT EXISTS idx_birds_workspace_memorialized + ON birds (workspace_id, memorialized_on DESC, name) + WHERE memorialized_at IS NOT NULL; + + CREATE INDEX IF NOT EXISTS idx_birds_tag_lookup_active + ON birds (LOWER(tag_id), created_at) + WHERE tag_id IS NOT NULL + AND BTRIM(tag_id) <> '' + AND LOWER(BTRIM(tag_id)) NOT IN ('unknown', 'not recorded', 'n/a', 'na', 'none') + AND memorialized_at IS NULL; + CREATE TABLE IF NOT EXISTS pending_bird_transfers ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), bird_id UUID NOT NULL REFERENCES birds(id) ON DELETE CASCADE, @@ -374,6 +404,9 @@ export const ensureSchema = async (database: DatabaseClient = db) => { CREATE INDEX IF NOT EXISTS idx_medication_administrations_bird_administered_on ON medication_administrations (bird_id, administered_on DESC); + CREATE INDEX IF NOT EXISTS idx_medication_administrations_medication_date + ON medication_administrations (medication_id, administered_on DESC, created_at DESC); + DO $$ BEGIN IF EXISTS ( diff --git a/backend/src/queues/birdMilestoneReminderQueue.ts b/backend/src/queues/birdMilestoneReminderQueue.ts new file mode 100644 index 0000000..bcc6714 --- /dev/null +++ b/backend/src/queues/birdMilestoneReminderQueue.ts @@ -0,0 +1,53 @@ +import { Queue, type Job } from 'bullmq'; + +import { redisConnection } from './redisConnection.js'; + +export type BirdMilestoneReminderJobData = { + runDate: string; + requestedBy: 'scheduler'; +}; + +export type BirdMilestoneReminderJobResult = { + runDate: string; + checked: number; + sent: number; + skipped: number; + failed: number; +}; + +export const birdMilestoneReminderQueueName = 'bird-milestone-reminders'; + +export const birdMilestoneReminderQueue = new Queue( + birdMilestoneReminderQueueName, + { + connection: redisConnection, + defaultJobOptions: { + attempts: 3, + backoff: { + type: 'exponential', + delay: 60_000, + }, + removeOnComplete: 100, + removeOnFail: 1_000, + }, + }, +); + +export const enqueueBirdMilestoneReminderJob = (runDate: string): Promise> => + birdMilestoneReminderQueue.add( + 'run-daily-reminders', + { + runDate, + requestedBy: 'scheduler', + }, + { + jobId: `bird-milestone-reminders:${runDate}`, + }, + ); + +export const closeBirdMilestoneReminderQueue = async () => { + await birdMilestoneReminderQueue.close(); +}; + +export const getBirdMilestoneReminderQueueCounts = () => + birdMilestoneReminderQueue.getJobCounts('waiting', 'active', 'delayed', 'completed', 'failed'); diff --git a/backend/src/queues/redisConnection.ts b/backend/src/queues/redisConnection.ts new file mode 100644 index 0000000..7faf852 --- /dev/null +++ b/backend/src/queues/redisConnection.ts @@ -0,0 +1,19 @@ +import type { RedisOptions } from 'bullmq'; + +const redisUrl = process.env.REDIS_URL?.trim() || 'redis://localhost:6379'; + +const parseRedisConnection = (): RedisOptions => { + const url = new URL(redisUrl); + const db = url.pathname && url.pathname !== '/' ? Number(url.pathname.slice(1)) : undefined; + + return { + host: url.hostname, + port: url.port ? Number(url.port) : 6379, + username: url.username ? decodeURIComponent(url.username) : undefined, + password: url.password ? decodeURIComponent(url.password) : undefined, + db: Number.isFinite(db) ? db : undefined, + maxRetriesPerRequest: null, + }; +}; + +export const redisConnection = parseRedisConnection(); diff --git a/backend/src/worker.ts b/backend/src/worker.ts new file mode 100644 index 0000000..f0f435c --- /dev/null +++ b/backend/src/worker.ts @@ -0,0 +1,61 @@ +import { Worker } from 'bullmq'; + +import { ensureSchema } from './db/schema.js'; +import { db } from './db/client.js'; +import { runBirdMilestoneReminders, startBirdMilestoneReminderScheduler } from './app.js'; +import { + birdMilestoneReminderQueueName, + closeBirdMilestoneReminderQueue, + type BirdMilestoneReminderJobData, + type BirdMilestoneReminderJobResult, +} from './queues/birdMilestoneReminderQueue.js'; +import { redisConnection } from './queues/redisConnection.js'; + +let birdMilestoneWorker: Worker | null = null; + +const startWorker = async () => { + await ensureSchema(); + + birdMilestoneWorker = new Worker( + birdMilestoneReminderQueueName, + async (job) => { + const result = await runBirdMilestoneReminders(job.data.runDate); + console.log( + `Bird milestone reminder job completed for ${result.runDate}: checked=${result.checked}, sent=${result.sent}, skipped=${result.skipped}, failed=${result.failed}`, + ); + return result; + }, + { + connection: redisConnection, + concurrency: 1, + }, + ); + + birdMilestoneWorker.on('failed', (job, error) => { + console.error(`Bird milestone reminder job failed: id=${job?.id ?? 'unknown'}`, error); + }); + + startBirdMilestoneReminderScheduler(); + console.log('FlockPal worker started.'); +}; + +const shutdown = async (signal: string) => { + console.log(`FlockPal worker received ${signal}; shutting down.`); + await birdMilestoneWorker?.close(); + await closeBirdMilestoneReminderQueue(); + await db.close(); + process.exit(0); +}; + +process.on('SIGINT', () => { + void shutdown('SIGINT'); +}); + +process.on('SIGTERM', () => { + void shutdown('SIGTERM'); +}); + +startWorker().catch((error) => { + console.error('Failed to start FlockPal worker', error); + process.exit(1); +}); diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 91cff9f..f12bb87 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -15,6 +15,19 @@ services: retries: 10 restart: unless-stopped + redis: + image: redis:7-alpine + container_name: flockpal-redis + command: ["redis-server", "--appendonly", "yes"] + volumes: + - ./data/redis:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 10 + restart: unless-stopped + backend: build: context: ./backend @@ -29,10 +42,12 @@ services: POSTGRES_DB: ${POSTGRES_DB:-flockpal} POSTGRES_USER: ${POSTGRES_USER:-flockpal} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD for production} + REDIS_URL: ${REDIS_URL:-redis://redis:6379} FRONTEND_URL: ${FRONTEND_URL:?set FRONTEND_URL for production} BACKEND_URL: ${BACKEND_URL:?set BACKEND_URL for production} ADMIN_EMAILS: ${ADMIN_EMAILS:-} RESCUE_STATUS_NOTIFICATION_EMAIL: ${RESCUE_STATUS_NOTIFICATION_EMAIL:-appadmin@flockpal.app} + RESCUE_ONBOARDING_WEBHOOK_URL: ${RESCUE_ONBOARDING_WEBHOOK_URL:-https://n8n.blaishome.online/webhook/395cd538-5e0d-4e89-8070-9e66f571b7ee} MILESTONE_REMINDERS_ENABLED: ${MILESTONE_REMINDERS_ENABLED:-true} MILESTONE_REMINDER_TIME_ZONE: ${MILESTONE_REMINDER_TIME_ZONE:-America/New_York} GOOGLE_CLIENT_ID: ${GOOGLE_CLIENT_ID:-} @@ -65,6 +80,8 @@ services: depends_on: postgres: condition: service_healthy + redis: + condition: service_healthy labels: - traefik.enable=true - traefik.docker.network=traefik @@ -79,6 +96,42 @@ services: - traefik restart: unless-stopped + worker: + build: + context: ./backend + dockerfile: Dockerfile + container_name: flockpal-worker + command: ["npm", "run", "worker"] + environment: + NODE_ENV: production + TRUST_PROXY: ${TRUST_PROXY:-1} + POSTGRES_HOST: postgres + POSTGRES_PORT: 5432 + POSTGRES_DB: ${POSTGRES_DB:-flockpal} + POSTGRES_USER: ${POSTGRES_USER:-flockpal} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD for production} + REDIS_URL: ${REDIS_URL:-redis://redis:6379} + FRONTEND_URL: ${FRONTEND_URL:?set FRONTEND_URL for production} + BACKEND_URL: ${BACKEND_URL:?set BACKEND_URL for production} + ADMIN_EMAILS: ${ADMIN_EMAILS:-} + RESCUE_STATUS_NOTIFICATION_EMAIL: ${RESCUE_STATUS_NOTIFICATION_EMAIL:-appadmin@flockpal.app} + RESCUE_ONBOARDING_WEBHOOK_URL: ${RESCUE_ONBOARDING_WEBHOOK_URL:-https://n8n.blaishome.online/webhook/395cd538-5e0d-4e89-8070-9e66f571b7ee} + MILESTONE_REMINDERS_ENABLED: ${MILESTONE_REMINDERS_ENABLED:-true} + MILESTONE_REMINDER_TIME_ZONE: ${MILESTONE_REMINDER_TIME_ZONE:-America/New_York} + SMTP_HOST: ${SMTP_HOST:-} + SMTP_PORT: ${SMTP_PORT:-587} + SMTP_SECURE: ${SMTP_SECURE:-false} + SMTP_USER: ${SMTP_USER:-} + SMTP_PASS: ${SMTP_PASS:-} + SMTP_FROM_EMAIL: ${SMTP_FROM_EMAIL:-} + SMTP_FROM_NAME: ${SMTP_FROM_NAME:-FlockPal} + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + restart: unless-stopped + frontend: build: context: ./frontend diff --git a/docker-compose.yml b/docker-compose.yml index 08f65b1..409332c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,18 @@ services: timeout: 5s retries: 10 + redis: + image: redis:7-alpine + container_name: flockpal-redis + command: ["redis-server", "--appendonly", "yes"] + volumes: + - ./data/redis:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 10 + backend: build: context: ./backend @@ -28,10 +40,12 @@ services: POSTGRES_DB: ${POSTGRES_DB:-flockpal} POSTGRES_USER: ${POSTGRES_USER:-flockpal} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-flockpal_dev_password} + REDIS_URL: ${REDIS_URL:-redis://redis:6379} FRONTEND_URL: ${FRONTEND_URL:-http://localhost:3000} BACKEND_URL: ${BACKEND_URL:-http://localhost:5000} ADMIN_EMAILS: ${ADMIN_EMAILS:-} RESCUE_STATUS_NOTIFICATION_EMAIL: ${RESCUE_STATUS_NOTIFICATION_EMAIL:-appadmin@flockpal.app} + RESCUE_ONBOARDING_WEBHOOK_URL: ${RESCUE_ONBOARDING_WEBHOOK_URL:-https://n8n.blaishome.online/webhook/395cd538-5e0d-4e89-8070-9e66f571b7ee} MILESTONE_REMINDERS_ENABLED: ${MILESTONE_REMINDERS_ENABLED:-true} MILESTONE_REMINDER_TIME_ZONE: ${MILESTONE_REMINDER_TIME_ZONE:-America/New_York} GOOGLE_CLIENT_ID: ${GOOGLE_CLIENT_ID:-} @@ -64,6 +78,8 @@ services: depends_on: postgres: condition: service_healthy + redis: + condition: service_healthy ports: - "5000:5000" command: > @@ -73,6 +89,47 @@ services: - ./backend:/app - /app/node_modules + worker: + build: + context: ./backend + dockerfile: Dockerfile.dev + container_name: flockpal-worker + environment: + PORT: 5000 + NODE_ENV: development + TRUST_PROXY: ${TRUST_PROXY:-} + POSTGRES_HOST: postgres + POSTGRES_PORT: 5432 + POSTGRES_DB: ${POSTGRES_DB:-flockpal} + POSTGRES_USER: ${POSTGRES_USER:-flockpal} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-flockpal_dev_password} + REDIS_URL: ${REDIS_URL:-redis://redis:6379} + FRONTEND_URL: ${FRONTEND_URL:-http://localhost:3000} + BACKEND_URL: ${BACKEND_URL:-http://localhost:5000} + ADMIN_EMAILS: ${ADMIN_EMAILS:-} + RESCUE_STATUS_NOTIFICATION_EMAIL: ${RESCUE_STATUS_NOTIFICATION_EMAIL:-appadmin@flockpal.app} + RESCUE_ONBOARDING_WEBHOOK_URL: ${RESCUE_ONBOARDING_WEBHOOK_URL:-https://n8n.blaishome.online/webhook/395cd538-5e0d-4e89-8070-9e66f571b7ee} + MILESTONE_REMINDERS_ENABLED: ${MILESTONE_REMINDERS_ENABLED:-true} + MILESTONE_REMINDER_TIME_ZONE: ${MILESTONE_REMINDER_TIME_ZONE:-America/New_York} + SMTP_HOST: ${SMTP_HOST:-} + SMTP_PORT: ${SMTP_PORT:-587} + SMTP_SECURE: ${SMTP_SECURE:-false} + SMTP_USER: ${SMTP_USER:-} + SMTP_PASS: ${SMTP_PASS:-} + SMTP_FROM_EMAIL: ${SMTP_FROM_EMAIL:-} + SMTP_FROM_NAME: ${SMTP_FROM_NAME:-FlockPal} + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + command: > + sh -c "npm install --include=dev && + npm run worker:dev" + volumes: + - ./backend:/app + - /app/node_modules + frontend: build: context: ./frontend diff --git a/docs/API_REFERENCE.md b/docs/API_REFERENCE.md index cb0a171..14e18fd 100644 --- a/docs/API_REFERENCE.md +++ b/docs/API_REFERENCE.md @@ -323,6 +323,61 @@ Response `200`: { "ok": true } ``` +### Metrics + +#### `GET /api/metrics` + +Requires auth and an admin user. Returns lightweight in-memory process and request counters for the current backend process. + +Response `200`: + +```json +{ + "startedAt": "2026-05-01T00:00:00.000Z", + "uptimeSeconds": 120, + "requests": { + "total": 10, + "inFlight": 0, + "errors": 0, + "averageDurationMs": 12.5, + "byStatus": { "2xx": 10 }, + "byRoute": { "GET /api/health": 10 } + }, + "memory": { + "rss": 0, + "heapTotal": 0, + "heapUsed": 0, + "external": 0, + "arrayBuffers": 0 + }, + "queues": { + "birdMilestoneReminders": { + "waiting": 0, + "active": 0, + "delayed": 0, + "completed": 0, + "failed": 0 + } + } +} +``` + +### Admin + +#### `PATCH /api/admin/rescue-workspaces/:workspaceId` + +Requires an admin user. Browser session tokens and admin-owned `read_write` integration tokens are accepted, so automation tools can approve or reject rescue claims after external validation. + +Request body: + +```json +{ + "rescueVerificationStatus": "approved" +} +``` + +Allowed statuses are `pending`, `approved`, and `rejected`. + ### Authentication #### `GET /api/auth/providers` diff --git a/scripts/backup-postgres.sh b/scripts/backup-postgres.sh new file mode 100755 index 0000000..7191590 --- /dev/null +++ b/scripts/backup-postgres.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail + +compose_file="${COMPOSE_FILE:-docker-compose.prod.yml}" +backup_dir="${BACKUP_DIR:-backups/postgres}" +timestamp="$(date -u +%Y%m%dT%H%M%SZ)" +backup_path="${backup_dir}/flockpal-${timestamp}.dump" + +mkdir -p "$backup_dir" + +docker compose -f "$compose_file" exec -T postgres sh -c 'pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=custom --no-owner --no-privileges' > "$backup_path" + +echo "$backup_path" diff --git a/scripts/restore-test-postgres.sh b/scripts/restore-test-postgres.sh new file mode 100755 index 0000000..aa6e357 --- /dev/null +++ b/scripts/restore-test-postgres.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 backups/postgres/flockpal-YYYYMMDDTHHMMSSZ.dump" >&2 + exit 2 +fi + +backup_path="$1" +compose_file="${COMPOSE_FILE:-docker-compose.prod.yml}" +restore_db="flockpal_restore_test_$(date -u +%Y%m%d%H%M%S)" + +if [[ ! -f "$backup_path" ]]; then + echo "Backup file not found: $backup_path" >&2 + exit 2 +fi + +cleanup() { + docker compose -f "$compose_file" exec -T postgres sh -c "dropdb -U \"\$POSTGRES_USER\" --if-exists \"$restore_db\"" >/dev/null +} +trap cleanup EXIT + +docker compose -f "$compose_file" exec -T postgres sh -c "createdb -U \"\$POSTGRES_USER\" \"$restore_db\"" +docker compose -f "$compose_file" exec -T postgres sh -c "pg_restore -U \"\$POSTGRES_USER\" -d \"$restore_db\" --no-owner --no-privileges" < "$backup_path" +docker compose -f "$compose_file" exec -T postgres sh -c "psql -U \"\$POSTGRES_USER\" -d \"$restore_db\" -v ON_ERROR_STOP=1 -c 'SELECT COUNT(*) AS workspaces FROM workspaces;'" + +echo "Restore test passed for $backup_path" From 1bb3002bafe77d786af5f76c2b24aac30386f680 Mon Sep 17 00:00:00 2001 From: blaisadmin Date: Sat, 2 May 2026 01:17:43 -0400 Subject: [PATCH 2/7] Adding Wasabi --- .env.example | 8 +++ README.md | 28 +++++++++ backend/src/app.ts | 3 + backend/src/db/schema.ts | 10 ++++ backend/src/repositories/birdRepository.ts | 13 +++-- backend/src/storage/imageStorageConfig.ts | 67 ++++++++++++++++++++++ backend/src/types.ts | 3 + docker-compose.prod.yml | 16 ++++++ docker-compose.yml | 16 ++++++ 9 files changed, 159 insertions(+), 5 deletions(-) create mode 100644 backend/src/storage/imageStorageConfig.ts diff --git a/.env.example b/.env.example index 3849fbf..cbbe7e0 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,14 @@ POSTGRES_DB=flockpal POSTGRES_USER=flockpal POSTGRES_PASSWORD=change_me_for_production REDIS_URL=redis://redis:6379 +IMAGE_STORAGE_PROVIDER=database +S3_ENDPOINT= +S3_REGION= +S3_BUCKET= +S3_ACCESS_KEY_ID= +S3_SECRET_ACCESS_KEY= +S3_PUBLIC_BASE_URL= +S3_KEY_PREFIX=bird-photos FRONTEND_URL=http://localhost:3000 BACKEND_URL=http://localhost:5000 VITE_API_BASE_URL=http://localhost:5000/api diff --git a/README.md b/README.md index 9e2092e..fa80dcc 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,12 @@ curl -H "Authorization: Bearer " https://your-host/api/metrics - `BACKEND_URL` - `VITE_API_BASE_URL` - `REDIS_URL` + - `IMAGE_STORAGE_PROVIDER` + - `S3_ENDPOINT` + - `S3_REGION` + - `S3_BUCKET` + - `S3_ACCESS_KEY_ID` + - `S3_SECRET_ACCESS_KEY` - `RESCUE_ONBOARDING_WEBHOOK_URL` 2. Build and start the production stack: @@ -104,6 +110,28 @@ Compose includes a Redis service at `redis://redis:6379` and passes that value t Scheduled milestone reminders are enqueued through Redis with a per-date job id, then processed by the worker. This keeps scheduled work out of API containers and prevents duplicate scheduled jobs when the API is scaled horizontally. Redis can also support later shared rate-limit state and short-lived cache entries. +## Image storage + +FlockPal currently keeps bird photos in Postgres as `photo_data_url`. The schema also has S3 object metadata columns so image storage can move to Wasabi/S3 without changing the bird record contract. + +Set these when Wasabi image storage is ready: + +- `IMAGE_STORAGE_PROVIDER=s3` +- `S3_ENDPOINT=https://s3..wasabisys.com` +- `S3_REGION=` +- `S3_BUCKET=` +- `S3_ACCESS_KEY_ID=` +- `S3_SECRET_ACCESS_KEY=` +- `S3_PUBLIC_BASE_URL=` +- `S3_KEY_PREFIX=bird-photos` + +Use a dedicated bucket and access key for FlockPal images. Grant only the S3 permissions the app needs for that bucket. + +Bucket settings recommendation: + +- Enable bucket versioning if you want rollback protection from accidental overwrites or deletes. Add a lifecycle policy once upload volume is known because every object version contributes to stored data. +- Do not enable Object Lock on the primary app image bucket unless there is a strict legal/compliance retention requirement. Object Lock must be enabled when creating the bucket, depends on versioning, and can make user-requested image deletion or replacement harder. + ## Worker process The API container does not run scheduled reminder loops. Background reminders run in the `worker` service so the API can be scaled horizontally without multiple API containers sending duplicate scheduled emails. diff --git a/backend/src/app.ts b/backend/src/app.ts index 86a1eb2..d5c2e49 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -482,6 +482,9 @@ const normalizeBird = (row: BirdRow) => ({ gotchaDay: row.gotcha_day, chartColor: row.chart_color, photoDataUrl: row.photo_data_url, + photoObjectKey: row.photo_object_key, + photoContentType: row.photo_content_type, + photoUpdatedAt: row.photo_updated_at, notifyOnDob: row.notify_on_dob, notifyOnGotchaDay: row.notify_on_gotcha_day, memorializedAt: row.memorialized_at, diff --git a/backend/src/db/schema.ts b/backend/src/db/schema.ts index e4bdfd6..05a41db 100644 --- a/backend/src/db/schema.ts +++ b/backend/src/db/schema.ts @@ -221,6 +221,9 @@ export const ensureSchema = async (database: DatabaseClient = db) => { gotcha_day DATE, chart_color VARCHAR(7) NOT NULL DEFAULT '#cb3a35', photo_data_url TEXT, + photo_object_key TEXT, + photo_content_type VARCHAR(80), + photo_updated_at TIMESTAMPTZ, notify_on_dob BOOLEAN NOT NULL DEFAULT FALSE, notify_on_gotcha_day BOOLEAN NOT NULL DEFAULT FALSE, memorialized_at TIMESTAMPTZ, @@ -237,6 +240,9 @@ export const ensureSchema = async (database: DatabaseClient = db) => { ADD COLUMN IF NOT EXISTS gotcha_day DATE, ADD COLUMN IF NOT EXISTS chart_color VARCHAR(7) NOT NULL DEFAULT '#cb3a35', ADD COLUMN IF NOT EXISTS photo_data_url TEXT, + ADD COLUMN IF NOT EXISTS photo_object_key TEXT, + ADD COLUMN IF NOT EXISTS photo_content_type VARCHAR(80), + ADD COLUMN IF NOT EXISTS photo_updated_at TIMESTAMPTZ, ADD COLUMN IF NOT EXISTS notify_on_dob BOOLEAN NOT NULL DEFAULT FALSE, ADD COLUMN IF NOT EXISTS notify_on_gotcha_day BOOLEAN NOT NULL DEFAULT FALSE, ADD COLUMN IF NOT EXISTS memorialized_at TIMESTAMPTZ, @@ -287,6 +293,10 @@ export const ensureSchema = async (database: DatabaseClient = db) => { AND LOWER(BTRIM(tag_id)) NOT IN ('unknown', 'not recorded', 'n/a', 'na', 'none') AND memorialized_at IS NULL; + CREATE INDEX IF NOT EXISTS idx_birds_photo_object_key + ON birds (photo_object_key) + WHERE photo_object_key IS NOT NULL; + CREATE TABLE IF NOT EXISTS pending_bird_transfers ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), bird_id UUID NOT NULL REFERENCES birds(id) ON DELETE CASCADE, diff --git a/backend/src/repositories/birdRepository.ts b/backend/src/repositories/birdRepository.ts index 400fcb7..d0e6414 100644 --- a/backend/src/repositories/birdRepository.ts +++ b/backend/src/repositories/birdRepository.ts @@ -25,6 +25,9 @@ const birdSelectFields = ` birds.gotcha_day::text, birds.chart_color, birds.photo_data_url, + birds.photo_object_key, + birds.photo_content_type, + birds.photo_updated_at, birds.notify_on_dob, birds.notify_on_gotcha_day, birds.memorialized_at, @@ -277,7 +280,7 @@ export const createBird = async ({ const result = await db.query( `INSERT INTO birds (workspace_id, name, tag_id, species, gender, date_of_birth, gotcha_day, chart_color, photo_data_url, notify_on_dob, notify_on_gotcha_day) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) - RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, NULL::text AS latest_weight_grams, NULL::text AS latest_recorded_on`, + RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, photo_object_key, photo_content_type, photo_updated_at, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, NULL::text AS latest_weight_grams, NULL::text AS latest_recorded_on`, [workspaceId, name, tagId, species, gender, dateOfBirth, gotchaDay, chartColor, photoDataUrl, notifyOnDob, notifyOnGotchaDay], ); @@ -326,7 +329,7 @@ export const updateBird = async ({ WHERE id = $1 AND workspace_id = $12 AND memorialized_at IS NULL - RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, + RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, photo_object_key, photo_content_type, photo_updated_at, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, ( SELECT weight_grams::text FROM weight_records @@ -369,7 +372,7 @@ export const memorializeBird = async ({ WHERE id = $1 AND workspace_id = $2 AND memorialized_at IS NULL - RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, + RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, photo_object_key, photo_content_type, photo_updated_at, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, ( SELECT weight_grams::text FROM weight_records @@ -405,7 +408,7 @@ export const updateMemorialReminderPreference = async ({ WHERE id = $1 AND workspace_id = $2 AND memorialized_at IS NOT NULL - RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, + RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, photo_object_key, photo_content_type, photo_updated_at, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, ( SELECT weight_grams::text FROM weight_records @@ -445,7 +448,7 @@ export const transferBirdToWorkspace = async (birdId: string, sourceWorkspaceId: WHERE id = $1 AND workspace_id = $2 AND memorialized_at IS NULL - RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, + RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, photo_object_key, photo_content_type, photo_updated_at, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, ( SELECT weight_grams::text FROM weight_records diff --git a/backend/src/storage/imageStorageConfig.ts b/backend/src/storage/imageStorageConfig.ts new file mode 100644 index 0000000..10e3f5e --- /dev/null +++ b/backend/src/storage/imageStorageConfig.ts @@ -0,0 +1,67 @@ +export type ImageStorageProvider = 'database' | 's3'; + +export type S3ImageStorageConfig = { + provider: 's3'; + endpoint: string; + region: string; + bucket: string; + accessKeyId: string; + secretAccessKey: string; + publicBaseUrl: string | null; + keyPrefix: string; +}; + +const trimOptional = (value: string | undefined) => { + const trimmed = value?.trim(); + return trimmed ? trimmed : null; +}; + +export const getImageStorageProvider = (): ImageStorageProvider => + process.env.IMAGE_STORAGE_PROVIDER === 's3' ? 's3' : 'database'; + +export const getS3ImageStorageConfig = (): S3ImageStorageConfig | null => { + if (getImageStorageProvider() !== 's3') { + return null; + } + + const endpoint = trimOptional(process.env.S3_ENDPOINT); + const region = trimOptional(process.env.S3_REGION); + const bucket = trimOptional(process.env.S3_BUCKET); + const accessKeyId = trimOptional(process.env.S3_ACCESS_KEY_ID); + const secretAccessKey = trimOptional(process.env.S3_SECRET_ACCESS_KEY); + + if (!endpoint || !region || !bucket || !accessKeyId || !secretAccessKey) { + return null; + } + + return { + provider: 's3', + endpoint, + region, + bucket, + accessKeyId, + secretAccessKey, + publicBaseUrl: trimOptional(process.env.S3_PUBLIC_BASE_URL), + keyPrefix: trimOptional(process.env.S3_KEY_PREFIX) ?? 'bird-photos', + }; +}; + +export const isS3ImageStorageConfigured = () => getS3ImageStorageConfig() !== null; + +export const buildBirdPhotoObjectKey = ({ + workspaceId, + birdId, + extension, + now = new Date(), +}: { + workspaceId: number; + birdId: string; + extension: string; + now?: Date; +}) => { + const prefix = trimOptional(process.env.S3_KEY_PREFIX) ?? 'bird-photos'; + const safeExtension = extension.replace(/^\./, '').toLowerCase() || 'bin'; + const timestamp = now.toISOString().replace(/[:.]/g, '-'); + + return `${prefix}/workspace-${workspaceId}/${birdId}/${timestamp}.${safeExtension}`; +}; diff --git a/backend/src/types.ts b/backend/src/types.ts index 72e9b2b..b27850f 100644 --- a/backend/src/types.ts +++ b/backend/src/types.ts @@ -103,6 +103,9 @@ export type BirdRow = { gotcha_day: string | null; chart_color: string; photo_data_url: string | null; + photo_object_key: string | null; + photo_content_type: string | null; + photo_updated_at: string | null; notify_on_dob: boolean; notify_on_gotcha_day: boolean; memorialized_at: string | null; diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index f12bb87..63ae0cc 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -43,6 +43,14 @@ services: POSTGRES_USER: ${POSTGRES_USER:-flockpal} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD for production} REDIS_URL: ${REDIS_URL:-redis://redis:6379} + IMAGE_STORAGE_PROVIDER: ${IMAGE_STORAGE_PROVIDER:-database} + S3_ENDPOINT: ${S3_ENDPOINT:-} + S3_REGION: ${S3_REGION:-} + S3_BUCKET: ${S3_BUCKET:-} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-} + S3_PUBLIC_BASE_URL: ${S3_PUBLIC_BASE_URL:-} + S3_KEY_PREFIX: ${S3_KEY_PREFIX:-bird-photos} FRONTEND_URL: ${FRONTEND_URL:?set FRONTEND_URL for production} BACKEND_URL: ${BACKEND_URL:?set BACKEND_URL for production} ADMIN_EMAILS: ${ADMIN_EMAILS:-} @@ -111,6 +119,14 @@ services: POSTGRES_USER: ${POSTGRES_USER:-flockpal} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD for production} REDIS_URL: ${REDIS_URL:-redis://redis:6379} + IMAGE_STORAGE_PROVIDER: ${IMAGE_STORAGE_PROVIDER:-database} + S3_ENDPOINT: ${S3_ENDPOINT:-} + S3_REGION: ${S3_REGION:-} + S3_BUCKET: ${S3_BUCKET:-} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-} + S3_PUBLIC_BASE_URL: ${S3_PUBLIC_BASE_URL:-} + S3_KEY_PREFIX: ${S3_KEY_PREFIX:-bird-photos} FRONTEND_URL: ${FRONTEND_URL:?set FRONTEND_URL for production} BACKEND_URL: ${BACKEND_URL:?set BACKEND_URL for production} ADMIN_EMAILS: ${ADMIN_EMAILS:-} diff --git a/docker-compose.yml b/docker-compose.yml index 409332c..1e8ce9c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,6 +41,14 @@ services: POSTGRES_USER: ${POSTGRES_USER:-flockpal} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-flockpal_dev_password} REDIS_URL: ${REDIS_URL:-redis://redis:6379} + IMAGE_STORAGE_PROVIDER: ${IMAGE_STORAGE_PROVIDER:-database} + S3_ENDPOINT: ${S3_ENDPOINT:-} + S3_REGION: ${S3_REGION:-} + S3_BUCKET: ${S3_BUCKET:-} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-} + S3_PUBLIC_BASE_URL: ${S3_PUBLIC_BASE_URL:-} + S3_KEY_PREFIX: ${S3_KEY_PREFIX:-bird-photos} FRONTEND_URL: ${FRONTEND_URL:-http://localhost:3000} BACKEND_URL: ${BACKEND_URL:-http://localhost:5000} ADMIN_EMAILS: ${ADMIN_EMAILS:-} @@ -104,6 +112,14 @@ services: POSTGRES_USER: ${POSTGRES_USER:-flockpal} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-flockpal_dev_password} REDIS_URL: ${REDIS_URL:-redis://redis:6379} + IMAGE_STORAGE_PROVIDER: ${IMAGE_STORAGE_PROVIDER:-database} + S3_ENDPOINT: ${S3_ENDPOINT:-} + S3_REGION: ${S3_REGION:-} + S3_BUCKET: ${S3_BUCKET:-} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-} + S3_PUBLIC_BASE_URL: ${S3_PUBLIC_BASE_URL:-} + S3_KEY_PREFIX: ${S3_KEY_PREFIX:-bird-photos} FRONTEND_URL: ${FRONTEND_URL:-http://localhost:3000} BACKEND_URL: ${BACKEND_URL:-http://localhost:5000} ADMIN_EMAILS: ${ADMIN_EMAILS:-} From 22f344a998fb5ef2172d8153e04187708cf99f6b Mon Sep 17 00:00:00 2001 From: blaisadmin Date: Sat, 2 May 2026 10:24:58 -0400 Subject: [PATCH 3/7] fixed db scripts --- README.md | 4 +- backend/src/app.ts | 169 +++++++++++++++++- .../src/queues/birdMilestoneReminderQueue.ts | 2 +- backend/src/repositories/birdRepository.ts | 63 ++++++- backend/src/storage/imageStorageConfig.ts | 16 ++ backend/src/storage/s3Client.ts | 161 +++++++++++++++++ scripts/backup-postgres.sh | 4 + scripts/restore-test-postgres.sh | 4 + 8 files changed, 409 insertions(+), 14 deletions(-) create mode 100644 backend/src/storage/s3Client.ts diff --git a/README.md b/README.md index fa80dcc..11a83d8 100644 --- a/README.md +++ b/README.md @@ -122,10 +122,10 @@ Set these when Wasabi image storage is ready: - `S3_BUCKET=` - `S3_ACCESS_KEY_ID=` - `S3_SECRET_ACCESS_KEY=` -- `S3_PUBLIC_BASE_URL=` +- `S3_PUBLIC_BASE_URL=` - `S3_KEY_PREFIX=bird-photos` -Use a dedicated bucket and access key for FlockPal images. Grant only the S3 permissions the app needs for that bucket. +Use a dedicated private bucket and access key for FlockPal images. Grant only the S3 permissions the app needs for that bucket. When `S3_PUBLIC_BASE_URL` is blank, FlockPal stores private object keys and returns short-lived signed URLs for bird photos. Bucket settings recommendation: diff --git a/backend/src/app.ts b/backend/src/app.ts index d5c2e49..bf6b1a8 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -60,6 +60,13 @@ import { updateVetVisitForBird, } from './repositories/birdRepository.js'; import { createIntegrationTokenRecord, listIntegrationTokens, revokeIntegrationToken } from './repositories/integrationTokenRepository.js'; +import { + buildBirdPhotoObjectKey, + getImageExtensionFromContentType, + getImageStorageProvider, + getS3ImageStorageConfig, +} from './storage/imageStorageConfig.js'; +import { deleteS3Object, getSignedS3ObjectUrl, putS3Object } from './storage/s3Client.js'; import { cancelRescueVerificationRequest, claimWorkspaceInvites, @@ -160,6 +167,7 @@ const photoDataUrlSchema = z .string() .regex(/^data:image\/(?:png|jpeg|jpg|webp|gif);base64,[A-Za-z0-9+/=]+$/) .max(1_500_000); +const photoUrlSchema = z.string().trim().url().max(2000); const magicLinkRequestSchema = z.object({ name: z.string().trim().max(160).optional().or(z.literal('')), @@ -230,7 +238,7 @@ const birdSchema = z.object({ dateOfBirth: dateStringSchema.optional().or(z.literal('')), gotchaDay: dateStringSchema.optional().or(z.literal('')), chartColor: chartColorSchema.optional(), - photoDataUrl: photoDataUrlSchema.optional().or(z.literal('')), + photoDataUrl: z.union([photoDataUrlSchema, photoUrlSchema, z.literal('')]).optional(), notifyOnDob: z.boolean().optional(), notifyOnGotchaDay: z.boolean().optional(), }); @@ -471,6 +479,24 @@ const normalizeWorkspaceMember = (row: WorkspaceMemberRow) => ({ createdAt: row.created_at, }); +const getBirdPhotoUrl = (row: BirdRow) => { + if (!row.photo_object_key) { + return row.photo_data_url; + } + + const s3Config = getS3ImageStorageConfig(); + + if (!s3Config) { + return row.photo_data_url; + } + + return getSignedS3ObjectUrl({ + config: s3Config, + objectKey: row.photo_object_key, + expiresInSeconds: 15 * 60, + }); +}; + const normalizeBird = (row: BirdRow) => ({ id: row.id, workspaceId: row.workspace_id, @@ -481,7 +507,7 @@ const normalizeBird = (row: BirdRow) => ({ dateOfBirth: row.date_of_birth, gotchaDay: row.gotcha_day, chartColor: row.chart_color, - photoDataUrl: row.photo_data_url, + photoDataUrl: getBirdPhotoUrl(row), photoObjectKey: row.photo_object_key, photoContentType: row.photo_content_type, photoUpdatedAt: row.photo_updated_at, @@ -1021,6 +1047,107 @@ const parseDataImage = (dataUrl: string) => { }; }; +const isDataImageUrl = (value: string | null | undefined) => Boolean(value && value.startsWith('data:image/')); + +const resolveBirdPhotoStorage = async ({ + birdId, + workspaceId, + photoDataUrl, + existingBird, +}: { + birdId: string; + workspaceId: number; + photoDataUrl: string | null; + existingBird?: BirdRow | null; +}) => { + if (!photoDataUrl) { + return { + photoDataUrl: null, + photoObjectKey: null, + photoContentType: null, + photoUpdatedAt: null, + objectKeyToDelete: existingBird?.photo_object_key ?? null, + }; + } + + if (!isDataImageUrl(photoDataUrl)) { + if (existingBird?.photo_object_key) { + return { + photoDataUrl: null, + photoObjectKey: existingBird.photo_object_key, + photoContentType: existingBird.photo_content_type, + photoUpdatedAt: existingBird.photo_updated_at, + objectKeyToDelete: null, + }; + } + + return { + photoDataUrl, + photoObjectKey: null, + photoContentType: null, + photoUpdatedAt: null, + objectKeyToDelete: existingBird?.photo_object_key ?? null, + }; + } + + const parsedImage = parseDataImage(photoDataUrl); + + if (!parsedImage) { + throw new Error('Unable to process bird photo.'); + } + + if (getImageStorageProvider() !== 's3') { + return { + photoDataUrl, + photoObjectKey: null, + photoContentType: null, + photoUpdatedAt: null, + objectKeyToDelete: existingBird?.photo_object_key ?? null, + }; + } + + const s3Config = getS3ImageStorageConfig(); + + if (!s3Config) { + throw new Error('S3 image storage is enabled but not fully configured.'); + } + + const extension = getImageExtensionFromContentType(parsedImage.contentType); + const objectKey = buildBirdPhotoObjectKey({ workspaceId, birdId, extension }); + await putS3Object({ + config: s3Config, + objectKey, + content: parsedImage.content, + contentType: parsedImage.contentType, + }); + + return { + photoDataUrl: null, + photoObjectKey: objectKey, + photoContentType: parsedImage.contentType, + photoUpdatedAt: new Date().toISOString(), + objectKeyToDelete: existingBird?.photo_object_key ?? null, + }; +}; + +const deleteBirdPhotoObjectIfNeeded = async (objectKey: string | null) => { + if (!objectKey) { + return; + } + + const s3Config = getS3ImageStorageConfig(); + + if (!s3Config) { + return; + } + + try { + await deleteS3Object({ config: s3Config, objectKey }); + } catch (error) { + console.warn(`Unable to delete old bird photo object ${objectKey}:`, error); + } +}; + const getDefaultBirdPhotoAttachment = () => { const defaultPhotoPath = path.join(process.cwd(), 'assets', 'yoda-default.png'); @@ -2558,8 +2685,18 @@ app.post('/api/birds', requireAuth, requireWriteAccess, requireWorkspaceRole(['o return; } + let uploadedObjectKeyToCleanup: string | null = null; + try { + const birdId = crypto.randomUUID(); + const photoStorage = await resolveBirdPhotoStorage({ + birdId, + workspaceId: req.auth!.workspace.id, + photoDataUrl: emptyToNull(parsed.data.photoDataUrl), + }); + uploadedObjectKeyToCleanup = photoStorage.photoObjectKey; const bird = await createBird({ + birdId, workspaceId: req.auth!.workspace.id, name: parsed.data.name, tagId: normalizeBandId(parsed.data.tagId), @@ -2568,13 +2705,19 @@ app.post('/api/birds', requireAuth, requireWriteAccess, requireWorkspaceRole(['o dateOfBirth: emptyToNull(parsed.data.dateOfBirth), gotchaDay: emptyToNull(parsed.data.gotchaDay), chartColor: parsed.data.chartColor ?? '#cb3a35', - photoDataUrl: emptyToNull(parsed.data.photoDataUrl), + photoDataUrl: photoStorage.photoDataUrl, + photoObjectKey: photoStorage.photoObjectKey, + photoContentType: photoStorage.photoContentType, + photoUpdatedAt: photoStorage.photoUpdatedAt, notifyOnDob: parsed.data.notifyOnDob ?? false, notifyOnGotchaDay: parsed.data.notifyOnGotchaDay ?? false, }); + uploadedObjectKeyToCleanup = null; res.status(201).json({ bird: normalizeBird(bird!) }); } catch (error) { + await deleteBirdPhotoObjectIfNeeded(uploadedObjectKeyToCleanup); + if (typeof error === 'object' && error && 'code' in error && error.code === '23505') { res.status(409).json({ error: 'That band/tag ID is already in use in this flock.' }); return; @@ -2667,6 +2810,8 @@ app.put('/api/birds/:birdId', requireAuth, requireWriteAccess, requireWorkspaceR return; } + let uploadedObjectKeyToCleanup: string | null = null; + try { const existingBird = await getBirdById(req.params.birdId, req.auth!.workspace.id); @@ -2679,6 +2824,14 @@ app.put('/api/birds/:birdId', requireAuth, requireWriteAccess, requireWorkspaceR return; } + const photoStorage = await resolveBirdPhotoStorage({ + birdId: req.params.birdId, + workspaceId: req.auth!.workspace.id, + photoDataUrl: emptyToNull(parsed.data.photoDataUrl), + existingBird, + }); + uploadedObjectKeyToCleanup = + photoStorage.photoObjectKey && photoStorage.photoObjectKey !== existingBird.photo_object_key ? photoStorage.photoObjectKey : null; const bird = await updateBird({ birdId: req.params.birdId, workspaceId: req.auth!.workspace.id, @@ -2689,7 +2842,10 @@ app.put('/api/birds/:birdId', requireAuth, requireWriteAccess, requireWorkspaceR dateOfBirth: emptyToNull(parsed.data.dateOfBirth), gotchaDay: emptyToNull(parsed.data.gotchaDay), chartColor: parsed.data.chartColor ?? '#cb3a35', - photoDataUrl: emptyToNull(parsed.data.photoDataUrl), + photoDataUrl: photoStorage.photoDataUrl, + photoObjectKey: photoStorage.photoObjectKey, + photoContentType: photoStorage.photoContentType, + photoUpdatedAt: photoStorage.photoUpdatedAt, notifyOnDob: parsed.data.notifyOnDob ?? false, notifyOnGotchaDay: parsed.data.notifyOnGotchaDay ?? false, }); @@ -2699,8 +2855,12 @@ app.put('/api/birds/:birdId', requireAuth, requireWriteAccess, requireWorkspaceR return; } + uploadedObjectKeyToCleanup = null; + await deleteBirdPhotoObjectIfNeeded(photoStorage.objectKeyToDelete); res.json({ bird: normalizeBird(bird) }); } catch (error) { + await deleteBirdPhotoObjectIfNeeded(uploadedObjectKeyToCleanup); + if (typeof error === 'object' && error && 'code' in error && error.code === '23505') { res.status(409).json({ error: 'That band/tag ID is already in use in this flock.' }); return; @@ -2731,6 +2891,7 @@ app.delete('/api/birds/:birdId', requireAuth, requireWriteAccess, requireWorkspa } res.status(204).send(); + await deleteBirdPhotoObjectIfNeeded(bird.photo_object_key); } catch (error) { next(error); } diff --git a/backend/src/queues/birdMilestoneReminderQueue.ts b/backend/src/queues/birdMilestoneReminderQueue.ts index bcc6714..6f3ac49 100644 --- a/backend/src/queues/birdMilestoneReminderQueue.ts +++ b/backend/src/queues/birdMilestoneReminderQueue.ts @@ -41,7 +41,7 @@ export const enqueueBirdMilestoneReminderJob = (runDate: string): Promise { const result = await db.query( - `INSERT INTO birds (workspace_id, name, tag_id, species, gender, date_of_birth, gotcha_day, chart_color, photo_data_url, notify_on_dob, notify_on_gotcha_day) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + `INSERT INTO birds (id, workspace_id, name, tag_id, species, gender, date_of_birth, gotcha_day, chart_color, photo_data_url, photo_object_key, photo_content_type, photo_updated_at, notify_on_dob, notify_on_gotcha_day) + VALUES (COALESCE($1::uuid, gen_random_uuid()), $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, photo_object_key, photo_content_type, photo_updated_at, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, NULL::text AS latest_weight_grams, NULL::text AS latest_recorded_on`, - [workspaceId, name, tagId, species, gender, dateOfBirth, gotchaDay, chartColor, photoDataUrl, notifyOnDob, notifyOnGotchaDay], + [ + birdId ?? null, + workspaceId, + name, + tagId, + species, + gender, + dateOfBirth, + gotchaDay, + chartColor, + photoDataUrl, + photoObjectKey, + photoContentType, + photoUpdatedAt, + notifyOnDob, + notifyOnGotchaDay, + ], ); return result.rows[0] ?? null; @@ -298,6 +322,9 @@ export const updateBird = async ({ gotchaDay, chartColor, photoDataUrl, + photoObjectKey = null, + photoContentType = null, + photoUpdatedAt = null, notifyOnDob, notifyOnGotchaDay, }: { @@ -311,6 +338,9 @@ export const updateBird = async ({ gotchaDay: string | null; chartColor: string; photoDataUrl: string | null; + photoObjectKey?: string | null; + photoContentType?: string | null; + photoUpdatedAt?: string | null; notifyOnDob: boolean; notifyOnGotchaDay: boolean; }) => { @@ -324,10 +354,13 @@ export const updateBird = async ({ gotcha_day = $7, chart_color = $8, photo_data_url = $9, - notify_on_dob = $10, - notify_on_gotcha_day = $11 + photo_object_key = $10, + photo_content_type = $11, + photo_updated_at = $12, + notify_on_dob = $13, + notify_on_gotcha_day = $14 WHERE id = $1 - AND workspace_id = $12 + AND workspace_id = $15 AND memorialized_at IS NULL RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, photo_object_key, photo_content_type, photo_updated_at, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, ( @@ -344,7 +377,23 @@ export const updateBird = async ({ ORDER BY recorded_on DESC LIMIT 1 ) AS latest_recorded_on`, - [birdId, name, tagId, species, gender, dateOfBirth, gotchaDay, chartColor, photoDataUrl, notifyOnDob, notifyOnGotchaDay, workspaceId], + [ + birdId, + name, + tagId, + species, + gender, + dateOfBirth, + gotchaDay, + chartColor, + photoDataUrl, + photoObjectKey, + photoContentType, + photoUpdatedAt, + notifyOnDob, + notifyOnGotchaDay, + workspaceId, + ], ); return result.rows[0] ?? null; diff --git a/backend/src/storage/imageStorageConfig.ts b/backend/src/storage/imageStorageConfig.ts index 10e3f5e..8d47d57 100644 --- a/backend/src/storage/imageStorageConfig.ts +++ b/backend/src/storage/imageStorageConfig.ts @@ -65,3 +65,19 @@ export const buildBirdPhotoObjectKey = ({ return `${prefix}/workspace-${workspaceId}/${birdId}/${timestamp}.${safeExtension}`; }; + +export const getImageExtensionFromContentType = (contentType: string) => { + switch (contentType.toLowerCase()) { + case 'image/jpeg': + case 'image/jpg': + return 'jpg'; + case 'image/png': + return 'png'; + case 'image/webp': + return 'webp'; + case 'image/gif': + return 'gif'; + default: + return 'bin'; + } +}; diff --git a/backend/src/storage/s3Client.ts b/backend/src/storage/s3Client.ts new file mode 100644 index 0000000..ab584c6 --- /dev/null +++ b/backend/src/storage/s3Client.ts @@ -0,0 +1,161 @@ +import crypto from 'crypto'; + +import type { S3ImageStorageConfig } from './imageStorageConfig.js'; + +const awsDate = (date: Date) => date.toISOString().replace(/[:-]|\.\d{3}/g, ''); +const shortDate = (date: Date) => date.toISOString().slice(0, 10).replace(/-/g, ''); + +const hmac = (key: crypto.BinaryLike, value: string) => crypto.createHmac('sha256', key).update(value).digest(); +const sha256Hex = (value: crypto.BinaryLike) => crypto.createHash('sha256').update(value).digest('hex'); + +const encodeObjectKey = (key: string) => key.split('/').map(encodeURIComponent).join('/'); +const encodeQueryValue = (value: string) => encodeURIComponent(value).replace(/[!'()*]/g, (character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}`); + +const getSigningKey = (secretAccessKey: string, date: string, region: string) => { + const dateKey = hmac(`AWS4${secretAccessKey}`, date); + const regionKey = hmac(dateKey, region); + const serviceKey = hmac(regionKey, 's3'); + return hmac(serviceKey, 'aws4_request'); +}; + +const buildObjectUrl = (config: S3ImageStorageConfig, objectKey: string) => { + const endpoint = config.endpoint.replace(/\/+$/, ''); + return new URL(`${endpoint}/${encodeURIComponent(config.bucket)}/${encodeObjectKey(objectKey)}`); +}; + +const signS3Request = ({ + config, + method, + objectKey, + contentHash, + contentType, + now = new Date(), +}: { + config: S3ImageStorageConfig; + method: 'DELETE' | 'PUT'; + objectKey: string; + contentHash: string; + contentType?: string; + now?: Date; +}) => { + const url = buildObjectUrl(config, objectKey); + const amzDate = awsDate(now); + const date = shortDate(now); + const credentialScope = `${date}/${config.region}/s3/aws4_request`; + const headers: Record = { + host: url.host, + 'x-amz-content-sha256': contentHash, + 'x-amz-date': amzDate, + }; + + if (contentType) { + headers['content-type'] = contentType; + } + + const signedHeaders = Object.keys(headers).sort().join(';'); + const canonicalHeaders = Object.keys(headers) + .sort() + .map((key) => `${key}:${headers[key]}\n`) + .join(''); + const canonicalRequest = [method, url.pathname, url.search.slice(1), canonicalHeaders, signedHeaders, contentHash].join('\n'); + const stringToSign = ['AWS4-HMAC-SHA256', amzDate, credentialScope, sha256Hex(canonicalRequest)].join('\n'); + const signature = crypto.createHmac('sha256', getSigningKey(config.secretAccessKey, date, config.region)).update(stringToSign).digest('hex'); + + return { + url, + headers: { + ...headers, + Authorization: `AWS4-HMAC-SHA256 Credential=${config.accessKeyId}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}`, + }, + }; +}; + +export const getPublicObjectUrl = (config: S3ImageStorageConfig, objectKey: string) => { + if (config.publicBaseUrl) { + return `${config.publicBaseUrl.replace(/\/+$/, '')}/${encodeObjectKey(objectKey)}`; + } + + return buildObjectUrl(config, objectKey).toString(); +}; + +export const getSignedS3ObjectUrl = ({ + config, + objectKey, + expiresInSeconds = 900, + now = new Date(), +}: { + config: S3ImageStorageConfig; + objectKey: string; + expiresInSeconds?: number; + now?: Date; +}) => { + const url = buildObjectUrl(config, objectKey); + const amzDate = awsDate(now); + const date = shortDate(now); + const credentialScope = `${date}/${config.region}/s3/aws4_request`; + const credential = `${config.accessKeyId}/${credentialScope}`; + const queryParams: Record = { + 'X-Amz-Algorithm': 'AWS4-HMAC-SHA256', + 'X-Amz-Credential': credential, + 'X-Amz-Date': amzDate, + 'X-Amz-Expires': String(Math.min(Math.max(expiresInSeconds, 1), 604800)), + 'X-Amz-SignedHeaders': 'host', + }; + const canonicalQuery = Object.keys(queryParams) + .sort() + .map((key) => `${encodeQueryValue(key)}=${encodeQueryValue(queryParams[key])}`) + .join('&'); + const canonicalHeaders = `host:${url.host}\n`; + const canonicalRequest = ['GET', url.pathname, canonicalQuery, canonicalHeaders, 'host', 'UNSIGNED-PAYLOAD'].join('\n'); + const stringToSign = ['AWS4-HMAC-SHA256', amzDate, credentialScope, sha256Hex(canonicalRequest)].join('\n'); + const signature = crypto.createHmac('sha256', getSigningKey(config.secretAccessKey, date, config.region)).update(stringToSign).digest('hex'); + + url.search = `${canonicalQuery}&X-Amz-Signature=${signature}`; + return url.toString(); +}; + +export const putS3Object = async ({ + config, + objectKey, + content, + contentType, +}: { + config: S3ImageStorageConfig; + objectKey: string; + content: Buffer; + contentType: string; +}) => { + const contentHash = sha256Hex(content); + const signed = signS3Request({ config, method: 'PUT', objectKey, contentHash, contentType }); + const response = await fetch(signed.url, { + method: 'PUT', + headers: signed.headers, + body: content, + }); + + if (!response.ok) { + const errorText = await response.text().catch(() => ''); + throw new Error(`Wasabi upload failed with ${response.status}${errorText ? `: ${errorText.slice(0, 300)}` : ''}`); + } + + return getPublicObjectUrl(config, objectKey); +}; + +export const deleteS3Object = async ({ + config, + objectKey, +}: { + config: S3ImageStorageConfig; + objectKey: string; +}) => { + const signed = signS3Request({ config, method: 'DELETE', objectKey, contentHash: sha256Hex('') }); + const response = await fetch(signed.url, { + method: 'DELETE', + headers: signed.headers, + }); + + if (!response.ok && response.status !== 404) { + const errorText = await response.text().catch(() => ''); + throw new Error(`Wasabi delete failed with ${response.status}${errorText ? `: ${errorText.slice(0, 300)}` : ''}`); + } +}; diff --git a/scripts/backup-postgres.sh b/scripts/backup-postgres.sh index 7191590..341522a 100755 --- a/scripts/backup-postgres.sh +++ b/scripts/backup-postgres.sh @@ -1,4 +1,8 @@ #!/usr/bin/env bash +if [ -z "${BASH_VERSION:-}" ]; then + exec bash "$0" "$@" +fi + set -euo pipefail compose_file="${COMPOSE_FILE:-docker-compose.prod.yml}" diff --git a/scripts/restore-test-postgres.sh b/scripts/restore-test-postgres.sh index aa6e357..63dae2b 100755 --- a/scripts/restore-test-postgres.sh +++ b/scripts/restore-test-postgres.sh @@ -1,4 +1,8 @@ #!/usr/bin/env bash +if [ -z "${BASH_VERSION:-}" ]; then + exec bash "$0" "$@" +fi + set -euo pipefail if [[ $# -ne 1 ]]; then From fc6d7c2762549de618f1d6a395f758346eff6cb5 Mon Sep 17 00:00:00 2001 From: blaisadmin Date: Sat, 2 May 2026 10:35:11 -0400 Subject: [PATCH 4/7] Updating migration script --- README.md | 8 ++ backend/package.json | 2 + backend/src/scripts/migrateBirdPhotosToS3.ts | 134 +++++++++++++++++++ 3 files changed, 144 insertions(+) create mode 100644 backend/src/scripts/migrateBirdPhotosToS3.ts diff --git a/README.md b/README.md index 11a83d8..5be53ff 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,14 @@ Set these when Wasabi image storage is ready: Use a dedicated private bucket and access key for FlockPal images. Grant only the S3 permissions the app needs for that bucket. When `S3_PUBLIC_BASE_URL` is blank, FlockPal stores private object keys and returns short-lived signed URLs for bird photos. +Migrate existing Postgres-stored bird photos after deploying S3 image storage: + +```bash +docker compose -f docker-compose.prod.yml exec backend npm run migrate:bird-photos-to-s3 -- --apply +``` + +Run a dry run first by omitting `--apply`. Use `--limit=10` to migrate a small batch, and `--keep-data-url` if you want to leave the original inline image in Postgres during an initial verification pass. + Bucket settings recommendation: - Enable bucket versioning if you want rollback protection from accidental overwrites or deletes. Add a lifecycle policy once upload volume is known because every object version contributes to stored data. diff --git a/backend/package.json b/backend/package.json index 2e7023f..7adcb12 100644 --- a/backend/package.json +++ b/backend/package.json @@ -8,6 +8,8 @@ "worker:dev": "tsx watch src/worker.ts", "build": "tsc", "test": "tsx --test src/**/*.test.ts", + "migrate:bird-photos-to-s3": "node dist/scripts/migrateBirdPhotosToS3.js", + "migrate:bird-photos-to-s3:dev": "tsx src/scripts/migrateBirdPhotosToS3.ts", "start": "node dist/app.js", "worker": "node dist/worker.js" }, diff --git a/backend/src/scripts/migrateBirdPhotosToS3.ts b/backend/src/scripts/migrateBirdPhotosToS3.ts new file mode 100644 index 0000000..85fa41e --- /dev/null +++ b/backend/src/scripts/migrateBirdPhotosToS3.ts @@ -0,0 +1,134 @@ +import { db } from '../db/client.js'; +import { ensureSchema } from '../db/schema.js'; +import { buildBirdPhotoObjectKey, getImageExtensionFromContentType, getS3ImageStorageConfig } from '../storage/imageStorageConfig.js'; +import { deleteS3Object, putS3Object } from '../storage/s3Client.js'; + +type BirdPhotoMigrationRow = { + id: string; + workspace_id: number; + name: string; + photo_data_url: string; +}; + +const parseDataImage = (dataUrl: string) => { + const match = /^data:(image\/[a-zA-Z0-9.+-]+);base64,(.+)$/.exec(dataUrl); + if (!match) { + return null; + } + + return { + contentType: match[1], + content: Buffer.from(match[2], 'base64'), + }; +}; + +const getArgValue = (name: string) => { + const prefix = `${name}=`; + const match = process.argv.find((arg) => arg.startsWith(prefix)); + return match ? match.slice(prefix.length) : null; +}; + +const dryRun = !process.argv.includes('--apply'); +const keepDataUrl = process.argv.includes('--keep-data-url'); +const limitArg = getArgValue('--limit'); +const limit = limitArg ? Number(limitArg) : null; + +if (limit !== null && (!Number.isInteger(limit) || limit <= 0)) { + console.error('Invalid --limit value. Use a positive integer.'); + process.exit(2); +} + +const run = async () => { + await ensureSchema(); + + const s3Config = getS3ImageStorageConfig(); + + if (!s3Config) { + throw new Error('S3 image storage is not fully configured. Set IMAGE_STORAGE_PROVIDER=s3 and the S3_* environment variables.'); + } + + const result = await db.query( + `SELECT id, workspace_id, name, photo_data_url + FROM birds + WHERE photo_object_key IS NULL + AND photo_data_url LIKE 'data:image/%' + ORDER BY created_at ASC + ${limit ? 'LIMIT $1' : ''}`, + limit ? [limit] : undefined, + ); + + if (dryRun) { + console.log(`Dry run: ${result.rows.length} bird photo(s) would be migrated to bucket ${s3Config.bucket}.`); + console.log('Run with --apply to upload objects and update rows.'); + return; + } + + let migrated = 0; + let skipped = 0; + let failed = 0; + + for (const bird of result.rows) { + const parsedImage = parseDataImage(bird.photo_data_url); + + if (!parsedImage) { + skipped += 1; + console.warn(`Skipping bird ${bird.id} (${bird.name}): invalid data URL.`); + continue; + } + + const objectKey = buildBirdPhotoObjectKey({ + workspaceId: bird.workspace_id, + birdId: bird.id, + extension: getImageExtensionFromContentType(parsedImage.contentType), + }); + + try { + await putS3Object({ + config: s3Config, + objectKey, + content: parsedImage.content, + contentType: parsedImage.contentType, + }); + + const updateResult = await db.query( + `UPDATE birds + SET photo_object_key = $2, + photo_content_type = $3, + photo_updated_at = CURRENT_TIMESTAMP, + photo_data_url = CASE WHEN $4::boolean THEN photo_data_url ELSE NULL END + WHERE id = $1 + AND photo_object_key IS NULL + AND photo_data_url LIKE 'data:image/%'`, + [bird.id, objectKey, parsedImage.contentType, keepDataUrl], + ); + + if (updateResult.rowCount !== 1) { + await deleteS3Object({ config: s3Config, objectKey }); + skipped += 1; + console.warn(`Skipped bird ${bird.id} (${bird.name}): row changed before update.`); + continue; + } + + migrated += 1; + console.log(`Migrated bird ${bird.id} (${bird.name}) -> ${objectKey}`); + } catch (error) { + failed += 1; + console.error(`Failed to migrate bird ${bird.id} (${bird.name}):`, error); + } + } + + console.log(`Migration complete: migrated=${migrated}, skipped=${skipped}, failed=${failed}`); + + if (failed > 0) { + process.exitCode = 1; + } +}; + +run() + .catch((error) => { + console.error('Bird photo migration failed:', error); + process.exitCode = 1; + }) + .finally(async () => { + await db.close(); + }); From 01541c5f5c787edbe4d95eee4f8859457e91e046 Mon Sep 17 00:00:00 2001 From: blaisadmin Date: Sat, 2 May 2026 11:50:31 -0400 Subject: [PATCH 5/7] Fixing images --- backend/src/app.ts | 122 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 117 insertions(+), 5 deletions(-) diff --git a/backend/src/app.ts b/backend/src/app.ts index bf6b1a8..3e74e13 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -319,6 +319,7 @@ const normalizeBandId = (value?: string | null) => { const normalizeEmail = (value: string) => value.trim().toLowerCase(); const hashToken = (token: string) => crypto.createHash('sha256').update(token).digest('hex'); const createSessionToken = () => crypto.randomBytes(32).toString('hex'); +const photoAccessSecret = process.env.PHOTO_ACCESS_SECRET?.trim() || process.env.S3_SECRET_ACCESS_KEY?.trim() || createSessionToken(); const createIntegrationToken = () => `flpt_${crypto.randomBytes(24).toString('hex')}`; const createRandomId = () => crypto.randomUUID(); const createCodeVerifier = () => crypto.randomBytes(32).toString('base64url'); @@ -479,6 +480,66 @@ const normalizeWorkspaceMember = (row: WorkspaceMemberRow) => ({ createdAt: row.created_at, }); +const signBirdPhotoAccessToken = (row: BirdRow) => { + if (!row.photo_object_key) { + return ''; + } + + const expiresAt = Math.floor(Date.now() / 1000) + 15 * 60; + const payload = Buffer.from( + JSON.stringify({ + birdId: row.id, + workspaceId: row.workspace_id, + objectKey: row.photo_object_key, + expiresAt, + }), + ).toString('base64url'); + const signature = crypto.createHmac('sha256', photoAccessSecret).update(payload).digest('base64url'); + + return `${payload}.${signature}`; +}; + +const verifyBirdPhotoAccessToken = (token: string) => { + const [payload, signature] = token.split('.'); + + if (!payload || !signature) { + return null; + } + + const expectedSignature = crypto.createHmac('sha256', photoAccessSecret).update(payload).digest('base64url'); + + const signatureBuffer = Buffer.from(signature); + const expectedSignatureBuffer = Buffer.from(expectedSignature); + + if (signatureBuffer.length !== expectedSignatureBuffer.length || !crypto.timingSafeEqual(signatureBuffer, expectedSignatureBuffer)) { + return null; + } + + const parsed = JSON.parse(Buffer.from(payload, 'base64url').toString('utf8')) as { + birdId?: unknown; + workspaceId?: unknown; + objectKey?: unknown; + expiresAt?: unknown; + }; + + if ( + typeof parsed.birdId !== 'string' || + typeof parsed.workspaceId !== 'number' || + typeof parsed.objectKey !== 'string' || + typeof parsed.expiresAt !== 'number' || + parsed.expiresAt < Math.floor(Date.now() / 1000) + ) { + return null; + } + + return parsed as { + birdId: string; + workspaceId: number; + objectKey: string; + expiresAt: number; + }; +}; + const getBirdPhotoUrl = (row: BirdRow) => { if (!row.photo_object_key) { return row.photo_data_url; @@ -490,11 +551,9 @@ const getBirdPhotoUrl = (row: BirdRow) => { return row.photo_data_url; } - return getSignedS3ObjectUrl({ - config: s3Config, - objectKey: row.photo_object_key, - expiresInSeconds: 15 * 60, - }); + const photoUrl = new URL(`${backendBaseUrl}/api/birds/${row.id}/photo`); + photoUrl.searchParams.set('token', signBirdPhotoAccessToken(row)); + return photoUrl.toString(); }; const normalizeBird = (row: BirdRow) => ({ @@ -2677,6 +2736,59 @@ app.get('/api/birds', requireAuth, async (req: Request, res: Response, next: Nex } }); +app.get('/api/birds/:birdId/photo', async (req: Request, res: Response, next: NextFunction) => { + try { + const token = typeof req.query.token === 'string' ? req.query.token : ''; + const photoAccess = verifyBirdPhotoAccessToken(token); + + if (!photoAccess || photoAccess.birdId !== req.params.birdId) { + res.status(403).json({ error: 'Photo link expired or invalid.' }); + return; + } + + const bird = await getBirdById(photoAccess.birdId, photoAccess.workspaceId); + + if (!bird || bird.photo_object_key !== photoAccess.objectKey) { + res.status(404).json({ error: 'Photo not found.' }); + return; + } + + const s3Config = getS3ImageStorageConfig(); + + if (!s3Config) { + res.status(503).json({ error: 'Image storage is not configured.' }); + return; + } + + const signedUrl = getSignedS3ObjectUrl({ + config: s3Config, + objectKey: bird.photo_object_key, + expiresInSeconds: 60, + }); + const imageResponse = await fetch(signedUrl); + + if (!imageResponse.ok) { + res.status(imageResponse.status).json({ error: 'Unable to load bird photo.' }); + return; + } + + const contentType = imageResponse.headers.get('content-type') || bird.photo_content_type || 'application/octet-stream'; + const contentLength = imageResponse.headers.get('content-length'); + const imageBuffer = Buffer.from(await imageResponse.arrayBuffer()); + + res.setHeader('Content-Type', contentType); + res.setHeader('Cache-Control', 'private, max-age=300'); + + if (contentLength) { + res.setHeader('Content-Length', contentLength); + } + + res.send(imageBuffer); + } catch (error) { + next(error); + } +}); + app.post('/api/birds', requireAuth, requireWriteAccess, requireWorkspaceRole(['owner', 'assistant', 'caregiver']), async (req: Request, res: Response, next: NextFunction) => { const parsed = birdSchema.safeParse(req.body); From ac1afc613fe77d3742baf9b6f7428707051c16ac Mon Sep 17 00:00:00 2001 From: blaisadmin Date: Sat, 2 May 2026 12:06:41 -0400 Subject: [PATCH 6/7] improved image handling --- backend/src/app.ts | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/backend/src/app.ts b/backend/src/app.ts index 3e74e13..7928140 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -2763,27 +2763,11 @@ app.get('/api/birds/:birdId/photo', async (req: Request, res: Response, next: Ne const signedUrl = getSignedS3ObjectUrl({ config: s3Config, objectKey: bird.photo_object_key, - expiresInSeconds: 60, + expiresInSeconds: 5 * 60, }); - const imageResponse = await fetch(signedUrl); - if (!imageResponse.ok) { - res.status(imageResponse.status).json({ error: 'Unable to load bird photo.' }); - return; - } - - const contentType = imageResponse.headers.get('content-type') || bird.photo_content_type || 'application/octet-stream'; - const contentLength = imageResponse.headers.get('content-length'); - const imageBuffer = Buffer.from(await imageResponse.arrayBuffer()); - - res.setHeader('Content-Type', contentType); res.setHeader('Cache-Control', 'private, max-age=300'); - - if (contentLength) { - res.setHeader('Content-Length', contentLength); - } - - res.send(imageBuffer); + res.redirect(302, signedUrl); } catch (error) { next(error); } From 6dbe51410c00d943b4456579ef6b20fce2deed44 Mon Sep 17 00:00:00 2001 From: blaisadmin Date: Sat, 2 May 2026 12:11:20 -0400 Subject: [PATCH 7/7] additional image changes --- .env.example | 1 + README.md | 3 ++- backend/src/app.ts | 28 ++++++++++++++++++++++++++-- docker-compose.prod.yml | 2 ++ docker-compose.yml | 2 ++ 5 files changed, 33 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index cbbe7e0..c12db60 100644 --- a/.env.example +++ b/.env.example @@ -10,6 +10,7 @@ S3_ACCESS_KEY_ID= S3_SECRET_ACCESS_KEY= S3_PUBLIC_BASE_URL= S3_KEY_PREFIX=bird-photos +PHOTO_DELIVERY_MODE=proxy FRONTEND_URL=http://localhost:3000 BACKEND_URL=http://localhost:5000 VITE_API_BASE_URL=http://localhost:5000/api diff --git a/README.md b/README.md index 5be53ff..9cb26ff 100644 --- a/README.md +++ b/README.md @@ -124,8 +124,9 @@ Set these when Wasabi image storage is ready: - `S3_SECRET_ACCESS_KEY=` - `S3_PUBLIC_BASE_URL=` - `S3_KEY_PREFIX=bird-photos` +- `PHOTO_DELIVERY_MODE=proxy` -Use a dedicated private bucket and access key for FlockPal images. Grant only the S3 permissions the app needs for that bucket. When `S3_PUBLIC_BASE_URL` is blank, FlockPal stores private object keys and returns short-lived signed URLs for bird photos. +Use a dedicated private bucket and access key for FlockPal images. Grant only the S3 permissions the app needs for that bucket. When `S3_PUBLIC_BASE_URL` is blank, FlockPal stores private object keys. `PHOTO_DELIVERY_MODE=proxy` streams images through the backend after validating the app photo token; `PHOTO_DELIVERY_MODE=redirect` validates the app token and redirects to a short-lived Wasabi signed URL. Migrate existing Postgres-stored bird photos after deploying S3 image storage: diff --git a/backend/src/app.ts b/backend/src/app.ts index 7928140..e6d988d 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -134,6 +134,7 @@ const trustProxy = process.env.TRUST_PROXY?.trim() ?? ''; const milestoneRemindersEnabled = (process.env.MILESTONE_REMINDERS_ENABLED ?? 'true').toLowerCase() !== 'false'; const milestoneReminderTimeZone = process.env.MILESTONE_REMINDER_TIME_ZONE?.trim() || 'America/New_York'; const milestoneReminderCheckIntervalMs = 60 * 60 * 1000; +const photoDeliveryMode = process.env.PHOTO_DELIVERY_MODE === 'redirect' ? 'redirect' : 'proxy'; if (trustProxy) { app.set('trust proxy', trustProxy === 'true' ? true : Number(trustProxy) || trustProxy); @@ -2766,8 +2767,31 @@ app.get('/api/birds/:birdId/photo', async (req: Request, res: Response, next: Ne expiresInSeconds: 5 * 60, }); - res.setHeader('Cache-Control', 'private, max-age=300'); - res.redirect(302, signedUrl); + res.setHeader('Cache-Control', 'private, max-age=900'); + + if (photoDeliveryMode === 'redirect') { + res.redirect(302, signedUrl); + return; + } + + const imageResponse = await fetch(signedUrl); + + if (!imageResponse.ok) { + res.status(imageResponse.status).json({ error: 'Unable to load bird photo.' }); + return; + } + + const contentType = imageResponse.headers.get('content-type') || bird.photo_content_type || 'application/octet-stream'; + const contentLength = imageResponse.headers.get('content-length'); + const imageBuffer = Buffer.from(await imageResponse.arrayBuffer()); + + res.setHeader('Content-Type', contentType); + + if (contentLength) { + res.setHeader('Content-Length', contentLength); + } + + res.send(imageBuffer); } catch (error) { next(error); } diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 63ae0cc..167a94b 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -51,6 +51,7 @@ services: S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-} S3_PUBLIC_BASE_URL: ${S3_PUBLIC_BASE_URL:-} S3_KEY_PREFIX: ${S3_KEY_PREFIX:-bird-photos} + PHOTO_DELIVERY_MODE: ${PHOTO_DELIVERY_MODE:-proxy} FRONTEND_URL: ${FRONTEND_URL:?set FRONTEND_URL for production} BACKEND_URL: ${BACKEND_URL:?set BACKEND_URL for production} ADMIN_EMAILS: ${ADMIN_EMAILS:-} @@ -127,6 +128,7 @@ services: S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-} S3_PUBLIC_BASE_URL: ${S3_PUBLIC_BASE_URL:-} S3_KEY_PREFIX: ${S3_KEY_PREFIX:-bird-photos} + PHOTO_DELIVERY_MODE: ${PHOTO_DELIVERY_MODE:-proxy} FRONTEND_URL: ${FRONTEND_URL:?set FRONTEND_URL for production} BACKEND_URL: ${BACKEND_URL:?set BACKEND_URL for production} ADMIN_EMAILS: ${ADMIN_EMAILS:-} diff --git a/docker-compose.yml b/docker-compose.yml index 1e8ce9c..b7cd0cc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,6 +49,7 @@ services: S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-} S3_PUBLIC_BASE_URL: ${S3_PUBLIC_BASE_URL:-} S3_KEY_PREFIX: ${S3_KEY_PREFIX:-bird-photos} + PHOTO_DELIVERY_MODE: ${PHOTO_DELIVERY_MODE:-proxy} FRONTEND_URL: ${FRONTEND_URL:-http://localhost:3000} BACKEND_URL: ${BACKEND_URL:-http://localhost:5000} ADMIN_EMAILS: ${ADMIN_EMAILS:-} @@ -120,6 +121,7 @@ services: S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-} S3_PUBLIC_BASE_URL: ${S3_PUBLIC_BASE_URL:-} S3_KEY_PREFIX: ${S3_KEY_PREFIX:-bird-photos} + PHOTO_DELIVERY_MODE: ${PHOTO_DELIVERY_MODE:-proxy} FRONTEND_URL: ${FRONTEND_URL:-http://localhost:3000} BACKEND_URL: ${BACKEND_URL:-http://localhost:5000} ADMIN_EMAILS: ${ADMIN_EMAILS:-}