Merge branch 'main' of https://git.blaishome.online/blaisadmin/FlockPal

# Conflicts:
#	backend/src/repositories/birdRepository.ts

backend/src/app.ts

@@ -1,6 +1,7 @@
 import crypto from 'crypto';
 import { existsSync } from 'fs';
 import path from 'path';
+import { fileURLToPath } from 'url';
 import cors from 'cors';
 import dotenv from 'dotenv';
 import express, { type NextFunction, type Request, type Response } from 'express';
@@ -12,6 +13,7 @@ import Stripe from 'stripe';
 import { z } from 'zod';
 
 import { ensureSchema } from './db/schema.js';
+import { enqueueBirdMilestoneReminderJob, getBirdMilestoneReminderQueueCounts } from './queues/birdMilestoneReminderQueue.js';
 import {
   consumeMagicLinkToken,
   consumeOAuthState,
@@ -58,6 +60,13 @@ import {
   updateVetVisitForBird,
 } from './repositories/birdRepository.js';
 import { createIntegrationTokenRecord, listIntegrationTokens, revokeIntegrationToken } from './repositories/integrationTokenRepository.js';
+import {
+  buildBirdPhotoObjectKey,
+  getImageExtensionFromContentType,
+  getImageStorageProvider,
+  getS3ImageStorageConfig,
+} from './storage/imageStorageConfig.js';
+import { deleteS3Object, getSignedS3ObjectUrl, putS3Object } from './storage/s3Client.js';
 import {
   cancelRescueVerificationRequest,
   claimWorkspaceInvites,
@@ -125,6 +134,7 @@ const trustProxy = process.env.TRUST_PROXY?.trim() ?? '';
 const milestoneRemindersEnabled = (process.env.MILESTONE_REMINDERS_ENABLED ?? 'true').toLowerCase() !== 'false';
 const milestoneReminderTimeZone = process.env.MILESTONE_REMINDER_TIME_ZONE?.trim() || 'America/New_York';
 const milestoneReminderCheckIntervalMs = 60 * 60 * 1000;
+const photoDeliveryMode = process.env.PHOTO_DELIVERY_MODE === 'redirect' ? 'redirect' : 'proxy';
 
 if (trustProxy) {
   app.set('trust proxy', trustProxy === 'true' ? true : Number(trustProxy) || trustProxy);
@@ -158,6 +168,7 @@ const photoDataUrlSchema = z
   .string()
   .regex(/^data:image\/(?:png|jpeg|jpg|webp|gif);base64,[A-Za-z0-9+/=]+$/)
   .max(1_500_000);
+const photoUrlSchema = z.string().trim().url().max(2000);
 
 const magicLinkRequestSchema = z.object({
   name: z.string().trim().max(160).optional().or(z.literal('')),
@@ -231,7 +242,7 @@ const birdSchema = z.object({
   dateOfBirth: dateStringSchema.optional().or(z.literal('')),
   gotchaDay: dateStringSchema.optional().or(z.literal('')),
   chartColor: chartColorSchema.optional(),
-  photoDataUrl: photoDataUrlSchema.optional().or(z.literal('')),
+  photoDataUrl: z.union([photoDataUrlSchema, photoUrlSchema, z.literal('')]).optional(),
   notifyOnDob: z.boolean().optional(),
   notifyOnGotchaDay: z.boolean().optional(),
 });
@@ -312,6 +323,7 @@ const normalizeBandId = (value?: string | null) => {
 const normalizeEmail = (value: string) => value.trim().toLowerCase();
 const hashToken = (token: string) => crypto.createHash('sha256').update(token).digest('hex');
 const createSessionToken = () => crypto.randomBytes(32).toString('hex');
+const photoAccessSecret = process.env.PHOTO_ACCESS_SECRET?.trim() || process.env.S3_SECRET_ACCESS_KEY?.trim() || createSessionToken();
 const createIntegrationToken = () => `flpt_${crypto.randomBytes(24).toString('hex')}`;
 const createRandomId = () => crypto.randomUUID();
 const createCodeVerifier = () => crypto.randomBytes(32).toString('base64url');
@@ -340,7 +352,8 @@ const smtpPass = process.env.SMTP_PASS?.trim() ?? '';
 const smtpFromEmail = process.env.SMTP_FROM_EMAIL?.trim() ?? '';
 const smtpFromName = process.env.SMTP_FROM_NAME?.trim() || 'FlockPal';
 const rescueStatusNotificationEmail = process.env.RESCUE_STATUS_NOTIFICATION_EMAIL?.trim() || 'appadmin@flockpal.app';
-const rescueOnboardingWebhookUrl = 'https://n8n.blaishome.online/webhook/395cd538-5e0d-4e89-8070-9e66f571b7ee';
+const rescueOnboardingWebhookUrl =
+  process.env.RESCUE_ONBOARDING_WEBHOOK_URL?.trim() || 'https://n8n.blaishome.online/webhook/395cd538-5e0d-4e89-8070-9e66f571b7ee';
 const stripeSecretKey = process.env.STRIPE_SECRET_KEY?.trim() ?? '';
 const stripeWebhookSecret = process.env.STRIPE_WEBHOOK_SECRET?.trim() ?? '';
 const withBillingRedirectState = (url: string, billingState: 'success' | 'cancelled' | 'portal') => {
@@ -471,6 +484,82 @@ const normalizeWorkspaceMember = (row: WorkspaceMemberRow) => ({
   createdAt: row.created_at,
 });
 
+const signBirdPhotoAccessToken = (row: BirdRow) => {
+  if (!row.photo_object_key) {
+    return '';
+  }
+
+  const expiresAt = Math.floor(Date.now() / 1000) + 15 * 60;
+  const payload = Buffer.from(
+    JSON.stringify({
+      birdId: row.id,
+      workspaceId: row.workspace_id,
+      objectKey: row.photo_object_key,
+      expiresAt,
+    }),
+  ).toString('base64url');
+  const signature = crypto.createHmac('sha256', photoAccessSecret).update(payload).digest('base64url');
+
+  return `${payload}.${signature}`;
+};
+
+const verifyBirdPhotoAccessToken = (token: string) => {
+  const [payload, signature] = token.split('.');
+
+  if (!payload || !signature) {
+    return null;
+  }
+
+  const expectedSignature = crypto.createHmac('sha256', photoAccessSecret).update(payload).digest('base64url');
+
+  const signatureBuffer = Buffer.from(signature);
+  const expectedSignatureBuffer = Buffer.from(expectedSignature);
+
+  if (signatureBuffer.length !== expectedSignatureBuffer.length || !crypto.timingSafeEqual(signatureBuffer, expectedSignatureBuffer)) {
+    return null;
+  }
+
+  const parsed = JSON.parse(Buffer.from(payload, 'base64url').toString('utf8')) as {
+    birdId?: unknown;
+    workspaceId?: unknown;
+    objectKey?: unknown;
+    expiresAt?: unknown;
+  };
+
+  if (
+    typeof parsed.birdId !== 'string' ||
+    typeof parsed.workspaceId !== 'number' ||
+    typeof parsed.objectKey !== 'string' ||
+    typeof parsed.expiresAt !== 'number' ||
+    parsed.expiresAt < Math.floor(Date.now() / 1000)
+  ) {
+    return null;
+  }
+
+  return parsed as {
+    birdId: string;
+    workspaceId: number;
+    objectKey: string;
+    expiresAt: number;
+  };
+};
+
+const getBirdPhotoUrl = (row: BirdRow) => {
+  if (!row.photo_object_key) {
+    return row.photo_data_url;
+  }
+
+  const s3Config = getS3ImageStorageConfig();
+
+  if (!s3Config) {
+    return row.photo_data_url;
+  }
+
+  const photoUrl = new URL(`${backendBaseUrl}/api/birds/${row.id}/photo`);
+  photoUrl.searchParams.set('token', signBirdPhotoAccessToken(row));
+  return photoUrl.toString();
+};
+
 const normalizeBird = (row: BirdRow) => ({
   id: row.id,
   workspaceId: row.workspace_id,
@@ -484,7 +573,10 @@ const normalizeBird = (row: BirdRow) => ({
   dateOfBirth: row.date_of_birth,
   gotchaDay: row.gotcha_day,
   chartColor: row.chart_color,
-  photoDataUrl: row.photo_data_url,
+  photoDataUrl: getBirdPhotoUrl(row),
+  photoObjectKey: row.photo_object_key,
+  photoContentType: row.photo_content_type,
+  photoUpdatedAt: row.photo_updated_at,
   notifyOnDob: row.notify_on_dob,
   notifyOnGotchaDay: row.notify_on_gotcha_day,
   memorializedAt: row.memorialized_at,
@@ -696,6 +788,40 @@ app.use(express.json({ limit: '2mb' }));
 app.use(express.urlencoded({ extended: false }));
 app.use(morgan(process.env.NODE_ENV === 'production' ? 'combined' : 'dev'));
 
+const requestMetrics = {
+  startedAt: new Date().toISOString(),
+  totalRequests: 0,
+  totalErrors: 0,
+  inFlightRequests: 0,
+  totalDurationMs: 0,
+  byStatus: {} as Record<string, number>,
+  byRoute: {} as Record<string, number>,
+};
+
+app.use((req: Request, res: Response, next: NextFunction) => {
+  const startedAt = process.hrtime.bigint();
+  requestMetrics.totalRequests += 1;
+  requestMetrics.inFlightRequests += 1;
+
+  res.on('finish', () => {
+    const durationMs = Number(process.hrtime.bigint() - startedAt) / 1_000_000;
+    requestMetrics.inFlightRequests -= 1;
+    requestMetrics.totalDurationMs += durationMs;
+
+    const statusBucket = `${Math.floor(res.statusCode / 100)}xx`;
+    requestMetrics.byStatus[statusBucket] = (requestMetrics.byStatus[statusBucket] ?? 0) + 1;
+
+    if (res.statusCode >= 500) {
+      requestMetrics.totalErrors += 1;
+    }
+
+    const routeKey = `${req.method} ${req.route?.path ?? req.path}`;
+    requestMetrics.byRoute[routeKey] = (requestMetrics.byRoute[routeKey] ?? 0) + 1;
+  });
+
+  next();
+});
+
 const normalizeWorkspaceMembershipList = async (userId: string) =>
   (await listMembershipsForUser(userId)).map((row) => ({
     membership: normalizeWorkspaceMember(row),
@@ -1002,6 +1128,107 @@ const parseDataImage = (dataUrl: string) => {
   };
 };
 
+const isDataImageUrl = (value: string | null | undefined) => Boolean(value && value.startsWith('data:image/'));
+
+const resolveBirdPhotoStorage = async ({
+  birdId,
+  workspaceId,
+  photoDataUrl,
+  existingBird,
+}: {
+  birdId: string;
+  workspaceId: number;
+  photoDataUrl: string | null;
+  existingBird?: BirdRow | null;
+}) => {
+  if (!photoDataUrl) {
+    return {
+      photoDataUrl: null,
+      photoObjectKey: null,
+      photoContentType: null,
+      photoUpdatedAt: null,
+      objectKeyToDelete: existingBird?.photo_object_key ?? null,
+    };
+  }
+
+  if (!isDataImageUrl(photoDataUrl)) {
+    if (existingBird?.photo_object_key) {
+      return {
+        photoDataUrl: null,
+        photoObjectKey: existingBird.photo_object_key,
+        photoContentType: existingBird.photo_content_type,
+        photoUpdatedAt: existingBird.photo_updated_at,
+        objectKeyToDelete: null,
+      };
+    }
+
+    return {
+      photoDataUrl,
+      photoObjectKey: null,
+      photoContentType: null,
+      photoUpdatedAt: null,
+      objectKeyToDelete: existingBird?.photo_object_key ?? null,
+    };
+  }
+
+  const parsedImage = parseDataImage(photoDataUrl);
+
+  if (!parsedImage) {
+    throw new Error('Unable to process bird photo.');
+  }
+
+  if (getImageStorageProvider() !== 's3') {
+    return {
+      photoDataUrl,
+      photoObjectKey: null,
+      photoContentType: null,
+      photoUpdatedAt: null,
+      objectKeyToDelete: existingBird?.photo_object_key ?? null,
+    };
+  }
+
+  const s3Config = getS3ImageStorageConfig();
+
+  if (!s3Config) {
+    throw new Error('S3 image storage is enabled but not fully configured.');
+  }
+
+  const extension = getImageExtensionFromContentType(parsedImage.contentType);
+  const objectKey = buildBirdPhotoObjectKey({ workspaceId, birdId, extension });
+  await putS3Object({
+    config: s3Config,
+    objectKey,
+    content: parsedImage.content,
+    contentType: parsedImage.contentType,
+  });
+
+  return {
+    photoDataUrl: null,
+    photoObjectKey: objectKey,
+    photoContentType: parsedImage.contentType,
+    photoUpdatedAt: new Date().toISOString(),
+    objectKeyToDelete: existingBird?.photo_object_key ?? null,
+  };
+};
+
+const deleteBirdPhotoObjectIfNeeded = async (objectKey: string | null) => {
+  if (!objectKey) {
+    return;
+  }
+
+  const s3Config = getS3ImageStorageConfig();
+
+  if (!s3Config) {
+    return;
+  }
+
+  try {
+    await deleteS3Object({ config: s3Config, objectKey });
+  } catch (error) {
+    console.warn(`Unable to delete old bird photo object ${objectKey}:`, error);
+  }
+};
+
 const getDefaultBirdPhotoAttachment = () => {
   const defaultPhotoPath = path.join(process.cwd(), 'assets', 'yoda-default.png');
 
@@ -1433,7 +1660,7 @@ const sendBirdMilestoneReminderNotification = async ({
   return { delivered: true };
 };
 
-const runBirdMilestoneReminders = async (runDate = getDateInTimeZone()) => {
+export const runBirdMilestoneReminders = async (runDate = getDateInTimeZone()) => {
   const reminders = await listDueBirdMilestoneReminders(runDate);
   let sent = 0;
   let skipped = 0;
@@ -1479,7 +1706,7 @@ const runBirdMilestoneReminders = async (runDate = getDateInTimeZone()) => {
 
 let lastMilestoneReminderRunDate = '';
 
-const startBirdMilestoneReminderScheduler = () => {
+export const startBirdMilestoneReminderScheduler = () => {
   if (!milestoneRemindersEnabled) {
     console.log('Bird milestone reminders are disabled.');
     return;
@@ -1492,10 +1719,8 @@ const startBirdMilestoneReminderScheduler = () => {
     }
 
     lastMilestoneReminderRunDate = runDate;
-    const result = await runBirdMilestoneReminders(runDate);
-    console.log(
-      `Bird milestone reminders completed for ${result.runDate}: checked=${result.checked}, sent=${result.sent}, skipped=${result.skipped}, failed=${result.failed}`,
-    );
+    const job = await enqueueBirdMilestoneReminderJob(runDate);
+    console.log(`Bird milestone reminder job queued for ${runDate}: id=${job.id ?? 'unknown'}`);
   };
 
   setTimeout(() => {
@@ -1635,6 +1860,40 @@ app.get('/api/health', (_req: Request, res: Response) => {
   res.json({ ok: true });
 });
 
+app.get('/api/metrics', requireAuth, requireAdmin, async (_req: Request, res: Response, next: NextFunction) => {
+  const memoryUsage = process.memoryUsage();
+  const averageDurationMs = requestMetrics.totalRequests > 0 ? requestMetrics.totalDurationMs / requestMetrics.totalRequests : 0;
+
+  try {
+    const birdMilestoneReminderQueueCounts = await getBirdMilestoneReminderQueueCounts();
+
+    res.json({
+      startedAt: requestMetrics.startedAt,
+      uptimeSeconds: Math.round(process.uptime()),
+      requests: {
+        total: requestMetrics.totalRequests,
+        inFlight: requestMetrics.inFlightRequests,
+        errors: requestMetrics.totalErrors,
+        averageDurationMs: Number(averageDurationMs.toFixed(2)),
+        byStatus: requestMetrics.byStatus,
+        byRoute: requestMetrics.byRoute,
+      },
+      memory: {
+        rss: memoryUsage.rss,
+        heapTotal: memoryUsage.heapTotal,
+        heapUsed: memoryUsage.heapUsed,
+        external: memoryUsage.external,
+        arrayBuffers: memoryUsage.arrayBuffers,
+      },
+      queues: {
+        birdMilestoneReminders: birdMilestoneReminderQueueCounts,
+      },
+    });
+  } catch (error) {
+    next(error);
+  }
+});
+
 app.post('/api/lost-bird/report', lostBirdReportLimiter, async (req: Request, res: Response) => {
   const parsed = lostBirdReportSchema.safeParse(req.body);
 
@@ -2502,6 +2761,66 @@ app.get('/api/birds', requireAuth, async (req: Request, res: Response, next: Nex
   }
 });
 
+app.get('/api/birds/:birdId/photo', async (req: Request, res: Response, next: NextFunction) => {
+  try {
+    const token = typeof req.query.token === 'string' ? req.query.token : '';
+    const photoAccess = verifyBirdPhotoAccessToken(token);
+
+    if (!photoAccess || photoAccess.birdId !== req.params.birdId) {
+      res.status(403).json({ error: 'Photo link expired or invalid.' });
+      return;
+    }
+
+    const bird = await getBirdById(photoAccess.birdId, photoAccess.workspaceId);
+
+    if (!bird || bird.photo_object_key !== photoAccess.objectKey) {
+      res.status(404).json({ error: 'Photo not found.' });
+      return;
+    }
+
+    const s3Config = getS3ImageStorageConfig();
+
+    if (!s3Config) {
+      res.status(503).json({ error: 'Image storage is not configured.' });
+      return;
+    }
+
+    const signedUrl = getSignedS3ObjectUrl({
+      config: s3Config,
+      objectKey: bird.photo_object_key,
+      expiresInSeconds: 5 * 60,
+    });
+
+    res.setHeader('Cache-Control', 'private, max-age=900');
+
+    if (photoDeliveryMode === 'redirect') {
+      res.redirect(302, signedUrl);
+      return;
+    }
+
+    const imageResponse = await fetch(signedUrl);
+
+    if (!imageResponse.ok) {
+      res.status(imageResponse.status).json({ error: 'Unable to load bird photo.' });
+      return;
+    }
+
+    const contentType = imageResponse.headers.get('content-type') || bird.photo_content_type || 'application/octet-stream';
+    const contentLength = imageResponse.headers.get('content-length');
+    const imageBuffer = Buffer.from(await imageResponse.arrayBuffer());
+
+    res.setHeader('Content-Type', contentType);
+
+    if (contentLength) {
+      res.setHeader('Content-Length', contentLength);
+    }
+
+    res.send(imageBuffer);
+  } catch (error) {
+    next(error);
+  }
+});
+
 app.post('/api/birds', requireAuth, requireWriteAccess, requireWorkspaceRole(['owner', 'assistant', 'caregiver']), async (req: Request, res: Response, next: NextFunction) => {
   const parsed = birdSchema.safeParse(req.body);
 
@@ -2510,8 +2829,18 @@ app.post('/api/birds', requireAuth, requireWriteAccess, requireWorkspaceRole(['o
     return;
   }
 
+  let uploadedObjectKeyToCleanup: string | null = null;
+
   try {
+    const birdId = crypto.randomUUID();
+    const photoStorage = await resolveBirdPhotoStorage({
+      birdId,
+      workspaceId: req.auth!.workspace.id,
+      photoDataUrl: emptyToNull(parsed.data.photoDataUrl),
+    });
+    uploadedObjectKeyToCleanup = photoStorage.photoObjectKey;
     const bird = await createBird({
+      birdId,
       workspaceId: req.auth!.workspace.id,
       name: parsed.data.name,
       tagId: normalizeBandId(parsed.data.tagId),
@@ -2523,13 +2852,19 @@ app.post('/api/birds', requireAuth, requireWriteAccess, requireWorkspaceRole(['o
       dateOfBirth: emptyToNull(parsed.data.dateOfBirth),
       gotchaDay: emptyToNull(parsed.data.gotchaDay),
       chartColor: parsed.data.chartColor ?? '#cb3a35',
-      photoDataUrl: emptyToNull(parsed.data.photoDataUrl),
+      photoDataUrl: photoStorage.photoDataUrl,
+      photoObjectKey: photoStorage.photoObjectKey,
+      photoContentType: photoStorage.photoContentType,
+      photoUpdatedAt: photoStorage.photoUpdatedAt,
       notifyOnDob: parsed.data.notifyOnDob ?? false,
       notifyOnGotchaDay: parsed.data.notifyOnGotchaDay ?? false,
     });
 
+    uploadedObjectKeyToCleanup = null;
     res.status(201).json({ bird: normalizeBird(bird!) });
   } catch (error) {
+    await deleteBirdPhotoObjectIfNeeded(uploadedObjectKeyToCleanup);
+
     if (typeof error === 'object' && error && 'code' in error && error.code === '23505') {
       res.status(409).json({ error: 'That band/tag ID is already in use in this flock.' });
       return;
@@ -2622,6 +2957,8 @@ app.put('/api/birds/:birdId', requireAuth, requireWriteAccess, requireWorkspaceR
     return;
   }
 
+  let uploadedObjectKeyToCleanup: string | null = null;
+
   try {
     const existingBird = await getBirdById(req.params.birdId, req.auth!.workspace.id);
 
@@ -2634,6 +2971,14 @@ app.put('/api/birds/:birdId', requireAuth, requireWriteAccess, requireWorkspaceR
       return;
     }
 
+    const photoStorage = await resolveBirdPhotoStorage({
+      birdId: req.params.birdId,
+      workspaceId: req.auth!.workspace.id,
+      photoDataUrl: emptyToNull(parsed.data.photoDataUrl),
+      existingBird,
+    });
+    uploadedObjectKeyToCleanup =
+      photoStorage.photoObjectKey && photoStorage.photoObjectKey !== existingBird.photo_object_key ? photoStorage.photoObjectKey : null;
     const bird = await updateBird({
       birdId: req.params.birdId,
       workspaceId: req.auth!.workspace.id,
@@ -2647,7 +2992,10 @@ app.put('/api/birds/:birdId', requireAuth, requireWriteAccess, requireWorkspaceR
       dateOfBirth: emptyToNull(parsed.data.dateOfBirth),
       gotchaDay: emptyToNull(parsed.data.gotchaDay),
       chartColor: parsed.data.chartColor ?? '#cb3a35',
-      photoDataUrl: emptyToNull(parsed.data.photoDataUrl),
+      photoDataUrl: photoStorage.photoDataUrl,
+      photoObjectKey: photoStorage.photoObjectKey,
+      photoContentType: photoStorage.photoContentType,
+      photoUpdatedAt: photoStorage.photoUpdatedAt,
       notifyOnDob: parsed.data.notifyOnDob ?? false,
       notifyOnGotchaDay: parsed.data.notifyOnGotchaDay ?? false,
     });
@@ -2657,8 +3005,12 @@ app.put('/api/birds/:birdId', requireAuth, requireWriteAccess, requireWorkspaceR
       return;
     }
 
+    uploadedObjectKeyToCleanup = null;
+    await deleteBirdPhotoObjectIfNeeded(photoStorage.objectKeyToDelete);
     res.json({ bird: normalizeBird(bird) });
   } catch (error) {
+    await deleteBirdPhotoObjectIfNeeded(uploadedObjectKeyToCleanup);
+
     if (typeof error === 'object' && error && 'code' in error && error.code === '23505') {
       res.status(409).json({ error: 'That band/tag ID is already in use in this flock.' });
       return;
@@ -2689,6 +3041,7 @@ app.delete('/api/birds/:birdId', requireAuth, requireWriteAccess, requireWorkspa
     }
 
     res.status(204).send();
+    await deleteBirdPhotoObjectIfNeeded(bird.photo_object_key);
   } catch (error) {
     next(error);
   }
@@ -3103,15 +3456,18 @@ app.use((error: unknown, _req: Request, res: Response, _next: NextFunction) => {
   res.status(500).json({ error: error instanceof Error ? error.message : 'Internal server error' });
 });
 
-const start = async () => {
+export const startApiServer = async () => {
   await ensureSchema();
   app.listen(port, () => {
     console.log(`FlockPal backend listening on port ${port}`);
   });
-  startBirdMilestoneReminderScheduler();
 };
 
-start().catch((error) => {
-  console.error('Failed to start backend', error);
-  process.exit(1);
-});
+const currentModulePath = fileURLToPath(import.meta.url);
+
+if (process.argv[1] === currentModulePath) {
+  startApiServer().catch((error) => {
+    console.error('Failed to start backend', error);
+    process.exit(1);
+  });
+}