blaisadmin/FlockPal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixed db scripts

Browse Source
This commit is contained in:
blaisadmin
2026-05-02 10:24:58 -04:00
parent 1bb3002baf
commit 22f344a998
8 changed files with 409 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+2 -2
View File
@@ -122,10 +122,10 @@ Set these when Wasabi image storage is ready:
- `S3_BUCKET=<bucket-name>`
- `S3_ACCESS_KEY_ID=<access-key>`
- `S3_SECRET_ACCESS_KEY=<secret-key>`
- `S3_PUBLIC_BASE_URL=<optional CDN or public bucket base URL>`
- `S3_PUBLIC_BASE_URL=<optional CDN or public bucket base URL; leave blank for private signed URLs>`
- `S3_KEY_PREFIX=bird-photos`
Use a dedicated bucket and access key for FlockPal images. Grant only the S3 permissions the app needs for that bucket.
Use a dedicated private bucket and access key for FlockPal images. Grant only the S3 permissions the app needs for that bucket. When `S3_PUBLIC_BASE_URL` is blank, FlockPal stores private object keys and returns short-lived signed URLs for bird photos.
Bucket settings recommendation:
backend/src/app.ts
+165 -4
View File
@@ -60,6 +60,13 @@ import {
updateVetVisitForBird,
} from './repositories/birdRepository.js';
import { createIntegrationTokenRecord, listIntegrationTokens, revokeIntegrationToken } from './repositories/integrationTokenRepository.js';
import {
buildBirdPhotoObjectKey,
getImageExtensionFromContentType,
getImageStorageProvider,
getS3ImageStorageConfig,
} from './storage/imageStorageConfig.js';
import { deleteS3Object, getSignedS3ObjectUrl, putS3Object } from './storage/s3Client.js';
import {
cancelRescueVerificationRequest,
claimWorkspaceInvites,
@@ -160,6 +167,7 @@ const photoDataUrlSchema = z
.string()
.regex(/^data:image\/(?:png|jpeg|jpg|webp|gif);base64,[A-Za-z0-9+/=]+$/)
.max(1_500_000);
const photoUrlSchema = z.string().trim().url().max(2000);
const magicLinkRequestSchema = z.object({
name: z.string().trim().max(160).optional().or(z.literal('')),
@@ -230,7 +238,7 @@ const birdSchema = z.object({
dateOfBirth: dateStringSchema.optional().or(z.literal('')),
gotchaDay: dateStringSchema.optional().or(z.literal('')),
chartColor: chartColorSchema.optional(),
photoDataUrl: photoDataUrlSchema.optional().or(z.literal('')),
photoDataUrl: z.union([photoDataUrlSchema, photoUrlSchema, z.literal('')]).optional(),
notifyOnDob: z.boolean().optional(),
notifyOnGotchaDay: z.boolean().optional(),
});
@@ -471,6 +479,24 @@ const normalizeWorkspaceMember = (row: WorkspaceMemberRow) => ({
createdAt: row.created_at,
});
const getBirdPhotoUrl = (row: BirdRow) => {
if (!row.photo_object_key) {
return row.photo_data_url;
}
const s3Config = getS3ImageStorageConfig();
if (!s3Config) {
return row.photo_data_url;
}
return getSignedS3ObjectUrl({
config: s3Config,
objectKey: row.photo_object_key,
expiresInSeconds: 15 * 60,
});
};
const normalizeBird = (row: BirdRow) => ({
id: row.id,
workspaceId: row.workspace_id,
@@ -481,7 +507,7 @@ const normalizeBird = (row: BirdRow) => ({
dateOfBirth: row.date_of_birth,
gotchaDay: row.gotcha_day,
chartColor: row.chart_color,
photoDataUrl: row.photo_data_url,
photoDataUrl: getBirdPhotoUrl(row),
photoObjectKey: row.photo_object_key,
photoContentType: row.photo_content_type,
photoUpdatedAt: row.photo_updated_at,
@@ -1021,6 +1047,107 @@ const parseDataImage = (dataUrl: string) => {
};
};
const isDataImageUrl = (value: string | null | undefined) => Boolean(value && value.startsWith('data:image/'));
const resolveBirdPhotoStorage = async ({
birdId,
workspaceId,
photoDataUrl,
existingBird,
}: {
birdId: string;
workspaceId: number;
photoDataUrl: string | null;
existingBird?: BirdRow | null;
}) => {
if (!photoDataUrl) {
return {
photoDataUrl: null,
photoObjectKey: null,
photoContentType: null,
photoUpdatedAt: null,
objectKeyToDelete: existingBird?.photo_object_key ?? null,
};
}
if (!isDataImageUrl(photoDataUrl)) {
if (existingBird?.photo_object_key) {
return {
photoDataUrl: null,
photoObjectKey: existingBird.photo_object_key,
photoContentType: existingBird.photo_content_type,
photoUpdatedAt: existingBird.photo_updated_at,
objectKeyToDelete: null,
};
}
return {
photoDataUrl,
photoObjectKey: null,
photoContentType: null,
photoUpdatedAt: null,
objectKeyToDelete: existingBird?.photo_object_key ?? null,
};
}
const parsedImage = parseDataImage(photoDataUrl);
if (!parsedImage) {
throw new Error('Unable to process bird photo.');
}
if (getImageStorageProvider() !== 's3') {
return {
photoDataUrl,
photoObjectKey: null,
photoContentType: null,
photoUpdatedAt: null,
objectKeyToDelete: existingBird?.photo_object_key ?? null,
};
}
const s3Config = getS3ImageStorageConfig();
if (!s3Config) {
throw new Error('S3 image storage is enabled but not fully configured.');
}
const extension = getImageExtensionFromContentType(parsedImage.contentType);
const objectKey = buildBirdPhotoObjectKey({ workspaceId, birdId, extension });
await putS3Object({
config: s3Config,
objectKey,
content: parsedImage.content,
contentType: parsedImage.contentType,
});
return {
photoDataUrl: null,
photoObjectKey: objectKey,
photoContentType: parsedImage.contentType,
photoUpdatedAt: new Date().toISOString(),
objectKeyToDelete: existingBird?.photo_object_key ?? null,
};
};
const deleteBirdPhotoObjectIfNeeded = async (objectKey: string | null) => {
if (!objectKey) {
return;
}
const s3Config = getS3ImageStorageConfig();
if (!s3Config) {
return;
}
try {
await deleteS3Object({ config: s3Config, objectKey });
} catch (error) {
console.warn(`Unable to delete old bird photo object ${objectKey}:`, error);
}
};
const getDefaultBirdPhotoAttachment = () => {
const defaultPhotoPath = path.join(process.cwd(), 'assets', 'yoda-default.png');
@@ -2558,8 +2685,18 @@ app.post('/api/birds', requireAuth, requireWriteAccess, requireWorkspaceRole(['o
return;
}
let uploadedObjectKeyToCleanup: string | null = null;
try {
const birdId = crypto.randomUUID();
const photoStorage = await resolveBirdPhotoStorage({
birdId,
workspaceId: req.auth!.workspace.id,
photoDataUrl: emptyToNull(parsed.data.photoDataUrl),
});
uploadedObjectKeyToCleanup = photoStorage.photoObjectKey;
const bird = await createBird({
birdId,
workspaceId: req.auth!.workspace.id,
name: parsed.data.name,
tagId: normalizeBandId(parsed.data.tagId),
@@ -2568,13 +2705,19 @@ app.post('/api/birds', requireAuth, requireWriteAccess, requireWorkspaceRole(['o
dateOfBirth: emptyToNull(parsed.data.dateOfBirth),
gotchaDay: emptyToNull(parsed.data.gotchaDay),
chartColor: parsed.data.chartColor ?? '#cb3a35',
photoDataUrl: emptyToNull(parsed.data.photoDataUrl),
photoDataUrl: photoStorage.photoDataUrl,
photoObjectKey: photoStorage.photoObjectKey,
photoContentType: photoStorage.photoContentType,
photoUpdatedAt: photoStorage.photoUpdatedAt,
notifyOnDob: parsed.data.notifyOnDob ?? false,
notifyOnGotchaDay: parsed.data.notifyOnGotchaDay ?? false,
});
uploadedObjectKeyToCleanup = null;
res.status(201).json({ bird: normalizeBird(bird!) });
} catch (error) {
await deleteBirdPhotoObjectIfNeeded(uploadedObjectKeyToCleanup);
if (typeof error === 'object' && error && 'code' in error && error.code === '23505') {
res.status(409).json({ error: 'That band/tag ID is already in use in this flock.' });
return;
@@ -2667,6 +2810,8 @@ app.put('/api/birds/:birdId', requireAuth, requireWriteAccess, requireWorkspaceR
return;
}
let uploadedObjectKeyToCleanup: string | null = null;
try {
const existingBird = await getBirdById(req.params.birdId, req.auth!.workspace.id);
@@ -2679,6 +2824,14 @@ app.put('/api/birds/:birdId', requireAuth, requireWriteAccess, requireWorkspaceR
return;
}
const photoStorage = await resolveBirdPhotoStorage({
birdId: req.params.birdId,
workspaceId: req.auth!.workspace.id,
photoDataUrl: emptyToNull(parsed.data.photoDataUrl),
existingBird,
});
uploadedObjectKeyToCleanup =
photoStorage.photoObjectKey && photoStorage.photoObjectKey !== existingBird.photo_object_key ? photoStorage.photoObjectKey : null;
const bird = await updateBird({
birdId: req.params.birdId,
workspaceId: req.auth!.workspace.id,
@@ -2689,7 +2842,10 @@ app.put('/api/birds/:birdId', requireAuth, requireWriteAccess, requireWorkspaceR
dateOfBirth: emptyToNull(parsed.data.dateOfBirth),
gotchaDay: emptyToNull(parsed.data.gotchaDay),
chartColor: parsed.data.chartColor ?? '#cb3a35',
photoDataUrl: emptyToNull(parsed.data.photoDataUrl),
photoDataUrl: photoStorage.photoDataUrl,
photoObjectKey: photoStorage.photoObjectKey,
photoContentType: photoStorage.photoContentType,
photoUpdatedAt: photoStorage.photoUpdatedAt,
notifyOnDob: parsed.data.notifyOnDob ?? false,
notifyOnGotchaDay: parsed.data.notifyOnGotchaDay ?? false,
});
@@ -2699,8 +2855,12 @@ app.put('/api/birds/:birdId', requireAuth, requireWriteAccess, requireWorkspaceR
return;
}
uploadedObjectKeyToCleanup = null;
await deleteBirdPhotoObjectIfNeeded(photoStorage.objectKeyToDelete);
res.json({ bird: normalizeBird(bird) });
} catch (error) {
await deleteBirdPhotoObjectIfNeeded(uploadedObjectKeyToCleanup);
if (typeof error === 'object' && error && 'code' in error && error.code === '23505') {
res.status(409).json({ error: 'That band/tag ID is already in use in this flock.' });
return;
@@ -2731,6 +2891,7 @@ app.delete('/api/birds/:birdId', requireAuth, requireWriteAccess, requireWorkspa
}
res.status(204).send();
await deleteBirdPhotoObjectIfNeeded(bird.photo_object_key);
} catch (error) {
next(error);
}
backend/src/queues/birdMilestoneReminderQueue.ts
+1 -1
View File
@@ -41,7 +41,7 @@ export const enqueueBirdMilestoneReminderJob = (runDate: string): Promise<Job<Bi
requestedBy: 'scheduler',
},
{
jobId: `bird-milestone-reminders:${runDate}`,
jobId: `bird-milestone-reminders-${runDate}`,
},
);
backend/src/repositories/birdRepository.ts
+56 -7
View File
@@ -253,6 +253,7 @@ export const createBirdMilestoneReminderDelivery = async ({
};
export const createBird = async ({
birdId,
workspaceId,
name,
tagId,
@@ -262,9 +263,13 @@ export const createBird = async ({
gotchaDay,
chartColor,
photoDataUrl,
photoObjectKey = null,
photoContentType = null,
photoUpdatedAt = null,
notifyOnDob,
notifyOnGotchaDay,
}: {
birdId?: string;
workspaceId: number;
name: string;
tagId: string | null;
@@ -274,14 +279,33 @@ export const createBird = async ({
gotchaDay: string | null;
chartColor: string;
photoDataUrl: string | null;
photoObjectKey?: string | null;
photoContentType?: string | null;
photoUpdatedAt?: string | null;
notifyOnDob: boolean;
notifyOnGotchaDay: boolean;
}) => {
const result = await db.query<BirdRow>(
`INSERT INTO birds (workspace_id, name, tag_id, species, gender, date_of_birth, gotcha_day, chart_color, photo_data_url, notify_on_dob, notify_on_gotcha_day)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
`INSERT INTO birds (id, workspace_id, name, tag_id, species, gender, date_of_birth, gotcha_day, chart_color, photo_data_url, photo_object_key, photo_content_type, photo_updated_at, notify_on_dob, notify_on_gotcha_day)
VALUES (COALESCE($1::uuid, gen_random_uuid()), $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, photo_object_key, photo_content_type, photo_updated_at, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at, NULL::text AS latest_weight_grams, NULL::text AS latest_recorded_on`,
[workspaceId, name, tagId, species, gender, dateOfBirth, gotchaDay, chartColor, photoDataUrl, notifyOnDob, notifyOnGotchaDay],
[
birdId ?? null,
workspaceId,
name,
tagId,
species,
gender,
dateOfBirth,
gotchaDay,
chartColor,
photoDataUrl,
photoObjectKey,
photoContentType,
photoUpdatedAt,
notifyOnDob,
notifyOnGotchaDay,
],
);
return result.rows[0] ?? null;
@@ -298,6 +322,9 @@ export const updateBird = async ({
gotchaDay,
chartColor,
photoDataUrl,
photoObjectKey = null,
photoContentType = null,
photoUpdatedAt = null,
notifyOnDob,
notifyOnGotchaDay,
}: {
@@ -311,6 +338,9 @@ export const updateBird = async ({
gotchaDay: string | null;
chartColor: string;
photoDataUrl: string | null;
photoObjectKey?: string | null;
photoContentType?: string | null;
photoUpdatedAt?: string | null;
notifyOnDob: boolean;
notifyOnGotchaDay: boolean;
}) => {
@@ -324,10 +354,13 @@ export const updateBird = async ({
gotcha_day = $7,
chart_color = $8,
photo_data_url = $9,
notify_on_dob = $10,
notify_on_gotcha_day = $11
photo_object_key = $10,
photo_content_type = $11,
photo_updated_at = $12,
notify_on_dob = $13,
notify_on_gotcha_day = $14
WHERE id = $1
AND workspace_id = $12
AND workspace_id = $15
AND memorialized_at IS NULL
RETURNING id, workspace_id, name, tag_id, species, gender, date_of_birth::text, gotcha_day::text, chart_color, photo_data_url, photo_object_key, photo_content_type, photo_updated_at, notify_on_dob, notify_on_gotcha_day, memorialized_at, memorialized_on::text, memorial_note, notify_on_memorial_day, created_at,
(
@@ -344,7 +377,23 @@ export const updateBird = async ({
ORDER BY recorded_on DESC
LIMIT 1
) AS latest_recorded_on`,
[birdId, name, tagId, species, gender, dateOfBirth, gotchaDay, chartColor, photoDataUrl, notifyOnDob, notifyOnGotchaDay, workspaceId],
[
birdId,
name,
tagId,
species,
gender,
dateOfBirth,
gotchaDay,
chartColor,
photoDataUrl,
photoObjectKey,
photoContentType,
photoUpdatedAt,
notifyOnDob,
notifyOnGotchaDay,
workspaceId,
],
);
return result.rows[0] ?? null;
backend/src/storage/imageStorageConfig.ts
+16
View File
@@ -65,3 +65,19 @@ export const buildBirdPhotoObjectKey = ({
return `${prefix}/workspace-${workspaceId}/${birdId}/${timestamp}.${safeExtension}`;
};
export const getImageExtensionFromContentType = (contentType: string) => {
switch (contentType.toLowerCase()) {
case 'image/jpeg':
case 'image/jpg':
return 'jpg';
case 'image/png':
return 'png';
case 'image/webp':
return 'webp';
case 'image/gif':
return 'gif';
default:
return 'bin';
}
};
backend/src/storage/s3Client.ts
+161
View File
@@ -0,0 +1,161 @@
import crypto from 'crypto';
import type { S3ImageStorageConfig } from './imageStorageConfig.js';
const awsDate = (date: Date) => date.toISOString().replace(/[:-]|\.\d{3}/g, '');
const shortDate = (date: Date) => date.toISOString().slice(0, 10).replace(/-/g, '');
const hmac = (key: crypto.BinaryLike, value: string) => crypto.createHmac('sha256', key).update(value).digest();
const sha256Hex = (value: crypto.BinaryLike) => crypto.createHash('sha256').update(value).digest('hex');
const encodeObjectKey = (key: string) => key.split('/').map(encodeURIComponent).join('/');
const encodeQueryValue = (value: string) => encodeURIComponent(value).replace(/[!'()*]/g, (character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}`);
const getSigningKey = (secretAccessKey: string, date: string, region: string) => {
const dateKey = hmac(`AWS4${secretAccessKey}`, date);
const regionKey = hmac(dateKey, region);
const serviceKey = hmac(regionKey, 's3');
return hmac(serviceKey, 'aws4_request');
};
const buildObjectUrl = (config: S3ImageStorageConfig, objectKey: string) => {
const endpoint = config.endpoint.replace(/\/+$/, '');
return new URL(`${endpoint}/${encodeURIComponent(config.bucket)}/${encodeObjectKey(objectKey)}`);
};
const signS3Request = ({
config,
method,
objectKey,
contentHash,
contentType,
now = new Date(),
}: {
config: S3ImageStorageConfig;
method: 'DELETE' | 'PUT';
objectKey: string;
contentHash: string;
contentType?: string;
now?: Date;
}) => {
const url = buildObjectUrl(config, objectKey);
const amzDate = awsDate(now);
const date = shortDate(now);
const credentialScope = `${date}/${config.region}/s3/aws4_request`;
const headers: Record<string, string> = {
host: url.host,
'x-amz-content-sha256': contentHash,
'x-amz-date': amzDate,
};
if (contentType) {
headers['content-type'] = contentType;
}
const signedHeaders = Object.keys(headers).sort().join(';');
const canonicalHeaders = Object.keys(headers)
.sort()
.map((key) => `${key}:${headers[key]}\n`)
.join('');
const canonicalRequest = [method, url.pathname, url.search.slice(1), canonicalHeaders, signedHeaders, contentHash].join('\n');
const stringToSign = ['AWS4-HMAC-SHA256', amzDate, credentialScope, sha256Hex(canonicalRequest)].join('\n');
const signature = crypto.createHmac('sha256', getSigningKey(config.secretAccessKey, date, config.region)).update(stringToSign).digest('hex');
return {
url,
headers: {
...headers,
Authorization: `AWS4-HMAC-SHA256 Credential=${config.accessKeyId}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}`,
},
};
};
export const getPublicObjectUrl = (config: S3ImageStorageConfig, objectKey: string) => {
if (config.publicBaseUrl) {
return `${config.publicBaseUrl.replace(/\/+$/, '')}/${encodeObjectKey(objectKey)}`;
}
return buildObjectUrl(config, objectKey).toString();
};
export const getSignedS3ObjectUrl = ({
config,
objectKey,
expiresInSeconds = 900,
now = new Date(),
}: {
config: S3ImageStorageConfig;
objectKey: string;
expiresInSeconds?: number;
now?: Date;
}) => {
const url = buildObjectUrl(config, objectKey);
const amzDate = awsDate(now);
const date = shortDate(now);
const credentialScope = `${date}/${config.region}/s3/aws4_request`;
const credential = `${config.accessKeyId}/${credentialScope}`;
const queryParams: Record<string, string> = {
'X-Amz-Algorithm': 'AWS4-HMAC-SHA256',
'X-Amz-Credential': credential,
'X-Amz-Date': amzDate,
'X-Amz-Expires': String(Math.min(Math.max(expiresInSeconds, 1), 604800)),
'X-Amz-SignedHeaders': 'host',
};
const canonicalQuery = Object.keys(queryParams)
.sort()
.map((key) => `${encodeQueryValue(key)}=${encodeQueryValue(queryParams[key])}`)
.join('&');
const canonicalHeaders = `host:${url.host}\n`;
const canonicalRequest = ['GET', url.pathname, canonicalQuery, canonicalHeaders, 'host', 'UNSIGNED-PAYLOAD'].join('\n');
const stringToSign = ['AWS4-HMAC-SHA256', amzDate, credentialScope, sha256Hex(canonicalRequest)].join('\n');
const signature = crypto.createHmac('sha256', getSigningKey(config.secretAccessKey, date, config.region)).update(stringToSign).digest('hex');
url.search = `${canonicalQuery}&X-Amz-Signature=${signature}`;
return url.toString();
};
export const putS3Object = async ({
config,
objectKey,
content,
contentType,
}: {
config: S3ImageStorageConfig;
objectKey: string;
content: Buffer;
contentType: string;
}) => {
const contentHash = sha256Hex(content);
const signed = signS3Request({ config, method: 'PUT', objectKey, contentHash, contentType });
const response = await fetch(signed.url, {
method: 'PUT',
headers: signed.headers,
body: content,
});
if (!response.ok) {
const errorText = await response.text().catch(() => '');
throw new Error(`Wasabi upload failed with ${response.status}${errorText ? `: ${errorText.slice(0, 300)}` : ''}`);
}
return getPublicObjectUrl(config, objectKey);
};
export const deleteS3Object = async ({
config,
objectKey,
}: {
config: S3ImageStorageConfig;
objectKey: string;
}) => {
const signed = signS3Request({ config, method: 'DELETE', objectKey, contentHash: sha256Hex('') });
const response = await fetch(signed.url, {
method: 'DELETE',
headers: signed.headers,
});
if (!response.ok && response.status !== 404) {
const errorText = await response.text().catch(() => '');
throw new Error(`Wasabi delete failed with ${response.status}${errorText ? `: ${errorText.slice(0, 300)}` : ''}`);
}
};
scripts/backup-postgres.sh
+4
View File
@@ -1,4 +1,8 @@
#!/usr/bin/env bash
if [ -z "${BASH_VERSION:-}" ]; then
exec bash "$0" "$@"
fi
set -euo pipefail
compose_file="${COMPOSE_FILE:-docker-compose.prod.yml}"
scripts/restore-test-postgres.sh
+4
View File
@@ -1,4 +1,8 @@
#!/usr/bin/env bash
if [ -z "${BASH_VERSION:-}" ]; then
exec bash "$0" "$@"
fi
set -euo pipefail
if [[ $# -ne 1 ]]; then