Files
TREK/shared/src/storage/storage.schema.ts
T
Maurice 0a25de2c6d feat(storage): confirm the reclaim count the admin was shown, name a running backup as a skip, bound a trip cover
The confirm request now carries the unreferenced count the admin saw, so the
server can refuse a count its last report no longer holds. A pass that found
objects due while a backup read the uploads reports backup_running for them.
A trip cover is bounded to 2048 characters on REST, RPC and the MCP tool, as a
place image already is, since a replaced cover is queued for release by every
upload path it spells out.
2026-11-01 18:11:25 +01:00

431 lines
18 KiB
TypeScript

import { z } from 'zod';
/**
* Storage admin contract (spec: docs/superpowers/specs/2026-08-19-storage-admin-config-design.md).
*
* STORAGE_BACKEND_TYPES is the single schema-driven registry: each backend
* type declares its fields once, and both the Zod options schema (generated
* below) and the admin form's field metadata render from that one definition.
* Adding a backend type = one entry here + one driver class + one registry
* construction line. The registry is deliberately consumer-agnostic — the
* admin panel is its first renderer and a standalone seed-file generator is
* the planned second; do not specialize it toward either.
*
* Shared Zod validates SHAPE only. Semantics (name references, mirror
* composition, mirror-only-on-backups, no nesting, key rules) stay in the
* server registry's validators — the settled two-layer pattern.
*/
/**
* Canonical CONFIGURABLE category list — relocated here from server
* storage.types.ts (which re-exports it) because the wire schema needs it and
* a second copy is forbidden. The legacy `photos` directory is served and
* backed up but not configurable — the server's SERVED_CATEGORIES extension
* (storage.types.ts) carries it.
*/
export const STORAGE_CATEGORIES = [
'files',
'journey',
'covers',
'avatars',
'places',
'photos-google',
'photos-trek',
'backups',
] as const;
export type StorageCategory = (typeof STORAGE_CATEGORIES)[number];
export const storageCategorySchema = z.enum(STORAGE_CATEGORIES);
export const STORAGE_BACKEND_TYPE_IDS = ['local', 's3', 'mirror'] as const;
export type StorageBackendTypeId = (typeof STORAGE_BACKEND_TYPE_IDS)[number];
export interface StorageBackendFieldDef {
key: string;
kind: 'text' | 'path' | 'secret' | 'number' | 'backend-ref' | 'backend-ref-list';
/** i18n key under the `storage` domain (locale files land with the client slice). */
labelKey: string;
helpKey?: string;
required: boolean;
defaultValue?: string | number;
/** Optional per-field refinement piped onto the kind's base Zod type. */
refine?: { schema: z.ZodTypeAny };
}
function fieldSchema(field: StorageBackendFieldDef): z.ZodTypeAny {
let base: z.ZodTypeAny;
switch (field.kind) {
case 'number':
base = z.number();
break;
case 'backend-ref':
base = z.string().min(1);
break;
case 'backend-ref-list':
base = z.array(z.string().min(1));
break;
default: // text | path | secret
base = field.required ? z.string().min(1) : z.string();
}
if (field.defaultValue !== undefined) base = base.default(field.defaultValue);
if (field.refine) base = base.pipe(field.refine.schema);
return base;
}
type FieldValue<F extends StorageBackendFieldDef> = F['kind'] extends 'number'
? number
: F['kind'] extends 'backend-ref-list'
? string[]
: string;
/** Parsed options are total: non-required fields must declare a defaultValue (spec pin). */
export type StorageOptionsOf<Fs extends readonly StorageBackendFieldDef[]> = {
[F in Fs[number] as F['key']]: FieldValue<F>;
};
function optionsSchemaFor<const Fs extends readonly StorageBackendFieldDef[]>(
fields: Fs,
): z.ZodType<StorageOptionsOf<Fs>> {
const shape: Record<string, z.ZodTypeAny> = {};
for (const field of fields) shape[field.key] = fieldSchema(field);
// The one sanctioned cast: the runtime schema is generated from the same
// field defs the mapped type reads, and the parity spec pins the two.
return z.strictObject(shape) as unknown as z.ZodType<StorageOptionsOf<Fs>>;
}
const LOCAL_FIELDS = [
{
key: 'root',
kind: 'path',
labelKey: 'storage.field.root',
helpKey: 'storage.help.root',
required: true,
},
] as const satisfies readonly StorageBackendFieldDef[];
const S3_FIELDS = [
{
key: 'endpoint',
kind: 'text',
labelKey: 'storage.field.endpoint',
helpKey: 'storage.help.endpoint',
required: true,
refine: { schema: z.url() },
},
{ key: 'bucket', kind: 'text', labelKey: 'storage.field.bucket', required: true },
{ key: 'accessKeyId', kind: 'text', labelKey: 'storage.field.accessKeyId', required: true },
{ key: 'secretAccessKey', kind: 'secret', labelKey: 'storage.field.secretAccessKey', required: true },
{
key: 'region',
kind: 'text',
labelKey: 'storage.field.region',
helpKey: 'storage.help.region',
required: false,
defaultValue: 'us-east-1',
},
{
key: 'keyPrefix',
kind: 'text',
labelKey: 'storage.field.keyPrefix',
helpKey: 'storage.help.keyPrefix',
required: false,
defaultValue: '',
},
{
key: 'retries',
kind: 'number',
labelKey: 'storage.field.retries',
required: false,
defaultValue: 1,
refine: { schema: z.number().int().min(0) },
},
{
key: 'timeoutMs',
kind: 'number',
labelKey: 'storage.field.timeoutMs',
required: false,
defaultValue: 30000,
refine: { schema: z.number().int().positive() },
},
] as const satisfies readonly StorageBackendFieldDef[];
const MIRROR_FIELDS = [
{ key: 'primary', kind: 'backend-ref', labelKey: 'storage.field.primary', required: true },
{ key: 'replicas', kind: 'backend-ref-list', labelKey: 'storage.field.replicas', required: true },
] as const satisfies readonly StorageBackendFieldDef[];
export const STORAGE_BACKEND_TYPES = {
local: { fields: LOCAL_FIELDS, optionsSchema: optionsSchemaFor(LOCAL_FIELDS) },
s3: { fields: S3_FIELDS, optionsSchema: optionsSchemaFor(S3_FIELDS) },
mirror: { fields: MIRROR_FIELDS, optionsSchema: optionsSchemaFor(MIRROR_FIELDS) },
} as const;
export type StorageLocalOptions = z.infer<(typeof STORAGE_BACKEND_TYPES)['local']['optionsSchema']>;
export type StorageS3Options = z.infer<(typeof STORAGE_BACKEND_TYPES)['s3']['optionsSchema']>;
export type StorageMirrorOptions = z.infer<(typeof STORAGE_BACKEND_TYPES)['mirror']['optionsSchema']>;
/** Field keys the type marks `secret` — the exact set encrypted at rest and masked on GET. */
export function storageSecretFields(type: StorageBackendTypeId): readonly string[] {
return STORAGE_BACKEND_TYPES[type].fields.filter((f) => f.kind === 'secret').map((f) => f.key);
}
export const storageBackendSchema = z.discriminatedUnion('type', [
z.strictObject({
name: z.string().min(1),
type: z.literal('local'),
options: STORAGE_BACKEND_TYPES.local.optionsSchema,
}),
z.strictObject({
name: z.string().min(1),
type: z.literal('s3'),
options: STORAGE_BACKEND_TYPES.s3.optionsSchema,
}),
z.strictObject({
name: z.string().min(1),
type: z.literal('mirror'),
options: STORAGE_BACKEND_TYPES.mirror.optionsSchema,
}),
]);
export type StorageBackend = z.infer<typeof storageBackendSchema>;
/**
* The two app_settings rows as one document — the PUT body and the seed-file
* shape. `.strict()` at every level: unknown keys are rejected, including the
* reserved top-level `readOnly` seed key (loud until a standing file-override
* mode is actually implemented). Byte-compatible with rows hand-written
* during the S3 slice (pinned in the sibling spec file).
*/
export const storageConfigSchema = z.strictObject({
backends: z.array(storageBackendSchema),
categories: z.partialRecord(storageCategorySchema, z.string().min(1)),
});
export type StorageConfig = z.infer<typeof storageConfigSchema>;
/**
* The PUT wire shape: the config document plus the optimistic-concurrency
* version the client read it at (audit #7 — a stale PUT can otherwise silently
* undo a category migration's just-flipped assignment). `storageConfigSchema`
* itself stays untouched — it doubles as the seed-file shape, which has no
* version to compare against.
*/
export const storageConfigPutSchema = storageConfigSchema.extend({
version: z.number().int().nonnegative(),
});
export type StorageConfigPut = z.infer<typeof storageConfigPutSchema>;
// ── GET /api/admin/storage state + POST /test wire shapes ─────────────────────
export const storageBackendSourceSchema = z.enum(['built-in', 'env', 'settings']);
export const storageCategorySourceSchema = z.enum(['default', 'settings']);
export const storageReplicaFailureSchema = z.object({
backend: z.string(),
key: z.string(),
/** 'list' is the backfill sweep phase's replica.list() failing. */
op: z.enum(['put', 'delete', 'stat', 'list']),
error: z.string(),
at: z.number(),
});
// ── Backfill + usage (backfill/stats/notifications spec) ─────────────────────
export const storageBackfillStatusSchema = z.object({
/** The mirror's wire name — user-facing copy says "Sync", never this. */
backend: z.string(),
status: z.enum(['running', 'done', 'error', 'cancelled']),
/** Objects examined in the copy phase; total is 0 while enumerating. */
done: z.number(),
total: z.number(),
copied: z.number(),
skipped: z.number(),
failed: z.number(),
/** Replica objects removed by the sweep phase — absent from the primary and not raced back. */
deleted: z.number(),
startedAt: z.number(),
finishedAt: z.number().optional(),
/** status 'error' only: the aborting (primary-side) error. */
error: z.string().optional(),
});
export type StorageBackfillStatus = z.infer<typeof storageBackfillStatusSchema>;
// ── Category migration (copy → flip → delta sweep) ────────────────────────────
export const storageMigrationStatusSchema = z.object({
category: storageCategorySchema,
from: z.string(),
to: z.string(),
status: z.enum(['running', 'done', 'failed', 'cancelled']),
/** Keys examined in the copy phase; total is 0 while enumerating. */
done: z.number(),
total: z.number(),
copied: z.number(),
skipped: z.number(),
failed: z.number(),
startedAt: z.number(),
finishedAt: z.number().optional(),
/** status 'failed' only: the aborting error (copy failures also set counts). */
error: z.string().optional(),
/** Terminal 'done' only: objects/bytes left on the old backend. */
reclaimable: z.object({ objects: z.number(), bytes: z.number() }).optional(),
});
export type StorageMigrationStatus = z.infer<typeof storageMigrationStatusSchema>;
export const storageUsageSchema = z.object({
computedAt: z.number(),
categories: z.record(storageCategorySchema, z.object({ objects: z.number(), bytes: z.number() })),
/** The served-legacy photos directory — attributed to whatever backend `photos` resolves to. */
legacyPhotos: z.object({ objects: z.number(), bytes: z.number() }),
});
export type StorageUsage = z.infer<typeof storageUsageSchema>;
// ── Reclaim: objects no row names any more ────────────────────────────────────
/** Every category the byte-paths serve, the legacy `photos` folder included: the reclaim report covers each. */
export const STORAGE_RECLAIM_CATEGORIES = [...STORAGE_CATEGORIES, 'photos'] as const;
export const storageReclaimCategorySchema = z.enum(STORAGE_RECLAIM_CATEGORIES);
/**
* Why a category was not judged on a pass, or stopped short, whatever the switch says:
* - `not_reclaimed`: the category is never reclaimed (a cache, the backups, the legacy photos folder);
* - `sources_missing`: a domain whose rows can name its objects did not answer, so nothing can be proven unreferenced;
* - `shared_location`: its objects sit where another category's do, or where they cannot be told apart;
* - `foreign_install`: another TREK installation keeps its uploads in the same place (its owner marker is there);
* - `owner_unknown`: this installation's owner marker could not be written there, so a shared place cannot be ruled out;
* - `list_failed` / `source_failed`: the listing or a reference lookup failed, so the pass stopped there;
* - `delete_failed`: deleting failed on the backend, so the pass stopped deleting in this category;
* - `backup_running`: a backup was reading the uploads when this category's objects were due, so the pass
* deleted nothing there (from then on, for the rest of the pass); the next pass tries again;
* - `interrupted`: the pass stopped (`stopped` says why) before it finished this category; its figures are partial.
* A skip keeps the marks it found: only a pass that judges the category moves them.
*/
export const STORAGE_RECLAIM_SKIP_REASONS = [
'not_reclaimed',
'sources_missing',
'shared_location',
'foreign_install',
'owner_unknown',
'list_failed',
'source_failed',
'delete_failed',
'backup_running',
'interrupted',
] as const;
const storageObjectCountSchema = z.object({ objects: z.number().int().nonnegative(), bytes: z.number().nonnegative() });
export const storageReclaimCategoryReportSchema = z.object({
/** Objects the pass looked at; 0 when the category was skipped before it was listed. */
objects: z.number().int().nonnegative(),
/** Older than a day and named by no row. */
unreferenced: storageObjectCountSchema,
/** Unreferenced on every pass for at least a week: what a pass with deletion on removes. */
eligible: storageObjectCountSchema,
deleted: storageObjectCountSchema,
/**
* More than a fifth of the category is unreferenced and no admin confirmation
* covers that count: deletion waits for one (POST .../reclaim/confirm). The
* admin state reads false once a confirmation covers it, before the next pass.
*/
confirmationRequired: z.boolean(),
skipped: z.enum(STORAGE_RECLAIM_SKIP_REASONS).nullable(),
});
export type StorageReclaimCategoryReport = z.infer<typeof storageReclaimCategoryReportSchema>;
export const storageReclaimReportSchema = z.object({
startedAt: z.number(),
finishedAt: z.number(),
/** `delete` when the pass started with deletion switched on; a check started by hand always only reports. */
mode: z.enum(['report', 'delete']),
/**
* Why the pass stopped before the end: the database was swapped under it (a
* restore), or an unexpected error. Null when it ran through.
*/
stopped: z.enum(['database_reopened', 'failed']).nullable(),
categories: z.record(storageReclaimCategorySchema, storageReclaimCategoryReportSchema),
});
export type StorageReclaimReport = z.infer<typeof storageReclaimReportSchema>;
export const storageReclaimStateSchema = z.object({
/** Off after install, upgrade and every backup restore: a pass then only reports. */
deleteEnabled: z.boolean(),
/** A pass is running now (the nightly one or a check started by hand). */
running: z.boolean(),
/** The last pass of any kind; null before the first. */
lastRun: storageReclaimReportSchema.nullable(),
/** The last pass that ran with deletion on, which a check started by hand never replaces; null before one. */
lastDeleteRun: storageReclaimReportSchema.nullable(),
});
export type StorageReclaimState = z.infer<typeof storageReclaimStateSchema>;
export const storageReclaimSettingsRequestSchema = z.strictObject({ deleteEnabled: z.boolean() });
export type StorageReclaimSettingsRequest = z.infer<typeof storageReclaimSettingsRequestSchema>;
/** POST .../reclaim/run answers 202 at once; the pass reports through the admin state. */
export const storageReclaimRunStartedSchema = z.object({ started: z.literal(true) });
/**
* Lets the pass delete in a category whose unreferenced share is over a
* fifth. `unreferenced` is the count the admin was shown; the server refuses
* (409 `reclaim_confirm_stale`) when its last report's count for the category
* differs from it by more than a tenth or by more than 50 objects. The
* confirmation then covers up to one and a half times that count (at least 100
* more) for 14 days, and is used up once the share falls under a fifth again.
*/
export const storageReclaimConfirmRequestSchema = z.strictObject({
category: storageReclaimCategorySchema,
unreferenced: z.number().int().nonnegative(),
});
export type StorageReclaimConfirmRequest = z.infer<typeof storageReclaimConfirmRequestSchema>;
export const storageAdminStateSchema = z.object({
backends: z.array(
z.object({
name: z.string(),
type: z.enum(STORAGE_BACKEND_TYPE_IDS),
source: storageBackendSourceSchema,
/** Secret-kind fields carry MASKED_SETTING_VALUE — never a stored value. */
options: z.record(z.string(), z.union([z.string(), z.number(), z.array(z.string())])),
categories: z.array(storageCategorySchema),
}),
),
// Exhaustive record: the effective world always maps all eight configurable categories.
categories: z.record(
storageCategorySchema,
z.object({ backend: z.string().min(1), source: storageCategorySourceSchema }),
),
health: z.object({ replicaFailures: z.array(storageReplicaFailureSchema) }),
seedFilePresent: z.boolean(),
usage: storageUsageSchema.nullable(),
/**
* The reclaim switch and the last pass's report (`lastRun` null before the
* first pass). Nullable so a client reading an older server copes with it.
*/
reclaim: storageReclaimStateSchema.nullable(),
backfills: z.array(storageBackfillStatusSchema),
migrations: z.array(storageMigrationStatusSchema),
/** Optimistic-concurrency counter for the two settings rows (audit #7) — echo it back on the next PUT. */
version: z.number().int().nonnegative(),
/**
* Non-null when the stored storage settings failed to parse/validate and
* the registry fell back to the last-good config (or built-in defaults) —
* the admin panel renders this as a warning banner so a save is never a
* silent replacement of a config the operator never actually saw (audit
* minor). Null once the stored settings load cleanly.
*/
configError: z.string().nullable(),
});
export type StorageAdminState = z.infer<typeof storageAdminStateSchema>;
export const storageTestRequestSchema = z.strictObject({ backend: storageBackendSchema });
export type StorageTestRequest = z.infer<typeof storageTestRequestSchema>;
export const storageMigrationRequestSchema = z.strictObject({
category: storageCategorySchema,
to: z.string().min(1),
});
export type StorageMigrationRequest = z.infer<typeof storageMigrationRequestSchema>;
export const storageTestResponseSchema = z.object({
ok: z.boolean(),
targets: z.array(z.object({ name: z.string(), ok: z.boolean(), error: z.string().optional() })),
});
export type StorageTestResponse = z.infer<typeof storageTestResponseSchema>;