mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 19:16:08 +00:00
0a25de2c6d
The confirm request now carries the unreferenced count the admin saw, so the server can refuse a count its last report no longer holds. A pass that found objects due while a backup read the uploads reports backup_running for them. A trip cover is bounded to 2048 characters on REST, RPC and the MCP tool, as a place image already is, since a replaced cover is queued for release by every upload path it spells out.
431 lines
18 KiB
TypeScript
431 lines
18 KiB
TypeScript
import { z } from 'zod';
|
|
|
|
/**
|
|
* Storage admin contract (spec: docs/superpowers/specs/2026-08-19-storage-admin-config-design.md).
|
|
*
|
|
* STORAGE_BACKEND_TYPES is the single schema-driven registry: each backend
|
|
* type declares its fields once, and both the Zod options schema (generated
|
|
* below) and the admin form's field metadata render from that one definition.
|
|
* Adding a backend type = one entry here + one driver class + one registry
|
|
* construction line. The registry is deliberately consumer-agnostic — the
|
|
* admin panel is its first renderer and a standalone seed-file generator is
|
|
* the planned second; do not specialize it toward either.
|
|
*
|
|
* Shared Zod validates SHAPE only. Semantics (name references, mirror
|
|
* composition, mirror-only-on-backups, no nesting, key rules) stay in the
|
|
* server registry's validators — the settled two-layer pattern.
|
|
*/
|
|
|
|
/**
|
|
* Canonical CONFIGURABLE category list — relocated here from server
|
|
* storage.types.ts (which re-exports it) because the wire schema needs it and
|
|
* a second copy is forbidden. The legacy `photos` directory is served and
|
|
* backed up but not configurable — the server's SERVED_CATEGORIES extension
|
|
* (storage.types.ts) carries it.
|
|
*/
|
|
export const STORAGE_CATEGORIES = [
|
|
'files',
|
|
'journey',
|
|
'covers',
|
|
'avatars',
|
|
'places',
|
|
'photos-google',
|
|
'photos-trek',
|
|
'backups',
|
|
] as const;
|
|
export type StorageCategory = (typeof STORAGE_CATEGORIES)[number];
|
|
export const storageCategorySchema = z.enum(STORAGE_CATEGORIES);
|
|
|
|
export const STORAGE_BACKEND_TYPE_IDS = ['local', 's3', 'mirror'] as const;
|
|
export type StorageBackendTypeId = (typeof STORAGE_BACKEND_TYPE_IDS)[number];
|
|
|
|
export interface StorageBackendFieldDef {
|
|
key: string;
|
|
kind: 'text' | 'path' | 'secret' | 'number' | 'backend-ref' | 'backend-ref-list';
|
|
/** i18n key under the `storage` domain (locale files land with the client slice). */
|
|
labelKey: string;
|
|
helpKey?: string;
|
|
required: boolean;
|
|
defaultValue?: string | number;
|
|
/** Optional per-field refinement piped onto the kind's base Zod type. */
|
|
refine?: { schema: z.ZodTypeAny };
|
|
}
|
|
|
|
function fieldSchema(field: StorageBackendFieldDef): z.ZodTypeAny {
|
|
let base: z.ZodTypeAny;
|
|
switch (field.kind) {
|
|
case 'number':
|
|
base = z.number();
|
|
break;
|
|
case 'backend-ref':
|
|
base = z.string().min(1);
|
|
break;
|
|
case 'backend-ref-list':
|
|
base = z.array(z.string().min(1));
|
|
break;
|
|
default: // text | path | secret
|
|
base = field.required ? z.string().min(1) : z.string();
|
|
}
|
|
if (field.defaultValue !== undefined) base = base.default(field.defaultValue);
|
|
if (field.refine) base = base.pipe(field.refine.schema);
|
|
return base;
|
|
}
|
|
|
|
type FieldValue<F extends StorageBackendFieldDef> = F['kind'] extends 'number'
|
|
? number
|
|
: F['kind'] extends 'backend-ref-list'
|
|
? string[]
|
|
: string;
|
|
/** Parsed options are total: non-required fields must declare a defaultValue (spec pin). */
|
|
export type StorageOptionsOf<Fs extends readonly StorageBackendFieldDef[]> = {
|
|
[F in Fs[number] as F['key']]: FieldValue<F>;
|
|
};
|
|
|
|
function optionsSchemaFor<const Fs extends readonly StorageBackendFieldDef[]>(
|
|
fields: Fs,
|
|
): z.ZodType<StorageOptionsOf<Fs>> {
|
|
const shape: Record<string, z.ZodTypeAny> = {};
|
|
for (const field of fields) shape[field.key] = fieldSchema(field);
|
|
// The one sanctioned cast: the runtime schema is generated from the same
|
|
// field defs the mapped type reads, and the parity spec pins the two.
|
|
return z.strictObject(shape) as unknown as z.ZodType<StorageOptionsOf<Fs>>;
|
|
}
|
|
|
|
const LOCAL_FIELDS = [
|
|
{
|
|
key: 'root',
|
|
kind: 'path',
|
|
labelKey: 'storage.field.root',
|
|
helpKey: 'storage.help.root',
|
|
required: true,
|
|
},
|
|
] as const satisfies readonly StorageBackendFieldDef[];
|
|
|
|
const S3_FIELDS = [
|
|
{
|
|
key: 'endpoint',
|
|
kind: 'text',
|
|
labelKey: 'storage.field.endpoint',
|
|
helpKey: 'storage.help.endpoint',
|
|
required: true,
|
|
refine: { schema: z.url() },
|
|
},
|
|
{ key: 'bucket', kind: 'text', labelKey: 'storage.field.bucket', required: true },
|
|
{ key: 'accessKeyId', kind: 'text', labelKey: 'storage.field.accessKeyId', required: true },
|
|
{ key: 'secretAccessKey', kind: 'secret', labelKey: 'storage.field.secretAccessKey', required: true },
|
|
{
|
|
key: 'region',
|
|
kind: 'text',
|
|
labelKey: 'storage.field.region',
|
|
helpKey: 'storage.help.region',
|
|
required: false,
|
|
defaultValue: 'us-east-1',
|
|
},
|
|
{
|
|
key: 'keyPrefix',
|
|
kind: 'text',
|
|
labelKey: 'storage.field.keyPrefix',
|
|
helpKey: 'storage.help.keyPrefix',
|
|
required: false,
|
|
defaultValue: '',
|
|
},
|
|
{
|
|
key: 'retries',
|
|
kind: 'number',
|
|
labelKey: 'storage.field.retries',
|
|
required: false,
|
|
defaultValue: 1,
|
|
refine: { schema: z.number().int().min(0) },
|
|
},
|
|
{
|
|
key: 'timeoutMs',
|
|
kind: 'number',
|
|
labelKey: 'storage.field.timeoutMs',
|
|
required: false,
|
|
defaultValue: 30000,
|
|
refine: { schema: z.number().int().positive() },
|
|
},
|
|
] as const satisfies readonly StorageBackendFieldDef[];
|
|
|
|
const MIRROR_FIELDS = [
|
|
{ key: 'primary', kind: 'backend-ref', labelKey: 'storage.field.primary', required: true },
|
|
{ key: 'replicas', kind: 'backend-ref-list', labelKey: 'storage.field.replicas', required: true },
|
|
] as const satisfies readonly StorageBackendFieldDef[];
|
|
|
|
export const STORAGE_BACKEND_TYPES = {
|
|
local: { fields: LOCAL_FIELDS, optionsSchema: optionsSchemaFor(LOCAL_FIELDS) },
|
|
s3: { fields: S3_FIELDS, optionsSchema: optionsSchemaFor(S3_FIELDS) },
|
|
mirror: { fields: MIRROR_FIELDS, optionsSchema: optionsSchemaFor(MIRROR_FIELDS) },
|
|
} as const;
|
|
|
|
export type StorageLocalOptions = z.infer<(typeof STORAGE_BACKEND_TYPES)['local']['optionsSchema']>;
|
|
export type StorageS3Options = z.infer<(typeof STORAGE_BACKEND_TYPES)['s3']['optionsSchema']>;
|
|
export type StorageMirrorOptions = z.infer<(typeof STORAGE_BACKEND_TYPES)['mirror']['optionsSchema']>;
|
|
|
|
/** Field keys the type marks `secret` — the exact set encrypted at rest and masked on GET. */
|
|
export function storageSecretFields(type: StorageBackendTypeId): readonly string[] {
|
|
return STORAGE_BACKEND_TYPES[type].fields.filter((f) => f.kind === 'secret').map((f) => f.key);
|
|
}
|
|
|
|
export const storageBackendSchema = z.discriminatedUnion('type', [
|
|
z.strictObject({
|
|
name: z.string().min(1),
|
|
type: z.literal('local'),
|
|
options: STORAGE_BACKEND_TYPES.local.optionsSchema,
|
|
}),
|
|
z.strictObject({
|
|
name: z.string().min(1),
|
|
type: z.literal('s3'),
|
|
options: STORAGE_BACKEND_TYPES.s3.optionsSchema,
|
|
}),
|
|
z.strictObject({
|
|
name: z.string().min(1),
|
|
type: z.literal('mirror'),
|
|
options: STORAGE_BACKEND_TYPES.mirror.optionsSchema,
|
|
}),
|
|
]);
|
|
export type StorageBackend = z.infer<typeof storageBackendSchema>;
|
|
|
|
/**
|
|
* The two app_settings rows as one document — the PUT body and the seed-file
|
|
* shape. `.strict()` at every level: unknown keys are rejected, including the
|
|
* reserved top-level `readOnly` seed key (loud until a standing file-override
|
|
* mode is actually implemented). Byte-compatible with rows hand-written
|
|
* during the S3 slice (pinned in the sibling spec file).
|
|
*/
|
|
export const storageConfigSchema = z.strictObject({
|
|
backends: z.array(storageBackendSchema),
|
|
categories: z.partialRecord(storageCategorySchema, z.string().min(1)),
|
|
});
|
|
export type StorageConfig = z.infer<typeof storageConfigSchema>;
|
|
|
|
/**
|
|
* The PUT wire shape: the config document plus the optimistic-concurrency
|
|
* version the client read it at (audit #7 — a stale PUT can otherwise silently
|
|
* undo a category migration's just-flipped assignment). `storageConfigSchema`
|
|
* itself stays untouched — it doubles as the seed-file shape, which has no
|
|
* version to compare against.
|
|
*/
|
|
export const storageConfigPutSchema = storageConfigSchema.extend({
|
|
version: z.number().int().nonnegative(),
|
|
});
|
|
export type StorageConfigPut = z.infer<typeof storageConfigPutSchema>;
|
|
|
|
// ── GET /api/admin/storage state + POST /test wire shapes ─────────────────────
|
|
|
|
export const storageBackendSourceSchema = z.enum(['built-in', 'env', 'settings']);
|
|
export const storageCategorySourceSchema = z.enum(['default', 'settings']);
|
|
|
|
export const storageReplicaFailureSchema = z.object({
|
|
backend: z.string(),
|
|
key: z.string(),
|
|
/** 'list' is the backfill sweep phase's replica.list() failing. */
|
|
op: z.enum(['put', 'delete', 'stat', 'list']),
|
|
error: z.string(),
|
|
at: z.number(),
|
|
});
|
|
|
|
// ── Backfill + usage (backfill/stats/notifications spec) ─────────────────────
|
|
|
|
export const storageBackfillStatusSchema = z.object({
|
|
/** The mirror's wire name — user-facing copy says "Sync", never this. */
|
|
backend: z.string(),
|
|
status: z.enum(['running', 'done', 'error', 'cancelled']),
|
|
/** Objects examined in the copy phase; total is 0 while enumerating. */
|
|
done: z.number(),
|
|
total: z.number(),
|
|
copied: z.number(),
|
|
skipped: z.number(),
|
|
failed: z.number(),
|
|
/** Replica objects removed by the sweep phase — absent from the primary and not raced back. */
|
|
deleted: z.number(),
|
|
startedAt: z.number(),
|
|
finishedAt: z.number().optional(),
|
|
/** status 'error' only: the aborting (primary-side) error. */
|
|
error: z.string().optional(),
|
|
});
|
|
export type StorageBackfillStatus = z.infer<typeof storageBackfillStatusSchema>;
|
|
|
|
// ── Category migration (copy → flip → delta sweep) ────────────────────────────
|
|
|
|
export const storageMigrationStatusSchema = z.object({
|
|
category: storageCategorySchema,
|
|
from: z.string(),
|
|
to: z.string(),
|
|
status: z.enum(['running', 'done', 'failed', 'cancelled']),
|
|
/** Keys examined in the copy phase; total is 0 while enumerating. */
|
|
done: z.number(),
|
|
total: z.number(),
|
|
copied: z.number(),
|
|
skipped: z.number(),
|
|
failed: z.number(),
|
|
startedAt: z.number(),
|
|
finishedAt: z.number().optional(),
|
|
/** status 'failed' only: the aborting error (copy failures also set counts). */
|
|
error: z.string().optional(),
|
|
/** Terminal 'done' only: objects/bytes left on the old backend. */
|
|
reclaimable: z.object({ objects: z.number(), bytes: z.number() }).optional(),
|
|
});
|
|
export type StorageMigrationStatus = z.infer<typeof storageMigrationStatusSchema>;
|
|
|
|
export const storageUsageSchema = z.object({
|
|
computedAt: z.number(),
|
|
categories: z.record(storageCategorySchema, z.object({ objects: z.number(), bytes: z.number() })),
|
|
/** The served-legacy photos directory — attributed to whatever backend `photos` resolves to. */
|
|
legacyPhotos: z.object({ objects: z.number(), bytes: z.number() }),
|
|
});
|
|
export type StorageUsage = z.infer<typeof storageUsageSchema>;
|
|
|
|
// ── Reclaim: objects no row names any more ────────────────────────────────────
|
|
|
|
/** Every category the byte-paths serve, the legacy `photos` folder included: the reclaim report covers each. */
|
|
export const STORAGE_RECLAIM_CATEGORIES = [...STORAGE_CATEGORIES, 'photos'] as const;
|
|
export const storageReclaimCategorySchema = z.enum(STORAGE_RECLAIM_CATEGORIES);
|
|
|
|
/**
|
|
* Why a category was not judged on a pass, or stopped short, whatever the switch says:
|
|
* - `not_reclaimed`: the category is never reclaimed (a cache, the backups, the legacy photos folder);
|
|
* - `sources_missing`: a domain whose rows can name its objects did not answer, so nothing can be proven unreferenced;
|
|
* - `shared_location`: its objects sit where another category's do, or where they cannot be told apart;
|
|
* - `foreign_install`: another TREK installation keeps its uploads in the same place (its owner marker is there);
|
|
* - `owner_unknown`: this installation's owner marker could not be written there, so a shared place cannot be ruled out;
|
|
* - `list_failed` / `source_failed`: the listing or a reference lookup failed, so the pass stopped there;
|
|
* - `delete_failed`: deleting failed on the backend, so the pass stopped deleting in this category;
|
|
* - `backup_running`: a backup was reading the uploads when this category's objects were due, so the pass
|
|
* deleted nothing there (from then on, for the rest of the pass); the next pass tries again;
|
|
* - `interrupted`: the pass stopped (`stopped` says why) before it finished this category; its figures are partial.
|
|
* A skip keeps the marks it found: only a pass that judges the category moves them.
|
|
*/
|
|
export const STORAGE_RECLAIM_SKIP_REASONS = [
|
|
'not_reclaimed',
|
|
'sources_missing',
|
|
'shared_location',
|
|
'foreign_install',
|
|
'owner_unknown',
|
|
'list_failed',
|
|
'source_failed',
|
|
'delete_failed',
|
|
'backup_running',
|
|
'interrupted',
|
|
] as const;
|
|
|
|
const storageObjectCountSchema = z.object({ objects: z.number().int().nonnegative(), bytes: z.number().nonnegative() });
|
|
|
|
export const storageReclaimCategoryReportSchema = z.object({
|
|
/** Objects the pass looked at; 0 when the category was skipped before it was listed. */
|
|
objects: z.number().int().nonnegative(),
|
|
/** Older than a day and named by no row. */
|
|
unreferenced: storageObjectCountSchema,
|
|
/** Unreferenced on every pass for at least a week: what a pass with deletion on removes. */
|
|
eligible: storageObjectCountSchema,
|
|
deleted: storageObjectCountSchema,
|
|
/**
|
|
* More than a fifth of the category is unreferenced and no admin confirmation
|
|
* covers that count: deletion waits for one (POST .../reclaim/confirm). The
|
|
* admin state reads false once a confirmation covers it, before the next pass.
|
|
*/
|
|
confirmationRequired: z.boolean(),
|
|
skipped: z.enum(STORAGE_RECLAIM_SKIP_REASONS).nullable(),
|
|
});
|
|
export type StorageReclaimCategoryReport = z.infer<typeof storageReclaimCategoryReportSchema>;
|
|
|
|
export const storageReclaimReportSchema = z.object({
|
|
startedAt: z.number(),
|
|
finishedAt: z.number(),
|
|
/** `delete` when the pass started with deletion switched on; a check started by hand always only reports. */
|
|
mode: z.enum(['report', 'delete']),
|
|
/**
|
|
* Why the pass stopped before the end: the database was swapped under it (a
|
|
* restore), or an unexpected error. Null when it ran through.
|
|
*/
|
|
stopped: z.enum(['database_reopened', 'failed']).nullable(),
|
|
categories: z.record(storageReclaimCategorySchema, storageReclaimCategoryReportSchema),
|
|
});
|
|
export type StorageReclaimReport = z.infer<typeof storageReclaimReportSchema>;
|
|
|
|
export const storageReclaimStateSchema = z.object({
|
|
/** Off after install, upgrade and every backup restore: a pass then only reports. */
|
|
deleteEnabled: z.boolean(),
|
|
/** A pass is running now (the nightly one or a check started by hand). */
|
|
running: z.boolean(),
|
|
/** The last pass of any kind; null before the first. */
|
|
lastRun: storageReclaimReportSchema.nullable(),
|
|
/** The last pass that ran with deletion on, which a check started by hand never replaces; null before one. */
|
|
lastDeleteRun: storageReclaimReportSchema.nullable(),
|
|
});
|
|
export type StorageReclaimState = z.infer<typeof storageReclaimStateSchema>;
|
|
|
|
export const storageReclaimSettingsRequestSchema = z.strictObject({ deleteEnabled: z.boolean() });
|
|
export type StorageReclaimSettingsRequest = z.infer<typeof storageReclaimSettingsRequestSchema>;
|
|
|
|
/** POST .../reclaim/run answers 202 at once; the pass reports through the admin state. */
|
|
export const storageReclaimRunStartedSchema = z.object({ started: z.literal(true) });
|
|
|
|
/**
|
|
* Lets the pass delete in a category whose unreferenced share is over a
|
|
* fifth. `unreferenced` is the count the admin was shown; the server refuses
|
|
* (409 `reclaim_confirm_stale`) when its last report's count for the category
|
|
* differs from it by more than a tenth or by more than 50 objects. The
|
|
* confirmation then covers up to one and a half times that count (at least 100
|
|
* more) for 14 days, and is used up once the share falls under a fifth again.
|
|
*/
|
|
export const storageReclaimConfirmRequestSchema = z.strictObject({
|
|
category: storageReclaimCategorySchema,
|
|
unreferenced: z.number().int().nonnegative(),
|
|
});
|
|
export type StorageReclaimConfirmRequest = z.infer<typeof storageReclaimConfirmRequestSchema>;
|
|
|
|
export const storageAdminStateSchema = z.object({
|
|
backends: z.array(
|
|
z.object({
|
|
name: z.string(),
|
|
type: z.enum(STORAGE_BACKEND_TYPE_IDS),
|
|
source: storageBackendSourceSchema,
|
|
/** Secret-kind fields carry MASKED_SETTING_VALUE — never a stored value. */
|
|
options: z.record(z.string(), z.union([z.string(), z.number(), z.array(z.string())])),
|
|
categories: z.array(storageCategorySchema),
|
|
}),
|
|
),
|
|
// Exhaustive record: the effective world always maps all eight configurable categories.
|
|
categories: z.record(
|
|
storageCategorySchema,
|
|
z.object({ backend: z.string().min(1), source: storageCategorySourceSchema }),
|
|
),
|
|
health: z.object({ replicaFailures: z.array(storageReplicaFailureSchema) }),
|
|
seedFilePresent: z.boolean(),
|
|
usage: storageUsageSchema.nullable(),
|
|
/**
|
|
* The reclaim switch and the last pass's report (`lastRun` null before the
|
|
* first pass). Nullable so a client reading an older server copes with it.
|
|
*/
|
|
reclaim: storageReclaimStateSchema.nullable(),
|
|
backfills: z.array(storageBackfillStatusSchema),
|
|
migrations: z.array(storageMigrationStatusSchema),
|
|
/** Optimistic-concurrency counter for the two settings rows (audit #7) — echo it back on the next PUT. */
|
|
version: z.number().int().nonnegative(),
|
|
/**
|
|
* Non-null when the stored storage settings failed to parse/validate and
|
|
* the registry fell back to the last-good config (or built-in defaults) —
|
|
* the admin panel renders this as a warning banner so a save is never a
|
|
* silent replacement of a config the operator never actually saw (audit
|
|
* minor). Null once the stored settings load cleanly.
|
|
*/
|
|
configError: z.string().nullable(),
|
|
});
|
|
export type StorageAdminState = z.infer<typeof storageAdminStateSchema>;
|
|
|
|
export const storageTestRequestSchema = z.strictObject({ backend: storageBackendSchema });
|
|
export type StorageTestRequest = z.infer<typeof storageTestRequestSchema>;
|
|
|
|
export const storageMigrationRequestSchema = z.strictObject({
|
|
category: storageCategorySchema,
|
|
to: z.string().min(1),
|
|
});
|
|
export type StorageMigrationRequest = z.infer<typeof storageMigrationRequestSchema>;
|
|
|
|
export const storageTestResponseSchema = z.object({
|
|
ok: z.boolean(),
|
|
targets: z.array(z.object({ name: z.string(), ok: z.boolean(), error: z.string().optional() })),
|
|
});
|
|
export type StorageTestResponse = z.infer<typeof storageTestResponseSchema>;
|