diff --git a/server/scripts/repository-owners.json b/server/scripts/repository-owners.json index 3cec95439..dd1b82c8a 100644 --- a/server/scripts/repository-owners.json +++ b/server/scripts/repository-owners.json @@ -119,6 +119,7 @@ "SchoolHolidayRegions": "school-holidays", "Settings": "settings", "ShareTokens": "share", + "StorageReclaimMarks": "storage", "Tags": "tags", "TodoCategoryAssignees": "todo", "TodoItems": "todo", diff --git a/server/src/db/entities/StorageReclaimMarks.entity.ts b/server/src/db/entities/StorageReclaimMarks.entity.ts new file mode 100644 index 000000000..09c9d32cd --- /dev/null +++ b/server/src/db/entities/StorageReclaimMarks.entity.ts @@ -0,0 +1,23 @@ +import { EntityRepositoryType, PrimaryKeyProp, defineEntity, p } from '@mikro-orm/core'; +import { StorageReclaimMarksRepository } from '../repositories/StorageReclaimMarks.repository'; +import { DbTimestampType } from '../types'; + +export class StorageReclaimMarks { + [EntityRepositoryType]?: StorageReclaimMarksRepository; + [PrimaryKeyProp]?: ['category', 'key']; + category!: string; + key!: string; + first_seen_at!: string; + size!: number; +} + +export const StorageReclaimMarksSchema = defineEntity({ + class: StorageReclaimMarks, + repository: () => StorageReclaimMarksRepository, + properties: { + category: p.text().primary(), + key: p.text().primary(), + first_seen_at: p.type(DbTimestampType), + size: p.integer(), + }, +}); diff --git a/server/src/db/entities/index.ts b/server/src/db/entities/index.ts index 134e7e21c..5d77e026c 100644 --- a/server/src/db/entities/index.ts +++ b/server/src/db/entities/index.ts @@ -118,6 +118,7 @@ import { SchoolHolidayPeriodsSchema } from './SchoolHolidayPeriods.entity'; import { SchoolHolidayRegionsSchema } from './SchoolHolidayRegions.entity'; import { SettingsSchema } from './Settings.entity'; import { ShareTokensSchema } from './ShareTokens.entity'; +import { StorageReclaimMarksSchema } from './StorageReclaimMarks.entity'; import { TagsSchema } from './Tags.entity'; import { TodoCategoryAssigneesSchema } from './TodoCategoryAssignees.entity'; import { TodoItemsSchema } from './TodoItems.entity'; @@ -285,6 +286,7 @@ export const ALL_ENTITIES: readonly EntitySchema[] = [ SchoolHolidayRegionsSchema, SettingsSchema, ShareTokensSchema, + StorageReclaimMarksSchema, TagsSchema, TodoCategoryAssigneesSchema, TodoItemsSchema, diff --git a/server/src/db/kysely/StorageReclaimMarks.table.ts b/server/src/db/kysely/StorageReclaimMarks.table.ts new file mode 100644 index 000000000..0769e63fb --- /dev/null +++ b/server/src/db/kysely/StorageReclaimMarks.table.ts @@ -0,0 +1,9 @@ +// Generated by scripts/generate-db-types.ts from the migrated schema. Do not edit by hand: +// change the migration, then run `npm run gen:db-types`. + +export interface StorageReclaimMarksTable { + category: string; + key: string; + first_seen_at: string; + size: number; +} diff --git a/server/src/db/kysely/db.ts b/server/src/db/kysely/db.ts index cd8ab820e..575fbc16e 100644 --- a/server/src/db/kysely/db.ts +++ b/server/src/db/kysely/db.ts @@ -124,6 +124,7 @@ import type { SchoolHolidayPeriodsTable } from './SchoolHolidayPeriods.table'; import type { SchoolHolidayRegionsTable } from './SchoolHolidayRegions.table'; import type { SettingsTable } from './Settings.table'; import type { ShareTokensTable } from './ShareTokens.table'; +import type { StorageReclaimMarksTable } from './StorageReclaimMarks.table'; import type { TagsTable } from './Tags.table'; import type { TodoCategoryAssigneesTable } from './TodoCategoryAssignees.table'; import type { TodoItemsTable } from './TodoItems.table'; @@ -297,6 +298,7 @@ export interface DB { school_holiday_regions: SchoolHolidayRegionsTable; settings: SettingsTable; share_tokens: ShareTokensTable; + storage_reclaim_marks: StorageReclaimMarksTable; tags: TagsTable; todo_category_assignees: TodoCategoryAssigneesTable; todo_items: TodoItemsTable; diff --git a/server/src/db/migrations/Migration20200101046330_an_object_no_row_names_is_marked_before_it_goes.ts b/server/src/db/migrations/Migration20200101046330_an_object_no_row_names_is_marked_before_it_goes.ts new file mode 100644 index 000000000..17d689162 --- /dev/null +++ b/server/src/db/migrations/Migration20200101046330_an_object_no_row_names_is_marked_before_it_goes.ts @@ -0,0 +1,24 @@ +import { Migration } from '@mikro-orm/migrations'; + +/** + * The storage reclaim pass remembers, per stored object, since when no row has + * named it. An object is deleted only once it has stayed unreferenced on every + * pass for a week (and only when the admin switched deletion on), so a value a + * client writes back late, an offline replay or a copy made in between keeps + * its object. A row goes as soon as its object is referenced again or gone. + */ +export class Migration20200101046330_an_object_no_row_names_is_marked_before_it_goes extends Migration { + override name = 'Migration20200101046330_an_object_no_row_names_is_marked_before_it_goes'; + + override up(): void { + this.addSql(` + CREATE TABLE IF NOT EXISTS storage_reclaim_marks ( + category TEXT NOT NULL, + key TEXT NOT NULL, + first_seen_at DATETIME NOT NULL, + size INTEGER NOT NULL, + PRIMARY KEY (category, key) + ) + `); + } +} diff --git a/server/src/db/repositories/StorageReclaimMarks.repository.ts b/server/src/db/repositories/StorageReclaimMarks.repository.ts new file mode 100644 index 000000000..1bedc7986 --- /dev/null +++ b/server/src/db/repositories/StorageReclaimMarks.repository.ts @@ -0,0 +1,63 @@ +import type { StorageReclaimMarks } from '../entities/StorageReclaimMarks.entity'; +import type { DB } from '../kysely/db'; +import { TrekRepository } from './_shared/trek-repository'; + +type MarksKyselyDB = Pick; + +/** Since when an object has been unreferenced, as the reclaim pass first saw it. */ +export interface ReclaimMarkRow { + key: string; + first_seen_at: string; + size: number; +} + +/** + * `storage_reclaim_marks`: one row per stored object the reclaim pass found + * no row naming, keyed by category and category-relative key. The pass reads a + * category's marks once, inserts the new ones and drops the ones whose object + * is referenced again or gone; a mark is never moved forward, so its age is how + * long the object has been unreferenced without a break. + */ +export class StorageReclaimMarksRepository extends TrekRepository { + private db() { + return this.kysely(); + } + + async marksOf(category: string): Promise { + return this.db() + .selectFrom('storage_reclaim_marks') + .select(['key', 'first_seen_at', 'size']) + .where('category', '=', category) + .execute(); + } + + /** Marks these keys first seen at `firstSeenAt`; a key marked already keeps its first sighting. */ + async insertMissing( + category: string, + rows: Array<{ key: string; size: number }>, + firstSeenAt: string, + ): Promise { + if (rows.length === 0) return; + await this.db() + .insertInto('storage_reclaim_marks') + .values(rows.map((row) => ({ category, key: row.key, size: row.size, first_seen_at: firstSeenAt }))) + .onConflict((oc) => oc.columns(['category', 'key']).doNothing()) + .execute(); + } + + /** Drops the marks of these keys; answers how many went. */ + async deleteKeys(category: string, keys: string[]): Promise { + if (keys.length === 0) return 0; + const result = await this.db() + .deleteFrom('storage_reclaim_marks') + .where('category', '=', category) + .where('key', 'in', keys) + .executeTakeFirst(); + return Number(result.numDeletedRows); + } + + /** Drops every mark of a category, for a pass that could not judge it: the week starts over. */ + async deleteCategory(category: string): Promise { + return await this.nativeDelete({ category }); + } +} diff --git a/server/tests/unit/db/entities-index.test.ts b/server/tests/unit/db/entities-index.test.ts index f7150c306..c63c27a61 100644 --- a/server/tests/unit/db/entities-index.test.ts +++ b/server/tests/unit/db/entities-index.test.ts @@ -63,8 +63,8 @@ describe('ALL_ENTITIES', () => { * `entity-schema-parity.test.ts` can't see — it iterates the metadata map, * which de-duplicates by class name). */ - it('ENT-003: exactly 165 entities, every class name unique', () => { - expect(ALL_ENTITIES.length).toBe(165); + it('ENT-003: exactly 166 entities, every class name unique', () => { + expect(ALL_ENTITIES.length).toBe(166); const names = ALL_ENTITIES.map((s) => s.meta.className); expect(new Set(names).size).toBe(names.length); }); diff --git a/server/tests/unit/db/entity-class-fields.test.ts b/server/tests/unit/db/entity-class-fields.test.ts index 3cc29f943..c81e6d633 100644 --- a/server/tests/unit/db/entity-class-fields.test.ts +++ b/server/tests/unit/db/entity-class-fields.test.ts @@ -59,7 +59,8 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; // + trip_member_removals: id, trip_id, user_id (a removal note always names its trip and its person). // + assistant_attachments: id, thread_id, user_id, trip_id, name, mime, size, sha256, storage_key, text_status // (an attached file is only ever written whole, by the turn it was attached in). -const KNOWN_UNINITIALISED_NOT_NULL_SCALARS = 604; +// + storage_reclaim_marks: category, key, first_seen_at, size (a mark is only ever written whole, by the reclaim pass). +const KNOWN_UNINITIALISED_NOT_NULL_SCALARS = 608; describe('entity class fields never carry a NaN initialiser (task-4-review-shape.md, Important 1)', () => { it('CLASSFIELD-001: new X() has no own property whose value is NaN, across all 133 entities', () => { diff --git a/server/tests/unit/db/legacy-baseline.test.ts b/server/tests/unit/db/legacy-baseline.test.ts index d21fe58f0..c18b3ce87 100644 --- a/server/tests/unit/db/legacy-baseline.test.ts +++ b/server/tests/unit/db/legacy-baseline.test.ts @@ -100,6 +100,7 @@ const UNNUMBERED = [ 'Migration20200101046300_an_attached_file_stays_with_its_conversation', 'Migration20200101046310_an_admin_can_run_a_plugin_outside_its_trek_range', 'Migration20200101046320_a_place_knows_where_its_details_came_from', + 'Migration20200101046330_an_object_no_row_names_is_marked_before_it_goes', ]; // The last step the positional runner ever had; planLegacyBaseline refuses a // schema_version past it. diff --git a/server/tests/unit/db/repositories/StorageReclaimMarks.repository.test.ts b/server/tests/unit/db/repositories/StorageReclaimMarks.repository.test.ts new file mode 100644 index 000000000..779523100 --- /dev/null +++ b/server/tests/unit/db/repositories/StorageReclaimMarks.repository.test.ts @@ -0,0 +1,79 @@ +/** + * StorageReclaimMarksRepository: since when the reclaim pass has seen an + * object unreferenced. RCLM-001 to RCLM-004. + */ +import { StorageReclaimMarks } from '../../../../src/db/entities/StorageReclaimMarks.entity'; +import type { StorageReclaimMarksRepository } from '../../../../src/db/repositories/StorageReclaimMarks.repository'; +import { createSnapshotTestDb } from '../../../helpers/db-mock'; +import { deleteRows } from '../../../helpers/factories/rows'; +import { createTestOrm, type TestOrm } from '../../../helpers/test-orm'; + +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; + +const testDb = createSnapshotTestDb(); +let t: TestOrm; +let marks: StorageReclaimMarksRepository; + +beforeAll(async () => { + t = await createTestOrm(testDb); + marks = t.repo(StorageReclaimMarks); +}); +beforeEach(async () => { + await deleteRows(t, StorageReclaimMarks); + t.clear(); +}); +afterAll(async () => { + await t.close(); + testDb.close(); +}); + +describe('StorageReclaimMarksRepository', () => { + it('RCLM-001: inserts marks per category and reads them back', async () => { + await marks.insertMissing( + 'covers', + [ + { key: 'a.jpg', size: 10 }, + { key: 'b.jpg', size: 20 }, + ], + '2026-01-01 00:00:00', + ); + await marks.insertMissing('places', [{ key: 'a.jpg', size: 5 }], '2026-01-02 00:00:00'); + const covers = await marks.marksOf('covers'); + expect(covers.sort((x, y) => x.key.localeCompare(y.key))).toEqual([ + { key: 'a.jpg', first_seen_at: '2026-01-01 00:00:00', size: 10 }, + { key: 'b.jpg', first_seen_at: '2026-01-01 00:00:00', size: 20 }, + ]); + expect(await marks.marksOf('places')).toEqual([{ key: 'a.jpg', first_seen_at: '2026-01-02 00:00:00', size: 5 }]); + }); + + it('RCLM-002: a key marked already keeps its first sighting; an empty batch writes nothing', async () => { + await marks.insertMissing('covers', [{ key: 'a.jpg', size: 10 }], '2026-01-01 00:00:00'); + await marks.insertMissing('covers', [{ key: 'a.jpg', size: 99 }], '2026-02-01 00:00:00'); + await marks.insertMissing('covers', [], '2026-03-01 00:00:00'); + expect(await marks.marksOf('covers')).toEqual([{ key: 'a.jpg', first_seen_at: '2026-01-01 00:00:00', size: 10 }]); + }); + + it('RCLM-003: drops the marks of the keys named, in their category only', async () => { + await marks.insertMissing( + 'covers', + [ + { key: 'a.jpg', size: 1 }, + { key: 'b.jpg', size: 1 }, + ], + '2026-01-01 00:00:00', + ); + await marks.insertMissing('places', [{ key: 'a.jpg', size: 1 }], '2026-01-01 00:00:00'); + expect(await marks.deleteKeys('covers', ['a.jpg', 'nope.jpg'])).toBe(1); + expect(await marks.deleteKeys('covers', [])).toBe(0); + expect((await marks.marksOf('covers')).map((m) => m.key)).toEqual(['b.jpg']); + expect((await marks.marksOf('places')).map((m) => m.key)).toEqual(['a.jpg']); + }); + + it('RCLM-004: drops every mark of one category', async () => { + await marks.insertMissing('covers', [{ key: 'a.jpg', size: 1 }], '2026-01-01 00:00:00'); + await marks.insertMissing('places', [{ key: 'a.jpg', size: 1 }], '2026-01-01 00:00:00'); + expect(await marks.deleteCategory('covers')).toBe(1); + expect(await marks.marksOf('covers')).toEqual([]); + expect(await marks.marksOf('places')).toHaveLength(1); + }); +});