feat(storage): keep a mark per stored object no row names

storage_reclaim_marks records since when the reclaim pass has seen an object
unreferenced, keyed by category and key. A mark is never moved forward, so
its age is how long the object has been unreferenced without a break; it goes
once the object is named again or gone. Owned by the storage domain.
This commit is contained in:
Maurice
2026-11-01 13:37:44 +01:00
parent 630f15b5ea
commit 29298dcfe5
11 changed files with 208 additions and 3 deletions
+1
View File
@@ -119,6 +119,7 @@
"SchoolHolidayRegions": "school-holidays",
"Settings": "settings",
"ShareTokens": "share",
"StorageReclaimMarks": "storage",
"Tags": "tags",
"TodoCategoryAssignees": "todo",
"TodoItems": "todo",
@@ -0,0 +1,23 @@
import { EntityRepositoryType, PrimaryKeyProp, defineEntity, p } from '@mikro-orm/core';
import { StorageReclaimMarksRepository } from '../repositories/StorageReclaimMarks.repository';
import { DbTimestampType } from '../types';
export class StorageReclaimMarks {
[EntityRepositoryType]?: StorageReclaimMarksRepository;
[PrimaryKeyProp]?: ['category', 'key'];
category!: string;
key!: string;
first_seen_at!: string;
size!: number;
}
export const StorageReclaimMarksSchema = defineEntity({
class: StorageReclaimMarks,
repository: () => StorageReclaimMarksRepository,
properties: {
category: p.text().primary(),
key: p.text().primary(),
first_seen_at: p.type(DbTimestampType),
size: p.integer(),
},
});
+2
View File
@@ -118,6 +118,7 @@ import { SchoolHolidayPeriodsSchema } from './SchoolHolidayPeriods.entity';
import { SchoolHolidayRegionsSchema } from './SchoolHolidayRegions.entity';
import { SettingsSchema } from './Settings.entity';
import { ShareTokensSchema } from './ShareTokens.entity';
import { StorageReclaimMarksSchema } from './StorageReclaimMarks.entity';
import { TagsSchema } from './Tags.entity';
import { TodoCategoryAssigneesSchema } from './TodoCategoryAssignees.entity';
import { TodoItemsSchema } from './TodoItems.entity';
@@ -285,6 +286,7 @@ export const ALL_ENTITIES: readonly EntitySchema[] = [
SchoolHolidayRegionsSchema,
SettingsSchema,
ShareTokensSchema,
StorageReclaimMarksSchema,
TagsSchema,
TodoCategoryAssigneesSchema,
TodoItemsSchema,
@@ -0,0 +1,9 @@
// Generated by scripts/generate-db-types.ts from the migrated schema. Do not edit by hand:
// change the migration, then run `npm run gen:db-types`.
export interface StorageReclaimMarksTable {
category: string;
key: string;
first_seen_at: string;
size: number;
}
+2
View File
@@ -124,6 +124,7 @@ import type { SchoolHolidayPeriodsTable } from './SchoolHolidayPeriods.table';
import type { SchoolHolidayRegionsTable } from './SchoolHolidayRegions.table';
import type { SettingsTable } from './Settings.table';
import type { ShareTokensTable } from './ShareTokens.table';
import type { StorageReclaimMarksTable } from './StorageReclaimMarks.table';
import type { TagsTable } from './Tags.table';
import type { TodoCategoryAssigneesTable } from './TodoCategoryAssignees.table';
import type { TodoItemsTable } from './TodoItems.table';
@@ -297,6 +298,7 @@ export interface DB {
school_holiday_regions: SchoolHolidayRegionsTable;
settings: SettingsTable;
share_tokens: ShareTokensTable;
storage_reclaim_marks: StorageReclaimMarksTable;
tags: TagsTable;
todo_category_assignees: TodoCategoryAssigneesTable;
todo_items: TodoItemsTable;
@@ -0,0 +1,24 @@
import { Migration } from '@mikro-orm/migrations';
/**
* The storage reclaim pass remembers, per stored object, since when no row has
* named it. An object is deleted only once it has stayed unreferenced on every
* pass for a week (and only when the admin switched deletion on), so a value a
* client writes back late, an offline replay or a copy made in between keeps
* its object. A row goes as soon as its object is referenced again or gone.
*/
export class Migration20200101046330_an_object_no_row_names_is_marked_before_it_goes extends Migration {
override name = 'Migration20200101046330_an_object_no_row_names_is_marked_before_it_goes';
override up(): void {
this.addSql(`
CREATE TABLE IF NOT EXISTS storage_reclaim_marks (
category TEXT NOT NULL,
key TEXT NOT NULL,
first_seen_at DATETIME NOT NULL,
size INTEGER NOT NULL,
PRIMARY KEY (category, key)
)
`);
}
}
@@ -0,0 +1,63 @@
import type { StorageReclaimMarks } from '../entities/StorageReclaimMarks.entity';
import type { DB } from '../kysely/db';
import { TrekRepository } from './_shared/trek-repository';
type MarksKyselyDB = Pick<DB, 'storage_reclaim_marks'>;
/** Since when an object has been unreferenced, as the reclaim pass first saw it. */
export interface ReclaimMarkRow {
key: string;
first_seen_at: string;
size: number;
}
/**
* `storage_reclaim_marks`: one row per stored object the reclaim pass found
* no row naming, keyed by category and category-relative key. The pass reads a
* category's marks once, inserts the new ones and drops the ones whose object
* is referenced again or gone; a mark is never moved forward, so its age is how
* long the object has been unreferenced without a break.
*/
export class StorageReclaimMarksRepository extends TrekRepository<StorageReclaimMarks> {
private db() {
return this.kysely<MarksKyselyDB>();
}
async marksOf(category: string): Promise<ReclaimMarkRow[]> {
return this.db()
.selectFrom('storage_reclaim_marks')
.select(['key', 'first_seen_at', 'size'])
.where('category', '=', category)
.execute();
}
/** Marks these keys first seen at `firstSeenAt`; a key marked already keeps its first sighting. */
async insertMissing(
category: string,
rows: Array<{ key: string; size: number }>,
firstSeenAt: string,
): Promise<void> {
if (rows.length === 0) return;
await this.db()
.insertInto('storage_reclaim_marks')
.values(rows.map((row) => ({ category, key: row.key, size: row.size, first_seen_at: firstSeenAt })))
.onConflict((oc) => oc.columns(['category', 'key']).doNothing())
.execute();
}
/** Drops the marks of these keys; answers how many went. */
async deleteKeys(category: string, keys: string[]): Promise<number> {
if (keys.length === 0) return 0;
const result = await this.db()
.deleteFrom('storage_reclaim_marks')
.where('category', '=', category)
.where('key', 'in', keys)
.executeTakeFirst();
return Number(result.numDeletedRows);
}
/** Drops every mark of a category, for a pass that could not judge it: the week starts over. */
async deleteCategory(category: string): Promise<number> {
return await this.nativeDelete({ category });
}
}
+2 -2
View File
@@ -63,8 +63,8 @@ describe('ALL_ENTITIES', () => {
* `entity-schema-parity.test.ts` can't see — it iterates the metadata map,
* which de-duplicates by class name).
*/
it('ENT-003: exactly 165 entities, every class name unique', () => {
expect(ALL_ENTITIES.length).toBe(165);
it('ENT-003: exactly 166 entities, every class name unique', () => {
expect(ALL_ENTITIES.length).toBe(166);
const names = ALL_ENTITIES.map((s) => s.meta.className);
expect(new Set(names).size).toBe(names.length);
});
@@ -59,7 +59,8 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
// + trip_member_removals: id, trip_id, user_id (a removal note always names its trip and its person).
// + assistant_attachments: id, thread_id, user_id, trip_id, name, mime, size, sha256, storage_key, text_status
// (an attached file is only ever written whole, by the turn it was attached in).
const KNOWN_UNINITIALISED_NOT_NULL_SCALARS = 604;
// + storage_reclaim_marks: category, key, first_seen_at, size (a mark is only ever written whole, by the reclaim pass).
const KNOWN_UNINITIALISED_NOT_NULL_SCALARS = 608;
describe('entity class fields never carry a NaN initialiser (task-4-review-shape.md, Important 1)', () => {
it('CLASSFIELD-001: new X() has no own property whose value is NaN, across all 133 entities', () => {
@@ -100,6 +100,7 @@ const UNNUMBERED = [
'Migration20200101046300_an_attached_file_stays_with_its_conversation',
'Migration20200101046310_an_admin_can_run_a_plugin_outside_its_trek_range',
'Migration20200101046320_a_place_knows_where_its_details_came_from',
'Migration20200101046330_an_object_no_row_names_is_marked_before_it_goes',
];
// The last step the positional runner ever had; planLegacyBaseline refuses a
// schema_version past it.
@@ -0,0 +1,79 @@
/**
* StorageReclaimMarksRepository: since when the reclaim pass has seen an
* object unreferenced. RCLM-001 to RCLM-004.
*/
import { StorageReclaimMarks } from '../../../../src/db/entities/StorageReclaimMarks.entity';
import type { StorageReclaimMarksRepository } from '../../../../src/db/repositories/StorageReclaimMarks.repository';
import { createSnapshotTestDb } from '../../../helpers/db-mock';
import { deleteRows } from '../../../helpers/factories/rows';
import { createTestOrm, type TestOrm } from '../../../helpers/test-orm';
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
const testDb = createSnapshotTestDb();
let t: TestOrm;
let marks: StorageReclaimMarksRepository;
beforeAll(async () => {
t = await createTestOrm(testDb);
marks = t.repo(StorageReclaimMarks);
});
beforeEach(async () => {
await deleteRows(t, StorageReclaimMarks);
t.clear();
});
afterAll(async () => {
await t.close();
testDb.close();
});
describe('StorageReclaimMarksRepository', () => {
it('RCLM-001: inserts marks per category and reads them back', async () => {
await marks.insertMissing(
'covers',
[
{ key: 'a.jpg', size: 10 },
{ key: 'b.jpg', size: 20 },
],
'2026-01-01 00:00:00',
);
await marks.insertMissing('places', [{ key: 'a.jpg', size: 5 }], '2026-01-02 00:00:00');
const covers = await marks.marksOf('covers');
expect(covers.sort((x, y) => x.key.localeCompare(y.key))).toEqual([
{ key: 'a.jpg', first_seen_at: '2026-01-01 00:00:00', size: 10 },
{ key: 'b.jpg', first_seen_at: '2026-01-01 00:00:00', size: 20 },
]);
expect(await marks.marksOf('places')).toEqual([{ key: 'a.jpg', first_seen_at: '2026-01-02 00:00:00', size: 5 }]);
});
it('RCLM-002: a key marked already keeps its first sighting; an empty batch writes nothing', async () => {
await marks.insertMissing('covers', [{ key: 'a.jpg', size: 10 }], '2026-01-01 00:00:00');
await marks.insertMissing('covers', [{ key: 'a.jpg', size: 99 }], '2026-02-01 00:00:00');
await marks.insertMissing('covers', [], '2026-03-01 00:00:00');
expect(await marks.marksOf('covers')).toEqual([{ key: 'a.jpg', first_seen_at: '2026-01-01 00:00:00', size: 10 }]);
});
it('RCLM-003: drops the marks of the keys named, in their category only', async () => {
await marks.insertMissing(
'covers',
[
{ key: 'a.jpg', size: 1 },
{ key: 'b.jpg', size: 1 },
],
'2026-01-01 00:00:00',
);
await marks.insertMissing('places', [{ key: 'a.jpg', size: 1 }], '2026-01-01 00:00:00');
expect(await marks.deleteKeys('covers', ['a.jpg', 'nope.jpg'])).toBe(1);
expect(await marks.deleteKeys('covers', [])).toBe(0);
expect((await marks.marksOf('covers')).map((m) => m.key)).toEqual(['b.jpg']);
expect((await marks.marksOf('places')).map((m) => m.key)).toEqual(['a.jpg']);
});
it('RCLM-004: drops every mark of one category', async () => {
await marks.insertMissing('covers', [{ key: 'a.jpg', size: 1 }], '2026-01-01 00:00:00');
await marks.insertMissing('places', [{ key: 'a.jpg', size: 1 }], '2026-01-01 00:00:00');
expect(await marks.deleteCategory('covers')).toBe(1);
expect(await marks.marksOf('covers')).toEqual([]);
expect(await marks.marksOf('places')).toHaveLength(1);
});
});