mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-21 11:36:05 +00:00
feat(storage): keep a mark per stored object no row names
storage_reclaim_marks records since when the reclaim pass has seen an object unreferenced, keyed by category and key. A mark is never moved forward, so its age is how long the object has been unreferenced without a break; it goes once the object is named again or gone. Owned by the storage domain.
This commit is contained in:
@@ -119,6 +119,7 @@
|
||||
"SchoolHolidayRegions": "school-holidays",
|
||||
"Settings": "settings",
|
||||
"ShareTokens": "share",
|
||||
"StorageReclaimMarks": "storage",
|
||||
"Tags": "tags",
|
||||
"TodoCategoryAssignees": "todo",
|
||||
"TodoItems": "todo",
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
import { EntityRepositoryType, PrimaryKeyProp, defineEntity, p } from '@mikro-orm/core';
|
||||
import { StorageReclaimMarksRepository } from '../repositories/StorageReclaimMarks.repository';
|
||||
import { DbTimestampType } from '../types';
|
||||
|
||||
export class StorageReclaimMarks {
|
||||
[EntityRepositoryType]?: StorageReclaimMarksRepository;
|
||||
[PrimaryKeyProp]?: ['category', 'key'];
|
||||
category!: string;
|
||||
key!: string;
|
||||
first_seen_at!: string;
|
||||
size!: number;
|
||||
}
|
||||
|
||||
export const StorageReclaimMarksSchema = defineEntity({
|
||||
class: StorageReclaimMarks,
|
||||
repository: () => StorageReclaimMarksRepository,
|
||||
properties: {
|
||||
category: p.text().primary(),
|
||||
key: p.text().primary(),
|
||||
first_seen_at: p.type(DbTimestampType),
|
||||
size: p.integer(),
|
||||
},
|
||||
});
|
||||
@@ -118,6 +118,7 @@ import { SchoolHolidayPeriodsSchema } from './SchoolHolidayPeriods.entity';
|
||||
import { SchoolHolidayRegionsSchema } from './SchoolHolidayRegions.entity';
|
||||
import { SettingsSchema } from './Settings.entity';
|
||||
import { ShareTokensSchema } from './ShareTokens.entity';
|
||||
import { StorageReclaimMarksSchema } from './StorageReclaimMarks.entity';
|
||||
import { TagsSchema } from './Tags.entity';
|
||||
import { TodoCategoryAssigneesSchema } from './TodoCategoryAssignees.entity';
|
||||
import { TodoItemsSchema } from './TodoItems.entity';
|
||||
@@ -285,6 +286,7 @@ export const ALL_ENTITIES: readonly EntitySchema[] = [
|
||||
SchoolHolidayRegionsSchema,
|
||||
SettingsSchema,
|
||||
ShareTokensSchema,
|
||||
StorageReclaimMarksSchema,
|
||||
TagsSchema,
|
||||
TodoCategoryAssigneesSchema,
|
||||
TodoItemsSchema,
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
// Generated by scripts/generate-db-types.ts from the migrated schema. Do not edit by hand:
|
||||
// change the migration, then run `npm run gen:db-types`.
|
||||
|
||||
export interface StorageReclaimMarksTable {
|
||||
category: string;
|
||||
key: string;
|
||||
first_seen_at: string;
|
||||
size: number;
|
||||
}
|
||||
@@ -124,6 +124,7 @@ import type { SchoolHolidayPeriodsTable } from './SchoolHolidayPeriods.table';
|
||||
import type { SchoolHolidayRegionsTable } from './SchoolHolidayRegions.table';
|
||||
import type { SettingsTable } from './Settings.table';
|
||||
import type { ShareTokensTable } from './ShareTokens.table';
|
||||
import type { StorageReclaimMarksTable } from './StorageReclaimMarks.table';
|
||||
import type { TagsTable } from './Tags.table';
|
||||
import type { TodoCategoryAssigneesTable } from './TodoCategoryAssignees.table';
|
||||
import type { TodoItemsTable } from './TodoItems.table';
|
||||
@@ -297,6 +298,7 @@ export interface DB {
|
||||
school_holiday_regions: SchoolHolidayRegionsTable;
|
||||
settings: SettingsTable;
|
||||
share_tokens: ShareTokensTable;
|
||||
storage_reclaim_marks: StorageReclaimMarksTable;
|
||||
tags: TagsTable;
|
||||
todo_category_assignees: TodoCategoryAssigneesTable;
|
||||
todo_items: TodoItemsTable;
|
||||
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
import { Migration } from '@mikro-orm/migrations';
|
||||
|
||||
/**
|
||||
* The storage reclaim pass remembers, per stored object, since when no row has
|
||||
* named it. An object is deleted only once it has stayed unreferenced on every
|
||||
* pass for a week (and only when the admin switched deletion on), so a value a
|
||||
* client writes back late, an offline replay or a copy made in between keeps
|
||||
* its object. A row goes as soon as its object is referenced again or gone.
|
||||
*/
|
||||
export class Migration20200101046330_an_object_no_row_names_is_marked_before_it_goes extends Migration {
|
||||
override name = 'Migration20200101046330_an_object_no_row_names_is_marked_before_it_goes';
|
||||
|
||||
override up(): void {
|
||||
this.addSql(`
|
||||
CREATE TABLE IF NOT EXISTS storage_reclaim_marks (
|
||||
category TEXT NOT NULL,
|
||||
key TEXT NOT NULL,
|
||||
first_seen_at DATETIME NOT NULL,
|
||||
size INTEGER NOT NULL,
|
||||
PRIMARY KEY (category, key)
|
||||
)
|
||||
`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
import type { StorageReclaimMarks } from '../entities/StorageReclaimMarks.entity';
|
||||
import type { DB } from '../kysely/db';
|
||||
import { TrekRepository } from './_shared/trek-repository';
|
||||
|
||||
type MarksKyselyDB = Pick<DB, 'storage_reclaim_marks'>;
|
||||
|
||||
/** Since when an object has been unreferenced, as the reclaim pass first saw it. */
|
||||
export interface ReclaimMarkRow {
|
||||
key: string;
|
||||
first_seen_at: string;
|
||||
size: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* `storage_reclaim_marks`: one row per stored object the reclaim pass found
|
||||
* no row naming, keyed by category and category-relative key. The pass reads a
|
||||
* category's marks once, inserts the new ones and drops the ones whose object
|
||||
* is referenced again or gone; a mark is never moved forward, so its age is how
|
||||
* long the object has been unreferenced without a break.
|
||||
*/
|
||||
export class StorageReclaimMarksRepository extends TrekRepository<StorageReclaimMarks> {
|
||||
private db() {
|
||||
return this.kysely<MarksKyselyDB>();
|
||||
}
|
||||
|
||||
async marksOf(category: string): Promise<ReclaimMarkRow[]> {
|
||||
return this.db()
|
||||
.selectFrom('storage_reclaim_marks')
|
||||
.select(['key', 'first_seen_at', 'size'])
|
||||
.where('category', '=', category)
|
||||
.execute();
|
||||
}
|
||||
|
||||
/** Marks these keys first seen at `firstSeenAt`; a key marked already keeps its first sighting. */
|
||||
async insertMissing(
|
||||
category: string,
|
||||
rows: Array<{ key: string; size: number }>,
|
||||
firstSeenAt: string,
|
||||
): Promise<void> {
|
||||
if (rows.length === 0) return;
|
||||
await this.db()
|
||||
.insertInto('storage_reclaim_marks')
|
||||
.values(rows.map((row) => ({ category, key: row.key, size: row.size, first_seen_at: firstSeenAt })))
|
||||
.onConflict((oc) => oc.columns(['category', 'key']).doNothing())
|
||||
.execute();
|
||||
}
|
||||
|
||||
/** Drops the marks of these keys; answers how many went. */
|
||||
async deleteKeys(category: string, keys: string[]): Promise<number> {
|
||||
if (keys.length === 0) return 0;
|
||||
const result = await this.db()
|
||||
.deleteFrom('storage_reclaim_marks')
|
||||
.where('category', '=', category)
|
||||
.where('key', 'in', keys)
|
||||
.executeTakeFirst();
|
||||
return Number(result.numDeletedRows);
|
||||
}
|
||||
|
||||
/** Drops every mark of a category, for a pass that could not judge it: the week starts over. */
|
||||
async deleteCategory(category: string): Promise<number> {
|
||||
return await this.nativeDelete({ category });
|
||||
}
|
||||
}
|
||||
@@ -63,8 +63,8 @@ describe('ALL_ENTITIES', () => {
|
||||
* `entity-schema-parity.test.ts` can't see — it iterates the metadata map,
|
||||
* which de-duplicates by class name).
|
||||
*/
|
||||
it('ENT-003: exactly 165 entities, every class name unique', () => {
|
||||
expect(ALL_ENTITIES.length).toBe(165);
|
||||
it('ENT-003: exactly 166 entities, every class name unique', () => {
|
||||
expect(ALL_ENTITIES.length).toBe(166);
|
||||
const names = ALL_ENTITIES.map((s) => s.meta.className);
|
||||
expect(new Set(names).size).toBe(names.length);
|
||||
});
|
||||
|
||||
@@ -59,7 +59,8 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
|
||||
// + trip_member_removals: id, trip_id, user_id (a removal note always names its trip and its person).
|
||||
// + assistant_attachments: id, thread_id, user_id, trip_id, name, mime, size, sha256, storage_key, text_status
|
||||
// (an attached file is only ever written whole, by the turn it was attached in).
|
||||
const KNOWN_UNINITIALISED_NOT_NULL_SCALARS = 604;
|
||||
// + storage_reclaim_marks: category, key, first_seen_at, size (a mark is only ever written whole, by the reclaim pass).
|
||||
const KNOWN_UNINITIALISED_NOT_NULL_SCALARS = 608;
|
||||
|
||||
describe('entity class fields never carry a NaN initialiser (task-4-review-shape.md, Important 1)', () => {
|
||||
it('CLASSFIELD-001: new X() has no own property whose value is NaN, across all 133 entities', () => {
|
||||
|
||||
@@ -100,6 +100,7 @@ const UNNUMBERED = [
|
||||
'Migration20200101046300_an_attached_file_stays_with_its_conversation',
|
||||
'Migration20200101046310_an_admin_can_run_a_plugin_outside_its_trek_range',
|
||||
'Migration20200101046320_a_place_knows_where_its_details_came_from',
|
||||
'Migration20200101046330_an_object_no_row_names_is_marked_before_it_goes',
|
||||
];
|
||||
// The last step the positional runner ever had; planLegacyBaseline refuses a
|
||||
// schema_version past it.
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
/**
|
||||
* StorageReclaimMarksRepository: since when the reclaim pass has seen an
|
||||
* object unreferenced. RCLM-001 to RCLM-004.
|
||||
*/
|
||||
import { StorageReclaimMarks } from '../../../../src/db/entities/StorageReclaimMarks.entity';
|
||||
import type { StorageReclaimMarksRepository } from '../../../../src/db/repositories/StorageReclaimMarks.repository';
|
||||
import { createSnapshotTestDb } from '../../../helpers/db-mock';
|
||||
import { deleteRows } from '../../../helpers/factories/rows';
|
||||
import { createTestOrm, type TestOrm } from '../../../helpers/test-orm';
|
||||
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
|
||||
|
||||
const testDb = createSnapshotTestDb();
|
||||
let t: TestOrm;
|
||||
let marks: StorageReclaimMarksRepository;
|
||||
|
||||
beforeAll(async () => {
|
||||
t = await createTestOrm(testDb);
|
||||
marks = t.repo(StorageReclaimMarks);
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await deleteRows(t, StorageReclaimMarks);
|
||||
t.clear();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await t.close();
|
||||
testDb.close();
|
||||
});
|
||||
|
||||
describe('StorageReclaimMarksRepository', () => {
|
||||
it('RCLM-001: inserts marks per category and reads them back', async () => {
|
||||
await marks.insertMissing(
|
||||
'covers',
|
||||
[
|
||||
{ key: 'a.jpg', size: 10 },
|
||||
{ key: 'b.jpg', size: 20 },
|
||||
],
|
||||
'2026-01-01 00:00:00',
|
||||
);
|
||||
await marks.insertMissing('places', [{ key: 'a.jpg', size: 5 }], '2026-01-02 00:00:00');
|
||||
const covers = await marks.marksOf('covers');
|
||||
expect(covers.sort((x, y) => x.key.localeCompare(y.key))).toEqual([
|
||||
{ key: 'a.jpg', first_seen_at: '2026-01-01 00:00:00', size: 10 },
|
||||
{ key: 'b.jpg', first_seen_at: '2026-01-01 00:00:00', size: 20 },
|
||||
]);
|
||||
expect(await marks.marksOf('places')).toEqual([{ key: 'a.jpg', first_seen_at: '2026-01-02 00:00:00', size: 5 }]);
|
||||
});
|
||||
|
||||
it('RCLM-002: a key marked already keeps its first sighting; an empty batch writes nothing', async () => {
|
||||
await marks.insertMissing('covers', [{ key: 'a.jpg', size: 10 }], '2026-01-01 00:00:00');
|
||||
await marks.insertMissing('covers', [{ key: 'a.jpg', size: 99 }], '2026-02-01 00:00:00');
|
||||
await marks.insertMissing('covers', [], '2026-03-01 00:00:00');
|
||||
expect(await marks.marksOf('covers')).toEqual([{ key: 'a.jpg', first_seen_at: '2026-01-01 00:00:00', size: 10 }]);
|
||||
});
|
||||
|
||||
it('RCLM-003: drops the marks of the keys named, in their category only', async () => {
|
||||
await marks.insertMissing(
|
||||
'covers',
|
||||
[
|
||||
{ key: 'a.jpg', size: 1 },
|
||||
{ key: 'b.jpg', size: 1 },
|
||||
],
|
||||
'2026-01-01 00:00:00',
|
||||
);
|
||||
await marks.insertMissing('places', [{ key: 'a.jpg', size: 1 }], '2026-01-01 00:00:00');
|
||||
expect(await marks.deleteKeys('covers', ['a.jpg', 'nope.jpg'])).toBe(1);
|
||||
expect(await marks.deleteKeys('covers', [])).toBe(0);
|
||||
expect((await marks.marksOf('covers')).map((m) => m.key)).toEqual(['b.jpg']);
|
||||
expect((await marks.marksOf('places')).map((m) => m.key)).toEqual(['a.jpg']);
|
||||
});
|
||||
|
||||
it('RCLM-004: drops every mark of one category', async () => {
|
||||
await marks.insertMissing('covers', [{ key: 'a.jpg', size: 1 }], '2026-01-01 00:00:00');
|
||||
await marks.insertMissing('places', [{ key: 'a.jpg', size: 1 }], '2026-01-01 00:00:00');
|
||||
expect(await marks.deleteCategory('covers')).toBe(1);
|
||||
expect(await marks.marksOf('covers')).toEqual([]);
|
||||
expect(await marks.marksOf('places')).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user