Feat/storage driver (#1949)

* feat(storage): storage key validation, driver contract suite, LocalDriver

* feat(storage): MirrorDriver composite backend

* feat(server): add TREK_UPLOADS_DIR env var for the uploads root

* feat(storage): settings-backed registry, StorageService facade, module wiring

* test(storage): pin src/nest/storage coverage thresholds

* feat(storage): multer upload options factory

* refactor(files): spool trip-file uploads through storage

* refactor(collab): spool note attachments through storage; normalize files/ prefix

* refactor(journey): spool journey media uploads through storage

* refactor(trips): spool trip cover upload through storage

* refactor(places): spool place-image and collection uploads through storage

* refactor(auth): spool avatar upload through storage

* fix(server): consolidate allowed_file_types onto AllowedFileTypesService

* fix(storage): default defParamCharset to utf8 in the upload factory

* test(db): re-point the trailing-migration guard at the trip_files prefix strip

* fix(storage): age-gate the boot spool sweep

* test(files): key FILE-P02 residue checks on upload size

* test(storage): raise src/nest/storage coverage pin

* refactor(platform): serve /uploads routes through storage

* refactor(files): serve trip-file downloads through storage

* refactor(memories): serve photo bytes through storage

* refactor(maps): stream place-photo bytes through storage

* refactor(share): stream shared place-photo bytes through storage

* refactor(journey): serve public journey assets through storage

* fix(share): answer place-photo cache miss with 204 like the maps proxy

* test(server): close serving-swap gaps surfaced by the full tier

* test(storage): raise src/nest/storage branches pin

* refactor(place-photos): route place-photo cache through storage

* refactor(places): reclaim place images and collection covers through storage

* refactor(trips): trip covers through storage

* refactor(auth): avatar reclaim through storage

* refactor(collab): note-file deletion through storage

* refactor(files): trash deletion through storage

* refactor(memories): trek photo cache internals through storage

* refactor(memories): thumbnail service category-addressed

* refactor(memories): immich upload reads through storage; drop uploads-root

* refactor(journey): photo deletion through storage

* fix(memories): reclaim orphaned trek photo cache objects in the ttl sweep

* feat(journey): journey-thumbs orphan sweep and delete-site derived-artifact reclaim

* test(server): close slice-4 gaps surfaced by the full tier

* test(coverage): raise slice-4 domain pins

* refactor(backup): list, existence and delete through storage

* refactor(backup): download through storage sendToResponse

* refactor(backup): create backup through per-category storage enumeration

* refactor(backup): restore and upload-restore through storage

* refactor(backup): auto-backup retention through storage

* refactor(server): boot dir-creation to driver init

* test(server): close slice-5 gaps surfaced by the coverage tier

* test(coverage): raise slice-5 backup pin

* test(server): resolve async backupFileExists mocks for the tests typecheck

* test(server): pin ranged-read stat and zero-byte roundtrip in the storage driver contract

* feat(server): TREK_S3_* env wiring — schema, s3 namespace, all-or-nothing boot rule

* feat(server): S3 driver core — aws-lite client seam, stat/delete, driver-owned deadlines

* refactor(server): type the aws-lite S3 client via s3-types reference

* feat(server): S3 driver getStream and list — ranged full-object stat, iterator pagination

* feat(server): S3 driver put — bounded-peek routing, Upload inactivity deadline

* fix(server): S3 driver Upload monitoring — Transform interposition, stream cleanup

* feat(server): env-declared s3-main storage backend; settings-declared s3 rejected

* test(server): env-gated S3 driver contract suite against MinIO

* fix(server): S3 driver zero-byte put — work around aws4 duplicate Content-Length signing defect

* fix(server): scope the zero-byte put workaround to buffered bodies; bind test MinIO to localhost

* ci(server): MinIO-backed S3 driver contract job

* ci(server): align s3-contract shim comment and MinIO bind with references

* docs: S3 storage backend configuration and off-box backup mirroring

* docs: restore README formatting, keep S3 additions only

* ci(server): make live-MinIO contract runs manual-only (AIStor licensing)

MinIO's community line is discontinued entirely; the maintained successor, AIStor, requires a license key that can't safely live in a shared CI runner. The s3-contract CI job is removed; live-provider contract runs against MinIO/AIStor are now a manual, license-gated procedure documented in docker-compose.minio-test.yml's header.

* chore(server): final-review polish — stale CI refs, dockerignore key, boundary pins, zero-byte tempfile contract case

* expose admin ui

* feat(shared): storage backend type registry, wire schemas, category relocation

* feat(server): accept settings-declared s3 backends — shared-schema options, decrypt-at-build, registry preview()

* feat(server): remove TREK_S3_*/TREK_UPLOADS_DIR — tombstone boot rule, built-in override as the relocation path

* feat(server): deriveSecurity.encryptionKeySet and storage secret plumbing (gate, encrypt, unmask, mask, redact)

* chore(server): drop storage-secrets import that only Task 8's probe helper will use

* feat(server): seed-once storage-config.json boot import — fail closed on any invalid seed

* feat(server): registry snapshot() — effective storage config with provenance

* feat(server): api/admin/storage GET/PUT — masked secrets, preview-gated saves, wired reload, audits

* feat(server): api/admin/storage/test — per-target ephemeral probes, mirror targets probed individually

* test(server): e2e coverage for api/admin/storage — guards, managed refusal, envelope, probes

* test(server): give e2e harness DBs the app_settings table the storage registry reads at boot

* test(coverage): raise pins after the storage admin-config slice

* fix(server): final-review polish — encryption-gate wording, all-field sentinel guard, README/e2e notes

* test(server): storage slice-1 review follow-ups — mirror s3-replica probe decrypt, all-encrypted seed without key, test-route 401

* feat(i18n): storage admin domain + tab label in every locale

* feat(client): adminApi storage endpoints — getStorage, updateStorage, testStorageBackend

* feat(client): storage admin draft-document helpers + relative-time util

* feat(client): schema-driven storage BackendForm — fields by kind, mask echo, encryption banner

* feat(client): admin storage panel — backends list, category map, health strip, one-PUT save

* feat(client): storage tab on the admin page

* feat(client): storage admin panel and section on the phone shell

* fix(client): block renaming a storage backend onto an existing name

* feat(server): allow mirrors on any storage category — lift the backups-only rule

* feat(i18n): replicas-on-primary mirror strings in every locale

* feat(client): mirror fold/synthesize model helpers for replicas-on-primary

* feat(client): BackendForm mirror-targets composer; raw mirror type hidden

* feat(client): desktop storage panel folds mirrors into their primaries

* feat(client): phone storage panel folds mirrors into their primaries

* fix(client): strip removed backends from remaining mirror replicas — no phantom refs

* feat(storage): retire photos as a configurable category — served-only via the ServedCategory seam

* feat(i18n): descriptive storage category names and descriptions in every locale

* feat(client): descriptive category names, descriptions and id badges in the storage panel

* docs(storage): fix stale nine-category comments after the eight-category shrink

* docs: storage configuration via the admin UI and seed file — README + compose

* docs(wiki): Admin Storage page and storage cross-links

* docs(server): storage domain blueprint entry; TREK_PLACE_PHOTO_DIR tombstone clarifier

* refactor(server): drop the storage env-var tombstones — the variables never shipped

* docs: storage config is simply new in v4 — drop the removed-variables story

* feat(shared): backfill-status and usage wire schemas on the storage admin state

* feat(server): MirrorDriver.backfill — prefix-scoped, size-compared, cancellable replica catch-up

* feat(server): one-at-a-time storage backfill jobs + the storage events bridge

* feat(server): storage usage scan — nightly cron, on-demand service, persisted with computedAt

* feat(server): replica_failure admin notifications — events bridge, hourly debounce, channel registration

* feat(i18n): replica-failure notification text in every locale

* feat(server): storage backfill and stats-refresh admin endpoints

* feat(client): adminApi backfill and stats-refresh endpoints

* feat(i18n): storage sync and usage strings in every locale

* feat(client): desktop storage panel — sync now, usage display, backfill polling

* feat(client): phone storage panel — sync now, usage display, backfill polling

* fix(client): per-category usage line on the phone storage panel

* docs: sync now, usage stats and replica-failure notifications

* test(server): admin scope now includes replica_failure, not just version_available

Commit 6db135da added replica_failure to ADMIN_SCOPED_EVENTS but left
NPREF-005b asserting the admin scope contained only version_available.
Update the expectation to match.

* test(server): cover storage backfill/stats/jobs branches to hold the coverage pin

* fix(client): guard storage admin polling against stale responses racing a save

A poll GET issued before a save's PUT resolves could land after the save
applied its fresher response, overwriting it with the pre-save world and
re-deriving the draft from that stale snapshot. useStorageAdmin now tracks
a state-application sequence number and drops any refreshState() response
whose sequence has moved on, or that resolves while a save is in flight.
Also guard the fire-and-forget refreshState() calls in startBackfill/
cancelBackfill against unhandled rejections, matching the poll's style.

* fix(server): report 404 before 409 when a backfill start names an unknown mirror

startBackfill checked the global busy flag before validating that :name is
a mirror routed by some category, so POST .../backends/garbage/backfill
answered 409 instead of 404 whenever a sync was already running. Resolve
and validate the target first (BackfillTargetError -> 404), then check
busy (BackfillBusyError -> 409).

* feat(client): two-column category grid on the storage panel

* feat(server): plugin files.getContent reads through the storage layer

* feat(server): plugin files.create writes through the storage layer

* refactor(server): drop FilesService.resolveFilePath, last direct-fs seam in the plugin surface

* fix(server): map StorageInvalidKeyError to the plugin file accessibility envelope

* feat(storage): allow credentialed backends without an explicit ENCRYPTION_KEY

Secrets are still always encrypted at rest; without an explicit key the
implicit data-directory key is used. Removes the plaintext-secret refusal
from the admin PUT and the seed import, the encryptionReady GET field, the
save-blocking banner in both admin shells, and the now-consumerless
security.encryptionKeySet env derivation. Docs now recommend (not require)
an explicit key, since the implicit key rides inside backup archives.

* test(client): pin the stale-poll release to the pre-save GET in FE-ADMIN-STOR-024

The stub deferred EVERY poll GET and rebound the release handle to the
newest one, so a tick landing after the save was released with the
pre-save payload — a response no real backend can produce, which the seq
guard rightly applied. Only the first (genuinely stale) GET is held now;
later GETs answer with the world the mock server currently holds. Also
drops the encryptionReady fixture field removed from the schema in the
previous commit.

* fix(storage): backfill puts carry content-type, stat failures report op stat, progress clamps

* feat(storage): category migration job — copy, flip, delta sweep, reclaimable tally

* feat(storage): migration start/cancel endpoints and client API

* test(client): storage admin fixtures carry the new migrations field

* fix(storage): migration POST answers 200 and validates its body via the shared schema

* fix(storage): cancel running storage jobs when a config save removes their backend

* fix(notifications): in-app replica-failure row carries the suppressed count

* feat(client): migration prompt, queue, and progress on the storage panel

* fix(client): migration queue holds a synchronous single-flight lock

* feat(client): migration prompt and progress on the mobile storage panel

* fix(server): register category-migration routes in the managed-route allow list

StorageAdminController.migrationStart/migrationCancel were added under the
controller-level @ManagedForbidden() decorator but never added to
MANAGED_ROUTE_ALLOW_LIST, so validateManagedRoutes() threw "Managed surface
changed" on every buildApp() call, failing 46 test files (any test booting
the real app).

* test(server): cover category-migration copy/sweep branches to hold the storage coverage pin

Adds MIG-005 (a target object already matching size/key is skipped in the
copy phase itself, not just the delta sweep) and MIG-006 (a copy failure
during the delta sweep is counted without undoing the flip that already
happened). src/nest/storage/**'s branch coverage was 83.91%, just under the
84% pin.

* docs(storage): category migration — move existing objects on reassign

* fix(storage): close the pre-flip cancel window and cancel re-routed migrations

A cancel landing between the last copy iteration and the category flip was
silently ignored, and a config save that re-routed the migrating category to
a third, still-defined backend never stopped the in-flight migration. Also
guards assignCategory against writing an unknown backend name, and fixes a
missing `suppressed` notification param being treated as truthy suppression.

* fix(client): migrations keep mirrors, queue waits on backfills, sweep failures surface

Reassigning a category onto a mirrored primary now posts the migration to the
mirror's wire name instead of the bare primary, so replication survives the
move. The queued-migration effect also waits out a running backfill (the
server's one-job rule spans both), and a done migration with sweep failures
now surfaces a doneFailures line in both shells, translated into every locale.

* ci(server): run the S3 contract suite against MinIO

Resurrects the s3-contract job removed in 6fbff835, adapted for AIStor:
writes the MINIO_LICENSE_KEY secret to a license file, boots the licensed
container, and runs the contract suite hermetically (no --coverage) so the
storage coverage pin keeps holding from unit tests alone. Gated to same-repo
PRs since fork PRs have no access to the license secret.

* fix(server): backup archives uploads from a remote primary

createBackup() enumerated uploads through withLocalFile, which downloads a
remote object to a temp file, hands it to the callback, then deletes it
before archiver ever reads it — archiver reads entries lazily during
finalize(), so a remote (S3) primary silently backed up zero upload bytes.

Add StorageService.getLocalPathOrNull(category, name): a narrow locality
probe that returns a real path only when one exists on disk right now (the
fs-existence check is the fail-safe half of the contract — a driver-reported
path that vanished between list() and here falls back too, not just a
path-less remote driver). The enumeration loop uses it directly when
available (zero-copy, unchanged for a local primary) and otherwise streams
the object into a per-backup staging dir under the backups spool, cleaned up
in the existing finally alongside zipSpool/dbSnap.

Also wire archive.on('warning', reject): archiver treats a missing/unreadable
entry as a warning, not an error, and skips it by default — which would
silently drop the entry from an otherwise "successful" backup.

* fix(server): client aborts stop 500ing and leaking streams

storage.service.ts sendToResponse: the local res.sendFile branch now treats
ECONNABORTED/ECONNRESET/ERR_STREAM_PREMATURE_CLOSE/syscall=write as success,
mirroring res.sendFile's own default callback (the contract
storageStaticHandler already implements caller-side). The remote branch
replaces the hand-rolled pipe+Promise with pipeline(stream, res), which
destroys the source stream on any failure — fixing the leaked S3 keep-alive
socket on a client abort — while swallowing the same client-abort error set;
a real source error still rejects so callers keep their miss contracts.

This centralizes the fix for every caller that streams through
sendToResponse: files-download, journey-public, photo-resolver,
trek-photo-cache, backup downloads, and /uploads/photos.

trek-exception.filter.ts now no-ops (destroying the socket) when headers are
already sent, so no filter path ever writes onto a response in flight.

maps.controller.ts and share.controller.ts's place-photo bytes proxies
switch their direct getStream+pipe to pipeline() with the same abort-tolerant
catch, keeping the existing 204-empty fallback for pre-header failures.

* fix(storage): mirror read fallback consults replicas on primary miss

MirrorDriver.getStream/stat now fall back to replicas on a primary miss
(StorageNotFoundError / null stat), not just an infra error — binding
decision: a read succeeds if ANY member holds the object. Only a
deterministic StorageInvalidKeyError skips fallback.

stat() gets a dedicated fallback loop instead of reusing fromReplicas:
a replica returning null means "keep looking," which the generic
catch-and-continue helper would have short-circuited on.

StorageService.sendToResponse/withLocalFile stop treating composite
driver.stat() as proof the object is at localPath; both now fs-check
the path itself and fall through to the stream branch when it's
missing, which is what actually reaches a MirrorDriver's replicas.

* feat(storage): remote serving parity — content-type, validators, range

Local backends get content type, ETag/Last-Modified, conditional GET, Range
and HEAD for free from res.sendFile; the proxying branch every remote
backend (and a local primary whose file vanished) falls through to had none
of it — every request re-streamed the whole object with no cache validator
and no way to seek.

sendStreamed() reproduces the contract from ONE pre-flight driver.stat():
a miss throws before any header is set (the caller's 404/204/next()
contract), and that same stat then answers 304s, 416s, HEADs and suffix
ranges without ever opening a body. The decisions live in a new pure module,
http-serving.ts:

- validatorsFor: the driver's ETag, else express/send's weak
  W/"<size hex>-<mtime hex>". mtimeMs 0 (S3 can report it) means no
  Last-Modified and no derived tag — serving uncached beats a 1970 header
  that reads as fresh forever.
- isNotModified: If-None-Match (weak compare, *, list) wins over
  If-Modified-Since, per RFC 9110 §13.2.2.
- decideRange: single range only, inclusive clamp, suffix computed from the
  pre-stat size, unsatisfiable → 416, If-Range strong-exact or full entity.
  A multi-range or malformed spec is answered with the full object rather
  than multipart/byteranges. If-Match/If-Unmodified-Since stay out of scope.

Range math uses the pre-stat size as the single source of truth, the ranged
getStream stat is never re-consulted, and the pipeline abort semantics are
unchanged.

S3Driver now surfaces the bucket's ETag on stat()/getStream() (on a ranged
GET it is still the full object's tag). aws-lite strips the tag's
surrounding quotes — verified live against AIStor RELEASE.2026-08-07 — and
an unquoted entity-tag is not a valid HTTP ETag, so the driver restores
exactly the quotes the SDK removed and nothing else.

* fix(server): key rotation covers storage.backends

The S3 storage backend's secretAccessKey is encrypted at rest inside the
app_settings row keyed storage.backends, but scripts/migrate-encryption.ts
didn't know about it — after a key rotation that secret stayed encrypted
under the old key and silently failed to decrypt on the next read.

The script now re-encrypts secretAccessKey for every s3 backend inside the
storage.backends JSON array, using a pinned copy of the secret-field list
(it deliberately can't import @trek/shared). The parity test pins that copy
against storageSecretFields('s3') so a new secret field can't be missed
silently.

* fix(server): restore reloads the storage registry

restoreFromZip closed and reopened the DB but never reloaded the
storage registry, so uploads rehydration ran against the STALE
pre-restore driver/category map instead of the config the restored
archive's own travel.db carries (audit #4, major).

StorageService gains reloadConfig(), a narrow passthrough to the
(unexported) registry's reload() — same shape as health(). backup.impl's
restoreFromZip now calls it right after reinitialize() and right before
the uploads wipe/rehydrateUploads/plugin staging, so rehydrated bytes
land where the restored config says. When reopen fails (reinitFailed),
reload and rehydration are skipped entirely — already reported as
"restart required", and there is no live DB handle for the registry to
read anyway.

* fix(server): category migration rewrites keys across prefix changes

Category migration copied objects to the destination backend using the
SOURCE key verbatim. A mode-A ('' prefix) source migrating onto a
prefixed backend (or vice versa) landed objects at keys the registry
could never resolve post-flip (audit #8).

Extracts StorageRegistryService.keyPrefixFor(category, backendName) from
build()'s inline conditional so startMigration can compute the
destination prefix before the flip. runMigration now maps every
destination key (destPrefix + key.slice(sourcePrefix.length)) in the
copy skip-check, the put, and the delta sweep; the reclaimable tally
stays source-keyed. Equal-prefix migrations are unaffected (identity
mapping).

* fix(plugins): await the now-async place/collection services in RPCs

PlacesService.update/remove and CollectionsService.deletePlace became
async when they started deleting storage objects, but their plugin RPC
handlers still called them synchronously: the null/false refusal check
compared against an always-truthy Promise, and places.update broadcast
a Promise instead of the resolved place. Both handlers now await the
service call; mapCollectionError is async-aware (accepts a sync-or-
async thunk) so the sync callers (create/update/savePlace/copyToTrip)
keep working through the same path. Verified the RPC host already
awaits handler results, so no change was needed there.

* fix(storage): backfill Sync now gains a deletion sweep

The Sync now backfill copied primary objects to replicas but never removed
replica objects the primary no longer has, leaving mirror read-fallback able
to serve stale data indefinitely after a failed replica delete. backfill()
now sweeps each replica post-copy: any replica key absent from the primary
key-set is re-checked against primary.stat (a race gate against the primary
regaining the key) before being deleted. Adds a deleted counter end-to-end
(driver, jobs service, shared schema, both admin shells, all 23 locales) and
a new ReplicaFailure 'list' op for a replica whose listing itself fails
during the sweep.

* feat(storage): optimistic concurrency for admin config saves

A stale admin PUT of storage settings could silently undo a category
migration's just-flipped assignment. Adds a version counter
(app_settings row storage.config_version) bumped inside the same
transaction by both writers — applyConfig and the migration flip
(assignCategory). applyConfig compares the submitted version first
and throws StorageConflictError, mapped to 409 ahead of the blanket
400. The client carries the draft's loaded version through every
local edit and maps a 409 to a distinct "settings changed underneath
you" message, refreshing state while preserving the dirty draft for
review.

* fix(storage): surface a failed config load + redact secretAccessKey

Registry load() now records the exact parse/validate failure as
lastLoadError() instead of silently falling back to last-good config or
built-in defaults; StorageAdminService exposes it as configError on the
admin state (storageAdminStateSchema extended additively), and
AdminStoragePanel renders it as a warning banner naming the error. Save
stays enabled — it's the recovery path — but an admin can no longer save
over a broken stored config without knowing.

Also adds 'secretaccesskey' to globalMiddleware's SENSITIVE_KEYS request-log
redaction list, so storage admin PUT bodies never leak the S3 secret into
debug logs.

* fix(client): admin storage panel state races

Migration prompt now recomputes computeMigrationCandidates(draft, state)
on render and again at confirm time, so a stale open-time snapshot never
drives stripCategories or the migration queue; adds a Cancel button to
the dialog. A failed startMigration POST explicitly clears the remaining
queue with a toast naming the dropped categories and refreshes state,
instead of leaving it invisibly stranded; a "Queued: X, Y" line renders
while the queue is non-empty. Testing a mirrored row now expands the
draft primary + replicas client-side and probes each individually,
instead of sending the mirror stub to the server (which resolved
targets from its own live snapshot, silently testing saved options over
unsaved draft edits). A completed/cancelled/errored backfill now renders
its terminal status alongside the Sync now button instead of hiding it,
so it can be re-run without a page reload.

* fix(storage): refuse shared replica backends so the sync sweep cannot cross categories

A mirror's "Sync now" sweep makes each replica MATCH the primary: it lists the
replica under every prefix the mirror's categories occupy and deletes the keys
the primary doesn't hold. The backups category's prefix is '' — the replica's
entire root — so making a category-serving backend (say uploads-local) a
replica of a backups mirror meant the first sync deleted every user upload.
Nothing forbade the config: validateConfig only checked that mirror targets
existed and weren't nested, and the admin picker offered every backend.

The sweep is correct, so the fix is at the config level:

- validateConfig refuses a backend that is both a mirror replica and a
  category target (directly, or as the primary of a mirror a category routes
  to), and refuses a backend replicating two mirrors whose swept prefixes
  overlap — '' overlaps everything. Disjoint prefixes stay legal.
- assignCategory (the migration flip, the other writer of the settings rows)
  refuses routing a category onto a backend that is currently a replica.
- The admin mirror-target picker (both shells) only offers backends that serve
  no category; already-selected targets stay listed so they can be unchecked.
  A category assigned to a replica still surfaces the server's 400 verbatim.

Existing shared-replica fixtures were rewritten onto dedicated replicas.

* fix(client): a 409 conflict save is recoverable without a page reload

After a 409 the admin storage panel was a dead end. refreshState leaves a
dirty draft's version alone on purpose (audit #7: the background poll must
never float a mid-edit draft past a migration flip it never carried), setDraft
re-attaches that same version to every later edit, and save() resubmits it —
so every retry 409s again and only a full page reload got the operator out,
while the banner told them to "review and save again".

The version block stays. Instead the conflict banner now offers the one action
that can clear it: discard the local edits and reload the saved settings —
a fresh GET applied through applyState, re-deriving the draft (and its
version) and clearing dirty/saveError. Bumping the version in refreshState,
or only in the 409 branch, was rejected: either resurrects audit #7, since a
full-document save would then revert a flip the draft never carried.

storage.saveConflict now points at that action instead of implying a plain
re-save works (all 23 locales). The escape lives in the shared hook, so the
phone panel renders it too.

* fix(client): deflake the queued-migration-waits-out-backfill panel test

FE-ADMIN-STOR-037 seeds a running backfill, so the panel polls GET from
first render — but the stub served the post-save world (files: off-box)
before the PUT had landed, which no real server would do. A poll
resolving while the confirm dialog was open made the confirm-time
candidate recompute see from === to, so nothing was queued and the
migration never POSTed.

The GET stub now stays faithful (pre-save world until the PUT lands,
usage included so candidates keep their object counts), and the test
deterministically forces the once-flaky interleaving: it waits for
polls to land while the dialog is open and asserts the candidate
survives them.

* fix(storage): surface the underlying cause in probe target errors

The admin storage Test probe reported only the driver wrapper's outer
message ("put failed for '…' on 'probe-s3'"), dropping the cause that
says WHY — connection refused vs missing bucket. Flatten the error's
cause chain into the probe result, appending an S3-style code when it
isn't already in the message.

* fix(storage): surface the underlying cause in replica-failure reports

Same defect as the probe fix, one layer over: MirrorDriver's
reportReplicaFailure captured only the wrapper's message, so
replica-failure notifications said "put failed" without the why.
Move the probe's cause-chain flattener into storage.types.ts as
describeError and use it in both call sites.

* fix(storage): strip ContentType from multipart Upload calls (MalformedXML)

Every multipart S3 put carrying a contentType — every backup zip and
photo over the 10MB threshold — failed with 400 MalformedXML. Upstream
@aws-lite/s3's Upload leaks the ContentType param into its final
CompleteMultipartUpload headers ({ ...xml, ...paramHeaders }), clobbering
application/xml, which flips @aws-lite/client's payload serializer to
JSON — so the server receives a JSON parts list where XML belongs.
Confirmed live against MinIO AIStor; any S3 refuses the same.

uploadMonitored now drops ContentType before s3.Upload. Multipart
objects are stored without a content-type, which TREK never reads back
(serving derives Content-Type from the key/caller, never object
metadata); single-request PutObject paths keep sending it.

Also closes the coverage gap that let this ship: the live contract
suite never crossed MULTIPART_THRESHOLD, so s3.Upload had never run
against a real server — it now round-trips a >10MB put carrying a
contentType.

---------

Co-authored-by: Maurice <mauriceboe@icloud.com>
This commit is contained in:
jubnl
2026-08-22 11:33:55 +02:00
committed by GitHub
parent 942e1932c9
commit 305b113d5f
369 changed files with 26211 additions and 2718 deletions
+2
View File
@@ -15,6 +15,8 @@ uploads
chart/
docs/
docker-compose.yml
docker-compose.minio-test.yml
minio-licence.key
unraid-template.xml
*.sqlite
*.sqlite-shm
+69
View File
@@ -175,6 +175,75 @@ jobs:
path: server/coverage/
retention-days: 7
s3-contract:
name: S3 Driver Contract (MinIO)
runs-on: ubuntu-latest
# Fork PRs don't get repo secrets, so MINIO_LICENSE_KEY would be empty
# and the AIStor container would fail to boot.
if: github.event.pull_request.head.repo.full_name == github.repository
steps:
- uses: actions/checkout@v6
- uses: actions/setup-node@v6
with:
node-version: 24
cache: npm
cache-dependency-path: package-lock.json
- name: Install dependencies
run: npm ci
- name: Ensure @swc/core's Linux binary for unplugin-swc
# The lockfile was generated on Windows and omits @swc/core's Linux
# optional native binary, so npm ci/install skips it on the runner.
# Install the matching version explicitly so the server's SWC transform
# (server/vitest.config.ts) can load.
run: |
SWC_VERSION=$(node -p "require('@swc/core/package.json').version")
npm install --no-save --legacy-peer-deps "@swc/core-linux-x64-gnu@$SWC_VERSION"
- name: Build shared
run: npm run build --workspace=shared
- name: Write MinIO license
run: printf '%s' "${{ secrets.MINIO_LICENSE_KEY }}" > minio-licence.key
- name: Start MinIO
# `services:` can't pass the `server /data --license ...` command to
# the AIStor image, so it starts via docker run. 127.0.0.1 endpoint is
# load-bearing: aws-lite picks path-style only for IP/localhost hosts,
# which is all stock MinIO/AIStor speaks.
run: |
docker run -d --name minio -p 127.0.0.1:9000:9000 \
-v "$PWD/minio-licence.key:/minio.license:ro" \
-e MINIO_ROOT_USER=trekci -e MINIO_ROOT_PASSWORD=trekci-secret \
quay.io/minio/aistor/minio:RELEASE.2026-08-07T18-34-35Z server /data --license /minio.license
- name: Wait for MinIO
# curl was removed from the AIStor image — poll from the runner instead.
# 60 x 2s gives a wider margin on cold runners than the 30 x 1s the
# pre-6fbff835 job used.
run: |
for i in $(seq 1 60); do
curl -sf http://127.0.0.1:9000/minio/health/live && exit 0
sleep 2
done
echo "MinIO never became healthy" >&2
docker logs minio >&2
exit 1
- name: Run the S3 driver contract suite
# No --coverage: the src/nest/storage/** 94/84/97/94 pin must keep
# holding from hermetic tests alone. The suite's beforeAll creates
# the trek-test bucket idempotently — no separate bucket-init step.
env:
TREK_TEST_S3_ENDPOINT: http://127.0.0.1:9000
TREK_TEST_S3_BUCKET: trek-test
TREK_TEST_S3_ACCESS_KEY_ID: trekci
TREK_TEST_S3_SECRET_ACCESS_KEY: trekci-secret
run: cd server && npx vitest run tests/unit/nest/storage/s3.driver.contract.test.ts
client-quality:
# Split out of client-tests: the suite takes ~11 minutes, and having the
# gates in front of it meant a single lint finding threw away the whole test
+3
View File
@@ -65,5 +65,8 @@ coverage-*/
.scannerwork
test-data
# Live-MinIO/AIStor manual test license key — never commit, operators obtain their own free key
minio-licence.key
.run
.full-review
+106 -2
View File
@@ -162,8 +162,9 @@ Most of what follows is an addon an admin switches on or off. Lists, Costs, Docu
- **2FA**: TOTP with ten single-use backup codes, and an admin can require it instance-wide
- **Passkeys**: WebAuthn login by fingerprint, face, PIN or security key, off until an admin enables it. A passkey also satisfies the 2FA requirement
- **Hardening**: per-IP limits on login, password reset and 2FA attempts, a password policy, secrets encrypted at rest and masked on read, and an SSRF guard on every URL you configure
- **Admin panel**: users and invites, the permission matrix, packing templates, categories, addons, plugins, API keys, MCP tokens and OAuth sessions, backups, audit log, and GitHub releases
- **Admin panel**: users and invites, the permission matrix, packing templates, categories, addons, plugins, API keys, MCP tokens and OAuth sessions, backups, storage, audit log, and GitHub releases
- **Backups**: manual or scheduled hourly, daily, weekly or monthly, with retention in days. The zip carries the at-rest key, so a restore can decrypt its own secrets
- **Storage**: pluggable storage backends per content category — keep everything on local disk, or add S3-compatible backends and replicate any category to them, configured entirely from the admin panel
- **Notifications**: a per-user matrix of events against in-app, email (SMTP), webhook and ntfy, plus any channel a plugin registers
- **Appearance**: light, dark or follow the OS, seven colour schemes plus a custom accent, transparency, compact density, reduce motion, and text size per tier
- **23 languages**: en, de, es, fr, it, nl, hu, ru, zh, zh-TW, pl, cs, ar (right to left), br, id, tr, ja, ko, uk, gr, sv, vi, ca
@@ -488,10 +489,113 @@ Caddy handles TLS and WebSockets automatically.
<br />
<h2 id="storage">Storage</h2>
TREK separates *what* it stores (eight content categories — trip documents,
journey photos, cover images, profile pictures, place images, the two photo
caches, and backups) from *where* it stores it (named backends). Out of the
box everything lives on local disk (`uploads/` and `data/backups`). From
**Admin → Storage** you can:
- **Add S3-compatible backends** (AWS S3, Cloudflare R2, Backblaze B2, Garage,
MinIO/AIStor) and assign any category to them.
- **Replicate a backend**: edit it and pick **Mirror targets** — every write is
then also copied to each target. The classic off-site backup setup is:
add an S3 backend, edit `backups-local`, tick the S3 backend as a mirror
target, save. Replica writes happen one after another during each upload, so
a slow or unreachable target slows every upload of every category on that
backend — fine for backups, worth weighing for hot categories. Existing
objects are not replicated when you add a target — press **Sync now** on
the backend to copy them (one sync at a time; progress and cancel in the
panel).
- **Watch the Health strip**: replica failures never fail the original
request; they are recorded and shown there instead. All-clear means every
replicated write landed. Failures also notify admins over the configured
notification channels (first failure immediately, then hourly summaries) —
tune it under Admin → Notifications.
- **Test** any backend from its row (for a replicated backend it probes the
primary and each target individually). Targets must be saved before Test
can probe them.
- **Usage**: per-category and per-backend object counts and sizes, scanned
nightly and on demand.
Backend credentials are encrypted at rest. Without an explicit
`ENCRYPTION_KEY` the implicit key from the data directory is used — that
works, but the implicit key rides inside backups, so setting `ENCRYPTION_KEY`
explicitly is recommended for credentialed backends. On an existing install
that has been running without an explicit key, do not just set a new one —
that would orphan already-stored secrets; use the key-rotation procedure (see
*Rotating the Encryption Key* under Updating) to move to an explicit key.
Notes and limits: reassigning a populated category prompts to move its
existing objects — accepting copies everything to the new backend, flips the
category, then sweeps any writes that raced the copy; source objects are kept
(not deleted) and reported as reclaimable (count + size) to reclaim manually;
declining routes only new writes to the new backend, leaving old objects in
place as before. Only one storage job (a mirror sync or a category migration)
runs at a time, cancelling before the flip is always safe, and a failed copy
never flips the category; media categories on S3 work but every served byte
proxies through the server — the proxy answers conditional GETs (ETag /
Last-Modified) and a single `Range` request, while a multi-range request is
answered with the full object; AWS buckets
with Object-Lock or checksum-requiring policies
reject the server's uploads; self-hosted endpoints (MinIO/Garage) should be
addressed by IP or `localhost` unless configured for virtual-hosted buckets.
The legacy `/uploads/photos` directory from older TREK versions is still
served and included in backups, but it is not a configurable category.
Running the storage contract suite against a live MinIO/AIStor is a
license-gated procedure — CI runs it automatically for same-repo pull
requests (fork PRs skip it, since they have no access to the license
secret); to run it yourself locally, see `docker-compose.minio-test.yml`'s
header for the manual procedure.
### Provisioning at first boot (seed file)
For infrastructure-as-code setups, mount a `storage-config.json` into the
data directory — it is imported **once**, on the first boot that has no
stored storage configuration, and loudly ignored afterwards:
```jsonc
// storage-config.json — secrets may be plaintext (encrypted on import)
// or already-encrypted enc:v1: values.
{
"backends": [
{ "name": "off-site", "type": "s3", "options": {
"endpoint": "https://s3.example.com", "bucket": "trek",
"accessKeyId": "…", "secretAccessKey": "…" } },
{ "name": "backups-mirror", "type": "mirror", "options": {
"primary": "backups-local", "replicas": ["off-site"] } }
],
"categories": { "backups": "backups-mirror" }
}
```
(The panel presents this exact setup as **Mirror targets** on `backups-local`.)
```yaml
# docker-compose: add under the trek service's volumes
- ./storage-config.json:/app/data/storage-config.json:ro
```
An invalid seed file aborts boot with the exact validation error — an
actively-provisioning operator sees the problem instead of a silent default.
### Recovery and restore
To reset storage configuration to the built-in defaults (or to re-import a
seed file): stop the server, run
`sqlite3 data/travel.db "DELETE FROM app_settings WHERE key LIKE 'storage.%';"`,
and start it again.
Backups include the storage configuration (it lives in the database). If your
only backup sits on S3 and the credentials for it sit inside that backup:
start a fresh instance, enter the S3 credentials in **Admin → Storage** (or
mount a seed file), then restore the backup from the Backup panel.
## Data & Backups
- **Database** — SQLite, stored in `./data/travel.db`
- **Uploads** — stored in `./uploads/`
- **Uploads** — stored in `./uploads/` by default; every content category can be reassigned to another backend in **Admin → Storage**
- **Logs** — `./data/logs/trek.log` (auto-rotated)
- **Backups** — create and restore via Admin Panel
- **Auto-Backups** — configurable schedule and retention in Admin Panel
+8
View File
@@ -267,6 +267,14 @@ describe('client > endpoint wiring', () => {
{ n: 'updateNotificationPreferences', r: () => adminApi.updateNotificationPreferences({ email: { trip_invite: true } }), e: 'PUT /api/admin/notification-preferences' },
{ n: 'getDefaultUserSettings', r: () => adminApi.getDefaultUserSettings(), e: 'GET /api/admin/default-user-settings' },
{ n: 'updateDefaultUserSettings', r: () => adminApi.updateDefaultUserSettings({ language: 'de' }), e: 'PUT /api/admin/default-user-settings' },
{ n: 'getStorage', r: () => adminApi.getStorage(), e: 'GET /api/admin/storage' },
{ n: 'updateStorage', r: () => adminApi.updateStorage({ backends: [], categories: {}, version: 0 }), e: 'PUT /api/admin/storage' },
{ n: 'testStorageBackend', r: () => adminApi.testStorageBackend({ name: 'x', type: 'local', options: { root: '/data' } }), e: 'POST /api/admin/storage/test' },
{ n: 'startStorageBackfill', r: () => adminApi.startStorageBackfill('m'), e: 'POST /api/admin/storage/backends/m/backfill' },
{ n: 'cancelStorageBackfill', r: () => adminApi.cancelStorageBackfill('m'), e: 'DELETE /api/admin/storage/backends/m/backfill' },
{ n: 'startStorageMigration', r: () => adminApi.startStorageMigration('files', 'dest'), e: 'POST /api/admin/storage/migrations' },
{ n: 'cancelStorageMigration', r: () => adminApi.cancelStorageMigration('files'), e: 'DELETE /api/admin/storage/migrations/files' },
{ n: 'refreshStorageStats', r: () => adminApi.refreshStorageStats(), e: 'POST /api/admin/storage/stats/refresh' },
{ n: 'mcpTokens', r: () => adminApi.mcpTokens(), e: 'GET /api/admin/mcp-tokens' },
{ n: 'deleteMcpToken', r: () => adminApi.deleteMcpToken(4), e: 'DELETE /api/admin/mcp-tokens/4' },
{ n: 'oauthSessions', r: () => adminApi.oauthSessions(), e: 'GET /api/admin/oauth-sessions' },
+25
View File
@@ -47,6 +47,11 @@ import {
type BookingImportPreviewResponse,
type BookingImportConfirmResponse,
type BookingImportMode,
type StorageAdminState,
type StorageBackend,
type StorageConfigPut,
type StorageTestResponse,
type StorageUsage,
} from '@trek/shared'
import { getSocketId } from './websocket'
import { probeNow } from '../sync/connectivity'
@@ -637,6 +642,26 @@ export const adminApi = {
updateNotificationPreferences: (prefs: Record<string, Record<string, boolean>>) => apiClient.put('/admin/notification-preferences', prefs).then(r => r.data),
getDefaultUserSettings: () => apiClient.get('/admin/default-user-settings').then(r => r.data),
updateDefaultUserSettings: (settings: Record<string, unknown>) => apiClient.put('/admin/default-user-settings', settings).then(r => r.data),
getStorage: (): Promise<StorageAdminState> => apiClient.get('/admin/storage').then(r => r.data),
updateStorage: (config: StorageConfigPut): Promise<StorageAdminState> =>
apiClient.put('/admin/storage', config).then(r => r.data),
// A probe is bounded by the target driver's own timeout (default 30s, ×2 with
// one retry) and a mirror probes its targets sequentially — the instance-wide
// 8s axios timeout would abort a legitimate slow probe, so this call carries
// its own generous ceiling.
testStorageBackend: (backend: StorageBackend): Promise<StorageTestResponse> =>
apiClient.post('/admin/storage/test', { backend }, { timeout: 120_000 }).then(r => r.data),
startStorageBackfill: (backend: string): Promise<{ started: true }> =>
apiClient.post(`/admin/storage/backends/${encodeURIComponent(backend)}/backfill`).then(r => r.data),
cancelStorageBackfill: (backend: string): Promise<{ cancelled: true }> =>
apiClient.delete(`/admin/storage/backends/${encodeURIComponent(backend)}/backfill`).then(r => r.data),
startStorageMigration: (category: string, to: string): Promise<{ started: true }> =>
apiClient.post('/admin/storage/migrations', { category, to }).then(r => r.data),
cancelStorageMigration: (category: string): Promise<{ cancelled: true }> =>
apiClient.delete(`/admin/storage/migrations/${encodeURIComponent(category)}`).then(r => r.data),
// A full scan of a large install can exceed the 8s instance timeout.
refreshStorageStats: (): Promise<StorageUsage> =>
apiClient.post('/admin/storage/stats/refresh', undefined, { timeout: 120_000 }).then(r => r.data),
}
export const addonsApi = {
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,779 @@
import React, { useEffect, useRef, useState } from 'react'
import { Activity, FolderTree, HardDrive } from 'lucide-react'
import {
STORAGE_CATEGORIES,
type StorageBackend,
type StorageCategory,
type StorageConfig,
type StorageMigrationStatus,
type StorageTestResponse,
} from '@trek/shared'
import { useTranslation } from '../../../i18n'
import { formatBytes } from '../../../utils/formatBytes'
import { relativeTime } from '../../../utils/relativeTime'
import ConfirmDialog from '../../shared/ConfirmDialog'
import CustomSelect from '../../shared/CustomSelect'
import { useToast } from '../../shared/Toast'
import Section from '../../Settings/Section'
import BackendForm, { type BackendFormMirrorProps } from './BackendForm'
import {
CACHE_CATEGORIES,
adoptedMirrorFor,
computeMigrationCandidates,
effectiveCategoryMap,
foldBackends,
mirrorProbeTargets,
primaryNameOf,
removeBackend,
removeBackendAndMirrors,
renameBackendRefs,
replicaCandidates,
replicaOfPrimaries,
settingsDocumentOf,
setMirrorTargets,
stripCategories,
upsertBackend,
usageByBackend,
type FoldedBackendRow,
type MigrationCandidate,
} from './storageModel'
import { useStorageAdmin } from './useStorageAdmin'
/** Display-name mapper for joined category lists — the raw id renders only in the badge. */
const categoryNames = (t: (key: string) => string, ids: readonly string[]): string =>
ids.map((id) => t(`storage.category.${id}`)).join(', ')
function TestResult({ result, failedLabel, okLabel }: {
result: StorageTestResponse | undefined
failedLabel: string
okLabel: string
}): React.ReactElement | null {
if (result === undefined) return null
return (
<div className="mt-2 space-y-0.5">
<p className="text-xs font-semibold text-content">{result.ok ? okLabel : failedLabel}</p>
{result.targets.map((target) => (
<p key={target.name} className="text-xs text-content-faint">
{target.ok ? '✓' : '✗'} {target.name}
{target.error ? ` — ${target.error}` : ''}
</p>
))}
</div>
)
}
const LINK_BUTTON_STYLE: React.CSSProperties = { background: 'none', border: 'none', cursor: 'pointer' }
export default function AdminStoragePanel(): React.ReactElement {
const { t, locale } = useTranslation()
const toast = useToast()
const admin = useStorageAdmin(t('common.error'), t('storage.saveConflict'))
const [editing, setEditing] = useState<{
initial: StorageBackend | null
originalName: string | null
mirror: BackendFormMirrorProps
} | null>(null)
const [confirmRemove, setConfirmRemove] = useState<{ name: string; degenerate: boolean } | null>(null)
const [syncPrompt, setSyncPrompt] = useState<string | null>(null)
// Open/closed only — never a snapshot of candidates. Storing the candidate
// array here would go stale the moment the operator edits a category while
// the dialog is open; the render below and moveAndSave both recompute
// computeMigrationCandidates(draft, state) fresh instead.
const [migratePromptOpen, setMigratePromptOpen] = useState(false)
const [migrationQueue, setMigrationQueue] = useState<MigrationCandidate[]>([])
// Set by `save` right before it calls admin.save(): the pre-save mirror
// target count per row name. Consumed (and cleared) by the effect below the
// first time `admin.state` changes afterward — never touched otherwise, so
// unrelated state changes (the backfill poll included) are no-ops here.
const pendingPromptCheck = useRef<Map<string, number> | null>(null)
// Synchronous single-flight lock for the queue effect below: `admin.state`
// only reflects a just-started migration once startMigration's awaited
// refreshState() resolves, so `setMigrationQueue(rest)` re-firing the
// effect synchronously (still against the pre-POST `admin.state`) would
// otherwise dequeue and POST the next candidate before the server has
// confirmed the first — a ref (not state) so the guard is visible on that
// very next synchronous re-render, not just after a state-driven one.
const migrationStartInFlight = useRef(false)
useEffect(() => {
if (!pendingPromptCheck.current || !admin.state) return
const before = pendingPromptCheck.current
pendingPromptCheck.current = null
const { rows: afterRows } = foldBackends(admin.state, settingsDocumentOf(admin.state))
const grown = afterRows.find((r) => r.mirrorTargets.length > (before.get(r.name) ?? 0))
if (grown) setSyncPrompt(grown.name)
}, [admin.state])
// Queued category migrations run strictly sequentially: once a slot opens
// (no migration currently running), dequeue the next candidate and start
// it. Depends on admin.state so a poll landing a terminal status re-fires
// this without any user interaction. The in-flight lock closes the race
// where setMigrationQueue(rest) re-fires this effect synchronously while
// admin.state still shows the pre-POST world — without it, the next
// candidate could dequeue and POST before the first is server-confirmed.
// Also waits out a running BACKFILL — the server's one-storage-job-at-a-time
// rule spans backfills and migrations alike, so starting while one runs
// would 409, and the queued candidate would be lost (no retry).
//
// A failed startMigration POST would otherwise strand `rest`: it was
// already dequeued into local state before the POST, so on failure nothing
// ever changes `migrationQueue` or `admin.state` again — the effect never
// re-fires and the remaining candidates sit invisibly forever, silently
// never migrated. On failure the whole remaining queue is explicitly
// cleared (never retried — the operator re-triggers via Save), named in a
// toast, and admin.state is refreshed so the panel reflects whatever the
// failed attempt's category ended up as server-side.
useEffect(() => {
if (migrationQueue.length === 0 || !admin.state) return
if (migrationStartInFlight.current) return
if (admin.state.migrations.some((m) => m.status === 'running')) return
if (admin.state.backfills.some((b) => b.status === 'running')) return
const [next, ...rest] = migrationQueue
migrationStartInFlight.current = true
setMigrationQueue(rest)
void admin.startMigration(next!.category, next!.toWire).then((error) => {
migrationStartInFlight.current = false
if (error) {
toast.error(error)
if (rest.length > 0) {
toast.error(t('storage.migrate.queueDropped', { categories: categoryNames(t, rest.map((c) => c.category)) }))
setMigrationQueue([])
}
void admin.refreshState().catch(() => {})
}
})
}, [migrationQueue, admin.state])
if (admin.loading) {
return <p className="text-sm italic p-4 text-content-faint">{t('storage.loading')}</p>
}
if (!admin.state || !admin.draft) {
return (
<div role="alert" className="rounded-xl border p-4 text-sm border-edge bg-surface-card text-content">
{admin.loadError || t('common.error')}
</div>
)
}
const { state, draft } = admin
// Pure/cheap — recomputed every render so the migrate-prompt dialog (below)
// never shows a stale candidate set while it's open (fix: migration prompt
// staleness). moveAndSave recomputes independently at confirm time rather
// than reading this render-scoped value, since the click handler and the
// render that produced it are not guaranteed to be the same one.
const migrationCandidates = computeMigrationCandidates(draft, state)
// Duplicate pre-check needs EVERY wire name (hidden mirror names included);
// mirror-target candidates are the visible primaries only.
const backendNames = [...new Set([...state.backends.map((b) => b.name), ...draft.backends.map((b) => b.name)])]
const { rows, degenerate } = foldBackends(state, draft)
const effective = effectiveCategoryMap(state, draft)
const usageSums = usageByBackend(state, draft)
const startEdit = (row: FoldedBackendRow) => {
// Editing a built-in creates a settings override row bearing its name —
// the first-class relocation path (merge-by-name).
setEditing({
initial: row.backend,
originalName: row.name,
mirror: {
candidates: replicaCandidates(rows, row.name, row.mirrorTargets),
initialTargets: row.mirrorTargets,
},
})
}
const commitBackend = (backend: StorageBackend, mirrorTargets?: string[]) => {
const renamedFrom =
editing?.originalName && editing.originalName !== backend.name ? editing.originalName : null
// Widened to StorageConfig: these edit helpers are version-blind (they
// return plain StorageConfig), and admin.setDraft re-attaches the
// draft's own `version` regardless of what shape it's handed.
let next: StorageConfig | null = draft
if (renamedFrom) next = renameBackendRefs(removeBackend(next, renamedFrom), renamedFrom, backend.name)
next = upsertBackend(next, backend)
if (mirrorTargets !== undefined) next = setMirrorTargets(state, next, backend.name, mirrorTargets)
admin.setDraft(next)
setEditing(null)
}
const removeMessage = (name: string, isDegenerate: boolean): string => {
const row = rows.find((r) => r.name === name)
const assigned = isDegenerate
? degenerate.find((d) => d.backend.name === name)?.categories ?? []
: row?.categories ?? []
const usedAsReplicaBy = isDegenerate ? [] : replicaOfPrimaries(draft, name)
return [
t('storage.remove.body', { name }),
assigned.length > 0 ? t('storage.remove.stillAssigned', { categories: categoryNames(t, assigned) }) : '',
usedAsReplicaBy.length > 0 ? t('storage.remove.usedAsReplicaBy', { primaries: usedAsReplicaBy.join(', ') }) : '',
]
.filter(Boolean)
.join(' ')
}
const setCategory = (category: StorageCategory, primaryName: string) => {
// Picking a mirrored primary routes through its adopted mirror silently.
const target = adoptedMirrorFor(draft, primaryName)?.name ?? primaryName
// The admin state's category record is exhaustive by schema contract.
const stateEntry = state.categories[category]!
const categories = { ...draft.categories }
if (stateEntry.source === 'default' && target === stateEntry.backend) {
delete categories[category] // back to the default → no longer settings-owned
} else {
categories[category] = target
}
admin.setDraft({ ...draft, categories })
}
// Snapshot the LAST-CONFIRMED (pre-save `state`'s own fold, not the
// in-progress `draft`) mirror target counts; the effect above compares
// them against the fresh post-save state to detect newly-added
// replicas. Folding `draft` here would already include the unsaved
// edit and mask the very growth this is meant to detect.
const snapshotMirrorTargets = (): Map<string, number> => {
const { rows: beforeRows } = foldBackends(state, settingsDocumentOf(state))
return new Map(beforeRows.map((r) => [r.name, r.mirrorTargets.length]))
}
const doPlainSave = async () => {
pendingPromptCheck.current = snapshotMirrorTargets()
if (await admin.save()) toast.success(t('storage.saved'))
else pendingPromptCheck.current = null
}
// Recomputes the candidate set at confirm time — never trusts whatever was
// true when the dialog opened. The operator can edit categories (or a
// background poll can land a fresher usage scan) while the dialog is on
// screen; the recomputed set is what actually drives both stripCategories
// (what gets reverted from the PUT body) and the queue (what gets POSTed),
// so the two can never diverge from what the dialog most recently showed.
const moveAndSave = async () => {
const candidates = computeMigrationCandidates(draft, state)
pendingPromptCheck.current = snapshotMirrorTargets()
const ok = await admin.save(stripCategories(draft, state, candidates.map((c) => c.category)))
if (ok) {
setMigrationQueue(candidates)
toast.success(t('storage.saved'))
} else {
pendingPromptCheck.current = null
}
setMigratePromptOpen(false)
}
const routeOnlySave = async () => {
setMigratePromptOpen(false)
await doPlainSave()
}
const save = async () => {
if (migrationCandidates.length > 0) {
setMigratePromptOpen(true)
return
}
await doPlainSave()
}
const handleCancelMigration = async (m: StorageMigrationStatus) => {
const error = await admin.cancelMigration(m.category)
if (error) toast.error(error)
}
const testResultFor = (key: string): StorageTestResponse | 'running' | undefined => admin.testResults[key]
const handleRefreshStats = async () => {
const error = await admin.refreshStats()
if (error) toast.error(error)
}
const handleStartBackfill = async (row: FoldedBackendRow) => {
if (!row.mirrorName) return
setSyncPrompt((prev) => (prev === row.name ? null : prev))
const error = await admin.startBackfill(row.mirrorName)
if (error) toast.error(error)
}
const handleCancelBackfill = async (row: FoldedBackendRow) => {
if (!row.mirrorName) return
const error = await admin.cancelBackfill(row.mirrorName)
if (error) toast.error(error)
}
return (
<div>
{state.configError && (
<div
role="alert"
className="rounded-xl border p-3 mb-4 text-sm"
style={{ borderColor: 'var(--warning)', background: 'var(--warning-soft)', color: 'var(--text-primary)' }}
>
{t('storage.configError.banner', { error: state.configError })}
</div>
)}
<Section title={t('storage.health.title')} icon={Activity}>
{state.health.replicaFailures.length === 0 ? (
<p className="text-sm text-content-faint">{t('storage.health.allClear')}</p>
) : (
<ul className="space-y-1">
{state.health.replicaFailures.map((failure) => (
<li key={`${failure.backend}-${failure.key}-${failure.at}`} className="text-sm text-content">
{t('storage.health.failureLine', {
op: failure.op,
key: failure.key,
backend: failure.backend,
error: failure.error,
})}
<span className="text-content-faint"> · {relativeTime(failure.at, locale)}</span>
</li>
))}
</ul>
)}
{state.seedFilePresent && <p className="text-xs mt-2 text-content-faint">{t('storage.health.seedFile')}</p>}
</Section>
<Section title={t('storage.backends.title')} icon={HardDrive}>
<p className="text-sm text-content-faint" style={{ marginTop: -8 }}>{t('storage.description')}</p>
<p className="text-xs mb-2 text-content-faint">
{state.usage
? t('storage.usage.computed', { age: relativeTime(state.usage.computedAt, locale) })
: t('storage.usage.never')}
{' '}
<button className="text-xs underline text-content-secondary" style={LINK_BUTTON_STYLE} onClick={handleRefreshStats}>
{state.usage ? t('storage.usage.refresh') : t('storage.usage.compute')}
</button>
</p>
<div className="space-y-3">
{rows.map((row) => {
const resultKey = row.mirrorName ?? row.name
const result = testResultFor(resultKey)
// row.mirrorName only exists when foldBackends adopted a draft mirror
// for this row, so the draft lookup below cannot miss.
const testCandidate = row.mirrorName
? draft.backends.find((b) => b.name === row.mirrorName)!
: row.backend
const rowUsage = usageSums?.[row.name]
const backfill = row.mirrorName ? state.backfills.find((b) => b.backend === row.mirrorName) : undefined
return (
<div key={row.name} data-testid={`storage-backend-${row.name}`} className="rounded-xl border p-4 border-edge-secondary">
<div className="flex items-center gap-2 flex-wrap">
<span className="text-sm font-semibold text-content">{row.name}</span>
<span className="text-xs px-2 py-0.5 rounded-full border border-edge text-content-secondary">
{t(`storage.type.${row.type}`)}
</span>
<span className="text-xs px-2 py-0.5 rounded-full border border-edge text-content-faint">
{t(`storage.source.${row.source}`)}
</span>
<span className="flex-1" />
<button
className="text-xs underline text-content-secondary"
style={LINK_BUTTON_STYLE}
onClick={() =>
row.mirrorName
? admin.testMirror(resultKey, mirrorProbeTargets(draft, state, testCandidate))
: admin.test(testCandidate)
}
>
{t('storage.actions.test')}
</button>
{row.source !== 'env' && (
<button className="text-xs underline text-content-secondary" style={LINK_BUTTON_STYLE} onClick={() => startEdit(row)}>
{t('storage.actions.edit')}
</button>
)}
{row.source === 'settings' && (
<button className="text-xs underline text-content-secondary" style={LINK_BUTTON_STYLE} onClick={() => setConfirmRemove({ name: row.name, degenerate: false })}>
{t('storage.actions.remove')}
</button>
)}
</div>
<p className="text-xs mt-1 text-content-faint">
{row.categories.length > 0
? t('storage.backends.usedBy', { categories: categoryNames(t, row.categories) })
: t('storage.backends.unused')}
</p>
{rowUsage && (
<p className="text-xs mt-1 text-content-faint">
{t('storage.usage.line', { objects: String(rowUsage.objects), size: formatBytes(rowUsage.bytes) })}
{row.name === 'uploads-local' && state.usage!.legacyPhotos.objects > 0
? ` (${t('storage.usage.legacyNote')})`
: ''}
</p>
)}
{row.mirrorTargets.length > 0 && (
<p className="text-xs mt-1 text-content-faint">
{t('storage.mirror.mirroredTo', { targets: row.mirrorTargets.join(', ') })}
</p>
)}
{row.replicaOf.length > 0 && (
<p className="text-xs mt-1 text-content-faint">
{t('storage.mirror.replicaOf', { primaries: row.replicaOf.join(', ') })}
</p>
)}
{row.source === 'env' && <p className="text-xs mt-1 text-content-faint">{t('storage.backends.envReadOnly')}</p>}
{result === 'running' ? (
<p className="text-xs mt-2 text-content-faint">{t('storage.test.running')}</p>
) : (
<TestResult
result={result as StorageTestResponse | undefined}
okLabel={t('storage.test.ok')}
failedLabel={t('storage.test.failed')}
/>
)}
{row.mirrorTargets.length > 0 && (
<div className="mt-2">
{backfill?.status === 'running' ? (
<>
<p className="text-xs text-content-faint">
{t('storage.sync.running', { done: String(backfill.done), total: String(backfill.total) })}
</p>
<p className="text-xs text-content-faint">
{t('storage.sync.counts', {
copied: String(backfill.copied),
skipped: String(backfill.skipped),
failed: String(backfill.failed),
})}
</p>
<button
className="text-xs underline text-content-secondary mt-1"
style={LINK_BUTTON_STYLE}
onClick={() => handleCancelBackfill(row)}
>
{t('storage.sync.cancel')}
</button>
</>
) : (
<>
{/* Terminal statuses render ALONGSIDE the button (not instead-of) —
a completed/cancelled/errored backfill must stay re-runnable
without a page reload. */}
{backfill?.status === 'done' && (
<p className="text-xs text-content-faint">
{t('storage.sync.done', {
copied: String(backfill.copied),
deleted: String(backfill.deleted),
failed: String(backfill.failed),
})}
</p>
)}
{backfill?.status === 'cancelled' && (
<p className="text-xs text-content-faint">{t('storage.sync.cancelled')}</p>
)}
{backfill?.status === 'error' && (
<p className="text-xs text-content-faint">
{t('storage.sync.error', { error: backfill.error ?? '' })}
</p>
)}
{syncPrompt !== row.name && (
<button
className="text-xs underline text-content-secondary"
style={LINK_BUTTON_STYLE}
onClick={() => handleStartBackfill(row)}
>
{t('storage.sync.now')}
</button>
)}
</>
)}
</div>
)}
{syncPrompt === row.name && (
<div className="mt-2 rounded-lg border p-2 border-edge-secondary">
<p className="text-xs text-content">{t('storage.sync.prompt')}</p>
<div className="flex items-center gap-3 mt-1">
<button
className="text-xs underline text-content-secondary"
style={LINK_BUTTON_STYLE}
onClick={() => handleStartBackfill(row)}
>
{t('storage.sync.now')}
</button>
<button
className="text-xs underline text-content-secondary"
style={LINK_BUTTON_STYLE}
onClick={() => setSyncPrompt(null)}
>
{t('storage.sync.dismiss')}
</button>
</div>
</div>
)}
</div>
)
})}
{degenerate.map(({ backend, reason }) => {
const result = testResultFor(backend.name)
const primary = backend.type === 'mirror' ? backend.options.primary : ''
return (
<div key={backend.name} data-testid={`storage-backend-${backend.name}`} className="rounded-xl border p-4 border-edge-secondary">
<div className="flex items-center gap-2 flex-wrap">
<span className="text-sm font-semibold text-content">{backend.name}</span>
<span className="text-xs px-2 py-0.5 rounded-full border border-edge text-content-secondary">
{t('storage.type.mirror')}
</span>
<span className="flex-1" />
<button className="text-xs underline text-content-secondary" style={LINK_BUTTON_STYLE} onClick={() => admin.test(backend)}>
{t('storage.actions.test')}
</button>
<button className="text-xs underline text-content-secondary" style={LINK_BUTTON_STYLE} onClick={() => setConfirmRemove({ name: backend.name, degenerate: true })}>
{t('storage.actions.remove')}
</button>
</div>
<p className="text-xs mt-1 text-content-faint">{t(`storage.mirror.degenerate.${reason}`, { primary })}</p>
{result === 'running' ? (
<p className="text-xs mt-2 text-content-faint">{t('storage.test.running')}</p>
) : (
<TestResult
result={result as StorageTestResponse | undefined}
okLabel={t('storage.test.ok')}
failedLabel={t('storage.test.failed')}
/>
)}
</div>
)
})}
</div>
{state.migrations.length > 0 && (
<div className="space-y-3 mt-3">
{state.migrations.map((m) => (
<div
key={m.category}
data-testid={`storage-migration-${m.category}`}
className="rounded-xl border p-4 border-edge-secondary"
>
<p className="text-sm font-semibold text-content">{t(`storage.category.${m.category}`)}</p>
{m.status === 'running' ? (
<>
<p className="text-xs mt-1 text-content-faint">
{t('storage.migrate.running', {
category: t(`storage.category.${m.category}`),
done: String(m.done),
total: String(m.total),
})}
</p>
<button
className="text-xs underline text-content-secondary mt-1"
style={LINK_BUTTON_STYLE}
onClick={() => void handleCancelMigration(m)}
>
{t('storage.migrate.cancel')}
</button>
</>
) : m.status === 'done' ? (
<>
<p className="text-xs mt-1 text-content-faint">
{t('storage.migrate.done', { copied: String(m.copied), skipped: String(m.skipped) })}
</p>
{m.failed > 0 && (
<p className="text-xs mt-1 text-content-faint">
{t('storage.migrate.doneFailures', { failed: String(m.failed) })}
</p>
)}
{m.reclaimable && (
<p className="text-xs mt-1 text-content-faint">
{t('storage.migrate.reclaimable', {
objects: String(m.reclaimable.objects),
size: formatBytes(m.reclaimable.bytes),
from: m.from,
})}
</p>
)}
</>
) : m.status === 'failed' ? (
<p className="text-xs mt-1 text-content-faint">
{t('storage.migrate.failed', { error: m.error ?? '' })}
</p>
) : (
<p className="text-xs mt-1 text-content-faint">{t('storage.migrate.cancelled')}</p>
)}
</div>
))}
</div>
)}
{editing ? (
<BackendForm
initial={editing.initial}
backendNames={backendNames}
mirror={editing.mirror}
onCommit={commitBackend}
onCancel={() => setEditing(null)}
/>
) : (
<button
onClick={() =>
setEditing({
initial: null,
originalName: null,
mirror: { candidates: replicaCandidates(rows, null), initialTargets: [] },
})
}
style={{
padding: '8px 20px', borderRadius: 10, cursor: 'pointer', fontFamily: 'inherit',
fontSize: 'calc(14px * var(--fs-scale-body, 1))', fontWeight: 500,
border: '2px solid var(--border-primary)', background: 'var(--bg-card)', color: 'var(--text-primary)',
}}
>
{t('storage.backends.add')}
</button>
)}
</Section>
<Section title={t('storage.categories.title')} icon={FolderTree}>
<div className="grid grid-cols-1 sm:grid-cols-2 gap-x-6 gap-y-4">
{STORAGE_CATEGORIES.map((category) => {
// The admin state's category record is exhaustive by schema contract.
const stateEntry = state.categories[category]!
const selectedPrimary = primaryNameOf(state, draft, effective[category])
const changed = selectedPrimary !== primaryNameOf(state, draft, stateEntry.backend)
const viaMirror = effective[category] !== selectedPrimary
return (
<div key={category} data-testid={`storage-category-${category}`}>
<label className="block text-sm font-medium mb-1 text-content-secondary">
{t(`storage.category.${category}`)}
<span className="ml-2 px-1.5 py-0.5 rounded border font-mono font-normal text-xs border-edge text-content-faint">
{category}
</span>
</label>
<p className="text-xs mb-1.5 text-content-faint">{t(`storage.categoryDesc.${category}`)}</p>
{state.usage?.categories[category] && (
<p className="text-xs mb-1.5 text-content-faint">
{t('storage.usage.line', {
objects: String(state.usage.categories[category]!.objects),
size: formatBytes(state.usage.categories[category]!.bytes),
})}
</p>
)}
<CustomSelect
value={selectedPrimary}
onChange={(value) => setCategory(category, String(value))}
options={rows.map((row) => ({
value: row.name,
label:
stateEntry.source === 'default' && row.name === stateEntry.backend
? `${row.name} (${t('storage.categories.default')})`
: row.name,
}))}
size="sm"
style={{ maxWidth: 320 }}
/>
{changed && (
<p role="alert" className="text-xs mt-1 text-content-faint">
{t('storage.categories.reassignWarning')}
</p>
)}
{viaMirror && CACHE_CATEGORIES.includes(category) && (
<p role="note" className="text-xs mt-1 text-content-faint">
{t('storage.mirror.cacheWarning')}
</p>
)}
</div>
)
})}
</div>
</Section>
<div className="flex items-center gap-3">
<button
onClick={save}
disabled={!admin.dirty || admin.saving}
style={{
padding: '10px 20px', borderRadius: 10, cursor: admin.dirty && !admin.saving ? 'pointer' : 'default',
fontFamily: 'inherit', fontSize: 'calc(14px * var(--fs-scale-body, 1))', fontWeight: 600,
border: '2px solid var(--text-primary)', background: 'var(--bg-hover)', color: 'var(--text-primary)',
opacity: admin.dirty && !admin.saving ? 1 : 0.5,
}}
>
{t('storage.save')}
</button>
{admin.dirty && <span className="text-xs text-content-faint">{t('storage.unsaved')}</span>}
</div>
{migrationQueue.length > 0 && (
<p className="text-xs mt-2 text-content-faint">
{t('storage.migrate.queued', { categories: categoryNames(t, migrationQueue.map((c) => c.category)) })}
</p>
)}
{migratePromptOpen && (
<div className="rounded-lg border px-3 py-2 mt-2 border-edge bg-surface-secondary" role="alertdialog">
<p className="text-sm text-content">{t('storage.migrate.promptTitle')}</p>
{migrationCandidates.map((c) => (
<p key={c.category} className="text-xs mt-1 text-content-secondary">
{c.objects === null
? t('storage.migrate.promptLineUnknown', {
category: t(`storage.category.${c.category}`),
from: c.from,
to: c.to,
})
: t('storage.migrate.promptLine', {
category: t(`storage.category.${c.category}`),
objects: String(c.objects),
size: formatBytes(c.bytes ?? 0),
from: c.from,
to: c.to,
})}
</p>
))}
<div className="flex items-center gap-3 mt-2">
<button
className="text-xs underline text-content-secondary"
style={LINK_BUTTON_STYLE}
onClick={() => void moveAndSave()}
>
{t('storage.migrate.move')}
</button>
<button
className="text-xs underline text-content-secondary"
style={LINK_BUTTON_STYLE}
onClick={() => void routeOnlySave()}
>
{t('storage.migrate.routeOnly')}
</button>
<button
className="text-xs underline text-content-secondary"
style={LINK_BUTTON_STYLE}
onClick={() => setMigratePromptOpen(false)}
>
{t('storage.migrate.promptCancel')}
</button>
</div>
</div>
)}
{admin.saveError && (
<div role="alert" className="text-sm mt-2 text-content">
<p>{admin.saveError}</p>
{/* A 409 leaves the draft pinned to a version the server has moved
past; "save again" can never clear it, so the banner offers the
one action that can — see useStorageAdmin.discardDraft. */}
{admin.saveConflict && (
<button
className="text-xs underline text-content-secondary mt-1"
style={LINK_BUTTON_STYLE}
onClick={() => void admin.discardDraft()}
>
{t('storage.discardAndReload')}
</button>
)}
</div>
)}
<ConfirmDialog
isOpen={confirmRemove !== null}
onClose={() => setConfirmRemove(null)}
onConfirm={() => {
if (confirmRemove) {
admin.setDraft(
confirmRemove.degenerate
? removeBackend(draft, confirmRemove.name)
: removeBackendAndMirrors(state, draft, confirmRemove.name),
)
}
setConfirmRemove(null)
}}
title={t('storage.remove.title')}
message={confirmRemove ? removeMessage(confirmRemove.name, confirmRemove.degenerate) : ''}
confirmLabel={t('storage.remove.title')}
/>
</div>
)
}
@@ -0,0 +1,155 @@
import { describe, expect, it, vi } from 'vitest';
import { MASKED_SETTING_VALUE, type StorageBackend } from '@trek/shared';
import { fireEvent, render, screen } from '../../../../tests/helpers/render';
import BackendForm from './BackendForm';
const NAMES = ['uploads-local', 'backups-local', 'off-box'];
type FormProps = Parameters<typeof BackendForm>[0];
function renderForm(overrides: Partial<FormProps> = {}) {
const onCommit = vi.fn();
const onCancel = vi.fn();
render(
<BackendForm
initial={null}
backendNames={NAMES}
onCommit={onCommit}
onCancel={onCancel}
{...overrides}
/>,
);
return { onCommit, onCancel };
}
const S3_INITIAL: StorageBackend = {
name: 'off-box',
type: 's3',
options: {
endpoint: 'http://127.0.0.1:9000', bucket: 'trek', accessKeyId: 'ak',
secretAccessKey: MASKED_SETTING_VALUE, region: 'us-east-1', keyPrefix: '', retries: 1, timeoutMs: 30000,
},
};
describe('BackendForm', () => {
it('FE-ADMIN-STORF-001: local renders the root path field from the registry', () => {
renderForm();
expect(screen.getByLabelText(/Root directory/)).toBeInTheDocument();
expect(screen.queryByLabelText(/Bucket/)).not.toBeInTheDocument();
});
it('FE-ADMIN-STORF-002: s3 renders every field by kind — password secret, number inputs, defaultValue placeholders', () => {
renderForm({ initial: S3_INITIAL });
expect(screen.getByLabelText(/Endpoint URL/)).toHaveAttribute('type', 'text');
expect(screen.getByLabelText(/Secret access key/)).toHaveAttribute('type', 'password');
expect(screen.getByLabelText(/Region/)).toHaveAttribute('placeholder', 'us-east-1');
expect(screen.getByLabelText(/Retries/)).toHaveAttribute('type', 'number');
expect(screen.getByLabelText(/Retries/)).toHaveAttribute('placeholder', '1');
expect(screen.getByLabelText(/Timeout \(ms\)/)).toHaveAttribute('placeholder', '30000');
});
it('FE-ADMIN-STORF-004: the mask echoes back through commit untouched (no-op by contract)', () => {
const { onCommit } = renderForm({ initial: S3_INITIAL });
fireEvent.click(screen.getByRole('button', { name: 'Apply' }));
expect(onCommit).toHaveBeenCalledWith(
expect.objectContaining({
name: 'off-box',
type: 's3',
options: expect.objectContaining({ secretAccessKey: MASKED_SETTING_VALUE, retries: 1, timeoutMs: 30000 }),
}),
);
});
it('FE-ADMIN-STORF-005: a typed plaintext secret keeps Apply offered (no encryption-key gate)', () => {
renderForm({ initial: S3_INITIAL });
expect(screen.getByRole('button', { name: 'Apply' })).toBeInTheDocument();
fireEvent.change(screen.getByLabelText(/Secret access key/), { target: { value: 'sk-new' } });
expect(screen.getByRole('button', { name: 'Apply' })).toBeInTheDocument();
expect(screen.queryByRole('alert')).not.toBeInTheDocument();
});
it('FE-ADMIN-STORF-006: Apply stays disabled until name and every required field are filled', () => {
const { onCommit } = renderForm();
const apply = screen.getByRole('button', { name: 'Apply' });
expect(apply).toBeDisabled();
fireEvent.change(screen.getByLabelText(/Name/), { target: { value: 'nas' } });
expect(apply).toBeDisabled(); // root still empty
fireEvent.change(screen.getByLabelText(/Root directory/), { target: { value: '/mnt/nas' } });
expect(apply).toBeEnabled();
fireEvent.click(apply);
expect(onCommit).toHaveBeenCalledWith({ name: 'nas', type: 'local', options: { root: '/mnt/nas' } });
});
it('FE-ADMIN-STORF-007: a duplicate name on a NEW backend warns and blocks Apply', () => {
renderForm();
fireEvent.change(screen.getByLabelText(/Name/), { target: { value: 'off-box' } });
fireEvent.change(screen.getByLabelText(/Root directory/), { target: { value: '/x' } });
expect(screen.getByText(/A backend named off-box already exists/)).toBeInTheDocument();
expect(screen.getByRole('button', { name: 'Apply' })).toBeDisabled();
});
it('FE-ADMIN-STORF-008: empty optional fields are omitted so the shared schema defaults apply', () => {
const { onCommit } = renderForm();
// Switch to s3 via the type select (CustomSelect renders options into a portal).
fireEvent.click(screen.getByText('Local'));
fireEvent.click(screen.getByText('S3'));
fireEvent.change(screen.getByLabelText(/Name/), { target: { value: 'fresh' } });
fireEvent.change(screen.getByLabelText(/Endpoint URL/), { target: { value: 'http://127.0.0.1:9000' } });
fireEvent.change(screen.getByLabelText(/Bucket/), { target: { value: 'trek' } });
fireEvent.change(screen.getByLabelText(/Access key ID/), { target: { value: 'ak' } });
fireEvent.change(screen.getByLabelText(/Secret access key/), { target: { value: 'sk' } });
fireEvent.click(screen.getByRole('button', { name: 'Apply' }));
const committed = onCommit.mock.calls[0]![0] as StorageBackend;
expect(committed.options).toEqual({
endpoint: 'http://127.0.0.1:9000', bucket: 'trek', accessKeyId: 'ak', secretAccessKey: 'sk',
});
expect(committed.options).not.toHaveProperty('region');
});
it('FE-ADMIN-STORF-009: renaming onto another existing backend warns and blocks Apply', () => {
renderForm({ initial: S3_INITIAL });
expect(screen.getByRole('button', { name: 'Apply' })).toBeEnabled();
fireEvent.change(screen.getByLabelText(/Name/), { target: { value: 'uploads-local' } });
expect(screen.getByText(/A backend named uploads-local already exists/)).toBeInTheDocument();
expect(screen.getByRole('button', { name: 'Apply' })).toBeDisabled();
// Keeping the original name stays allowed.
fireEvent.change(screen.getByLabelText(/Name/), { target: { value: 'off-box' } });
expect(screen.getByRole('button', { name: 'Apply' })).toBeEnabled();
});
it('FE-ADMIN-STORF-010: the type select no longer offers Mirror', () => {
renderForm();
fireEvent.click(screen.getByText('Local')); // open the type select
expect(screen.queryByText('Mirror')).not.toBeInTheDocument();
expect(screen.getByText('S3')).toBeInTheDocument();
});
it('FE-ADMIN-STORF-011: the mirror composer renders candidates minus self and commits targets', () => {
const { onCommit } = renderForm({
initial: S3_INITIAL,
mirror: { candidates: ['uploads-local', 'backups-local', 'off-box'], initialTargets: ['backups-local'] },
});
expect(screen.getByText('Mirror targets')).toBeInTheDocument();
const boxes = screen.getAllByRole('checkbox');
expect(boxes).toHaveLength(2); // off-box (self) excluded
expect(screen.getByRole('checkbox', { name: 'backups-local' })).toBeChecked();
expect(screen.getByRole('note').textContent).toContain('slows every upload');
fireEvent.click(screen.getByRole('checkbox', { name: 'uploads-local' }));
fireEvent.click(screen.getByRole('button', { name: 'Apply' }));
expect(onCommit).toHaveBeenCalledWith(
expect.objectContaining({ name: 'off-box', type: 's3' }),
['backups-local', 'uploads-local'],
);
});
it('FE-ADMIN-STORF-012: unchecking every target commits an empty array (dissolve) and hides the latency note', () => {
const { onCommit } = renderForm({
initial: S3_INITIAL,
mirror: { candidates: ['uploads-local'], initialTargets: ['uploads-local'] },
});
fireEvent.click(screen.getByRole('checkbox', { name: 'uploads-local' }));
expect(screen.queryByRole('note')).not.toBeInTheDocument();
fireEvent.click(screen.getByRole('button', { name: 'Apply' }));
expect(onCommit).toHaveBeenCalledWith(expect.objectContaining({ name: 'off-box' }), []);
});
});
@@ -0,0 +1,264 @@
import React, { useState } from 'react'
import {
STORAGE_BACKEND_TYPES,
STORAGE_BACKEND_TYPE_IDS,
type StorageBackend,
type StorageBackendFieldDef,
type StorageBackendTypeId,
} from '@trek/shared'
import { useTranslation } from '../../../i18n'
import CustomSelect from '../../shared/CustomSelect'
type FieldValues = Record<string, string | string[]>
function valuesOf(backend: StorageBackend | null): FieldValues {
if (!backend) return {}
const values: FieldValues = {}
for (const [key, value] of Object.entries(backend.options)) {
values[key] = Array.isArray(value) ? value : String(value)
}
return values
}
export interface BackendFormMirrorProps {
/** Selectable replica targets (the form excludes its own name at render). */
candidates: string[]
initialTargets: string[]
}
interface BackendFormProps {
/** null = new backend */
initial: StorageBackend | null
/** Every defined backend name — mirror-target options and the duplicate pre-check. */
backendNames: string[]
/**
* Present on non-mirror backends: renders the Mirror-targets composer
* (replicas-on-primary — panel-supplied chrome, never a registry field).
* When present, onCommit's second argument is always an array.
*/
mirror?: BackendFormMirrorProps
onCommit: (backend: StorageBackend, mirrorTargets?: string[]) => void
onCancel: () => void
}
const LABEL_CLASS = 'block text-sm font-medium mb-1.5 text-content-secondary'
const INPUT_CLASS =
'mt-1.5 w-full px-3 py-2 border rounded-lg text-sm border-edge bg-surface-card text-content'
/**
* Renders whatever STORAGE_BACKEND_TYPES declares, by field kind. The raw
* `mirror` type is hidden from the type select — mirrors are composed via the
* Mirror-targets prop block and synthesized by the panels (the ref-kind
* renderers below stay for future registry types).
*/
export default function BackendForm({
initial,
backendNames,
mirror,
onCommit,
onCancel,
}: BackendFormProps): React.ReactElement {
const { t } = useTranslation()
const [type, setType] = useState<StorageBackendTypeId>(initial?.type ?? 'local')
const [name, setName] = useState(initial?.name ?? '')
const [values, setValues] = useState<FieldValues>(() => valuesOf(initial))
const [targets, setTargets] = useState<string[]>(mirror?.initialTargets ?? [])
const fields = STORAGE_BACKEND_TYPES[type].fields as readonly StorageBackendFieldDef[]
const refOptions = backendNames.filter((candidate) => candidate !== name.trim())
const setValue = (key: string, value: string | string[]) =>
setValues((prev) => ({ ...prev, [key]: value }))
const filled = (field: StorageBackendFieldDef): boolean => {
const value = values[field.key]
if (field.kind === 'backend-ref-list') return Array.isArray(value) && value.length > 0
return typeof value === 'string' && value.trim() !== ''
}
const duplicate = name.trim() !== (initial?.name ?? '') && backendNames.includes(name.trim())
const canApply =
name.trim() !== '' && !duplicate && fields.every((f) => !f.required || filled(f))
const apply = () => {
const options: Record<string, unknown> = {}
for (const field of fields) {
const value = values[field.key]
if (field.kind === 'backend-ref-list') {
options[field.key] = Array.isArray(value) ? value : []
continue
}
const text = typeof value === 'string' ? value : ''
if (text === '' && !field.required) continue // omitted → the shared schema default applies
options[field.key] = field.kind === 'number' ? Number(text) : text
}
// The options were built from the same field defs the schema is generated
// from — this is the same sanctioned cast storageModel.asWireBackend makes.
const payload = { name: name.trim(), type, options } as StorageBackend
// Arity matters: the landed tests pin single-argument calls when no mirror
// block is supplied (toHaveBeenCalledWith treats a trailing undefined as a
// mismatch), so the second argument exists only when the composer does.
if (mirror) onCommit(payload, targets)
else onCommit(payload)
}
const renderField = (field: StorageBackendFieldDef): React.ReactElement => {
const value = values[field.key]
if (field.kind === 'backend-ref') {
return (
<div key={field.key}>
<span className={LABEL_CLASS}>{t(field.labelKey)}</span>
<CustomSelect
value={typeof value === 'string' ? value : ''}
onChange={(next) => setValue(field.key, String(next))}
options={refOptions.map((candidate) => ({ value: candidate, label: candidate }))}
placeholder={t(field.labelKey)}
size="sm"
/>
</div>
)
}
if (field.kind === 'backend-ref-list') {
const selected = Array.isArray(value) ? value : []
return (
<div key={field.key}>
<span className={LABEL_CLASS}>{t(field.labelKey)}</span>
<div className="space-y-1">
{refOptions.map((candidate) => (
<label key={candidate} className="flex items-center gap-2 text-sm text-content">
<input
type="checkbox"
checked={selected.includes(candidate)}
onChange={(e) =>
setValue(
field.key,
e.target.checked
? [...selected, candidate]
: selected.filter((existing) => existing !== candidate),
)
}
/>
{candidate}
</label>
))}
</div>
</div>
)
}
const inputType = field.kind === 'secret' ? 'password' : field.kind === 'number' ? 'number' : 'text'
return (
<label key={field.key} className={LABEL_CLASS}>
{t(field.labelKey)}
<input
type={inputType}
value={typeof value === 'string' ? value : ''}
onChange={(e) => setValue(field.key, e.target.value)}
placeholder={field.defaultValue !== undefined ? String(field.defaultValue) : ''}
spellCheck={false}
autoComplete="off"
className={INPUT_CLASS}
/>
{field.helpKey && <span className="block text-xs mt-1 font-normal text-content-faint">{t(field.helpKey)}</span>}
</label>
)
}
return (
<div className="rounded-xl border p-4 space-y-4 border-edge bg-surface-card">
<p className="text-sm font-semibold text-content">
{initial ? t('storage.form.editTitle') : t('storage.form.addTitle')}
</p>
<label className={LABEL_CLASS}>
{t('storage.form.name')}
<input
type="text"
value={name}
onChange={(e) => setName(e.target.value)}
spellCheck={false}
autoComplete="off"
className={INPUT_CLASS}
/>
</label>
{duplicate && (
<p className="text-xs text-content-faint" role="alert">
{t('storage.form.duplicateName', { name: name.trim() })}
</p>
)}
<div>
<span className={LABEL_CLASS}>{t('storage.form.type')}</span>
<CustomSelect
value={type}
onChange={(next) => {
setType(next as StorageBackendTypeId)
setValues({}) // a different type has different fields
}}
options={STORAGE_BACKEND_TYPE_IDS.filter((id) => id !== 'mirror').map((id) => ({
value: id,
label: t(`storage.type.${id}`),
}))}
size="sm"
disabled={initial !== null}
/>
</div>
{fields.map(renderField)}
{mirror && (
<div>
<span className={LABEL_CLASS}>{t('storage.mirror.targets')}</span>
<p className="text-xs mb-1 text-content-faint">{t('storage.mirror.targetsHelp')}</p>
<div className="space-y-1">
{mirror.candidates
.filter((candidate) => candidate !== name.trim())
.map((candidate) => (
<label key={candidate} className="flex items-center gap-2 text-sm text-content">
<input
type="checkbox"
checked={targets.includes(candidate)}
onChange={(e) =>
setTargets(
e.target.checked
? [...targets, candidate]
: targets.filter((existing) => existing !== candidate),
)
}
/>
{candidate}
</label>
))}
</div>
{targets.length > 0 && (
<p className="text-xs mt-1 text-content-faint" role="note">
{t('storage.mirror.latencyNote')}
</p>
)}
</div>
)}
<div className="flex items-center gap-3">
<button
onClick={apply}
disabled={!canApply}
style={{
padding: '8px 20px', borderRadius: 10, cursor: canApply ? 'pointer' : 'default',
fontFamily: 'inherit', fontSize: 'calc(14px * var(--fs-scale-body, 1))', fontWeight: 600,
border: '2px solid var(--text-primary)', background: 'var(--bg-hover)',
color: 'var(--text-primary)', opacity: canApply ? 1 : 0.5,
}}
>
{t('storage.form.apply')}
</button>
<button
onClick={onCancel}
style={{
padding: '8px 20px', borderRadius: 10, cursor: 'pointer', fontFamily: 'inherit',
fontSize: 'calc(14px * var(--fs-scale-body, 1))', fontWeight: 500,
border: '2px solid var(--border-primary)', background: 'var(--bg-card)', color: 'var(--text-primary)',
}}
>
{t('storage.form.cancel')}
</button>
</div>
</div>
)
}
@@ -0,0 +1,407 @@
import { describe, expect, it } from 'vitest';
import {
MASKED_SETTING_VALUE,
STORAGE_CATEGORIES,
type StorageAdminState,
type StorageBackend,
type StorageCategory,
type StorageConfig,
} from '@trek/shared';
import {
categoriesPointingAt,
computeMigrationCandidates,
foldBackends,
mirrorProbeTargets,
mirrorsReferencing,
primaryNameOf,
removeBackend,
removeBackendAndMirrors,
renameBackendRefs,
replicaCandidates,
replicaOfPrimaries,
setMirrorTargets,
settingsDocumentOf,
stripCategories,
upsertBackend,
usageByBackend,
} from './storageModel';
const STATE: StorageAdminState = {
backends: [
{ name: 'uploads-local', type: 'local', source: 'built-in', options: { root: '/data/uploads' }, categories: ['files'] },
{ name: 'place-photos-local', type: 'local', source: 'env', options: { root: '/photos' }, categories: ['places'] },
{
name: 'off-box',
type: 's3',
source: 'settings',
options: {
endpoint: 'http://127.0.0.1:9000', bucket: 'trek', accessKeyId: 'ak',
secretAccessKey: MASKED_SETTING_VALUE, region: 'us-east-1', keyPrefix: '', retries: 1, timeoutMs: 30000,
},
categories: ['covers'],
},
],
categories: {
files: { backend: 'uploads-local', source: 'default' },
journey: { backend: 'uploads-local', source: 'default' },
covers: { backend: 'off-box', source: 'settings' },
avatars: { backend: 'uploads-local', source: 'default' },
places: { backend: 'place-photos-local', source: 'default' },
'photos-google': { backend: 'place-photos-local', source: 'default' },
'photos-trek': { backend: 'uploads-local', source: 'default' },
backups: { backend: 'backups-local', source: 'default' },
},
health: { replicaFailures: [] },
seedFilePresent: false,
usage: null,
backfills: [],
migrations: [],
version: 5,
configError: null,
};
describe('settingsDocumentOf', () => {
it('FE-ADMIN-STORM-001: carries ONLY settings-sourced backends and categories (the PUT contract)', () => {
const doc = settingsDocumentOf(STATE);
expect(doc.backends.map((b) => b.name)).toEqual(['off-box']);
expect(doc.categories).toEqual({ covers: 'off-box' });
});
it('FE-ADMIN-STORM-001b: carries version from state (audit #7 — the PUT body\'s optimistic-concurrency check)', () => {
expect(settingsDocumentOf(STATE).version).toBe(5);
expect(settingsDocumentOf({ ...STATE, version: 12 }).version).toBe(12);
});
});
describe('remove pre-check helpers', () => {
const draft: StorageConfig = {
backends: [
{ name: 'nas', type: 'local', options: { root: '/mnt/nas' } },
{ name: 'mir', type: 'mirror', options: { primary: 'backups-local', replicas: ['nas'] } },
],
categories: { covers: 'nas', backups: 'mir' },
};
it('FE-ADMIN-STORM-002: lists categories pointing at a backend', () => {
expect(categoriesPointingAt(draft, 'nas')).toEqual(['covers']);
expect(categoriesPointingAt(draft, 'unused')).toEqual([]);
});
it('FE-ADMIN-STORM-003: lists mirrors referencing a backend (primary or replica)', () => {
expect(mirrorsReferencing(draft, 'nas')).toEqual(['mir']);
expect(mirrorsReferencing(draft, 'backups-local')).toEqual(['mir']);
expect(mirrorsReferencing(draft, 'unused')).toEqual([]);
});
});
describe('draft edits', () => {
const draft: StorageConfig = {
backends: [{ name: 'nas', type: 'local', options: { root: '/mnt/nas' } }],
categories: {},
};
it('FE-ADMIN-STORM-005: upsert replaces by name in place, appends new ones, never mutates', () => {
const edited = upsertBackend(draft, { name: 'nas', type: 'local', options: { root: '/mnt/nas2' } });
expect(edited.backends).toEqual([{ name: 'nas', type: 'local', options: { root: '/mnt/nas2' } }]);
const added = upsertBackend(draft, { name: 'other', type: 'local', options: { root: '/o' } });
expect(added.backends.map((b) => b.name)).toEqual(['nas', 'other']);
expect(draft.backends).toHaveLength(1); // untouched
});
it('FE-ADMIN-STORM-006: removeBackend filters by name', () => {
expect(removeBackend(draft, 'nas').backends).toEqual([]);
expect(removeBackend(draft, 'ghost').backends).toHaveLength(1);
});
});
// Hoisted to module scope (not just the mirror-fold describe below) so the
// usageByBackend suite can reuse the same mirrored fixture the brief's test
// snippet assumes is in scope.
const MIRRORED_STATE: StorageAdminState = {
...STATE,
backends: [
...STATE.backends,
{
name: 'mirror', type: 'mirror', source: 'settings',
options: { primary: 'backups-local', replicas: ['off-box'] }, categories: ['backups'],
},
{ name: 'backups-local', type: 'local', source: 'built-in', options: { root: '/data/backups' }, categories: [] },
],
categories: { ...STATE.categories, backups: { backend: 'mirror', source: 'settings' } },
};
const mirroredDraft = (): StorageConfig => settingsDocumentOf(MIRRORED_STATE);
describe('mirror fold/synthesize (replicas-on-primary)', () => {
it('FE-ADMIN-STORM-007: foldBackends hides the mirror, decorates primary and replica, unions categories', () => {
const { rows, degenerate } = foldBackends(MIRRORED_STATE, mirroredDraft());
expect(degenerate).toEqual([]);
expect(rows.map((r) => r.name)).not.toContain('mirror');
const primary = rows.find((r) => r.name === 'backups-local')!;
expect(primary.mirrorTargets).toEqual(['off-box']);
expect(primary.mirrorName).toBe('mirror');
expect(primary.categories).toContain('backups'); // via the mirror
const replica = rows.find((r) => r.name === 'off-box')!;
expect(replica.replicaOf).toEqual(['backups-local']);
expect(replica.categories).toContain('covers'); // its own direct assignment survives
});
it('FE-ADMIN-STORM-008: setMirrorTargets synthesizes a mirror and reroutes ALL effective categories of the primary, defaults included', () => {
const draft = settingsDocumentOf(STATE); // no mirror yet; backups is default → backups-local
const next = setMirrorTargets(STATE, draft, 'backups-local', ['off-box']);
const mirror = next.backends.find((b) => b.type === 'mirror')!;
expect(mirror.name).toBe('backups-local-mirror');
expect(mirror.options).toEqual({ primary: 'backups-local', replicas: ['off-box'] });
expect(next.categories.backups).toBe('backups-local-mirror'); // default-sourced category rewritten
expect(next.categories.covers).toBe('off-box'); // categories of OTHER backends untouched
});
it('FE-ADMIN-STORM-009: setMirrorTargets adopts a foreign-named mirror in place', () => {
const next = setMirrorTargets(MIRRORED_STATE, mirroredDraft(), 'backups-local', ['off-box', 'uploads-local']);
const mirrors = next.backends.filter((b) => b.type === 'mirror');
expect(mirrors).toHaveLength(1);
expect(mirrors[0]!.name).toBe('mirror'); // adopted, not renamed
expect(mirrors[0]!.options).toEqual({ primary: 'backups-local', replicas: ['off-box', 'uploads-local'] });
});
it('FE-ADMIN-STORM-010: empty targets dissolve the mirror and revert default-sourced categories', () => {
// Build the synthesized state locally: default backups routed via a fresh mirror…
const withMirror = setMirrorTargets(STATE, settingsDocumentOf(STATE), 'backups-local', ['off-box']);
// …then dissolve it: the backups entry must DROP (state says its default is backups-local).
const dissolved = setMirrorTargets(STATE, withMirror, 'backups-local', []);
expect(dissolved.backends.some((b) => b.type === 'mirror')).toBe(false);
expect(dissolved.categories).not.toHaveProperty('backups');
// Against MIRRORED_STATE (backups already settings-sourced at the mirror), dissolve re-points instead of dropping.
const repointed = setMirrorTargets(MIRRORED_STATE, mirroredDraft(), 'backups-local', []);
expect(repointed.categories.backups).toBe('backups-local');
});
it('FE-ADMIN-STORM-011: degenerate mirrors are classified, never hidden', () => {
const draft: StorageConfig = {
backends: [
{ name: 'off-box', type: 's3', options: STATE.backends[2]!.options } as StorageBackend,
{ name: 'm1', type: 'mirror', options: { primary: 'uploads-local', replicas: ['off-box'] } },
{ name: 'm2', type: 'mirror', options: { primary: 'uploads-local', replicas: ['off-box'] } },
{ name: 'm-env', type: 'mirror', options: { primary: 'place-photos-local', replicas: ['off-box'] } },
{ name: 'm-ghost', type: 'mirror', options: { primary: 'nope', replicas: [] } },
],
categories: {},
};
const { rows, degenerate } = foldBackends(STATE, draft);
expect(rows.find((r) => r.name === 'uploads-local')!.mirrorName).toBe('m1'); // first wins
expect(degenerate.map((d) => [d.backend.name, d.reason])).toEqual([
['m2', 'duplicate-mirror'],
['m-env', 'env-primary'],
['m-ghost', 'missing-primary'],
]);
});
it('FE-ADMIN-STORM-012: renameBackendRefs rewrites mirror primaries, replicas, and category entries', () => {
const draft: StorageConfig = {
backends: [
{ name: 'nas', type: 'local', options: { root: '/mnt/nas' } },
{ name: 'm', type: 'mirror', options: { primary: 'nas', replicas: ['nas'] } },
],
categories: { covers: 'nas' },
};
const renamed = renameBackendRefs(draft, 'nas', 'nas2');
const mirror = renamed.backends.find((b) => b.type === 'mirror')!;
expect(mirror.options).toEqual({ primary: 'nas2', replicas: ['nas2'] });
expect(renamed.categories.covers).toBe('nas2');
expect(renamed.backends[0]!.name).toBe('nas'); // the backend row itself is the caller's job
});
it('FE-ADMIN-STORM-013: removeBackendAndMirrors cascades; replicaOfPrimaries phrases in primary names', () => {
const draft = mirroredDraft();
expect(replicaOfPrimaries(draft, 'off-box')).toEqual(['backups-local']);
const removed = removeBackendAndMirrors(MIRRORED_STATE, draft, 'backups-local');
expect(removed.backends.some((b) => b.name === 'mirror')).toBe(false);
expect(removed.backends.some((b) => b.name === 'backups-local')).toBe(false);
});
it('FE-ADMIN-STORM-015: removing a replica strips it from remaining mirrors; a mirror stripped empty dissolves', () => {
// Single-replica mirror: stripping off-box empties it → dissolve + re-point
// (backups is settings-sourced at the mirror in MIRRORED_STATE, so it re-points, not drops).
const dissolved = removeBackendAndMirrors(MIRRORED_STATE, mirroredDraft(), 'off-box');
expect(dissolved.backends.some((b) => b.name === 'off-box')).toBe(false);
expect(dissolved.backends.some((b) => b.type === 'mirror')).toBe(false);
expect(dissolved.categories.backups).toBe('backups-local');
// Multi-replica mirror: stripping one target keeps the mirror alive with the rest.
const twoTargets = setMirrorTargets(MIRRORED_STATE, mirroredDraft(), 'backups-local', ['off-box', 'uploads-local']);
const survived = removeBackendAndMirrors(MIRRORED_STATE, twoTargets, 'off-box');
const mirror = survived.backends.find((b) => b.type === 'mirror')!;
expect(mirror.options).toEqual({ primary: 'backups-local', replicas: ['uploads-local'] });
expect(survived.categories.backups).toBe('mirror'); // still routed via the surviving mirror
});
it('FE-ADMIN-STORM-014: primaryNameOf maps mirror names (draft or state) to their primary; others pass through', () => {
expect(primaryNameOf(MIRRORED_STATE, mirroredDraft(), 'mirror')).toBe('backups-local');
expect(primaryNameOf(MIRRORED_STATE, { backends: [], categories: {} }, 'mirror')).toBe('backups-local'); // state fallback
expect(primaryNameOf(STATE, settingsDocumentOf(STATE), 'uploads-local')).toBe('uploads-local');
});
});
describe('replicaCandidates', () => {
it('FE-ADMIN-STORM-020: offers only rows that serve no category, never the row being edited', () => {
const { rows } = foldBackends(MIRRORED_STATE, mirroredDraft());
// uploads-local/place-photos-local serve categories directly;
// backups-local serves backups through the mirror it is the primary of;
// off-box carries a direct `covers` assignment. Nothing is offerable.
expect(replicaCandidates(rows, 'backups-local')).toEqual([]);
});
it('FE-ADMIN-STORM-021: keeps an already-selected target listed so it can be unchecked', () => {
const { rows } = foldBackends(MIRRORED_STATE, mirroredDraft());
expect(replicaCandidates(rows, 'backups-local', ['off-box'])).toEqual(['off-box']);
});
it('FE-ADMIN-STORM-022: an unassigned row is offered', () => {
const state: StorageAdminState = {
...MIRRORED_STATE,
backends: [
...MIRRORED_STATE.backends,
{ name: 'cold-store', type: 'local', source: 'settings', options: { root: '/data/cold' }, categories: [] },
],
};
const { rows } = foldBackends(state, settingsDocumentOf(state));
expect(replicaCandidates(rows, 'backups-local')).toEqual(['cold-store']);
});
});
describe('usageByBackend', () => {
it('FE-ADMIN-STORM-016: sums categories onto their effective backend rows, mirrors onto the primary, legacy onto uploads-local', () => {
const usage = {
computedAt: 1,
categories: Object.fromEntries(
STORAGE_CATEGORIES.map((c) => [c, { objects: 1, bytes: 10 }]),
) as Record<StorageCategory, { objects: number; bytes: number }>,
legacyPhotos: { objects: 5, bytes: 50 },
};
const state = { ...MIRRORED_STATE, usage };
const sums = usageByBackend(state, mirroredDraft())!;
// backups routes via the mirror → attributed to backups-local (the primary row).
expect(sums['backups-local']).toEqual({ objects: 1, bytes: 10 });
// uploads-local serves 4 direct categories (files, journey, avatars, photos-trek —
// this fixture's `places` and `photos-google` default to the env-sourced
// place-photos-local backend, not uploads-local) + legacy photos.
expect(sums['uploads-local']).toEqual({ objects: 4 + 5, bytes: 40 + 50 });
// places + photos-google both default to place-photos-local.
expect(sums['place-photos-local']).toEqual({ objects: 2, bytes: 20 });
// covers routes to off-box directly.
expect(sums['off-box']).toEqual({ objects: 1, bytes: 10 });
});
it('FE-ADMIN-STORM-017: null usage yields null', () => {
expect(usageByBackend({ ...STATE, usage: null }, settingsDocumentOf(STATE))).toBeNull();
});
});
describe('computeMigrationCandidates', () => {
it('FE-ADMIN-STORM-018: lists changed categories with usage counts; zero-object ones are excluded; null usage keeps them with null counts', () => {
const state: StorageAdminState = {
...STATE,
usage: {
computedAt: 1,
categories: {
files: { objects: 3, bytes: 300 },
journey: { objects: 0, bytes: 0 },
} as Record<StorageCategory, { objects: number; bytes: number }>,
legacyPhotos: { objects: 0, bytes: 0 },
},
};
const base = settingsDocumentOf(state);
const draft: StorageConfig = {
...base,
categories: { ...base.categories, files: 'off-box', journey: 'off-box', avatars: 'off-box' },
};
const candidates = computeMigrationCandidates(draft, state);
expect(candidates).toEqual([
{ category: 'files', from: 'uploads-local', to: 'off-box', toWire: 'off-box', objects: 3, bytes: 300 },
{ category: 'avatars', from: 'uploads-local', to: 'off-box', toWire: 'off-box', objects: null, bytes: null },
]);
});
it('FE-ADMIN-STORM-018b: reassigning onto a mirrored primary yields toWire = the mirror wire name while `to` stays the display primary', () => {
const state: StorageAdminState = {
...STATE,
backends: [
...STATE.backends,
{ name: 'nas', type: 'local', source: 'settings', options: { root: '/mnt/nas' }, categories: [] },
{
name: 'off-box-mirror',
type: 'mirror',
source: 'settings',
options: { primary: 'off-box', replicas: ['nas'] },
categories: [],
},
],
usage: {
computedAt: 1,
categories: { files: { objects: 3, bytes: 300 } } as Record<StorageCategory, { objects: number; bytes: number }>,
legacyPhotos: { objects: 0, bytes: 0 },
},
};
const base = settingsDocumentOf(state);
// Reassign 'files' to the mirrored primary's WIRE name — the same value
// setCategory() writes into the draft when the operator picks a mirrored
// primary in the UI (adoptedMirrorFor(draft, 'off-box')?.name).
const draft: StorageConfig = { ...base, categories: { ...base.categories, files: 'off-box-mirror' } };
const candidates = computeMigrationCandidates(draft, state);
expect(candidates).toEqual([
{ category: 'files', from: 'uploads-local', to: 'off-box', toWire: 'off-box-mirror', objects: 3, bytes: 300 },
]);
});
it('FE-ADMIN-STORM-019: stripCategories restores the saved assignment for exactly the named categories', () => {
const base = settingsDocumentOf(STATE);
const draft: StorageConfig = {
...base,
categories: { ...base.categories, files: 'off-box', covers: 'uploads-local' },
};
const strippedFiles = stripCategories(draft, STATE, ['files']);
expect(strippedFiles.categories.files).toBeUndefined(); // default-sourced: saved value is "no override"
expect(strippedFiles.categories.covers).toBe('uploads-local'); // untouched — not in the strip list
const strippedCovers = stripCategories(draft, STATE, ['covers']);
expect(strippedCovers.categories.covers).toBe('off-box'); // settings-sourced saved value restored explicitly
expect(strippedCovers.categories.files).toBe('off-box'); // untouched — not in the strip list
});
});
describe('mirrorProbeTargets', () => {
it('FE-ADMIN-STORM-020: a non-mirror candidate passes through unchanged', () => {
const local: StorageBackend = { name: 'uploads-local', type: 'local', options: { root: '/data/uploads' } };
expect(mirrorProbeTargets(settingsDocumentOf(STATE), STATE, local)).toEqual([local]);
});
it('FE-ADMIN-STORM-021: expands primary + replicas — draft override wins per name, falling back to state for names the draft never touched', () => {
const mirror: StorageBackend = {
name: 'mirror', type: 'mirror',
options: { primary: 'uploads-local', replicas: ['off-box'] },
};
// The draft carries an EDITED off-box (unsaved endpoint change) but never
// touches uploads-local (a built-in, never draft-owned).
const editedOffBox = {
name: 'off-box', type: 's3',
options: { ...(STATE.backends[2]!.options as Record<string, unknown>), endpoint: 'http://edited:9000' },
} as StorageBackend;
const draft: StorageConfig = { backends: [editedOffBox, mirror], categories: {} };
const targets = mirrorProbeTargets(draft, STATE, mirror);
expect(targets.map((t) => t.name)).toEqual(['uploads-local', 'off-box']);
// Primary: no draft entry — resolved from state (the live built-in).
expect(targets[0]).toEqual({ name: 'uploads-local', type: 'local', options: { root: '/data/uploads' } });
// Replica: draft override wins — the unsaved edit, not the saved options.
expect((targets[1]!.options as Record<string, unknown>).endpoint).toBe('http://edited:9000');
});
it('FE-ADMIN-STORM-022: a name resolving to neither draft nor state is dropped, not thrown', () => {
const mirror: StorageBackend = {
name: 'mirror', type: 'mirror',
options: { primary: 'uploads-local', replicas: ['ghost'] },
};
const targets = mirrorProbeTargets(settingsDocumentOf(STATE), STATE, mirror);
expect(targets.map((t) => t.name)).toEqual(['uploads-local']);
});
});
@@ -0,0 +1,440 @@
import {
STORAGE_CATEGORIES,
type StorageAdminState,
type StorageBackend,
type StorageCategory,
type StorageConfig,
type StorageConfigPut,
} from '@trek/shared'
/** One entry of the effective world (GET state), as the panels render it. */
export type StateBackend = StorageAdminState['backends'][number]
/**
* The settings-owned document the PUT carries: settings-sourced backends and
* settings-sourced category assignments ONLY — built-ins and env backends are
* never in the body unless the operator overrides one by name (in which case
* the override already reports source 'settings'). Carries `version` from
* the state it was built at (audit #7 — optimistic concurrency): the hook
* re-attaches this same version on every subsequent local edit, so a PUT
* always submits the version the operator's form was loaded at, not
* whatever the server currently holds.
*/
export function settingsDocumentOf(state: StorageAdminState): StorageConfigPut {
return {
backends: state.backends.filter((b) => b.source === 'settings').map(asWireBackend),
categories: Object.fromEntries(
Object.entries(state.categories)
.filter(([, entry]) => entry.source === 'settings')
.map(([category, entry]) => [category, entry.backend]),
) as StorageConfig['categories'],
version: state.version,
}
}
/**
* State options are already schema-shaped (the wire contract guarantees it;
* secret fields carry the mask, which the server unmasks by name on PUT/test) —
* the cast re-attaches the discriminated-union type the loose state record dropped.
*/
export function asWireBackend(backend: Pick<StateBackend, 'name' | 'type' | 'options'>): StorageBackend {
return { name: backend.name, type: backend.type, options: backend.options } as StorageBackend
}
/** Draft categories pointing at a backend name (the friendly remove pre-check). */
export function categoriesPointingAt(draft: StorageConfig, name: string): StorageCategory[] {
return (Object.entries(draft.categories) as Array<[StorageCategory, string]>)
.filter(([, backend]) => backend === name)
.map(([category]) => category)
}
/** Draft mirrors referencing a backend name as primary or replica (remove pre-check). */
export function mirrorsReferencing(draft: StorageConfig, name: string): string[] {
return draft.backends
.filter((b) => b.type === 'mirror' && (b.options.primary === name || b.options.replicas.includes(name)))
.map((b) => b.name)
}
/** Replace by name in place, or append. Never mutates. */
export function upsertBackend(draft: StorageConfig, backend: StorageBackend): StorageConfig {
const exists = draft.backends.some((b) => b.name === backend.name)
return {
...draft,
backends: exists
? draft.backends.map((b) => (b.name === backend.name ? backend : b))
: [...draft.backends, backend],
}
}
export function removeBackend(draft: StorageConfig, name: string): StorageConfig {
return { ...draft, backends: draft.backends.filter((b) => b.name !== name) }
}
// ── Mirror fold/synthesize (replicas-on-primary spec) ─────────────────────────
// The draft stays the wire document; folding is a view rule, synthesis a set
// of pure draft operations. Mirror NAMES never reach the UI — every helper
// that renders phrases mirrors as their primary.
type MirrorBackend = Extract<StorageBackend, { type: 'mirror' }>
const isMirror = (b: StorageBackend): b is MirrorBackend => b.type === 'mirror'
export interface FoldedBackendRow {
name: string
type: 'local' | 's3'
source: 'built-in' | 'env' | 'settings'
backend: StorageBackend
/** Direct assignments ∪ assignments routed via the adopted mirror. */
categories: StorageCategory[]
/** Adopted mirror's replicas — [] when unmirrored. */
mirrorTargets: string[]
/** Adopted mirror's wire name — needed by draft ops, hidden from the UI. */
mirrorName: string | null
/** Primaries whose adopted mirror lists this backend as a replica. */
replicaOf: string[]
}
export interface DegenerateMirror {
backend: StorageBackend
reason: 'duplicate-mirror' | 'env-primary' | 'missing-primary'
categories: StorageCategory[]
}
/**
* Re-fetchable content — replicating it is usually wasteful. photos-google is
* the Google photo cache (place-photo-cache.service.ts); places is mostly
* provider-derived imagery.
*/
export const CACHE_CATEGORIES: readonly StorageCategory[] = ['photos-google', 'places']
/** The configurable categories resolved draft-over-state (draft entries win; defaults fill the rest). */
export function effectiveCategoryMap(state: StorageAdminState, draft: StorageConfig): Record<StorageCategory, string> {
const map = {} as Record<StorageCategory, string>
for (const category of STORAGE_CATEGORIES) {
// The admin state's category record is exhaustive by schema contract.
map[category] = draft.categories[category] ?? state.categories[category]!.backend
}
return map
}
/** First draft mirror wrapping the named primary — the one the UI manages. */
export function adoptedMirrorFor(draft: StorageConfig, primaryName: string): MirrorBackend | undefined {
return draft.backends.filter(isMirror).find((m) => m.options.primary === primaryName)
}
/**
* Client-side mirror expansion for the test probe (audit fix — the server's
* `testBackend` resolves a mirror candidate's primary/replicas against its
* OWN live snapshot by name, so probing a mirrored row with unsaved draft
* edits to those targets would silently test the SAVED options instead of
* what's on the form). Draft overrides win per name, falling back to state
* (built-in/env); a name that resolves to neither is dropped — that
* primary/replica no longer exists, and the row already renders as
* degenerate in that case.
*/
export function mirrorProbeTargets(
draft: StorageConfig,
state: StorageAdminState,
mirror: StorageBackend,
): StorageBackend[] {
if (mirror.type !== 'mirror') return [mirror]
const names = [mirror.options.primary, ...mirror.options.replicas]
return names
.map((name): StorageBackend | null => {
const draftMatch = draft.backends.find((b) => b.name === name)
if (draftMatch) return draftMatch
const stateMatch = state.backends.find((b) => b.name === name)
return stateMatch ? asWireBackend(stateMatch) : null
})
.filter((b): b is StorageBackend => b !== null)
}
export function foldBackends(
state: StorageAdminState,
draft: StorageConfig,
): { rows: FoldedBackendRow[]; degenerate: DegenerateMirror[] } {
const draftNames = draft.backends.map((b) => b.name)
const effective = effectiveCategoryMap(state, draft)
const categoriesOf = (name: string): StorageCategory[] =>
STORAGE_CATEGORIES.filter((category) => effective[category] === name)
// Base (non-mirror) rows, slice-2 precedence: built-ins/env from the state
// unless a draft row overrides the name; settings rows from the draft.
const base: Array<Pick<FoldedBackendRow, 'name' | 'type' | 'source' | 'backend'>> = []
for (const b of state.backends) {
if (b.type === 'mirror') continue // mirrors fold below; the draft owns them
if (draftNames.includes(b.name)) continue
if (b.source === 'settings') continue // removed from the draft, pending save
base.push({ name: b.name, type: b.type, source: b.source, backend: asWireBackend(b) })
}
for (const b of draft.backends) {
if (isMirror(b)) continue
base.push({ name: b.name, type: b.type, source: 'settings', backend: b })
}
// Adoption: the first mirror per editable primary; the rest are degenerate —
// rendered unfolded with a reason, never silently dropped.
const adopted = new Map<string, MirrorBackend>()
const degenerate: DegenerateMirror[] = []
for (const mirror of draft.backends.filter(isMirror)) {
const primary = base.find((row) => row.name === mirror.options.primary)
if (!primary) degenerate.push({ backend: mirror, reason: 'missing-primary', categories: categoriesOf(mirror.name) })
else if (primary.source === 'env') degenerate.push({ backend: mirror, reason: 'env-primary', categories: categoriesOf(mirror.name) })
else if (adopted.has(primary.name)) degenerate.push({ backend: mirror, reason: 'duplicate-mirror', categories: categoriesOf(mirror.name) })
else adopted.set(primary.name, mirror)
}
const rows = base.map((row): FoldedBackendRow => {
const mirror = adopted.get(row.name)
return {
...row,
categories: [...categoriesOf(row.name), ...(mirror ? categoriesOf(mirror.name) : [])],
mirrorTargets: mirror ? [...mirror.options.replicas] : [],
mirrorName: mirror?.name ?? null,
replicaOf: [...adopted.values()]
.filter((m) => m.options.replicas.includes(row.name))
.map((m) => m.options.primary),
}
})
return { rows, degenerate }
}
/**
* The rows offerable as mirror replicas (the "Mirror targets" picker).
*
* A backend that already serves a category — directly, or as the primary of a
* mirror a category routes to, which is exactly what `row.categories` unions —
* can never also be a replica: the mirror's sync sweep makes each replica
* MATCH the primary, deleting whatever the primary doesn't hold, and `backups`
* sweeps the replica's entire root. The server refuses such a config outright
* (storage-registry.service.ts, assertNoSharedReplicas); the picker simply
* never offers one, so the operator can't walk into the refusal.
*
* Targets already selected on this mirror stay listed regardless — they must
* remain visible to be unchecked (and a config that predates this rule is
* repaired here, not hidden).
*/
export function replicaCandidates(
rows: FoldedBackendRow[],
selfName: string | null,
currentTargets: readonly string[] = [],
): string[] {
return rows
.filter((row) => row.name !== selfName)
.filter((row) => row.categories.length === 0 || currentTargets.includes(row.name))
.map((row) => row.name)
}
function uniqueMirrorName(state: StorageAdminState, draft: StorageConfig, primaryName: string): string {
const taken = new Set([...draft.backends.map((b) => b.name), ...state.backends.map((b) => b.name)])
const base = `${primaryName}-mirror`
if (!taken.has(base)) return base
for (let n = 2; ; n++) {
if (!taken.has(`${base}-${n}`)) return `${base}-${n}`
}
}
/**
* targets non-empty: update the adopted mirror in place (or synthesize one)
* and route every category that effectively resolves to the primary —
* default-sourced included — through it. targets empty: dissolve, re-pointing
* the mirror's categories at the primary and dropping entries that revert to
* their built-in default.
*/
export function setMirrorTargets(
state: StorageAdminState,
draft: StorageConfig,
primaryName: string,
targets: string[],
): StorageConfig {
const adopted = adoptedMirrorFor(draft, primaryName)
if (targets.length === 0) {
if (!adopted) return draft
const categories = { ...draft.categories }
for (const category of STORAGE_CATEGORIES) {
if (categories[category] !== adopted.name) continue
const stateEntry = state.categories[category]! // exhaustive by schema contract
if (stateEntry.source === 'default' && stateEntry.backend === primaryName) delete categories[category]
else categories[category] = primaryName
}
return { backends: draft.backends.filter((b) => b.name !== adopted.name), categories }
}
const effective = effectiveCategoryMap(state, draft)
const mirror: StorageBackend = {
name: adopted?.name ?? uniqueMirrorName(state, draft, primaryName),
type: 'mirror',
options: { primary: primaryName, replicas: [...targets] },
}
const backends = adopted
? draft.backends.map((b) => (b.name === adopted.name ? mirror : b))
: [...draft.backends, mirror]
const categories = { ...draft.categories }
for (const category of STORAGE_CATEGORIES) {
if (effective[category] === primaryName) categories[category] = mirror.name
}
return { backends, categories }
}
/** Rewrite every reference to a renamed backend: mirror primaries, replicas, category entries. */
export function renameBackendRefs(draft: StorageConfig, oldName: string, newName: string): StorageConfig {
return {
backends: draft.backends.map((b) => {
if (!isMirror(b)) return b
return {
...b,
options: {
primary: b.options.primary === oldName ? newName : b.options.primary,
replicas: b.options.replicas.map((r) => (r === oldName ? newName : r)),
},
}
}),
categories: Object.fromEntries(
Object.entries(draft.categories).map(([category, backend]) => [category, backend === oldName ? newName : backend]),
) as StorageConfig['categories'],
}
}
/**
* Remove a backend, every mirror wrapping it as primary, and every replica
* reference to it in the remaining mirrors — a mirror stripped of its last
* replica dissolves (with the same category re-pointing as an explicit
* dissolve), so the draft never carries a phantom reference the composer
* cannot render or repair.
*/
export function removeBackendAndMirrors(
state: StorageAdminState,
draft: StorageConfig,
name: string,
): StorageConfig {
const stripped = draft.backends
.filter((b) => b.name !== name && !(isMirror(b) && b.options.primary === name))
.map((b) =>
isMirror(b) && b.options.replicas.includes(name)
? { ...b, options: { ...b.options, replicas: b.options.replicas.filter((r) => r !== name) } }
: b,
)
let next: StorageConfig = { ...draft, backends: stripped }
for (const mirror of stripped.filter(isMirror)) {
if (mirror.options.replicas.length > 0) continue
if (adoptedMirrorFor(next, mirror.options.primary)?.name === mirror.name) {
next = setMirrorTargets(state, next, mirror.options.primary, [])
} else {
// A degenerate duplicate stripped empty: drop it and re-point its categories.
const categories = { ...next.categories }
for (const category of STORAGE_CATEGORIES) {
if (categories[category] === mirror.name) categories[category] = mirror.options.primary
}
next = { backends: next.backends.filter((b) => b.name !== mirror.name), categories }
}
}
return next
}
/** Primaries whose mirrors use the named backend as a replica — pre-check copy stays in primary names. */
export function replicaOfPrimaries(draft: StorageConfig, name: string): string[] {
return draft.backends
.filter(isMirror)
.filter((m) => m.options.replicas.includes(name))
.map((m) => m.options.primary)
}
/** Map a mirror name (draft first, then state) to its primary; anything else passes through. */
export function primaryNameOf(state: StorageAdminState, draft: StorageConfig, backendName: string): string {
const draftMirror = draft.backends.filter(isMirror).find((m) => m.name === backendName)
if (draftMirror) return draftMirror.options.primary
const stateMirror = state.backends.find((b) => b.name === backendName && b.type === 'mirror')
if (stateMirror) return String((stateMirror.options as Record<string, unknown>).primary)
return backendName
}
/**
* Usage sums per VISIBLE backend row: each category's numbers land on the row
* that serves it (a mirror-routed category on the mirror's PRIMARY row), and
* the legacy photos directory lands on uploads-local — the only backend the
* served-legacy `photos` category ever resolves to.
*/
export function usageByBackend(
state: StorageAdminState,
draft: StorageConfig,
): Record<string, { objects: number; bytes: number }> | null {
if (!state.usage) return null
const sums: Record<string, { objects: number; bytes: number }> = {}
const add = (row: string, entry: { objects: number; bytes: number }) => {
const current = (sums[row] ??= { objects: 0, bytes: 0 })
current.objects += entry.objects
current.bytes += entry.bytes
}
const effective = effectiveCategoryMap(state, draft)
for (const category of STORAGE_CATEGORIES) {
const entry = state.usage.categories[category]
if (entry) add(primaryNameOf(state, draft, effective[category]), entry)
}
// The served-legacy `photos` directory always resolves to uploads-local —
// hardcoded, not derived, because it is not one of the configurable categories.
add(primaryNameOf(state, draft, 'uploads-local'), state.usage.legacyPhotos)
return sums
}
// ── Category migration (copy → flip → delta sweep) ────────────────────────────
/** One reassigned category the operator might want to move existing objects for. */
export interface MigrationCandidate {
category: StorageCategory
from: string
/** Display primary — never a hidden mirror name; used by the prompt lines and comparisons. */
to: string
/**
* The RAW effective draft backend name for this category — possibly a
* mirror's wire name when `to` is a mirrored primary. The migration POST
* must carry this, not `to`: posting the bare primary would silently drop
* replication for a category that was just routed onto a mirror.
*/
toWire: string
/** null when usage was never scanned — the prompt still fires, with an unknown-size line. */
objects: number | null
bytes: number | null
}
/**
* Every category whose effective (draft-over-state) PRIMARY backend differs
* from what's currently saved — normalized through `primaryNameOf` so simply
* wrapping (or unwrapping) a mirror around the same primary never counts as
* a candidate; mirroring is the existing sync/backfill flow, not a category
* migration. Zero-object categories (usage present, 0 objects) are excluded —
* nothing to move; null usage (never scanned) keeps the category as a
* candidate with null counts, per spec ("unknown still prompts").
*/
export function computeMigrationCandidates(draft: StorageConfig, state: StorageAdminState): MigrationCandidate[] {
const effective = effectiveCategoryMap(state, draft)
const candidates: MigrationCandidate[] = []
for (const category of STORAGE_CATEGORIES) {
const from = primaryNameOf(state, draft, state.categories[category]!.backend) // exhaustive by schema contract
const toWire = effective[category]
const to = primaryNameOf(state, draft, toWire)
if (from === to) continue
const usage = state.usage?.categories[category]
candidates.push({ category, from, to, toWire, objects: usage ? usage.objects : null, bytes: usage ? usage.bytes : null })
}
return candidates.filter((c) => c.objects !== 0)
}
/**
* Revert exactly the named categories in `draft` back to what's currently
* saved in `saved` — same default/settings convention as the rest of this
* file (drop the override when the saved entry is default-sourced, write the
* explicit backend name otherwise). Every other draft edit is untouched.
*/
export function stripCategories(
draft: StorageConfig,
saved: StorageAdminState,
categories: StorageCategory[],
): StorageConfig {
const next = { ...draft.categories }
for (const category of categories) {
const savedEntry = saved.categories[category]! // exhaustive by schema contract
if (savedEntry.source === 'default') delete next[category]
else next[category] = savedEntry.backend
}
return { ...draft, categories: next }
}
@@ -0,0 +1,349 @@
import { useCallback, useEffect, useRef, useState } from 'react'
import type { StorageAdminState, StorageBackend, StorageConfig, StorageConfigPut, StorageTestResponse } from '@trek/shared'
import { adminApi } from '../../../api/client'
import { getApiErrorMessage, type ApiError } from '../../../types'
import { settingsDocumentOf } from './storageModel'
export type StorageTestResults = Record<string, StorageTestResponse | 'running'>
/** 50ms under vitest so poll tests run on real timers (MSW + fake timers don't mix). */
export const BACKFILL_POLL_MS = import.meta.env.MODE === 'test' ? 50 : 5000
/** True for a 409 response — the version this draft was built at is no longer current (audit #7). */
function isConflictError(err: unknown): boolean {
return typeof err === 'object' && err !== null && 'response' in err && (err as ApiError).response?.status === 409
}
export interface StorageAdmin {
state: StorageAdminState | null
/** The settings-owned document (the PUT body), with local edits layered in. Carries the version the draft was built at. */
draft: StorageConfigPut | null
dirty: boolean
loading: boolean
loadError: string | null
/** Server 400s/409s land here VERBATIM (400) or as the distinct conflict copy (409) — long registry messages outlive a toast. */
saveError: string | null
/** True while `saveError` is the 409 copy — the panels render the discard-and-reload escape next to it. */
saveConflict: boolean
saving: boolean
testResults: StorageTestResults
setDraft: (next: StorageConfig) => void
/** PUTs `overrideDraft ?? draft` — an override lets a caller strip categories from the wire body without staging them (and marking dirty) via setDraft. */
save: (overrideDraft?: StorageConfig) => Promise<boolean>
test: (backend: StorageBackend) => Promise<void>
/**
* Probe each target individually (client-expanded mirror primary +
* replicas — see `mirrorProbeTargets`) and merge the results into one
* `StorageTestResponse` under `key` (the row's mirror wire name). Never
* sends the mirror stub itself — each call probes a concrete backend.
*/
testMirror: (key: string, targets: StorageBackend[]) => Promise<void>
/** null on success; the verbatim server error otherwise. */
startBackfill: (mirrorName: string) => Promise<string | null>
cancelBackfill: (mirrorName: string) => Promise<string | null>
startMigration: (category: string, to: string) => Promise<string | null>
cancelMigration: (category: string) => Promise<string | null>
refreshStats: () => Promise<string | null>
/** Re-GETs admin state (poll-safe — see the implementation comment below). Exposed for the panel's failed-queue recovery path. */
refreshState: () => Promise<void>
/**
* The 409's only exit: throw the local edits away and reload the saved
* settings (fresh GET → fresh draft → fresh version), clearing dirty and
* saveError. See the implementation comment for why refreshState can't do
* this itself.
*/
discardDraft: () => Promise<void>
}
/**
* Shared by the desktop and phone storage panels (DefaultUserSettingsTab
* model: self-contained, adminApi directly, deliberately no offline core —
* hoster-level config is online-only).
*/
export function useStorageAdmin(genericError: string, conflictError: string): StorageAdmin {
const [state, setState] = useState<StorageAdminState | null>(null)
const [draft, setDraftState] = useState<StorageConfigPut | null>(null)
const [dirty, setDirty] = useState(false)
const [loading, setLoading] = useState(true)
const [loadError, setLoadError] = useState<string | null>(null)
const [saveError, setSaveError] = useState<string | null>(null)
const [saveConflict, setSaveConflict] = useState(false)
const [saving, setSaving] = useState(false)
const [testResults, setTestResults] = useState<StorageTestResults>({})
// Read imperatively from the poll effect/refreshState so a stale closure
// never re-derives the draft out from under an in-flight edit.
const dirtyRef = useRef(dirty)
useEffect(() => {
dirtyRef.current = dirty
}, [dirty])
// Read imperatively at refreshState's resolve time so a poll GET that was
// in flight when a save started never applies after the save's own state.
const savingRef = useRef(saving)
useEffect(() => {
savingRef.current = saving
}, [saving])
// Bumped by every deliberate state application (initial load, save
// success). A refreshState() call captures the seq before its GET and
// drops the response if the seq moved on — i.e. a save applied its own
// fresher state — while that GET was in flight.
const stateSeq = useRef(0)
const applyState = useCallback((next: StorageAdminState) => {
stateSeq.current += 1
setState(next)
setDraftState(settingsDocumentOf(next))
setDirty(false)
setSaveError(null)
setSaveConflict(false)
}, [])
useEffect(() => {
let cancelled = false
adminApi
.getStorage()
.then((data) => {
if (!cancelled) applyState(data)
})
.catch((err: unknown) => {
if (!cancelled) setLoadError(getApiErrorMessage(err, genericError))
})
.finally(() => {
if (!cancelled) setLoading(false)
})
return () => {
cancelled = true
}
}, [applyState, genericError])
// Re-attaches the CURRENT draft's version onto whatever shape the caller
// hands in (the storageModel edit helpers are version-blind — they operate
// on plain StorageConfig — so this is the one place that keeps `version`
// pinned to what the operator's form was loaded at, no matter how many
// local edits run in between).
const setDraft = useCallback((next: StorageConfig) => {
setDraftState((prev) => ({ ...next, version: prev?.version ?? 0 }))
setDirty(true)
}, [])
// Poll-safe refresh: replaces `state` unconditionally, but only re-derives
// `draft` when the operator has no unsaved edits in flight — never touches
// `dirty`/`saveError`, so a running poll cannot clobber a dirty draft or
// mask a pending save error.
//
// Race guard: a poll GET issued before a save's PUT resolves can resolve
// AFTER the save applied its own (fresher) response, overwriting it with
// the pre-save world. Capture the state-application sequence number before
// the await and drop the response if it moved on, or if a save is
// in-flight/just landed (read via a ref so it's current at resolve time).
//
// Declared before `save` (which calls it on a 409) — a hook's `const`s
// must exist before another callback's dependency array references them.
const refreshState = useCallback(async (): Promise<void> => {
const seq = stateSeq.current
const next = await adminApi.getStorage()
if (stateSeq.current !== seq || savingRef.current) return
setState(next)
if (!dirtyRef.current) setDraftState(settingsDocumentOf(next))
}, [])
const save = useCallback(
async (overrideDraft?: StorageConfig): Promise<boolean> => {
if (!draft) return false
// Always the DRAFT's own version, even for an overrideDraft (a variant
// of the same draft with categories stripped, per stripCategories) —
// both share the same version basis: what the operator's form was
// loaded/refreshed at, never re-read from `state` here (that would
// defeat the whole check — see setDraft's re-attach comment above).
const body: StorageConfigPut = { ...(overrideDraft ?? draft), version: draft.version }
setSaving(true)
setSaveError(null)
setSaveConflict(false)
try {
// The response is the fresh effective world, never the request echo.
applyState(await adminApi.updateStorage(body))
return true
} catch (err: unknown) {
if (isConflictError(err)) {
// Something else (another admin's save, or a category migration's
// flip) wrote since this draft's version was captured. Refresh
// `state` so the operator sees the fresh world, but the dirty
// draft itself is left untouched for review (dirty is never
// cleared here) — refreshState() already respects dirtyRef.
setSaveError(conflictError)
setSaveConflict(true)
void refreshState().catch(() => {})
} else {
setSaveError(getApiErrorMessage(err, genericError))
}
return false
} finally {
setSaving(false)
}
},
[draft, applyState, genericError, conflictError, refreshState],
)
/**
* Conflict recovery (final-review fix): after a 409 the dirty draft keeps
* the version it was built at. refreshState deliberately leaves a dirty
* draft's version alone — bumping it there would resurrect audit #7, where
* the background poll floats a mid-edit draft past a migration flip it
* never carried — so nothing else can ever move it, and every retry 409s
* again. The operator's way out is to say so explicitly: discard the local
* edits, reload the saved settings, and start over from the fresh version.
* A fresh GET (not the possibly-dropped post-409 refresh) guarantees that
* version is current, and applyState clears dirty/saveError with it.
*/
const discardDraft = useCallback(async (): Promise<void> => {
try {
applyState(await adminApi.getStorage())
} catch (err: unknown) {
setSaveError(getApiErrorMessage(err, genericError))
setSaveConflict(false)
}
}, [applyState, genericError])
const test = useCallback(
async (backend: StorageBackend): Promise<void> => {
setTestResults((prev) => ({ ...prev, [backend.name]: 'running' }))
try {
const result = await adminApi.testStorageBackend(backend)
setTestResults((prev) => ({ ...prev, [backend.name]: result }))
} catch (err: unknown) {
const error = getApiErrorMessage(err, genericError)
setTestResults((prev) => ({
...prev,
[backend.name]: { ok: false, targets: [{ name: backend.name, ok: false, error }] },
}))
}
},
[genericError],
)
const testMirror = useCallback(
async (key: string, targets: StorageBackend[]): Promise<void> => {
setTestResults((prev) => ({ ...prev, [key]: 'running' }))
const settled = await Promise.all(
targets.map(async (target) => {
try {
const result = await adminApi.testStorageBackend(target)
return result.targets[0] ?? { name: target.name, ok: result.ok }
} catch (err: unknown) {
return { name: target.name, ok: false, error: getApiErrorMessage(err, genericError) }
}
}),
)
setTestResults((prev) => ({ ...prev, [key]: { ok: settled.every((r) => r.ok), targets: settled } }))
},
[genericError],
)
// While any backfill or category migration is running, poll GET state so
// progress/counts advance without the operator refreshing the page.
useEffect(() => {
if (
!state?.backfills.some((b) => b.status === 'running') &&
!state?.migrations.some((m) => m.status === 'running')
)
return
const iv = setInterval(() => {
// A transient poll failure is retried on the next tick — it must never
// surface as a toast or interrupt the operator's in-progress edit.
void refreshState().catch(() => {})
}, BACKFILL_POLL_MS)
return () => clearInterval(iv)
}, [state, refreshState])
const startBackfill = useCallback(
async (mirrorName: string): Promise<string | null> => {
try {
await adminApi.startStorageBackfill(mirrorName)
void refreshState().catch(() => {})
return null
} catch (err: unknown) {
return getApiErrorMessage(err, genericError)
}
},
[refreshState, genericError],
)
const cancelBackfill = useCallback(
async (mirrorName: string): Promise<string | null> => {
try {
await adminApi.cancelStorageBackfill(mirrorName)
void refreshState().catch(() => {})
return null
} catch (err: unknown) {
return getApiErrorMessage(err, genericError)
}
},
[refreshState, genericError],
)
const startMigration = useCallback(
async (category: string, to: string): Promise<string | null> => {
try {
await adminApi.startStorageMigration(category, to)
// Awaited (unlike cancelMigration/startBackfill/cancelBackfill): the
// queue effect's in-flight lock releases as soon as this promise
// resolves, so admin.state must already reflect the just-started
// migration by then — otherwise the next queued candidate can
// dequeue and POST before the server has confirmed this one.
await refreshState().catch(() => {})
return null
} catch (err: unknown) {
return getApiErrorMessage(err, genericError)
}
},
[refreshState, genericError],
)
const cancelMigration = useCallback(
async (category: string): Promise<string | null> => {
try {
await adminApi.cancelStorageMigration(category)
void refreshState().catch(() => {})
return null
} catch (err: unknown) {
return getApiErrorMessage(err, genericError)
}
},
[refreshState, genericError],
)
const refreshStats = useCallback(async (): Promise<string | null> => {
try {
const usage = await adminApi.refreshStorageStats()
setState((prev) => (prev ? { ...prev, usage } : prev))
return null
} catch (err: unknown) {
return getApiErrorMessage(err, genericError)
}
}, [genericError])
return {
state,
draft,
dirty,
loading,
loadError,
saveError,
saveConflict,
saving,
testResults,
setDraft,
save,
test,
testMirror,
startBackfill,
cancelBackfill,
startMigration,
cancelMigration,
refreshStats,
refreshState,
discardDraft,
}
}
@@ -10,6 +10,7 @@ import {
Database,
FileText,
Github,
HardDrive,
Map,
Plug,
Puzzle,
@@ -39,6 +40,7 @@ import MAdminPackingTemplateManager from './MAdminPackingTemplateManager'
import MAdminNotificationsSection from './MAdminNotificationsSection'
import MAdminSettingsSection from './MAdminSettingsSection'
import MAdminSheets from './MAdminSheets'
import MAdminStoragePanel from './MAdminStoragePanel'
import MAdminUsersSection from './MAdminUsersSection'
import { MAdminButton } from './MAdminUi'
@@ -86,6 +88,7 @@ export default function MAdmin() {
{ id: 'settings', label: t('admin.tabs.settings'), icon: SettingsIcon },
{ id: 'addons', label: t('admin.tabs.addons'), icon: Blocks },
{ id: 'plugins', label: t('admin.tabs.plugins'), icon: Puzzle },
...(managed ? [] : [{ id: 'storage', label: t('admin.tabs.storage'), icon: HardDrive }]),
{ id: 'notifications', label: t('admin.tabs.notifications'), icon: Bell },
...(mcpEnabled ? [{ id: 'mcp-tokens', label: t('admin.tabs.mcpTokens'), icon: Plug }] : []),
// Same two the desktop list drops: releases and backup schedule belong to
@@ -238,6 +241,7 @@ export default function MAdmin() {
/>
)}
{activeTab === 'plugins' && <MAdminPluginsPanel />}
{activeTab === 'storage' && <MAdminStoragePanel />}
{activeTab === 'mcp-tokens' && <MAdminMcpTokensPanel />}
{activeTab === 'github' && <MAdminGitHubPanel isPrerelease={updateInfo?.is_prerelease ?? false} />}
{activeTab === 'backup' && <MAdminBackupPanel />}
@@ -0,0 +1,890 @@
import React, { useEffect, useRef, useState } from 'react'
import { ChevronDown } from 'lucide-react'
import {
STORAGE_BACKEND_TYPES,
STORAGE_BACKEND_TYPE_IDS,
STORAGE_CATEGORIES,
type StorageBackend,
type StorageBackendFieldDef,
type StorageBackendTypeId,
type StorageCategory,
type StorageConfig,
type StorageMigrationStatus,
} from '@trek/shared'
import { useTranslation } from '../../../i18n'
import { useToast } from '../../../components/shared/Toast'
import { formatBytes } from '../../../utils/formatBytes'
import { relativeTime } from '../../../utils/relativeTime'
import {
CACHE_CATEGORIES,
adoptedMirrorFor,
computeMigrationCandidates,
effectiveCategoryMap,
foldBackends,
primaryNameOf,
removeBackend,
removeBackendAndMirrors,
renameBackendRefs,
replicaCandidates,
replicaOfPrimaries,
settingsDocumentOf,
setMirrorTargets,
stripCategories,
upsertBackend,
usageByBackend,
type FoldedBackendRow,
type MigrationCandidate,
} from '../../../components/Admin/storage/storageModel'
import { useStorageAdmin } from '../../../components/Admin/storage/useStorageAdmin'
import MToggle from '../../components/MToggle'
import MSetPickerSheet from '../settings/MSetPickerSheet'
import MConfirmSheet from '../settings/MConfirmSheet'
import { MSetSelectRow } from '../settings/MSettingsUi'
import { MAdminButton, MAdminCard, MAdminCardHead, MAdminField, MAdminInput, MAdminSecretInput } from './MAdminUi'
type FieldValues = Record<string, string | string[]>
function valuesOf(backend: StorageBackend | null): FieldValues {
if (!backend) return {}
const values: FieldValues = {}
for (const [key, value] of Object.entries(backend.options)) {
values[key] = Array.isArray(value) ? value : String(value)
}
return values
}
/** Display-name mapper for joined category lists — the raw id renders only in the badge. */
const categoryNames = (t: (key: string) => string, ids: readonly string[]): string =>
ids.map((id) => t(`storage.category.${id}`)).join(', ')
/** Same behavior contract as the desktop BackendForm, rendered on the M* primitives. */
function MBackendForm({
initial,
backendNames,
mirror,
onCommit,
onCancel,
}: {
initial: StorageBackend | null
backendNames: string[]
mirror: { candidates: string[]; initialTargets: string[] }
onCommit: (backend: StorageBackend, mirrorTargets: string[]) => void
onCancel: () => void
}): React.ReactElement {
const { t } = useTranslation()
const [type, setType] = useState<StorageBackendTypeId>(initial?.type ?? 'local')
const [name, setName] = useState(initial?.name ?? '')
const [values, setValues] = useState<FieldValues>(() => valuesOf(initial))
const [targets, setTargets] = useState<string[]>(mirror.initialTargets)
const [picker, setPicker] = useState<string | null>(null)
const fields = STORAGE_BACKEND_TYPES[type].fields as readonly StorageBackendFieldDef[]
const refOptions = backendNames.filter((candidate) => candidate !== name.trim())
const setValue = (key: string, value: string | string[]) => setValues((prev) => ({ ...prev, [key]: value }))
const filled = (field: StorageBackendFieldDef): boolean => {
const value = values[field.key]
if (field.kind === 'backend-ref-list') return Array.isArray(value) && value.length > 0
return typeof value === 'string' && value.trim() !== ''
}
const duplicate = name.trim() !== (initial?.name ?? '') && backendNames.includes(name.trim())
const canApply = name.trim() !== '' && !duplicate && fields.every((f) => !f.required || filled(f))
const apply = () => {
const options: Record<string, unknown> = {}
for (const field of fields) {
const value = values[field.key]
if (field.kind === 'backend-ref-list') {
options[field.key] = Array.isArray(value) ? value : []
continue
}
const text = typeof value === 'string' ? value : ''
if (text === '' && !field.required) continue
options[field.key] = field.kind === 'number' ? Number(text) : text
}
onCommit({ name: name.trim(), type, options } as StorageBackend, targets)
}
return (
<MAdminCard className="space-y-3">
<MAdminCardHead title={initial ? t('storage.form.editTitle') : t('storage.form.addTitle')} />
<MAdminField label={t('storage.form.name')}>
<MAdminInput value={name} onChange={(e) => setName(e.target.value)} spellCheck={false} autoComplete="off" />
</MAdminField>
{duplicate && (
<p role="alert" className="font-geist text-[0.625rem] text-m-muted">
{t('storage.form.duplicateName', { name: name.trim() })}
</p>
)}
{initial === null && (
<MAdminField label={t('storage.form.type')}>
<MSetSelectRow
label={t(`storage.type.${type}`)}
trailing={<ChevronDown size={14} className="text-m-faint" />}
onClick={() => setPicker('type')}
/>
<MSetPickerSheet
open={picker === 'type'}
onClose={() => setPicker(null)}
title={t('storage.form.type')}
options={STORAGE_BACKEND_TYPE_IDS.filter((id) => id !== 'mirror').map((id) => ({
value: id,
label: t(`storage.type.${id}`),
}))}
value={type}
onSelect={(next) => {
setType(next as StorageBackendTypeId)
setValues({})
}}
/>
</MAdminField>
)}
{fields.map((field) => {
const value = values[field.key]
if (field.kind === 'backend-ref') {
const current = typeof value === 'string' ? value : ''
return (
<MAdminField key={field.key} label={t(field.labelKey)}>
<MSetSelectRow
label={current || t(field.labelKey)}
trailing={<ChevronDown size={14} className="text-m-faint" />}
onClick={() => setPicker(`field:${field.key}`)}
/>
<MSetPickerSheet
open={picker === `field:${field.key}`}
onClose={() => setPicker(null)}
title={t(field.labelKey)}
options={refOptions.map((candidate) => ({ value: candidate, label: candidate }))}
value={current}
onSelect={(next) => setValue(field.key, next)}
/>
</MAdminField>
)
}
if (field.kind === 'backend-ref-list') {
const selected = Array.isArray(value) ? value : []
return (
<MAdminField key={field.key} label={t(field.labelKey)}>
<div className="space-y-2">
{refOptions.map((candidate) => (
<div key={candidate} className="flex items-center justify-between gap-2">
<span className="text-[0.8125rem] font-semibold text-m-ink">{candidate}</span>
<MToggle
checked={selected.includes(candidate)}
ariaLabel={candidate}
onChange={(checked) =>
setValue(
field.key,
checked ? [...selected, candidate] : selected.filter((existing) => existing !== candidate),
)
}
/>
</div>
))}
</div>
</MAdminField>
)
}
const text = typeof value === 'string' ? value : ''
const shared = {
value: text,
onChange: (e: React.ChangeEvent<HTMLInputElement>) => setValue(field.key, e.target.value),
placeholder: field.defaultValue !== undefined ? String(field.defaultValue) : '',
spellCheck: false,
autoComplete: 'off',
}
return (
<MAdminField key={field.key} label={t(field.labelKey)} hint={field.helpKey ? t(field.helpKey) : undefined}>
{field.kind === 'secret' ? (
<MAdminSecretInput {...shared} />
) : (
<MAdminInput {...shared} type={field.kind === 'number' ? 'number' : 'text'} />
)}
</MAdminField>
)
})}
<MAdminField label={t('storage.mirror.targets')} hint={t('storage.mirror.targetsHelp')}>
<div className="space-y-2">
{mirror.candidates
.filter((candidate) => candidate !== name.trim())
.map((candidate) => (
<div key={candidate} className="flex items-center justify-between gap-2">
<span className="text-[0.8125rem] font-semibold text-m-ink">{candidate}</span>
<MToggle
checked={targets.includes(candidate)}
ariaLabel={candidate}
onChange={(checked) =>
setTargets(checked ? [...targets, candidate] : targets.filter((existing) => existing !== candidate))
}
/>
</div>
))}
</div>
{targets.length > 0 && (
<p role="note" className="mt-1 font-geist text-[0.625rem] leading-relaxed text-m-muted">
{t('storage.mirror.latencyNote')}
</p>
)}
</MAdminField>
<div className="flex items-center gap-2">
<MAdminButton onClick={apply} disabled={!canApply}>
{t('storage.form.apply')}
</MAdminButton>
<MAdminButton variant="ghost" onClick={onCancel}>
{t('storage.form.cancel')}
</MAdminButton>
</div>
</MAdminCard>
)
}
export default function MAdminStoragePanel(): React.ReactElement {
const { t, locale } = useTranslation()
const toast = useToast()
const admin = useStorageAdmin(t('common.error'), t('storage.saveConflict'))
const [editing, setEditing] = useState<{
initial: StorageBackend | null
originalName: string | null
mirror: { candidates: string[]; initialTargets: string[] }
} | null>(null)
const [confirmRemove, setConfirmRemove] = useState<{ name: string; degenerate: boolean } | null>(null)
const [categoryPicker, setCategoryPicker] = useState<StorageCategory | null>(null)
const [syncPrompt, setSyncPrompt] = useState<string | null>(null)
const [migratePrompt, setMigratePrompt] = useState<MigrationCandidate[] | null>(null)
const [migrationQueue, setMigrationQueue] = useState<MigrationCandidate[]>([])
// Set by `save` right before it calls admin.save(): the pre-save mirror
// target count per row name. Consumed (and cleared) by the effect below the
// first time `admin.state` changes afterward — never touched otherwise, so
// unrelated state changes (the backfill poll included) are no-ops here.
const pendingPromptCheck = useRef<Map<string, number> | null>(null)
// Synchronous single-flight lock for the queue effect below: `admin.state`
// only reflects a just-started migration once startMigration's awaited
// refreshState() resolves, so `setMigrationQueue(rest)` re-firing the
// effect synchronously (still against the pre-POST `admin.state`) would
// otherwise dequeue and POST the next candidate before the server has
// confirmed the first — a ref (not state) so the guard is visible on that
// very next synchronous re-render, not just after a state-driven one.
const migrationStartInFlight = useRef(false)
useEffect(() => {
if (!pendingPromptCheck.current || !admin.state) return
const before = pendingPromptCheck.current
pendingPromptCheck.current = null
const { rows: afterRows } = foldBackends(admin.state, settingsDocumentOf(admin.state))
const grown = afterRows.find((r) => r.mirrorTargets.length > (before.get(r.name) ?? 0))
if (grown) setSyncPrompt(grown.name)
}, [admin.state])
// Queued category migrations run strictly sequentially: once a slot opens
// (no migration currently running), dequeue the next candidate and start
// it. Depends on admin.state so a poll landing a terminal status re-fires
// this without any user interaction. The in-flight lock closes the race
// where setMigrationQueue(rest) re-fires this effect synchronously while
// admin.state still shows the pre-POST world — without it, the next
// candidate could dequeue and POST before the first is server-confirmed.
// Also waits out a running BACKFILL — the server's one-storage-job-at-a-time
// rule spans backfills and migrations alike, so starting while one runs
// would 409, and the queued candidate would be lost (no retry).
useEffect(() => {
if (migrationQueue.length === 0 || !admin.state) return
if (migrationStartInFlight.current) return
if (admin.state.migrations.some((m) => m.status === 'running')) return
if (admin.state.backfills.some((b) => b.status === 'running')) return
const [next, ...rest] = migrationQueue
migrationStartInFlight.current = true
setMigrationQueue(rest)
void admin.startMigration(next!.category, next!.toWire).then((error) => {
migrationStartInFlight.current = false
if (error) toast.error(error)
})
}, [migrationQueue, admin.state])
if (admin.loading) {
return (
<MAdminCard>
<p className="font-geist text-[0.75rem] italic text-m-faint">{t('storage.loading')}</p>
</MAdminCard>
)
}
if (!admin.state || !admin.draft) {
return (
<MAdminCard>
<p role="alert" className="text-[0.8125rem] text-m-ink">{admin.loadError || t('common.error')}</p>
</MAdminCard>
)
}
const { state, draft } = admin
const backendNames = [...new Set([...state.backends.map((b) => b.name), ...draft.backends.map((b) => b.name)])]
const { rows, degenerate } = foldBackends(state, draft)
const effective = effectiveCategoryMap(state, draft)
const usageSums = usageByBackend(state, draft)
const startEdit = (row: FoldedBackendRow) => {
setEditing({
initial: row.backend,
originalName: row.name,
mirror: {
candidates: replicaCandidates(rows, row.name, row.mirrorTargets),
initialTargets: row.mirrorTargets,
},
})
}
const commitBackend = (backend: StorageBackend, mirrorTargets: string[]) => {
const renamedFrom = editing?.originalName && editing.originalName !== backend.name ? editing.originalName : null
// Widened to StorageConfig: these edit helpers are version-blind (they
// return plain StorageConfig), and admin.setDraft re-attaches the
// draft's own `version` regardless of what shape it's handed.
let next: StorageConfig | null = draft
if (renamedFrom) next = renameBackendRefs(removeBackend(next, renamedFrom), renamedFrom, backend.name)
next = upsertBackend(next, backend)
next = setMirrorTargets(state, next, backend.name, mirrorTargets)
admin.setDraft(next)
setEditing(null)
}
const removeMessage = (name: string, isDegenerate: boolean): string => {
const row = rows.find((r) => r.name === name)
const assigned = isDegenerate
? degenerate.find((d) => d.backend.name === name)?.categories ?? []
: row?.categories ?? []
const usedAsReplicaBy = isDegenerate ? [] : replicaOfPrimaries(draft, name)
return [
t('storage.remove.body', { name }),
assigned.length > 0 ? t('storage.remove.stillAssigned', { categories: categoryNames(t, assigned) }) : '',
usedAsReplicaBy.length > 0 ? t('storage.remove.usedAsReplicaBy', { primaries: usedAsReplicaBy.join(', ') }) : '',
]
.filter(Boolean)
.join(' ')
}
const setCategory = (category: StorageCategory, primaryName: string) => {
const target = adoptedMirrorFor(draft, primaryName)?.name ?? primaryName
// The admin state's category record is exhaustive by schema contract.
const stateEntry = state.categories[category]!
const categories = { ...draft.categories }
if (stateEntry.source === 'default' && target === stateEntry.backend) {
delete categories[category]
} else {
categories[category] = target
}
admin.setDraft({ ...draft, categories })
}
// Snapshot the LAST-CONFIRMED (pre-save `state`'s own fold, not the
// in-progress `draft`) mirror target counts; the effect above compares
// them against the fresh post-save state to detect newly-added
// replicas. Folding `draft` here would already include the unsaved
// edit and mask the very growth this is meant to detect.
const snapshotMirrorTargets = (): Map<string, number> => {
const { rows: beforeRows } = foldBackends(state, settingsDocumentOf(state))
return new Map(beforeRows.map((r) => [r.name, r.mirrorTargets.length]))
}
const doPlainSave = async () => {
pendingPromptCheck.current = snapshotMirrorTargets()
if (await admin.save()) toast.success(t('storage.saved'))
else pendingPromptCheck.current = null
}
const moveAndSave = async (candidates: MigrationCandidate[]) => {
pendingPromptCheck.current = snapshotMirrorTargets()
const ok = await admin.save(stripCategories(draft, state, candidates.map((c) => c.category)))
if (ok) {
setMigrationQueue(candidates)
toast.success(t('storage.saved'))
} else {
pendingPromptCheck.current = null
}
setMigratePrompt(null)
}
const routeOnlySave = async () => {
setMigratePrompt(null)
await doPlainSave()
}
const save = async () => {
const candidates = computeMigrationCandidates(draft, state)
if (candidates.length > 0) {
setMigratePrompt(candidates)
return
}
await doPlainSave()
}
const handleCancelMigration = async (m: StorageMigrationStatus) => {
const error = await admin.cancelMigration(m.category)
if (error) toast.error(error)
}
const handleRefreshStats = async () => {
const error = await admin.refreshStats()
if (error) toast.error(error)
}
const handleStartBackfill = async (row: FoldedBackendRow) => {
if (!row.mirrorName) return
setSyncPrompt((prev) => (prev === row.name ? null : prev))
const error = await admin.startBackfill(row.mirrorName)
if (error) toast.error(error)
}
const handleCancelBackfill = async (row: FoldedBackendRow) => {
if (!row.mirrorName) return
const error = await admin.cancelBackfill(row.mirrorName)
if (error) toast.error(error)
}
return (
<div className="space-y-3">
<MAdminCard>
<MAdminCardHead title={t('storage.health.title')} />
{state.health.replicaFailures.length === 0 ? (
<p className="font-geist text-[0.75rem] text-m-faint">{t('storage.health.allClear')}</p>
) : (
<div className="space-y-1">
{state.health.replicaFailures.map((failure) => (
<p key={`${failure.backend}-${failure.key}-${failure.at}`} className="text-[0.75rem] text-m-ink">
{t('storage.health.failureLine', {
op: failure.op,
key: failure.key,
backend: failure.backend,
error: failure.error,
})}{' '}
<span className="text-m-faint">· {relativeTime(failure.at, locale)}</span>
</p>
))}
</div>
)}
{state.seedFilePresent && (
<p className="mt-1 font-geist text-[0.625rem] text-m-muted">{t('storage.health.seedFile')}</p>
)}
</MAdminCard>
<MAdminCard>
<MAdminCardHead title={t('storage.backends.title')} hint={t('storage.description')} />
<div className="mb-2 flex items-center gap-2">
<p className="font-geist text-[0.625rem] text-m-muted">
{state.usage
? t('storage.usage.computed', { age: relativeTime(state.usage.computedAt, locale) })
: t('storage.usage.never')}
</p>
<MAdminButton variant="ghost" onClick={handleRefreshStats}>
{state.usage ? t('storage.usage.refresh') : t('storage.usage.compute')}
</MAdminButton>
</div>
<div className="space-y-2">
{rows.map((row) => {
const resultKey = row.mirrorName ?? row.name
const result = admin.testResults[resultKey]
// row.mirrorName only exists when foldBackends adopted a draft mirror
// for this row, so the draft lookup below cannot miss.
const testCandidate = row.mirrorName
? draft.backends.find((b) => b.name === row.mirrorName)!
: row.backend
const rowUsage = usageSums?.[row.name]
const backfill = row.mirrorName ? state.backfills.find((b) => b.backend === row.mirrorName) : undefined
return (
<div
key={row.name}
data-testid={`m-storage-backend-${row.name}`}
className="rounded-xl border border-[color:var(--m-rowbr)] bg-[color:var(--m-sheet)] p-3"
>
<div className="flex flex-wrap items-center gap-2">
<span className="text-[0.8125rem] font-bold text-m-ink">{row.name}</span>
<span className="rounded-full border border-[color:var(--m-rowbr)] px-2 py-[1px] font-geist text-[0.625rem] text-m-muted">
{t(`storage.type.${row.type}`)}
</span>
<span className="rounded-full border border-[color:var(--m-rowbr)] px-2 py-[1px] font-geist text-[0.625rem] text-m-muted">
{t(`storage.source.${row.source}`)}
</span>
</div>
<p className="mt-1 font-geist text-[0.625rem] text-m-muted">
{row.categories.length > 0
? t('storage.backends.usedBy', { categories: categoryNames(t, row.categories) })
: t('storage.backends.unused')}
</p>
{rowUsage && (
<p className="mt-1 font-geist text-[0.625rem] text-m-muted">
{t('storage.usage.line', { objects: String(rowUsage.objects), size: formatBytes(rowUsage.bytes) })}
{row.name === 'uploads-local' && state.usage!.legacyPhotos.objects > 0
? ` (${t('storage.usage.legacyNote')})`
: ''}
</p>
)}
{row.mirrorTargets.length > 0 && (
<p className="mt-1 font-geist text-[0.625rem] text-m-muted">
{t('storage.mirror.mirroredTo', { targets: row.mirrorTargets.join(', ') })}
</p>
)}
{row.replicaOf.length > 0 && (
<p className="mt-1 font-geist text-[0.625rem] text-m-muted">
{t('storage.mirror.replicaOf', { primaries: row.replicaOf.join(', ') })}
</p>
)}
{row.source === 'env' && (
<p className="mt-1 font-geist text-[0.625rem] text-m-muted">{t('storage.backends.envReadOnly')}</p>
)}
<div className="mt-2 flex gap-2">
<MAdminButton variant="ghost" onClick={() => admin.test(testCandidate)}>
{t('storage.actions.test')}
</MAdminButton>
{row.source !== 'env' && (
<MAdminButton variant="ghost" onClick={() => startEdit(row)}>
{t('storage.actions.edit')}
</MAdminButton>
)}
{row.source === 'settings' && (
<MAdminButton variant="danger" onClick={() => setConfirmRemove({ name: row.name, degenerate: false })}>
{t('storage.actions.remove')}
</MAdminButton>
)}
</div>
{result === 'running' ? (
<p className="mt-2 font-geist text-[0.625rem] text-m-muted">{t('storage.test.running')}</p>
) : (
result && (
<div className="mt-2 space-y-0.5">
<p className="text-[0.75rem] font-bold text-m-ink">
{result.ok ? t('storage.test.ok') : t('storage.test.failed')}
</p>
{result.targets.map((target) => (
<p key={target.name} className="font-geist text-[0.625rem] text-m-muted">
{target.ok ? '✓' : '✗'} {target.name}
{target.error ? ` — ${target.error}` : ''}
</p>
))}
</div>
)
)}
{row.mirrorTargets.length > 0 && (
<div className="mt-2">
{backfill?.status === 'running' ? (
<>
<p className="font-geist text-[0.625rem] text-m-muted">
{t('storage.sync.running', { done: String(backfill.done), total: String(backfill.total) })}
</p>
<p className="font-geist text-[0.625rem] text-m-muted">
{t('storage.sync.counts', {
copied: String(backfill.copied),
skipped: String(backfill.skipped),
failed: String(backfill.failed),
})}
</p>
<div className="mt-1">
<MAdminButton variant="ghost" onClick={() => handleCancelBackfill(row)}>
{t('storage.sync.cancel')}
</MAdminButton>
</div>
</>
) : backfill?.status === 'done' ? (
<p className="font-geist text-[0.625rem] text-m-muted">
{t('storage.sync.done', {
copied: String(backfill.copied),
deleted: String(backfill.deleted),
failed: String(backfill.failed),
})}
</p>
) : backfill?.status === 'cancelled' ? (
<p className="font-geist text-[0.625rem] text-m-muted">{t('storage.sync.cancelled')}</p>
) : backfill?.status === 'error' ? (
<p className="font-geist text-[0.625rem] text-m-muted">
{t('storage.sync.error', { error: backfill.error ?? '' })}
</p>
) : syncPrompt !== row.name ? (
<div className="mt-1">
<MAdminButton variant="ghost" onClick={() => handleStartBackfill(row)}>
{t('storage.sync.now')}
</MAdminButton>
</div>
) : null}
</div>
)}
{syncPrompt === row.name && (
<div className="mt-2 rounded-lg border border-[color:var(--m-rowbr)] p-2">
<p className="text-[0.75rem] text-m-ink">{t('storage.sync.prompt')}</p>
<div className="mt-1 flex items-center gap-2">
<MAdminButton variant="ghost" onClick={() => handleStartBackfill(row)}>
{t('storage.sync.now')}
</MAdminButton>
<MAdminButton variant="ghost" onClick={() => setSyncPrompt(null)}>
{t('storage.sync.dismiss')}
</MAdminButton>
</div>
</div>
)}
</div>
)
})}
{degenerate.map(({ backend, reason }) => {
const result = admin.testResults[backend.name]
const primary = backend.type === 'mirror' ? backend.options.primary : ''
return (
<div
key={backend.name}
data-testid={`m-storage-backend-${backend.name}`}
className="rounded-xl border border-[color:var(--m-rowbr)] bg-[color:var(--m-sheet)] p-3"
>
<div className="flex flex-wrap items-center gap-2">
<span className="text-[0.8125rem] font-bold text-m-ink">{backend.name}</span>
<span className="rounded-full border border-[color:var(--m-rowbr)] px-2 py-[1px] font-geist text-[0.625rem] text-m-muted">
{t('storage.type.mirror')}
</span>
</div>
<p className="mt-1 font-geist text-[0.625rem] text-m-muted">
{t(`storage.mirror.degenerate.${reason}`, { primary })}
</p>
<div className="mt-2 flex gap-2">
<MAdminButton variant="ghost" onClick={() => admin.test(backend)}>
{t('storage.actions.test')}
</MAdminButton>
<MAdminButton variant="danger" onClick={() => setConfirmRemove({ name: backend.name, degenerate: true })}>
{t('storage.actions.remove')}
</MAdminButton>
</div>
{result === 'running' ? (
<p className="mt-2 font-geist text-[0.625rem] text-m-muted">{t('storage.test.running')}</p>
) : (
result && (
<div className="mt-2 space-y-0.5">
<p className="text-[0.75rem] font-bold text-m-ink">
{result.ok ? t('storage.test.ok') : t('storage.test.failed')}
</p>
{result.targets.map((target) => (
<p key={target.name} className="font-geist text-[0.625rem] text-m-muted">
{target.ok ? '✓' : '✗'} {target.name}
{target.error ? ` — ${target.error}` : ''}
</p>
))}
</div>
)
)}
</div>
)
})}
{state.migrations.map((m) => (
<div
key={m.category}
data-testid={`m-storage-migration-${m.category}`}
className="rounded-xl border border-[color:var(--m-rowbr)] bg-[color:var(--m-sheet)] p-3"
>
<p className="text-[0.8125rem] font-bold text-m-ink">{t(`storage.category.${m.category}`)}</p>
{m.status === 'running' ? (
<>
<p className="mt-1 font-geist text-[0.625rem] text-m-muted">
{t('storage.migrate.running', {
category: t(`storage.category.${m.category}`),
done: String(m.done),
total: String(m.total),
})}
</p>
<div className="mt-1">
<MAdminButton variant="ghost" onClick={() => void handleCancelMigration(m)}>
{t('storage.migrate.cancel')}
</MAdminButton>
</div>
</>
) : m.status === 'done' ? (
<>
<p className="mt-1 font-geist text-[0.625rem] text-m-muted">
{t('storage.migrate.done', { copied: String(m.copied), skipped: String(m.skipped) })}
</p>
{m.failed > 0 && (
<p className="mt-1 font-geist text-[0.625rem] text-m-muted">
{t('storage.migrate.doneFailures', { failed: String(m.failed) })}
</p>
)}
{m.reclaimable && (
<p className="mt-1 font-geist text-[0.625rem] text-m-muted">
{t('storage.migrate.reclaimable', {
objects: String(m.reclaimable.objects),
size: formatBytes(m.reclaimable.bytes),
from: m.from,
})}
</p>
)}
</>
) : m.status === 'failed' ? (
<p className="mt-1 font-geist text-[0.625rem] text-m-muted">
{t('storage.migrate.failed', { error: m.error ?? '' })}
</p>
) : (
<p className="mt-1 font-geist text-[0.625rem] text-m-muted">{t('storage.migrate.cancelled')}</p>
)}
</div>
))}
</div>
{editing === null && (
<div className="mt-3">
<MAdminButton
onClick={() =>
setEditing({
initial: null,
originalName: null,
mirror: { candidates: replicaCandidates(rows, null), initialTargets: [] },
})
}
>
{t('storage.backends.add')}
</MAdminButton>
</div>
)}
</MAdminCard>
{editing !== null && (
<MBackendForm
initial={editing.initial}
backendNames={backendNames}
mirror={editing.mirror}
onCommit={commitBackend}
onCancel={() => setEditing(null)}
/>
)}
<MAdminCard>
<MAdminCardHead title={t('storage.categories.title')} />
<div className="space-y-2">
{STORAGE_CATEGORIES.map((category) => {
// The admin state's category record is exhaustive by schema contract.
const stateEntry = state.categories[category]!
const selectedPrimary = primaryNameOf(state, draft, effective[category])
const changed = selectedPrimary !== primaryNameOf(state, draft, stateEntry.backend)
const viaMirror = effective[category] !== selectedPrimary
return (
<MAdminField
key={category}
label={
<>
{t(`storage.category.${category}`)}{' '}
<span className="rounded border border-[color:var(--m-rowbr)] px-1 font-geist text-[0.625rem] font-normal text-m-faint">
{category}
</span>
</>
}
hint={t(`storage.categoryDesc.${category}`)}
>
{state.usage?.categories[category] && (
<p className="mb-1 font-geist text-[0.625rem] text-m-muted">
{t('storage.usage.line', {
objects: String(state.usage.categories[category]!.objects),
size: formatBytes(state.usage.categories[category]!.bytes),
})}
</p>
)}
<div data-testid={`m-storage-category-${category}`} className="contents">
<MSetSelectRow
label={
stateEntry.source === 'default' && selectedPrimary === stateEntry.backend
? `${selectedPrimary} (${t('storage.categories.default')})`
: selectedPrimary
}
trailing={<ChevronDown size={14} className="text-m-faint" />}
onClick={() => setCategoryPicker(category)}
/>
</div>
{changed && (
<p role="alert" className="mt-1 font-geist text-[0.625rem] text-m-muted">
{t('storage.categories.reassignWarning')}
</p>
)}
{viaMirror && CACHE_CATEGORIES.includes(category) && (
<p role="note" className="mt-1 font-geist text-[0.625rem] text-m-muted">
{t('storage.mirror.cacheWarning')}
</p>
)}
</MAdminField>
)
})}
</div>
<MSetPickerSheet
open={categoryPicker !== null}
onClose={() => setCategoryPicker(null)}
title={categoryPicker ? t(`storage.category.${categoryPicker}`) : ''}
options={rows.map((row) => ({ value: row.name, label: row.name }))}
value={categoryPicker ? primaryNameOf(state, draft, effective[categoryPicker]) : ''}
onSelect={(name) => {
if (categoryPicker) setCategory(categoryPicker, name)
}}
/>
</MAdminCard>
<div className="flex items-center gap-2">
<MAdminButton onClick={save} disabled={!admin.dirty} busy={admin.saving}>
{t('storage.save')}
</MAdminButton>
{admin.dirty && <span className="font-geist text-[0.625rem] text-m-muted">{t('storage.unsaved')}</span>}
</div>
{migratePrompt && (
<div role="alertdialog">
<MAdminCard>
<p className="text-[0.8125rem] text-m-ink">{t('storage.migrate.promptTitle')}</p>
{migratePrompt.map((c) => (
<p key={c.category} className="mt-1 font-geist text-[0.625rem] text-m-muted">
{c.objects === null
? t('storage.migrate.promptLineUnknown', {
category: t(`storage.category.${c.category}`),
from: c.from,
to: c.to,
})
: t('storage.migrate.promptLine', {
category: t(`storage.category.${c.category}`),
objects: String(c.objects),
size: formatBytes(c.bytes ?? 0),
from: c.from,
to: c.to,
})}
</p>
))}
<div className="mt-2 flex items-center gap-2">
<MAdminButton variant="ghost" onClick={() => void moveAndSave(migratePrompt)}>
{t('storage.migrate.move')}
</MAdminButton>
<MAdminButton variant="ghost" onClick={() => void routeOnlySave()}>
{t('storage.migrate.routeOnly')}
</MAdminButton>
</div>
</MAdminCard>
</div>
)}
{admin.saveError && (
<div role="alert" className="text-[0.8125rem] text-m-ink">
<p>{admin.saveError}</p>
{/* A 409 pins the draft to a version the server moved past — saving
again can never clear it, so offer the one action that can. */}
{admin.saveConflict && (
<MAdminButton variant="ghost" onClick={() => void admin.discardDraft()}>
{t('storage.discardAndReload')}
</MAdminButton>
)}
</div>
)}
<MConfirmSheet
open={confirmRemove !== null}
onClose={() => setConfirmRemove(null)}
title={t('storage.remove.title')}
message={confirmRemove ? removeMessage(confirmRemove.name, confirmRemove.degenerate) : ''}
confirmLabel={t('storage.actions.remove')}
cancelLabel={t('storage.form.cancel')}
danger
onConfirm={() => {
if (confirmRemove) {
admin.setDraft(
confirmRemove.degenerate
? removeBackend(draft, confirmRemove.name)
: removeBackendAndMirrors(state, draft, confirmRemove.name),
)
}
setConfirmRemove(null)
}}
/>
</div>
)
}
+19
View File
@@ -69,6 +69,10 @@ vi.mock('../components/Admin/DefaultUserSettingsTab', () => ({
default: () => <div data-testid="default-user-settings" />,
}));
vi.mock('../components/Admin/storage/AdminStoragePanel', () => ({
default: () => <div data-testid="admin-storage-panel" />,
}));
beforeEach(() => {
resetAllStores();
});
@@ -1631,4 +1635,19 @@ describe('AdminPage', () => {
expect(screen.queryByText('Files')).not.toBeInTheDocument();
});
});
it('FE-PAGE-ADMIN-STOR-001: the Storage tab renders the storage panel', async () => {
seedStore(useAuthStore, { isAuthenticated: true, user: buildAdmin() });
render(<AdminPage />);
const tab = await screen.findByRole('button', { name: /^storage$/i });
fireEvent.click(tab);
expect(screen.getByTestId('admin-storage-panel')).toBeInTheDocument();
});
it('FE-PAGE-ADMIN-STOR-002: managed mode hides the Storage tab (hoster-level config)', async () => {
seedStore(useAuthStore, { isAuthenticated: true, user: buildAdmin(), managed: true });
render(<AdminPage />);
await screen.findByRole('button', { name: /^users$/i });
expect(screen.queryByRole('button', { name: /^storage$/i })).not.toBeInTheDocument();
});
});
+7 -1
View File
@@ -12,7 +12,8 @@ import PackingTemplateManager from '../components/Admin/PackingTemplateManager'
import AuditLogPanel from '../components/Admin/AuditLogPanel'
import AdminMcpTokensPanel from '../components/Admin/AdminMcpTokensPanel'
import AdminPluginsPanel from '../components/Admin/AdminPluginsPanel'
import { Users, Map, Briefcase, Shield, FileText, SlidersHorizontal, UserCog, Puzzle, Blocks, Settings as SettingsIcon, Bell, Database, ScrollText, KeyRound, GitBranch, Bug } from 'lucide-react'
import AdminStoragePanel from '../components/Admin/storage/AdminStoragePanel'
import { Users, Map, Briefcase, Shield, FileText, SlidersHorizontal, UserCog, Puzzle, Blocks, Settings as SettingsIcon, Bell, Database, ScrollText, KeyRound, GitBranch, Bug, HardDrive } from 'lucide-react'
import PageSidebar, { type PageSidebarTab } from '../components/Layout/PageSidebar'
import { useAdmin } from './admin/useAdmin'
import AdminUpdateBanner from './admin/AdminUpdateBanner'
@@ -55,6 +56,9 @@ function AdminPageDesktop(): React.ReactElement {
{ id: 'settings', label: t('admin.tabs.settings'), icon: SettingsIcon, group: gConfig },
{ id: 'addons', label: t('admin.tabs.addons'), icon: Puzzle, group: gConfig },
{ id: 'plugins', label: t('admin.tabs.plugins'), icon: Blocks, group: gConfig },
// Storage backends and their credentials are hoster-level configuration —
// the server refuses the whole surface in managed mode (MANAGED_FORBIDDEN).
...(managed ? [] : [{ id: 'storage', label: t('admin.tabs.storage'), icon: HardDrive, group: gConfig }]),
{ id: 'notifications', label: t('admin.tabs.notifications'), icon: Bell, group: gIntegration },
...(mcpEnabled ? [{ id: 'mcp-tokens', label: t('admin.tabs.mcpTokens'), icon: KeyRound, group: gIntegration }] : []),
// Releases and update cadence belong to whoever operates the install.
@@ -180,6 +184,8 @@ function AdminPageDesktop(): React.ReactElement {
{activeTab === 'plugins' && <AdminPluginsPanel />}
{activeTab === 'storage' && <AdminStoragePanel />}
{activeTab === 'github' && <GitHubPanel isPrerelease={updateInfo?.is_prerelease ?? false} />}
{activeTab === 'defaults' && <DefaultUserSettingsTab />}
+13
View File
@@ -0,0 +1,13 @@
import { describe, expect, it } from 'vitest'
import { formatBytes } from './formatBytes'
describe('formatBytes', () => {
it('FE-UTIL-BYTES-001: scales B→TB with one decimal above bytes', () => {
expect(formatBytes(0)).toBe('0 B')
expect(formatBytes(512)).toBe('512 B')
expect(formatBytes(3_174)).toBe('3.1 KB')
expect(formatBytes(2 * 1024 * 1024)).toBe('2.0 MB')
expect(formatBytes(1.5 * 1024 ** 3)).toBe('1.5 GB')
expect(formatBytes(1.2 * 1024 ** 4)).toBe('1.2 TB')
})
})
+13
View File
@@ -0,0 +1,13 @@
const UNITS = ['B', 'KB', 'MB', 'GB', 'TB'] as const
/** B→TB, one decimal above bytes. (FileManager's MB-capped formatSize predates this; not consolidated here.) */
export function formatBytes(bytes: number): string {
if (bytes < 1024) return `${Math.round(bytes)} B`
let value = bytes
let unit = 0
while (value >= 1024 && unit < UNITS.length - 1) {
value /= 1024
unit += 1
}
return `${value.toFixed(1)} ${UNITS[unit]}`
}
+21
View File
@@ -0,0 +1,21 @@
import { describe, expect, it } from 'vitest';
import { relativeTime } from './relativeTime';
describe('relativeTime', () => {
const now = 1_700_000_000_000;
it('FE-UTIL-RELTIME-001: formats seconds, minutes, hours and days', () => {
expect(relativeTime(now - 30_000, 'en', now)).toBe('30 seconds ago');
expect(relativeTime(now - 120_000, 'en', now)).toBe('2 minutes ago');
expect(relativeTime(now - 2 * 3_600_000, 'en', now)).toBe('2 hours ago');
expect(relativeTime(now - 3 * 86_400_000, 'en', now)).toBe('3 days ago');
});
it('FE-UTIL-RELTIME-002: clamps future timestamps to "now"', () => {
expect(relativeTime(now + 5_000, 'en', now)).toBe('now');
});
it('FE-UTIL-RELTIME-003: localizes via Intl (de)', () => {
expect(relativeTime(now - 120_000, 'de', now)).toBe('vor 2 Minuten');
});
});
+15
View File
@@ -0,0 +1,15 @@
const STEPS: Array<{ limit: number; div: number; unit: Intl.RelativeTimeFormatUnit }> = [
{ limit: 60_000, div: 1_000, unit: 'second' },
{ limit: 3_600_000, div: 60_000, unit: 'minute' },
{ limit: 86_400_000, div: 3_600_000, unit: 'hour' },
{ limit: Number.POSITIVE_INFINITY, div: 86_400_000, unit: 'day' },
]
/** "2 minutes ago" from an epoch-ms timestamp, localized. Future stamps clamp to "now". */
export function relativeTime(at: number, locale: string, now: number = Date.now()): string {
const elapsed = now - at
if (elapsed < 1_000) return new Intl.RelativeTimeFormat(locale, { numeric: 'auto' }).format(0, 'second')
const step = STEPS.find((s) => elapsed < s.limit)!
const value = Math.round(elapsed / step.div)
return new Intl.RelativeTimeFormat(locale, { numeric: 'auto' }).format(-value, step.unit)
}
@@ -55,6 +55,9 @@ vi.mock('../../../../src/mobile/screens/admin/MAdminGitHubPanel', () => ({
<div data-testid="github-panel">{String(isPrerelease)}</div>
),
}));
vi.mock('../../../../src/mobile/screens/admin/MAdminStoragePanel', () => ({
default: () => <div data-testid="m-admin-storage-panel" />,
}));
// The addon manager is reduced to the two controls MAdmin wires itself.
vi.mock('../../../../src/mobile/screens/admin/MAdminAddonManager', () => ({
@@ -213,6 +216,9 @@ describe('MAdmin', () => {
await openSection('Plugins');
expect(screen.getByTestId('plugins-panel')).toBeInTheDocument();
await openSection('Storage');
expect(screen.getByTestId('m-admin-storage-panel')).toBeInTheDocument();
await openSection('Notifications');
expect(screen.getByTestId('notifications-section')).toBeInTheDocument();
@@ -380,4 +386,11 @@ describe('MAdmin', () => {
expect(root.className).not.toContain('h-full');
expect(root.className).not.toContain('overflow-y-auto');
});
it('FE-MOB-ADMIN-STOR-001: managed mode hides the Storage section', async () => {
seedStore(useAuthStore, { isAuthenticated: true, user: buildAdmin(), managed: true });
renderAdmin();
fireEvent.click(await screen.findByRole('button', { name: 'Administration' }));
expect(screen.queryByRole('button', { name: 'Storage' })).not.toBeInTheDocument();
});
});
@@ -0,0 +1,431 @@
import { http, HttpResponse } from 'msw';
import { describe, expect, it } from 'vitest';
import { MASKED_SETTING_VALUE, type StorageAdminState, type StorageCategory, type StorageConfig } from '@trek/shared';
import { server } from '../../../helpers/msw/server';
import { fireEvent, render, screen, waitFor, within } from '../../../helpers/render';
import { ToastContainer } from '../../../../src/components/shared/Toast';
import MAdminStoragePanel from '../../../../src/mobile/screens/admin/MAdminStoragePanel';
function baseState(overrides: Partial<StorageAdminState> = {}): StorageAdminState {
return {
backends: [
{ name: 'uploads-local', type: 'local', source: 'built-in', options: { root: '/data/uploads' }, categories: ['files', 'journey', 'covers', 'avatars', 'photos-google', 'photos-trek'] },
{ name: 'backups-local', type: 'local', source: 'built-in', options: { root: '/data/backups' }, categories: ['backups'] },
{
name: 'off-box', type: 's3', source: 'settings',
options: { endpoint: 'http://127.0.0.1:9000', bucket: 'trek', accessKeyId: 'ak', secretAccessKey: MASKED_SETTING_VALUE, region: 'us-east-1', keyPrefix: '', retries: 1, timeoutMs: 30000 },
categories: [],
},
],
categories: {
files: { backend: 'uploads-local', source: 'default' },
journey: { backend: 'uploads-local', source: 'default' },
covers: { backend: 'uploads-local', source: 'default' },
avatars: { backend: 'uploads-local', source: 'default' },
places: { backend: 'uploads-local', source: 'default' },
'photos-google': { backend: 'uploads-local', source: 'default' },
'photos-trek': { backend: 'uploads-local', source: 'default' },
backups: { backend: 'backups-local', source: 'default' },
},
health: { replicaFailures: [] },
seedFilePresent: false,
usage: null,
backfills: [],
migrations: [],
version: 0,
...overrides,
};
}
function mirroredState(): StorageAdminState {
const state = baseState();
state.backends.push({
name: 'mirror', type: 'mirror', source: 'settings',
options: { primary: 'backups-local', replicas: ['off-box'] }, categories: ['backups'],
});
state.categories.backups = { backend: 'mirror', source: 'settings' };
return state;
}
async function renderPanel(state: StorageAdminState = baseState()) {
server.use(http.get('/api/admin/storage', () => HttpResponse.json(state)));
render(
<>
<ToastContainer />
<MAdminStoragePanel />
</>,
);
await waitFor(() => expect(screen.getByText('Backends')).toBeInTheDocument());
}
describe('MAdminStoragePanel', () => {
it('FE-MOB-MSTOR-001: renders every backend with type and source, env rows read-only', async () => {
await renderPanel();
const row = screen.getByTestId('m-storage-backend-off-box');
expect(within(row).getByText('S3')).toBeInTheDocument();
expect(within(row).getByText('Settings')).toBeInTheDocument();
expect(screen.getByText('No replica failures recorded.')).toBeInTheDocument();
});
it('FE-MOB-MSTOR-002: reassigning a category via the picker sheet warns and saves through one PUT', async () => {
let putBody: unknown;
await renderPanel();
server.use(
http.put('/api/admin/storage', async ({ request }) => {
putBody = await request.json();
return HttpResponse.json(baseState());
}),
);
fireEvent.click(within(screen.getByTestId('m-storage-category-files')).getByRole('button'));
fireEvent.click(screen.getByRole('button', { name: 'off-box' })); // picker option; selecting closes
expect(screen.getByText(/Existing objects do not move/)).toBeInTheDocument();
fireEvent.click(screen.getByRole('button', { name: 'Save changes' }));
// Usage was never scanned (baseState().usage === null) — the migrate
// prompt fires; this test is about the picker + PUT shape, not the
// prompt, so route new writes only (today's save behavior, unstripped).
await screen.findByRole('alertdialog');
fireEvent.click(screen.getByRole('button', { name: 'Just route new writes' }));
await screen.findByText('Storage configuration saved');
expect((putBody as { categories: Record<string, string> }).categories.files).toBe('off-box');
});
it('FE-MOB-MSTOR-003: a typed plaintext secret keeps Apply offered (no encryption-key gate)', async () => {
await renderPanel();
fireEvent.click(within(screen.getByTestId('m-storage-backend-off-box')).getByRole('button', { name: 'Edit' }));
fireEvent.change(screen.getByPlaceholderText('us-east-1'), { target: { value: 'eu-west-1' } }); // sanity: form is open
const secret = screen.getByDisplayValue(MASKED_SETTING_VALUE);
fireEvent.change(secret, { target: { value: 'sk-new' } });
expect(screen.getByRole('button', { name: 'Apply' })).toBeInTheDocument();
expect(screen.queryByRole('alert')).not.toBeInTheDocument();
});
it('FE-MOB-MSTOR-004: a 400 renders the server message verbatim', async () => {
await renderPanel();
server.use(http.put('/api/admin/storage', () => HttpResponse.json({ error: 'registry says no' }, { status: 400 })));
fireEvent.click(within(screen.getByTestId('m-storage-category-files')).getByRole('button'));
fireEvent.click(screen.getByRole('button', { name: 'off-box' }));
fireEvent.click(screen.getByRole('button', { name: 'Save changes' }));
// Usage was never scanned — the migrate prompt fires first; route new
// writes only, matching this test's focus on the save-error rendering.
await screen.findByRole('alertdialog');
fireEvent.click(screen.getByRole('button', { name: 'Just route new writes' }));
expect(await screen.findByText('registry says no')).toBeInTheDocument();
});
it('FE-MOB-MSTOR-005: renaming onto another existing backend warns and blocks Apply', async () => {
await renderPanel();
fireEvent.click(within(screen.getByTestId('m-storage-backend-off-box')).getByRole('button', { name: 'Edit' }));
const nameInput = screen.getByDisplayValue('off-box');
fireEvent.change(nameInput, { target: { value: 'uploads-local' } });
expect(screen.getByText(/A backend named uploads-local already exists/)).toBeInTheDocument();
expect(screen.getByRole('button', { name: 'Apply' })).toBeDisabled();
});
it('FE-MOB-MSTOR-006: mirrors fold — no mirror card, primary and replica decorated', async () => {
await renderPanel(mirroredState());
expect(screen.queryByTestId('m-storage-backend-mirror')).not.toBeInTheDocument();
const primary = screen.getByTestId('m-storage-backend-backups-local');
expect(within(primary).getByText('Mirrored to: off-box')).toBeInTheDocument();
const replica = screen.getByTestId('m-storage-backend-off-box');
expect(within(replica).getByText('Replica of: backups-local')).toBeInTheDocument();
});
it('FE-MOB-MSTOR-007: toggling a target on a primary synthesizes the mirror in the PUT', async () => {
let putBody: unknown;
await renderPanel();
server.use(
http.put('/api/admin/storage', async ({ request }) => {
putBody = await request.json();
return HttpResponse.json(baseState());
}),
);
fireEvent.click(within(screen.getByTestId('m-storage-backend-backups-local')).getByRole('button', { name: 'Edit' }));
fireEvent.click(screen.getByRole('switch', { name: 'off-box' }));
expect(screen.getByText(/slows every upload/)).toBeInTheDocument();
fireEvent.click(screen.getByRole('button', { name: 'Apply' }));
fireEvent.click(screen.getByRole('button', { name: 'Save changes' }));
await screen.findByText('Storage configuration saved');
const body = putBody as { backends: Array<{ name: string; type: string; options: unknown }>; categories: Record<string, string> };
expect(body.backends.find((b) => b.name === 'backups-local-mirror')!.options).toEqual({
primary: 'backups-local', replicas: ['off-box'],
});
expect(body.categories.backups).toBe('backups-local-mirror');
});
it('FE-MOB-MSTOR-008: the category picker deals in primaries and warns on cache categories', async () => {
let putBody: unknown;
await renderPanel(mirroredState());
server.use(
http.put('/api/admin/storage', async ({ request }) => {
putBody = await request.json();
return HttpResponse.json(mirroredState());
}),
);
fireEvent.click(within(screen.getByTestId('m-storage-category-places')).getByRole('button'));
fireEvent.click(within(screen.getByRole('dialog', { name: 'Place images' })).getByRole('button', { name: 'backups-local' })); // picker option = primary name
// getByText(/re-fetchable/) is now ambiguous — the photos-google/photos-trek category
// descriptions also say "re-fetchable". The cache warning is the only role="note" on
// screen here (the mirror-targets latency note only renders while a form is open).
expect(screen.getByRole('note')).toHaveTextContent(/re-fetchable/);
fireEvent.click(screen.getByRole('button', { name: 'Save changes' }));
// Usage was never scanned — the migrate prompt fires first; route new
// writes only, matching this test's focus on the picker/cache-warning.
await screen.findByRole('alertdialog');
fireEvent.click(screen.getByRole('button', { name: 'Just route new writes' }));
await screen.findByText('Storage configuration saved');
expect((putBody as { categories: Record<string, string> }).categories.places).toBe('mirror');
});
it('FE-MOB-MSTOR-009: category rows show display name, id badge and description; photos is gone', async () => {
await renderPanel();
const files = screen.getByTestId('m-storage-category-files');
// The testid wraps only the select row — assert name/badge/description through the field around it.
expect(screen.getByText('Trip documents')).toBeInTheDocument();
expect(screen.getByText('files')).toBeInTheDocument();
expect(screen.getByText(/tickets, PDFs, booking confirmations/)).toBeInTheDocument();
expect(within(files).getByRole('button')).toBeInTheDocument(); // row still tappable
expect(screen.queryByTestId('m-storage-category-photos')).not.toBeInTheDocument();
});
it('FE-MOB-MSTOR-010: usage renders on rows and the header; never-computed shows the compute prompt', async () => {
const usage = {
computedAt: Date.now() - 3_600_000,
categories: Object.fromEntries(
(['files', 'journey', 'covers', 'avatars', 'places', 'photos-google', 'photos-trek', 'backups'] as const).map(
(c) => [c, { objects: 2, bytes: 1024 * 1024 }],
),
),
legacyPhotos: { objects: 0, bytes: 0 },
};
usage.categories.files = { objects: 3, bytes: 2048 };
await renderPanel({ ...baseState(), usage } as StorageAdminState);
expect(screen.getByText(/Usage computed/)).toBeInTheDocument();
expect(within(screen.getByTestId('m-storage-backend-backups-local')).getByText(/2 objects · 1\.0 MB/)).toBeInTheDocument();
// Per-category usage line (the testid wraps only the select row, so assert by text).
expect(screen.getByText('3 objects · 2.0 KB')).toBeInTheDocument();
});
it('FE-MOB-MSTOR-011: Sync now → running → done via the test poll', async () => {
let polls = 0;
await renderPanel(mirroredState());
server.use(
http.post('/api/admin/storage/backends/mirror/backfill', () => HttpResponse.json({ started: true })),
http.get('/api/admin/storage', () => {
polls += 1;
const state = mirroredState();
(state as StorageAdminState).backfills =
polls < 3
? [{ backend: 'mirror', status: 'running', done: 1, total: 4, copied: 1, skipped: 0, failed: 0, deleted: 0, startedAt: 1 }]
: [{ backend: 'mirror', status: 'done', done: 4, total: 4, copied: 4, skipped: 0, failed: 0, deleted: 2, startedAt: 1, finishedAt: 2 }];
return HttpResponse.json(state);
}),
);
const row = screen.getByTestId('m-storage-backend-backups-local');
fireEvent.click(within(row).getByRole('button', { name: 'Sync now' }));
await within(row).findByText(/Syncing… 1\/4/);
await within(row).findByText(/Sync finished: 4 copied, 2 deleted, 0 failed/);
});
it('FE-MOB-MSTOR-012: a save that added targets raises the sync prompt', async () => {
await renderPanel();
server.use(http.put('/api/admin/storage', async () => HttpResponse.json(mirroredState())));
fireEvent.click(within(screen.getByTestId('m-storage-backend-backups-local')).getByRole('button', { name: 'Edit' }));
fireEvent.click(screen.getByRole('switch', { name: 'off-box' }));
fireEvent.click(screen.getByRole('button', { name: 'Apply' }));
fireEvent.click(screen.getByRole('button', { name: 'Save changes' }));
await screen.findByText(/Existing objects are not replicated yet/);
});
it('FE-MOB-MSTOR-013: reassigning a populated category prompts before saving; Move strips it from the PUT and POSTs the migration after save', async () => {
let putBody: unknown;
let putCalled = false;
let migrationBody: unknown;
const usage = {
computedAt: Date.now(),
categories: { files: { objects: 3, bytes: 3072 } },
legacyPhotos: { objects: 0, bytes: 0 },
};
await renderPanel({ ...baseState(), usage } as StorageAdminState);
server.use(
http.put('/api/admin/storage', async ({ request }) => {
putCalled = true;
putBody = await request.json();
const saved = baseState();
saved.categories.files = { backend: 'off-box', source: 'settings' };
return HttpResponse.json(saved);
}),
http.post('/api/admin/storage/migrations', async ({ request }) => {
migrationBody = await request.json();
return HttpResponse.json({ started: true });
}),
);
fireEvent.click(within(screen.getByTestId('m-storage-category-files')).getByRole('button'));
fireEvent.click(screen.getByRole('button', { name: 'off-box' }));
fireEvent.click(screen.getByRole('button', { name: 'Save changes' }));
expect(await screen.findByRole('alertdialog')).toBeInTheDocument();
expect(putCalled).toBe(false);
expect(screen.getByText(/Trip documents: 3 objects \(3\.0 KB\) from uploads-local to off-box/)).toBeInTheDocument();
fireEvent.click(screen.getByRole('button', { name: 'Move existing objects' }));
await screen.findByText('Storage configuration saved');
// files is default-sourced (uploads-local) in baseState() — stripping restores "no override".
expect((putBody as StorageConfig).categories.files).toBeUndefined();
expect(migrationBody).toEqual({ category: 'files', to: 'off-box' });
expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument();
});
it('FE-MOB-MSTOR-014: Just route new writes saves the reassignment as-is, no migration POST', async () => {
let putBody: unknown;
let migrationPosted = false;
const usage = {
computedAt: Date.now(),
categories: { files: { objects: 3, bytes: 3072 } },
legacyPhotos: { objects: 0, bytes: 0 },
};
await renderPanel({ ...baseState(), usage } as StorageAdminState);
server.use(
http.put('/api/admin/storage', async ({ request }) => {
putBody = await request.json();
const saved = baseState();
saved.categories.files = { backend: 'off-box', source: 'settings' };
return HttpResponse.json(saved);
}),
http.post('/api/admin/storage/migrations', () => {
migrationPosted = true;
return HttpResponse.json({ started: true });
}),
);
fireEvent.click(within(screen.getByTestId('m-storage-category-files')).getByRole('button'));
fireEvent.click(screen.getByRole('button', { name: 'off-box' }));
fireEvent.click(screen.getByRole('button', { name: 'Save changes' }));
await screen.findByRole('alertdialog');
fireEvent.click(screen.getByRole('button', { name: 'Just route new writes' }));
await screen.findByText('Storage configuration saved');
expect((putBody as StorageConfig).categories.files).toBe('off-box');
expect(migrationPosted).toBe(false);
expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument();
});
it('FE-MOB-MSTOR-015: two candidates queue sequentially: the second POSTs only after the first turns terminal', async () => {
const usage = {
computedAt: Date.now(),
categories: {
files: { objects: 3, bytes: 3072 },
journey: { objects: 2, bytes: 2048 },
},
legacyPhotos: { objects: 0, bytes: 0 },
};
await renderPanel({ ...baseState(), usage } as StorageAdminState);
const savedState = baseState();
savedState.categories.files = { backend: 'off-box', source: 'settings' };
savedState.categories.journey = { backend: 'off-box', source: 'settings' };
const migrationPosts: Array<{ category: StorageCategory; to: string }> = [];
// Held 'running' until the test explicitly flips this — deterministic,
// unlike counting poll ticks against the 50ms test interval.
let firstDone = false;
let postCount = 0;
server.use(
http.put('/api/admin/storage', () => HttpResponse.json(savedState)),
http.post('/api/admin/storage/migrations', async ({ request }) => {
postCount += 1;
const body = (await request.json()) as { category: StorageCategory; to: string };
migrationPosts.push(body);
// Pins the single-flight fix: the second POST must never arrive
// before the first migration is server-confirmed terminal — a
// regression here means the queue's synchronous re-fire raced ahead
// of admin.state showing the first as done.
if (postCount === 2) expect(firstDone).toBe(true);
return HttpResponse.json({ started: true });
}),
http.get('/api/admin/storage', () => {
const state = { ...savedState };
state.migrations =
migrationPosts.length === 0
? []
: [
{
category: migrationPosts[0]!.category, from: 'uploads-local', to: migrationPosts[0]!.to,
status: firstDone ? 'done' : 'running', done: firstDone ? 3 : 1, total: 3,
copied: firstDone ? 3 : 1, skipped: 0, failed: 0, startedAt: 1,
...(firstDone ? { finishedAt: 2 } : {}),
},
];
return HttpResponse.json(state);
}),
);
fireEvent.click(within(screen.getByTestId('m-storage-category-files')).getByRole('button'));
fireEvent.click(within(screen.getByRole('dialog', { name: 'Trip documents' })).getByRole('button', { name: 'off-box' }));
fireEvent.click(within(screen.getByTestId('m-storage-category-journey')).getByRole('button'));
fireEvent.click(within(screen.getByRole('dialog', { name: 'Journey photos' })).getByRole('button', { name: 'off-box' }));
fireEvent.click(screen.getByRole('button', { name: 'Save changes' }));
await screen.findByRole('alertdialog');
fireEvent.click(screen.getByRole('button', { name: 'Move existing objects' }));
await screen.findByText('Storage configuration saved');
await waitFor(() => expect(postCount).toBe(1));
expect(migrationPosts[0]!.category).toBe('files');
// At the moment the first POST arrives, exactly one POST has happened —
// and the second must not fire while the first is still running, no
// matter how many poll ticks elapse.
expect(postCount).toBe(1);
await new Promise((r) => setTimeout(r, 200));
expect(postCount).toBe(1);
// Once the first turns terminal, the second is dequeued — the POST
// handler's own assertion (above) pins that it never arrives early.
firstDone = true;
await waitFor(() => expect(postCount).toBe(2), { timeout: 2000 });
expect(migrationPosts[1]!.category).toBe('journey');
});
it('FE-MOB-MSTOR-016: a done migration row shows the reclaimable line; a running one shows progress + cancel wired to DELETE', async () => {
let cancelledCategory: string | null = null;
const state = baseState();
(state as StorageAdminState).migrations = [
{
category: 'files', from: 'uploads-local', to: 'off-box',
status: 'running', done: 2, total: 5, copied: 1, skipped: 1, failed: 0, startedAt: 1,
},
{
category: 'journey', from: 'uploads-local', to: 'off-box',
status: 'done', done: 4, total: 4, copied: 4, skipped: 0, failed: 0, startedAt: 1, finishedAt: 2,
reclaimable: { objects: 4, bytes: 4096 },
},
];
await renderPanel(state);
server.use(
http.delete('/api/admin/storage/migrations/files', () => {
cancelledCategory = 'files';
return HttpResponse.json({ cancelled: true });
}),
);
expect(screen.getByText(/Moving Trip documents… 2\/5/)).toBeInTheDocument();
expect(screen.getByText(/Move finished: 4 copied, 0 skipped/)).toBeInTheDocument();
expect(screen.getByText(/4 objects \(4\.0 KB\) remain on uploads-local — reclaim manually/)).toBeInTheDocument();
fireEvent.click(screen.getByRole('button', { name: 'Cancel move' }));
await waitFor(() => expect(cancelledCategory).toBe('files'));
});
it('FE-MOB-MSTOR-017: a failed save disarms the sync-prompt snapshot', async () => {
await renderPanel();
server.use(http.put('/api/admin/storage', () => HttpResponse.json({ error: 'nope' }, { status: 400 })));
fireEvent.click(within(screen.getByTestId('m-storage-backend-backups-local')).getByRole('button', { name: 'Edit' }));
fireEvent.click(screen.getByRole('switch', { name: 'off-box' }));
fireEvent.click(screen.getByRole('button', { name: 'Apply' }));
fireEvent.click(screen.getByRole('button', { name: 'Save changes' }));
await screen.findByText('nope');
// The draft-only mirror (never saved) still renders its "Sync now" —
// clicking it drives a genuine admin.state refresh (via refreshState())
// that lands a world whose mirror targets grew. With the pending-prompt
// snapshot disarmed by the failed save, the sync prompt must never appear.
server.use(
http.post('/api/admin/storage/backends/backups-local-mirror/backfill', () => HttpResponse.json({ started: true })),
http.get('/api/admin/storage', () => HttpResponse.json(mirroredState())),
);
fireEvent.click(within(screen.getByTestId('m-storage-backend-backups-local')).getByRole('button', { name: 'Sync now' }));
await new Promise((r) => setTimeout(r, 100));
expect(screen.queryByText(/Existing objects are not replicated yet/)).not.toBeInTheDocument();
});
});
+64
View File
@@ -0,0 +1,64 @@
# Manual, license-gated live-provider test for the env-gated S3 driver
# contract suite, for local/operator use. CI runs this exact suite for
# same-repo pull requests, via the MINIO_LICENSE_KEY repo secret (see
# .github/workflows/test.yml's s3-contract job) — this compose file stays
# the manual procedure for local runs and for fork PRs, which don't get
# repo secrets and so still need it run by hand.
#
# MinIO fully dropped the community line (quay.io/minio/minio gets no
# further updates). The maintained successor is AIStor
# (quay.io/minio/aistor/minio), which requires a license key to boot at all.
#
# REQUIRED: a `minio-licence.key` file at the repo root (same directory as
# this compose file). It is git-ignored and must never be committed — obtain
# your own free license key from MinIO (https://min.io) and place it there.
# The AIStor container will not start without it.
#
# Verified empirically against RELEASE.2026-08-07T18-34-35Z:
# - MINIO_ROOT_USER / MINIO_ROOT_PASSWORD still set the root credentials
# (unchanged from community MinIO).
# - Command shape: the image entrypoint expects `minio` as its first
# argument, so `command: minio server /data --license /minio.license`
# is passed through as-is.
# - `mc ready local` works as a healthcheck (mc ships in the image; curl
# does not).
#
# 127.0.0.1 is load-bearing on both sides: aws-lite picks path-style only
# for IP/localhost hosts (stock MinIO/AIStor only speaks path-style, no
# MINIO_DOMAIN here), and the port publish below binds 127.0.0.1 only — not
# 0.0.0.0 — so this fixed-credential test instance is never reachable from
# the LAN.
#
# Procedure:
# docker compose -f docker-compose.minio-test.yml up -d
# docker compose -f docker-compose.minio-test.yml ps # wait for "healthy"
# cd server && TREK_TEST_S3_ENDPOINT=http://127.0.0.1:9000 \
# TREK_TEST_S3_BUCKET=trek-test TREK_TEST_S3_ACCESS_KEY_ID=trekci \
# TREK_TEST_S3_SECRET_ACCESS_KEY=trekci-secret \
# npx vitest run tests/unit/nest/storage/s3.driver.contract.test.ts
# (optional) UI check: with the app running against this instance, open
# Admin → Storage, add an S3 backend (endpoint http://127.0.0.1:9000,
# bucket trek-test, access key trekci, secret trekci-secret), press Test
# on its row and confirm "Connection OK".
# docker compose -f docker-compose.minio-test.yml down -v
services:
minio:
# Docker Hub's minio/minio repo is archived, and quay.io/minio/minio (the
# former community line) receives no further updates — MinIO dropped it
# entirely. AIStor is the maintained successor; it requires a license
# (see header above). Pinned tag.
image: quay.io/minio/aistor/minio:RELEASE.2026-08-07T18-34-35Z
command: minio server /data --license /minio.license
ports:
- "127.0.0.1:9000:9000"
- "127.0.0.1:9001:9001"
volumes:
- ./minio-licence.key:/minio.license:ro
environment:
- MINIO_ROOT_USER=trekci
- MINIO_ROOT_PASSWORD=trekci-secret
healthcheck:
test: ["CMD", "mc", "ready", "local"] # curl was removed from the image
interval: 2s
timeout: 5s
retries: 15
+1
View File
@@ -53,6 +53,7 @@ services:
volumes:
- ./data:/app/data
- ./uploads:/app/uploads
# - ./storage-config.json:/app/data/storage-config.json:ro # optional seed-once storage provisioning (README → Storage)
restart: unless-stopped
healthcheck:
test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/health"]
+501
View File
@@ -394,6 +394,404 @@
"integrity": "sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q==",
"license": "MIT"
},
"node_modules/@aws-lite/client": {
"version": "0.23.7",
"resolved": "https://registry.npmjs.org/@aws-lite/client/-/client-0.23.7.tgz",
"integrity": "sha512-Qo/lq96DS8UOg1yFccolMuKG1hKnNStXVYhzVl4tZk/7m+OtANVk2JB8iuYJISc4DF7Dm3Qy+e91L6BGJ1dfkw==",
"license": "Apache-2.0",
"workspaces": [
"plugins/acm",
"plugins/acm/types",
"plugins/apigateway",
"plugins/apigateway/types",
"plugins/apigatewaymanagementapi",
"plugins/apigatewaymanagementapi/types",
"plugins/apigatewayv2",
"plugins/apigatewayv2/types",
"plugins/cloudformation",
"plugins/cloudformation/types",
"plugins/cloudfront",
"plugins/cloudfront/types",
"plugins/cloudwatch-logs",
"plugins/cloudwatch-logs/types",
"plugins/dynamodb",
"plugins/dynamodb/types",
"plugins/iam",
"plugins/iam/types",
"plugins/lambda",
"plugins/lambda/types",
"plugins/organizations",
"plugins/organizations/types",
"plugins/rds-data",
"plugins/rds-data/types",
"plugins/route53",
"plugins/route53/types",
"plugins/s3",
"plugins/s3/types",
"plugins/sns",
"plugins/sns/types",
"plugins/sqs",
"plugins/sqs/types",
"plugins/ssm",
"plugins/ssm/types",
"plugins/sts",
"plugins/sts/types"
],
"dependencies": {
"aws4": "^1.13.2"
},
"engines": {
"node": ">=16"
}
},
"node_modules/@aws-lite/s3": {
"version": "0.2.7",
"resolved": "https://registry.npmjs.org/@aws-lite/s3/-/s3-0.2.7.tgz",
"integrity": "sha512-NPQZXMcHIwrcyZWrHUeoyjl3353xIoYA/6XoUu4KJs2NQKqF6xx5YO+7nXE05vSSygNSgUHetaue/x2vr2430A==",
"license": "Apache-2.0",
"engines": {
"node": ">=16"
}
},
"node_modules/@aws-lite/s3-types": {
"version": "0.2.8",
"resolved": "https://registry.npmjs.org/@aws-lite/s3-types/-/s3-types-0.2.8.tgz",
"integrity": "sha512-e+wf4m83Qk9d+4e/on3d6dSkqC2eueL6LM7iLpZQ+U4f6Ey5tRuvirqbj8iOQcUGxJpP7UiVXAv+P61WIql/OA==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/client-s3": "3"
},
"engines": {
"node": ">=16"
}
},
"node_modules/@aws-sdk/checksums": {
"version": "3.1000.28",
"resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1000.28.tgz",
"integrity": "sha512-VCpnmyHQ1IH49ni3LXnQj7DPr7rmcJmzYeiCkYdCcfgNtkvOj38cdcL9lapBWoItZWFACJPFJlymqC7/gem3Gw==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.977.8",
"@aws-sdk/types": "^3.974.4",
"@smithy/core": "^3.31.1",
"@smithy/types": "^4.16.1",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/client-s3": {
"version": "3.1113.0",
"resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1113.0.tgz",
"integrity": "sha512-NRqdtohoMRyWkEeeznfG1KPN08dclCbl+HFuLPB2v8qPcgoNmTFlLKl9ELiR6hsGXQ4Ur3qvRnoJVEk8ND74pg==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/checksums": "^3.1000.28",
"@aws-sdk/core": "^3.977.8",
"@aws-sdk/credential-provider-node": "^3.972.80",
"@aws-sdk/middleware-sdk-s3": "^3.972.74",
"@aws-sdk/signature-v4-multi-region": "^3.996.45",
"@aws-sdk/types": "^3.974.4",
"@smithy/core": "^3.31.1",
"@smithy/fetch-http-handler": "^5.6.13",
"@smithy/node-http-handler": "^4.9.13",
"@smithy/types": "^4.16.1",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/core": {
"version": "3.977.8",
"resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.977.8.tgz",
"integrity": "sha512-7+Kcrkvrk9lM/m7jRhHpT4jCdvzGHsuaSRbF8TdzzkY1mRzp/Ogwf9c7H29k4gGhey0BBWhCWr16+t0J61gwmg==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/types": "^3.974.4",
"@aws-sdk/xml-builder": "^3.972.39",
"@aws/lambda-invoke-store": "^0.3.0",
"@smithy/core": "^3.31.1",
"@smithy/signature-v4": "^5.6.12",
"@smithy/types": "^4.16.1",
"bowser": "^2.11.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-env": {
"version": "3.972.69",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.69.tgz",
"integrity": "sha512-AreCFzcB4kH2HF9031Ot0jSJr3KXvRg6e8uDeub20JEVdZU3Bv0sTq1plc7VsT3KiqutlzH7l0j50UcCWHUioA==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.977.8",
"@aws-sdk/types": "^3.974.4",
"@smithy/core": "^3.31.1",
"@smithy/types": "^4.16.1",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-http": {
"version": "3.972.71",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.71.tgz",
"integrity": "sha512-A8ObcqVmDMnk4F9NozZ7JwmUu9Q4xyBJkmyq1C5U+wNM9ht9J7+EuuyabsLWXZnOoTqFaJuYBYTKf5CTipkEjA==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.977.8",
"@aws-sdk/types": "^3.974.4",
"@smithy/core": "^3.31.1",
"@smithy/fetch-http-handler": "^5.6.13",
"@smithy/node-http-handler": "^4.9.13",
"@smithy/types": "^4.16.1",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-ini": {
"version": "3.973.14",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.14.tgz",
"integrity": "sha512-7c+Wti2LsERNWMfm7ySz3/6RPopFW3Nmn7s63Xpcq6R/tRuY5hpvkHA2xVgi5ukJbvok9l0IDtVEvqTtg+X7dw==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.977.8",
"@aws-sdk/credential-provider-env": "^3.972.69",
"@aws-sdk/credential-provider-http": "^3.972.71",
"@aws-sdk/credential-provider-login": "^3.972.76",
"@aws-sdk/credential-provider-process": "^3.972.69",
"@aws-sdk/credential-provider-sso": "^3.973.13",
"@aws-sdk/credential-provider-web-identity": "^3.972.75",
"@aws-sdk/nested-clients": "^3.997.43",
"@aws-sdk/types": "^3.974.4",
"@smithy/core": "^3.31.1",
"@smithy/credential-provider-imds": "^4.4.16",
"@smithy/types": "^4.16.1",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-login": {
"version": "3.972.76",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.76.tgz",
"integrity": "sha512-LVixwOnEJfrrfKHeZjBA8pIMTZjNDq8ak8VpcoWUuCJDrSnBNU8POJksULMgvN089P0MXtQYH2Zs627/MK1K0g==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.977.8",
"@aws-sdk/nested-clients": "^3.997.43",
"@aws-sdk/types": "^3.974.4",
"@smithy/core": "^3.31.1",
"@smithy/types": "^4.16.1",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-node": {
"version": "3.972.80",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.80.tgz",
"integrity": "sha512-bE2qh8ww4iClO1jHsBXdOE8FUgzDbdxbyorNjSCoPSkQd51k3jODItuPZfuwcLHZqDXsH+bI4AMHhqtuyR7mSg==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/credential-provider-env": "^3.972.69",
"@aws-sdk/credential-provider-http": "^3.972.71",
"@aws-sdk/credential-provider-ini": "^3.973.14",
"@aws-sdk/credential-provider-process": "^3.972.69",
"@aws-sdk/credential-provider-sso": "^3.973.13",
"@aws-sdk/credential-provider-web-identity": "^3.972.75",
"@aws-sdk/types": "^3.974.4",
"@smithy/core": "^3.31.1",
"@smithy/credential-provider-imds": "^4.4.16",
"@smithy/types": "^4.16.1",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-process": {
"version": "3.972.69",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.69.tgz",
"integrity": "sha512-9kpTNdZTrcqXTfhxM7fgl9Z68ek3Fu5oe3Yf+A/pJGibEqpgZxz2tSY7SinmyCIU2PJ+ygY4FPoBBnLpocMtrQ==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.977.8",
"@aws-sdk/types": "^3.974.4",
"@smithy/core": "^3.31.1",
"@smithy/types": "^4.16.1",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-sso": {
"version": "3.973.13",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.13.tgz",
"integrity": "sha512-Oc81qauMPzUoTnAS2YKpNwY6sY/LUyQTEeaf6yP197WMxkEBQfcKLR1MFpD7+pNTubXnfkH6gwpji+Gc7iyD2Q==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.977.8",
"@aws-sdk/nested-clients": "^3.997.43",
"@aws-sdk/token-providers": "3.1111.0",
"@aws-sdk/types": "^3.974.4",
"@smithy/core": "^3.31.1",
"@smithy/types": "^4.16.1",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-web-identity": {
"version": "3.972.75",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.75.tgz",
"integrity": "sha512-YPN6uoGDgjjjeVFZrcOeCJqmB6zpXoeeNgIjqe+DexJaWqdjVfCCe+VAZwli9Z2h8KhFW8oxkO39emQ1tyz/Mw==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.977.8",
"@aws-sdk/nested-clients": "^3.997.43",
"@aws-sdk/types": "^3.974.4",
"@smithy/core": "^3.31.1",
"@smithy/types": "^4.16.1",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/middleware-sdk-s3": {
"version": "3.972.74",
"resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.74.tgz",
"integrity": "sha512-2lzoV2z2QO5KJZYGOCnIZ1WVQgzMECvwuzr1xb034a++8QW4U4eGrmC2u4yg1xvNv4TLL/Uv5DLyuAiw0b9z7Q==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.977.8",
"@aws-sdk/signature-v4-multi-region": "^3.996.45",
"@aws-sdk/types": "^3.974.4",
"@smithy/core": "^3.31.1",
"@smithy/types": "^4.16.1",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/nested-clients": {
"version": "3.997.43",
"resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.43.tgz",
"integrity": "sha512-bit+VpqWNyi3wHxFoTsTliNXimCSL2r2OeDTm7ZrG+YsTZ2D7ofDJ6r/t9PVBn80i6/v0X2h9Tgw6QP2MAKfPw==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.977.8",
"@aws-sdk/signature-v4-multi-region": "^3.996.45",
"@aws-sdk/types": "^3.974.4",
"@smithy/core": "^3.31.1",
"@smithy/fetch-http-handler": "^5.6.13",
"@smithy/node-http-handler": "^4.9.13",
"@smithy/types": "^4.16.1",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/signature-v4-multi-region": {
"version": "3.996.45",
"resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.45.tgz",
"integrity": "sha512-bBuyztukzXq6plzFGHAWiQt0QXo+HL8b8lX5cFTzkez/74PtS1c0qPFCIVuHkyoT+miH2qOjAcm1/yoro2ESPA==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/types": "^3.974.4",
"@smithy/signature-v4": "^5.6.12",
"@smithy/types": "^4.16.1",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/token-providers": {
"version": "3.1111.0",
"resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1111.0.tgz",
"integrity": "sha512-JfljgoVtl+s3Qy21n9a7Z48uCQaOXcN74KJ3TEQfPoB293GrXFSt6HSQJF1sTZ8c/5QedEvd3NjJQMO4u9qa5A==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.977.8",
"@aws-sdk/nested-clients": "^3.997.43",
"@aws-sdk/types": "^3.974.4",
"@smithy/core": "^3.31.1",
"@smithy/types": "^4.16.1",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/types": {
"version": "3.974.4",
"resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.4.tgz",
"integrity": "sha512-dSFDNG00MEz0/xl5gxL62giLd1iYyJsTxZ1I1DOj6lC+bbgLB4TRsYClJg3b62dhXT1uATzsTNXPnC+33EJV3A==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@smithy/types": "^4.16.1",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/xml-builder": {
"version": "3.972.39",
"resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.39.tgz",
"integrity": "sha512-FTti8DS5MMWXNUWiRwXAJeYS+0GHHiMy0+7XOhcwk63ILHmfS2UFy2z/HNpZCSOJJ3P3dnWY6hfYNW3DF0nXUA==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@smithy/types": "^4.16.1",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws/lambda-invoke-store": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz",
"integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==",
"dev": true,
"license": "Apache-2.0",
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@babel/code-frame": {
"version": "7.29.7",
"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz",
@@ -6774,6 +7172,93 @@
"node": ">=20.0.0"
}
},
"node_modules/@smithy/core": {
"version": "3.33.2",
"resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.33.2.tgz",
"integrity": "sha512-CUGXpnPkVdjUCbix+83sWLW9VFgQOm44MDOx/ihITJMAnOZKvL8YYIc7DR9pP/tZ8CIRvMiON/TucvygqbHO3w==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@smithy/types": "^4.17.2",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/credential-provider-imds": {
"version": "4.5.2",
"resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.5.2.tgz",
"integrity": "sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@smithy/core": "^3.33.2",
"@smithy/types": "^4.17.2",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/fetch-http-handler": {
"version": "5.7.2",
"resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.7.2.tgz",
"integrity": "sha512-nZyWTmSpJEXl6VtWVMBJve/7x12DZu6sIX1z1a+ZMaHlQQRs9Zpu6NbTe/gmxYXVRpkjxyDYpZ5gx2IM6f/Wkw==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@smithy/core": "^3.33.2",
"@smithy/types": "^4.17.2",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/node-http-handler": {
"version": "4.11.2",
"resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.11.2.tgz",
"integrity": "sha512-avwAh9HM3h2lcfjvP3zYIZGf+XVgLQ91wOJ2qoFbNpW1UZeZb33aGlhTZvtkANHfcGhJroRY64525OjfgOg30g==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@smithy/core": "^3.33.2",
"@smithy/types": "^4.17.2",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/signature-v4": {
"version": "5.7.2",
"resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.7.2.tgz",
"integrity": "sha512-P7Ki6px6OOrxVtx8K7nLmyx4SlXUW/uTKDdMG44UHefmPGSRMBKe2v+TM59WdLcpUIrBrnuCsIqiM2MbsZjmhw==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@smithy/core": "^3.33.2",
"@smithy/types": "^4.17.2",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/types": {
"version": "4.17.2",
"resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.17.2.tgz",
"integrity": "sha512-FOKpVZob9MPTn2znRzGrnsMHv7BOsKVw3XiP/cOyYLDVZ9qKp4nifIiSCuUU/fIj5Vu0UOAxCFr+qRAtG0NUkA==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@standard-schema/spec": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz",
@@ -8668,6 +9153,12 @@
"node": ">=6.0.0"
}
},
"node_modules/aws4": {
"version": "1.13.2",
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.13.2.tgz",
"integrity": "sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw==",
"license": "MIT"
},
"node_modules/axios": {
"version": "1.18.0",
"resolved": "https://registry.npmjs.org/axios/-/axios-1.18.0.tgz",
@@ -9025,6 +9516,13 @@
"node": ">= 0.8"
}
},
"node_modules/bowser": {
"version": "2.14.1",
"resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz",
"integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==",
"dev": true,
"license": "MIT"
},
"node_modules/brace-expansion": {
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz",
@@ -21289,6 +21787,8 @@
"name": "@trek/server",
"version": "3.4.1",
"dependencies": {
"@aws-lite/client": "^0.23.7",
"@aws-lite/s3": "^0.2.7",
"@modelcontextprotocol/sdk": "^1.30.0",
"@nestjs/common": "^11.1.24",
"@nestjs/config": "^4.0.4",
@@ -21335,6 +21835,7 @@
"zod": "^4.3.6"
},
"devDependencies": {
"@aws-lite/s3-types": "^0.2.8",
"@eslint/js": "^10.0.1",
"@nestjs/testing": "^11.1.24",
"@swc/core": "^1.15.40",
+3
View File
@@ -25,6 +25,8 @@
"test:coverage": "vitest run --coverage"
},
"dependencies": {
"@aws-lite/client": "^0.23.7",
"@aws-lite/s3": "^0.2.7",
"@modelcontextprotocol/sdk": "^1.30.0",
"@nestjs/common": "^11.1.24",
"@nestjs/config": "^4.0.4",
@@ -81,6 +83,7 @@
"file-type": "^21.3.4"
},
"devDependencies": {
"@aws-lite/s3-types": "^0.2.8",
"@eslint/js": "^10.0.1",
"@nestjs/testing": "^11.1.24",
"@swc/core": "^1.15.40",
+62
View File
@@ -326,6 +326,68 @@ async function main() {
}
}
}
// --- app_settings: storage.backends ---
// Storage backend secrets live encrypted INSIDE the storage.backends JSON
// array (BACKENDS_KEY in src/nest/storage/storage-registry.service.ts is
// its own app_settings row — 'storage.categories' is a sibling row with no
// secrets), on the fields the shared type registry marks `secret` (only
// options.secretAccessKey for s3 today; src/nest/storage/storage-secrets.ts
// encrypts it with the same enc:v1: scheme as the api-key columns above).
// This script can't import storageSecretFields from @trek/shared (kept
// independent of the app, see the header note), so the field list per
// backend type is pinned here. Miss one and that backend's secret stays
// encrypted under the old key after rotation, then fails to decrypt on the
// next read (StorageBackendError) — pinned by
// tests/unit/db/migrate-encryption-parity.test.ts.
const STORAGE_BACKEND_SECRET_FIELDS: Record<string, readonly string[]> = {
s3: ['secretAccessKey'],
};
{
const row = db.prepare("SELECT value FROM app_settings WHERE key = 'storage.backends'").get() as
| { value: string }
| undefined;
if (row?.value) {
let backends: unknown;
try {
backends = JSON.parse(row.value);
} catch {
result.errors.push('app_settings.storage.backends: invalid JSON — skipping');
backends = undefined;
}
if (backends !== undefined) {
if (Array.isArray(backends)) {
let changed = false;
for (const backend of backends as Record<string, unknown>[]) {
if (!backend || typeof backend !== 'object') continue;
const type = typeof backend.type === 'string' ? backend.type : undefined;
const fields = type ? STORAGE_BACKEND_SECRET_FIELDS[type] : undefined;
if (!fields || fields.length === 0) continue;
const options = backend.options;
if (!options || typeof options !== 'object') continue;
const name = typeof backend.name === 'string' ? backend.name : '?';
for (const field of fields) {
const raw = (options as Record<string, unknown>)[field];
if (typeof raw !== 'string' || !raw) continue;
const newVal = migrateApiKeyValue(raw, `app_settings.storage.backends[${name}].${field}`);
if (newVal !== null) {
(options as Record<string, unknown>)[field] = newVal;
changed = true;
}
}
}
if (changed) {
db.prepare("UPDATE app_settings SET value = ? WHERE key = 'storage.backends'").run(
JSON.stringify(backends),
);
}
} else {
result.errors.push('app_settings.storage.backends: not an array — skipping');
}
}
}
}
// --- settings: per-user encrypted keys ---
const encryptedSettingKeys = ['webhook_url', 'ntfy_token', 'mapbox_access_token'];
const settingRows = db.prepare(
+4 -1
View File
@@ -68,9 +68,12 @@ PORT, HOST, TRUST_PROXY, SESSION_DURATION(_REMEMBER), MCP_SESSION_TTL,
MCP_MAX_SESSION_PER_USER, MCP_SSE_KEEPALIVE, TREK_PLUGIN_RPC_*/LOG_*/MAX_RSS_MB,
TREK_PLUGIN_REGISTRY_URL, TREK_WIKI_DIR*, TREK_PLACE_PHOTO_DIR, BACKUP_*,
TRANSIT_API_URL, LOG_LEVEL*, ALLOW_INTERNAL_NETWORK*, DEFAULT_LANGUAGE,
TREK_DB_FILE, TREK_DB_JOURNAL_MODE, TREK_DB_SYNCHRONOUS, ENCRYPTION_KEY.
TREK_DB_FILE, TREK_DB_JOURNAL_MODE, TREK_DB_SYNCHRONOUS, ENCRYPTION_KEY**.
(* frozen today because the consuming module captures it at import; tests that
override these set them at file top, before the SUT import.)
(** ENCRYPTION_KEY is boot-stable for the cipher key material itself —
`src/config.ts` resolves and freezes it at process start, per the exemption
below.)
**Runtime-toggled** (read live on every access via `readEnv()` /
`RuntimeEnvService`; tests mutate these mid-lifetime):
+7
View File
@@ -46,6 +46,13 @@ export function positiveIntOr(raw: string | undefined, fallback: number): number
return Number.isFinite(n) && n > 0 ? n : fallback;
}
/** Integer ≥ 0 (zero is meaningful — e.g. retries disabled), else the fallback. */
export function nonNegativeIntOr(raw: string | undefined, fallback: number): number {
if (raw === undefined || raw.trim() === '') return fallback;
const n = Number.parseInt(raw, 10);
return Number.isFinite(n) && n >= 0 ? n : fallback;
}
/** Comma-split + trim, null when unset — the websocket ALLOWED_ORIGINS variant (no empty-entry filter). */
export function csvList(raw: string | undefined): string[] | null {
return raw ? raw.split(',').map((o) => o.trim()) : null;
+5 -1
View File
@@ -16,6 +16,7 @@ import { validateBodyContracts } from './nest/common/validate-body-contracts';
import { validateRouteGuards } from './nest/common/validate-route-guards';
import { validateManagedRoutes } from './nest/common/validate-managed-routes';
import { TrekWsAdapter } from './nest/realtime/trek-ws.adapter';
import { StorageService } from './nest/storage/storage.service';
/**
* Builds the unified TREK NestJS application that serves the ENTIRE surface — the
@@ -84,7 +85,10 @@ export async function buildApp(): Promise<INestApplication> {
// config instead of reading process.env itself.
const http = app.get<ConfigType<typeof httpConfig>>(httpConfig.KEY);
applyGlobalMiddleware(instance, { http });
applyPlatformUploads(instance);
// Same pre-init consumption bridge as httpConfig above: the StorageService
// instance is resolvable before init, and the handlers only *register* here —
// per-request resolution runs after app.init() completed the registry load.
applyPlatformUploads(instance, app.get(StorageService));
// The SDK discovery router (+ its addon gate). Container-built so its deps
// are injected (same pre-init consumption bridge as httpConfig above), but
// applied here as a PATHLESS app.use: the SDK router matches absolute
+18 -2
View File
@@ -3895,8 +3895,8 @@ function runMigrations(db: Database.Database): void {
// its day are on the same trip. The table carries no trip_id and its two
// foreign keys only ask that the ids exist, so pairs that never belonged
// together could accumulate; the writer refuses them now, and this clears
// whatever an older build let through. Appended LAST — the array is
// index-addressed against schema_version.
// whatever an older build let through. This slot keeps the index it shipped
// with on dev — the array is index-addressed against schema_version.
() => {
db.exec(`
DELETE FROM reservation_day_positions
@@ -4067,6 +4067,22 @@ function runMigrations(db: Database.Database): void {
if (byName) update.run(byName, row.id);
}
},
// Storage slice 2 — collab note attachments historically stored 'files/<name>'
// in trip_files.filename while the file manager stored bare names in the same
// column; the storage layer addresses objects as category + bare name, so
// normalize the legacy rows. substr is 1-indexed: 7 drops the six chars of
// 'files/'. The LIKE guard makes it a no-op on already-bare rows.
//
// Appended LAST again, and for the same reason as the note this replaces:
// the array is index-addressed against schema_version, so a slot that has
// shipped in dev keeps the index it shipped with and anything from this
// branch goes after it. A database that already ran this migration at its
// pre-merge index replays it harmlessly (the LIKE guard) but skips whatever
// now occupies that index. Acceptable only because this branch has never
// been published; never do this with a released slot.
() => {
db.exec("UPDATE trip_files SET filename = substr(filename, 7) WHERE filename LIKE 'files/%'");
},
];
if (currentVersion < migrations.length) {
+9 -21
View File
@@ -10,28 +10,16 @@ import http from 'node:http';
import type { INestApplication } from '@nestjs/common';
import { buildApp, getHttpServer } from './bootstrap';
// Create upload and data directories on startup.
// Every uploads subdir the app writes to must be listed here (#1762): a dir
// created lazily on first upload needs write permission on the uploads mount
// point itself at request time, which fails with EACCES on Docker hosts whose
// bind-mounted uploads dir isn't writable by the runtime user — while every
// feature writing into an already-existing subdir keeps working. Creating them
// at boot turns that into one loud startup failure instead of a stray 500.
// Keep in sync with the `mkdir -p` list in the Dockerfile (guarded by
// tests/unit/uploads-dirs.test.ts).
const uploadsDir = path.join(__dirname, '../uploads');
const photosDir = path.join(uploadsDir, 'photos');
const filesDir = path.join(uploadsDir, 'files');
const coversDir = path.join(uploadsDir, 'covers');
const avatarsDir = path.join(uploadsDir, 'avatars');
const journeyDir = path.join(uploadsDir, 'journey');
const placesDir = path.join(uploadsDir, 'places');
const backupsDir = path.join(__dirname, '../data/backups');
// data/tmp is the driver-agnostic global scratch dir (restore-upload spool,
// mirror stream staging) and stays boot-created here. Driver-owned roots — the
// uploads/ category tree and data/backups — are ensured by LocalDriver.init on
// every storage-registry load (boot and reload(), storage-registry.service.ts),
// which keeps the #1762 EACCES guarantee: a non-writable bind mount still
// fails loudly at startup, inside app.init(), instead of as a stray 500 on
// first upload. The Dockerfile `mkdir -p` list is pinned to the registry's
// category prefixes by tests/unit/uploads-dirs.test.ts.
const tmpDir = path.join(__dirname, '../data/tmp');
[uploadsDir, photosDir, filesDir, coversDir, avatarsDir, journeyDir, placesDir, backupsDir, tmpDir].forEach(dir => {
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
});
if (!fs.existsSync(tmpDir)) fs.mkdirSync(tmpDir, { recursive: true });
import { getAppUrl, getMcpSafeUrl, readEnv } from './app-config';
+36 -12
View File
@@ -6,6 +6,41 @@ import cookieParser from 'cookie-parser';
import { readEnv, type AppEnv } from '../app-config';
import { logDebug, logWarn, logError } from '../nest/audit/audit-log.logger';
/**
* Field names redacted from request-log query/body dumps (case-insensitive —
* lookup lowercases the key first). `secretaccesskey` covers the S3 storage
* backend's secret field (storage-admin PUT bodies land in the same debug
* log line as any other request body).
*
* Exported (module-level, not per-request) so it can be unit-tested directly
* without threading LOG_LEVEL through the real logger pipeline.
*/
export const SENSITIVE_KEYS = new Set([
'password',
'new_password',
'current_password',
'token',
'jwt',
'authorization',
'cookie',
'client_secret',
'mfa_token',
'code',
'smtp_pass',
'secretaccesskey',
]);
/** Deep-redacts every key in `SENSITIVE_KEYS` (case-insensitive) from a request-log value. */
export function redact(value: unknown): unknown {
if (!value || typeof value !== 'object') return value;
if (Array.isArray(value)) return (value as unknown[]).map(redact);
const out: Record<string, unknown> = {};
for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
out[k] = SENSITIVE_KEYS.has(k.toLowerCase()) ? '[REDACTED]' : redact(v);
}
return out;
}
/**
* The global request pipeline shared by the legacy Express app and the NestJS
* instance. Both mount the *exact same* config so a request hitting a migrated
@@ -172,18 +207,7 @@ export function applyGlobalMiddleware(
app.use(cookieParser());
// Request logging with sensitive field redaction
const SENSITIVE_KEYS = new Set(['password', 'new_password', 'current_password', 'token', 'jwt', 'authorization', 'cookie', 'client_secret', 'mfa_token', 'code', 'smtp_pass']);
const redact = (value: unknown): unknown => {
if (!value || typeof value !== 'object') return value;
if (Array.isArray(value)) return (value as unknown[]).map(redact);
const out: Record<string, unknown> = {};
for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
out[k] = SENSITIVE_KEYS.has(k.toLowerCase()) ? '[REDACTED]' : redact(v);
}
return out;
};
// Request logging with sensitive field redaction (SENSITIVE_KEYS/redact above)
app.use((req: Request, res: Response, next: NextFunction) => {
if (req.path === '/api/health') return next();
const startedAt = Date.now();
+21
View File
@@ -125,6 +125,27 @@ tools, which never pass through an HTTP guard. In the five domains piloted for
answer 404 to anonymous callers, so the addon check must beat the 401.
- `common/demo-write.ts` — `isDemoWriteBlocked`, the demo-mode upload block. A
function and not a guard on purpose; see the gotcha below.
- `storage/` — the storage abstraction and its admin surface. `storage.types.ts`
defines the driver contract (opaque keys, no rename/append — S3 constraints
baked in) plus the `ServedCategory` seam: `STORAGE_CATEGORIES` (shared, 8,
configurable) + the legacy `photos` directory that is served and backed up
but not configurable. `drivers/` (local, s3, mirror) all pass one contract
suite (`tests/unit/nest/storage/storage-driver.contract.ts`).
`storage-registry.service.ts` merges built-ins ← env (`place-photos-local`)
← `app_settings` rows, validates semantics (references, no mirror nesting),
falls back to last-good/defaults at boot but NEVER on admin writes
(`preview()` runs the real build first), seeds once from
`data/storage-config.json`, and hot-swaps on `reload()`.
`storage.service.ts` is the facade every byte-path uses.
`storage-admin.{controller,service}.ts` (+ `storage-secrets.ts`,
`storage-probe.ts`) carry `api/admin/storage`: masked GET of the effective
world, full-document PUT (unmask → preview → encrypt → one transaction →
reload → audit), and per-target test probes with ephemeral drivers. Secrets
live `enc:v1:`-encrypted inside the `storage.backends` JSON (the implicit
key covers installs without an explicit `ENCRYPTION_KEY`). The client
presents mirrors as replicas-on-primary; the wire keeps explicit mirror
backends. `storage-jobs.service.ts` runs backfill and category-migration
jobs (one at a time, TTL'd statuses).
- `memories/photo-provider.ts` + `providers/` — the PhotoProvider seam (#584).
Dispatch to a photo backend was a `switch (photo.provider)` in three places
(`photo-resolver.service.ts` twice, `journey-public.controller.ts` once, where
+2 -1
View File
@@ -46,6 +46,7 @@ import { TripInviteModule } from './trip-invite/trip-invite.module';
import { TransitModule } from './transit/transit.module';
import { FeedsModule } from './feeds/feeds.module';
import { SettingsModule } from './settings/settings.module';
import { StorageModule } from './storage/storage.module';
import { BackupModule } from './backup/backup.module';
import { BookingImportModule } from './booking-import/booking-import.module';
import { ReservationImportModule } from './reservation-import/reservation-import.module';
@@ -74,7 +75,7 @@ import { RealtimeGatewayModule } from './realtime/realtime-gateway.module';
* migrated.
*/
@Module({
imports: [AppConfigModule, DatabaseModule, RealtimeModule, RealtimeGatewayModule, SchedulingModule, McpModule.forRoot({ accessPolicy: trekMcpAccessPolicy, validateAccess: trekMcpValidateAccess }), HealthModule, PlatformModule, McpTransportModule, WeatherModule, HelpModule, AirportsModule, ConfigModule, SystemNoticesModule, GeoModule, MapsModule, PlaceEnrichmentModule, CategoriesModule, TagsModule, NotificationsModule, AtlasModule, VacayModule, PackingModule, TodoModule, BudgetModule, ReservationsModule, DaysModule, DayNotesModule, AccommodationsModule, AssignmentsModule, PlacesModule, TripsModule, CollabModule, FilesModule, PhotosModule, MemoriesModule, AirtrailModule, JourneyModule, CollectionsModule, ShareModule, TripInviteModule, TransitModule, FeedsModule, SettingsModule, BackupModule, AuthModule, OidcModule, OauthModule, AdminModule, AddonsModule, AuditModule, PermissionsModule, PluginsModule, BookingImportModule, ReservationImportModule, LlmParseModule, ManagedExtModule],
imports: [AppConfigModule, DatabaseModule, RealtimeModule, RealtimeGatewayModule, SchedulingModule, McpModule.forRoot({ accessPolicy: trekMcpAccessPolicy, validateAccess: trekMcpValidateAccess }), HealthModule, PlatformModule, McpTransportModule, WeatherModule, HelpModule, AirportsModule, ConfigModule, SystemNoticesModule, GeoModule, MapsModule, PlaceEnrichmentModule, CategoriesModule, TagsModule, NotificationsModule, AtlasModule, VacayModule, PackingModule, TodoModule, BudgetModule, ReservationsModule, DaysModule, DayNotesModule, AccommodationsModule, AssignmentsModule, PlacesModule, TripsModule, CollabModule, FilesModule, PhotosModule, MemoriesModule, AirtrailModule, JourneyModule, CollectionsModule, ShareModule, TripInviteModule, TransitModule, FeedsModule, SettingsModule, StorageModule, BackupModule, AuthModule, OidcModule, OauthModule, AdminModule, AddonsModule, AuditModule, PermissionsModule, PluginsModule, BookingImportModule, ReservationImportModule, LlmParseModule, ManagedExtModule],
providers: [
// Default-deny: a route is authenticated unless it carries @Public() or
// @OptionalAuth(), or declares its own @UseGuards chain. Protection used to
+5
View File
@@ -18,6 +18,11 @@ const ACTION_LABELS: Record<string, string> = {
'admin.user_delete': 'deleted user',
'admin.plugin_retrust': "re-trusted a plugin's author signing key",
'admin.invite_create': 'created invite',
'admin.storage_update': 'updated storage configuration',
'admin.storage_test': 'tested a storage backend',
'admin.storage_backfill': 'started a storage sync',
'admin.storage_backfill_cancel': 'cancelled a storage sync',
'admin.storage_stats_refresh': 'refreshed storage usage stats',
'immich.private_ip_configured': 'configured Immich with private IP',
};
+18 -21
View File
@@ -17,15 +17,13 @@ import {
import { FileInterceptor } from '@nestjs/platform-express';
import { isDemoWriteBlocked, DEMO_WRITE_ERROR } from '../common/demo-write';
import { RuntimeEnvService } from '../app-config/runtime-env.service';
import { diskStorage } from 'multer';
import type { Options } from 'multer';
import type { Request, Response } from 'express';
import path from 'path';
import fs from 'fs';
import { v4 as uuid } from 'uuid';
import { AuthService } from './auth.service';
import { TokenService } from '../tokens/token.service';
import { UserProfileService } from './user-profile.service';
import { avatarDir } from './auth.helpers';
import { StorageService } from '../storage/storage.service';
import {
ChangePasswordDto,
MapsKeyUpdateDto,
@@ -49,21 +47,17 @@ import { ManagedForbidden } from '../common/managed';
const WINDOW = 15 * 60 * 1000;
const ALLOWED_AVATAR_EXTS = ['.jpg', '.jpeg', '.png', '.gif', '.webp'];
const AVATAR_UPLOAD = {
storage: diskStorage({
destination: (_req, _file, cb) => { if (!fs.existsSync(avatarDir)) fs.mkdirSync(avatarDir, { recursive: true }); cb(null, avatarDir); },
filename: (_req, file, cb) => cb(null, uuid() + path.extname(file.originalname)),
}),
limits: { fileSize: 5 * 1024 * 1024 },
fileFilter: (_req: unknown, file: Express.Multer.File, cb: (err: Error | null, accept: boolean) => void) => {
const ext = path.extname(file.originalname).toLowerCase();
if (!file.mimetype.startsWith('image/') || !ALLOWED_AVATAR_EXTS.includes(ext)) {
const err: Error & { statusCode?: number } = new Error('Only image files (jpg, png, gif, webp) are allowed');
err.statusCode = 400;
return cb(err, false);
}
cb(null, true);
},
export const MAX_AVATAR_SIZE = 5 * 1024 * 1024;
// Consumed by auth.module.ts's MulterModule factory; the storage engine (spool
// destination + UUID filename) comes from the storage upload factory.
export const AVATAR_FILE_FILTER: Options['fileFilter'] = (_req, file, cb) => {
const ext = path.extname(file.originalname).toLowerCase();
if (!file.mimetype.startsWith('image/') || !ALLOWED_AVATAR_EXTS.includes(ext)) {
const err: Error & { statusCode?: number } = new Error('Only image files (jpg, png, gif, webp) are allowed');
err.statusCode = 400;
return cb(err);
}
cb(null, true);
};
/**
@@ -77,7 +71,7 @@ const AVATAR_UPLOAD = {
@Controller('api/auth')
@UseGuards(JwtAuthGuard)
export class AuthController {
constructor(private readonly auth: AuthService, private readonly profile: UserProfileService, private readonly tokens: TokenService, private readonly rl: RateLimitService, private readonly audit: AuditService, private readonly env: RuntimeEnvService) {}
constructor(private readonly auth: AuthService, private readonly profile: UserProfileService, private readonly tokens: TokenService, private readonly rl: RateLimitService, private readonly audit: AuditService, private readonly env: RuntimeEnvService, private readonly storage: StorageService) {}
private limit(bucket: string, req: Request, max: number): void {
if (!this.rl.check(bucket, req.ip || 'unknown', max, WINDOW, Date.now())) {
@@ -182,7 +176,7 @@ export class AuthController {
@Post('avatar')
@HttpCode(200)
@UseInterceptors(FileInterceptor('avatar', AVATAR_UPLOAD))
@UseInterceptors(FileInterceptor('avatar'))
async avatar(@CurrentUser() user: User, @UploadedFile() file: Express.Multer.File | undefined) {
if (isDemoWriteBlocked(this.env, user.email)) {
throw new HttpException(DEMO_WRITE_ERROR, 403);
@@ -190,6 +184,9 @@ export class AuthController {
if (!file) {
throw new HttpException({ error: 'No image uploaded' }, 400);
}
// Commit the spooled upload to its final storage location (atomic
// same-volume rename) before the DB row references the final path.
await this.storage.put('avatars', file.filename, { tmpPath: file.path });
return this.profile.saveAvatar(user.id, file.filename);
}
-9
View File
@@ -1,7 +1,5 @@
import bcrypt from 'bcryptjs';
import crypto from 'crypto';
import path from 'path';
import fs from 'fs';
import { User } from '../../types';
import { decrypt_api_key } from '../common/crypto/apiKeyCrypto';
@@ -38,13 +36,6 @@ export const ADMIN_SETTINGS_KEYS = [
'passkey_login', 'webauthn_rp_id', 'webauthn_origins',
];
// One directory deeper than the legacy src/services location, so the extra
// '../' keeps resolving to server/uploads/avatars (same anchor the multer
// diskStorage in auth.controller.ts uses). Import-time mkdir on purpose
// (legacy parity): the upload dir must exist before the first request.
export const avatarDir = path.join(__dirname, '../../../uploads/avatars');
if (!fs.existsSync(avatarDir)) fs.mkdirSync(avatarDir, { recursive: true });
// ---------------------------------------------------------------------------
// Helpers (exported for route-level use where needed)
// ---------------------------------------------------------------------------
+20 -1
View File
@@ -18,6 +18,12 @@ import { MailerModule } from '../notifications/mailer/mailer.module';
import { PermissionsModule } from '../permissions/permissions.module';
import { TripMembershipModule } from '../trip-membership/trip-membership.module';
import { EphemeralTokenModule } from './ephemeral-token.module';
import { MulterModule } from '@nestjs/platform-express';
import { StorageModule } from '../storage/storage.module';
import { StorageService } from '../storage/storage.service';
import { buildStorageUploadOptions } from '../storage/storage-upload.factory';
import { AVATAR_FILE_FILTER, MAX_AVATAR_SIZE } from './auth.controller';
import { AllowedFileTypesModule } from '../files/allowed-file-types.module';
/**
* Auth module — public flows (login/register/reset/mfa-verify/logout) and the
@@ -43,7 +49,20 @@ import { EphemeralTokenModule } from './ephemeral-token.module';
* through auth.bridge.ts.
*/
@Module({
imports: [EphemeralTokenModule, RateLimitModule, AuditModule, PermissionsModule, TripMembershipModule, MailerModule, AppConfigModule, TokensModule, BudgetModule],
imports: [
MulterModule.registerAsync({
imports: [StorageModule],
inject: [StorageService],
useFactory: (storage: StorageService) =>
buildStorageUploadOptions(storage, {
category: 'avatars',
maxSize: MAX_AVATAR_SIZE,
fileFilter: AVATAR_FILE_FILTER,
}),
}),
StorageModule,
AllowedFileTypesModule,
EphemeralTokenModule, RateLimitModule, AuditModule, PermissionsModule, TripMembershipModule, MailerModule, AppConfigModule, TokensModule, BudgetModule],
controllers: [AuthPublicController, AuthController, PasskeyController],
providers: [AuthService, UserProfileService, RegistrationInvitesService, PasskeyService, UserCleanupService, WebauthnConfigService, AuthMcp],
exports: [AuthService, RegistrationInvitesService, PasskeyService, UserCleanupService],
+7 -2
View File
@@ -31,13 +31,13 @@ import { TripMembershipService } from '../trip-membership/trip-membership.servic
import { WebauthnConfigService } from './webauthn-config.service';
import { setAuthCookie, clearAuthCookie } from '../common/cookie';
import { MailerService } from '../notifications/mailer/mailer.service';
import { AllowedFileTypesService } from '../files/allowed-file-types.service';
import { getAppUrl } from '../../app-config';
import {
ADMIN_SETTINGS_KEYS,
BCRYPT_COST,
DUMMY_PASSWORD_HASH,
EMAIL_REGEX,
avatarDir,
generateBackupCodes,
hashBackupCodeBcrypt,
mask_stored_api_key,
@@ -128,6 +128,7 @@ export class AuthService {
private readonly userCleanup: UserCleanupService,
private readonly mailer: MailerService,
private readonly tokens: EphemeralTokenService,
private readonly allowedFileTypes: AllowedFileTypesService,
) {}
// Cookie
@@ -290,7 +291,11 @@ export class AuthService {
oidc_configured: oidcConfigured,
oidc_display_name: oidcConfigured ? (oidcDisplayName || 'SSO') : undefined,
require_mfa: requireMfaRow?.value === 'true',
allowed_file_types: this.db.get<{ value: string }>("SELECT value FROM app_settings WHERE key = 'allowed_file_types'")?.value || 'jpg,jpeg,png,gif,webp,heic,pdf,doc,docx,xls,xlsx,txt,csv',
// The canonical live-read: same query + DEFAULT_ALLOWED_EXTENSIONS
// fallback the upload filters use, so the client's picker and the
// server's acceptance can never drift (the historical inline copy here
// dropped pkpass, pkpasses, md and markdown).
allowed_file_types: this.allowedFileTypes.get(),
// Whether the configuration belongs to whoever operates this install
// rather than to its admin. The client uses it to stop offering settings
// the server would refuse anyway — it is an honesty flag for the UI, never
+13 -12
View File
@@ -1,11 +1,10 @@
import { Injectable } from '@nestjs/common';
import path from 'path';
import fs from 'fs';
import { readEnv, getAppUrl } from '../../app-config';
import { DatabaseService } from '../database/database.service';
import { StorageService } from '../storage/storage.service';
import { decrypt_api_key, maybe_encrypt_api_key } from '../common/crypto/apiKeyCrypto';
import { avatarUrl } from '../common/avatarUrl';
import { EMAIL_REGEX, avatarDir, mask_stored_api_key } from './auth.helpers';
import { EMAIL_REGEX, mask_stored_api_key } from './auth.helpers';
import { splitManagedKeys, MANAGED_LOCKED_PROFILE_KEYS } from '../common/managed';
import {
INSTANCE_API_KEY_NAMES,
@@ -33,7 +32,10 @@ import { User } from '../../types';
*/
@Injectable()
export class UserProfileService {
constructor(private readonly db: DatabaseService) {}
constructor(
private readonly db: DatabaseService,
private readonly storage: StorageService,
) {}
/**
* On a centrally administered install the three key columns belong to the
@@ -281,12 +283,12 @@ export class UserProfileService {
async saveAvatar(userId: number, filename: string) {
const current = this.db.get<{ avatar: string | null }>('SELECT avatar FROM users WHERE id = ?', userId);
// Only a locally uploaded file has something to clean up. An OIDC picture URL
// (#1399) has no file on disk, so skip the rm — path.join on a URL is meaningless.
// (#1399) has no storage object, so skip the delete entirely.
if (current?.avatar && !/^https:\/\//i.test(current.avatar)) {
// Fire-and-forget: leftover files are harmless; the DB update is
// the source of truth for which avatar is current.
const oldPath = path.join(avatarDir, current.avatar);
await fs.promises.rm(oldPath, { force: true }).catch(() => {});
// Fire-and-forget parity: leftover objects are harmless; the DB update is
// the source of truth for which avatar is current. The catch also
// swallows a hostile stored value the central key validation rejects.
await this.storage.delete('avatars', current.avatar).catch(() => {});
}
this.db.run('UPDATE users SET avatar = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?', filename, userId);
@@ -297,10 +299,9 @@ export class UserProfileService {
async deleteAvatar(userId: number) {
const current = this.db.get<{ avatar: string | null }>('SELECT avatar FROM users WHERE id = ?', userId);
// An OIDC picture URL (#1399) has no local file — only rm an uploaded one.
// An OIDC picture URL (#1399) has no storage object — only delete an uploaded one.
if (current?.avatar && !/^https:\/\//i.test(current.avatar)) {
const filePath = path.join(avatarDir, current.avatar);
await fs.promises.rm(filePath, { force: true }).catch(() => {});
await this.storage.delete('avatars', current.avatar).catch(() => {});
}
this.db.run('UPDATE users SET avatar = NULL, updated_at = CURRENT_TIMESTAMP WHERE id = ?', userId);
return { success: true };
+3 -1
View File
@@ -3,6 +3,7 @@ import { readEnv } from '../../app-config';
import { logInfo, logError } from '../audit/audit-log.logger';
import { BackupService } from './backup.service';
import { CronRegistrarService } from '../scheduling/cron-registrar.service';
import { StorageService } from '../storage/storage.service';
import { parseAutoBackupBody } from './backup.impl';
import {
buildCronExpression,
@@ -24,6 +25,7 @@ export class AutoBackupJob implements OnApplicationBootstrap {
constructor(
private readonly backup: BackupService,
private readonly registrar: CronRegistrarService,
private readonly storage: StorageService,
) {}
onApplicationBootstrap(): void {
@@ -66,7 +68,7 @@ export class AutoBackupJob implements OnApplicationBootstrap {
const settings = loadSettings();
if (settings.keep_days > 0) {
cleanupOldBackups(settings.keep_days);
await cleanupOldBackups(this.storage, settings.keep_days);
}
}
+13 -10
View File
@@ -1,16 +1,19 @@
import path from 'node:path';
import fs from 'node:fs';
import { logInfo, logError } from '../audit/audit-log.logger';
import type { StorageService } from '../storage/storage.service';
/**
* Auto-backup settings and retention — the pure half of the auto-backup cron
* (moved from src/scheduler.ts). Stays a plain module beside backup.impl.ts
* for the same reason that does: file I/O against data/backup-settings.json
* and data/backups, no container state. AutoBackupJob owns the scheduling.
* for the same reason that does: file I/O against data/backup-settings.json,
* no container state. AutoBackupJob owns the scheduling and passes its
* injected StorageService into cleanupOldBackups — retention addresses the
* backups category through the facade so expired archives leave mirror
* replicas too.
*/
const dataDir = path.join(__dirname, '../../../data');
const backupsDir = path.join(dataDir, 'backups');
const settingsFile = path.join(dataDir, 'backup-settings.json');
export const VALID_INTERVALS = ['hourly', 'daily', 'weekly', 'monthly'];
@@ -70,16 +73,16 @@ function autoBackupTimestampMs(filename: string): number | null {
return Number.isNaN(ms) ? null : ms;
}
export function cleanupOldBackups(keepDays: number, now: number = Date.now()): void {
export async function cleanupOldBackups(storage: StorageService, keepDays: number, now: number = Date.now()): Promise<void> {
try {
const cutoff = now - keepDays * 24 * 60 * 60 * 1000;
const files = fs.readdirSync(backupsDir).filter(f => f.startsWith('auto-backup-') && f.endsWith('.zip'));
for (const file of files) {
const filePath = path.join(backupsDir, file);
const ageMs = autoBackupTimestampMs(file) ?? fs.statSync(filePath).mtimeMs;
for await (const obj of storage.list('backups')) {
if (obj.key.includes('/')) continue; // list() recurses; retention is top-level-only like the readdir it replaces
if (!obj.key.startsWith('auto-backup-') || !obj.key.endsWith('.zip')) continue; // manual backup-*.zip is never auto-deleted
const ageMs = autoBackupTimestampMs(obj.key) ?? obj.mtimeMs;
if (ageMs < cutoff) {
fs.unlinkSync(filePath);
logInfo(`Auto-Backup old backup deleted: ${file}`);
await storage.delete('backups', obj.key); // fans out to mirror replicas too
logInfo(`Auto-Backup old backup deleted: ${obj.key}`);
}
}
} catch (err: unknown) {
+21 -21
View File
@@ -27,19 +27,10 @@ import { CurrentUser } from '../auth/current-user.decorator';
import { AutoBackupSettingsDto } from './backup.dto';
import { getClientIp } from '../audit/client-ip';
import { AuditService } from '../audit/audit.service';
import { getUploadTmpDir, MAX_BACKUP_UPLOAD_SIZE } from './backup.impl';
import { StorageNotFoundError } from '../storage/storage.types';
import { ManagedForbidden, isManagedBlocked, MANAGED_FORBIDDEN_ERROR } from '../common/managed';
import { RuntimeEnvService } from '../app-config/runtime-env.service';
const UPLOAD = {
dest: getUploadTmpDir(),
fileFilter: (_req: unknown, file: Express.Multer.File, cb: (err: Error | null, accept: boolean) => void) => {
if (file.originalname.endsWith('.zip')) return cb(null, true);
cb(new Error('Only ZIP files allowed'), false);
},
limits: { fileSize: MAX_BACKUP_UPLOAD_SIZE },
};
/**
* /api/backup — admin-only database backup management (list, create, download,
* restore from a stored or uploaded zip, auto-backup settings, delete).
@@ -60,9 +51,9 @@ export class BackupController {
) {}
@Get('list')
list() {
async list() {
try {
return { backups: this.backup.listBackups() };
return { backups: await this.backup.listBackups() };
} catch {
throw new HttpException({ error: 'Error loading backups' }, 500);
}
@@ -84,14 +75,23 @@ export class BackupController {
}
@Get('download/:filename')
download(@Param('filename') filename: string, @Res() res: Response): void {
async download(@Param('filename') filename: string, @Res() res: Response): Promise<void> {
if (!this.backup.isValidBackupFilename(filename)) {
throw new HttpException({ error: 'Invalid filename' }, 400);
}
if (!this.backup.backupFileExists(filename)) {
if (!(await this.backup.backupFileExists(filename))) {
throw new HttpException({ error: 'Backup not found' }, 404);
}
res.download(this.backup.backupFilePath(filename), filename);
try {
await this.backup.sendBackupToResponse(filename, res);
} catch (err) {
// The pre-check above owns the normal miss; this only covers a delete
// racing between the check and the send.
if (err instanceof StorageNotFoundError) {
throw new HttpException({ error: 'Backup not found' }, 404);
}
throw err;
}
}
@ManagedForbidden('a restore replaces database and uploads, and the operator owns the recovery point')
@@ -101,11 +101,11 @@ export class BackupController {
if (!this.backup.isValidBackupFilename(filename)) {
throw new HttpException({ error: 'Invalid filename' }, 400);
}
if (!this.backup.backupFileExists(filename)) {
if (!(await this.backup.backupFileExists(filename))) {
throw new HttpException({ error: 'Backup not found' }, 404);
}
try {
const result = await this.backup.restoreFromZip(this.backup.backupFilePath(filename));
const result = await this.backup.restoreBackup(filename);
if (!result.success) {
throw new HttpException({ error: result.error }, result.status || 400);
}
@@ -123,7 +123,7 @@ export class BackupController {
)
@Post('upload-restore')
@HttpCode(200) // Express answers upload-restore with res.json (200).
@UseInterceptors(FileInterceptor('backup', UPLOAD))
@UseInterceptors(FileInterceptor('backup'))
async uploadRestore(@CurrentUser() user: User, @UploadedFile() file: Express.Multer.File | undefined, @Req() req: Request) {
// Checked here rather than in the guard: a guard runs before the multipart
// parser, so throwing there leaves the body unread and the client sees an
@@ -177,14 +177,14 @@ export class BackupController {
}
@Delete(':filename')
remove(@CurrentUser() user: User, @Param('filename') filename: string, @Req() req: Request) {
async remove(@CurrentUser() user: User, @Param('filename') filename: string, @Req() req: Request) {
if (!this.backup.isValidBackupFilename(filename)) {
throw new HttpException({ error: 'Invalid filename' }, 400);
}
if (!this.backup.backupFileExists(filename)) {
if (!(await this.backup.backupFileExists(filename))) {
throw new HttpException({ error: 'Backup not found' }, 404);
}
this.backup.deleteBackup(filename);
await this.backup.deleteBackup(filename);
this.audit.writeAudit({ userId: user.id, action: 'backup.delete', resource: filename, ip: getClientIp(req) });
return { success: true };
}
+197 -109
View File
@@ -1,6 +1,7 @@
import archiver from 'archiver';
import unzipper from 'unzipper';
import path from 'path';
import { pipeline } from 'node:stream/promises';
import { readEnv } from '../../app-config';
import fs from 'fs';
import Database from 'better-sqlite3';
@@ -10,14 +11,15 @@ import { invalidatePermissionsCache } from '../permissions/permissions-cache';
import { pluginsCodeRoot, pluginsDataRoot } from '../plugins/paths';
import { stageExtractedPluginTrees, applyStagedRestoreNow } from '../plugins/plugin-backup';
import { snapshotAllPluginDataDbs } from '../plugins/host/plugin-data.service';
import type { Response } from 'express';
import type { StorageService } from '../storage/storage.service';
import { StorageInvalidKeyError } from '../storage/storage.types';
// ---------------------------------------------------------------------------
// Paths
// ---------------------------------------------------------------------------
const dataDir = path.join(__dirname, '../../../data');
const backupsDir = path.join(dataDir, 'backups');
const uploadsDir = path.join(__dirname, '../../../uploads');
// Compressed upload cap for restore archives. Defaults to 500 MB, raisable via
// BACKUP_UPLOAD_LIMIT_MB for instances whose backups (uploads/ included) grow
@@ -36,10 +38,6 @@ export const MAX_BACKUP_DECOMPRESSED_SIZE = backupEnv.maxDecompressedMb * 1024 *
// Helpers
// ---------------------------------------------------------------------------
export function ensureBackupsDir(): void {
if (!fs.existsSync(backupsDir)) fs.mkdirSync(backupsDir, { recursive: true });
}
export function formatSize(bytes: number): string {
if (bytes < 1024) return `${bytes} B`;
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
@@ -80,12 +78,19 @@ export function isValidBackupFilename(filename: string): boolean {
return /^(?:auto-)?backup-[\w-]+\.zip$/.test(filename);
}
export function backupFilePath(filename: string): string {
return path.join(backupsDir, filename);
export function backupFileExists(storage: StorageService, filename: string): Promise<boolean> {
return storage.exists('backups', filename);
}
export function backupFileExists(filename: string): boolean {
return fs.existsSync(path.join(backupsDir, filename));
/**
* The codebase's only res.download becomes the storage equivalent: root-relative
* sendFile via sendToResponse, with res.download's attachment header rebuilt by
* hand (filenames are regex-gated ASCII — no encoding cases).
*/
export function sendBackupToResponse(storage: StorageService, filename: string, res: Response): Promise<void> {
return storage.sendToResponse('backups', filename, res, {
disposition: `attachment; filename="${filename}"`,
});
}
// ---------------------------------------------------------------------------
@@ -122,27 +127,32 @@ export interface BackupInfo {
created_at: string;
}
export function listBackups(): BackupInfo[] {
ensureBackupsDir();
return fs.readdirSync(backupsDir)
.filter(f => f.endsWith('.zip'))
.map(filename => {
const filePath = path.join(backupsDir, filename);
const stat = fs.statSync(filePath);
return {
filename,
size: stat.size,
sizeText: formatSize(stat.size),
created_at: stat.mtime.toISOString(),
};
})
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
export async function listBackups(storage: StorageService): Promise<BackupInfo[]> {
const backups: BackupInfo[] = [];
for await (const obj of storage.list('backups')) {
// storage.list() recurses; the legacy readdir was single-level. Nested keys
// (a restore-* staging tree when data and uploads map to the same dir) and
// non-zip files must not surface.
if (obj.key.includes('/') || !obj.key.endsWith('.zip')) continue;
backups.push({
filename: obj.key,
size: obj.size,
sizeText: formatSize(obj.size),
created_at: new Date(obj.mtimeMs).toISOString(),
});
}
return backups.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
}
// ---------------------------------------------------------------------------
// Create backup
// ---------------------------------------------------------------------------
/** The categories a backup archives — everything else under uploads/ is a
* re-derivable cache (photos-google, photos-trek) or not uploads at all
* (backups). Restore's rehydration walks the same list. */
export const BACKUP_UPLOAD_CATEGORIES = ['files', 'journey', 'covers', 'avatars', 'places', 'photos'] as const;
/**
* Writes a full backup zip and returns its BackupInfo.
*
@@ -151,27 +161,71 @@ export function listBackups(): BackupInfo[] {
* only auto-backup-*.zip, and the admin panel badges them as automatic. Manual
* backups keep the default.
*/
export async function createBackup(prefix: 'backup' | 'auto-backup' = 'backup'): Promise<BackupInfo> {
ensureBackupsDir();
export async function createBackup(storage: StorageService, prefix: 'backup' | 'auto-backup' = 'backup'): Promise<BackupInfo> {
const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
const filename = `${prefix}-${timestamp}.zip`;
const outputPath = path.join(backupsDir, filename);
// The scratch names carry the prefix too: a scheduled run and a manual one that
// start in the same second would otherwise share a snapshot path, and the first
// to finish would delete the other's staging copy mid-archive.
const pdataSnap = path.join(backupsDir, `.plugins-snap-${prefix}-${timestamp}`);
const dbSnap = path.join(backupsDir, `.travel-snap-${prefix}-${timestamp}.db`);
// All staging lives in the backups backend's own spool: same volume as the
// destination (the put commit stays an atomic rename) and crash leftovers are
// reaped by LocalDriver's boot spool-cleanup. The scratch names carry the
// prefix too: a scheduled run and a manual one that start in the same second
// would otherwise share a snapshot path, and the first to finish would delete
// the other's staging copy mid-archive.
const spoolDir = storage.spoolDirFor('backups');
const zipSpool = path.join(spoolDir, `zip-build-${prefix}-${timestamp}`);
const pdataSnap = path.join(spoolDir, `plugins-snap-${prefix}-${timestamp}`);
const dbSnap = path.join(spoolDir, `travel-snap-${prefix}-${timestamp}.db`);
// Per-backup staging for uploads with no local path (a remote/S3 primary, or
// a local path that vanished between listing and archiving — see
// getLocalPathOrNull). Same spool as the rest of the build, same cleanup.
const stagingDir = path.join(spoolDir, `staging-${prefix}-${timestamp}`);
try {
try { db.exec('PRAGMA wal_checkpoint(TRUNCATE)'); } catch (e) {}
// Enumerate the archived categories up front (the archiver reads entries
// lazily during finalize(), so the promise executor below must stay
// synchronous). Everything NOT in BACKUP_UPLOAD_CATEGORIES is excluded by
// construction: the re-derivable caches (photos-google in both
// TREK_PLACE_PHOTO_DIR modes, photos-trek) and backups itself are never
// enumerated, which is what makes the same-dir-misconfig guard (#1358)
// structural instead of pattern-based.
const uploadEntries: { absPath: string; name: string }[] = [];
for (const category of BACKUP_UPLOAD_CATEGORIES) {
for await (const obj of storage.list(category)) {
// In mode A the google/trek caches nest under the photos/ prefix — the
// category walk would sweep them back in without this skip.
if (category === 'photos' && (obj.key.startsWith('google/') || obj.key.startsWith('trek/'))) continue;
// Local path available (exists on disk right now — the fail-safe half
// of getLocalPathOrNull's contract) → push it directly: zero-copy, the
// default-install path. Otherwise (a remote/S3 primary, or a local
// path that vanished between the list() and here) stream the object
// into this backup's own staging dir so archiver has a real file to
// read lazily during finalize() — the temp file withLocalFile would
// have produced is gone by the time archiver gets to it.
const localPath = await storage.getLocalPathOrNull(category, obj.key);
if (localPath !== null) {
uploadEntries.push({ absPath: localPath, name: `uploads/${category}/${obj.key}` });
continue;
}
const stagedPath = path.join(stagingDir, category, obj.key);
fs.mkdirSync(path.dirname(stagedPath), { recursive: true });
const { stream } = await storage.getStream(category, obj.key);
await pipeline(stream, fs.createWriteStream(stagedPath));
uploadEntries.push({ absPath: stagedPath, name: `uploads/${category}/${obj.key}` });
}
}
await new Promise<void>((resolve, reject) => {
const output = fs.createWriteStream(outputPath);
const output = fs.createWriteStream(zipSpool);
const archive = archiver('zip', { zlib: { level: 9 } });
output.on('close', resolve);
archive.on('error', reject);
// archiver emits 'warning' (not 'error') for entries it couldn't
// stat/read — a stale staged path, a permission error — and by default
// just skips them, silently dropping bytes from the backup. Fail the
// backup instead: a dropped entry must never pass as a success.
archive.on('warning', reject);
archive.pipe(output);
@@ -205,28 +259,7 @@ export async function createBackup(prefix: 'backup' | 'auto-backup' = 'backup'):
archive.file(encKeyPath, { name: '.encryption_key' });
}
if (fs.existsSync(uploadsDir)) {
// Exclude the place-photo and trek-memory caches: both are re-derivable
// (re-fetched on demand, keyed on stable ids) and would otherwise dominate
// backup size. Restores self-heal — the cache dirs are recreated at startup.
//
// Also exclude backups/ and restore-*/: these live under data/, not uploads/,
// but when an install maps data and uploads to the SAME directory (a
// misconfiguration, but a catastrophic one) the glob would otherwise sweep
// every prior backup zip into the new archive — each run embedding all
// previous runs, so size compounds without bound (see issue #1358). Ignoring
// them keeps the backup bounded regardless of how the volumes are mounted.
archive.glob(
'**/*',
{
cwd: uploadsDir,
ignore: ['photos/google/**', 'photos/trek/**', 'backups/**', 'restore-*/**'],
nodir: true,
dot: true,
},
{ prefix: 'uploads' },
);
}
for (const entry of uploadEntries) archive.file(entry.absPath, { name: entry.name });
// Plugin data — each plugin's own SQLite file and any blobs. This is the ONLY
// copy of the user data a plugin holds, so it belongs in the backup. Checkpoint
@@ -262,23 +295,29 @@ export async function createBackup(prefix: 'backup' | 'auto-backup' = 'backup'):
archive.finalize();
});
const stat = fs.statSync(outputPath);
// The commit — and, under a mirror backend, the replica fan-out point.
await storage.put('backups', filename, { tmpPath: zipSpool });
const stat = await storage.stat('backups', filename);
if (!stat) throw new Error(`Backup vanished after commit: ${filename}`);
return {
filename,
size: stat.size,
sizeText: formatSize(stat.size),
created_at: stat.birthtime.toISOString(),
created_at: new Date(stat.mtimeMs).toISOString(),
};
} catch (err: unknown) {
console.error('Backup error:', err);
if (fs.existsSync(outputPath)) fs.unlinkSync(outputPath);
throw err;
} finally {
// The snapshots were staging copies for the archiver only; drop them once streaming is
// done (the await above resolves on the output stream's 'close', so the archive is
// complete).
// put commits by rename, so on success the zip spool file is already gone;
// on failure these clean the half-built staging. The destination needs no
// unlink anymore — nothing lands there until put succeeds. (The await on
// the build promise resolves on the output stream's 'close', so the
// snapshots are no longer being read.)
fs.rmSync(zipSpool, { force: true });
fs.rmSync(pdataSnap, { recursive: true, force: true });
fs.rmSync(dbSnap, { force: true });
fs.rmSync(stagingDir, { recursive: true, force: true });
}
}
@@ -292,7 +331,51 @@ export interface RestoreResult {
status?: number;
}
export async function restoreFromZip(zipPath: string): Promise<RestoreResult> {
/** Restore a zip that already sits in the backups store, reading it through
* the storage facade (primary-local in v1; a remote backend downloads to
* tempDir via withLocalFile — the seam is in place, resumability is not). */
export function restoreBackup(storage: StorageService, filename: string): Promise<RestoreResult> {
return storage.withLocalFile('backups', filename, (zipPath) => restoreFromZip(storage, zipPath));
}
const isBackupCategory = (dir: string): dir is (typeof BACKUP_UPLOAD_CATEGORIES)[number] =>
(BACKUP_UPLOAD_CATEGORIES as readonly string[]).includes(dir);
/**
* Per-entry storage.put replaces the old wipe-and-cpSync (and with it the
* realpathSync symlinked-uploads workaround — the driver resolves its own
* root at init). Entries that cannot map to a storage key — an unknown
* top-level dir, or dot-segments from old `dot: true` archives — are skipped
* with a warning (2026-08-17 decision): new archives never contain them, and
* the category mapping stays structural in both directions.
*/
async function rehydrateUploads(storage: StorageService, extractedUploads: string): Promise<void> {
const walk = (dir: string): string[] =>
fs.readdirSync(dir, { withFileTypes: true }).flatMap((e) => {
const p = path.join(dir, e.name);
return e.isDirectory() ? walk(p) : e.isFile() ? [p] : [];
});
for (const absPath of walk(extractedUploads)) {
const rel = path.relative(extractedUploads, absPath).split(path.sep).join('/');
const slash = rel.indexOf('/');
const top = slash === -1 ? '' : rel.slice(0, slash);
if (slash === -1 || !isBackupCategory(top)) {
console.warn(`Restore: skipping upload entry outside a storage category: ${rel}`);
continue;
}
try {
await storage.put(top, rel.slice(slash + 1), { tmpPath: absPath });
} catch (err) {
if (err instanceof StorageInvalidKeyError) {
console.warn(`Restore: skipping upload entry with an invalid key: ${rel}`);
continue;
}
throw err;
}
}
}
export async function restoreFromZip(storage: StorageService, zipPath: string): Promise<RestoreResult> {
const extractDir = path.join(dataDir, `restore-${Date.now()}`);
let reinitFailed: unknown = null;
try {
@@ -404,41 +487,6 @@ export async function restoreFromZip(zipPath: string): Promise<RestoreResult> {
if (fs.existsSync(extractedEncKey)) {
fs.copyFileSync(extractedEncKey, path.join(dataDir, '.encryption_key'));
}
const extractedUploads = path.join(extractDir, 'uploads');
if (fs.existsSync(extractedUploads)) {
for (const sub of fs.readdirSync(uploadsDir)) {
const subPath = path.join(uploadsDir, sub);
if (fs.statSync(subPath).isDirectory()) {
for (const file of fs.readdirSync(subPath)) {
try { fs.unlinkSync(path.join(subPath, file)); } catch (e) {}
}
}
}
// Copy into the real directory behind uploadsDir. In Docker, uploadsDir
// (/app/server/uploads) is a symlink to the mounted /app/uploads volume;
// cpSync(dereference:false) would otherwise try to overwrite the symlink
// node with a directory and throw ERR_FS_CP_DIR_TO_NON_DIR. realpathSync
// is a no-op when uploadsDir is a plain directory (dev/non-Docker).
fs.cpSync(extractedUploads, fs.realpathSync(uploadsDir), { recursive: true, force: true });
}
// Plugin trees can't be swapped while the runtime holds their DBs open, so stage
// them beside the live trees, then ask the runtime to quiesce its plugins and apply
// the swap NOW. If the runtime isn't up (plugins disabled / restore during boot),
// the staging waits for the boot reconcile — with nothing running, no data diverges.
// Best-effort: a staging error must not fail an otherwise-good core restore.
try {
stageExtractedPluginTrees(extractDir);
// Quiesce regardless of whether trees were staged: the restored travel.db carries
// a different `plugins` table, so any plugin still running with its pre-restore
// identity/grants is now a ghost — invisible in the restored UI, unstoppable short
// of a process restart. applyStagedRestoreNow closes those handles; the tree swap
// it also performs is a no-op when nothing was staged (e.g. an older archive).
await applyStagedRestoreNow();
} catch (e) {
console.error('Restore: staging plugin trees failed:', e);
}
} finally {
// Reopening the DB must always run (even if the copy above threw) so the
// process is never left without a connection. Capture a reopen failure
@@ -458,6 +506,54 @@ export async function restoreFromZip(zipPath: string): Promise<RestoreResult> {
invalidatePermissionsCache();
}
if (!reinitFailed) {
// The registry reads storage.* app_settings through the DB handle that
// was just closed and reopened above — reload it now, AFTER reinitialize()
// and BEFORE any byte moves, so rehydrated uploads land where the RESTORED
// config says rather than the stale pre-restore one (audit #4). Skipped
// entirely when reopen failed: with no live DB handle the registry has
// nothing to read, and the restore is already reported as "restart
// required" below — rehydrating into a stale/guessed config would be worse.
storage.reloadConfig();
const extractedUploads = path.join(extractDir, 'uploads');
if (fs.existsSync(extractedUploads)) {
// Parity with the legacy wipe: it unlinked one level deep only (nested
// files — journey/thumbs, photos/google — survived until overwritten by
// the copy) and swallowed per-file errors.
for (const category of BACKUP_UPLOAD_CATEGORIES) {
for await (const obj of storage.list(category)) {
if (obj.key.includes('/')) continue;
await storage.delete(category, obj.key).catch(() => { /* best-effort, as the old unlink loop was */ });
}
}
await rehydrateUploads(storage, extractedUploads);
}
}
// Plugin trees can't be swapped while the runtime holds their DBs open, so stage
// them beside the live trees, then ask the runtime to quiesce its plugins and apply
// the swap NOW. If the runtime isn't up (plugins disabled / restore during boot),
// the staging waits for the boot reconcile — with nothing running, no data diverges.
// Best-effort: a staging error must not fail an otherwise-good core restore. Runs
// UNCONDITIONALLY, even when reinitFailed — unlike reload/rehydration this is pure
// filesystem staging with no DB dependency (plugin-backup.ts), so a failed reopen
// must not cost the archive's plugin data: extractDir is unlinked right below, and
// an un-staged tree there would be gone for good with no recovery path.
try {
stageExtractedPluginTrees(extractDir);
// Quiesce regardless of whether trees were staged: the restored travel.db carries
// a different `plugins` table, so any plugin still running with its pre-restore
// identity/grants is now a ghost — invisible in the restored UI, unstoppable short
// of a process restart. applyStagedRestoreNow closes those handles; the tree swap
// it also performs is a no-op when nothing was staged (e.g. an older archive). It
// degrades gracefully when the DB isn't reopened, same as any other best-effort
// failure here.
await applyStagedRestoreNow();
} catch (e) {
console.error('Restore: staging plugin trees failed:', e);
}
fs.rmSync(extractDir, { recursive: true, force: true });
if (reinitFailed) {
console.error('Restore: database reopen failed after file swap:', reinitFailed);
@@ -482,15 +578,7 @@ export async function restoreFromZip(zipPath: string): Promise<RestoreResult> {
// Delete backup
// ---------------------------------------------------------------------------
export function deleteBackup(filename: string): void {
const filePath = path.join(backupsDir, filename);
fs.unlinkSync(filePath);
export function deleteBackup(storage: StorageService, filename: string): Promise<void> {
return storage.delete('backups', filename);
}
// ---------------------------------------------------------------------------
// Upload config (multer dest)
// ---------------------------------------------------------------------------
export function getUploadTmpDir(): string {
return path.join(dataDir, 'tmp/');
}
+33 -1
View File
@@ -1,13 +1,45 @@
import { Module } from '@nestjs/common';
import { MulterModule } from '@nestjs/platform-express';
import type { MulterOptions } from '@nestjs/platform-express/multer/interfaces/multer-options.interface';
import { BackupController } from './backup.controller';
import { BackupService } from './backup.service';
import { AutoBackupJob } from './auto-backup.job';
import { AuditModule } from '../audit/audit.module';
import { SchedulingModule } from '../scheduling/scheduling.module';
import { AppConfigModule } from '../app-config/app-config.module';
import { StorageModule } from '../storage/storage.module';
import { StorageService } from '../storage/storage.service';
import { MAX_BACKUP_UPLOAD_SIZE } from './backup.impl';
/**
* Multer options for the restore upload. The uploaded zip is a restore INPUT,
* not a stored object — it gets no storage category and spools to the
* driver-agnostic scratch dir (data/tmp), where the handler consumes and
* unlinks it. Resolved through DI so no path is computed at module load.
*/
export function buildBackupUploadOptions(storage: StorageService): MulterOptions {
return {
dest: storage.tempDir(),
fileFilter: (_req: unknown, file: Express.Multer.File, cb: (err: Error | null, accept: boolean) => void) => {
if (file.originalname.endsWith('.zip')) return cb(null, true);
cb(new Error('Only ZIP files allowed'), false);
},
limits: { fileSize: MAX_BACKUP_UPLOAD_SIZE },
};
}
@Module({
imports: [AppConfigModule, AuditModule, SchedulingModule],
imports: [
AppConfigModule,
AuditModule,
SchedulingModule,
StorageModule,
MulterModule.registerAsync({
imports: [StorageModule],
inject: [StorageService],
useFactory: buildBackupUploadOptions,
}),
],
controllers: [BackupController],
providers: [BackupService, AutoBackupJob],
})
+11 -6
View File
@@ -1,4 +1,6 @@
import { Injectable } from '@nestjs/common';
import type { Response } from 'express';
import { StorageService } from '../storage/storage.service';
import * as svc from './backup.impl';
/**
@@ -13,14 +15,17 @@ import * as svc from './backup.impl';
*/
@Injectable()
export class BackupService {
listBackups() { return svc.listBackups(); }
createBackup(prefix?: 'backup' | 'auto-backup') { return svc.createBackup(prefix); }
restoreFromZip(zipPath: string) { return svc.restoreFromZip(zipPath); }
deleteBackup(filename: string) { return svc.deleteBackup(filename); }
constructor(private readonly storage: StorageService) {}
listBackups() { return svc.listBackups(this.storage); }
createBackup(prefix?: 'backup' | 'auto-backup') { return svc.createBackup(this.storage, prefix); }
restoreFromZip(zipPath: string) { return svc.restoreFromZip(this.storage, zipPath); }
restoreBackup(filename: string) { return svc.restoreBackup(this.storage, filename); }
deleteBackup(filename: string) { return svc.deleteBackup(this.storage, filename); }
isValidBackupFilename(filename: string) { return svc.isValidBackupFilename(filename); }
backupFilePath(filename: string) { return svc.backupFilePath(filename); }
backupFileExists(filename: string) { return svc.backupFileExists(filename); }
backupFileExists(filename: string) { return svc.backupFileExists(this.storage, filename); }
sendBackupToResponse(filename: string, res: Response) { return svc.sendBackupToResponse(this.storage, filename, res); }
checkRateLimit(key: string, maxAttempts: number, windowMs: number) { return svc.checkRateLimit(key, maxAttempts, windowMs); }
get rateWindow() { return svc.BACKUP_RATE_WINDOW; }
+27 -28
View File
@@ -15,12 +15,11 @@ import {
UseInterceptors,
} from '@nestjs/common';
import { FileInterceptor } from '@nestjs/platform-express';
import { diskStorage } from 'multer';
import type { Options } from 'multer';
import path from 'path';
import fs from 'fs';
import { v4 as uuidv4 } from 'uuid';
import type { User } from '../../types';
import { CollabService } from './collab.service';
import { StorageService } from '../storage/storage.service';
import {
CollabNoteCreateDto,
CollabNoteUpdateDto,
@@ -34,24 +33,18 @@ import { CurrentUser } from '../auth/current-user.decorator';
import { RequirePermission, TripAccessGuard } from '../permissions/trip-access.guard';
import { BLOCKED_EXTENSIONS } from '../files/files.constants';
const MAX_NOTE_FILE_SIZE = 50 * 1024 * 1024;
const filesDir = path.join(__dirname, '../../../uploads/files');
const NOTE_UPLOAD = {
storage: diskStorage({
destination: (_req, _file, cb) => { if (!fs.existsSync(filesDir)) fs.mkdirSync(filesDir, { recursive: true }); cb(null, filesDir); },
filename: (_req, file, cb) => cb(null, `${uuidv4()}${path.extname(file.originalname)}`),
}),
limits: { fileSize: MAX_NOTE_FILE_SIZE },
defParamCharset: 'utf8', // parity with legacy routes/collab.ts — preserve non-ASCII original filenames
fileFilter: (_req: unknown, file: Express.Multer.File, cb: (err: Error | null, accept: boolean) => void) => {
const ext = path.extname(file.originalname).toLowerCase();
if (BLOCKED_EXTENSIONS.includes(ext) || file.mimetype.includes('svg') || file.mimetype.includes('html') || file.mimetype.includes('javascript')) {
const err: Error & { statusCode?: number } = new Error('File type not allowed');
err.statusCode = 400;
return cb(err, false);
}
cb(null, true);
},
export const MAX_NOTE_FILE_SIZE = 50 * 1024 * 1024;
// Consumed by collab.module.ts's MulterModule factory; the rest of the multer
// options (spool destination, filename, limits) come from the storage upload
// factory.
export const collabNoteFileFilter: Options['fileFilter'] = (_req, file, cb) => {
const ext = path.extname(file.originalname).toLowerCase();
if (BLOCKED_EXTENSIONS.includes(ext) || file.mimetype.includes('svg') || file.mimetype.includes('html') || file.mimetype.includes('javascript')) {
const err: Error & { statusCode?: number } = new Error('File type not allowed');
err.statusCode = 400;
return cb(err);
}
cb(null, true);
};
/**
@@ -78,7 +71,10 @@ const NOTE_UPLOAD = {
// caller then sees ECONNRESET instead of the 404). Same as files.controller.ts.
@UseGuards(JwtAuthGuard)
export class CollabController {
constructor(private readonly collab: CollabService) {}
constructor(
private readonly collab: CollabService,
private readonly storage: StorageService,
) {}
private requireTrip(tripId: string, user: User) {
const trip = this.collab.verifyTripAccess(tripId, user.id);
@@ -139,8 +135,8 @@ export class CollabController {
@UseGuards(TripAccessGuard)
@RequirePermission('collab_edit')
@Delete('notes/:id')
deleteNote(@CurrentUser() user: User, @Param('tripId') tripId: string, @Param('id') id: string, @Headers('x-socket-id') socketId?: string) {
if (!this.collab.deleteNote(tripId, id)) {
async deleteNote(@CurrentUser() user: User, @Param('tripId') tripId: string, @Param('id') id: string, @Headers('x-socket-id') socketId?: string) {
if (!(await this.collab.deleteNote(tripId, id))) {
throw new HttpException({ error: 'Note not found' }, 404);
}
this.collab.broadcast(tripId, 'collab:note:deleted', { noteId: Number(id) }, socketId);
@@ -148,8 +144,8 @@ export class CollabController {
}
@Post('notes/:id/files')
@UseInterceptors(FileInterceptor('file', NOTE_UPLOAD))
addNoteFile(@CurrentUser() user: User, @Param('tripId') tripId: string, @Param('id') id: string, @UploadedFile() file: Express.Multer.File | undefined, @Headers('x-socket-id') socketId?: string) {
@UseInterceptors(FileInterceptor('file'))
async addNoteFile(@CurrentUser() user: User, @Param('tripId') tripId: string, @Param('id') id: string, @UploadedFile() file: Express.Multer.File | undefined, @Headers('x-socket-id') socketId?: string) {
const trip = this.requireTrip(tripId, user);
if (!this.collab.canUploadFiles(trip, user)) {
throw new HttpException({ error: 'No permission to upload files' }, 403);
@@ -157,6 +153,9 @@ export class CollabController {
if (!file) {
throw new HttpException({ error: 'No file uploaded' }, 400);
}
// Commit the spooled upload to its final storage location (atomic
// same-volume rename) before the DB row references the final path.
await this.storage.put('files', file.filename, { tmpPath: file.path });
const result = this.collab.addNoteFile(tripId, id, file);
if (!result) {
throw new HttpException({ error: 'Note not found' }, 404);
@@ -168,8 +167,8 @@ export class CollabController {
@UseGuards(TripAccessGuard)
@RequirePermission('collab_edit')
@Delete('notes/:id/files/:fileId')
deleteNoteFile(@CurrentUser() user: User, @Param('tripId') tripId: string, @Param('id') id: string, @Param('fileId') fileId: string, @Headers('x-socket-id') socketId?: string) {
if (!this.collab.deleteNoteFile(tripId, id, fileId)) {
async deleteNoteFile(@CurrentUser() user: User, @Param('tripId') tripId: string, @Param('id') id: string, @Param('fileId') fileId: string, @Headers('x-socket-id') socketId?: string) {
if (!(await this.collab.deleteNoteFile(tripId, id, fileId))) {
throw new HttpException({ error: 'File not found' }, 404);
}
this.collab.broadcast(tripId, 'collab:note:updated', { note: this.collab.getFormattedNoteById(tripId, id) }, socketId);
+1 -1
View File
@@ -145,7 +145,7 @@ export class CollabMcp {
if (this.auth.isDemoUser(ctx.userId)) return demoDenied();
if (!this.collab.verifyTripAccess(tripId, ctx.userId)) return noAccess();
if (!this.guards.hasTripPermission('collab_edit', tripId, ctx.userId)) return permissionDenied();
const deleted = this.collab.deleteNote(tripId, noteId);
const deleted = await this.collab.deleteNote(tripId, noteId);
if (!deleted) return errorResult('Note not found.');
this.guards.safeBroadcast(tripId, 'collab:note:deleted', { noteId });
return ok({ success: true });
+19 -2
View File
@@ -1,5 +1,6 @@
import { Module } from '@nestjs/common';
import { CollabController } from './collab.controller';
import { MulterModule } from '@nestjs/platform-express';
import { CollabController, collabNoteFileFilter, MAX_NOTE_FILE_SIZE } from './collab.controller';
import { CollabService } from './collab.service';
import { CollabRpc } from './collab.rpc';
import { PluginGuardsModule } from '../plugins/host/plugin-guards.module';
@@ -10,9 +11,25 @@ import { AuthModule } from '../auth/auth.module';
import { AddonsModule } from '../addons/addons.module';
import { NotificationsModule } from '../notifications/notifications.module';
import { McpSharedModule } from '../mcp-shared/mcp-shared.module';
import { StorageModule } from '../storage/storage.module';
import { StorageService } from '../storage/storage.service';
import { buildStorageUploadOptions } from '../storage/storage-upload.factory';
@Module({
imports: [McpSharedModule, NotificationsModule, PermissionsModule, AuthModule, RealtimeModule, PluginGuardsModule, AddonsModule],
imports: [
MulterModule.registerAsync({
imports: [StorageModule],
inject: [StorageService],
useFactory: (storage: StorageService) =>
buildStorageUploadOptions(storage, {
category: 'files',
maxSize: MAX_NOTE_FILE_SIZE,
defParamCharset: 'utf8', // parity with legacy routes/collab.ts — preserve non-ASCII original filenames
fileFilter: collabNoteFileFilter,
}),
}),
StorageModule,
McpSharedModule, NotificationsModule, PermissionsModule, AuthModule, RealtimeModule, PluginGuardsModule, AddonsModule],
controllers: [CollabController],
providers: [CollabService, CollabMcp, CollabRpc],
// For in-container consumers (CollabRpc).
+11 -10
View File
@@ -1,5 +1,4 @@
import path from 'path';
import fs from 'fs';
import { Injectable } from '@nestjs/common';
import { DatabaseService, type TripAccess } from '../database/database.service';
import type { TrekWsPayload, TrekWsTripEventName } from '@trek/shared';
@@ -9,6 +8,7 @@ import { avatarUrl } from '../common/avatarUrl';
import { checkSsrf, createPinnedDispatcher } from '../../utils/ssrfGuard';
import type { CollabNote, CollabPoll, CollabMessage, TripFile, User } from '../../types';
import { NotificationsService } from '../notifications/notifications.service';
import { StorageService } from '../storage/storage.service';
type Trip = TripAccess;
@@ -69,6 +69,7 @@ export class CollabService {
private readonly permissions: PermissionsService,
private readonly realtime: RealtimeService,
private readonly notifications: NotificationsService,
private readonly storage: StorageService,
) {}
verifyTripAccess(tripId: string | number, userId: number) {
@@ -193,16 +194,17 @@ export class CollabService {
return this.formatNote(note);
}
deleteNote(tripId: string | number, noteId: string | number): boolean {
async deleteNote(tripId: string | number, noteId: string | number): Promise<boolean> {
const existing = this.db.get('SELECT id FROM collab_notes WHERE id = ? AND trip_id = ?', noteId, tripId);
if (!existing) return false;
// Clean up attached files from disk (unlink-first is intentional — a
// failed row delete leaves dangling rows, never orphaned files).
// Clean up attached objects first (delete-first is intentional — a failed
// row delete leaves dangling rows, never orphaned files). basename()
// tolerates any legacy 'files/'-prefixed row the boot migration has not
// seen.
const noteFiles = this.db.all<NoteFileRow>('SELECT id, filename FROM trip_files WHERE note_id = ?', noteId);
for (const f of noteFiles) {
const filePath = path.join(__dirname, '../../../uploads', f.filename);
try { fs.unlinkSync(filePath); } catch { /* ignore */ }
await this.storage.delete('files', path.basename(f.filename)).catch(() => { /* ignore */ });
}
this.db.transaction(() => {
this.db.run('DELETE FROM trip_files WHERE note_id = ?', noteId);
@@ -221,7 +223,7 @@ export class CollabService {
const result = this.db.run(
'INSERT INTO trip_files (trip_id, note_id, filename, original_name, file_size, mime_type) VALUES (?, ?, ?, ?, ?, ?)',
tripId, noteId, `files/${file.filename}`, file.originalname, file.size, file.mimetype
tripId, noteId, file.filename, file.originalname, file.size, file.mimetype
);
const saved = this.db.get<TripFile>('SELECT * FROM trip_files WHERE id = ?', result.lastInsertRowid)!;
@@ -234,7 +236,7 @@ export class CollabService {
return this.formatNote(note);
}
deleteNoteFile(tripId: string | number, noteId: string | number, fileId: string | number): boolean {
async deleteNoteFile(tripId: string | number, noteId: string | number, fileId: string | number): Promise<boolean> {
// Scope to the trip — like every sibling collab op — so a caller authorized for THEIR
// trip can't delete a note-file that belongs to someone else's trip (IDOR). trip_files
// carries trip_id, so this ties the deleted object to the URL's :tripId the controller
@@ -242,8 +244,7 @@ export class CollabService {
const file = this.db.get<TripFile>('SELECT * FROM trip_files WHERE id = ? AND note_id = ? AND trip_id = ?', fileId, noteId, tripId);
if (!file) return false;
const filePath = path.join(__dirname, '../../../uploads', file.filename);
try { fs.unlinkSync(filePath); } catch { /* ignore */ }
await this.storage.delete('files', path.basename(file.filename)).catch(() => { /* ignore */ });
this.db.run('DELETE FROM trip_files WHERE id = ?', fileId);
return true;
@@ -18,11 +18,8 @@ import {
import { FileInterceptor } from '@nestjs/platform-express';
import { isDemoWriteBlocked, DEMO_WRITE_ERROR } from '../common/demo-write';
import { RuntimeEnvService } from '../app-config/runtime-env.service';
import type { Request } from 'express';
import { diskStorage } from 'multer';
import type { Options } from 'multer';
import path from 'path';
import fs from 'fs';
import { v4 as uuidv4 } from 'uuid';
import type { User } from '../../types';
import { CollectionsService } from './collections.service';
import { AddonGuard } from '../addons/addon.guard';
@@ -30,7 +27,8 @@ import { RequireAddon } from '../addons/require-addon.decorator';
import { ADDON_IDS } from '../../addons';
import { JwtAuthGuard } from '../auth/jwt-auth.guard';
import { CurrentUser } from '../auth/current-user.decorator';
import { PLACE_IMAGE_UPLOAD } from '../common/place-image-upload';
import { PLACE_IMAGE_FILE_FILTER } from '../common/place-image-upload';
import { StorageService } from '../storage/storage.service';
import { placeImageUrl } from '../places/place-image';
import {
CollectionCreateDto,
@@ -56,23 +54,16 @@ import {
} from './collections.dto';
import { PlaceRatingDto } from '../places/places.dto';
const MAX_COVER_SIZE = 20 * 1024 * 1024;
const coversDir = path.join(__dirname, '../../../uploads/covers');
const COVER_UPLOAD = {
storage: diskStorage({
destination: (_req, _file, cb) => {
if (!fs.existsSync(coversDir)) fs.mkdirSync(coversDir, { recursive: true });
cb(null, coversDir);
},
filename: (_req, file, cb) => cb(null, `${uuidv4()}${path.extname(file.originalname)}`),
}),
limits: { fileSize: MAX_COVER_SIZE },
fileFilter: (_req: Request, file: Express.Multer.File, cb: (err: Error | null, accept: boolean) => void) => {
const ext = path.extname(file.originalname).toLowerCase();
const allowed = ['.jpg', '.jpeg', '.png', '.gif', '.webp'];
if (file.mimetype.startsWith('image/') && !file.mimetype.includes('svg') && allowed.includes(ext)) cb(null, true);
else cb(new Error('Only jpg, png, gif, webp images allowed'), false);
},
export const MAX_COVER_SIZE = 20 * 1024 * 1024;
// Duplicated on purpose from trips.controller.ts (historical parity — no
// drive-by consolidation). Quirk preserved: a plain Error without statusCode
// maps to 500, not 400. Passed inline on the cover route; the storage engine
// comes from collections.module.ts's storage-upload factory options.
const COVER_FILE_FILTER: Options['fileFilter'] = (_req, file, cb) => {
const ext = path.extname(file.originalname).toLowerCase();
const allowed = ['.jpg', '.jpeg', '.png', '.gif', '.webp'];
if (file.mimetype.startsWith('image/') && !file.mimetype.includes('svg') && allowed.includes(ext)) cb(null, true);
else cb(new Error('Only jpg, png, gif, webp images allowed'));
};
/**
@@ -89,7 +80,7 @@ const COVER_UPLOAD = {
@UseGuards(AddonGuard, JwtAuthGuard)
@RequireAddon(ADDON_IDS.COLLECTIONS, 'Collections')
export class CollectionsController {
constructor(private readonly collections: CollectionsService, private readonly env: RuntimeEnvService) {}
constructor(private readonly collections: CollectionsService, private readonly env: RuntimeEnvService, private readonly storage: StorageService) {}
// ── Lists ─────────────────────────────────────────────────────────────────
@Get()
@@ -130,8 +121,8 @@ export class CollectionsController {
@Post('places/delete-many')
@HttpCode(200)
deleteMany(@CurrentUser() user: User, @Body() body: CollectionDeleteManyDto, @Headers('x-socket-id') socketId?: string) {
return { deleted: this.collections.deletePlacesMany(user.id, body.ids, socketId) };
async deleteMany(@CurrentUser() user: User, @Body() body: CollectionDeleteManyDto, @Headers('x-socket-id') socketId?: string) {
return { deleted: await this.collections.deletePlacesMany(user.id, body.ids, socketId) };
}
@Post('places/status-many')
@@ -184,8 +175,8 @@ export class CollectionsController {
@Post('places/:pid/image')
@HttpCode(200)
@UseInterceptors(FileInterceptor('image', PLACE_IMAGE_UPLOAD))
uploadPlaceImage(
@UseInterceptors(FileInterceptor('image', { fileFilter: PLACE_IMAGE_FILE_FILTER }))
async uploadPlaceImage(
@CurrentUser() user: User,
@Param('pid') pid: string,
@UploadedFile() file: Express.Multer.File | undefined,
@@ -195,12 +186,15 @@ export class CollectionsController {
throw new HttpException(DEMO_WRITE_ERROR, 403);
}
if (!file) throw new HttpException({ error: 'No image uploaded' }, 400);
// Commit the spooled upload to its final storage location (atomic
// same-volume rename) before the DB row references the final path.
await this.storage.put('places', file.filename, { tmpPath: file.path });
return this.collections.setPlaceImage(user.id, Number(pid), placeImageUrl(file.filename), socketId);
}
@Delete('places/:pid')
deletePlace(@CurrentUser() user: User, @Param('pid') pid: string, @Headers('x-socket-id') socketId?: string) {
this.collections.deletePlace(user.id, Number(pid), socketId);
async deletePlace(@CurrentUser() user: User, @Param('pid') pid: string, @Headers('x-socket-id') socketId?: string) {
await this.collections.deletePlace(user.id, Number(pid), socketId);
return { success: true };
}
@@ -360,12 +354,15 @@ export class CollectionsController {
}
@Post(':id/cover')
@UseInterceptors(FileInterceptor('cover', COVER_UPLOAD))
uploadCover(@CurrentUser() user: User, @Param('id') id: string, @UploadedFile() file: Express.Multer.File | undefined, @Headers('x-socket-id') socketId?: string) {
@UseInterceptors(FileInterceptor('cover', { fileFilter: COVER_FILE_FILTER }))
async uploadCover(@CurrentUser() user: User, @Param('id') id: string, @UploadedFile() file: Express.Multer.File | undefined, @Headers('x-socket-id') socketId?: string) {
if (isDemoWriteBlocked(this.env, user.email)) {
throw new HttpException(DEMO_WRITE_ERROR, 403);
}
if (!file) throw new HttpException({ error: 'No image uploaded' }, 400);
// Commit the spooled upload to its final storage location (atomic
// same-volume rename) before the DB row references the final path.
await this.storage.put('covers', file.filename, { tmpPath: file.path });
const coverUrl = `/uploads/covers/${file.filename}`;
return this.collections.setCollectionCover(user.id, Number(id), coverUrl, socketId);
}
@@ -211,7 +211,7 @@ export class CollectionsMcp {
})
async updateCollectionPlace({ placeId, ...body }: { placeId: number } & CollectionPlaceUpdateRequest, ctx: McpContext) {
const demo = this.denyDemo(ctx.userId); if (demo) return demo;
try { return ok({ place: this.collections.updatePlace(ctx.userId, placeId, body) }); } catch (err) { return fail(err); }
try { return ok({ place: await this.collections.updatePlace(ctx.userId, placeId, body) }); } catch (err) { return fail(err); }
}
@Tool({
@@ -253,7 +253,7 @@ export class CollectionsMcp {
})
async deleteCollectionPlace({ placeId }: { placeId: number }, ctx: McpContext) {
const demo = this.denyDemo(ctx.userId); if (demo) return demo;
try { this.collections.deletePlace(ctx.userId, placeId); return ok({ success: true }); } catch (err) { return fail(err); }
try { await this.collections.deletePlace(ctx.userId, placeId); return ok({ success: true }); } catch (err) { return fail(err); }
}
@Tool({
@@ -9,12 +9,33 @@ import { AddonsModule } from '../addons/addons.module';
import { PermissionsModule } from '../permissions/permissions.module';
import { AuthModule } from '../auth/auth.module';
import { NotificationsModule } from '../notifications/notifications.module';
import { MulterModule } from '@nestjs/platform-express';
import { StorageModule } from '../storage/storage.module';
import { StorageService } from '../storage/storage.service';
import { buildStorageUploadOptions } from '../storage/storage-upload.factory';
import { MAX_COVER_SIZE } from './collections.controller';
/** Collections domain (saved-places library). Registered in AppModule.
* Exports CollectionsService for in-container consumers (PluginsModule's
* RPC host deps factory). */
@Module({
imports: [NotificationsModule, AddonsModule, PermissionsModule, AuthModule, AppConfigModule, PluginGuardsModule],
imports: [
// Two upload routes, two categories: the per-file resolver routes each part
// to the right spool. Fieldname routing is exact — FileInterceptor('cover')/
// FileInterceptor('image') reject any other part with LIMIT_UNEXPECTED_FILE
// before the storage callbacks run. Both routes pass their fileFilter
// inline (cover keeps its plain-Error 500 quirk; image its statusCode 400).
MulterModule.registerAsync({
imports: [StorageModule],
inject: [StorageService],
useFactory: (storage: StorageService) =>
buildStorageUploadOptions(storage, {
category: (_req, file) => (file.fieldname === 'image' ? 'places' : 'covers'),
maxSize: MAX_COVER_SIZE, // same 20 MB cap for covers and place images
}),
}),
StorageModule,
NotificationsModule, AddonsModule, PermissionsModule, AuthModule, AppConfigModule, PluginGuardsModule],
controllers: [CollectionsController],
providers: [CollectionsService, CollectionsMcp, CollectionsRpc],
exports: [CollectionsService],
+15 -5
View File
@@ -83,11 +83,14 @@ export class CollectionsRpc {
}
@PluginMethod('collections.deletePlace', { permission: 'db:write:collections' })
deletePlace(params: Record<string, unknown>, ctx: PluginRpcContext): unknown {
async deletePlace(params: Record<string, unknown>, ctx: PluginRpcContext): Promise<unknown> {
const userId = this.requireCollectionsUser(ctx, 'writes');
const placeId = num(params.placeId, 'placeId');
this.requireCollectionsAddon();
this.mapCollectionError(() => this.collections.deletePlace(userId, placeId, undefined));
// deletePlace is async (it deletes the underlying storage object): await it so a
// refusal actually reaches the plugin as RESOURCE_FORBIDDEN/BAD_PARAMS instead of
// being dropped as an unhandled rejection while this returns {deleted: true} anyway.
await this.mapCollectionError(() => this.collections.deletePlace(userId, placeId, undefined));
return { deleted: true };
}
@@ -102,10 +105,17 @@ export class CollectionsRpc {
this.guards.requireAddon(ADDON_IDS.COLLECTIONS, 'collections');
}
/** The service's status-tagged errors, mapped onto the RPC error taxonomy. */
private mapCollectionError<T>(fn: () => T): T {
/**
* The service's status-tagged errors, mapped onto the RPC error taxonomy.
*
* Async-aware: `deletePlace` is async (it deletes the underlying storage object),
* while `create`/`update`/`savePlace`/`copyToTrip` stay sync. Awaiting inside always
* works for both — a sync throw from `fn()` is caught by this function's own
* try/catch same as an awaited rejection — so every caller goes through one path.
*/
private async mapCollectionError<T>(fn: () => T | Promise<T>): Promise<T> {
try {
return fn();
return await fn();
} catch (e) {
const status = (e as { status?: number })?.status;
const msg = e instanceof Error ? e.message : 'collection error';
@@ -1,10 +1,10 @@
import fs from 'fs';
import path from 'path';
import { Injectable } from '@nestjs/common';
import { DatabaseService } from '../database/database.service';
import { PermissionsService } from '../permissions/permissions.service';
import { RealtimeService } from '../realtime/realtime.service';
import { reclaimPlaceImage } from '../places/place-image';
import { StorageService } from '../storage/storage.service';
import {
COORD_DEDUP_TOLERANCE,
externalIdsOf,
@@ -47,17 +47,6 @@ function serializeLinks(links: CollectionLink[] | undefined): string | null {
return links && links.length ? JSON.stringify(links) : null;
}
/**
* Reclaim a replaced cover file. Path-confined to uploads/covers (mirrors
* tripService.deleteOldCover — kept local so this service doesn't pull the
* trips import graph). Collection + trip covers share the same directory.
*/
function deleteOldCollectionCover(coverImage: string | null | undefined): void {
if (!coverImage) return;
const coversDir = path.resolve(__dirname, '../../../uploads/covers');
const resolved = path.resolve(path.join(coversDir, path.basename(coverImage)));
if (resolved.startsWith(coversDir + path.sep) && fs.existsSync(resolved)) fs.unlinkSync(resolved);
}
// ---------------------------------------------------------------------------
// Errors — thrown as plain Errors carrying a status; TrekExceptionFilter maps
@@ -96,8 +85,24 @@ export class CollectionsService {
private readonly permissions: PermissionsService,
private readonly realtime: RealtimeService,
private readonly notifications: NotificationsService,
private readonly storage: StorageService,
) {}
/**
* Reclaim a replaced cover object (mirrors tripService.deleteOldCover — kept
* local so this service doesn't pull the trips import graph). Collection +
* trip covers share the 'covers' category; basename() tolerates the stored
* /uploads/covers/<name> URL form and any external URL the client saved
* (central key validation rejects hostile values; the catch swallows them
* like the old containment guard did).
*/
private async deleteOldCollectionCover(coverImage: string | null | undefined): Promise<void> {
if (!coverImage) return;
await this.storage.delete('covers', path.basename(coverImage)).catch(() => {
/* external URL or already gone */
});
}
// -------------------------------------------------------------------------
// Visibility — a user may see/edit a collection if they own it OR are an
// accepted member. Every read/write goes through assertAccess.
@@ -374,11 +379,11 @@ export class CollectionsService {
}
/** Set (or clear) a list's cover image, reclaiming the previous file. */
setCollectionCover(userId: number, id: number, coverUrl: string | null, socketId?: string): Collection {
async setCollectionCover(userId: number, id: number, coverUrl: string | null, socketId?: string): Promise<Collection> {
this.assertCanEdit(userId, id);
const prev = this.db.get<{ cover_image: string | null }>('SELECT cover_image FROM collections WHERE id = ?', id)?.cover_image ?? null;
this.db.run('UPDATE collections SET cover_image = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?', coverUrl, id);
if (prev && prev !== coverUrl) deleteOldCollectionCover(prev);
if (prev && prev !== coverUrl) await this.deleteOldCollectionCover(prev);
this.notifyCollectionUsers(id, socketId, 'collections:updated');
const col = this.getCollectionRow(id);
return { ...col, is_owner: col.owner_id === userId };
@@ -649,7 +654,7 @@ export class CollectionsService {
return { copied, skipped };
}
updatePlace(userId: number, placeId: number, body: CollectionPlaceUpdateRequest, socketId?: string): CollectionPlace {
async updatePlace(userId: number, placeId: number, body: CollectionPlaceUpdateRequest, socketId?: string): Promise<CollectionPlace> {
const currentCollection = this.collectionIdOfPlace(placeId);
this.assertCanEdit(userId, currentCollection);
@@ -701,7 +706,7 @@ export class CollectionsService {
});
if (body.image_url !== undefined && prevImage !== (body.image_url ?? null)) {
reclaimPlaceImage(prevImage);
await reclaimPlaceImage(this.storage, prevImage);
}
this.notifyCollectionUsers(currentCollection, socketId, 'collections:updated');
@@ -737,16 +742,16 @@ export class CollectionsService {
return this.getPlaceById(placeId);
}
deletePlace(userId: number, placeId: number, socketId?: string): void {
async deletePlace(userId: number, placeId: number, socketId?: string): Promise<void> {
const collectionId = this.collectionIdOfPlace(placeId);
this.assertCanDelete(userId, collectionId);
const image = this.db.get<{ image_url: string | null }>('SELECT image_url FROM collection_places WHERE id = ?', placeId)?.image_url ?? null;
this.db.run('DELETE FROM collection_places WHERE id = ?', placeId); // CASCADE drops tags. NO photo-cache reclaim.
reclaimPlaceImage(image);
await reclaimPlaceImage(this.storage, image);
this.notifyCollectionUsers(collectionId, socketId, 'collections:updated');
}
deletePlacesMany(userId: number, ids: number[], socketId?: string): number[] {
async deletePlacesMany(userId: number, ids: number[], socketId?: string): Promise<number[]> {
// All-or-nothing since the post-fold quirk pass: every id is resolved and
// permission-checked BEFORE any delete (the relocated legacy interleaved
// checks with deletes, so a mid-list 403/404 left earlier deletes committed),
@@ -766,7 +771,7 @@ export class CollectionsService {
deleted.push(id);
}
});
for (const image of images) reclaimPlaceImage(image);
for (const image of images) await reclaimPlaceImage(this.storage, image);
touched.forEach(cid => this.notifyCollectionUsers(cid, socketId, 'collections:updated'));
return deleted;
}
@@ -875,12 +880,12 @@ export class CollectionsService {
}
/** Set (or clear) a saved place's custom thumbnail, reclaiming the previous upload. */
setPlaceImage(userId: number, placeId: number, imageUrl: string | null, socketId?: string): CollectionPlace {
async setPlaceImage(userId: number, placeId: number, imageUrl: string | null, socketId?: string): Promise<CollectionPlace> {
const collectionId = this.collectionIdOfPlace(placeId);
this.assertCanEdit(userId, collectionId);
const prev = this.db.get<{ image_url: string | null }>('SELECT image_url FROM collection_places WHERE id = ?', placeId)?.image_url ?? null;
this.db.run('UPDATE collection_places SET image_url = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?', imageUrl, placeId);
if (prev !== imageUrl) reclaimPlaceImage(prev);
if (prev !== imageUrl) await reclaimPlaceImage(this.storage, prev);
this.notifyCollectionUsers(collectionId, socketId, 'collections:updated');
return this.getPlaceById(placeId);
}
@@ -41,3 +41,8 @@ export function maybe_encrypt_api_key(value: unknown) {
return encrypt_api_key(trimmed);
}
/** True exactly for values carrying the enc:v1: envelope produced by encrypt_api_key. */
export function is_encrypted_api_key(value: unknown): value is string {
return typeof value === 'string' && value.startsWith(ENCRYPTED_PREFIX);
}
+18 -32
View File
@@ -1,38 +1,24 @@
import fs from 'fs';
import path from 'path';
import { diskStorage } from 'multer';
import { v4 as uuidv4 } from 'uuid';
import type { Request } from 'express';
import { PLACE_IMAGES_DIR } from '../places/place-image';
import type { Options } from 'multer';
const MAX_PLACE_IMAGE_SIZE = 20 * 1024 * 1024; // 20 MB — same cap as covers.
export const MAX_PLACE_IMAGE_SIZE = 20 * 1024 * 1024; // 20 MB — same cap as covers.
/**
* Multer config for the custom place-image upload (#1136). Mirrors the collection
* COVER_UPLOAD: server-chosen UUID filename, image-only filter, written to the
* dedicated uploads/places dir. Shared by the trip-place and collection-place
* upload endpoints.
* fileFilter for the custom place-image upload (#1136), shared by the
* trip-place and collection-place upload endpoints (passed inline on each
* route; the storage engine — spool destination + UUID filename — comes from
* the owning module's storage-upload factory options).
*/
export const PLACE_IMAGE_UPLOAD = {
storage: diskStorage({
destination: (_req, _file, cb) => {
if (!fs.existsSync(PLACE_IMAGES_DIR)) fs.mkdirSync(PLACE_IMAGES_DIR, { recursive: true });
cb(null, PLACE_IMAGES_DIR);
},
filename: (_req, file, cb) => cb(null, `${uuidv4()}${path.extname(file.originalname)}`),
}),
limits: { fileSize: MAX_PLACE_IMAGE_SIZE },
fileFilter: (_req: Request, file: Express.Multer.File, cb: (err: Error | null, accept: boolean) => void) => {
const ext = path.extname(file.originalname).toLowerCase();
const allowed = ['.jpg', '.jpeg', '.png', '.gif', '.webp'];
if (file.mimetype.startsWith('image/') && !file.mimetype.includes('svg') && allowed.includes(ext)) {
cb(null, true);
} else {
// Carry statusCode so TrekExceptionFilter maps the rejection to a 400 rather
// than a 500 (same contract as the avatar upload's fileFilter).
const err: Error & { statusCode?: number } = new Error('Only jpg, png, gif, webp images allowed');
err.statusCode = 400;
cb(err, false);
}
},
export const PLACE_IMAGE_FILE_FILTER: Options['fileFilter'] = (_req, file, cb) => {
const ext = path.extname(file.originalname).toLowerCase();
const allowed = ['.jpg', '.jpeg', '.png', '.gif', '.webp'];
if (file.mimetype.startsWith('image/') && !file.mimetype.includes('svg') && allowed.includes(ext)) {
cb(null, true);
} else {
// Carry statusCode so TrekExceptionFilter maps the rejection to a 400 rather
// than a 500 (same contract as the avatar upload's fileFilter).
const err: Error & { statusCode?: number } = new Error('Only jpg, png, gif, webp images allowed');
err.statusCode = 400;
cb(err);
}
};
@@ -22,6 +22,22 @@ export class TrekExceptionFilter implements ExceptionFilter {
catch(exception: unknown, host: ArgumentsHost): void {
const res = host.switchToHttp().getResponse<Response>();
// 0. A response already streaming to the client (headers sent) cannot be
// rewritten with a JSON error envelope — this fires when a storage
// stream (or any other in-flight write) errors after
// sendToResponse()/pipeline() has already flushed headers, e.g. a
// client abort mid-download. Writing here would throw
// ERR_HTTP_HEADERS_SENT or corrupt a response the client is mid-read
// on. Log it first — this is the ONLY place a post-header failure is
// ever recorded, since destroying the socket below skips every other
// branch's logging (including case 2's `status >= 500` log) — then
// destroy the socket so the connection doesn't hang open.
if (res.headersSent) {
console.error('Unhandled error after headers sent:', exception);
res.destroy();
return;
}
// 1. Raw multer errors that slipped past @nestjs/platform-express's
// transformException (it leaves codes it does not recognise untouched).
// Legacy: LIMIT_FILE_SIZE -> 413, everything else -> 400, body { error: message }.
@@ -79,6 +79,14 @@ export const MANAGED_ROUTE_ALLOW_LIST: string[] = [
'PluginsController.link',
'PluginsController.reload',
'PluginsController.upload',
'StorageAdminController.backfillCancel',
'StorageAdminController.backfillStart',
'StorageAdminController.get',
'StorageAdminController.migrationCancel',
'StorageAdminController.migrationStart',
'StorageAdminController.statsRefresh',
'StorageAdminController.test',
'StorageAdminController.update',
];
/**
@@ -1,9 +1,9 @@
import { Controller, Get, HttpException, Param, Req, Res } from '@nestjs/common';
import type { Request, Response } from 'express';
import path from 'path';
import fs from 'fs';
import { FilesService } from './files.service';
import { Public } from '../auth/public.decorator';
import { StorageService } from '../storage/storage.service';
/**
* GET /api/trips/:tripId/files/:id/download — authenticated file download.
@@ -11,15 +11,25 @@ import { Public } from '../auth/public.decorator';
* Deliberately NOT behind the JwtAuthGuard: it accepts a cookie, a Bearer header
* OR a one-shot `?token=` query param (so links can be opened directly), all via
* the legacy authenticateDownload helper. Byte-identical to the legacy route:
* 401 token, 404 trip/file, 403 path traversal, .pkpass served inline for Wallet.
* 401 token, 404 trip/file, .pkpass served inline for Wallet. Bytes come from
* storage.sendToResponse, whose local branch is the root-relative res.sendFile
* form this route needs under the ExpressAdapter.
*/
@Public('authenticates itself: the download link carries a short-lived ?token the client mints')
@Controller('api/trips/:tripId/files')
export class FilesDownloadController {
constructor(private readonly files: FilesService) {}
constructor(
private readonly files: FilesService,
private readonly storage: StorageService,
) {}
@Get(':id/download')
download(@Req() req: Request, @Res() res: Response, @Param('tripId') tripId: string, @Param('id') id: string): void {
async download(
@Req() req: Request,
@Res() res: Response,
@Param('tripId') tripId: string,
@Param('id') id: string,
): Promise<void> {
const auth = this.files.authenticateDownload(req);
if ('error' in auth) {
throw new HttpException({ error: auth.error }, auth.status);
@@ -35,11 +45,10 @@ export class FilesDownloadController {
throw new HttpException({ error: 'File not found' }, 404);
}
const { resolved, safe } = this.files.resolveFilePath(file.filename);
if (!safe) {
throw new HttpException({ error: 'Forbidden' }, 403);
}
if (!fs.existsSync(resolved)) {
// basename() tolerates a stray prefixed row (legacy rows sometimes stored
// a relative path); the storage layer's key validation refuses the rest.
const name = path.basename(file.filename);
if (!(await this.storage.exists('files', name).catch(() => false))) {
throw new HttpException({ error: 'File not found' }, 404);
}
@@ -47,22 +56,20 @@ export class FilesDownloadController {
// (iOS/macOS) hands them to Wallet instead of downloading as a blob. A
// `.pkpasses` bundle (a ZIP of multiple passes) is a distinct type with its
// own plural MIME type — without it Wallet won't offer to add the passes.
const ext = path.extname(name).toLowerCase();
const walletMime =
path.extname(resolved).toLowerCase() === '.pkpass'
ext === '.pkpass'
? 'application/vnd.apple.pkpass'
: path.extname(resolved).toLowerCase() === '.pkpasses'
: ext === '.pkpasses'
? 'application/vnd.apple.pkpasses'
: null;
if (walletMime) {
res.setHeader('Content-Type', walletMime);
res.setHeader('Content-Disposition', `inline; filename="${path.basename(file.original_name || resolved)}"`);
}
// Serve with an explicit { root } + basename rather than an absolute path:
// under the Nest ExpressAdapter, res.sendFile(absolutePath) resolves the
// file relative to the (rewritten) req.url and fails with a spurious
// "Not Found", whereas the root-relative form streams correctly. The
// resolveFilePath guard above already pins this to the uploads dir.
res.sendFile(path.basename(resolved), { root: path.dirname(resolved) });
await this.storage.sendToResponse(
'files',
name,
res,
walletMime
? { contentType: walletMime, disposition: `inline; filename="${path.basename(file.original_name || name)}"` }
: undefined,
);
}
}
+35 -36
View File
@@ -18,55 +18,45 @@ import {
import { FileInterceptor } from '@nestjs/platform-express';
import { isDemoWriteBlocked, DEMO_WRITE_ERROR } from '../common/demo-write';
import { RuntimeEnvService } from '../app-config/runtime-env.service';
import { diskStorage } from 'multer';
import type { Options } from 'multer';
import path from 'path';
import fs from 'fs';
import { v4 as uuidv4 } from 'uuid';
import type { User } from '../../types';
import { StorageService } from '../storage/storage.service';
import { FilesService } from './files.service';
import { JwtAuthGuard } from '../auth/jwt-auth.guard';
import { CurrentUser } from '../auth/current-user.decorator';
import { TripAccessGuard } from '../permissions/trip-access.guard';
import type { TripAccess } from '../database/database.service';
import { Trip } from '../permissions/trip.decorator';
import { MAX_FILE_SIZE, MAX_VIDEO_SIZE, BLOCKED_EXTENSIONS, filesDir, isVideoExtension } from './files.constants';
import { MAX_FILE_SIZE, BLOCKED_EXTENSIONS, isVideoExtension } from './files.constants';
import { FileUploadDto, FileUpdateDto, FileLinkDto } from './files.dto';
import { AllowedFileTypesService } from './allowed-file-types.service';
/**
* Trip-file upload options, built from the container.
* Trip-file upload filter, built from the container.
*
* A factory rather than a module-scope literal because the fileFilter reads the
* operator's allowed-extension list at request time, and that list lives in the
* database. It used to reach it through files.bridge.ts, which built a second
* FilesService outside the injector; MulterModule.registerAsync gives the
* closure the real one.
* A factory rather than a module-scope literal because it reads the operator's
* allowed-extension list at request time, and that list lives in the database.
* The rest of the multer options come from the storage upload factory
* (buildStorageUploadOptions in files.module.ts); this closure is passed
* through untouched.
*/
export function buildFilesUploadOptions(allowedTypes: AllowedFileTypesService) {
return {
storage: diskStorage({
destination: (_req, _file, cb) => { if (!fs.existsSync(filesDir)) fs.mkdirSync(filesDir, { recursive: true }); cb(null, filesDir); },
filename: (_req, file, cb) => cb(null, `${uuidv4()}${path.extname(file.originalname)}`),
}),
// Allow up to the video cap; non-video files are still held to MAX_FILE_SIZE by
// the per-type guard in the upload handler (#823).
limits: { fileSize: MAX_VIDEO_SIZE },
defParamCharset: 'utf8', // parity with legacy routes/files.ts — preserve non-ASCII original filenames
fileFilter: (_req: unknown, file: Express.Multer.File, cb: (err: Error | null, accept: boolean) => void) => {
const ext = path.extname(file.originalname).toLowerCase();
const reject = () => {
// i18n key — the client resolves it via t() (see translateApiError).
const err: Error & { statusCode?: number } = new Error('files.uploadErrorType');
err.statusCode = 400;
cb(err, false);
};
if (BLOCKED_EXTENSIONS.includes(ext) || file.mimetype.includes('svg')) return reject();
const allowed = allowedTypes.get().split(',').map((e) => e.trim().toLowerCase());
const fileExt = ext.replace('.', '');
// Video is accepted as media regardless of the admin doc-types allowlist (#823).
if (allowed.includes(fileExt) || isVideoExtension(fileExt) || (allowed.includes('*') && !BLOCKED_EXTENSIONS.includes(ext))) return cb(null, true);
reject();
},
export function filesUploadFileFilter(allowedTypes: AllowedFileTypesService): Options['fileFilter'] {
return (_req, file, cb) => {
const ext = path.extname(file.originalname).toLowerCase();
const reject = () => {
// i18n key — the client resolves it via t() (see translateApiError).
const err: Error & { statusCode?: number } = new Error('files.uploadErrorType');
err.statusCode = 400;
cb(err);
};
if (BLOCKED_EXTENSIONS.includes(ext) || file.mimetype.includes('svg')) return reject();
const allowed = allowedTypes.get().split(',').map((e) => e.trim().toLowerCase());
const fileExt = ext.replace('.', '');
// Video is accepted as media regardless of the admin doc-types allowlist (#823).
if (allowed.includes(fileExt) || isVideoExtension(fileExt) || (allowed.includes('*') && !BLOCKED_EXTENSIONS.includes(ext))) return cb(null, true);
reject();
};
}
@@ -97,6 +87,7 @@ export class FilesController {
constructor(
private readonly files: FilesService,
private readonly env: RuntimeEnvService,
private readonly storage: StorageService,
) {}
@@ -117,14 +108,14 @@ export class FilesController {
@Post()
@UseInterceptors(FileInterceptor('file'))
upload(
async upload(
@CurrentUser() user: User,
@Param('tripId') tripId: string,
@UploadedFile() file: Express.Multer.File | undefined,
@Body() body: FileUploadDto,
@Headers('x-socket-id') socketId?: string,
) {
// multer (diskStorage) has already written the upload by the time we get here,
// multer (diskStorage) has already spooled the upload by the time we get here,
// so every rejection below must remove the orphaned bytes — otherwise a 404/403
// leaves up to the 500 MB video cap on disk (#823).
const cleanup = () => { if (file?.path) { try { fs.unlinkSync(file.path); } catch { /* best-effort */ } } };
@@ -162,6 +153,14 @@ export class FilesController {
cleanup();
throw err;
}
// Commit the spooled upload to its final storage location (atomic
// same-volume rename) before anything references the final path.
try {
await this.storage.put('files', file.filename, { tmpPath: file.path });
} catch (err) {
cleanup();
throw err;
}
const created = this.files.createFile(tripId, file, user.id, {
place_id: body.place_id,
description: body.description,
+17 -4
View File
@@ -11,15 +11,28 @@ import { EphemeralTokenModule } from '../auth/ephemeral-token.module';
import { MulterModule } from '@nestjs/platform-express';
import { AllowedFileTypesModule } from './allowed-file-types.module';
import { AllowedFileTypesService } from './allowed-file-types.service';
import { buildFilesUploadOptions } from './files.controller';
import { StorageModule } from '../storage/storage.module';
import { StorageService } from '../storage/storage.service';
import { buildStorageUploadOptions } from '../storage/storage-upload.factory';
import { filesUploadFileFilter } from './files.controller';
import { MAX_VIDEO_SIZE } from './files.constants';
@Module({
imports: [
MulterModule.registerAsync({
imports: [AllowedFileTypesModule],
inject: [AllowedFileTypesService],
useFactory: (allowedTypes: AllowedFileTypesService) => buildFilesUploadOptions(allowedTypes),
imports: [StorageModule, AllowedFileTypesModule],
inject: [StorageService, AllowedFileTypesService],
useFactory: (storage: StorageService, allowedTypes: AllowedFileTypesService) =>
buildStorageUploadOptions(storage, {
category: 'files',
// Allow up to the video cap; non-video files are still held to
// MAX_FILE_SIZE by the per-type guard in the upload handler (#823).
maxSize: MAX_VIDEO_SIZE,
defParamCharset: 'utf8', // parity with legacy routes/files.ts — preserve non-ASCII original filenames
fileFilter: filesUploadFileFilter(allowedTypes),
}),
}),
StorageModule,
EphemeralTokenModule, PermissionsModule, AppConfigModule, RealtimeModule, PluginGuardsModule],
controllers: [FilesController, FilesDownloadController],
providers: [FilesService, FilesRpc],
+48 -14
View File
@@ -1,6 +1,6 @@
import fsMod from 'node:fs';
import pathMod from 'node:path';
import { randomUUID } from 'node:crypto';
import { Readable } from 'node:stream';
import { PluginController, PluginMethod } from '../plugins/host/rpc-kit/decorators';
import { PluginGuards } from '../plugins/host/plugin-guards.service';
import { BadParams, ForbiddenResource } from '../plugins/host/rpc-errors';
@@ -10,8 +10,10 @@ import { RealtimeService } from '../realtime/realtime.service';
import { DatabaseService } from '../database/database.service';
import { readEnv } from '../../app-config';
import { isDemoEmail } from '../common/demo';
import { BLOCKED_EXTENSIONS, filesDir } from './files.constants';
import { BLOCKED_EXTENSIONS } from './files.constants';
import { FilesService } from './files.service';
import { StorageService } from '../storage/storage.service';
import { StorageNotFoundError, StorageInvalidKeyError, type ObjectStat } from '../storage/storage.types';
/** Files use three separate rights, one per operation, unlike every other domain. */
const UPLOAD_ACTION = 'file_upload';
@@ -46,9 +48,10 @@ type CreateInput = {
* sensitive than its filename. And the three write operations use three distinct
* rights (file_upload / file_edit / file_delete) rather than one domain action.
*
* The upload path is the only one in the plugin surface that touches disk, so the
* extension is checked against the central blocklist and the size is capped before
* anything is written, and a demo user is refused outright.
* This surface holds no direct fs access: both byte-paths — read and write — go
* through the storage layer, so `ctx.files` behaves identically whether the backend
* is local disk, S3, or a mirrored pair. The extension blocklist and size cap still
* run before the put, and a demo user is refused outright.
*/
@PluginController()
export class FilesRpc {
@@ -57,6 +60,7 @@ export class FilesRpc {
private readonly realtime: RealtimeService,
private readonly db: DatabaseService,
private readonly guards: PluginGuards,
private readonly storage: StorageService,
) {}
@PluginMethod('files.list', { permission: 'db:read:files' })
@@ -74,7 +78,8 @@ export class FilesRpc {
/**
* Size-capped BEFORE the read, so a 500MB video cannot be pulled through the IPC
* pipe as ~667MB of base64. The read itself runs off the event loop: 10MB of
* pipe as ~667MB of base64. The bytes come from the storage layer, so this read
* works on remote backends too. The read itself runs off the event loop: 10MB of
* readFile plus base64 on the host thread would stall every other plugin RPC and
* every HTTP request for its duration.
*/
@@ -84,10 +89,35 @@ export class FilesRpc {
if ((file.file_size ?? 0) > CONTENT_MAX) {
throw new BadParams(`file too large to read (>${CONTENT_MAX} bytes); use the download UI`);
}
const { resolved, safe } = this.files.resolveFilePath(file.filename);
if (!safe) throw new ForbiddenResource('file path is not accessible');
const buf = await fsMod.promises.readFile(resolved);
if (buf.length > CONTENT_MAX) throw new BadParams('file too large to read');
let stream: Readable;
let stat: ObjectStat;
try {
({ stream, stat } = await this.storage.getStream('files', pathMod.basename(file.filename)));
} catch (err) {
if (err instanceof StorageNotFoundError || err instanceof StorageInvalidKeyError) {
throw new ForbiddenResource('file path is not accessible');
}
throw err;
}
// Re-checked against the OBJECT, not the DB row: file_size can drift.
if (stat.size > CONTENT_MAX) {
stream.destroy();
throw new BadParams(`file too large to read (>${CONTENT_MAX} bytes); use the download UI`);
}
const chunks: Buffer[] = [];
let total = 0;
for await (const chunk of stream) {
const part = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array);
total += part.length;
// A driver whose stat under-reports must not push an oversized payload
// through the IPC pipe: abort as soon as the running total crosses the cap.
if (total > CONTENT_MAX) {
stream.destroy();
throw new BadParams('file too large to read');
}
chunks.push(part);
}
const buf = Buffer.concat(chunks);
return {
name: file.original_name,
mimetype: file.mime_type ?? 'application/octet-stream',
@@ -97,7 +127,7 @@ export class FilesRpc {
}
@PluginMethod('files.create', { permission: 'db:write:files' })
create(params: Record<string, unknown>, ctx: PluginRpcContext): unknown {
async create(params: Record<string, unknown>, ctx: PluginRpcContext): Promise<unknown> {
const tripId = num(params.tripId, 'tripId');
const actor = this.guards.requireActor(ctx, 'file');
const input = asPayload(params.input);
@@ -116,7 +146,7 @@ export class FilesRpc {
return this.writeFile(tripId, input as unknown as CreateInput, actor);
}
private writeFile(tripId: number, input: CreateInput, actingUserId: number): unknown {
private async writeFile(tripId: number, input: CreateInput, actingUserId: number): Promise<unknown> {
// Mirrors the REST upload guard: a demo user must not write bytes to the shared
// demo instance, not even through a plugin's db:write:files. The email is only
// resolved when demo mode is actually on, so self-hosted installs pay nothing.
@@ -138,8 +168,12 @@ export class FilesRpc {
});
if (foreign) throw new ForbiddenResource(`${foreign} does not belong to trip ${tripId}`);
const filename = `${randomUUID()}${ext}`;
fsMod.mkdirSync(filesDir, { recursive: true });
fsMod.writeFileSync(pathMod.join(filesDir, filename), buf);
// Same order as the REST upload (files.controller.ts): the object is
// committed to storage before anything references it — a put failure can
// orphan a blob at worst, never create a row pointing at missing bytes.
await this.storage.put('files', filename, Readable.from(buf), {
contentType: input.mimetype || 'application/octet-stream',
});
const file = this.files.createFile(
tripId,
{ filename, originalname: original, size: buf.length, mimetype: input.mimetype || 'application/octet-stream' },
+12 -29
View File
@@ -1,6 +1,5 @@
import { Injectable } from '@nestjs/common';
import path from 'path';
import fs from 'fs';
import type { Request } from 'express';
import type { TrekWsPayload, TrekWsTripEventName } from '@trek/shared';
import { RealtimeService } from '../realtime/realtime.service';
@@ -10,7 +9,8 @@ import { EphemeralTokenService } from '../auth/ephemeral-token.service';
import { verifyJwtAndLoadUser } from '../auth/jwt-verify';
import type { User, TripFile } from '../../types';
import { DatabaseService, type TripAccess } from '../database/database.service';
import { DEFAULT_ALLOWED_EXTENSIONS, filesDir } from './files.constants';
import { DEFAULT_ALLOWED_EXTENSIONS } from './files.constants';
import { StorageService } from '../storage/storage.service';
type Trip = TripAccess;
type FilePermission = 'file_upload' | 'file_edit' | 'file_delete';
@@ -45,9 +45,9 @@ export interface FileLink {
* DatabaseService's canAccessTrip (the legacy verifyTripAccess re-export was a
* plain wrapper over the same helper); the file_* permissions, path-resolution
* guard, download-token auth and WebSocket broadcasts are unchanged. The
* load-time constants live in files.constants.ts; the one out-of-DI consumer
* path (module-scope multer configs) reaches getAllowedExtensions through
* files.bridge.ts.
* load-time constants live in files.constants.ts; the admin allowed-types
* live-read is AllowedFileTypesService (the single query owner since the
* storage slice-2 consolidation).
*
* Two deliberate post-migration fixes over the legacy behavior: createFileLink
* no longer swallows insert errors, and updateFile coerces an empty-string
@@ -60,6 +60,7 @@ export class FilesService {
private readonly permissions: PermissionsService,
private readonly realtime: RealtimeService,
private readonly tokens: EphemeralTokenService,
private readonly storage: StorageService,
) {}
verifyTripAccess(tripId: string | number, userId: number) {
@@ -74,13 +75,6 @@ export class FilesService {
this.realtime.broadcast(tripId, event, payload, socketId);
}
getAllowedExtensions(): string {
try {
const row = this.db.get<{ value: string }>("SELECT value FROM app_settings WHERE key = 'allowed_file_types'");
return row?.value || DEFAULT_ALLOWED_EXTENSIONS;
} catch { return DEFAULT_ALLOWED_EXTENSIONS; }
}
// ---------------------------------------------------------------------------
// Token-based download auth & safe path resolution (used by the unguarded
// download route)
@@ -112,14 +106,6 @@ export class FilesService {
return { error: 'Authentication required', status: 401 };
}
resolveFilePath(filename: string): { resolved: string; safe: boolean } {
const safeName = path.basename(filename);
const filePath = path.join(filesDir, safeName);
const resolved = path.resolve(filePath);
const safe = resolved.startsWith(path.resolve(filesDir));
return { resolved, safe };
}
// ---------------------------------------------------------------------------
// Trip-scoped link validation
// ---------------------------------------------------------------------------
@@ -251,14 +237,12 @@ export class FilesService {
}
async permanentDeleteFile(file: TripFile): Promise<void> {
const { resolved } = this.resolveFilePath(file.filename);
// `force: true` swallows ENOENT, replacing the prior existsSync+unlink
// double-call that blocked the event loop twice per deletion. Only
// drop the DB row when the on-disk unlink either succeeded or the
// file was already gone — otherwise a permission / ENOSPC failure
// would orphan the bytes on disk with no DB pointer left to clean it.
// storage.delete is idempotent on a missing object (the old rm force:true
// contract). Only drop the DB row when the delete either succeeded or the
// object was already gone — otherwise a permission / ENOSPC failure
// would orphan the bytes with no DB pointer left to clean them.
try {
await fs.promises.rm(resolved, { force: true });
await this.storage.delete('files', path.basename(file.filename));
} catch (e) {
console.error(`[files] unlink failed for ${file.filename}, keeping DB row:`, e);
throw e;
@@ -273,9 +257,8 @@ export class FilesService {
// and a retry via the single-file delete path can try again.
const successfullyUnlinked: number[] = [];
await Promise.all(trashed.map(async (file) => {
const { resolved } = this.resolveFilePath(file.filename);
try {
await fs.promises.rm(resolved, { force: true });
await this.storage.delete('files', path.basename(file.filename));
successfullyUnlinked.push(Number(file.id));
} catch (e) {
console.error(`[files] unlink failed for ${file.filename}, keeping DB row:`, e);
@@ -1468,22 +1468,22 @@ export class JourneyDomainService {
deleteGalleryPhoto(
journeyPhotoId: number,
userId: number,
): { photo_id: number; file_path?: string | null } | null {
): { photo_id: number; file_path?: string | null; thumbnail_path?: string | null } | null {
const row = this.db.prepare('SELECT * FROM journey_photos WHERE id = ?').get(journeyPhotoId) as
| { id: number; journey_id: number; photo_id: number }
| undefined;
if (!row) return null;
if (!this.canEdit(row.journey_id, userId)) return null;
const trekRow = this.db.prepare('SELECT file_path, provider FROM trek_photos WHERE id = ?').get(row.photo_id) as
| { file_path?: string; provider?: string }
const trekRow = this.db.prepare('SELECT file_path, thumbnail_path, provider FROM trek_photos WHERE id = ?').get(row.photo_id) as
| { file_path?: string; thumbnail_path?: string; provider?: string }
| undefined;
// cascade on journey_entry_photos.journey_photo_id handles junction cleanup
this.db.prepare('DELETE FROM journey_photos WHERE id = ?').run(journeyPhotoId);
this.photos.deleteIfOrphan(row.photo_id);
return { photo_id: row.photo_id, file_path: trekRow?.file_path ?? null };
return { photo_id: row.photo_id, file_path: trekRow?.file_path ?? null, thumbnail_path: trekRow?.thumbnail_path ?? null };
}
setPhotoProvider(photoId: number, provider: string, assetId: string, ownerId: number) {
@@ -1533,21 +1533,21 @@ export class JourneyDomainService {
deletePhoto(
photoId: number,
userId: number,
): { id: number; photo_id: number; file_path?: string | null; journey_id: number } | null {
): { id: number; photo_id: number; file_path?: string | null; thumbnail_path?: string | null; journey_id: number } | null {
const row = this.db.prepare('SELECT id, journey_id, photo_id FROM journey_photos WHERE id = ?').get(photoId) as
| { id: number; journey_id: number; photo_id: number }
| undefined;
if (!row) return null;
if (!this.canEdit(row.journey_id, userId)) return null;
const trekRow = this.db.prepare('SELECT file_path, provider FROM trek_photos WHERE id = ?').get(row.photo_id) as
| { file_path?: string; provider?: string }
const trekRow = this.db.prepare('SELECT file_path, thumbnail_path, provider FROM trek_photos WHERE id = ?').get(row.photo_id) as
| { file_path?: string; thumbnail_path?: string; provider?: string }
| undefined;
this.db.prepare('DELETE FROM journey_photos WHERE id = ?').run(photoId);
this.photos.deleteIfOrphan(row.photo_id);
return { id: row.id, photo_id: row.photo_id, file_path: trekRow?.file_path ?? null, journey_id: row.journey_id };
return { id: row.id, photo_id: row.photo_id, file_path: trekRow?.file_path ?? null, thumbnail_path: trekRow?.thumbnail_path ?? null, journey_id: row.journey_id };
}
// ── Contributors ─────────────────────────────────────────────────────────
@@ -1,8 +1,8 @@
import { Controller, Get, HttpException, Param, Res } from '@nestjs/common';
import type { Response } from 'express';
import path from 'node:path';
import fs from 'node:fs';
import { JourneyService } from './journey.service';
import { StorageService } from '../storage/storage.service';
import { Public } from '../auth/public.decorator';
/**
@@ -17,7 +17,10 @@ import { Public } from '../auth/public.decorator';
@Public('share-token validated: the whole point is a link that works without an account')
@Controller('api/public/journey')
export class JourneyPublicController {
constructor(private readonly journey: JourneyService) {}
constructor(
private readonly journey: JourneyService,
private readonly storage: StorageService,
) {}
@Get(':token')
get(@Param('token') token: string) {
@@ -54,17 +57,18 @@ export class JourneyPublicController {
const wantThumb = kind === 'thumbnail' ? 'thumbnail' : 'original';
if (provider === 'local') {
// Local journey assets are flat filenames; use basename() and confine the
// resolved path to the journey upload directory.
const journeyDir = path.resolve(__dirname, '../../../uploads/journey');
const resolved = path.resolve(path.join(journeyDir, path.basename(assetId)));
if (!resolved.startsWith(journeyDir + path.sep) || !fs.existsSync(resolved)) {
// Local journey assets are flat filenames. basename() kept for shape;
// containment now lives in central storage key validation
// (storage-keys.ts), which subsumes the old journeyDir startsWith guard.
const name = path.basename(assetId);
if (!(await this.storage.exists('journey', name).catch(() => false))) {
throw new HttpException({ error: 'Not found' }, 404);
}
res.set('Cache-Control', 'public, max-age=86400');
// Root-relative, see photo-resolver.service.ts — an absolute path 404s under
// the Nest ExpressAdapter. The guard above already confined `resolved`.
res.sendFile(path.basename(resolved), { root: path.dirname(resolved) });
// The storage layer serves it, whichever driver is configured. dev's fix
// for the absolute-path 404 under the Nest ExpressAdapter does not apply
// here any more: nothing on this path calls sendFile.
await this.storage.sendToResponse('journey', name, res);
return;
}
+138 -75
View File
@@ -16,11 +16,14 @@ import {
UseInterceptors,
} from '@nestjs/common';
import { FileInterceptor, FilesInterceptor, FileFieldsInterceptor } from '@nestjs/platform-express';
import { diskStorage } from 'multer';
import type { Options } from 'multer';
import type { Request } from 'express';
import path from 'node:path';
import fs from 'node:fs';
import crypto from 'node:crypto';
import type { User } from '../../types';
import { StorageService } from '../storage/storage.service';
import { journeyThumbName } from '../memories/thumbnail.service';
import { JourneyService } from './journey.service';
import { JourneyBookService } from './journey-book.service';
import { PhotoCaptureBackfillService } from '../memories/photo-capture-backfill.service';
@@ -39,73 +42,73 @@ import {
import { isVideoMime, isVideoExtension, MAX_VIDEO_SIZE } from '../files/files.constants';
import { AllowedFileTypesService } from '../files/allowed-file-types.service';
const uploadsBase = path.join(__dirname, '../../../uploads/journey');
/**
* Journey image upload options, built from the container.
*
* Same reason as the trip-file factory: the fileFilter reads the operator's
* allowed-extension list at request time. Note this config carries NO
* defParamCharset, unlike the trip-file one; that difference is deliberate and
* predates the move, so the two are not folded into one shared object.
* One filename hook for all four journey upload routes (consumed by
* journey.module.ts's storage-upload factory). Field routing is exact: the
* image routes only ever admit 'photos'/'cover' fields and the video route
* only 'video'/'poster' — any other part dies in multer with
* LIMIT_UNEXPECTED_FILE before this hook runs.
*/
export function buildJourneyImageUploadOptions(allowedTypes: AllowedFileTypesService) {
return {
storage: diskStorage({
destination: (_req, _file, cb) => { if (!fs.existsSync(uploadsBase)) fs.mkdirSync(uploadsBase, { recursive: true }); cb(null, uploadsBase); },
filename: (_req, file, cb) => cb(null, `${crypto.randomUUID()}${path.extname(file.originalname).toLowerCase() || '.jpg'}`),
}),
limits: { fileSize: 20 * 1024 * 1024 },
fileFilter: (_req: unknown, file: Express.Multer.File, cb: (err: Error | null, accept: boolean) => void) => {
if (!file.mimetype.startsWith('image/') || file.mimetype.includes('svg')) {
const err: Error & { statusCode?: number } = new Error('Only image files are allowed');
err.statusCode = 400;
return cb(err, false);
}
const ext = path.extname(file.originalname).toLowerCase().replace('.', '');
const allowed = allowedTypes.get().split(',').map((e) => e.trim().toLowerCase());
if (!allowed.includes('*') && !allowed.includes(ext)) {
const err: Error & { statusCode?: number } = new Error(`File type .${ext} is not allowed`);
err.statusCode = 400;
return cb(err, false);
}
cb(null, true);
},
export const journeyUploadFilename = (_req: Request, file: Express.Multer.File): string => {
// The poster is ALWAYS stored as .jpg, never the client-supplied extension:
// otherwise a poster declared image/* but named x.html / x.js would land on
// disk with that extension and be served inline same-origin (stored XSS,
// reachable via the public share proxy). The video extension is validated by
// the fileFilter, so it is safe to keep.
const ext =
file.fieldname === 'poster' ? '.jpg'
: file.fieldname === 'video' ? (path.extname(file.originalname).toLowerCase() || '.mp4')
: (path.extname(file.originalname).toLowerCase() || '.jpg');
return `${crypto.randomUUID()}${ext}`;
};
/**
* Journey image upload filter, built from the container.
*
* Same reason as the trip-file factory: it reads the operator's
* allowed-extension list at request time. The module's multer options carry NO
* defParamCharset, unlike the trip-file ones; that difference is deliberate
* and predates the move.
*/
export function journeyImageFileFilter(allowedTypes: AllowedFileTypesService): Options['fileFilter'] {
return (_req, file, cb) => {
if (!file.mimetype.startsWith('image/') || file.mimetype.includes('svg')) {
const err: Error & { statusCode?: number } = new Error('Only image files are allowed');
err.statusCode = 400;
return cb(err);
}
const ext = path.extname(file.originalname).toLowerCase().replace('.', '');
const allowed = allowedTypes.get().split(',').map((e) => e.trim().toLowerCase());
if (!allowed.includes('*') && !allowed.includes(ext)) {
const err: Error & { statusCode?: number } = new Error(`File type .${ext} is not allowed`);
err.statusCode = 400;
return cb(err);
}
cb(null, true);
};
}
// Gallery video upload (#823): one video plus an optional client-captured poster
// image, written to the same uploads/journey store. Larger cap than images since
// phone clips are big; videos are stored as-is and streamed with HTTP Range.
const VIDEO_UPLOAD = {
storage: diskStorage({
destination: (_req, _file, cb) => { if (!fs.existsSync(uploadsBase)) fs.mkdirSync(uploadsBase, { recursive: true }); cb(null, uploadsBase); },
filename: (_req, file, cb) => {
// The poster is ALWAYS stored as .jpg, never the client-supplied extension:
// otherwise a poster declared image/* but named x.html / x.js would land on
// disk with that extension and be served inline same-origin (stored XSS,
// reachable via the public share proxy). The video extension is validated by
// the fileFilter, so it is safe to keep.
const ext = file.fieldname === 'poster' ? '.jpg' : (path.extname(file.originalname).toLowerCase() || '.mp4');
cb(null, `${crypto.randomUUID()}${ext}`);
},
}),
limits: { fileSize: MAX_VIDEO_SIZE },
fileFilter: (_req: unknown, file: Express.Multer.File, cb: (err: Error | null, accept: boolean) => void) => {
const reject = (msg: string) => {
const err: Error & { statusCode?: number } = new Error(msg);
err.statusCode = 400;
cb(err, false);
};
if (file.fieldname === 'poster') {
if (!file.mimetype.startsWith('image/') || file.mimetype.includes('svg')) return reject('Poster must be an image');
return cb(null, true);
}
// 'video' field
const ext = path.extname(file.originalname).toLowerCase().replace('.', '');
if (!isVideoMime(file.mimetype)) return reject('Only video files are allowed');
if (!isVideoExtension(ext)) return reject(`Video type .${ext} is not allowed`);
cb(null, true);
},
// Passed inline on the video route; the storage engine (spool destination +
// journeyUploadFilename) is inherited from the module options via the
// interceptor's shallow merge.
const VIDEO_FILE_FILTER: Options['fileFilter'] = (_req, file, cb) => {
const reject = (msg: string) => {
const err: Error & { statusCode?: number } = new Error(msg);
err.statusCode = 400;
cb(err);
};
if (file.fieldname === 'poster') {
if (!file.mimetype.startsWith('image/') || file.mimetype.includes('svg')) return reject('Poster must be an image');
return cb(null, true);
}
// 'video' field
const ext = path.extname(file.originalname).toLowerCase().replace('.', '');
if (!isVideoMime(file.mimetype)) return reject('Only video files are allowed');
if (!isVideoExtension(ext)) return reject(`Video type .${ext} is not allowed`);
cb(null, true);
};
/**
@@ -125,10 +128,30 @@ const VIDEO_UPLOAD = {
export class JourneyController {
constructor(
private readonly journey: JourneyService,
private readonly storage: StorageService,
private readonly books: JourneyBookService,
private readonly captureBackfill: PhotoCaptureBackfillService,
) {}
/**
* Commit spooled multer files to their final uploads/journey location. On any
* failure: best-effort removal of both spool leftovers and already-committed
* names, then rethrow (500) — so a DB row never references a missing file.
*/
private async commitJourneyUploads(files: Express.Multer.File[]): Promise<void> {
try {
for (const f of files) await this.storage.put('journey', f.filename, { tmpPath: f.path });
} catch (err) {
for (const f of files) {
if (f.path) { try { fs.unlinkSync(f.path); } catch { /* best-effort */ } }
}
for (const f of files) {
await this.storage.delete('journey', f.filename).catch(() => {});
}
throw err;
}
}
// The add call carries only an asset id, so when and where the picture was taken
// are fetched from the provider afterwards rather than trusted from the client
// (#1614). Detached: a slow or unreachable provider must not hold up the add.
@@ -193,6 +216,9 @@ export class JourneyController {
if (!files?.length) {
throw new HttpException({ error: 'No files uploaded' }, 400);
}
// Commit to final storage BEFORE the addPhoto loop: the Immich mirror below
// reads each file at its final uploads/journey path.
await this.commitJourneyUploads(files);
const results: unknown[] = [];
for (const file of files) {
const relativePath = `journey/${file.filename}`;
@@ -274,24 +300,51 @@ export class JourneyController {
}
@Delete('photos/:photoId')
deletePhoto(@CurrentUser() user: User, @Param('photoId') photoId: string) {
async deletePhoto(@CurrentUser() user: User, @Param('photoId') photoId: string) {
const photo = this.journey.deletePhoto(Number(photoId), user.id);
if (!photo) {
throw new HttpException({ error: 'Photo not found' }, 404);
}
if (photo.file_path) {
try { fs.unlinkSync(path.join(__dirname, '../../../uploads', photo.file_path)); } catch { /* file already gone */ }
}
await this.deletePhotoObject(photo.file_path);
await this.reclaimDerived(photo.file_path, photo.thumbnail_path);
return { success: true };
}
/**
* Best-effort removal of a deleted photo's bytes. file_path rows are the
* uploads-relative 'journey/<file>' form every writer stores; anything else
* (provider rows have null) has no local object.
*/
private async deletePhotoObject(filePath: string | null | undefined): Promise<void> {
if (!filePath) return;
const name = filePath.startsWith('journey/') ? filePath.slice('journey/'.length) : null;
if (!name) return;
await this.storage.delete('journey', name).catch(() => { /* file already gone */ });
}
/**
* Delete a photo's derived artifacts with it (spec In-scope fix #2): the
* hash-named thumbnail derivable from file_path, and whatever
* thumbnail_path recorded — a generated thumb (usually the same object) or
* a video's uploaded poster. Both best-effort, both idempotent.
*/
private async reclaimDerived(filePath: string | null | undefined, thumbnailPath: string | null | undefined): Promise<void> {
if (filePath?.startsWith('journey/')) {
await this.storage.delete('journey', journeyThumbName(filePath)).catch(() => { /* already gone */ });
}
if (thumbnailPath?.startsWith('journey/')) {
await this.storage.delete('journey', thumbnailPath.slice('journey/'.length)).catch(() => { /* already gone */ });
}
}
// ── Gallery (prefix /:id/gallery — before /:id) ─────────────────────────
@Post(':id/gallery/photos')
@UseInterceptors(FilesInterceptor('photos'))
uploadGalleryPhotos(@CurrentUser() user: User, @Param('id') id: string, @UploadedFiles() files: Express.Multer.File[] | undefined) {
async uploadGalleryPhotos(@CurrentUser() user: User, @Param('id') id: string, @UploadedFiles() files: Express.Multer.File[] | undefined) {
if (!files?.length) {
throw new HttpException({ error: 'No files uploaded' }, 400);
}
await this.commitJourneyUploads(files);
const filePaths = files.map((f) => ({ path: `journey/${f.filename}` }));
const photos = this.journey.uploadGalleryPhotos(Number(id), user.id, filePaths);
if (!photos.length) {
@@ -304,8 +357,13 @@ export class JourneyController {
}
@Post(':id/gallery/video')
@UseInterceptors(FileFieldsInterceptor([{ name: 'video', maxCount: 1 }, { name: 'poster', maxCount: 1 }], VIDEO_UPLOAD))
uploadGalleryVideo(
@UseInterceptors(FileFieldsInterceptor(
[{ name: 'video', maxCount: 1 }, { name: 'poster', maxCount: 1 }],
// Inherits the module's storage engine (spool + journeyUploadFilename) via
// the interceptor's shallow options merge; overrides only cap and filter.
{ limits: { fileSize: MAX_VIDEO_SIZE }, fileFilter: VIDEO_FILE_FILTER },
))
async uploadGalleryVideo(
@CurrentUser() user: User,
@Param('id') id: string,
@UploadedFiles() files: { video?: Express.Multer.File[]; poster?: Express.Multer.File[] } | undefined,
@@ -313,8 +371,8 @@ export class JourneyController {
) {
const video = files?.video?.[0];
const poster = files?.poster?.[0];
// multer already wrote both parts; clean them up on any rejection so a POST to
// a journey the user can't edit doesn't orphan a 500 MB clip on disk (#823).
// multer already spooled both parts; clean them up on a pre-commit rejection
// so a bad POST doesn't orphan a 500 MB clip (#823).
const cleanup = () => {
for (const f of [video, poster]) {
if (f?.path) { try { fs.unlinkSync(f.path); } catch { /* best-effort */ } }
@@ -324,6 +382,7 @@ export class JourneyController {
cleanup();
throw new HttpException({ error: 'No video uploaded' }, 400);
}
await this.commitJourneyUploads(poster ? [video, poster] : [video]);
const durationMs = body?.duration_ms != null ? Number(body.duration_ms) : null;
const photos = this.journey.uploadGalleryPhotos(Number(id), user.id, [{
path: `journey/${video.filename}`,
@@ -332,7 +391,11 @@ export class JourneyController {
durationMs: durationMs != null && Number.isFinite(durationMs) ? durationMs : null,
}]);
if (!photos.length) {
cleanup();
// The parts are already committed (file.path is consumed), so the
// pre-commit unlink cleanup would silently orphan them — delete the
// final objects instead. Same observable outcome as before: bytes gone.
await this.storage.delete('journey', video.filename).catch(() => {});
if (poster) await this.storage.delete('journey', poster.filename).catch(() => {});
throw new HttpException({ error: 'Not allowed' }, 403);
}
return { photos };
@@ -364,14 +427,13 @@ export class JourneyController {
@Delete(':id/gallery/:journeyPhotoId')
@HttpCode(204)
deleteGalleryPhoto(@CurrentUser() user: User, @Param('journeyPhotoId') journeyPhotoId: string): void {
async deleteGalleryPhoto(@CurrentUser() user: User, @Param('journeyPhotoId') journeyPhotoId: string): Promise<void> {
const photo = this.journey.deleteGalleryPhoto(Number(journeyPhotoId), user.id);
if (!photo) {
throw new HttpException({ error: 'Photo not found or not allowed' }, 404);
}
if (photo.file_path) {
try { fs.unlinkSync(path.join(__dirname, '../../../uploads', photo.file_path)); } catch { /* file already gone */ }
}
await this.deletePhotoObject(photo.file_path);
await this.reclaimDerived(photo.file_path, photo.thumbnail_path);
}
// ── Journeys /:id ───────────────────────────────────────────────────────
@@ -396,10 +458,11 @@ export class JourneyController {
@Post(':id/cover')
@HttpCode(200) // Express answers cover with res.json (200).
@UseInterceptors(FileInterceptor('cover'))
cover(@CurrentUser() user: User, @Param('id') id: string, @UploadedFile() file: Express.Multer.File | undefined) {
async cover(@CurrentUser() user: User, @Param('id') id: string, @UploadedFile() file: Express.Multer.File | undefined) {
if (!file) {
throw new HttpException({ error: 'No file uploaded' }, 400);
}
await this.commitJourneyUploads([file]);
const result = this.journey.updateJourney(Number(id), user.id, { cover_image: `journey/${file.filename}` });
if (!result) {
throw new HttpException({ error: 'Journey not found' }, 404);
+16 -4
View File
@@ -11,16 +11,28 @@ import { AuthModule } from '../auth/auth.module';
import { MulterModule } from '@nestjs/platform-express';
import { AllowedFileTypesModule } from '../files/allowed-file-types.module';
import { AllowedFileTypesService } from '../files/allowed-file-types.service';
import { buildJourneyImageUploadOptions } from './journey.controller';
import { StorageModule } from '../storage/storage.module';
import { StorageService } from '../storage/storage.service';
import { buildStorageUploadOptions } from '../storage/storage-upload.factory';
import { journeyImageFileFilter, journeyUploadFilename } from './journey.controller';
@Module({
// MemoriesModule: the journey gallery streams provider assets and uploads to Immich.
imports: [
MulterModule.registerAsync({
imports: [AllowedFileTypesModule],
inject: [AllowedFileTypesService],
useFactory: (allowedTypes: AllowedFileTypesService) => buildJourneyImageUploadOptions(allowedTypes),
imports: [StorageModule, AllowedFileTypesModule],
inject: [StorageService, AllowedFileTypesService],
// NO defParamCharset here — deliberate, documented asymmetry with the
// trip-file options (see journey.controller.ts).
useFactory: (storage: StorageService, allowedTypes: AllowedFileTypesService) =>
buildStorageUploadOptions(storage, {
category: 'journey',
maxSize: 20 * 1024 * 1024,
fileFilter: journeyImageFileFilter(allowedTypes),
filename: journeyUploadFilename,
}),
}),
StorageModule,
AuthModule, AddonsModule, MemoriesModule, JourneyDomainModule],
controllers: [JourneyController, JourneyPublicController],
providers: [JourneyService, JourneyBookService, JourneyMcp],
+38 -15
View File
@@ -11,7 +11,8 @@ import {
UseGuards,
} from '@nestjs/common';
import type { Response } from 'express';
import { createReadStream } from 'node:fs';
import type { Readable } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import type {
MapsAutocompleteResult,
MapsPlaceDetailsResult,
@@ -22,6 +23,8 @@ import type {
} from '@trek/shared';
import type { User } from '../../types';
import { MapsService } from './maps.service';
import { StorageService } from '../storage/storage.service';
import { isClientAbortError } from '../storage/storage.types';
import { JwtAuthGuard } from '../auth/jwt-auth.guard';
import { CurrentUser } from '../auth/current-user.decorator';
import { MapsSearchDto, MapsAutocompleteDto, MapsResolveUrlDto } from './maps.dto';
@@ -60,7 +63,10 @@ function toHttpException(err: unknown, fallbackMessage: string, defaultStatus: n
@Controller('api/maps')
@UseGuards(JwtAuthGuard)
export class MapsController {
constructor(private readonly maps: MapsService) {}
constructor(
private readonly maps: MapsService,
private readonly storage: StorageService,
) {}
@Post('search')
@HttpCode(200) // Express answers with res.json (200); Nest would otherwise default POST to 201.
@@ -166,9 +172,9 @@ export class MapsController {
}
@Get('place-photo/:placeId/bytes')
placePhotoBytes(@Param('placeId') placeId: string, @Res() res: Response): void {
const fp = this.maps.photoBytesPath(placeId);
if (!fp) {
async placePhotoBytes(@Param('placeId') placeId: string, @Res() res: Response): Promise<void> {
const key = await this.maps.photoBytesKey(placeId);
if (!key) {
// Same reasoning as the JSON endpoint above, and the bigger half of #1727:
// places keep this URL in image_url, so a trip render asks for one photo per
// place and every entry the cache sweep dropped answered 404 — a whole map
@@ -177,19 +183,36 @@ export class MapsController {
this.emptyPhoto(res);
return;
}
// Stream the cached file directly instead of res.sendFile(): the send library
// bundled under @nestjs/platform-express rejects absolute Windows paths (drive
// letter, no `root`) with a NotFoundError that surfaced as an unhandled 500,
// even though the file exists. A plain read stream serves the bytes
// cross-platform. Cached photos are always JPEG (placePhotoCache writes
// `<hash>.jpg`).
// Bytes are stream-piped (spec §Serving pins this route's stream contract);
// headers go on before the stream attempt, matching the old createReadStream
// ordering — an early failure overrides them via emptyPhoto exactly as the
// old error event did. Cached photos are always JPEG (placePhotoCache
// writes `<hash>.jpg`).
res.set('Cache-Control', 'public, max-age=2592000, immutable');
res.type('image/jpeg');
const stream = createReadStream(fp);
stream.on('error', () => {
let stream: Readable;
try {
({ stream } = await this.storage.getStream('photos-google', key));
} catch {
// Cache-delete race — same terminal state as the old stream error path.
if (!res.headersSent) this.emptyPhoto(res);
});
stream.pipe(res);
return;
}
try {
// pipeline destroys the source on any failure (no leaked read handle),
// matching storage.service.ts's sendToResponse contract.
await pipeline(stream, res);
} catch (err) {
if (!res.headersSent) {
// Same terminal state as the old pre-flush stream error path.
this.emptyPhoto(res);
return;
}
// Bytes are already on the wire — a client abort mid-download is the
// client's problem now, not ours; a real source error still surfaces
// (to the exception filter, which now no-ops safely on headersSent).
if (!isClientAbortError(err)) throw err;
}
}
// 204 for "no bytes to serve". Overrides the immutable Cache-Control the hit
+2 -1
View File
@@ -3,6 +3,7 @@ import { MapsController } from './maps.controller';
import { MapsService } from './maps.service';
import { MapsMcp } from './maps.mcp';
import { PlacePhotosModule } from '../place-photos/place-photos.module';
import { StorageModule } from '../storage/storage.module';
/**
* Maps / geo domain (L3 leaf module). Registered in AppModule. Exports
@@ -11,7 +12,7 @@ import { PlacePhotosModule } from '../place-photos/place-photos.module';
* Nothing outside the container consumes this domain, so there is no bridge.
*/
@Module({
imports: [PlacePhotosModule],
imports: [PlacePhotosModule, StorageModule],
controllers: [MapsController],
providers: [MapsService, MapsMcp],
exports: [MapsService],
+8 -8
View File
@@ -542,8 +542,8 @@ export class MapsService {
return this.getPlacePhoto(userId, placeId, lat, lng, name) as Promise<MapsPlacePhotoResult>;
}
photoBytesPath(placeId: string): string | null {
return this.photoCache.serveFilePath(placeId);
photoBytesKey(placeId: string): Promise<string | null> {
return this.photoCache.serveKey(placeId);
}
reverse(lat: string, lng: string, lang?: string): Promise<MapsReverseResult> {
@@ -1816,7 +1816,7 @@ export class MapsService {
name?: string,
): Promise<{ photoUrl: string | null; attribution: string | null }> {
// Disk cache hit — serve immediately, no Google call
const diskHit = this.photoCache.get(placeId);
const diskHit = await this.photoCache.get(placeId);
if (diskHit) return { photoUrl: diskHit.photoUrl, attribution: diskHit.attribution };
// "No photo for this place" is an empty result, not a missing resource: a trip
@@ -1842,7 +1842,7 @@ export class MapsService {
// or timed out only for a few minutes.
let providerFailed = false;
const fetchPromise = (async (): Promise<{ filePath: string; attribution: string | null } | null> => {
const fetchPromise = (async (): Promise<{ attribution: string | null } | null> => {
await acquirePhotoFetchSlot();
try {
const apiKey = this.getMapsKey(userId);
@@ -1851,7 +1851,7 @@ export class MapsService {
// (right-click) places and as a fallback when a Google place yields no photo,
// so a place added via search still gets a marker image when Google returns
// nothing. Returns null (without marking an error) so the caller decides.
const fetchWikimediaFallback = async (): Promise<{ filePath: string; attribution: string | null } | null> => {
const fetchWikimediaFallback = async (): Promise<{ attribution: string | null } | null> => {
if (isNaN(lat) || isNaN(lng)) return null;
try {
const wiki = await this.fetchWikimediaPhoto(lat, lng, name);
@@ -1865,7 +1865,7 @@ export class MapsService {
}
const bytes = Buffer.from(await imgRes.arrayBuffer());
const cached = await this.photoCache.put(placeId, bytes, wiki.attribution);
return { filePath: cached.filePath, attribution: cached.attribution };
return { attribution: cached.attribution };
} catch {
providerFailed = true;
return null;
@@ -1876,7 +1876,7 @@ export class MapsService {
// key, request rejected, no photos, or a failed media download — so the
// caller can fall back to Wikimedia; the misses that were Google's fault
// flag providerFailed on the way out.
const fetchGooglePhoto = async (): Promise<{ filePath: string; attribution: string | null } | null> => {
const fetchGooglePhoto = async (): Promise<{ attribution: string | null } | null> => {
if (!apiKey) return null;
// Fetch details to get the photo name
@@ -1940,7 +1940,7 @@ export class MapsService {
console.error('Failed to persist photo URL to database:', dbErr);
}
return { filePath: cached.filePath, attribution };
return { attribution };
};
// Prefer the Google photo (higher quality); if Google yields nothing, fall
+11 -7
View File
@@ -3,10 +3,12 @@ import type { Response } from 'express';
import { maybe_encrypt_api_key, decrypt_api_key } from '../common/crypto/apiKeyCrypto';
import { checkSsrf, safeFetch } from '../../utils/ssrfGuard';
import { AuditService } from '../audit/audit.service';
import { StorageService } from '../storage/storage.service';
import fsPromises from 'node:fs/promises';
import path from 'node:path';
import { DatabaseService } from '../database/database.service';
import { MemoriesAccessService } from './memories-access.service';
import { fail, handleServiceResult, pipeAsset, type Selection } from './memories.helpers';
import { UPLOADS_ROOT } from './uploads-root';
const ALBUM_PAGE_SIZE = 1000;
const ALBUM_MAX_PAGES = 20;
@@ -24,6 +26,7 @@ export class ImmichService {
private readonly db: DatabaseService,
private readonly audit: AuditService,
private readonly access: MemoriesAccessService,
private readonly storage: StorageService,
) {}
getImmichCredentials(userId: number) {
@@ -575,14 +578,15 @@ export class ImmichService {
const creds = this.getImmichCredentials(userId);
if (!creds) return null;
const fs = await import('node:fs');
const path = await import('node:path');
const fullPath = path.join(UPLOADS_ROOT, filePath);
if (!fs.existsSync(fullPath)) return null;
// Journey uploads store the uploads-relative 'journey/<file>' path; only
// those are local storage objects this can push.
const name = filePath.startsWith('journey/') ? filePath.slice('journey/'.length) : null;
if (!name) return null;
try {
const fileBuffer = fs.readFileSync(fullPath);
// A missing object throws StorageNotFoundError into the catch below —
// the same null the old existsSync guard answered.
const fileBuffer = await this.storage.withLocalFile('journey', name, (p) => fsPromises.readFile(p));
const boundary = '----ImmichUpload' + Date.now();
const ext = path.extname(fileName).toLowerCase();
const mimeTypes: Record<string, string> = {
@@ -0,0 +1,37 @@
import { Injectable, type OnApplicationBootstrap } from '@nestjs/common';
import { logInfo, logError } from '../audit/audit-log.logger';
import { CronRegistrarService } from '../scheduling/cron-registrar.service';
import { ThumbnailService } from './thumbnail.service';
/**
* Journey thumbnail cleanup: daily — reclaim journey/thumbs/ objects whose
* photo row is gone (spec In-scope fix #2: delete sites only started removing
* the derived thumb with this slice, so historical orphans need a sweeper).
* The Journey addon gate lives inside sweepOrphanThumbs, so toggling the
* addon takes effect without a restart.
*/
@Injectable()
export class JourneyThumbsJob implements OnApplicationBootstrap {
constructor(
private readonly thumbnails: ThumbnailService,
private readonly registrar: CronRegistrarService,
) {}
onApplicationBootstrap(): void {
if (!this.registrar.isEnabled()) return;
// Run once on startup to reclaim orphans left over from before this sweeper existed.
void this.sweep();
this.registrar.register('journey-thumbs', '0 4 * * *', () => {
void this.sweep();
});
}
async sweep(): Promise<void> {
try {
const removed = await this.thumbnails.sweepOrphanThumbs();
if (removed > 0) logInfo(`Journey thumbnail cleanup: removed ${removed} orphaned thumbnail(s)`);
} catch (err: unknown) {
logError(`Journey thumbnail cleanup: ${err instanceof Error ? err.message : err}`);
}
}
}
+4 -1
View File
@@ -9,6 +9,7 @@ import { PhotoCaptureBackfillService } from './photo-capture-backfill.service';
import { ThumbnailService } from './thumbnail.service';
import { TrekPhotoCacheService } from './trek-photo-cache.service';
import { TrekPhotoCacheJob } from './trek-photo-cache.job';
import { JourneyThumbsJob } from './journey-thumbs.job';
import { SchedulingModule } from '../scheduling/scheduling.module';
import { UnifiedMemoriesController } from './unified.controller';
import { ImmichMemoriesController } from './immich.controller';
@@ -22,6 +23,7 @@ import { PhotoProviderRegistry } from './photo-provider.registry';
import { ImmichPhotoProvider } from './providers/immich.provider';
import { SynologyPhotoProvider } from './providers/synology.provider';
import { NotificationsModule } from '../notifications/notifications.module';
import { StorageModule } from '../storage/storage.module';
/**
* Memories (photo-providers) domain — mounted at /api/integrations/memories.
@@ -44,7 +46,7 @@ import { NotificationsModule } from '../notifications/notifications.module';
* adapters themselves so the set is readable in one place.
*/
@Module({
imports: [NotificationsModule, AddonsModule, AuditModule, TrekPhotosModule, RealtimeModule, SchedulingModule],
imports: [NotificationsModule, AddonsModule, AuditModule, TrekPhotosModule, RealtimeModule, SchedulingModule, StorageModule],
controllers: [UnifiedMemoriesController, ImmichMemoriesController, SynologyMemoriesController],
providers: [
MemoriesService,
@@ -57,6 +59,7 @@ import { NotificationsModule } from '../notifications/notifications.module';
ThumbnailService,
TrekPhotoCacheService,
TrekPhotoCacheJob,
JourneyThumbsJob,
ImmichPhotoProvider,
SynologyPhotoProvider,
PhotoProviderRegistry,
@@ -1,8 +1,7 @@
import { Injectable } from '@nestjs/common';
import path from 'node:path';
import exifr from 'exifr';
import { UPLOADS_ROOT } from './uploads-root';
import { PhotoResolverService } from './photo-resolver.service';
import { StorageService } from '../storage/storage.service';
import { TrekPhotosRepository } from '../photos/trek-photos.repository';
/**
@@ -23,6 +22,7 @@ export class PhotoCaptureBackfillService {
constructor(
private readonly resolver: PhotoResolverService,
private readonly photos: TrekPhotosRepository,
private readonly storage: StorageService,
) {}
/** Fire-and-forget for a batch that was just added. */
@@ -74,18 +74,24 @@ export class PhotoCaptureBackfillService {
filePath?: string | null,
): Promise<{ takenAt: string | null; lat: number | null; lng: number | null } | null> {
if (!filePath) return null;
// Never let a stored path climb out of the uploads tree.
const abs = path.resolve(UPLOADS_ROOT, filePath);
if (abs !== UPLOADS_ROOT && !abs.startsWith(UPLOADS_ROOT + path.sep)) return null;
// photos.file_path is uploads-relative 'journey/<name>' by every writer;
// anything else reads as a miss (same rule as photo-resolver). Central key
// validation (storage-keys.ts) rejects a name that still carries a path,
// so a stored path can never climb out of the journey category.
if (!filePath.startsWith('journey/')) return null;
const name = filePath.slice('journey/'.length);
type Exif = { DateTimeOriginal?: Date; CreateDate?: Date; latitude?: number; longitude?: number };
let parsed: Exif | null;
try {
parsed = (await exifr.parse(abs, {
pick: ['DateTimeOriginal', 'CreateDate', 'latitude', 'longitude'],
})) as Exif | null;
parsed = await this.storage.withLocalFile('journey', name, async abs =>
(await exifr.parse(abs, {
pick: ['DateTimeOriginal', 'CreateDate', 'latitude', 'longitude'],
})) as Exif | null,
);
} catch {
// Not an image, a truncated upload, a video — none of it is an error here.
// A vanished object, an invalid key, not an image, a truncated upload,
// a video — none of it is an error here.
return null;
}
if (!parsed) return null;
@@ -1,16 +1,14 @@
import { Injectable } from '@nestjs/common';
import { Response } from 'express';
import path from 'path';
import fs from 'fs';
import type { TrekPhoto } from '../../types';
import { decrypt_api_key } from '../common/crypto/apiKeyCrypto';
import { TrekPhotosRepository } from '../photos/trek-photos.repository';
import { UPLOADS_ROOT } from './uploads-root';
import { ThumbnailService } from './thumbnail.service';
import { TrekPhotoCacheService } from './trek-photo-cache.service';
import { fail, success, type AssetInfo, type ServiceResult } from './memories.helpers';
import { PhotoProviderRegistry } from './photo-provider.registry';
import type { PhotoAssetRef } from './photo-provider';
import { StorageService } from '../storage/storage.service';
/**
* Resolves a stored trek_photo to bytes or metadata by asking whichever provider
@@ -29,8 +27,16 @@ export class PhotoResolverService {
private readonly thumbnails: ThumbnailService,
private readonly cache: TrekPhotoCacheService,
private readonly providers: PhotoProviderRegistry,
private readonly storage: StorageService,
) {}
/** photos.file_path / thumbnail_path are uploads-relative 'journey/…' by
* every writer; anything else reads as a local miss, matching the old
* existsSync(join(uploadsRoot, file_path)) === false path. */
private storageName(relPath: string): string | null {
return relPath.startsWith('journey/') ? relPath.slice('journey/'.length) : null;
}
// ── Streaming ────────────────────────────────────────────────────────────
private async streamCachedThumbnail(
@@ -41,12 +47,12 @@ export class PhotoResolverService {
): Promise<void> {
const key = this.cache.cacheKey(photo.provider!, photo.asset_id!, 'thumbnail', photo.owner_id!);
if (this.cache.serveFresh(res, key)) return;
if (await this.cache.serveFresh(res, key)) return;
const existing = this.cache.getInFlight(key);
if (existing) {
const bytes = await existing;
if (bytes && this.cache.serveFresh(res, key)) return;
if (bytes && (await this.cache.serveFresh(res, key))) return;
await fallback();
return;
}
@@ -59,7 +65,7 @@ export class PhotoResolverService {
this.cache.setInFlight(key, promise);
const bytes = await promise;
if (bytes && this.cache.serveFresh(res, key)) return;
if (bytes && (await this.cache.serveFresh(res, key))) return;
await fallback();
}
@@ -77,31 +83,24 @@ export class PhotoResolverService {
}
if (photo.file_path) {
const uploadsRoot = UPLOADS_ROOT;
if (kind === 'thumbnail') {
const isVideo = photo.media_type === 'video';
let thumbRel = photo.thumbnail_path ?? null;
// Only raster images get a lazily-generated Jimp thumbnail; Jimp can't decode
// video, so a video relies on the poster captured at upload (#823).
if (!thumbRel && !isVideo) {
const result = await this.thumbnails.ensureLocalThumbnail(uploadsRoot, photo.file_path);
const result = await this.thumbnails.ensureLocalThumbnail(photo.file_path);
if (result) {
thumbRel = result.thumbnailRelPath;
this.photos.recordLocalThumbnail(photo.id, thumbRel, result.width, result.height);
}
}
if (thumbRel) {
const thumbAbs = path.join(uploadsRoot, thumbRel);
if (fs.existsSync(thumbAbs)) {
const thumbName = this.storageName(thumbRel);
if (thumbName && (await this.storage.exists('journey', thumbName).catch(() => false))) {
res.set('Cache-Control', 'public, max-age=86400, immutable');
res.set('X-Content-Type-Options', 'nosniff');
// Explicit { root } + basename, not the absolute path: under the Nest
// ExpressAdapter, res.sendFile(absolutePath) resolves against the
// rewritten req.url and 404s spuriously even though the file is right
// there (same trap as files-download.controller.ts). The existsSync
// above already pins this inside the uploads tree.
res.sendFile(path.basename(thumbAbs), { root: path.dirname(thumbAbs) });
await this.storage.sendToResponse('journey', thumbName, res);
return;
}
}
@@ -114,11 +113,11 @@ export class PhotoResolverService {
// Images fall through to original if the thumbnail is unavailable.
}
const localPath = path.join(uploadsRoot, photo.file_path);
if (fs.existsSync(localPath)) {
const name = this.storageName(photo.file_path);
if (name && (await this.storage.exists('journey', name).catch(() => false))) {
res.set('Cache-Control', 'public, max-age=86400');
res.set('X-Content-Type-Options', 'nosniff');
res.sendFile(path.basename(localPath), { root: path.dirname(localPath) });
await this.storage.sendToResponse('journey', name, res);
return;
}
}
+73 -18
View File
@@ -1,55 +1,75 @@
import { Jimp } from 'jimp'
import path from 'path'
import fs from 'fs/promises'
import crypto from 'crypto'
import path from 'node:path'
import crypto from 'node:crypto'
import { Injectable } from '@nestjs/common'
import { ADDON_IDS } from '../../addons'
import { AddonsService } from '../addons/addons.service'
import { DatabaseService } from '../database/database.service'
import { StorageService } from '../storage/storage.service'
const THUMB_MAX = 800
const THUMB_QUALITY = 80
/**
* Storage name (category 'journey') of the derived thumbnail for an
* uploads-relative journey photo path ('journey/<file>'). The hash input is
* deliberately the uploads-relative path — the scheme predates the storage
* layer, and changing it would orphan every existing thumbnail.
*/
export function journeyThumbName(originalRelPath: string): string {
const hash = crypto.createHash('sha1').update(originalRelPath).digest('hex').slice(0, 16)
return `thumbs/${hash}.jpg`
}
/**
* Downscaled JPEGs for locally uploaded journey photos. Gated on the journey
* addon, because that is the only surface that uploads them.
* addon, because that is the only surface that uploads them. Category-addressed
* since storage slice 4: originals and thumbs are ('journey', <name>) objects.
*/
@Injectable()
export class ThumbnailService {
constructor(private readonly addons: AddonsService) {}
constructor(
private readonly addons: AddonsService,
private readonly storage: StorageService,
private readonly db: DatabaseService,
) {}
async ensureLocalThumbnail(
uploadsRoot: string,
originalRelPath: string,
): Promise<{ thumbnailRelPath: string; width: number; height: number } | null> {
if (!this.addons.isAddonEnabled(ADDON_IDS.JOURNEY)) return null
const originalAbs = path.join(uploadsRoot, originalRelPath)
try { await fs.access(originalAbs) } catch { return null }
// The DB stores uploads-relative paths ('journey/<file>'); anything else
// is not a local journey photo and has no thumbnail to derive.
const origName = originalRelPath.startsWith('journey/') ? originalRelPath.slice('journey/'.length) : null
if (!origName) return null
// Deterministic name so concurrent requests don't race on the same photo.
const hash = crypto.createHash('sha1').update(originalRelPath).digest('hex').slice(0, 16)
const thumbRel = `journey/thumbs/${hash}.jpg`
const thumbAbs = path.join(uploadsRoot, thumbRel)
const thumbName = journeyThumbName(originalRelPath)
const thumbRel = `journey/${thumbName}`
try {
const [srcStat, dstStat] = await Promise.all([
fs.stat(originalAbs),
fs.stat(thumbAbs).catch(() => null),
this.storage.stat('journey', origName),
this.storage.stat('journey', thumbName),
])
if (!srcStat) return null
if (dstStat && dstStat.mtimeMs >= srcStat.mtimeMs) {
const img = await Jimp.read(thumbAbs)
const img = await this.storage.withLocalFile('journey', thumbName, (p) => Jimp.read(p))
return { thumbnailRelPath: thumbRel, width: img.bitmap.width, height: img.bitmap.height }
}
await fs.mkdir(path.dirname(thumbAbs), { recursive: true })
// Jimp auto-applies EXIF orientation on read, matching sharp's .rotate() behavior.
const img = await Jimp.read(originalAbs)
const img = await this.storage.withLocalFile('journey', origName, (p) => Jimp.read(p))
const { width: w, height: h } = img.bitmap
if (w > THUMB_MAX || h > THUMB_MAX) {
img.scaleToFit({ w: THUMB_MAX, h: THUMB_MAX })
}
await img.write(thumbAbs as `${string}.jpg`, { quality: THUMB_QUALITY })
// Jimp writes straight to a path, so spool on the category's volume and
// commit through storage for the atomic rename.
const spool = path.join(this.storage.spoolDirFor('journey'), `thumb-${crypto.randomUUID()}.jpg`)
await img.write(spool as `${string}.jpg`, { quality: THUMB_QUALITY })
await this.storage.put('journey', thumbName, { tmpPath: spool })
return { thumbnailRelPath: thumbRel, width: img.bitmap.width, height: img.bitmap.height }
} catch {
@@ -57,4 +77,39 @@ export class ThumbnailService {
return null
}
}
/**
* Reclaim journey/thumbs/ objects no live trek_photos row can account for
* (spec In-scope fix #2). The live set is every hash derivable from a
* journey file_path plus every recorded thumbnail_path under thumbs/ —
* video posters live in journey/ proper and are reclaimed at the delete
* sites, not here. Gated on the Journey addon like the generator, inside
* the method so an admin toggle takes effect without a restart
* (airtrail-sync.job.ts precedent).
*/
async sweepOrphanThumbs(): Promise<number> {
if (!this.addons.isAddonEnabled(ADDON_IDS.JOURNEY)) return 0
const live = new Set<string>()
for (const row of this.db.all<{ file_path: string }>(
"SELECT file_path FROM trek_photos WHERE file_path LIKE 'journey/%'",
)) {
live.add(journeyThumbName(row.file_path))
}
for (const row of this.db.all<{ thumbnail_path: string }>(
"SELECT thumbnail_path FROM trek_photos WHERE thumbnail_path LIKE 'journey/thumbs/%'",
)) {
live.add(row.thumbnail_path.slice('journey/'.length))
}
let removed = 0
for await (const stat of this.storage.list('journey', 'thumbs/')) {
if (!stat.key.endsWith('.jpg') || live.has(stat.key)) continue
try {
await this.storage.delete('journey', stat.key)
removed++
} catch { /* race */ }
}
return removed
}
}
@@ -19,15 +19,19 @@ export class TrekPhotoCacheJob implements OnApplicationBootstrap {
onApplicationBootstrap(): void {
if (!this.registrar.isEnabled()) return;
// Run once immediately on startup to evict any entries left over from a previous run
try {
this.cache.sweepExpired();
} catch { /* cache dir may not exist yet — harmless */ }
this.registrar.register('trek-photo-cache', '0 */2 * * *', () => this.tick(), { timezone: 'none' });
void (async () => {
try {
await this.cache.sweepExpired();
} catch { /* cache dir may not exist yet — harmless */ }
})();
this.registrar.register('trek-photo-cache', '0 */2 * * *', () => {
void this.tick();
}, { timezone: 'none' });
}
tick(): void {
async tick(): Promise<void> {
try {
this.cache.sweepExpired();
await this.cache.sweepExpired();
} catch (err: unknown) {
logError(`Trek photo cache cleanup: ${err instanceof Error ? err.message : err}`);
}
@@ -1,13 +1,11 @@
import path from 'node:path';
import fs from 'node:fs';
import fsPromises from 'node:fs/promises';
import crypto from 'node:crypto';
import { Readable } from 'node:stream';
import { Injectable } from '@nestjs/common';
import { Response } from 'express';
import { DatabaseService } from '../database/database.service';
import { UPLOADS_ROOT } from './uploads-root';
import { StorageService } from '../storage/storage.service';
import { StorageNotFoundError } from '../storage/storage.types';
const TREK_PHOTO_DIR = path.join(UPLOADS_ROOT, 'photos/trek');
export const CACHE_TTL = 60 * 60 * 1000; // 1 hour
/**
@@ -23,28 +21,24 @@ export const CACHE_TTL = 60 * 60 * 1000; // 1 hour
*/
const inFlight = new Map<string, Promise<Buffer | null>>();
function ensureDir(): void {
if (!fs.existsSync(TREK_PHOTO_DIR)) {
fs.mkdirSync(TREK_PHOTO_DIR, { recursive: true });
}
/** Storage name (category 'photos-trek') for a cache key. */
function objectName(key: string): string {
return `${key}.bin`;
}
function cachedFilePath(key: string): string {
return path.join(TREK_PHOTO_DIR, `${key}.bin`);
}
/** Disk + metadata cache for provider thumbnails and originals. */
/** Storage + metadata cache for provider thumbnails and originals. */
@Injectable()
export class TrekPhotoCacheService {
constructor(private readonly db: DatabaseService) {}
constructor(
private readonly db: DatabaseService,
private readonly storage: StorageService,
) {}
cacheKey(provider: string, assetId: string, kind: string, ownerId: number): string {
return crypto.createHash('sha1').update(`${provider}:${assetId}:${kind}:${ownerId}`).digest('hex');
}
getFresh(key: string): { filePath: string; contentType: string } | null {
async getFresh(key: string): Promise<{ contentType: string } | null> {
const row = this.db.get<{ content_type: string; fetched_at: number }>(
'SELECT content_type, fetched_at FROM trek_photo_cache_meta WHERE cache_key = ?', key,
);
@@ -56,22 +50,16 @@ export class TrekPhotoCacheService {
return null;
}
const fp = cachedFilePath(key);
if (!fs.existsSync(fp)) {
if (!(await this.storage.exists('photos-trek', objectName(key)))) {
this.db.run('DELETE FROM trek_photo_cache_meta WHERE cache_key = ?', key);
return null;
}
return { filePath: fp, contentType: row.content_type };
return { contentType: row.content_type };
}
async put(key: string, bytes: Buffer, contentType: string): Promise<void> {
ensureDir();
const fp = cachedFilePath(key);
const tmp = fp + '.tmp';
await fsPromises.writeFile(tmp, bytes);
await fsPromises.rename(tmp, fp);
await this.storage.put('photos-trek', objectName(key), Readable.from(bytes));
this.db.run(
'INSERT OR REPLACE INTO trek_photo_cache_meta (cache_key, content_type, fetched_at) VALUES (?, ?, ?)',
@@ -79,15 +67,21 @@ export class TrekPhotoCacheService {
);
}
serveFresh(res: Response, key: string): boolean {
const entry = this.getFresh(key);
async serveFresh(res: Response, key: string): Promise<boolean> {
const entry = await this.getFresh(key);
if (!entry) return false;
res.set('Content-Type', entry.contentType);
res.set('Cache-Control', 'public, max-age=3600');
// Root-relative, see photo-resolver.service.ts — an absolute path 404s under
// the Nest ExpressAdapter.
res.sendFile(path.basename(entry.filePath), { root: path.dirname(entry.filePath) });
try {
// send() keeps a pre-set Content-Type, so entry.contentType survives the
// .bin extension.
await this.storage.sendToResponse('photos-trek', objectName(key), res);
} catch (err) {
// getFresh→send delete race: fall back like a cache miss.
if (err instanceof StorageNotFoundError && !res.headersSent) return false;
throw err;
}
return true;
}
@@ -100,7 +94,7 @@ export class TrekPhotoCacheService {
promise.finally(() => inFlight.delete(key));
}
sweepExpired(): void {
async sweepExpired(): Promise<void> {
const cutoff = Date.now() - CACHE_TTL * 2;
const stale = this.db.all<{ cache_key: string }>(
'SELECT cache_key FROM trek_photo_cache_meta WHERE fetched_at < ?', cutoff,
@@ -108,8 +102,21 @@ export class TrekPhotoCacheService {
for (const row of stale) {
this.db.run('DELETE FROM trek_photo_cache_meta WHERE cache_key = ?', row.cache_key);
const fp = cachedFilePath(row.cache_key);
if (fs.existsSync(fp)) fs.unlinkSync(fp);
await this.storage.delete('photos-trek', objectName(row.cache_key));
}
// Pass 2 (fix #4, spec rev 3.2): getFresh's expiry path deletes the meta
// row but never the object — reclaim row-less .bin objects past the same
// cutoff. The mtime guard spares an in-flight put (object lands before its
// row); nested keys are skipped because the old readdir-free sweep could
// never touch them either.
for await (const stat of this.storage.list('photos-trek')) {
if (stat.key.includes('/') || !stat.key.endsWith('.bin') || stat.mtimeMs >= cutoff) continue;
const key = stat.key.slice(0, -'.bin'.length);
const row = this.db.get('SELECT 1 FROM trek_photo_cache_meta WHERE cache_key = ?', key);
if (!row) {
await this.storage.delete('photos-trek', stat.key).catch(() => { /* race */ });
}
}
}
}
-14
View File
@@ -1,14 +0,0 @@
import path from 'node:path';
/**
* `<server>/uploads`, the one anchor for everything the app stores on disk.
*
* `__dirname` moves with the file, and three separate copies of this join used
* to live in services/memories/ — every one of them a silent breakage waiting
* for the next move (multer would have created the wrong directory without a
* word). One constant, one test (UPLOADS-ROOT-001), and the depth is stated
* rather than counted by hand: `server/{src,dist}/nest/memories` sits three
* levels under `server/`, the same depth `services/memories/` sat at — so the
* resolved path is unchanged by the move.
*/
export const UPLOADS_ROOT = path.resolve(__dirname, '..', '..', '..', 'uploads');
@@ -39,6 +39,7 @@ export type NotifEventType =
| 'collab_message'
| 'packing_tagged'
| 'version_available'
| 'replica_failure'
| 'synology_session_cleared'
| 'plugin_notification';
@@ -55,12 +56,13 @@ export const ALL_EVENT_TYPES: NotifEventType[] = [
'collab_message',
'packing_tagged',
'version_available',
'replica_failure',
'synology_session_cleared',
'plugin_notification',
];
/** Events that target admins only (shown in admin panel, not in user settings). */
export const ADMIN_SCOPED_EVENTS = new Set<NotifEventType>(['version_available']);
export const ADMIN_SCOPED_EVENTS = new Set<NotifEventType>(['version_available', 'replica_failure']);
// Compile-time guard: shared NotificationEventKey and server NotifEventType must
// stay in sync. It sits next to the union it guards — it used to live in the
@@ -4,23 +4,36 @@ import { NotificationsMcp } from './notifications.mcp';
import { NotificationsService } from './notifications.service';
import { NotificationPreferencesService } from './notification-preferences.service';
import { ReminderJobsService } from './reminder-jobs.service';
import { StorageHealthNotifierService } from './storage-health-notifier.service';
import { NtfyService } from './transports/ntfy.service';
import { WebhookService } from './transports/webhook.service';
import { MailerModule } from './mailer/mailer.module';
import { AuthModule } from '../auth/auth.module';
import { SchedulingModule } from '../scheduling/scheduling.module';
import { StorageModule } from '../storage/storage.module';
/** Notifications domain (L6 leaf module). Registered in AppModule.
* AuthModule feeds NotificationsMcp's demo gate; MailerModule carries SMTP,
* which lives outside this module so AuthService can send the password-reset
* mail without AuthModule and NotificationsModule importing each other.
* StorageModule feeds StorageHealthNotifierService, which bridges replica
* failures into admin notifications — this direction has no cycle (Storage
* imports only AppConfig+Audit+Scheduling).
* NotificationsService and NotificationPreferencesService are exported for
* in-container consumers (AdminController's dev test send and preferences tab,
* the plugin RPC surface, HostSurfaceRpc). */
@Module({
imports: [AuthModule, MailerModule, SchedulingModule],
imports: [AuthModule, MailerModule, SchedulingModule, StorageModule],
controllers: [NotificationsController, AdminNotificationPreferencesController],
providers: [NotificationsService, NotificationPreferencesService, WebhookService, NtfyService, NotificationsMcp, ReminderJobsService],
providers: [
NotificationsService,
NotificationPreferencesService,
WebhookService,
NtfyService,
NotificationsMcp,
ReminderJobsService,
StorageHealthNotifierService,
],
exports: [NotificationsService, NotificationPreferencesService],
})
export class NotificationsModule {}
@@ -96,7 +96,7 @@ type RespondResult = { success: boolean; error?: string; notification?: Notifica
interface EventNotifConfig {
inAppType: 'simple' | 'navigate';
titleKey: string;
textKey: string;
textKey: string | ((params: Record<string, string>) => string);
navigateTextKey?: string;
navigateTarget: (params: Record<string, string>) => string | null;
}
@@ -210,6 +210,13 @@ const EVENT_NOTIFICATION_CONFIG: Record<string, EventNotifConfig> = {
navigateTextKey: 'notif.action.view_admin',
navigateTarget: () => '/admin',
},
replica_failure: {
inAppType: 'navigate',
titleKey: 'notif.replica_failure.title',
textKey: (p) => (p.suppressed && p.suppressed !== '0' ? 'notif.replica_failure.textSuppressed' : 'notif.replica_failure.text'),
navigateTextKey: 'notif.action.view',
navigateTarget: () => '/admin?tab=storage',
},
synology_session_cleared: {
inAppType: 'simple',
titleKey: 'notifications.synologySessionCleared.title',
@@ -703,6 +710,10 @@ export class NotificationsService {
const navigateTarget = inApp?.navigateTarget ?? config.navigateTarget(params);
const fullLink = navigateTarget ? `${appUrl}${navigateTarget}` : undefined;
// Resolve the in-app text key once, so all three in-app branches below agree
// (e.g. replica_failure picks the "N more failures suppressed" phrasing off params.suppressed).
const textKey = typeof config.textKey === 'function' ? config.textKey(params) : config.textKey;
// Fetch sender info once for in-app WS payloads
const sender = actorId
? this.db.get<{ username: string; avatar: string | null }>('SELECT username, avatar FROM users WHERE id = ?', actorId) ?? null
@@ -728,7 +739,7 @@ export class NotificationsService {
event_type: event,
title_key: config.titleKey,
title_params: params,
text_key: config.textKey,
text_key: textKey,
text_params: params,
positive_text_key: inApp.positiveTextKey ?? 'notif.action.accept',
negative_text_key: inApp.negativeTextKey ?? 'notif.action.decline',
@@ -744,7 +755,7 @@ export class NotificationsService {
event_type: event,
title_key: config.titleKey,
title_params: params,
text_key: config.textKey,
text_key: textKey,
text_params: params,
navigate_text_key: config.navigateTextKey ?? 'notif.action.view',
navigate_target: navigateTarget,
@@ -758,7 +769,7 @@ export class NotificationsService {
event_type: event,
title_key: config.titleKey,
title_params: params,
text_key: config.textKey,
text_key: textKey,
text_params: params,
};
}
@@ -0,0 +1,28 @@
/**
* Lazy per-backend debounce (backfill/stats/notifications spec): the first
* failure notifies immediately; later failures accumulate and the next one
* past the window rides out as a summary carrying the suppressed count. No
* timers — the summary is sent on the next failure, which is the only moment
* anyone would care. Process-local, like the health ring itself.
*/
export class ReplicaFailureDebouncer {
private readonly windows = new Map<string, { windowStart: number; suppressed: number }>();
constructor(
private readonly windowMs: number,
private readonly now: () => number = Date.now,
) {}
/** null = suppress; otherwise the suppressed-count to report with this send. */
admit(backend: string): number | null {
const now = this.now();
const window = this.windows.get(backend);
if (!window || now - window.windowStart > this.windowMs) {
const suppressed = window?.suppressed ?? 0;
this.windows.set(backend, { windowStart: now, suppressed: 0 });
return suppressed;
}
window.suppressed += 1;
return null;
}
}
@@ -0,0 +1,49 @@
import { Injectable, Logger, OnApplicationBootstrap } from '@nestjs/common';
import { StorageEventsService } from '../storage/storage-events.service';
import { NotificationsService } from './notifications.service';
import { ReplicaFailureDebouncer } from './replica-failure-debouncer';
const DEBOUNCE_WINDOW_MS = 60 * 60 * 1000;
/**
* Bridges storage replica failures into the admin notification channels.
* Lives HERE (not in StorageModule) because NotificationsModule → AuthModule
* → StorageModule — the reverse import would cycle. Subscribes at bootstrap;
* the debouncer keeps an outage from becoming a notification storm.
*/
@Injectable()
export class StorageHealthNotifierService implements OnApplicationBootstrap {
private readonly logger = new Logger(StorageHealthNotifierService.name);
private readonly debouncer = new ReplicaFailureDebouncer(DEBOUNCE_WINDOW_MS);
constructor(
private readonly events: StorageEventsService,
private readonly notifications: NotificationsService,
) {}
onApplicationBootstrap(): void {
this.events.onReplicaFailure((failure) => {
const suppressed = this.debouncer.admit(failure.backend);
if (suppressed === null) return;
void this.notifications
.send({
event: 'replica_failure',
actorId: null,
scope: 'admin',
targetId: 0,
params: {
backend: failure.backend,
key: failure.key,
op: failure.op,
error: failure.error,
suppressed: String(suppressed),
},
})
.catch((err: unknown) => {
this.logger.error(
`replica_failure notification failed: ${err instanceof Error ? err.message : String(err)}`,
);
});
});
}
}
@@ -24,6 +24,7 @@ const NTFY_EVENT_META: Partial<Record<NotifEventType, { priority: 1 | 2 | 3 | 4
collab_message: { priority: 3, tags: ['speech_balloon'] },
packing_tagged: { priority: 3, tags: ['luggage'] },
version_available: { priority: 4, tags: ['package'] },
replica_failure: { priority: 5, tags: ['warning', 'floppy_disk'] },
synology_session_cleared: { priority: 3, tags: ['warning'] },
};
const NTFY_DEFAULT_META = { priority: 3 as const, tags: [] as string[] };
@@ -60,7 +60,7 @@ export class PlaceEnrichmentController {
* local — no provider is contacted, so it needs no rate limit.
*/
@Get('credit/:key')
credit(@Param('key') key: string): { credit: string | null } {
credit(@Param('key') key: string): Promise<{ credit: string | null }> {
return this.enrichment.credit(key);
}
}
@@ -269,7 +269,7 @@ export class PlaceEnrichmentService {
const placeId = req.placeId?.trim() || `coords:${req.lat}:${req.lng}`;
const lang = req.lang;
const cached = this.readCache(placeId, lang);
const cached = await this.readCache(placeId, lang);
if (cached) return cached;
// One details lookup feeds all three halves: the pictures need its Commons
@@ -318,8 +318,8 @@ export class PlaceEnrichmentService {
* obligation does not end there — the place inspector reads this so a picture
* that was chosen weeks ago still names whoever made it.
*/
credit(key: string): { credit: string | null } {
return { credit: this.photoCache.get(key)?.attribution ?? null };
async credit(key: string): Promise<{ credit: string | null }> {
return { credit: (await this.photoCache.get(key))?.attribution ?? null };
}
/**
@@ -492,7 +492,7 @@ export class PlaceEnrichmentService {
credit: string | null,
fetchBytes: () => Promise<Buffer | null>,
): Promise<string | null> {
const hit = this.photoCache.get(key);
const hit = await this.photoCache.get(key);
if (hit) return hit.photoUrl;
const bytes = await withPhotoFetchSlot(fetchBytes);
@@ -662,7 +662,7 @@ export class PlaceEnrichmentService {
// ── Result cache ───────────────────────────────────────────────────────────
private readCache(placeId: string, lang: string | undefined): CachedEnrichment | null {
private async readCache(placeId: string, lang: string | undefined): Promise<CachedEnrichment | null> {
try {
const row = this.database.get<{ payload_json: string; fetched_at: number }>(
'SELECT payload_json, fetched_at FROM place_details_cache WHERE place_id = ? AND lang = ? AND expanded = ?',
@@ -679,7 +679,8 @@ export class PlaceEnrichmentService {
// them. Treat any loss as a stale entry and rebuild, rather than serving
// the survivors: filtering alone would leave the column empty for the rest
// of the week for a place that has pictures perfectly available.
const photos = parsed.photos.filter((photo) => this.photoCache.get(photo.key));
const checks = await Promise.all(parsed.photos.map((photo) => this.photoCache.get(photo.key)));
const photos = parsed.photos.filter((_, i) => checks[i]);
if (photos.length !== parsed.photos.length) return null;
const value: CachedEnrichment = {
@@ -20,13 +20,15 @@ export class PlacePhotoCacheJob implements OnApplicationBootstrap {
onApplicationBootstrap(): void {
if (!this.registrar.isEnabled()) return;
// Run once on startup to reclaim orphans left over from before this sweeper existed.
this.sweep();
this.registrar.register('place-photo-cache', '30 3 * * *', () => this.sweep());
void this.sweep();
this.registrar.register('place-photo-cache', '30 3 * * *', () => {
void this.sweep();
});
}
sweep(): void {
async sweep(): Promise<void> {
try {
const removed = this.cache.sweepOrphans();
const removed = await this.cache.sweepOrphans();
if (removed > 0) logInfo(`Place-photo cache cleanup: removed ${removed} orphaned file(s)/row(s)`);
} catch (err: unknown) {
logError(`Place-photo cache cleanup: ${err instanceof Error ? err.message : err}`);
@@ -1,11 +1,9 @@
import { Injectable, OnModuleInit } from '@nestjs/common';
import { Injectable } from '@nestjs/common';
import { Jimp, JimpMime } from 'jimp';
import crypto from 'node:crypto';
import fs from 'node:fs';
import fsPromises from 'node:fs/promises';
import path from 'node:path';
import { Readable } from 'node:stream';
import { DatabaseService } from '../database/database.service';
import { RuntimeEnvService } from '../app-config/runtime-env.service';
import { StorageService } from '../storage/storage.service';
// How long a "no photo for this place" answer stays remembered. Nothing about it
// changes until a photo appears upstream, so it is worth keeping: without it every
@@ -31,7 +29,6 @@ const JPEG_QUALITY = 80;
interface CachedPhoto {
photoUrl: string;
filePath: string;
attribution: string | null;
}
@@ -44,51 +41,38 @@ interface CachedPhoto {
* simultaneous requests for the same uncached place would both call the
* provider. That is the reason the nightly sweep (PlacePhotoCacheJob) injects
* this container singleton rather than constructing its own.
*
* Bytes live in the 'photos-google' storage category as '<sha1>.jpg' names.
* The registry's mode-aware prefix reproduces both TREK_PLACE_PHOTO_DIR
* layouts (unset: uploads/photos/google; set: that dir, bare keys), so the
* cache itself is mode-agnostic.
*/
@Injectable()
export class PlacePhotoCacheService implements OnModuleInit {
export class PlacePhotoCacheService {
/** Provider calls that failed recently — in memory only, so a restart retries. */
private readonly recentFailures = new Map<string, number>();
/** In-flight dedup: stops a stampede when several requests miss the same placeId at once. */
private readonly inFlight = new Map<string, Promise<{ filePath: string; attribution: string | null } | null>>();
/** placeIds whose file was confirmed on disk this session — saves an existsSync per hit. */
private readonly inFlight = new Map<string, Promise<{ attribution: string | null } | null>>();
/** placeIds whose object was confirmed in storage this session — saves a stat per hit. */
private readonly knownOnDisk = new Set<string>();
constructor(
private readonly db: DatabaseService,
private readonly env: RuntimeEnvService,
private readonly storage: StorageService,
) {}
/**
* Overridable for tests (mirrors the TREK_DB_FILE seam) so the suite never
* touches the real uploads tree. Read per call rather than frozen at import:
* the tests that set TREK_PLACE_PHOTO_DIR do so after this module loads.
*/
private get photoDir(): string {
return this.env.env().paths.placePhotoDir || path.join(__dirname, '../../../uploads/photos/google');
}
/** Ensure the upload dir exists once at startup, so put() never needs a sync FS call. */
onModuleInit(): void {
try {
fs.mkdirSync(this.photoDir, { recursive: true });
} catch {
/* already exists */
}
}
private filePath(placeId: string): string {
private fileName(placeId: string): string {
// Hash to avoid filename collisions — coords:lat:lng pseudo-IDs contain characters that
// collapse identically under sanitization (e.g. ':' and '.' both → '_')
const hash = crypto.createHash('sha1').update(placeId).digest('hex');
return path.join(this.photoDir, `${hash}.jpg`);
return `${hash}.jpg`;
}
private proxyUrl(placeId: string): string {
return `/api/maps/place-photo/${encodeURIComponent(placeId)}/bytes`;
}
get(placeId: string): CachedPhoto | null {
async get(placeId: string): Promise<CachedPhoto | null> {
const row = this.db.get<{ attribution: string | null }>(
'SELECT attribution FROM google_place_photo_meta WHERE place_id = ? AND error_at IS NULL',
placeId,
@@ -96,19 +80,17 @@ export class PlacePhotoCacheService implements OnModuleInit {
if (!row) return null;
const fp = this.filePath(placeId);
if (!this.knownOnDisk.has(placeId)) {
// First time this placeId is checked this session — verify the file exists on disk.
// First time this placeId is checked this session — verify the object exists.
// (Guards against volume wipes or manual deletion between server restarts.)
if (!fs.existsSync(fp)) {
if (!(await this.storage.exists('photos-google', this.fileName(placeId)))) {
this.db.run('DELETE FROM google_place_photo_meta WHERE place_id = ?', placeId);
return null;
}
this.knownOnDisk.add(placeId);
}
return { photoUrl: this.proxyUrl(placeId), filePath: fp, attribution: row.attribution };
return { photoUrl: this.proxyUrl(placeId), attribution: row.attribution };
}
getErrored(placeId: string): boolean {
@@ -170,12 +152,8 @@ export class PlacePhotoCacheService implements OnModuleInit {
}
async put(placeId: string, bytes: Buffer, attribution: string | null): Promise<CachedPhoto> {
const fp = this.filePath(placeId);
const tmp = fp + '.tmp';
const resized = await this.downscale(bytes);
await fsPromises.writeFile(tmp, resized);
await fsPromises.rename(tmp, fp);
await this.storage.put('photos-google', this.fileName(placeId), Readable.from(resized));
this.knownOnDisk.add(placeId);
this.recentFailures.delete(placeId);
@@ -185,19 +163,14 @@ export class PlacePhotoCacheService implements OnModuleInit {
placeId, attribution, Date.now(),
);
return { photoUrl: this.proxyUrl(placeId), filePath: fp, attribution };
return { photoUrl: this.proxyUrl(placeId), attribution };
}
getInFlight(
placeId: string,
): Promise<{ filePath: string; attribution: string | null } | null> | undefined {
getInFlight(placeId: string): Promise<{ attribution: string | null } | null> | undefined {
return this.inFlight.get(placeId);
}
setInFlight(
placeId: string,
promise: Promise<{ filePath: string; attribution: string | null } | null>,
): void {
setInFlight(placeId: string, promise: Promise<{ attribution: string | null } | null>): void {
this.inFlight.set(placeId, promise);
promise
.finally(() => this.inFlight.delete(placeId))
@@ -206,12 +179,16 @@ export class PlacePhotoCacheService implements OnModuleInit {
});
}
serveFilePath(placeId: string): string | null {
if (this.knownOnDisk.has(placeId)) return this.filePath(placeId);
const fp = this.filePath(placeId);
if (!fs.existsSync(fp)) return null;
/**
* Storage name (category 'photos-google') for a cached photo, or null when
* the object is missing.
*/
async serveKey(placeId: string): Promise<string | null> {
const name = this.fileName(placeId);
if (this.knownOnDisk.has(placeId)) return name;
if (!(await this.storage.exists('photos-google', name))) return null;
this.knownOnDisk.add(placeId);
return fp;
return name;
}
/**
@@ -235,12 +212,10 @@ export class PlacePhotoCacheService implements OnModuleInit {
return !!row;
}
private deleteEntry(placeId: string): void {
try {
fs.unlinkSync(this.filePath(placeId));
} catch {
private async deleteEntry(placeId: string): Promise<void> {
await this.storage.delete('photos-google', this.fileName(placeId)).catch(() => {
/* already gone */
}
});
this.db.run('DELETE FROM google_place_photo_meta WHERE place_id = ?', placeId);
this.knownOnDisk.delete(placeId);
}
@@ -249,41 +224,37 @@ export class PlacePhotoCacheService implements OnModuleInit {
* Drop a cache entry if no place references it anymore. Called after a place delete
* for prompt reclamation; the nightly sweep is the catch-all for every other path.
*/
removeIfUnreferenced(placeId: string): void {
async removeIfUnreferenced(placeId: string): Promise<void> {
if (this.isReferenced(placeId)) return;
this.deleteEntry(placeId);
await this.deleteEntry(placeId);
}
/**
* Reclaim orphaned cache files + meta rows. Runs on startup and nightly (scheduler).
* Two passes: (1) meta rows no place references; (2) stray .jpg files with no meta row.
* Two passes: (1) meta rows no place references; (2) stray .jpg objects with no meta row.
*/
sweepOrphans(): number {
async sweepOrphans(): Promise<number> {
let removed = 0;
const rows = this.db.all<{ place_id: string }>('SELECT place_id FROM google_place_photo_meta');
const keepFiles = new Set<string>();
for (const { place_id } of rows) {
if (this.isReferenced(place_id)) {
keepFiles.add(`${crypto.createHash('sha1').update(place_id).digest('hex')}.jpg`);
keepFiles.add(this.fileName(place_id));
} else {
this.deleteEntry(place_id);
await this.deleteEntry(place_id);
removed++;
}
}
// Pass 2: files on disk that no surviving meta row maps to (e.g. left over from a
// crash between writeFile and the DB upsert, or a meta row deleted out-of-band).
let entries: string[];
try {
entries = fs.readdirSync(this.photoDir);
} catch {
entries = [];
}
for (const entry of entries) {
if (!entry.endsWith('.jpg') || keepFiles.has(entry)) continue;
// Pass 2: objects that no surviving meta row maps to (e.g. left over from a
// crash between the object write and the DB upsert, or a meta row deleted
// out-of-band). list() recurses where the old readdirSync didn't — skip
// nested keys so a subdirectory of a relocated cache dir is never swept.
for await (const stat of this.storage.list('photos-google')) {
if (stat.key.includes('/') || !stat.key.endsWith('.jpg') || keepFiles.has(stat.key)) continue;
try {
fs.unlinkSync(path.join(this.photoDir, entry));
await this.storage.delete('photos-google', stat.key);
removed++;
} catch {
/* race */
@@ -3,6 +3,7 @@ import { PlacePhotoCacheService } from './place-photo-cache.service';
import { PlacePhotoCacheJob } from './place-photo-cache.job';
import { AppConfigModule } from '../app-config/app-config.module';
import { SchedulingModule } from '../scheduling/scheduling.module';
import { StorageModule } from '../storage/storage.module';
/** The marker-photo cache. No controller of its own — maps serves the bytes,
* places and share read through it, and PlacePhotoCacheJob sweeps it nightly.
@@ -11,7 +12,7 @@ import { SchedulingModule } from '../scheduling/scheduling.module';
* imported explicitly because @Global only reaches modules that are in the
* graph — which a single-domain e2e TestingModule is not. */
@Module({
imports: [AppConfigModule, SchedulingModule],
imports: [AppConfigModule, SchedulingModule, StorageModule],
providers: [PlacePhotoCacheService, PlacePhotoCacheJob],
exports: [PlacePhotoCacheService],
})
+11 -14
View File
@@ -1,11 +1,7 @@
import fs from 'fs';
import path from 'path';
import path from 'node:path';
import { db } from '../../db/database';
import type { StorageService } from '../storage/storage.service';
// Custom place images (the user-uploaded thumbnails from #1136) live in their own
// uploads subdir, served statically like covers. Storing the full /uploads/places/…
// path in image_url lets every existing thumbnail renderer show it unchanged.
export const PLACE_IMAGES_DIR = path.resolve(__dirname, '../../../uploads/places');
const URL_PREFIX = '/uploads/places/';
export function placeImageUrl(filename: string): string {
@@ -17,20 +13,21 @@ export function isUploadedPlaceImage(url: string | null | undefined): url is str
}
/**
* Delete a custom place-image file once nothing references it any more. A trip
* Delete a custom place-image object once nothing references it any more. A trip
* place and a collection saved-place can share the same uploaded file — save-to-
* collection and copy-to-trip copy image_url by reference — so we ref-count across
* both tables before unlinking. The path is confined to PLACE_IMAGES_DIR via
* basename, mirroring tripService.deleteOldCover. Best-effort: never throws.
* both tables before deleting. basename() keeps the storage name confined to the
* 'places' category, mirroring tripService.deleteOldCover. Best-effort: never
* throws (central key validation rejects a hostile stored value; the catch
* swallows it exactly like the old unlink guard).
*/
export function reclaimPlaceImage(url: string | null | undefined): void {
export async function reclaimPlaceImage(storage: StorageService, url: string | null | undefined): Promise<void> {
if (!isUploadedPlaceImage(url)) return;
const referenced =
db.prepare('SELECT 1 FROM places WHERE image_url = ? LIMIT 1').get(url) ||
db.prepare('SELECT 1 FROM collection_places WHERE image_url = ? LIMIT 1').get(url);
if (referenced) return;
try {
const resolved = path.resolve(path.join(PLACE_IMAGES_DIR, path.basename(url)));
if (resolved.startsWith(PLACE_IMAGES_DIR + path.sep) && fs.existsSync(resolved)) fs.unlinkSync(resolved);
} catch { /* best-effort */ }
await storage.delete('places', path.basename(url)).catch(() => {
/* best-effort */
});
}
+17 -12
View File
@@ -27,7 +27,8 @@ import { isUpdateConflict } from '../common/conflictResult';
import { JwtAuthGuard } from '../auth/jwt-auth.guard';
import { CurrentUser } from '../auth/current-user.decorator';
import { RequirePermission, TripAccessGuard } from '../permissions/trip-access.guard';
import { PLACE_IMAGE_UPLOAD } from '../common/place-image-upload';
import { PLACE_IMAGE_FILE_FILTER } from '../common/place-image-upload';
import { StorageService } from '../storage/storage.service';
import { placeImageUrl } from './place-image';
import {
PlaceBulkDeleteDto,
@@ -112,6 +113,7 @@ export class PlacesController {
constructor(
private readonly places: PlacesService,
private readonly env: RuntimeEnvService,
private readonly storage: StorageService,
) {}
private requireTrip(tripId: string, user: User) {
@@ -290,7 +292,7 @@ export class PlacesController {
@Post('bulk-delete')
@HttpCode(200) // Express answers bulk-delete with res.json (200), unlike the 201 imports.
bulkDelete(
async bulkDelete(
@CurrentUser() user: User,
@Param('tripId') tripId: string,
@Body() body: PlaceBulkDeleteDto,
@@ -311,7 +313,7 @@ export class PlacesController {
for (const id of scoped) this.places.onDeleted(id);
// Read the linked expenses before the delete — afterwards the link is gone (#1298).
const expenseIds = this.places.linkedExpenseIds(tripId, scoped);
const deleted = this.places.removeMany(tripId, ids);
const deleted = await this.places.removeMany(tripId, ids);
for (const id of deleted) {
this.places.broadcast(tripId, 'place:deleted', { placeId: id }, socketId);
}
@@ -323,7 +325,7 @@ export class PlacesController {
@Post('bulk-update')
@HttpCode(200)
bulkUpdate(
async bulkUpdate(
@CurrentUser() user: User,
@Param('tripId') tripId: string,
@Body() body: PlaceBulkUpdateDto,
@@ -340,7 +342,7 @@ export class PlacesController {
if (!('category_id' in body)) {
throw new HttpException({ error: 'Provide at least one field to update' }, 400);
}
const updated = this.places.updateMany(tripId, ids, { category_id: body.category_id as number | null });
const updated = await this.places.updateMany(tripId, ids, { category_id: body.category_id as number | null });
for (const place of updated) {
this.places.broadcast(tripId, 'place:updated', { place }, socketId);
this.places.onUpdated(place.id);
@@ -360,8 +362,8 @@ export class PlacesController {
@Post(':id/image')
@HttpCode(200)
@UseInterceptors(FileInterceptor('image', PLACE_IMAGE_UPLOAD))
uploadImage(
@UseInterceptors(FileInterceptor('image', { fileFilter: PLACE_IMAGE_FILE_FILTER }))
async uploadImage(
@CurrentUser() user: User,
@Param('tripId') tripId: string,
@Param('id') id: string,
@@ -378,9 +380,12 @@ export class PlacesController {
if (!file) {
throw new HttpException({ error: 'No image uploaded' }, 400);
}
// Commit the spooled upload to its final storage location (atomic
// same-volume rename) before the DB row references the final path.
await this.storage.put('places', file.filename, { tmpPath: file.path });
// Reuse the existing image_url slot (the top-precedence thumbnail source); the
// update path reclaims any previously uploaded file it replaces.
const result = this.places.update(tripId, id, { image_url: placeImageUrl(file.filename) } as never);
const result = await this.places.update(tripId, id, { image_url: placeImageUrl(file.filename) } as never);
if (!result || isUpdateConflict(result)) {
throw new HttpException({ error: 'Place not found' }, 404);
}
@@ -437,7 +442,7 @@ export class PlacesController {
}
@Put(':id')
update(
async update(
@CurrentUser() user: User,
@Param('tripId') tripId: string,
@Param('id') id: string,
@@ -450,7 +455,7 @@ export class PlacesController {
validateRouteColor(body);
validateUrlFields(body);
this.requireEdit(trip, user);
const result = this.places.update(tripId, id, body as never, ifMatch);
const result = await this.places.update(tripId, id, body as never, ifMatch);
if (!result) {
throw new HttpException({ error: 'Place not found' }, 404);
}
@@ -468,7 +473,7 @@ export class PlacesController {
@Delete(':id')
@UseGuards(TripAccessGuard)
@RequirePermission('place_edit')
remove(@CurrentUser() user: User, @Param('tripId') tripId: string, @Param('id') id: string, @Headers('x-socket-id') socketId?: string) {
async remove(@CurrentUser() user: User, @Param('tripId') tripId: string, @Param('id') id: string, @Headers('x-socket-id') socketId?: string) {
// Scope the id to the trip before the hook (see bulkDelete), then sync the
// journey ahead of the actual delete.
if (!this.places.get(tripId, id)) {
@@ -476,7 +481,7 @@ export class PlacesController {
}
this.places.onDeleted(Number(id));
const expenseIds = this.places.linkedExpenseIds(tripId, [id]);
if (!this.places.remove(tripId, id)) {
if (!(await this.places.remove(tripId, id))) {
throw new HttpException({ error: 'Place not found' }, 404);
}
this.places.broadcast(tripId, 'place:deleted', { placeId: Number(id) }, socketId);
+2 -2
View File
@@ -101,13 +101,13 @@ export interface KmlImportOptions {
// Reclaim a deleted place's cached marker photo if nothing else references it.
// The cache key is the Google place_id, or — for coordinate-only places — the
// pseudo-id embedded in the stored proxy URL (/api/maps/place-photo/{id}/bytes).
export function reclaimPhotoCache(cache: PlacePhotoCacheService, googlePlaceId: string | null, imageUrl: string | null): void {
export async function reclaimPhotoCache(cache: PlacePhotoCacheService, googlePlaceId: string | null, imageUrl: string | null): Promise<void> {
const candidates = new Set<string>();
if (googlePlaceId) candidates.add(googlePlaceId);
const m = imageUrl?.match(/^\/api\/maps\/place-photo\/(.+)\/bytes$/);
if (m) { try { candidates.add(decodeURIComponent(m[1])); } catch { /* malformed url */ } }
for (const id of candidates) {
try { cache.removeIfUnreferenced(id); } catch { /* best-effort */ }
try { await cache.removeIfUnreferenced(id); } catch { /* best-effort */ }
}
}
+4 -4
View File
@@ -182,7 +182,7 @@ export class PlacesMcp {
if (this.auth.isDemoUser(ctx.userId)) return demoDenied();
if (!this.db.canAccessTrip(tripId, ctx.userId)) return noAccess();
if (!this.guards.hasTripPermission('place_edit', tripId, ctx.userId)) return permissionDenied();
const place = this.places.update(String(tripId), String(placeId), { name, description, lat, lng, address, category_id, price, currency, place_time, end_time, duration_minutes, notes, website, phone, transport_mode, osm_id, google_place_id, google_ftid });
const place = await this.places.update(String(tripId), String(placeId), { name, description, lat, lng, address, category_id, price, currency, place_time, end_time, duration_minutes, notes, website, phone, transport_mode, osm_id, google_place_id, google_ftid });
if (!place) return { content: [{ type: 'text' as const, text: 'Place not found.' }], isError: true };
this.guards.safeBroadcast(tripId, 'place:updated', { place });
return ok({ place });
@@ -235,7 +235,7 @@ export class PlacesMcp {
try { this.journey.onPlaceDeleted(placeId); } catch { /* non-fatal */ } // sync journeys before the row is gone
// The link is gone once the place is, so read it first (#1298).
const expenseIds = this.places.linkedExpenseIds(tripId, [placeId]);
const deleted = this.places.remove(String(tripId), String(placeId));
const deleted = await this.places.remove(String(tripId), String(placeId));
if (!deleted) return { content: [{ type: 'text' as const, text: 'Place not found.' }], isError: true };
this.guards.safeBroadcast(tripId, 'place:deleted', { placeId });
for (const itemId of expenseIds) this.guards.safeBroadcast(tripId, 'budget:deleted', { itemId });
@@ -347,7 +347,7 @@ export class PlacesMcp {
}
// The link is gone once the places are, so read it first (#1298).
const expenseIds = this.places.linkedExpenseIds(tripId, scoped);
const deleted = this.places.removeMany(String(tripId), placeIds);
const deleted = await this.places.removeMany(String(tripId), placeIds);
for (const id of deleted) this.guards.safeBroadcast(tripId, 'place:deleted', { placeId: id });
for (const itemId of expenseIds) this.guards.safeBroadcast(tripId, 'budget:deleted', { itemId });
return ok({ deleted, count: deleted.length });
@@ -392,7 +392,7 @@ export class PlacesMcp {
return { content: [{ type: 'text' as const, text: 'Provide at least one field to update.' }], isError: true };
}
const updated = this.places.updateMany(String(tripId), placeIds, fields);
const updated = await this.places.updateMany(String(tripId), placeIds, fields);
for (const place of updated) this.guards.safeBroadcast(tripId, 'place:updated', { place });
return ok({ count: updated.length, updatedIds: updated.map(p => p.id), skipped: placeIds.length - updated.length });
}
+18 -1
View File
@@ -15,6 +15,11 @@ import { QueryHelpersModule } from '../query-helpers/query-helpers.module';
import { MapsModule } from '../maps/maps.module';
import { AuthModule } from '../auth/auth.module';
import { McpSharedModule } from '../mcp-shared/mcp-shared.module';
import { MulterModule } from '@nestjs/platform-express';
import { StorageModule } from '../storage/storage.module';
import { StorageService } from '../storage/storage.service';
import { buildStorageUploadOptions } from '../storage/storage-upload.factory';
import { MAX_PLACE_IMAGE_SIZE } from '../common/place-image-upload';
/**
* Places domain (S8 — Phase 2 trip sub-domain). Depends on L4 Categories + L5
@@ -25,7 +30,19 @@ import { McpSharedModule } from '../mcp-shared/mcp-shared.module';
* is no places.bridge.ts: nothing outside the container consumes this domain.
*/
@Module({
imports: [McpSharedModule, PermissionsModule, QueryHelpersModule, MapsModule, AuthModule, AppConfigModule, UnsplashModule, PlacePhotosModule, JourneyDomainModule, RealtimeModule, PluginGuardsModule, AssignmentsDomainModule],
imports: [
// Module-level options deliberately carry NO fileFilter: the GPX/KML import
// route keeps its inline memoryStorage config and would inherit a
// module-level filter via the interceptor's shallow merge — the image
// route passes PLACE_IMAGE_FILE_FILTER inline instead.
MulterModule.registerAsync({
imports: [StorageModule],
inject: [StorageService],
useFactory: (storage: StorageService) =>
buildStorageUploadOptions(storage, { category: 'places', maxSize: MAX_PLACE_IMAGE_SIZE }),
}),
StorageModule,
McpSharedModule, PermissionsModule, QueryHelpersModule, MapsModule, AuthModule, AppConfigModule, UnsplashModule, PlacePhotosModule, JourneyDomainModule, RealtimeModule, PluginGuardsModule, AssignmentsDomainModule],
controllers: [PlacesController],
providers: [PlacesService, PlacesMcp, PlacesRpc],
exports: [PlacesService],
+9 -4
View File
@@ -67,7 +67,7 @@ export class PlacesRpc {
}
@PluginMethod('places.update', { permission: 'db:write:places' })
update(params: Record<string, unknown>, ctx: PluginRpcContext): unknown {
async update(params: Record<string, unknown>, ctx: PluginRpcContext): Promise<unknown> {
const tripId = num(params.tripId, 'tripId');
const placeId = num(params.placeId, 'placeId');
const actor = this.guards.requireActor(ctx, 'place');
@@ -76,7 +76,10 @@ export class PlacesRpc {
this.guards.capStrings(parsed.data as Record<string, unknown>, PLACE_STR_LIMITS);
capUrls(parsed.data as Record<string, unknown>);
this.guards.requireTripEdit(tripId, actor, PLACE_EDIT_ACTION);
const place = this.places.update(String(tripId), String(placeId), parsed.data as PlaceUpdateInput);
// update is async (it may delete a superseded storage object): await it so the
// null-refusal check compares against the resolved place, not an always-truthy
// Promise, and the broadcast below carries the actual place.
const place = await this.places.update(String(tripId), String(placeId), parsed.data as PlaceUpdateInput);
if (place === null) throw new ForbiddenResource(`no place ${placeId} on trip ${tripId}`);
this.realtime.broadcast(tripId, 'place:updated', { place });
this.mirrorJourneys(() => this.journey.onPlaceUpdated(placeId));
@@ -84,7 +87,7 @@ export class PlacesRpc {
}
@PluginMethod('places.delete', { permission: 'db:write:places' })
delete(params: Record<string, unknown>, ctx: PluginRpcContext): unknown {
async delete(params: Record<string, unknown>, ctx: PluginRpcContext): Promise<unknown> {
const tripId = num(params.tripId, 'tripId');
const placeId = num(params.placeId, 'placeId');
const actor = this.guards.requireActor(ctx, 'place');
@@ -101,7 +104,9 @@ export class PlacesRpc {
this.mirrorJourneys(() => this.journey.onPlaceDeleted(placeId));
// The link is gone once the place is, so read it first (#1298).
const expenseIds = this.places.linkedExpenseIds(tripId, [placeId]);
if (!this.places.remove(String(tripId), String(placeId))) {
// remove is async (it deletes the place's storage object): await it so the
// refusal check sees the resolved boolean, not an always-truthy Promise.
if (!(await this.places.remove(String(tripId), String(placeId)))) {
throw new ForbiddenResource(`no place ${placeId} on trip ${tripId}`);
}
this.realtime.broadcast(tripId, 'place:deleted', { placeId });
+43 -19
View File
@@ -26,6 +26,7 @@ import { PlacePhotoCacheService } from '../place-photos/place-photo-cache.servic
import { type UpdateConflict, isUpdateConflict } from '../common/conflictResult';
import { reclaimPlaceImage } from './place-image';
import { JourneyDomainService } from '../journey/journey-domain.service';
import { StorageService } from '../storage/storage.service';
import {
COORD_DEDUP_TOLERANCE,
ENRICH_CONCURRENCY,
@@ -113,6 +114,7 @@ export class PlacesService {
private readonly unsplash: UnsplashService,
private readonly photoCache: PlacePhotoCacheService,
private readonly journey: JourneyDomainService,
private readonly storage: StorageService,
) {}
verifyTripAccess(tripId: string, userId: number) {
@@ -269,20 +271,36 @@ export class PlacesService {
// Update place
// -------------------------------------------------------------------------
update(
async update(
tripId: string,
placeId: string,
body: PlaceUpdateInput,
ifMatch?: string,
): PlaceWithTags | UpdateConflict | null {
): Promise<PlaceWithTags | UpdateConflict | null> {
const { result, reclaim } = this.applyUpdate(tripId, placeId, body, ifMatch);
if (reclaim !== undefined) await reclaimPlaceImage(this.storage, reclaim);
return result;
}
/**
* The synchronous DB half of update(). Split out so updateMany() can run it
* inside a better-sqlite3 transaction (which cannot await) and settle the
* storage reclaims after the transaction commits.
*/
private applyUpdate(
tripId: string,
placeId: string,
body: PlaceUpdateInput,
ifMatch?: string,
): { result: PlaceWithTags | UpdateConflict | null; reclaim?: string | null } {
const existingPlace = this.dbs.get<Place>('SELECT * FROM places WHERE id = ? AND trip_id = ?', placeId, tripId);
if (!existingPlace) return null;
if (!existingPlace) return { result: null };
// Optimistic concurrency (#1135): when the caller sent the version it based its
// edit on and the row has moved on since, reject instead of clobbering. Absent
// token => unconditional update (back-compat — old clients keep last-write-wins).
if (ifMatch !== undefined && existingPlace.updated_at != null && String(existingPlace.updated_at) !== ifMatch) {
return { conflict: true, server: this.dbs.getPlaceWithTags(placeId) };
return { result: { conflict: true, server: this.dbs.getPlaceWithTags(placeId) } };
}
const {
@@ -355,12 +373,13 @@ export class PlacesService {
}
// A custom uploaded thumbnail (#1136) that was just replaced or cleared leaves
// an orphan file behind — reclaim it once nothing references it any more.
if (image_url !== undefined && image_url !== existingPlace.image_url) {
reclaimPlaceImage(existingPlace.image_url);
}
// an orphan file behind — reclaim it (in the caller, once any enclosing
// transaction committed) if nothing references it any more.
const reclaim = image_url !== undefined && image_url !== existingPlace.image_url
? existingPlace.image_url
: undefined;
return this.dbs.getPlaceWithTags(placeId);
return { result: this.dbs.getPlaceWithTags(placeId), reclaim };
}
// -------------------------------------------------------------------------
@@ -381,7 +400,7 @@ export class PlacesService {
return rows.map(r => r.id);
}
remove(tripId: string, placeId: string): boolean {
async remove(tripId: string, placeId: string): Promise<boolean> {
const place = this.dbs.get<{ google_place_id: string | null; image_url: string | null }>(
'SELECT google_place_id, image_url FROM places WHERE id = ? AND trip_id = ?', placeId, tripId,
);
@@ -393,12 +412,12 @@ export class PlacesService {
this.dbs.run('DELETE FROM budget_items WHERE trip_id = ? AND place_id = ?', tripId, placeId);
this.dbs.run('DELETE FROM places WHERE id = ?', placeId);
});
reclaimPhotoCache(this.photoCache, place.google_place_id, place.image_url);
reclaimPlaceImage(place.image_url);
await reclaimPhotoCache(this.photoCache, place.google_place_id, place.image_url);
await reclaimPlaceImage(this.storage, place.image_url);
return true;
}
removeMany(tripId: string, ids: number[]): number[] {
async removeMany(tripId: string, ids: number[]): Promise<number[]> {
if (ids.length === 0) return [];
const selectStmt = this.dbs.prepare('SELECT google_place_id, image_url FROM places WHERE id = ? AND trip_id = ?');
const deleteStmt = this.dbs.prepare('DELETE FROM places WHERE id = ?');
@@ -417,8 +436,8 @@ export class PlacesService {
});
// Reclaim after the transaction commits so isReferenced() sees the final place set.
for (const row of reclaimable) {
reclaimPhotoCache(this.photoCache, row.google_place_id, row.image_url);
reclaimPlaceImage(row.image_url);
await reclaimPhotoCache(this.photoCache, row.google_place_id, row.image_url);
await reclaimPlaceImage(this.storage, row.image_url);
}
return deleted;
}
@@ -449,17 +468,22 @@ export class PlacesService {
* fields change and everything else is preserved. IDs that don't belong to the
* trip are skipped. Returns the updated places.
*/
updateMany(tripId: string, ids: number[], body: PlaceUpdateInput): PlaceWithTags[] {
async updateMany(tripId: string, ids: number[], body: PlaceUpdateInput): Promise<PlaceWithTags[]> {
if (ids.length === 0) return [];
const updated: PlaceWithTags[] = [];
const reclaims: (string | null)[] = [];
this.dbs.transaction(() => {
for (const id of ids) {
// Bulk update sends no If-Match, so update() never returns a conflict
// here; the guard keeps the types honest.
const place = this.update(tripId, String(id), body);
// Bulk update sends no If-Match, so applyUpdate() never returns a
// conflict here; the guard keeps the types honest.
const { result: place, reclaim } = this.applyUpdate(tripId, String(id), body);
if (place && !isUpdateConflict(place)) updated.push(place);
if (reclaim !== undefined) reclaims.push(reclaim);
}
});
// Settle reclaims after the transaction commits, so the refcount sees the
// final image_url state.
for (const reclaim of reclaims) await reclaimPlaceImage(this.storage, reclaim);
return updated;
}

Some files were not shown because too many files have changed in this diff Show More