mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-10-19 13:36:48 +00:00
fix(server): bound every outbound fetch
The rule is a timeout, a size cap and boundary validation on anything that leaves the process, and several call sites had only the timeout. A link preview, a maps URL resolve, a wiki fallback and a holiday lookup all read the whole body into memory, so a caller who picks the URL picks how much memory the server spends - a few concurrent multi-hundred-megabyte responses is the whole heap. Two of them also fetched more than their name suggests. resolve-url fell through to a full page GET for any host once it found no coordinates, which makes the server a general-purpose outbound proxy wearing its own IP. The weather URLs took lat/lng straight from the caller into the query string. readCapped/exceedsDeclaredLength collect the pattern transit.service.ts and the plugin installer had each grown separately, so there is one of it now. Callers that only scan the head of a document (og: tags, a coordinate regex) keep the truncated bytes and find fewer fields; callers that must have the whole thing reject it.
This commit is contained in:
@@ -0,0 +1,78 @@
|
||||
/**
|
||||
* Response size caps for outbound fetches.
|
||||
*
|
||||
* Every outbound request has to bound what it buffers, and there are two
|
||||
* things worth bounding: the length the provider declares (cheap, but a
|
||||
* chunked response omits it) and the bytes that actually arrive. The
|
||||
* declared-length check is the pattern transit.service.ts established; the
|
||||
* streaming read is the one the plugin installer needed for codeload
|
||||
* archives, lifted here so the other fetch sites can share it.
|
||||
*/
|
||||
|
||||
/** Minimal shape of the bits of Response these helpers touch (test stubs are partial). */
|
||||
interface CappedReader {
|
||||
read(): Promise<{ done?: boolean; value?: Uint8Array }>;
|
||||
cancel(): Promise<unknown>;
|
||||
}
|
||||
interface CappedResponse {
|
||||
headers?: { get(name: string): string | null } | null;
|
||||
body?: { getReader(): CappedReader } | null;
|
||||
arrayBuffer?(): Promise<ArrayBuffer>;
|
||||
text?(): Promise<string>;
|
||||
}
|
||||
|
||||
/** True when the provider declares a body larger than the budget. */
|
||||
export function exceedsDeclaredLength(res: CappedResponse, maxBytes: number): boolean {
|
||||
return Number(res.headers?.get('content-length') ?? 0) > maxBytes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a body, stopping once maxBytes have arrived. Callers that must reject
|
||||
* an oversized payload look at `truncated`; callers that only scan the head of
|
||||
* the document (og: tags, a coordinate regex) can use the partial bytes and
|
||||
* degrade to whatever they find. Responses without a stream fall back to
|
||||
* arrayBuffer(), which is post-checked instead.
|
||||
*/
|
||||
export async function readCapped(res: CappedResponse, maxBytes: number): Promise<{ bytes: Buffer; truncated: boolean }> {
|
||||
const reader = res.body?.getReader();
|
||||
if (!reader) {
|
||||
const raw = res.arrayBuffer ? Buffer.from(await res.arrayBuffer()) : Buffer.from(res.text ? await res.text() : '');
|
||||
return raw.length > maxBytes
|
||||
? { bytes: raw.subarray(0, maxBytes), truncated: true }
|
||||
: { bytes: raw, truncated: false };
|
||||
}
|
||||
|
||||
const chunks: Buffer[] = [];
|
||||
let received = 0;
|
||||
let truncated = false;
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done || !value) break;
|
||||
chunks.push(Buffer.from(value));
|
||||
received += value.length;
|
||||
if (received > maxBytes) {
|
||||
truncated = true;
|
||||
await reader.cancel().catch(() => {});
|
||||
break;
|
||||
}
|
||||
}
|
||||
const bytes = Buffer.concat(chunks);
|
||||
return { bytes: truncated ? bytes.subarray(0, maxBytes) : bytes, truncated };
|
||||
}
|
||||
|
||||
/**
|
||||
* readCapped for text bodies. Decodes as UTF-8, so a chopped trailing sequence
|
||||
* comes back as a replacement character rather than as a throw. Without a
|
||||
* stream it reads text() directly instead of going through readCapped, which
|
||||
* prefers arrayBuffer().
|
||||
*/
|
||||
export async function readCappedText(res: CappedResponse, maxBytes: number): Promise<{ text: string; truncated: boolean }> {
|
||||
if (!res.body?.getReader && res.text) {
|
||||
const raw = Buffer.from(await res.text(), 'utf8');
|
||||
return raw.length > maxBytes
|
||||
? { text: new TextDecoder('utf-8').decode(raw.subarray(0, maxBytes)), truncated: true }
|
||||
: { text: raw.toString('utf8'), truncated: false };
|
||||
}
|
||||
const { bytes, truncated } = await readCapped(res, maxBytes);
|
||||
return { text: new TextDecoder('utf-8').decode(bytes), truncated };
|
||||
}
|
||||
Reference in New Issue
Block a user