fix(llm): retry OpenAI import with max_completion_tokens

Newer OpenAI models (gpt-5.x) reject max_tokens with a 400 and demand
max_completion_tokens, so AI booking import failed and no reservations
were created. Send max_tokens first (understood by the classic chat API
and every local server), and only on a 400 naming max_completion_tokens
resend the whole request with that parameter instead — mirroring the
existing json_schema->json_object fallback beside it. Detection is by
API response, not a model-name allowlist.

Fixes #1760
This commit is contained in:
Konstantinos Thermos
2026-08-04 14:14:42 +03:00
committed by Maurice
parent 17dd6b7caa
commit 28ffc814c0
2 changed files with 71 additions and 28 deletions
@@ -44,42 +44,58 @@ export class OpenAiCompatibleClient implements LlmExtractionClient {
});
}
const baseBody = {
model: input.model,
max_tokens: MAX_TOKENS,
// Extraction is a deterministic task — Ollama defaults to 0.7, which makes
// small models (NuExtract) drop fields or return empty. Pin to 0.
temperature: 0,
// NuExtract wants the template (in the user turn) to be the only instruction
// — a system prompt or a json_schema grammar derails it.
messages: nuextract
? [{ role: 'user', content: userContent }]
: [
{ role: 'system', content: input.prompt },
{ role: 'user', content: userContent },
],
// The token cap is `max_tokens` for the chat-completions API and for every
// local server (Ollama/vLLM/llama.cpp), but newer OpenAI models reject it
// with a 400 and demand `max_completion_tokens`. Start with the broadly
// supported spelling and swap on that specific rejection (#1760).
const buildBody = (tokenParam: 'max_tokens' | 'max_completion_tokens', jsonObject: boolean) => {
const baseBody = {
model: input.model,
[tokenParam]: MAX_TOKENS,
// Extraction is a deterministic task — Ollama defaults to 0.7, which makes
// small models (NuExtract) drop fields or return empty. Pin to 0.
temperature: 0,
// NuExtract wants the template (in the user turn) to be the only instruction
// — a system prompt or a json_schema grammar derails it.
messages: nuextract
? [{ role: 'user', content: userContent }]
: [
{ role: 'system', content: input.prompt },
{ role: 'user', content: userContent },
],
};
if (nuextract) return baseBody;
return {
...baseBody,
response_format: jsonObject
? { type: 'json_object' as const }
: { type: 'json_schema' as const, json_schema: { name: 'reservations', schema: input.jsonSchema, strict: false } },
};
};
const body = nuextract
? baseBody
: {
...baseBody,
response_format: {
type: 'json_schema' as const,
json_schema: { name: 'reservations', schema: input.jsonSchema, strict: false },
},
};
let res = await this.send(url, body, input.apiKey);
let tokenParam: 'max_tokens' | 'max_completion_tokens' = 'max_tokens';
let res = await this.send(url, buildBody(tokenParam, false), input.apiKey);
let detail = res.ok ? '' : await res.text().catch(() => '');
// Newer OpenAI models 400 on `max_tokens` — retry the whole request (schema
// and all) with `max_completion_tokens` before giving up.
if (!res.ok && res.status === 400 && detail.includes('max_completion_tokens')) {
tokenParam = 'max_completion_tokens';
res = await this.send(url, buildBody(tokenParam, false), input.apiKey);
detail = res.ok ? '' : await res.text().catch(() => '');
}
// Servers that only support `json_object` (DeepSeek, Mistral, some
// vLLM/llama.cpp) reject `json_schema` with a 400 — retry once in
// `json_object` mode. The system prompt already dictates the exact output
// shape (and mentions JSON, which json_object mode requires).
// `json_object` mode (keeping whichever token param stuck). The system
// prompt already dictates the exact output shape (and mentions JSON, which
// json_object mode requires).
if (!res.ok && res.status === 400 && !nuextract) {
res = await this.send(url, { ...baseBody, response_format: { type: 'json_object' as const } }, input.apiKey);
res = await this.send(url, buildBody(tokenParam, true), input.apiKey);
detail = res.ok ? '' : await res.text().catch(() => '');
}
if (!res.ok) {
const detail = await res.text().catch(() => '');
throw new Error(`LLM request failed (${res.status}): ${detail.slice(0, 300)}`);
}
@@ -93,6 +93,33 @@ describe('OpenAiCompatibleClient', () => {
expect(second.model).toBe(first.model);
});
it('retries with max_completion_tokens when the model rejects max_tokens (400, #1760)', async () => {
safeFetchLlmMock
.mockResolvedValueOnce(
jsonResponse(
{ error: { message: "Unsupported parameter: 'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead.", code: 'unsupported_parameter' } },
false,
400,
),
)
.mockResolvedValueOnce(
jsonResponse({ choices: [{ message: { content: '{"reservations":[{"@type":"FlightReservation"}]}' } }] }),
);
const out = await new OpenAiCompatibleClient().extract(baseInput);
expect(out).toEqual([{ '@type': 'FlightReservation' }]);
expect(safeFetchLlmMock).toHaveBeenCalledTimes(2);
const first = JSON.parse((safeFetchLlmMock.mock.calls[0][1] as RequestInit).body as string);
const second = JSON.parse((safeFetchLlmMock.mock.calls[1][1] as RequestInit).body as string);
expect(first.max_tokens).toBe(4096);
expect(first.max_completion_tokens).toBeUndefined();
// The retry swaps the token param but keeps everything else, json_schema included.
expect(second.max_completion_tokens).toBe(4096);
expect(second.max_tokens).toBeUndefined();
expect(second.response_format.type).toBe('json_schema');
expect(second.messages).toEqual(first.messages);
});
it('throws when the json_object retry also fails (400 twice)', async () => {
safeFetchLlmMock
.mockResolvedValueOnce(jsonResponse({ error: 'no json_schema' }, false, 400))