Setapp AI streaming issue

verify-setapp-streaming.mjs

#!/usr/bin/env node

import crypto from 'node:crypto';

const env = process.env;

function printHelp() {
  console.log(`
Usage:
  node verify-setapp-streaming.mjs [options]

This script compares streaming timing between OpenAI-compatible targets.
It focuses on when SSE text deltas arrive, not just the final output.

Environment variables:
  PROMPT
  STREAM_MODE                 responses | chat   (default: responses)
  MAX_OUTPUT_TOKENS           (default: 256)
  TIMEOUT_MS                  (default: 120000)

  OPENAI_API_KEY
  OPENAI_MODEL
  OPENAI_BASE_URL             (default: https://api.openai.com/v1)

  SETAPP_API_KEY
  SETAPP_MODEL
  SETAPP_BASE_URL
  SETAPP_OPENAI_PATH          optional, e.g. v1/responses for legacy Setapp

Options:
  --prompt <text>
  --mode <responses|chat>
  --max-output-tokens <n>
  --timeout-ms <n>
  --openai-api-key <key>
  --openai-model <model>
  --openai-base-url <url>
  --setapp-api-key <key>
  --setapp-model <model>
  --setapp-base-url <url>
  --setapp-openai-path <path>
  --only <openai|setapp>
  --help

Examples:
  OPENAI_API_KEY=... OPENAI_MODEL=gpt-4.1-mini \\
  SETAPP_API_KEY=... SETAPP_MODEL=openai/gpt-5-mini \\
  SETAPP_BASE_URL=https://api.macpaw.com/ai/api/v1 \\
  PROMPT="Count from 1 to 50, one number per token-ish chunk." \\
  node verify-setapp-streaming.mjs

  SETAPP_API_KEY=... SETAPP_MODEL=gpt-5-mini \\
  SETAPP_BASE_URL=https://vendor-api.setapp.com/resource/v1/ai/openai \\
  SETAPP_OPENAI_PATH=v1/responses \\
  PROMPT="Write a short story slowly." \\
  node verify-setapp-streaming.mjs --only setapp
`.trim());
}

function parseArgs(argv) {
  const args = argv.slice(2);
  const result = {};

  for (let i = 0; i < args.length; i += 1) {
    const arg = args[i];
    if (arg === '--help') {
      result.help = true;
      continue;
    }

    if (!arg.startsWith('--')) {
      throw new Error(`Unknown positional argument: ${arg}`);
    }

    const key = arg.slice(2);
    const value = args[i + 1];
    if (value == null || value.startsWith('--')) {
      throw new Error(`Missing value for --${key}`);
    }
    result[key] = value;
    i += 1;
  }

  return result;
}

function toInt(value, fallback) {
  const parsed = Number.parseInt(String(value ?? ''), 10);
  return Number.isFinite(parsed) ? parsed : fallback;
}

function nowMs() {
  return Number(process.hrtime.bigint() / 1000000n);
}

function trimTrailingSlash(url) {
  return String(url ?? '').trim().replace(/\/+$/, '');
}

function buildRequestBody({ mode, model, prompt, maxOutputTokens }) {
  if (mode === 'chat') {
    return {
      model,
      stream: true,
      temperature: 0,
      max_tokens: maxOutputTokens,
      messages: [
        {
          role: 'user',
          content: prompt,
        },
      ],
    };
  }

  return {
    model,
    stream: true,
    max_output_tokens: maxOutputTokens,
    input: prompt,
  };
}

function buildTargetUrl(baseURL, mode, openaiPath) {
  const trimmedBaseURL = trimTrailingSlash(baseURL);
  if (!trimmedBaseURL) {
    throw new Error('Missing base URL');
  }

  if (openaiPath) {
    return trimmedBaseURL;
  }

  const endpoint = mode === 'chat' ? '/chat/completions' : '/responses';
  return `${trimmedBaseURL}${endpoint}`;
}

function parseSSEFrame(frame) {
  const lines = frame.split(/\r?\n/);
  let eventName = '';
  const dataLines = [];

  for (const line of lines) {
    if (!line || line.startsWith(':')) continue;
    if (line.startsWith('event:')) {
      eventName = line.slice(6).trim();
      continue;
    }
    if (line.startsWith('data:')) {
      dataLines.push(line.slice(5).trimStart());
    }
  }

  return {
    eventName: eventName || null,
    dataText: dataLines.join('\n'),
  };
}

function extractDeltaText(parsed) {
  if (!parsed || typeof parsed !== 'object') return '';

  if (typeof parsed.delta === 'string') {
    return parsed.delta;
  }

  const choiceDelta = parsed.choices?.[0]?.delta;
  if (typeof choiceDelta?.content === 'string') {
    return choiceDelta.content;
  }

  if (Array.isArray(choiceDelta?.content)) {
    return choiceDelta.content
      .map((part) => (typeof part?.text === 'string' ? part.text : ''))
      .join('');
  }

  if (Array.isArray(parsed.output)) {
    return parsed.output
      .flatMap((item) => (Array.isArray(item?.content) ? item.content : []))
      .map((part) => (typeof part?.text === 'string' ? part.text : ''))
      .join('');
  }

  return '';
}

function summarizeTimings(deltaEvents) {
  const gaps = [];
  for (let i = 1; i < deltaEvents.length; i += 1) {
    gaps.push(deltaEvents[i].atMs - deltaEvents[i - 1].atMs);
  }

  const sortedGaps = [...gaps].sort((a, b) => a - b);
  const percentile = (values, p) => {
    if (values.length === 0) return null;
    const index = Math.min(values.length - 1, Math.floor(values.length * p));
    return values[index];
  };

  return {
    gapCount: gaps.length,
    minGapMs: gaps.length > 0 ? Math.min(...gaps) : null,
    medianGapMs: percentile(sortedGaps, 0.5),
    p95GapMs: percentile(sortedGaps, 0.95),
    maxGapMs: gaps.length > 0 ? Math.max(...gaps) : null,
  };
}

async function collectStream(target) {
  const mode = target.mode;
  const requestBody = buildRequestBody(target);
  const requestUrl = buildTargetUrl(target.baseURL, mode, target.openaiPath);

  const headers = {
    'content-type': 'application/json',
    authorization: `Bearer ${target.apiKey}`,
  };
  if (target.openaiPath) {
    headers.OpenAIPath = target.openaiPath;
  }

  const startedAt = nowMs();
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), target.timeoutMs);

  let response;
  try {
    response = await fetch(requestUrl, {
      method: 'POST',
      headers,
      body: JSON.stringify(requestBody),
      signal: controller.signal,
    });
  } finally {
    clearTimeout(timer);
  }

  const responseStartedAt = nowMs();
  if (!response.body) {
    throw new Error(`No response body for ${target.name}`);
  }

  const decoder = new TextDecoder();
  const reader = response.body.getReader();
  let buffer = '';
  let rawText = '';
  let firstChunkAt = null;
  const events = [];
  const deltaEvents = [];
  let assembledText = '';

  while (true) {
    const { done, value } = await reader.read();
    const chunkAt = nowMs();

    if (done) {
      break;
    }

    if (firstChunkAt == null) {
      firstChunkAt = chunkAt;
    }

    const decoded = decoder.decode(value, { stream: true });
    rawText += decoded;
    buffer += decoded;

    let boundaryIndex = buffer.search(/\r?\n\r?\n/);
    while (boundaryIndex !== -1) {
      const frame = buffer.slice(0, boundaryIndex);
      const separatorMatch = buffer.slice(boundaryIndex).match(/^\r?\n\r?\n/);
      const separatorLength = separatorMatch ? separatorMatch[0].length : 2;
      buffer = buffer.slice(boundaryIndex + separatorLength);

      const parsedFrame = parseSSEFrame(frame);
      const atMs = chunkAt - startedAt;
      if (!parsedFrame.dataText) {
        boundaryIndex = buffer.search(/\r?\n\r?\n/);
        continue;
      }

      let parsedJSON = null;
      try {
        parsedJSON = JSON.parse(parsedFrame.dataText);
      } catch {}

      const eventType =
        parsedJSON?.type ||
        parsedFrame.eventName ||
        (parsedFrame.dataText === '[DONE]' ? 'done' : 'message');
      const deltaText = extractDeltaText(parsedJSON);

      events.push({
        atMs,
        eventType,
        deltaLength: deltaText.length,
      });

      if (deltaText.length > 0) {
        assembledText += deltaText;
        deltaEvents.push({
          atMs,
          deltaLength: deltaText.length,
        });
      }

      boundaryIndex = buffer.search(/\r?\n\r?\n/);
    }
  }

  const finishedAt = nowMs();
  const timingSummary = summarizeTimings(deltaEvents);

  return {
    name: target.name,
    requestUrl,
    status: response.status,
    ok: response.ok,
    headers: {
      contentType: response.headers.get('content-type'),
      transferEncoding: response.headers.get('transfer-encoding'),
      cacheControl: response.headers.get('cache-control'),
      server: response.headers.get('server'),
    },
    startedAt,
    responseStartedAt,
    firstByteMs: firstChunkAt == null ? null : firstChunkAt - startedAt,
    responseHeaderMs: responseStartedAt - startedAt,
    totalDurationMs: finishedAt - startedAt,
    totalEvents: events.length,
    totalDeltaEvents: deltaEvents.length,
    totalTextChars: assembledText.length,
    assembledTextSha256: crypto.createHash('sha256').update(assembledText).digest('hex'),
    assembledTextPreview: assembledText.slice(0, 160),
    timingSummary,
    firstEventMs: events[0]?.atMs ?? null,
    firstDeltaMs: deltaEvents[0]?.atMs ?? null,
    sampleDeltaTimeline: deltaEvents.slice(0, 12),
    rawPreview: rawText.slice(0, 400),
  };
}

function printSummary(result) {
  console.log(`\n=== ${result.name.toUpperCase()} ===`);
  console.log(`url: ${result.requestUrl}`);
  console.log(`status: ${result.status}`);
  console.log(`content-type: ${result.headers.contentType ?? '-'}`);
  console.log(`transfer-encoding: ${result.headers.transferEncoding ?? '-'}`);
  console.log(`response headers: ${result.responseHeaderMs}ms`);
  console.log(`first byte: ${result.firstByteMs ?? '-'}ms`);
  console.log(`first SSE event: ${result.firstEventMs ?? '-'}ms`);
  console.log(`first text delta: ${result.firstDeltaMs ?? '-'}ms`);
  console.log(`total duration: ${result.totalDurationMs}ms`);
  console.log(`events: ${result.totalEvents}`);
  console.log(`text deltas: ${result.totalDeltaEvents}`);
  console.log(`text chars: ${result.totalTextChars}`);
  console.log(`sha256: ${result.assembledTextSha256}`);
  if (result.timingSummary.gapCount > 0) {
    console.log(
      `delta gaps ms: min=${result.timingSummary.minGapMs}, median=${result.timingSummary.medianGapMs}, p95=${result.timingSummary.p95GapMs}, max=${result.timingSummary.maxGapMs}`,
    );
  }
  if (result.sampleDeltaTimeline.length > 0) {
    console.log('first delta events:');
    for (const item of result.sampleDeltaTimeline) {
      console.log(`  t=${item.atMs}ms chars=${item.deltaLength}`);
    }
  }
  if (result.assembledTextPreview) {
    console.log(`text preview: ${JSON.stringify(result.assembledTextPreview)}`);
  }
}

function printComparison(results) {
  if (results.length < 2) return;

  console.log('\n=== COMPARISON ===');
  for (const result of results) {
    console.log(
      `${result.name}: firstDelta=${result.firstDeltaMs ?? '-'}ms, totalDuration=${result.totalDurationMs}ms, deltas=${result.totalDeltaEvents}, maxGap=${result.timingSummary.maxGapMs ?? '-'}ms`,
    );
  }

  const uniqueHashes = new Set(results.map((result) => result.assembledTextSha256));
  console.log(`same final text: ${uniqueHashes.size === 1 ? 'yes' : 'no'}`);
}

async function main() {
  const args = parseArgs(process.argv);
  if (args.help) {
    printHelp();
    return;
  }

  const mode = String(args.mode ?? env.STREAM_MODE ?? 'responses').trim().toLowerCase();
  if (mode !== 'responses' && mode !== 'chat') {
    throw new Error(`Unsupported mode: ${mode}`);
  }

  const prompt = String(
    args.prompt ??
      env.PROMPT ??
      'Write the numbers 1 through 40, with a short dash explanation after each number.',
  );
  const maxOutputTokens = toInt(args['max-output-tokens'] ?? env.MAX_OUTPUT_TOKENS, 256);
  const timeoutMs = toInt(args['timeout-ms'] ?? env.TIMEOUT_MS, 120000);
  const only = args.only ? String(args.only).trim().toLowerCase() : null;

  const targets = [
    {
      name: 'openai',
      apiKey: args['openai-api-key'] ?? env.OPENAI_API_KEY,
      model: args['openai-model'] ?? env.OPENAI_MODEL,
      baseURL: args['openai-base-url'] ?? env.OPENAI_BASE_URL ?? 'https://api.openai.com/v1',
      openaiPath: null,
      prompt,
      mode,
      maxOutputTokens,
      timeoutMs,
    },
    {
      name: 'setapp',
      apiKey: args['setapp-api-key'] ?? env.SETAPP_API_KEY,
      model: args['setapp-model'] ?? env.SETAPP_MODEL,
      baseURL: args['setapp-base-url'] ?? env.SETAPP_BASE_URL,
      openaiPath: args['setapp-openai-path'] ?? env.SETAPP_OPENAI_PATH ?? null,
      prompt,
      mode,
      maxOutputTokens,
      timeoutMs,
    },
  ]
    .filter((target) => (only ? target.name === only : true))
    .filter((target) => target.apiKey && target.model && target.baseURL);

  if (targets.length === 0) {
    printHelp();
    throw new Error('No runnable targets configured. Provide API key, model, and base URL.');
  }

  const results = [];
  for (const target of targets) {
    console.log(`\nRunning ${target.name}...`);
    const result = await collectStream(target);
    printSummary(result);
    results.push(result);
  }

  printComparison(results);
}

main().catch((error) => {
  console.error(`\nERROR: ${error?.message || String(error)}`);
  process.exit(1);
});

Results

OpenAI

== OPENAI ===
url: https://api.openai.com/v1/responses
status: 200
content-type: text/event-stream; charset=utf-8
transfer-encoding: chunked
response headers: 822ms
first byte: 823ms
first SSE event: 823ms
first text delta: 1299ms
total duration: 5434ms
events: 260
text deltas: 252
text chars: 798
sha256: ebe496bc3356c04a847c0bd687a1b49f95a5006d12ac787a73daee2cc397a0d5
delta gaps ms: min=0, median=2, p95=66, max=143
first delta events:
  t=1299ms chars=1
  t=1299ms chars=1
  t=1335ms chars=4
  t=1336ms chars=5
  t=1380ms chars=8
  t=1420ms chars=1
  t=1420ms chars=3
  t=1423ms chars=1
  t=1423ms chars=1
  t=1458ms chars=5
  t=1469ms chars=3
  t=1527ms chars=6
text preview: "1. One step forward.  \n2. Keep it going.  \n3. Small wins count.  \n4. Stay focused.  \n5. Nice progress.  \n6. Breathe and continue.  \n7. Momentum builds.  \n8. Alm"

Setapp old endpoint

url: https://vendor-api.setapp.com/resource/v1/ai/openai
status: 200
content-type: text/event-stream; charset=UTF-8
transfer-encoding: chunked
response headers: 7010ms
first byte: 7010ms
first SSE event: 7011ms
first text delta: 7011ms
total duration: 7434ms
events: 262
text deltas: 253
text chars: 908
sha256: 92c2707cb0207b429bb06d3ee20ff18869b1a0dcb90e38f24a6232a52487e16e
delta gaps ms: min=0, median=0, p95=0, max=263
first delta events:
  t=7011ms chars=1
  t=7011ms chars=1
  t=7011ms chars=4
  t=7011ms chars=2
  t=7011ms chars=8
  t=7011ms chars=8
  t=7011ms chars=1
  t=7011ms chars=3
  t=7011ms chars=1
  t=7012ms chars=1
  t=7012ms chars=4
  t=7012ms chars=2
text preview: "1. One — getting started.  \n2. Two — keep going.  \n3. Three — finding rhythm.  \n4. Four — steady pace.  \n5. Five — first milestone.  \n6. Six — staying focused. "

Setapp new endpoint:

url: https://api.macpaw.com/ai/api/v1/responses
status: 200
content-type: text/event-stream
transfer-encoding: chunked
response headers: 6046ms
first byte: 6048ms
first SSE event: 6048ms
first text delta: 6048ms
total duration: 6051ms
events: 262
text deltas: 253
text chars: 927
sha256: 832a3d0227ecb39c803872fd0e228bc206bddec1f22dab2297dadd4b3320fd42
delta gaps ms: min=0, median=0, p95=0, max=1
first delta events:
  t=6048ms chars=1
  t=6048ms chars=1
  t=6048ms chars=4
  t=6048ms chars=5
  t=6048ms chars=8
  t=6048ms chars=1
  t=6048ms chars=3
  t=6048ms chars=1
  t=6048ms chars=1
  t=6048ms chars=4
  t=6048ms chars=6
  t=6048ms chars=5
text preview: "1. One step forward.  \n2. Two quick taps.  \n3. Three deep breaths.  \n4. Four steady beats.  \n5. Five seconds of focus.  \n6. Six bright ideas.  \n7. Seven quiet m"