Stream-write JSON Lines (JSONL/NDJSON) to a file in Node.js safely using a temp file + atomic rename (no deps).

node-write-jsonl-atomic.js

#!/usr/bin/env node
/**
 * Stream-write JSONL (NDJSON) safely with atomic write.
 *
 * Pattern:
 * 1) Write to a temp file in the same directory
 * 2) fsync (best-effort) to reduce risk of partial writes
 * 3) Rename temp -> final (atomic on the same filesystem)
 *
 * Usage:
 *   node node-write-jsonl-atomic.js output.jsonl
 *
 * Optional:
 *   Pipe JSON objects (one per line) into this script:
 *     cat input.jsonl | node node-write-jsonl-atomic.js output.jsonl
 *
 * Or generate records in-code (see example).
 */

const fs = require("fs");
const path = require("path");
const crypto = require("crypto");
const readline = require("readline");

function usage() {
  console.error("Usage: node node-write-jsonl-atomic.js <output.jsonl>");
}

function tmpPathFor(finalPath) {
  const dir = path.dirname(finalPath);
  const base = path.basename(finalPath);
  const rand = crypto.randomBytes(6).toString("hex");
  return path.join(dir, `.${base}.tmp-${process.pid}-${rand}`);
}

async function fsyncFileHandle(fd) {
  // Best-effort fsync. On some environments it may not matter.
  try {
    await fd.sync();
  } catch {
    // ignore
  }
}

async function fsyncDir(dir) {
  // Best-effort directory fsync (helps ensure rename is persisted on some FS)
  // Not supported everywhere; safe to ignore errors.
  let dfd;
  try {
    dfd = await fs.promises.open(dir, "r");
    await dfd.sync();
  } catch {
    // ignore
  } finally {
    if (dfd) await dfd.close().catch(() => {});
  }
}

async function writeJSONLAtomic(finalPath, writeRecordsFn) {
  const absFinal = path.resolve(finalPath);
  const dir = path.dirname(absFinal);
  const tmp = tmpPathFor(absFinal);

  let fh;
  try {
    // Use 'wx' to avoid clobbering an existing temp file
    fh = await fs.promises.open(tmp, "wx");

    // Provide a writable stream backed by the file handle
    const stream = fh.createWriteStream({ encoding: "utf8" });

    // Let caller stream records into this file
    await writeRecordsFn(stream);

    // Ensure stream is finished
    await new Promise((resolve, reject) => {
      stream.end(() => resolve());
      stream.on("error", reject);
    });

    await fsyncFileHandle(fh);
    await fh.close();
    fh = null;

    // Atomic replace (same directory/filesystem)
    await fs.promises.rename(tmp, absFinal);

    // Best-effort: fsync directory entry
    await fsyncDir(dir);
  } catch (err) {
    // Cleanup temp file if anything fails
    try { if (fh) await fh.close(); } catch {}
    try { await fs.promises.unlink(tmp); } catch {}
    throw err;
  }
}

// ---------------------------------------------------------------------------
// Option A: Read JSONL from stdin and rewrite it atomically to a file
// ---------------------------------------------------------------------------
async function writeFromStdinJSONL(outStream) {
  const rl = readline.createInterface({
    input: process.stdin,
    crlfDelay: Infinity,
  });

  for await (const line of rl) {
    if (!line) continue;

    // Validate each line is valid JSON (optional but useful)
    JSON.parse(line);

    outStream.write(line + "\n");
  }
}

// ---------------------------------------------------------------------------
// Option B: Generate JSONL in code (example)
// ---------------------------------------------------------------------------
async function writeGeneratedJSONL(outStream) {
  for (let i = 1; i <= 5; i++) {
    const obj = { id: i, ts: new Date().toISOString() };
    outStream.write(JSON.stringify(obj) + "\n");
  }
}

async function main() {
  const outFile = process.argv[2];
  if (!outFile || outFile === "-h" || outFile === "--help") {
    usage();
    process.exit(outFile ? 0 : 2);
  }

  const isPiped = !process.stdin.isTTY;

  await writeJSONLAtomic(outFile, async (outStream) => {
    if (isPiped) {
      await writeFromStdinJSONL(outStream);
    } else {
      await writeGeneratedJSONL(outStream);
    }
  });

  console.error(`Wrote JSONL atomically to: ${path.resolve(outFile)}`);
}

main().catch((err) => {
  console.error("ERROR:", err && err.stack ? err.stack : err);
  process.exit(1);
});