Robust, ditto-like deduplicating recursive copy for macOS/APFS

apfs-dcopy.c

// ============================================================================
// dcopy — Robust, APFS-aware, ditto-like recursive copy for macOS
// ============================================================================
//
// dcopy is a robust, mission-grade copy/clone engine for macOS/APFS. It marries
// fast, low-syscall enumeration with parallel hashing and IO, opportunistic APFS
// clone paths, sparse-aware streaming, and multi-strategy deduplication. It
// prioritizes crash safety (temp-file + fsync + atomic rename), correctness, and
// predictable performance on very large trees.
//
// Problem statement (why this exists)
// -----------------------------------
// Moving multi-terabyte, high-value datasets on APFS efficiently and safely is
// non-trivial:
//
//   * Traditional tools either don’t exploit APFS clone-on-write (CoW) or do so
//     inconsistently across versions and mount types.
//   * rsync/cp may stream bytes even when an in-filesystem clone would suffice.
//   * Sparse files, hardlinks, xattrs, and directory durability are often
//     handled in ways that trade safety for speed, or vice versa, without
//     transparent control.
//   * Deduplication is typically content-hash only; APFS also enables physical
//     extent-based opportunities on a single volume.
//
// dcopy addresses these gaps with explicit APFS-aware fast paths, strict crash
// safety by default, and tunable concurrency/IO budgets.
//
// Contrast with ditto / cp / rsync / scp
// --------------------------------------
// * ditto: Good metadata handling on macOS; APFS clone usage varies by version
//   and options. dcopy always attempts clonefile(2)/fclonefileat(2) on same-FS,
//   else falls back. dcopy also adds physical-extent dedupe and persistent hash
//   caching.
// * cp: On macOS, `cp -c`/`--clone` exists on some versions; behavior differs
//   across releases. dcopy detects and falls back automatically, and adds
//   sparse-aware streaming, parallelism, and durable temp-rename semantics.
// * rsync: Powerful delta/remote sync; expensive on local CoW filesystems if
//   cloning would suffice. rsync’s metadata/durability knobs are numerous and
//   non-APFS-specific. dcopy is single-host/local only, APFS-aware, and built
//   for large local moves with strict atomicity/fdatasync discipline.
// * scp: Network copy; not CoW-aware, not sparse-aware on the remote FS.
//   dcopy focuses on local APFS volumes.
//
// APFS primer (concise)
// ---------------------
// * APFS is CoW: metadata and data are clone-on-write. File “cloning” creates a
//   new inode whose logical blocks reference the same physical extents as the
//   source. Writes to either inode copy-on-write new blocks, diverging storage.
// * APFS clone syscalls: clonefile(2)/fclonefileat(2) create a logical clone
//   within the same APFS volume; cross-volume returns EXDEV.
// * Sparse files: APFS supports holes. The logical size can far exceed allocated
//   physical bytes. On Darwin, SEEK_DATA/SEEK_HOLE and F_LOG2PHYS_EXT expose
//   sparsity and logical→physical extent mapping.
// * Integrity: rename(2) is atomic within a filesystem; combined with fsync on
//   file and directory, it provides strong crash-consistency for file creation.
//
// Interfaces used (Darwin/XNU & friends)
// --------------------------------------
// * Enumeration: getattrlistbulk(2) (fast batched metadata) with adaptive masks;
//   per-subtree fallback to FTS if EINVAL or partial support.
// * Cloning: clonefile(2)/fclonefileat(2) (APFS-native).
// * Kernel copy helpers: copyfile(3)/fcopyfile(3) with COPYFILE_CLONE where
//   possible, falling back to COPYFILE_ALL.
// * Sparse IO: SEEK_DATA/SEEK_HOLE scanning; F_PREALLOCATE to reserve space;
//   F_NOCACHE on both fds; F_RDAHEAD / F_RDADVISE for read-ahead hints.
// * Physical extent fingerprinting: fcntl(F_LOG2PHYS_EXT) to walk logical→
//   physical extents and derive a compact, volume-local “phys-fp” key.
// * Durability: fsync(2), fcntl(F_FULLFSYNC) when available; renameatx_np with
//   RENAME_EXCL when no-clobber is desired.
// * iCloud: SF_DATALESS flag filtering to avoid materialization (opt-in).
//
// Architectural overview
// ----------------------
// Phases (high-level):
//  1) Enumerate: Traverse all roots (bulk API; per-subtree FTS fallback). Build
//     a flat vector of Entry {path, rel, type, size, times, dev, ino, nlink}.
//  2) Pre-analysis:
//     a) Size binning: stable sort by size; sum bytes_total.
//     b) Sampling: for size-duplicate groups, compute 64-bit sample (head/mid/
//        tail XXH3). Groups with identical (size,sample) are candidates for hash.
//     c) Physical fingerprints: for groups with ≥3 items and size≥threshold,
//        compute phys-fp via F_LOG2PHYS_EXT (opportunistic dedupe on same volume).
//     d) Hashing: for candidate groups, compute XXH3-128 digests in parallel,
//        reusing a persistent append-only hash cache keyed by (dev, ino, size,
//        mtime, ctime).
//  3) Scheduling:
//     a) Emit directory creation jobs first, to construct tree topology.
//     b) Emit file jobs sorted by size descending (improve early throughput).
//     c) Emit “other” (symlinks/..). All jobs flow into an IO job queue.
//  4) IO execution (N workers):
//     a) For each file: try fast paths in order—same-volume clone; anchor-clone
//        by phys-fp; anchor-clone by content hash; kernel copy; sparse streaming;
//        dense streaming. All writes go to a uniquely named temp file in the
//        target dir, then fsync + atomic rename.
//     b) Hardlinks: track first “appearance” of (dev, ino) and link subsequent
//        entries to maintain hardlink topology.
//     c) Metadata: for files, copy with copyfile(COPYFILE_METADATA). Directories
//        obey the selected metadata policy (full|safe|minimal).
//     d) Durability: strict mode fsyncs target directory after rename; batch mode
//        defers per-directory fsync using a shard-local budget.
//     e) Optional deletion: if enabled, verify dst vs src (size|sample|hash), and
//        unlink the source only on successful verification (never when src==dst).
//
// Internal subsystems:
// * Persistent hash cache: append-only binary log on disk for XXH128 digests,
//   cheap in-memory index keyed by (dev, ino, size, mtimes). Avoids rehashing.
// * Three sharded lock-free-style maps (with per-shard mutex):
//    - DigestMap (content hash → anchor dst path), used for dedupe clone.
//    - InoMap    ((dev,ino)   → first dst path), used for hardlink preservation.
//    - PhysMap   (phys-fp     → anchor dst path), used for physical clone dedupe.
//   Sharding minimizes lock contention under high concurrency.
// * Token-bucket inflight limiter: global cap on concurrently “in-flight” bytes
//   across workers to bound page-cache and memory pressure.
// * Buffer pool: fixed-size aligned read buffers per IO worker to reduce malloc/
//   free churn and improve cache locality.
// * Directory FD cache: small LRU for parent dir fds to reduce open/close/fsync
//   overhead when many files land in the same directory.
// * Progress/logging: a dedicated progress updater renders a single status line;
//   all human logs (COPY/CLONE/LINK/DEL) synchronize on a global log mutex to
//   avoid interleaving.
//
// Failure handling & fallbacks
// ---------------------------
// * getattrlistbulk(2) → per-subtree FTS on EINVAL/incompatibility.
// * clonefile(2)/fclonefileat(2) → copyfile()/streaming on EXDEV/ENOTSUP/EPERM.
// * SEEK_DATA/SEEK_HOLE absent → dense streaming copy.
// * F_PREALLOCATE may fail (e.g., network FS) → best-effort, continue.
// * copyfile()/fcopyfile() errors → streaming fallback.
// * Hashing errors → mark un-hashed; copy still proceeds.
// * Deletion is strictly gated by verification and inode inequality (src!=dst).
//
// Safety & correctness model
// --------------------------
// * Writes use temp files (unique per destination directory). File content is
//   fully written and fsync’d (F_FULLFSYNC if available) before atomic rename.
// * In strict mode, parent directory is fsync’d after rename (and after mkdir).
// * No-clobber by default. `--overwrite` and `--update` opt into replacement.
// * Hardlinks preserved by default; this is essential for correctness on trees
//   with multi-link files (e.g., software installations).
// * Sparse semantics preserved unless `--no-sparse` is set.
// * Optional deletion uses a selectable verification policy:
//      VERIFY_SIZE   (fastest; equal size only),
//      VERIFY_SAMPLE (XXH3-64 of head/mid/tail; low FP risk, not cryptographic),
//      VERIFY_HASH   (XXH3-128 over full file; fast and strong; default).
// * iCloud dataless entries (SF_DATALESS) are skipped by default to avoid forced
//   network materialization; opt-in via `--include-icloud`.
//
// Performance characteristics (big-O-ish & practical)
// ---------------------------------------------------
// * Enumeration: O(n) with large constant improvements vs FTS via bulk batched
//   syscalls. Per-record lstat only where necessary for fidelity.
// * Sampling & hashing: O(k) over candidate groups; parallelized across CPUs.
// * Scheduling: O(n log n) sort by size (qsort), then stable bucket operations.
// * IO: bounded by inflight cap and per-worker buffer; throughput typically gated
//   by storage and clone support.
//
// Tunables (compile-time & constants)
// -----------------------------------
// * DCOPY_CACHELINE (default 64)     : alignment/padding for shard structs.
// * READ_BUFSZ_DEFAULT (4 MiB)       : per-IO buffer size default.
// * DIGEST_LEN (16)                  : XXH3-128 length.
// * MAX_THREADS (128)                : guardrail for worker counts.
// * DEFAULT_RDADVISE_MB (32)         : read-ahead window in MiB.
// * DEFAULT_INFLIGHT_MB (512)        : global inflight-bytes cap in MiB.
// * DIRFD_CACHE_MAX (64)             : LRU entries for directory FD cache.
// * PHYS_FP_MIN_SIZE (1 MiB)         : minimum file size for phys-fp pass.
// * MAP_SHARDS (64)                  : shard count for Digest/INO/Phys maps;
//   load factor thresholds (MAP_LF_NUM/DEN) control rehashing.
// * DMAP_INIT_TOTAL / IMAP_INIT_TOTAL: initial table sizing hints.
// * TMP_SUFFIX (\".dctmp\")           : temp file marker.
// * Feature guards: HAVE_RENAMEATX_NP, RENAME_EXCL, SEEK_DATA/HOLE defines,
//   XXH_STATIC_LINKING_ONLY, XXH_IMPLEMENTATION.
// Adjust in code if building for atypical environments (older SDKs, etc.).
//
// Defaults (runtime)
// ------------------
// * Metadata policy: SAFE (directories get mode only; file metadata always copied)
// * Durability: STRICT (file fsync + dir fsync after rename)
// * Enumeration: BULK (getattrlistbulk) with FTS fallback
// * Sparse: enabled; kernel copy: opt-in (`--kernel-copy`)
// * Hashing threads: = online CPUs (capped); IO threads: = min(4, CPUs)
// * Inflight cap: 512 MiB; per-IO buffer: 4 MiB; read-ahead: 32 MiB
// * Hardlinks preserved; iCloud dataless skipped; no-clobber by default
// * Deletion disabled unless `--delete-source` / `--delete-source-existing`
// * Delete verification default: HASH
//
// Data integrity notes
// --------------------
// * rename is atomic on the same filesystem; combined with fsync discipline it
//   ensures that after success, either the old file or the new complete file is
//   visible (never a partial). Strict mode fsyncs the parent dir to persist the
//   name binding. Batch mode coalesces directory fsyncs per budget.
// * Source deletion is never performed without a positive verification and
//   explicit user opt-in. The tool refuses to delete when src/dst are the same
//   inode (safeguard).
//
// Build
// -----
//   clang -std=c23 -O3 -ffast-math -fomit-frame-pointer -foptimize-sibling-calls \
//         -fno-stack-protector -fno-strict-overflow -fno-use-cxa-atexit -flto \
//         -Wall -Wextra -pthread -o dcopy dcopy.c
//
// Portability & environment
// -------------------------
// * Requires macOS with APFS for clone fast-paths and F_LOG2PHYS_EXT. Falls back
//   to portable paths on other filesystems/mounts (e.g., SMB/NFS).
// * copyfile(3) semantics on network mounts can vary; sparse and preallocation
//   hints may be ignored by remote servers. The engine adapts without failing.
// * F_FULLFSYNC behavior depends on underlying device driver; fsync fallback is
//   used when unavailable.
//
// Operational guidance (for developers)
// -------------------------------------
// * For maximum safety on mission-critical moves, use a two-pass approach:
//     1) run without deletion, verify results, 2) re-run with --delete-source
//        or --delete-source-existing, keeping VERIFY_HASH.
//
// * Increase --io-threads for many small files on SSD/NVMe. Increase --inflight
//   for deep NVMe queues, but watch memory pressure. For HDDs, conservative
//   thread counts typically perform better.
//
// * When measuring performance, disable excessive verbosity (-v) as log
//   serialization is deliberate and can limit throughput on very chatty runs.
//
// Internal ASCII sketch
// ---------------------
//   [Enumerate] → [Size bin] → [Sample groups] → [Phys-fp (optional)] → [Hash]
//        |                                                         |
//        └──> [Dirs → IOQ]   [Files(size↓) → IOQ]   [Other → IOQ]  |
//                                   ↓                              |
//                               [IO workers] — clone → copyfile → streaming
//                                   ↓
//                              [fsync + atomic rename] → [optional delete]
//
// License & support
// -----------------
// * Freely distributed. No warranty. No support. Use at your own risk. Read
//   the usage/help (-h) for detailed operational semantics and defaults.
//
// © 2025

#define _DARWIN_C_SOURCE 1

#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <float.h>
#include <limits.h>
#include <math.h>
#include <pthread.h>
#include <signal.h>
#include <stdarg.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdnoreturn.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include <stdalign.h>

#include <fts.h>

#include <sys/attr.h>
#include <sys/clonefile.h>
#include <sys/mount.h>   /* statfs / f_fstypename */
#include <sys/param.h>
#include <sys/resource.h>
#include <sys/stat.h>
#include <sys/sysctl.h>  /* hw.memsize for RAM-based inflight clamping */
#include <sys/time.h>
#include <sys/types.h>
#include <sys/vnode.h>

#include <copyfile.h>

// Use xxHash as a header-only library
#define XXH_STATIC_LINKING_ONLY 1
#define XXH_IMPLEMENTATION 1
#include "xxhash.h"

// Darwin extensions (renameatx_np flags).
#ifndef RENAME_EXCL
#define RENAME_EXCL 0x00000004u
#endif
#ifndef HAVE_RENAMEATX_NP
#define HAVE_RENAMEATX_NP 1
#endif
extern int renameatx_np(int fromfd, const char *from, int tofd, const char *to, unsigned int flags);

// SEEK_DATA/SEEK_HOLE (Solaris/FreeBSD/Darwin extension).
#ifndef SEEK_DATA
#define SEEK_DATA 3
#endif
#ifndef SEEK_HOLE
#define SEEK_HOLE 4
#endif

#if defined(__GNUC__) || defined(__clang__)
#define DCOPY_ALIGN_TYPE(n) __attribute__((aligned(n)))
#elif defined(_MSC_VER)
#define DCOPY_ALIGN_TYPE(n) __declspec(align(n))
#else
#define DCOPY_ALIGN_TYPE(n)
#endif

#ifndef DCOPY_CACHELINE
#define DCOPY_CACHELINE 64
#endif

// ============================ Config / defaults ============================

#define READ_BUFSZ_DEFAULT (4 << 20)        // 4 MiB
#define DIGEST_LEN 16                       // XXH128 is 128 bits = 16 bytes
#define TMP_SUFFIX ".dctmp"
#define MAX_THREADS 128
#define DEFAULT_RDADVISE_MB 32              // 32 MiB
#define DEFAULT_INFLIGHT_MB 512             // 512 MiB
#define DIRFD_CACHE_MAX 64
#define PHYS_FP_MIN_SIZE (1 << 20)          // 1 MiB: bound phys-fingerprint pass
#define MAP_SHARDS 64u
#define MAP_LF_NUM 7u
#define MAP_LF_DEN 10u
#define DMAP_INIT_TOTAL (1u << 14)
#define IMAP_INIT_TOTAL (1u << 13)
#define MIN_SHARD_CAP 16u
#define DIRB_SHARDS 64

static int g_verbose = 0;
static int g_quiet = 0;
static int g_progress_enabled = 1;

// ============================ Logging / errors ============================

static pthread_mutex_t g_log_mu = PTHREAD_MUTEX_INITIALIZER;

static noreturn void die(const char *fmt, ...) {
    va_list ap;
    va_start(ap, fmt);
    pthread_mutex_lock(&g_log_mu);
    fprintf(stderr, "fatal: ");
    vfprintf(stderr, fmt, ap);
    fprintf(stderr, "\n");
    fflush(stderr);
    pthread_mutex_unlock(&g_log_mu);
    va_end(ap);
    exit(2);
}

static void warnx(const char *fmt, ...) {
    va_list ap;
    va_start(ap, fmt);
    pthread_mutex_lock(&g_log_mu);
    fprintf(stderr, "warn: ");
    vfprintf(stderr, fmt, ap);
    fprintf(stderr, "\n");
    fflush(stderr);
    pthread_mutex_unlock(&g_log_mu);
    va_end(ap);
}

static void vmsg(const char *fmt, ...) {
    if (g_verbose == 0) return;
    va_list ap;
    va_start(ap, fmt);
    pthread_mutex_lock(&g_log_mu);
    vfprintf(stderr, fmt, ap);
    fprintf(stderr, "\n");
    fflush(stderr);
    pthread_mutex_unlock(&g_log_mu);
    va_end(ap);
}

/* --- File-operation prefix test (for conditional verbosity) ----------------- */
static inline int is_fileop_prefix(const char *fmt) {
    /* skip leading spaces/tabs just in case */
    while (*fmt == ' ' || *fmt == '\t') fmt++;
    /* match the operation tags we emit: COPY / CLONE / LINK / DEL / PLAN */
    return (strncmp(fmt, "COPY", 4)  == 0) ||
           (strncmp(fmt, "CLONE", 5) == 0) ||
           (strncmp(fmt, "LINK", 4)  == 0) ||
           (strncmp(fmt, "DEL", 3)   == 0) ||
           (strncmp(fmt, "PLAN", 4)  == 0);
}

/* ============================ Logging / info lines ========================== */
/* NOTE: File-operation lines (COPY/CLONE/LINK/DEL/PLAN …) only print when -v. */
static void pmsg(const char *fmt, ...) {
    if (g_quiet) return;

    /* Suppress noisy per-file ops unless verbose is enabled */
    if (g_verbose == 0 && is_fileop_prefix(fmt)) {
        return;
    }

    va_list ap;
    va_start(ap, fmt);
    pthread_mutex_lock(&g_log_mu);
    if (g_progress_enabled) {
        /* clear any dynamic progress line */
        fprintf(stderr, "\r%*s\r", 120, "");
    }
    vfprintf(stderr, fmt, ap);
    fprintf(stderr, "\n");
    fflush(stderr);
    pthread_mutex_unlock(&g_log_mu);
    va_end(ap);
}

/* ============================ Autotune status line ============================ */
/* Always-on status line (ignores g_quiet) to avoid "stuck" appearance. */
static void bench_statusf(const char *fmt, ...)
{
    va_list ap;
    va_start(ap, fmt);
    pthread_mutex_lock(&g_log_mu);
    fprintf(stderr, "\r");                 /* begin line */
    vfprintf(stderr, fmt, ap);
    fprintf(stderr, "  ");                 /* minimal padding to clear remnants */
    fflush(stderr);
    pthread_mutex_unlock(&g_log_mu);
    va_end(ap);
}

/* End the current status line with a newline (only when appropriate). */
static void bench_status_endline(void)
{
    pthread_mutex_lock(&g_log_mu);
    fprintf(stderr, "\n");
    fflush(stderr);
    pthread_mutex_unlock(&g_log_mu);
}

/* Outer-run context for status; set from orchestrator before each run. */
static int g_bench_outer_idx  = 0;   /* 1-based */
static int g_bench_outer_total = 1;  /* total runs */

// ============================ CLI options ============================

typedef enum { CLOBBER_NO = 0, CLOBBER_OVERWRITE = 1, CLOBBER_UPDATE = 2 } ClobberMode;
typedef enum { META_FULL = 0, META_SAFE = 1, META_MINIMAL = 2 } MetaMode;
typedef enum { DUR_STRICT = 0, DUR_BATCH = 1 } DurabilityMode;
typedef enum { ENUM_FTS = 0, ENUM_BULK = 1 } EnumMode;
typedef enum { VERIFY_SIZE = 0, VERIFY_SAMPLE = 1, VERIFY_HASH = 2 } VerifyMode;

typedef struct {
    /* Core behavior */
    bool            dry_run;
    bool            follow_symlinks;
    bool            preserve_hardlinks;
    bool            progress;
    bool            include_icloud;
    bool            one_file_system;
    bool            use_kernel_copy;
    bool            sparse_copy;

    /* Deletion model */
    bool            delete_source;            /* --delete-source */
    bool            delete_source_existing;   /* --delete-source-existing */
    VerifyMode      delete_verify;            /* --delete-verify size|sample|hash */

    /* Tunables */
    int             jobs_hash;
    int             jobs_io;
    ClobberMode     clobber;
    MetaMode        meta_mode;
    DurabilityMode  dur_mode;
    EnumMode        enum_mode;
    size_t          buf_size;
    size_t          rdadvise_window;
    size_t          inflight_cap;
    size_t          fsync_batch_files;
    size_t          fsync_batch_bytes;

    /* ===================== AUTOTUNE / BENCH KNOBS ===================== */
    bool            autotune;           /* --autotune */
    int             bench_seconds;      /* --bench-seconds */
    uint64_t        bench_byte_budget;  /* --bench-mib */
    size_t          bench_large_cap;    /* --bench-large */
    size_t          bench_small_cap;    /* --bench-small */
    bool            bench_show_table;   /* --bench-table */
    bool            bench_exhaustive;   /* --bench-exhaustive */
    size_t          bench_max_combos;   /* --bench-max */

    /* Search / noise control */
    double          bench_ci_mibs_pct;  /* --bench-ci-mibs (95% CI target) */
    double          bench_ci_files_pct; /* --bench-ci-files (95% CI target) */
    bool            bench_fast;         /* --bench-fast */
    int             bench_eta;          /* --bench-eta */

    /* outer repetitions (run the full autotune K times) */
    int             bench_runs;         /* --bench-runs (default 1) */

    /* Internal path forcing during benchmark runs */
    bool            force_streaming;
    bool            force_kernel_only;
} Options;

static int cpu_count(void) {
    long n = sysconf(_SC_NPROCESSORS_ONLN);
    if (n < 1) n = 1;
    if (n > MAX_THREADS) n = MAX_THREADS;
    return (int)n;
}

static bool parse_u64(const char *s, uint64_t *out) {
    if (!s || !*s) return false;
    char *end = NULL;
    errno = 0;
    unsigned long long v = strtoull(s, &end, 10);
    if (errno || end == s || *end != '\0') return false;
    *out = (uint64_t)v;
    return true;
}

static void usage(FILE *f) {
    fprintf(f,
        "SYNOPSIS\n"
        "  dcopy [options] <src> [<src> ...] <dst>\n"
        "\n"
        "OVERVIEW\n"
        "  dcopy is a robust, APFS-aware recursive copy tool focused on durability\n"
        "  (temp-file + fsync + atomic rename), minimal data movement (clone paths,\n"
        "  sparse-aware streaming, hash/physical dedupe), and predictable performance.\n"
        "\n"
        "ARGUMENTS\n"
        "  <src> ...    One or more input roots.\n"
        "  <dst>        Destination file/dir (dir if multiple sources or src is dir).\n"
        "\n"
        "COMMON OPTIONS\n"
        "  -n                       Dry-run (plan only).\n"
        "  -v                       Verbose; repeatable.\n"
        "  -q                       Quiet (no progress / non-error logs).\n"
        "  -j N                     Hashing threads (default: online CPUs).\n"
        "  --io-threads N           Copy/clone workers (default: min(4, CPUs)).\n"
        "  --overwrite              Replace existing files atomically.\n"
        "  --update                 Replace only if newer or size differs.\n"
        "  --follow-symlinks        Copy symlink targets instead of links.\n"
        "  --no-hardlinks           Do not preserve hardlink topology.\n"
        "  --no-progress            Disable dynamic progress rendering.\n"
        "  --no-sparse              Force dense streaming copy.\n"
        "  --one-file-system        Do not cross filesystem boundaries.\n"
        "  --include-icloud         Include iCloud dataless placeholders.\n"
        "  --kernel-copy            Prefer fcopyfile(3) kernel-assisted copy.\n"
        "  --metadata MODE          Directory metadata: full|safe|minimal.\n"
        "  --durability MODE        strict|batch.\n"
        "  --enum MODE              bulk|fts.\n"
        "  --buf-size MB            Per-IO-thread read buffer (default: 4).\n"
        "  --rdadvise MB            Read-ahead advisory window (default: 32).\n"
        "  --inflight MB            Global in-flight IO cap (default: 512).\n"
        "  --fsync-batch N          (batch) fsync dir after N children (default: 128).\n"
        "  --fsync-bytes MB         (batch) fsync dir after MB bytes (default: 128).\n"
        "\n"
        "DELETION & VERIFICATION\n"
        "  --delete-source          After verify, delete the source file.\n"
        "  --delete-source-existing If dst exists and matches, delete source.\n"
        "  --delete-verify M        size | sample | hash  (default: hash).\n"
        "\n"
        "AUTOTUNE / BENCHMARKING (non-destructive)\n"
        "  --autotune               Intelligent search over tunables on representative samples.\n"
        "  --bench-seconds SEC      Soft time budget for the whole search (default: 35; 5..600).\n"
        "  --bench-mib MiB          Byte budget per combo (default: 512; 64..16384).\n"
        "  --bench-large N          Max large files in the sample (default: 12; 1..256).\n"
        "  --bench-small N          Max small (<=1 MiB) files (default: 1024; 1..32768).\n"
        "  --bench-table            Also print a ranked table of tested combinations.\n"
        "  --bench-exhaustive       Test ALL parameter combinations; ignores time/max.\n"
        "  --bench-max N            Upper-bound the number of combos to evaluate (0 = no cap).\n"
        "  --bench-ci-mibs PCT      Target 95%% CI half-width for MiB/s (default: 3.0).\n"
        "  --bench-ci-files PCT     Target 95%% CI half-width for files/s (default: 10.0).\n"
        "  --bench-fast             Reuse cached recommendation if profile matches.\n"
        "  --bench-eta N            Successive Halving factor (default: 3).\n"
        "  --bench-runs N           Run the entire autotune N times and aggregate (default: 1; 1..50).\n"
        "\n"
        "EXAMPLES\n"
        "  dcopy --autotune --bench-table /SRC /DST\n"
        "  dcopy --autotune --bench-runs 5 --bench-table /SRC /DST\n"
        "\n"
    );
}

// ============================ Helpers ============================

static char *xstrdup(const char *s) {
    size_t n = strlen(s) + 1;
    char *p = (char *)malloc(n);

    if (!p) {
        die("OOM");
    }

    memcpy(p, s, n);

    return p;
}

static void path_dirname(const char *path, char *out, size_t outsz) {
    const char *slash = strrchr(path, '/');
    if (!slash) {
        snprintf(out, outsz, ".");
        return;
    }
    if (slash == path) {
        snprintf(out, outsz, "/");
        return;
    }
    size_t len = (size_t)(slash - path);
    if (len >= outsz) len = outsz - 1;
    memcpy(out, path, len);
    out[len] = '\0';
}

static const char *path_basename(const char *path) {
    const char *slash = strrchr(path, '/');
    return slash ? slash + 1 : path;
}

static void path_join2(const char *a, const char *b, char *out, size_t n) {
    if (!a || !*a) {
        snprintf(out, n, "%s", b ? b : "");
        return;
    }
    if (!b || !*b) {
        snprintf(out, n, "%s", a);
        return;
    }
    size_t la = strlen(a);
    bool need_slash = a[la - 1] != '/';
    snprintf(out, n, "%s%s%s", a, need_slash ? "/" : "", b);
}

static int fsync_full(int fd) {
#ifdef F_FULLFSYNC
    if (fcntl(fd, F_FULLFSYNC) == 0) {
        return 0;
    }
#endif
    return fsync(fd);
}

/* Treat these as "permission-like" errors where we should warn+skip, not abort. */
__attribute__((always_inline))
static inline int is_perm_error(int err) {
    return err == EPERM || err == EACCES || err == EROFS || err == ENOTSUP;
}

/* Best-effort: create all parents. Never fatal. Returns 0 on success/already-exists,
   -1 on error (errno preserved). */
static int
ensure_dir_exists_recursive(const char *path) {
    char tmp[PATH_MAX];
    snprintf(tmp, sizeof(tmp), "%s", path);
    size_t n = strlen(tmp);

    for (size_t i = 1; i < n; i++) {
        if (tmp[i] != '/') continue;
        tmp[i] = '\0';
        if (tmp[0] != '\0') {
            if (mkdir(tmp, 0777) != 0 && errno != EEXIST) {
                int e = errno;
                warnx("mkdir: %s: %s", tmp, strerror(e));
                tmp[i] = '/';
                errno = e;
                return is_perm_error(e) ? -2 : -1;
            }
        }
        tmp[i] = '/';
    }

    if (mkdir(tmp, 0777) != 0 && errno != EEXIST) {
        int e = errno;
        warnx("mkdir: %s: %s", tmp, strerror(e));
        errno = e;
        return is_perm_error(e) ? -2 : -1;
    }
    return 0;
}

static int ensure_parent_dirs_for(const char *final_path) {
    char dir[PATH_MAX];
    path_dirname(final_path, dir, sizeof(dir));
    return ensure_dir_exists_recursive(dir);
}

static const char *human_bytes(double b, char *buf, size_t n) {
    static const char *units[] = {"B", "KiB", "MiB", "GiB", "TiB", "PiB"};
    int i = 0;
    while (b >= 1024.0 && i < 5) {
        b /= 1024.0;
        i++;
    }
    snprintf(buf, n, "%.2f %s", b, units[i]);
    return buf;
}

static void
human_rate(double bytes_per_sec, char out[], size_t out_sz)
{
    char qty[32];
    human_bytes(bytes_per_sec, qty, sizeof(qty));
    snprintf(out, out_sz, "%s/s", qty);
}

/* --------------------------------------------------------------------------
 * Autotune helpers (RAM clamp, FS detection, scratch cleanup, time)
 * -------------------------------------------------------------------------- */

static uint64_t get_memsize_bytes(void) {
    uint64_t mem = 0;
    size_t   sz  = sizeof(mem);

    if (sysctlbyname("hw.memsize", &mem, &sz, NULL, 0) != 0 ||
        sz != sizeof(mem) ||
        mem == 0)
    {
        /* Sensible fallback */
        mem = 8ull << 30;
    }
    return mem;
}

static void remove_tree_recursive(const char *root) {
    char *roots[2] = { (char*)root, NULL };

    FTS *fts = fts_open(roots, FTS_PHYSICAL | FTS_NOCHDIR, NULL);
    if (!fts) return;

    for (FTSENT *e; (e = fts_read(fts)) != NULL; ) {
        switch (e->fts_info) {
            case FTS_D:    break;                 /* pre-order dir */
            case FTS_DP:   (void)rmdir(e->fts_path); break; /* post-order dir */
            case FTS_F:
            case FTS_SL:
            case FTS_SLNONE:
            case FTS_DEFAULT:
                (void)unlink(e->fts_path);
                break;
            default:
                break;
        }
    }
    (void)fts_close(fts);
}

static double now_sec(void) {
    struct timeval tv;
    gettimeofday(&tv, NULL);
    return (double)tv.tv_sec + (double)tv.tv_usec / 1e6;
}

static void clamp_inflight_to_ram(Options *o) {
    const uint64_t ram         = get_memsize_bytes();
    const uint64_t max_allowed = (ram >> 2);             /* 25% of RAM */
    const uint64_t min_floor   = (256ull << 20);         /* 256 MiB    */

    if ((uint64_t)o->inflight_cap > max_allowed) o->inflight_cap = (size_t)max_allowed;
    if ((uint64_t)o->inflight_cap < min_floor)   o->inflight_cap = (size_t)min_floor;
}

// ============================ Statistics ============================

typedef struct Stats {
    /* Totals discovered during planning/enqueue */
    uint64_t files_total;         /* number of regular files to process */
    uint64_t bytes_total;         /* logical bytes across files only */
    uint64_t symlinks_total;      /* number of symlinks to process      */
    uint64_t dirs_total_jobs;     /* number of directory jobs enqueued  */

    /* Live progress during IO (files only, keeps behavior for progress bar) */
    uint64_t files_done;          /* number of regular files completed  */
    uint64_t bytes_done;          /* logical progress bytes (includes holes/clones) */
    uint64_t bytes_phys_io;       /* true physical I/O bytes written/read */

    /* File operation accounting (regular files) */
    uint64_t files_copied;
    uint64_t bytes_copied;

    uint64_t files_cloned_src;
    uint64_t bytes_cloned_src;

    uint64_t files_cloned_dedupe;
    uint64_t bytes_cloned_dedupe;

    uint64_t files_hardlinked;
    uint64_t bytes_hardlinked;

    uint64_t files_skipped;
    uint64_t bytes_skipped;

    /* Directory + symlink creation accounting */
    uint64_t dirs_created;        /* directories created (not total)    */
    uint64_t symlinks_created;    /* symlinks created at destination    */

    /* Source deletion accounting */
    uint64_t files_deleted_src;     /* regular files deleted from source */
    uint64_t bytes_deleted_src;     /* logical bytes for files only      */
    uint64_t symlinks_deleted_src;  /* symlinks deleted from source      */

    /* Directory job progress accounting */
    uint64_t dirs_done_jobs;      /* number of directory jobs completed */

    /* Phase signal: 0 while only preparing directory structure,
       set to 1 immediately before enqueuing the first T_FILE job. */
    volatile sig_atomic_t copy_phase_started;

    struct timeval start_time;
} Stats;

// ============================ Dir FD LRU cache ============================

typedef struct {
    char path[PATH_MAX];
    int fd;
    uint64_t tick;
    bool used;
} DirFDEntry;

typedef struct {
    DirFDEntry v[DIRFD_CACHE_MAX];
    uint64_t tick;
    pthread_mutex_t mu;
} DirFDCache;

static DirFDCache g_dirfd_cache;

static void dirfd_cache_init(void) {
    memset(&g_dirfd_cache, 0, sizeof(g_dirfd_cache));
    pthread_mutex_init(&g_dirfd_cache.mu, NULL);
}

static int dirfd_cache_open(const char *dirpath) {
    pthread_mutex_lock(&g_dirfd_cache.mu);
    g_dirfd_cache.tick++;
    uint64_t now = g_dirfd_cache.tick;

    for (int i = 0; i < DIRFD_CACHE_MAX; i++) {
        if (g_dirfd_cache.v[i].used && strcmp(g_dirfd_cache.v[i].path, dirpath) == 0) {
            g_dirfd_cache.v[i].tick = now;
            int fd = g_dirfd_cache.v[i].fd;
            pthread_mutex_unlock(&g_dirfd_cache.mu);
            return fd;
        }
    }
    int fd = open(dirpath, O_RDONLY | O_CLOEXEC);
    if (fd < 0) {
        pthread_mutex_unlock(&g_dirfd_cache.mu);
        return -1;
    }

    int slot = -1;
    uint64_t oldtick = UINT64_MAX;
    for (int i = 0; i < DIRFD_CACHE_MAX; i++) {
        if (!g_dirfd_cache.v[i].used) {
            slot = i;
            break;
        }
        if (g_dirfd_cache.v[i].tick < oldtick) {
            oldtick = g_dirfd_cache.v[i].tick;
            slot = i;
        }
    }
    if (g_dirfd_cache.v[slot].used) {
        close(g_dirfd_cache.v[slot].fd);
    }
    snprintf(g_dirfd_cache.v[slot].path, sizeof(g_dirfd_cache.v[slot].path), "%s", dirpath);
    g_dirfd_cache.v[slot].fd = fd;
    g_dirfd_cache.v[slot].tick = now;
    g_dirfd_cache.v[slot].used = true;
    pthread_mutex_unlock(&g_dirfd_cache.mu);
    return fd;
}

static void dirfd_cache_close_all(void) {
    pthread_mutex_lock(&g_dirfd_cache.mu);

    for (int i = 0; i < DIRFD_CACHE_MAX; i++) {
        if (g_dirfd_cache.v[i].used) {
            close(g_dirfd_cache.v[i].fd);
            g_dirfd_cache.v[i].used = false;
        }
    }

    pthread_mutex_unlock(&g_dirfd_cache.mu);
}

static void dirfd_cache_fsync_all(void) {
    pthread_mutex_lock(&g_dirfd_cache.mu);

    for (int i = 0; i < DIRFD_CACHE_MAX; i++) {
        if (g_dirfd_cache.v[i].used) {
            (void)fsync_full(g_dirfd_cache.v[i].fd);
        }
    }
    pthread_mutex_unlock(&g_dirfd_cache.mu);
}

// ============================ Persistent Hash Cache ============================

typedef struct {
    dev_t dev;
    ino_t ino;
    uint64_t size;
    int64_t mtime_sec;
    int32_t mtime_nsec;
    int64_t ctime_sec;
    int32_t ctime_nsec;
    unsigned char digest[DIGEST_LEN];
} HashCacheEnt;

typedef struct {
    HashCacheEnt *v;
    size_t n, cap;
} HashCacheVec;

typedef struct {
    uint64_t h;
    size_t idx;
    bool used;
} HIdxSlot;

typedef struct {
    HIdxSlot *tab;
    size_t cap, fill;
} HIdx;

static HashCacheVec g_hcache = {0};
static char g_hcache_path[PATH_MAX];
static HIdx g_hidx = {0};

/* single lock for cache+index+file append */
static pthread_rwlock_t g_hcache_rw = PTHREAD_RWLOCK_INITIALIZER;

__attribute__((always_inline))
static uint64_t mix64(uint64_t x) {
    x ^= x >> 33;
    x *= 0xff51afd7ed558ccdULL;
    x ^= x >> 33;
    x *= 0xc4ceb9fe1a85ec53ULL;
    x ^= x >> 33;
    return x;
}

__attribute__((always_inline))
static uint64_t hkey(dev_t d, ino_t i, uint64_t sz, int64_t ms, int32_t mn, int64_t cs, int32_t cn) {
    uint64_t h = 0x9e3779b97f4a7c15ULL;

    h ^= mix64((uint64_t)d) + (h << 6) + (h >> 2);
    h ^= mix64((uint64_t)i) + (h << 6) + (h >> 2);

    h ^= mix64(sz) + (h << 6) + (h >> 2);

    h ^= mix64((uint64_t)ms) + (h << 6) + (h >> 2);
    h ^= mix64((uint64_t)mn) + (h << 6) + (h >> 2);
    h ^= mix64((uint64_t)cs) + (h << 6) + (h >> 2);
    h ^= mix64((uint64_t)cn) + (h << 6) + (h >> 2);

    return h;
}

static void hidx_init(void) {
    g_hidx.cap = 1u << 14; // 16384
    g_hidx.tab = (HIdxSlot *)calloc(g_hidx.cap, sizeof(HIdxSlot));

    if (!g_hidx.tab) {
        die("OOM");
    }
}

static void hidx_rehash(void) {
    size_t ncap = g_hidx.cap << 1;
    HIdxSlot *nt = (HIdxSlot *)calloc(ncap, sizeof(HIdxSlot));

    if (!nt) {
        die("OOM");
    }

    for (size_t i = 0; i < g_hidx.cap; i++) {
        if (g_hidx.tab[i].used) {
            uint64_t h = g_hidx.tab[i].h;
            size_t mask = ncap - 1, j = (size_t)(h & mask);

            while (nt[j].used) {
                j = (j + 1) & mask;
            }

            nt[j] = g_hidx.tab[i];
        }
    }

    free(g_hidx.tab);

    g_hidx.tab = nt;
    g_hidx.cap = ncap;
}

static void hidx_put(uint64_t h, size_t idx) {
    if ((g_hidx.fill * 10) >= (g_hidx.cap * 7)) {
        hidx_rehash();
    }

    size_t mask = g_hidx.cap - 1, j = (size_t)(h & mask);

    while (g_hidx.tab[j].used) {
        if (g_hidx.tab[j].h == h) {
            g_hidx.tab[j].idx = idx;
            return;
        }

        j = (j + 1) & mask;
    }

    g_hidx.tab[j].used = true;
    g_hidx.tab[j].h = h;
    g_hidx.tab[j].idx = idx;

    g_hidx.fill++;
}

static ssize_t hidx_get(uint64_t h) {
    size_t mask = g_hidx.cap - 1, j = (size_t)(h & mask);

    while (g_hidx.tab[j].used) {
        if (g_hidx.tab[j].h == h) {
            return (ssize_t)g_hidx.tab[j].idx;
        }

        j = (j + 1) & mask;
    }

    return -1;
}

static void hcache_load(const char *hint_dir) {
    const char *home = getenv("HOME");

    if (home && *home) {
        snprintf(g_hcache_path, sizeof(g_hcache_path),
                                "%s/Library/Caches/dcopy/hashcache.bin",
                                home);
    } else if (hint_dir && *hint_dir) {
        snprintf(g_hcache_path, sizeof(g_hcache_path),
                                    "%s/.dcopy-hashcache.bin", hint_dir);
    } else {
        snprintf(g_hcache_path, sizeof(g_hcache_path), ".dcopy-hashcache.bin");
    }

    pthread_rwlock_wrlock(&g_hcache_rw);

    hidx_init();

    FILE *f = fopen(g_hcache_path, "rb");

    if (!f) {
        pthread_rwlock_unlock(&g_hcache_rw);
        return;
    }

    HashCacheEnt ent;

    while (fread(&ent, sizeof(ent), 1, f) == 1) {
        if (g_hcache.n == g_hcache.cap) {
            size_t ncap = g_hcache.cap ? g_hcache.cap * 2 : 8192;

            HashCacheEnt *nv = (HashCacheEnt *)realloc(g_hcache.v, ncap * sizeof(HashCacheEnt));

            if (!nv) {
                fclose(f);
                pthread_rwlock_unlock(&g_hcache_rw);
                die("OOM");
            }

            g_hcache.v = nv;
            g_hcache.cap = ncap;
        }

        g_hcache.v[g_hcache.n] = ent;

        uint64_t hk = hkey(ent.dev, ent.ino, ent.size, ent.mtime_sec,
                           ent.mtime_nsec, ent.ctime_sec, ent.ctime_nsec);

        hidx_put(hk, g_hcache.n);

        g_hcache.n++;
    }

    fclose(f);

    pthread_rwlock_unlock(&g_hcache_rw);
}

static void hcache_append(const HashCacheEnt *ent) {
    pthread_rwlock_wrlock(&g_hcache_rw);

    int fd = open(g_hcache_path, O_CREAT | O_WRONLY | O_APPEND, 0600);

    if (fd >= 0) {
        /* best-effort; ignore short write errors (not fatal for correctness) */
        (void)write(fd, ent, (unsigned)sizeof(*ent));
        close(fd);
    }

    if (g_hcache.n == g_hcache.cap) {
        size_t ncap = g_hcache.cap ? g_hcache.cap * 2 : 8192;
        HashCacheEnt *nv = (HashCacheEnt *)realloc(g_hcache.v, ncap * sizeof(HashCacheEnt));

        if (!nv) {
            pthread_rwlock_unlock(&g_hcache_rw);
            die("OOM");
        }

        g_hcache.v = nv;
        g_hcache.cap = ncap;
    }

    g_hcache.v[g_hcache.n] = *ent;

    uint64_t hk = hkey(ent->dev, ent->ino, ent->size,
                       ent->mtime_sec, ent->mtime_nsec,
                       ent->ctime_sec, ent->ctime_nsec);

    hidx_put(hk, g_hcache.n);

    g_hcache.n++;

    pthread_rwlock_unlock(&g_hcache_rw);
}

static bool hcache_lookup(dev_t dev, ino_t ino, uint64_t size,
                          int64_t msec, int32_t mnsec,
                          int64_t csec, int32_t cnsec,
                          unsigned char out[DIGEST_LEN])
{
    uint64_t hk = hkey(dev, ino, size, msec, mnsec, csec, cnsec);

    pthread_rwlock_rdlock(&g_hcache_rw);

    ssize_t idx = hidx_get(hk);

    if (idx >= 0) {
        HashCacheEnt *e = &g_hcache.v[(size_t)idx];

        if (e->dev == dev && e->ino == ino && e->size == size &&
            e->mtime_sec == msec && e->mtime_nsec == mnsec &&
            e->ctime_sec == csec && e->ctime_nsec == cnsec) {
            memcpy(out, e->digest, DIGEST_LEN);

            pthread_rwlock_unlock(&g_hcache_rw);

            return true;
        }
    }

    pthread_rwlock_unlock(&g_hcache_rw);

    return false;
}

// ============================ Data model ============================

typedef enum { T_DIR, T_FILE, T_SYMLINK, T_OTHER } EntType;

typedef struct {
    char *src_path;
    char *rel_path;
    EntType type;
    mode_t mode;
    dev_t dev;
    ino_t ino;
    nlink_t nlink;
    off_t size;
    struct timespec mtime;
    struct timespec ctime;
    bool hashed;
    unsigned char digest[DIGEST_LEN];
    uint64_t sample64;
    bool has_physfp;
    uint64_t physfp;
} Entry;

typedef struct {
    Entry *v;
    size_t n, cap;
} VecEntry;

static void ve_push(VecEntry *ve, Entry e) {
    if (ve->n == ve->cap) {
        size_t ncap = ve->cap ? ve->cap * 2 : 4096;

        Entry *nv = (Entry *)realloc(ve->v, ncap * sizeof(Entry));

        if (!nv) {
            die("OOM");
        }

        ve->v = nv;
        ve->cap = ncap;
    }

    ve->v[ve->n++] = e;
}

static void ve_free(VecEntry *ve) {
    if (!ve) {
        return;
    }

    for (size_t i = 0; i < ve->n; i++) {
        free(ve->v[i].src_path);
        free(ve->v[i].rel_path);
    }

    free(ve->v);

    ve->v = NULL;
    ve->n = ve->cap = 0;
}

typedef struct {
    char *src_root;
    bool is_dir;
    char *inject_base;
} RootPlan;

// ============================ Enumeration ============================

static const uint32_t ATTR_CMN_MASK_BULK =
    ATTR_CMN_NAME
    | ATTR_CMN_DEVID
    | ATTR_CMN_FILEID
    | ATTR_CMN_OBJTYPE
    | ATTR_CMN_MODTIME
    | ATTR_CMN_CRTIME
    | ATTR_CMN_FLAGS;

static const uint32_t ATTR_DIR_MASK_BULK =
    ATTR_DIR_LINKCOUNT;

static const uint32_t ATTR_FILE_MASK_BULK =
    ATTR_FILE_LINKCOUNT
    | ATTR_FILE_TOTALSIZE;

static inline size_t ATTR_ALIGN4(size_t n) {
    return (n + 3u) & ~3u;
}

static void collect_entries_fts(VecEntry *out, const RootPlan *rp, const Options *opt);

static void collect_entries_bulk(VecEntry *out, const RootPlan *rp, const Options *opt) {
    typedef struct {
        char dpath[PATH_MAX];
        char relprefix[PATH_MAX];
        dev_t dev;  /* device of the directory at dpath (authoritative for XDEV checks) */
    } Work;

    Work *stack = NULL;
    size_t n = 0, cap = 0;

    struct stat st_root;

    if (lstat(rp->src_root, &st_root) != 0) {
        warnx("stat: %s: %s", rp->src_root, strerror(errno));
        return;
    }

    char base_rel[PATH_MAX];

    if (rp->is_dir) {
        snprintf(base_rel, sizeof(base_rel), "%s", rp->inject_base ? rp->inject_base : "");
    } else {
        snprintf(base_rel, sizeof(base_rel), "%s", rp->inject_base ? rp->inject_base : path_basename(rp->src_root));
    }

    /* Single-file root: just emit one entry and return */
    if (!S_ISDIR(st_root.st_mode)) {
        Entry e = (Entry){0};

        e.src_path = xstrdup(rp->src_root);
        e.rel_path = xstrdup(base_rel);
        e.mode     = st_root.st_mode;
        e.dev      = st_root.st_dev;
        e.ino      = st_root.st_ino;
        e.nlink    = st_root.st_nlink;
        e.size     = st_root.st_size;
        e.mtime    = st_root.st_mtimespec;
        e.ctime    = st_root.st_ctimespec;

        if (S_ISREG(e.mode))      e.type = T_FILE;
        else if (S_ISLNK(e.mode)) e.type = T_SYMLINK;
        else if (S_ISDIR(e.mode)) e.type = T_DIR;
        else                      e.type = T_OTHER;

#ifdef SF_DATALESS
        if (!opt->include_icloud) {
            if (st_root.st_flags & SF_DATALESS) {
                return; /* skip dataless items when requested */
            }
        }
#endif

        ve_push(out, e);
        return;
    }

    /* Emit the root directory itself (empty relpath allowed when inject_base == "") */
    {
        Entry rootE = (Entry){0};

        rootE.src_path = xstrdup(rp->src_root);
        rootE.rel_path = xstrdup(base_rel[0] ? base_rel : "");
        rootE.mode     = st_root.st_mode;
        rootE.dev      = st_root.st_dev;
        rootE.ino      = st_root.st_ino;
        rootE.nlink    = st_root.st_nlink;
        rootE.size     = st_root.st_size;
        rootE.mtime    = st_root.st_mtimespec;
        rootE.ctime    = st_root.st_ctimespec;
        rootE.type     = T_DIR;

        ve_push(out, rootE);
    }

    if (n == cap) {
        cap = 16;
        stack = (Work *)malloc(cap * sizeof(Work));

        if (!stack) {
            die("OOM");
        }
    }

    snprintf(stack[n].dpath, sizeof(stack[n].dpath), "%s", rp->src_root);
    snprintf(stack[n].relprefix, sizeof(stack[n].relprefix), "%s", base_rel);

    stack[n].dev = st_root.st_dev;

    n++;

    /* Prepare getattrlistbulk(2) attribute masks with fallbacks */
    struct attrlist al;
    memset(&al, 0, sizeof(al));

    al.bitmapcount = ATTR_BIT_MAP_COUNT;
    al.commonattr  = ATTR_CMN_RETURNED_ATTRS | ATTR_CMN_MASK_BULK; /* includes NAME/DEVID/FILEID/OBJTYPE/MODTIME/CRTIME/FLAGS */
    al.dirattr     = ATTR_DIR_MASK_BULK;                            /* DIR_LINKCOUNT */
    al.fileattr    = ATTR_FILE_MASK_BULK;                           /* FILE_LINKCOUNT / TOTALSIZE */

    const size_t bufsz = 64 * 1024;
    void *buf = malloc(bufsz);

    if (!buf) {
        die("OOM");
    }

    while (n > 0) {
        Work w = stack[--n];

        int dfd = dirfd_cache_open(w.dpath);

        if (dfd < 0) {
            warnx("open dir: %s: %s", w.dpath, strerror(errno));
            continue;
        }

        for (;;) {
            struct attrlist cur = al;

            int phase = 0;
            int nrec  = -1;

            for (;;) {
                errno = 0;
                nrec = getattrlistbulk(dfd, &cur, buf, bufsz, 0);

                if (nrec >= 0) {
                    break;
                }

                if (errno == EINVAL && phase == 0) {
                    /* Retry without devid/fileid/size/flags (minimal but still includes name + basic times) */
                    struct attrlist tmp = {0};

                    tmp.bitmapcount = ATTR_BIT_MAP_COUNT;
                    tmp.commonattr  = ATTR_CMN_NAME | ATTR_CMN_OBJTYPE | ATTR_CMN_MODTIME | ATTR_CMN_CRTIME;
                    tmp.dirattr     = ATTR_DIR_LINKCOUNT;
                    tmp.fileattr    = 0;

                    cur = tmp;

                    phase = 1;

                    if (g_verbose) {
                        vmsg("bulk: EINVAL -> retry reduced mask on %s", w.dpath);
                    }

                    continue;
                }

                if (errno == EINVAL && phase == 1) {
                    /* Last resort: only name + objtype (most portable) */
                    struct attrlist tmp = {0};

                    tmp.bitmapcount = ATTR_BIT_MAP_COUNT;
                    tmp.commonattr  = ATTR_CMN_NAME | ATTR_CMN_OBJTYPE;
                    tmp.dirattr     = 0;
                    tmp.fileattr    = 0;

                    cur = tmp;
                    phase = 2;

                    if (g_verbose) {
                        vmsg("bulk: EINVAL -> retry minimal mask on %s", w.dpath);
                    }

                    continue;
                }

                if (nrec < 0) {
                    /* Fall back to FTS for this subtree */
                    vmsg("getattrlistbulk failed on %s: %s -> fallback FTS", w.dpath, strerror(errno));

                    RootPlan subrp = {
                        .src_root = (char *)w.dpath,
                        .is_dir = true,
                        .inject_base = (char *)w.relprefix,
                    };

                    collect_entries_fts(out, &subrp, opt);

                    nrec = 0;
                }

                break;
            }

            if (nrec == 0) {
                break;
            }

            char *p = (char *)buf;

            for (int irec = 0; irec < nrec; irec++) {
                uint32_t reclen = *(uint32_t *)p;

                p += 4;

                char *rec = (char *)p;

                /* Returned groups (ATTR_CMN_RETURNED_ATTRS populates these) */
                attrgroup_t retc = 0, retv = 0, retd = 0, retf = 0, retfk = 0;

                memcpy(&retc, rec, sizeof(attrgroup_t)); rec += ATTR_ALIGN4(sizeof(attrgroup_t));
                memcpy(&retv, rec, sizeof(attrgroup_t)); rec += ATTR_ALIGN4(sizeof(attrgroup_t));
                memcpy(&retd, rec, sizeof(attrgroup_t)); rec += ATTR_ALIGN4(sizeof(attrgroup_t));
                memcpy(&retf, rec, sizeof(attrgroup_t)); rec += ATTR_ALIGN4(sizeof(attrgroup_t));
                memcpy(&retfk, rec, sizeof(attrgroup_t)); rec += ATTR_ALIGN4(sizeof(attrgroup_t));

                /* Name */
                char namebuf[NAME_MAX + 1] = {0};

                if (retc & ATTR_CMN_NAME) {
                    attrreference_t *ref = (attrreference_t *)rec;

                    const char *nm = ((char *)rec) + ref->attr_dataoffset;
                    size_t nn = (size_t)ref->attr_length;

                    if (nn > NAME_MAX) {
                        nn = NAME_MAX;
                    }

                    memcpy(namebuf, nm, nn);

                    namebuf[nn] = '\0';

                    rec += ATTR_ALIGN4(sizeof(attrreference_t));
                } else {
                    /* No name -> skip this record safely */
                    p += reclen - 4;
                    continue;
                }

                /* Basic attributes possibly present */
                dev_t         dev = 0;
                uint64_t      fileid = 0;
                fsobj_type_t  objtype = 0;
                struct timespec mt = {0}, ct = {0};
                uint32_t      cmn_flags = 0;
                uint32_t      dir_linkcount = 0;
                uint32_t      file_linkcount = 0;
                off_t         totsz = 0;

                if (retc & ATTR_CMN_DEVID)     { memcpy(&dev,     rec, sizeof(dev));     rec += ATTR_ALIGN4(sizeof(dev)); }
                if (retc & ATTR_CMN_FILEID)    { memcpy(&fileid,  rec, sizeof(fileid));  rec += ATTR_ALIGN4(sizeof(fileid)); }
                if (retc & ATTR_CMN_OBJTYPE)   { memcpy(&objtype, rec, sizeof(objtype)); rec += ATTR_ALIGN4(sizeof(objtype)); }
                if (retc & ATTR_CMN_MODTIME)   { memcpy(&mt,      rec, sizeof(mt));      rec += ATTR_ALIGN4(sizeof(mt)); }
                if (retc & ATTR_CMN_CRTIME)    { memcpy(&ct,      rec, sizeof(ct));      rec += ATTR_ALIGN4(sizeof(ct)); }
                if (retc & ATTR_CMN_FLAGS)     { memcpy(&cmn_flags, rec, sizeof(cmn_flags)); rec += ATTR_ALIGN4(sizeof(cmn_flags)); }
                if (retd & ATTR_DIR_LINKCOUNT) { memcpy(&dir_linkcount, rec, sizeof(dir_linkcount)); rec += ATTR_ALIGN4(sizeof(dir_linkcount)); }
                if (retf & ATTR_FILE_LINKCOUNT){ memcpy(&file_linkcount, rec, sizeof(file_linkcount)); rec += ATTR_ALIGN4(sizeof(file_linkcount)); }
                if (retf & ATTR_FILE_TOTALSIZE){ memcpy(&totsz,   rec, sizeof(totsz));   rec += ATTR_ALIGN4(sizeof(totsz)); }

                /* Join child path */
                char child[PATH_MAX];

                if (w.dpath[0] == '/' && w.dpath[1] == '\0') {
                    snprintf(child, sizeof(child), "/%s", namebuf);
                } else {
                    path_join2(w.dpath, namebuf, child, sizeof(child));
                }

                if (strcmp(namebuf, ".") == 0 || strcmp(namebuf, "..") == 0) {
                    p += reclen - 4;
                    continue;
                }

                /* Authoritative classification via lstat() */
                struct stat stx;
                int have_stat = (lstat(child, &stx) == 0);

                /* iCloud dataless filtering */
#ifdef SF_DATALESS
                if (!opt->include_icloud) {
                    bool dataless = false;
                    if (have_stat) {
                        dataless = (stx.st_flags & SF_DATALESS) != 0;
                    } else if (retc & ATTR_CMN_FLAGS) {
                        dataless = (cmn_flags & SF_DATALESS) != 0;
                    }
                    if (dataless) {
                        p += reclen - 4;
                        continue;
                    }
                }
#endif

                /* Resolve entry type */
                EntType ty = T_OTHER;

                if (have_stat) {
                    if      (S_ISDIR(stx.st_mode)) ty = T_DIR;
                    else if (S_ISREG(stx.st_mode)) ty = T_FILE;
                    else if (S_ISLNK(stx.st_mode)) ty = T_SYMLINK;
                    else                           ty = T_OTHER;
                } else {
                    /* Fallback to OBJTYPE/group hints if lstat failed */
                    if (retc & ATTR_CMN_OBJTYPE) {
                        uint32_t vt = (uint32_t)objtype;
                        if      (vt == VDIR) ty = T_DIR;
                        else if (vt == VREG) ty = T_FILE;
                        else if (vt == VLNK) ty = T_SYMLINK;
                        else                 ty = T_OTHER;
                    } else if (retd & ATTR_DIR_LINKCOUNT) {
                        ty = T_DIR;
                    } else if (retf & (ATTR_FILE_LINKCOUNT | ATTR_FILE_TOTALSIZE)) {
                        ty = T_FILE;
                    } else {
                        ty = T_OTHER;
                    }
                }

                /* Normalize attributes from lstat if bulk record omitted them */
                if (have_stat) {
                    if (!(retc & ATTR_CMN_DEVID))    dev    = stx.st_dev;
                    if (!(retc & ATTR_CMN_FILEID))   fileid = (uint64_t)stx.st_ino;
                    if (!(retc & ATTR_CMN_MODTIME))  mt     = stx.st_mtimespec;
                    if (!(retc & ATTR_CMN_CRTIME))   ct     = stx.st_ctimespec;

                    if (ty == T_FILE) {
                        if (!(retf & ATTR_FILE_TOTALSIZE))  totsz         = stx.st_size;
                        if (!(retf & ATTR_FILE_LINKCOUNT))  file_linkcount = (uint32_t)stx.st_nlink;
                    } else if (ty == T_DIR) {
                        if (!(retd & ATTR_DIR_LINKCOUNT))   dir_linkcount  = (uint32_t)stx.st_nlink;
                    }
                }

                /* Build Entry */
                Entry e = (Entry){0};
                e.src_path = xstrdup(child);

                if (w.relprefix[0]) {
                    char rel[PATH_MAX];

                    path_join2(w.relprefix, namebuf, rel, sizeof(rel));

                    e.rel_path = xstrdup(rel);
                } else {
                    e.rel_path = xstrdup(namebuf);
                }

                e.type  = ty;
                e.mode  = have_stat ? stx.st_mode : 0;
                e.dev   = dev;
                e.ino   = (ino_t)fileid;
                e.mtime = mt;
                e.ctime = ct;

                if (ty == T_DIR)       e.nlink = (nlink_t)dir_linkcount;
                else if (ty == T_FILE) e.nlink = (nlink_t)file_linkcount;
                else                   e.nlink = (nlink_t)(have_stat ? stx.st_nlink : 1);

                e.size = (ty == T_FILE) ? totsz : 0;

                ve_push(out, e);

                /* Recurse into directories, honoring --one-file-system */
                if (ty == T_DIR) {
                    dev_t child_dev = have_stat ? stx.st_dev : dev;

                    if (opt->one_file_system && child_dev != w.dev) {
                        p += reclen - 4;
                        continue;
                    }

                    if (n == cap) {
                        cap = cap ? (cap << 1) : 16;

                        Work *nw = (Work *)realloc(stack, cap * sizeof(Work));

                        if (!nw) {
                            die("OOM");
                        }

                        stack = nw;
                    }

                    snprintf(stack[n].dpath, sizeof(stack[n].dpath), "%s", child);

                    if (w.relprefix[0]) {
                        snprintf(stack[n].relprefix, sizeof(stack[n].relprefix), "%s/%s", w.relprefix, namebuf);
                    } else {
                        snprintf(stack[n].relprefix, sizeof(stack[n].relprefix), "%s", namebuf);
                    }

                    stack[n].dev = child_dev;

                    n++;
                }

                /* advance to next record */
                p += reclen - 4;
            }
        }
    }

    free(buf);
    free(stack);
}

static void collect_entries_fts(VecEntry *out, const RootPlan *rp, const Options *opt) {
    int fts_opts = opt->follow_symlinks ? FTS_LOGICAL : FTS_PHYSICAL;

    char *roots[2] = {rp->src_root, NULL};

    FTS *fts = fts_open(roots, fts_opts | FTS_NOCHDIR, NULL);

    if (!fts) {
        die("fts_open: %s", strerror(errno));
    }

    char root_with_slash[PATH_MAX];

    size_t rl = strlen(rp->src_root);

    snprintf(root_with_slash, sizeof(root_with_slash), "%s%s", rp->src_root,
             (rl > 0 && rp->src_root[rl - 1] == '/') ? "" : "/");

    size_t base_trim = strlen(root_with_slash);

    FTSENT *ent;

    while ((ent = fts_read(fts)) != NULL) {
        if (opt->one_file_system && ent->fts_statp && ent->fts_level > 0 && ent->fts_statp->st_dev != fts->fts_dev) {
            (void)fts_set(fts, ent, FTS_SKIP);
            continue;
        }

        if (ent->fts_info == FTS_DNR || ent->fts_info == FTS_ERR || ent->fts_info == FTS_NS) {
            warnx("fts: %s: %s", ent->fts_path, strerror(ent->fts_errno));
            continue;
        }

        if (ent->fts_info == FTS_DP) {
            continue;
        }

        Entry e = {0};
        e.src_path = xstrdup(ent->fts_path);

        const char *suffix = ent->fts_path + base_trim;

        if (suffix < ent->fts_path) {
            suffix = ent->fts_path;
        }

        if (rp->inject_base && rp->inject_base[0]) {
            char rel[PATH_MAX];
            path_join2(rp->inject_base, suffix, rel, sizeof(rel));
            e.rel_path = xstrdup(rel);
        } else {
            e.rel_path = xstrdup(suffix);
        }

        struct stat *st = ent->fts_statp;
        e.mode = st->st_mode;
        e.dev = st->st_dev;
        e.ino = st->st_ino;
        e.nlink = st->st_nlink;
        e.size = st->st_size;
        e.mtime = st->st_mtimespec;
        e.ctime = st->st_ctimespec;

        if (S_ISDIR(st->st_mode)) e.type = T_DIR;
        else if (S_ISREG(st->st_mode)) e.type = T_FILE;
        else if (S_ISLNK(st->st_mode)) e.type = T_SYMLINK;
        else e.type = T_OTHER;

        ve_push(out, e);
    }

    if (errno != 0) {
        warnx("fts_read error: %s", strerror(errno));
    }

    (void)fts_close(fts);
}

// ============================ Fingerprint & Hash ============================

static uint64_t sample_fingerprint(const char *path, off_t fsize) {
    int fd = open(path, O_RDONLY | O_NOFOLLOW);

    if (fd < 0) {
        return 0;
    }

    const size_t sample_size = 4096;
    unsigned char buf[4096];
    bool ok = true;
    uint64_t result = 0;

    XXH3_state_t* const state = XXH3_createState();

    if (!state) {
        close(fd);
        return 0;
    }

    XXH3_64bits_reset(state);

#ifdef F_NOCACHE
    (void)fcntl(fd, F_NOCACHE, 1);
#endif

    // Head sample
    ssize_t r = pread(fd, buf, sample_size, 0);

    if (r > 0) {
        XXH3_64bits_update(state, buf, (size_t)r);
    } else if (r < 0) {
        ok = false;
    }

    // Middle sample
    if (ok && fsize > (off_t)(2 * sample_size)) {
        off_t mid = (fsize - (off_t)sample_size) / 2;

        r = pread(fd, buf, sample_size, mid);

        if (r > 0) {
            XXH3_64bits_update(state, buf, (size_t)r);
        } else if (r < 0) {
            ok = false;
        }
    }

    // Tail sample
    if (ok && fsize > (off_t)sample_size) {
        off_t tail = fsize - (off_t)sample_size;

        r = pread(fd, buf, sample_size, tail);

        if (r > 0) {
            XXH3_64bits_update(state, buf, (size_t)r);
        } else if (r < 0) {
            ok = false;
        }
    }

    if (ok) {
        result = XXH3_64bits_digest(state);
    }

    XXH3_freeState(state);
    close(fd);
    return result;
}

static ssize_t safe_read(int fd, void *buf, size_t n) {
    for (;;) {
        ssize_t r = read(fd, buf, n);

        if (r >= 0) {
            return r;
        }

        if (errno == EINTR) {
            continue;
        }

        return -1;
    }
}

static bool xxh128_file(const char *path, unsigned char out[DIGEST_LEN], int bufsize) {
    int fd = open(path, O_RDONLY | O_NOFOLLOW);

    if (fd < 0) {
        return false;
    }

#ifdef F_NOCACHE
    (void)fcntl(fd, F_NOCACHE, 1);
#endif

    XXH3_state_t* const state = XXH3_createState();

    if (!state) {
        close(fd);
        return false;
    }

    XXH3_128bits_reset(state);

    void *buf = malloc((size_t)bufsize);

    if (!buf) {
        XXH3_freeState(state);
        close(fd);

        return false;
    }

    for (;;) {
        ssize_t r = safe_read(fd, buf, (size_t)bufsize);

        if (r < 0) {
            free(buf);
            XXH3_freeState(state);
            close(fd);
            return false;
        }

        if (r == 0) {
            break;
        }

        XXH3_128bits_update(state, buf, (size_t)r);
    }

    XXH128_hash_t const hash = XXH3_128bits_digest(state);

    memcpy(out, &hash, sizeof(hash));

    XXH3_freeState(state);
    free(buf);
    close(fd);
    return true;
}

__attribute__((always_inline))
static bool phys_fingerprint_fd(int fd, off_t fsize, uint64_t *out_h) {
#ifdef F_LOG2PHYS_EXT
    struct log2phys l2p;
    off_t off = 0;
    uint64_t h = 0x9e3779b97f4a7c15ULL;

    while (off < fsize) {
        memset(&l2p, 0, sizeof(l2p));

        l2p.l2p_devoffset = off;
        l2p.l2p_contigbytes = fsize - off;

        if (fcntl(fd, F_LOG2PHYS_EXT, &l2p) == -1) {
            return false;
        }

        if (l2p.l2p_devoffset < 0 || l2p.l2p_contigbytes <= 0) {
            return false;
        }

        uint64_t P = (uint64_t)l2p.l2p_devoffset;
        uint64_t C = (uint64_t)l2p.l2p_contigbytes;

        h ^= mix64(P + 0x9e37) + (h << 6) + (h >> 2);
        h ^= mix64(C + 0x4a7c) + (h << 6) + (h >> 2);

        off += l2p.l2p_contigbytes;
    }

    *out_h = h;
    return true;
#else
    (void)fd;
    (void)fsize;
    (void)out_h;
    return false;
#endif
}

static uint64_t phys_fingerprint_path(const char *path, off_t fsize) {
    int fd = open(path, O_RDONLY | O_NOFOLLOW);

    if (fd < 0) {
        return 0;
    }

#ifdef F_NOCACHE
    (void)fcntl(fd, F_NOCACHE, 1);
#endif

    uint64_t h = 0;

    bool ok = phys_fingerprint_fd(fd, fsize, &h);

    close(fd);

    return ok ? h : 0;
}

// ============================ Verify & Source Deletion ============================

static bool file_xxh128_path(const char *path, unsigned char out[DIGEST_LEN], int bufsize) {
    return xxh128_file(path, out, bufsize);
}

static bool file_sample64_path(const char *path, off_t size, uint64_t *out) {
    *out = sample_fingerprint(path, size);
    return (*out != 0);
}

/* -------------------- Regular file equality -------------------- */

typedef enum {
    DEQ_OK = 0,
    DEQ_SIZE_MISMATCH,
    DEQ_HASH_MISMATCH,
    DEQ_SRC_STAT_FAIL,
    DEQ_DST_STAT_FAIL,
    DEQ_SRC_HASH_FAIL,
    DEQ_DST_HASH_FAIL,
    DEQ_SAME_INODE_GUARD
} DeqReason;

static bool files_equal_for_delete(const Options *opt,
                                   const char *src_path,
                                   const struct stat *st_src,
                                   const char *dst_path,
                                   const struct stat *st_dst)
{
    if (st_dst->st_size != st_src->st_size) {
        return false;
    }

    if (opt->delete_verify == VERIFY_SIZE) {
        return true;
    }

    if (opt->delete_verify == VERIFY_SAMPLE) {
        uint64_t s_src = 0, s_dst = 0;
        if (!file_sample64_path(src_path, (off_t)st_src->st_size, &s_src)) return false;
        if (!file_sample64_path(dst_path, (off_t)st_dst->st_size, &s_dst)) return false;
        return s_src == s_dst;
    }

    /* VERIFY_HASH */
    unsigned char src_h[DIGEST_LEN], dst_h[DIGEST_LEN];
    if (!file_xxh128_path(src_path, src_h, (int)opt->buf_size)) return false;
    if (!file_xxh128_path(dst_path, dst_h, (int)opt->buf_size)) return false;
    return memcmp(src_h, dst_h, DIGEST_LEN) == 0;
}

/* Same as files_equal_for_delete but records a concrete reason for mismatch,
   and optionally prints detailed diagnostics under -v. */
static DeqReason files_equal_with_reason(const Options *opt,
                                         const char *src_path,
                                         const struct stat *st_src,
                                         const char *dst_path,
                                         const struct stat *st_dst,
                                         bool verbose_log)
{
    if (st_src->st_dev == st_dst->st_dev && st_src->st_ino == st_dst->st_ino) {
        if (verbose_log) vmsg("existing-equal: same inode guard: %s == %s", src_path, dst_path);
        return DEQ_SAME_INODE_GUARD;
    }

    if (st_dst->st_size != st_src->st_size) {
        if (verbose_log) vmsg("existing-equal: size mismatch: src=%lld dst=%lld (%s)",
                              (long long)st_src->st_size, (long long)st_dst->st_size, dst_path);
        return DEQ_SIZE_MISMATCH;
    }

    if (opt->delete_verify == VERIFY_SIZE) {
        return DEQ_OK;
    }

    if (opt->delete_verify == VERIFY_SAMPLE) {
        uint64_t s_src = 0, s_dst = 0;
        if (!file_sample64_path(src_path, (off_t)st_src->st_size, &s_src)) {
            if (verbose_log) vmsg("existing-equal: sample src failed: %s", src_path);
            return DEQ_SRC_HASH_FAIL;
        }
        if (!file_sample64_path(dst_path, (off_t)st_dst->st_size, &s_dst)) {
            if (verbose_log) vmsg("existing-equal: sample dst failed: %s", dst_path);
            return DEQ_DST_HASH_FAIL;
        }
        if (s_src != s_dst) {
            if (verbose_log) vmsg("existing-equal: sample mismatch for %s", dst_path);
            return DEQ_HASH_MISMATCH;
        }
        return DEQ_OK;
    }

    /* VERIFY_HASH */
    unsigned char src_h[DIGEST_LEN], dst_h[DIGEST_LEN];
    if (!file_xxh128_path(src_path, src_h, (int)opt->buf_size)) {
        if (verbose_log) vmsg("existing-equal: hash src failed: %s", src_path);
        return DEQ_SRC_HASH_FAIL;
    }
    if (!file_xxh128_path(dst_path, dst_h, (int)opt->buf_size)) {
        if (verbose_log) vmsg("existing-equal: hash dst failed: %s", dst_path);
        return DEQ_DST_HASH_FAIL;
    }
    if (memcmp(src_h, dst_h, DIGEST_LEN) != 0) {
        if (verbose_log) {
            /* Print a short 64-bit prefix of each hash for quick triage. */
            uint64_t a = 0, b = 0;
            memcpy(&a, src_h, sizeof(uint64_t));
            memcpy(&b, dst_h, sizeof(uint64_t));
            vmsg("existing-equal: hash mismatch for %s (src=%016llx dst=%016llx)",
                 dst_path, (unsigned long long)a, (unsigned long long)b);
        }
        return DEQ_HASH_MISMATCH;
    }
    return DEQ_OK;
}

static void note_deleted_source(Stats *st, const Entry *e) {
    (void)__atomic_add_fetch(&st->files_deleted_src, 1, __ATOMIC_RELAXED);
    (void)__atomic_add_fetch(&st->bytes_deleted_src, (uint64_t)e->size, __ATOMIC_RELAXED);
}

static void plan_note_deleted_source(Stats *st, const Entry *e) {
    (void)__atomic_add_fetch(&st->files_deleted_src, 1, __ATOMIC_RELAXED);
    (void)__atomic_add_fetch(&st->bytes_deleted_src, (uint64_t)e->size, __ATOMIC_RELAXED);
}

/* Returns 1 if deleted/planned, 0 if not deleted, -1 on error (files) */
static int verify_and_delete_source(const Options *opt, const Entry *e, const char *dst_path, Stats *st) {
    if (!opt->delete_source) return 0;

    struct stat ss, sd;
    if (lstat(e->src_path, &ss) != 0) {
        warnx("delete: stat src failed: %s: %s", e->src_path, strerror(errno));
        return -1;
    }
    if (lstat(dst_path, &sd) != 0) {
        warnx("delete: stat dst failed: %s: %s", dst_path, strerror(errno));
        return -1;
    }

    if (ss.st_dev == sd.st_dev && ss.st_ino == sd.st_ino) {
        vmsg("delete: skip (src==dst inode) %s", e->src_path);
        return 0;
    }

    if (!files_equal_for_delete(opt, e->src_path, &ss, dst_path, &sd)) {
        return 0;
    }

    if (opt->dry_run) {
        pmsg("PLAN   DELSRC %s", e->src_path);
        plan_note_deleted_source(st, e);
        return 1;
    }

    if (unlink(e->src_path) == 0) {
        pmsg("DEL    %s", e->src_path);
        note_deleted_source(st, e);
        return 1;
    }

    warnx("unlink: %s: %s", e->src_path, strerror(errno));
    return -1;
}

/* Equality check against an already-existing destination for --delete-source-existing (files).
   Emits reasoned diagnostics under -v when mismatch prevents deletion. */
static bool equal_against_existing_for_delete(const Options *opt,
                                              const char *src_path,
                                              const struct stat *st_src,
                                              const char *dst_path,
                                              const struct stat *st_dst)
{
    DeqReason r = files_equal_with_reason(opt, src_path, st_src, dst_path, st_dst, /*verbose_log=*/(g_verbose > 0));
    return r == DEQ_OK;
}

/* -------------------- Symlink helpers -------------------- */

/* Read a symlink into a freshly-allocated, NUL-terminated buffer. Returns true on success. */
static bool readlink_malloc(const char *path, char **out, size_t *out_len)
{
    const size_t START = 256;
    size_t n = START;

    for (;;) {
        char *buf = (char *)malloc(n + 1);
        if (!buf) return false;

        ssize_t r = readlink(path, buf, n);
        if (r < 0) {
            free(buf);
            return false;
        }

        if ((size_t)r < n) {
            buf[r] = '\0';
            if (out) *out = buf;
            if (out_len) *out_len = (size_t)r;
            return true;
        }

        free(buf);
        if (n >= 32 * 1024) {
            return false;
        }
        n <<= 1;
    }
}

/* Compare the *contents* of two symlinks without following them. */
static bool symlink_targets_equal_paths(const char *a, const char *b)
{
    struct stat sa, sb;
    if (lstat(a, &sa) != 0 || lstat(b, &sb) != 0) return false;
    if (!S_ISLNK(sa.st_mode) || !S_ISLNK(sb.st_mode)) return false;

    char *ta = NULL, *tb = NULL;
    size_t la = 0, lb = 0;

    if (!readlink_malloc(a, &ta, &la)) return false;
    if (!readlink_malloc(b, &tb, &lb)) { free(ta); return false; }

    bool eq = (la == lb) && (memcmp(ta, tb, la) == 0);

    free(ta);
    free(tb);
    return eq;
}

/* Separate symlink delete accounting to keep the report accurate. */
static void note_deleted_symlink(Stats *st) {
    (void)__atomic_add_fetch(&st->symlinks_deleted_src, 1, __ATOMIC_RELAXED);
}
static void plan_note_deleted_symlink(Stats *st) {
    (void)__atomic_add_fetch(&st->symlinks_deleted_src, 1, __ATOMIC_RELAXED);
}

/* Delete a source symlink if the destination symlink exists and the *link bodies* are identical.
   Returns 1 if deleted/planned, 0 if not deleted, -1 on error. */
static int verify_and_delete_symlink_existing_equal(const Options *opt,
                                                    const Entry *e,
                                                    const char *dst_path,
                                                    Stats *st)
{
    if (!opt->delete_source_existing) return 0;

    struct stat ss, sd;
    if (lstat(e->src_path, &ss) != 0) return -1;
    if (lstat(dst_path, &sd) != 0)    return -1;

    if (!S_ISLNK(ss.st_mode) || !S_ISLNK(sd.st_mode)) return 0;

    if (ss.st_dev == sd.st_dev && ss.st_ino == sd.st_ino) return 0;

    if (!symlink_targets_equal_paths(e->src_path, dst_path)) return 0;

    if (opt->dry_run) {
        pmsg("PLAN   DELSRC %s (existing-equal symlink)", e->src_path);
        plan_note_deleted_symlink(st);
        return 1;
    }
    if (unlink(e->src_path) == 0) {
        pmsg("DEL    %s (existing-equal symlink)", e->src_path);
        note_deleted_symlink(st);
        return 1;
    }
    warnx("unlink: %s: %s", e->src_path, strerror(errno));
    return -1;
}

/* Delete a source symlink after we have successfully created the destination symlink
   and verified the link bodies match. Returns 1 if deleted/planned, 0 if not deleted, -1 on error. */
static int verify_and_delete_source_symlink(const Options *opt,
                                            const Entry *e,
                                            const char *dst_path,
                                            Stats *st)
{
    if (!opt->delete_source) return 0;

    struct stat ss, sd;
    if (lstat(e->src_path, &ss) != 0)          return -1;
    if (lstat(dst_path, &sd) != 0)             return -1;
    if (!S_ISLNK(ss.st_mode) || !S_ISLNK(sd.st_mode)) return 0;

    if (ss.st_dev == sd.st_dev && ss.st_ino == sd.st_ino) return 0;

    if (!symlink_targets_equal_paths(e->src_path, dst_path)) return 0;

    if (opt->dry_run) {
        pmsg("PLAN   DELSRC %s", e->src_path);
        plan_note_deleted_symlink(st);
        return 1;
    }
    if (unlink(e->src_path) == 0) {
        pmsg("DEL    %s", e->src_path);
        note_deleted_symlink(st);
        return 1;
    }
    warnx("unlink: %s: %s", e->src_path, strerror(errno));
    return -1;
}

// =========================== Path pruning ======================================

/* Count '/' components (for depth sort) */
static int path_depth(const char *p) {
    int d = 0;

    for (const char *s = p; *s; s++) {
        if (*s == '/') {
            d++;
        }
    }

    return d;
}

static int cmp_dir_depth_desc(const void *a, const void *b) {
    const Entry *ea = *(Entry * const *)a;
    const Entry *eb = *(Entry * const *)b;

    int da = path_depth(ea->src_path);
    int db = path_depth(eb->src_path);

    if (da != db) {
        /* deeper first */
        return (db - da);
    }

    /* stable-ish tie-breaker */
    return strcmp(eb->src_path, ea->src_path);
}

/* Attempt to prune all source directories that are now empty.
   Safe to run after IO workers (no concurrent unlinks left). */
static void prune_empty_source_dirs(const Options *opt, VecEntry *ents) {
    /* Prune when either delete mode may have removed all file children. */
    if (!(opt->delete_source || opt->delete_source_existing)) {
        return;
    }

    // Drop all cached dir FDs so rmdir() cannot hit EBUSY
    // on roots we previously opened during getattrlistbulk()/IO.
    dirfd_cache_close_all();

    /* Collect directory pointers */
    size_t n = 0;

    for (size_t i = 0; i < ents->n; i++) {
        if (ents->v[i].type == T_DIR) {
            n++;
        }
    }

    if (n == 0) {
        return;
    }

    Entry **dirs = (Entry **)malloc(n * sizeof(*dirs));

    if (!dirs) {
        die("OOM");
    }

    size_t k = 0;

    for (size_t i = 0; i < ents->n; i++) {
        if (ents->v[i].type == T_DIR) {
            dirs[k++] = &ents->v[i];
        }
    }

    qsort(dirs, n, sizeof(*dirs), cmp_dir_depth_desc);

    for (size_t i = 0; i < n; i++) {
        const char *d = dirs[i]->src_path;

        if (opt->dry_run) {
            /* Plan-only visibility is gated by -v via pmsg() prefix filter */
            pmsg("PLAN   DEL %s (dir, if empty)", d);
            continue;
        }

        if (rmdir(d) == 0) {
            /* only printed with -v */
            pmsg("DEL    %s", d);
            continue;
        }

        // ENOTEMPTY / EEXIST: directory has surviving children -> keep.
        // EBUSY commonly occurs if some process has CWD or we kept a dirfd
        // open; by this point we've closed ours. Log remaining oddities.
        if (errno == ENOTEMPTY || errno == EEXIST) {
            if (g_verbose) {
                /* List a few leftover entries to aid diagnosis (e.g., .DS_Store). */
                DIR *dirp = opendir(d);

                if (dirp) {
                    int shown = 0;

                    struct dirent *de;

                    fprintf(stderr, "KEEP   %s (non-empty):", d);

                    while ((de = readdir(dirp)) && shown < 3) {
                        if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) {
                            continue;
                        }

                        fprintf(stderr, " %s", de->d_name);

                        shown++;
                    }

                    fprintf(stderr, "%s\n", (shown == 3 ? " …" : ""));

                    closedir(dirp);
                } else {
                    vmsg("rmdir: %s: directory not empty", d);
                }
            }
        } else if (errno != ENOENT) {
            vmsg("rmdir: %s: %s", d, strerror(errno));
        }
    }

    free(dirs);
}

// ============================ In-flight I/O Limiter ============================

static pthread_mutex_t g_tb_mu = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t g_tb_cv = PTHREAD_COND_INITIALIZER;

static size_t g_inflight = 0;
static size_t g_inflight_cap = DEFAULT_INFLIGHT_MB << 20;

static void tb_reserve(size_t bytes) {
    pthread_mutex_lock(&g_tb_mu);

    while (g_inflight + bytes > g_inflight_cap) {
        pthread_cond_wait(&g_tb_cv, &g_tb_mu);
    }

    g_inflight += bytes;

    pthread_mutex_unlock(&g_tb_mu);
}

static void tb_release(size_t bytes) {
    pthread_mutex_lock(&g_tb_mu);

    if (g_inflight < bytes) {
        g_inflight = 0;
    } else {
        g_inflight -= bytes;
    }

    pthread_cond_broadcast(&g_tb_cv);

    pthread_mutex_unlock(&g_tb_mu);
}

// ============================ I/O Buffer Pool ============================

typedef struct BufNode {
    struct BufNode *next;
    size_t sz;
    char data[];
} BufNode;

static pthread_mutex_t g_bp_mu = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t g_bp_cv = PTHREAD_COND_INITIALIZER;
static BufNode *g_bp_free = NULL;
static size_t g_bp_bufsz = READ_BUFSZ_DEFAULT;

static void bp_init(size_t bufsz, int count) {
    g_bp_bufsz = bufsz;

    for (int i = 0; i < count; i++) {
        BufNode *n = (BufNode *)malloc(sizeof(BufNode) + bufsz);

        if (!n) {
            die("OOM");
        }

        n->sz = bufsz;
        n->next = g_bp_free;

        g_bp_free = n;
    }
}

static BufNode *bp_acquire(void) {
    pthread_mutex_lock(&g_bp_mu);

    while (!g_bp_free) {
        pthread_cond_wait(&g_bp_cv, &g_bp_mu);
    }

    BufNode *n = g_bp_free;

    g_bp_free = n->next;
    n->next = NULL;

    pthread_mutex_unlock(&g_bp_mu);

    return n;
}

static void bp_release(BufNode *n) {
    pthread_mutex_lock(&g_bp_mu);

    n->next = g_bp_free;
    g_bp_free = n;

    pthread_cond_signal(&g_bp_cv);

    pthread_mutex_unlock(&g_bp_mu);
}

// ============================ Batch Durability Management ============================

typedef struct {
    dev_t dev;
    ino_t ino;
    size_t files;
    size_t bytes;
} DirBudget;

typedef struct {
    DirBudget v[256];
    size_t n;
    pthread_mutex_t mu;
} DirbShard;

static DirbShard g_dirb[DIRB_SHARDS];

__attribute__((always_inline))
static inline size_t dirb_idx(dev_t d, ino_t i) {
    return (size_t)((mix64((uint64_t)d) ^ mix64((uint64_t)i)) & (DIRB_SHARDS - 1));
}

static void dirb_note(int dirfd, size_t bytes, const Options *opt) {
    if (opt->dur_mode != DUR_BATCH) {
        return;
    }

    struct stat st;

    if (fstat(dirfd, &st) != 0) {
        return;
    }

    size_t si = dirb_idx(st.st_dev, st.st_ino);

    DirbShard *S = &g_dirb[si];

    pthread_mutex_lock(&S->mu);

    size_t pos = SIZE_MAX;

    for (size_t k = 0; k < S->n; k++) {
        if (S->v[k].dev == st.st_dev && S->v[k].ino == st.st_ino) {
            pos = k;
            break;
        }
    }

    if (pos == SIZE_MAX) {
        if (S->n < (sizeof(S->v) / sizeof(S->v[0]))) {
            pos = S->n++;
        } else {
            pos = 0;
        }

        S->v[pos] = (DirBudget){
            .dev = st.st_dev,
            .ino = st.st_ino,
            .files = 0,
            .bytes = 0
        };
    }

    S->v[pos].files++;
    S->v[pos].bytes += bytes;

    if (S->v[pos].files >= opt->fsync_batch_files || S->v[pos].bytes >= opt->fsync_batch_bytes) {
        (void)fsync_full(dirfd);

        S->v[pos].files = 0;
        S->v[pos].bytes = 0;
    }

    pthread_mutex_unlock(&S->mu);
}

// ============================ Thread-safe Maps (Dedupe/Hardlink Tracking) ============================

#define IS_POW2(x) (((x) & ((x)-1u)) == 0u)
#if !IS_POW2(MAP_SHARDS)
#error "MAP_SHARDS must be a power-of-two"
#endif

__attribute__((always_inline))
static inline uint64_t fnv1a64_mix(const void *data, size_t len) {
    const unsigned char *p = (const unsigned char *)data;

    uint64_t h = 1469598103934665603ull;

    for (size_t i = 0; i < len; i++) {
        h ^= p[i];
        h *= 1099511628211ull;
    }

    h = mix64(h);

    if (h == 0) {
        h = 0x9e3779b97f4a7c15ull;
    }

    return h;
}

__attribute__((always_inline))
static inline uint64_t inode_hash(dev_t d, ino_t i) {
    uint64_t h = mix64((uint64_t)d) ^ mix64((uint64_t)i);

    if (h == 0) {
        h = 0x2545F4914F6CDD1Dull;
    }

    return h;
}

// ============================================================================
// Bit / math helpers
// ============================================================================

// smallest power-of-two >= x (x >= 1)
__attribute__((always_inline))
static inline size_t
next_pow2_size(size_t x)
{
    if (x <= 1) {
        return 1;
    }

    x--;

    /* Propagate top bit rightward */
    for (size_t shift = 1; shift < sizeof(size_t) * 8; shift <<= 1) {
        x |= (x >> shift);
    }

    return x + 1;
}

// count leading zeros for size_t
__attribute__((always_inline))
static inline unsigned
clzzu(size_t x)
{
#if defined(__GNUC__) || defined(__clang__)
#  if UINTPTR_MAX == 0xFFFFFFFFFFFFFFFFull
    return (x ? (unsigned)__builtin_clzll((unsigned long long)x) : (unsigned)(8*sizeof(size_t)));
#  else
    return (x ? (unsigned)__builtin_clz((unsigned)x) : (unsigned)(8*sizeof(size_t)));
#  endif
#else
    /* Portable fallback */
    unsigned n = 0;
    size_t   m = (size_t)1 << (sizeof(size_t) * 8 - 1);
    if (x == 0) return (unsigned)(8*sizeof(size_t));
    while ((x & m) == 0) { n++; m >>= 1; }
    return n;
#endif
}

// floor(log2(x)) for size_t, x > 0
__attribute__((always_inline))
static inline unsigned
ilog2u(size_t x)
{
    /* Precondition: x > 0 */
    return (unsigned)(8 * sizeof(size_t) - 1 - clzzu(x));
}

// --- DigestMap ---

typedef struct {
    unsigned char digest[DIGEST_LEN];
    char *dst_path;
    uint64_t hash;
} DigestSlot;

#define DIGEST_SHARD_FIELDS \
    DigestSlot *tab;        \
    size_t cap;             \
    size_t fill;            \
    pthread_mutex_t mu;

typedef struct {
    DIGEST_SHARD_FIELDS
} DigestShardCore;

enum {
    DigestShardPadRaw = (DCOPY_CACHELINE - (sizeof(DigestShardCore) % DCOPY_CACHELINE)) % DCOPY_CACHELINE,
    DigestShardPad = (DigestShardPadRaw == 0 ? DCOPY_CACHELINE : DigestShardPadRaw)
};

typedef struct {
    DIGEST_SHARD_FIELDS
    char _pad[DigestShardPad];
} DigestShard DCOPY_ALIGN_TYPE(DCOPY_CACHELINE);

_Static_assert(_Alignof(DigestShard) >= DCOPY_CACHELINE, "DigestShard alignment failed");
_Static_assert((sizeof(DigestShard) % DCOPY_CACHELINE) == 0, "DigestShard size must be multiple of cacheline");

#undef DIGEST_SHARD_FIELDS

typedef struct {
    DigestShard shard[MAP_SHARDS];
    size_t shard_mask;
    unsigned shard_bits;
} DigestMap;

static void dmap_shard_init(DigestShard *s, size_t cap) {
    if (cap < MIN_SHARD_CAP) {
        cap = MIN_SHARD_CAP;
    }

    cap = next_pow2_size(cap);

    s->tab = (DigestSlot *)calloc(cap, sizeof(DigestSlot));

    if (!s->tab) {
        die("OOM");
    }

    s->cap = cap;
    s->fill = 0;

    if (pthread_mutex_init(&s->mu, NULL) != 0) {
        die("pthread_mutex_init");
    }
}

static void dmap_shard_free(DigestShard *s) {
    if (!s->tab) {
        return;
    }

    for (size_t i = 0; i < s->cap; i++) {
        if (s->tab[i].hash != 0 && s->tab[i].dst_path) {
            free(s->tab[i].dst_path);
        }
    }

    free(s->tab);

    s->tab = NULL;
    s->cap = s->fill = 0;

    pthread_mutex_destroy(&s->mu);
}

static void dmap_shard_grow_unlocked(DigestShard *s) {
    size_t ncap = s->cap << 1;

    DigestSlot *nt = (DigestSlot *)calloc(ncap, sizeof(DigestSlot));

    if (!nt) {
        die("OOM");
    }

    size_t mask = ncap - 1;

    for (size_t i = 0; i < s->cap; i++) {
        DigestSlot *old = &s->tab[i];

        if (old->hash == 0) {
            continue;
        }

        size_t j = (size_t)(old->hash & mask);

        while (nt[j].hash != 0) {
            j = (j + 1) & mask;
        }

        nt[j] = *old;
    }

    free(s->tab);

    s->tab = nt;
    s->cap = ncap;
}

static inline bool dcmp_digest(const unsigned char a[DIGEST_LEN], const unsigned char b[DIGEST_LEN]) {
    return memcmp(a, b, DIGEST_LEN) == 0;
}

static inline const char *dmap_shard_get_unlocked(DigestShard *s, const unsigned char d[DIGEST_LEN], uint64_t h) {
    size_t mask = s->cap - 1;
    size_t j = (size_t)(h & mask);
#if defined(__GNUC__) || defined(__clang__)
    __builtin_prefetch(&s->tab[j], 0, 1);
#endif
    for (;;) {
        DigestSlot *sl = &s->tab[j];
        uint64_t sh = sl->hash;
        if (sh == 0) return NULL;
        if (sh == h && dcmp_digest(sl->digest, d)) return sl->dst_path;
        j = (j + 1) & mask;
    }
}

static inline const char *dmap_shard_put_unlocked(DigestShard *s, const unsigned char d[DIGEST_LEN], uint64_t h,
                                                  const char *dst) {
    if ((s->fill * MAP_LF_DEN) >= (s->cap * MAP_LF_NUM)) {
        dmap_shard_grow_unlocked(s);
    }

    size_t mask = s->cap - 1;
    size_t j = (size_t)(h & mask);

    for (;;) {
        DigestSlot *sl = &s->tab[j];

        if (sl->hash == 0) {
            sl->hash = h;

            memcpy(sl->digest, d, DIGEST_LEN);

            sl->dst_path = xstrdup(dst);
            s->fill++;

            return sl->dst_path;
        }

        if (sl->hash == h && dcmp_digest(sl->digest, d)) {
            return sl->dst_path;
        }

        j = (j + 1) & mask;
    }
}

static void dmap_init(DigestMap *m) {
    m->shard_mask = MAP_SHARDS - 1u;
    m->shard_bits = ilog2u(MAP_SHARDS);

    size_t per = (DMAP_INIT_TOTAL / MAP_SHARDS);

    if (per < MIN_SHARD_CAP) {
        per = MIN_SHARD_CAP;
    }

    for (size_t s = 0; s < MAP_SHARDS; s++) {
        dmap_shard_init(&m->shard[s], per);
    }
}

static void dmap_free(DigestMap *m) {
    for (size_t s = 0; s < MAP_SHARDS; s++) {
        dmap_shard_free(&m->shard[s]);
    }
}

static char *dmap_get(DigestMap *m, const unsigned char d[DIGEST_LEN]) {
    uint64_t h = fnv1a64_mix(d, DIGEST_LEN);
    size_t si = (size_t)((h >> (64 - m->shard_bits)) & m->shard_mask);

    DigestShard *S = &m->shard[si];

    pthread_mutex_lock(&S->mu);

    const char *p = dmap_shard_get_unlocked(S, d, h);

    pthread_mutex_unlock(&S->mu);

    return (char *)p;
}

static void dmap_put(DigestMap *m, const unsigned char d[DIGEST_LEN], const char *dst) {
    uint64_t h = fnv1a64_mix(d, DIGEST_LEN);
    size_t si = (size_t)((h >> (64 - m->shard_bits)) & m->shard_mask);

    DigestShard *S = &m->shard[si];

    pthread_mutex_lock(&S->mu);

    (void)dmap_shard_put_unlocked(S, d, h, dst);

    pthread_mutex_unlock(&S->mu);
}

// --- InoMap ---

typedef struct {
    dev_t dev;
    ino_t ino;
} InoKey;

typedef struct {
    InoKey key;
    char *dst;
    uint64_t hash;
} InoSlot;

#define INO_SHARD_FIELDS \
    InoSlot *tab;        \
    size_t cap;          \
    size_t fill;         \
    pthread_mutex_t mu;

typedef struct {
    INO_SHARD_FIELDS
} InoShardCore;

enum {
    InoShardPadRaw = (DCOPY_CACHELINE - (sizeof(InoShardCore) % DCOPY_CACHELINE)) % DCOPY_CACHELINE,
    InoShardPad = (InoShardPadRaw == 0 ? DCOPY_CACHELINE : InoShardPadRaw)
};

typedef struct {
    INO_SHARD_FIELDS
    char _pad[InoShardPad];
} InoShard DCOPY_ALIGN_TYPE(DCOPY_CACHELINE);

_Static_assert(_Alignof(InoShard) >= DCOPY_CACHELINE, "InoShard alignment failed");
_Static_assert((sizeof(InoShard) % DCOPY_CACHELINE) == 0, "InoShard size must be multiple of cacheline");

#undef INO_SHARD_FIELDS

typedef struct {
    InoShard shard[MAP_SHARDS];
    size_t shard_mask;
    unsigned shard_bits;
} InoMap;

static void imap_shard_init(InoShard *s, size_t cap) {
    if (cap < MIN_SHARD_CAP) {
        cap = MIN_SHARD_CAP;
    }

    cap = next_pow2_size(cap);

    s->tab = (InoSlot *)calloc(cap, sizeof(InoSlot));

    if (!s->tab) {
        die("OOM");
    }

    s->cap = cap;
    s->fill = 0;

    if (pthread_mutex_init(&s->mu, NULL) != 0) {
        die("pthread_mutex_init");
    }
}

static void imap_shard_free(InoShard *s) {
    if (!s->tab) {
        return;
    }

    for (size_t i = 0; i < s->cap; i++) {
        if (s->tab[i].hash != 0 && s->tab[i].dst) {
            free(s->tab[i].dst);
        }
    }

    free(s->tab);

    s->tab = NULL;
    s->cap = s->fill = 0;

    pthread_mutex_destroy(&s->mu);
}

static void imap_shard_grow_unlocked(InoShard *s) {
    size_t ncap = s->cap << 1;
    InoSlot *nt = (InoSlot *)calloc(ncap, sizeof(InoSlot));

    if (!nt) {
        die("OOM");
    }

    size_t mask = ncap - 1;

    for (size_t i = 0; i < s->cap; i++) {
        InoSlot *old = &s->tab[i];

        if (old->hash == 0) {
            continue;
        }

        size_t j = (size_t)(old->hash & mask);

        while (nt[j].hash != 0) {
            j = (j + 1) & mask;
        }

        nt[j] = *old;
    }

    free(s->tab);

    s->tab = nt;
    s->cap = ncap;
}

__attribute__((always_inline))
static inline bool inokey_eq(InoKey a, InoKey b) {
    return a.dev == b.dev && a.ino == b.ino;
}

static inline const char
*imap_shard_get_unlocked(InoShard *s, InoKey k, uint64_t h) {
    size_t mask = s->cap - 1;
    size_t j = (size_t)(h & mask);

#if defined(__GNUC__) || defined(__clang__)
    __builtin_prefetch(&s->tab[j], 0, 1);
#endif

    for (;;) {
        InoSlot *sl = &s->tab[j];
        uint64_t sh = sl->hash;

        if (sh == 0) {
            return NULL;
        }

        if (sh == h && inokey_eq(sl->key, k)) {
            return sl->dst;
        }

        j = (j + 1) & mask;
    }
}

static inline const char *imap_shard_put_unlocked(InoShard *s, InoKey k, uint64_t h, const char *dst) {
    if ((s->fill * MAP_LF_DEN) >= (s->cap * MAP_LF_NUM)) {
        imap_shard_grow_unlocked(s);
    }

    size_t mask = s->cap - 1;
    size_t j = (size_t)(h & mask);

    for (;;) {
        InoSlot *sl = &s->tab[j];

        if (sl->hash == 0) {
            sl->hash = h;
            sl->key = k;
            sl->dst = xstrdup(dst);
            s->fill++;
            return sl->dst;
        }

        if (sl->hash == h && inokey_eq(sl->key, k)) {
            return sl->dst;
        }

        j = (j + 1) & mask;
    }
}

static void imap_init(InoMap *m) {
    m->shard_mask = MAP_SHARDS - 1u;
    m->shard_bits = ilog2u(MAP_SHARDS);

    size_t per = (IMAP_INIT_TOTAL / MAP_SHARDS);

    if (per < MIN_SHARD_CAP) {
        per = MIN_SHARD_CAP;
    }

    for (size_t s = 0; s < MAP_SHARDS; s++) {
        imap_shard_init(&m->shard[s], per);
    }
}

static void imap_free(InoMap *m) {
    for (size_t s = 0; s < MAP_SHARDS; s++) {
        imap_shard_free(&m->shard[s]);
    }
}

static char *imap_get(InoMap *m, InoKey k) {
    uint64_t h = inode_hash(k.dev, k.ino);

    size_t si = (size_t)((h >> (64 - m->shard_bits)) & m->shard_mask);

    InoShard *S = &m->shard[si];

    pthread_mutex_lock(&S->mu);

    const char *p = imap_shard_get_unlocked(S, k, h);

    pthread_mutex_unlock(&S->mu);

    return (char *)p;
}

static void imap_put(InoMap *m, InoKey k, const char *dst) {
    uint64_t h = inode_hash(k.dev, k.ino);

    size_t si = (size_t)((h >> (64 - m->shard_bits)) & m->shard_mask);

    InoShard *S = &m->shard[si];

    pthread_mutex_lock(&S->mu);

    (void)imap_shard_put_unlocked(S, k, h, dst);

    pthread_mutex_unlock(&S->mu);
}

// --- PhysMap ---

typedef struct {
    uint64_t key;
    char *dst;
    uint64_t hash;
} PSlot;

typedef struct {
    PSlot *tab;
    size_t cap;
    size_t fill;
    pthread_mutex_t mu;
} PShardCore;

enum {
    PShardPadRaw = (DCOPY_CACHELINE - (sizeof(PShardCore) % DCOPY_CACHELINE)) % DCOPY_CACHELINE,
    PShardPad = (PShardPadRaw == 0 ? DCOPY_CACHELINE : PShardPadRaw)
};

typedef struct {
    PSlot *tab;
    size_t cap;
    size_t fill;
    pthread_mutex_t mu;
    char _pad[PShardPad];
} PShard DCOPY_ALIGN_TYPE(DCOPY_CACHELINE);

typedef struct {
    PShard shard[MAP_SHARDS];
    size_t shard_mask;
    unsigned shard_bits;
} PhysMap;

static inline uint64_t h64(uint64_t x) {
    uint64_t h = mix64(x);
    return h ? h : 0x9e3779b97f4a7c15ULL;
}

static void pshard_init(PShard *s, size_t cap) {
    if (cap < MIN_SHARD_CAP) {
        cap = MIN_SHARD_CAP;
    }

    cap = next_pow2_size(cap);

    s->tab = (PSlot *)calloc(cap, sizeof(PSlot));

    if (!s->tab) {
        die("OOM");
    }

    s->cap = cap;
    s->fill = 0;

    if (pthread_mutex_init(&s->mu, NULL) != 0) {
        die("pthread_mutex_init");
    }
}

static void pshard_free(PShard *s) {
    if (!s->tab) {
        return;
    }

    for (size_t i = 0; i < s->cap; i++) {
        if (s->tab[i].hash && s->tab[i].dst) {
            free(s->tab[i].dst);
        }
    }

    free(s->tab);

    s->tab = NULL;
    s->cap = s->fill = 0;

    pthread_mutex_destroy(&s->mu);
}

static void pshard_grow(PShard *s) {
    size_t ncap = s->cap << 1;
    PSlot *nt = (PSlot *)calloc(ncap, sizeof(PSlot));

    if (!nt) {
        die("OOM");
    }

    size_t mask = ncap - 1;

    for (size_t i = 0; i < s->cap; i++) {
        PSlot *old = &s->tab[i];

        if (!old->hash) {
            continue;
        }

        size_t j = (size_t)(old->hash & mask);

        while (nt[j].hash) {
            j = (j + 1) & mask;
        }

        nt[j] = *old;
    }

    free(s->tab);

    s->tab = nt;
    s->cap = ncap;
}

static void pmap_init(PhysMap *m) {
    m->shard_mask = MAP_SHARDS - 1u;
    m->shard_bits = ilog2u(MAP_SHARDS);

    size_t per = (IMAP_INIT_TOTAL / MAP_SHARDS);

    if (per < MIN_SHARD_CAP) {
        per = MIN_SHARD_CAP;
    }

    for (size_t s = 0; s < MAP_SHARDS; s++) {
        pshard_init(&m->shard[s], per);
    }
}

static void pmap_free(PhysMap *m) {
    for (size_t s = 0; s < MAP_SHARDS; s++) {
        pshard_free(&m->shard[s]);
    }
}

static char *pmap_get(PhysMap *m, uint64_t key) {
    uint64_t hv = h64(key);

    size_t si = (size_t)((hv >> (64 - m->shard_bits)) & m->shard_mask);
    PShard *S = &m->shard[si];

    pthread_mutex_lock(&S->mu);

    size_t mask = S->cap - 1;
    size_t j = (size_t)(hv & mask);

#if defined(__GNUC__) || defined(__clang__)
    __builtin_prefetch(&S->tab[j], 0, 1);
#endif

    while (S->tab[j].hash) {
        if (S->tab[j].hash == hv && S->tab[j].key == key) {
            char *p = S->tab[j].dst;

            pthread_mutex_unlock(&S->mu);

            return p;
        }

        j = (j + 1) & mask;
    }

    pthread_mutex_unlock(&S->mu);

    return NULL;
}

static void pmap_put(PhysMap *m, uint64_t key, const char *dst) {
    uint64_t hv = h64(key);
    size_t si = (size_t)((hv >> (64 - m->shard_bits)) & m->shard_mask);
    PShard *S = &m->shard[si];

    pthread_mutex_lock(&S->mu);

    if ((S->fill * MAP_LF_DEN) >= (S->cap * MAP_LF_NUM)) {
        pshard_grow(S);
    }

    size_t mask = S->cap - 1;
    size_t j = (size_t)(hv & mask);

    for (;;) {
        PSlot *sl = &S->tab[j];

        if (!sl->hash) {
            sl->hash = hv;
            sl->key = key;
            sl->dst = xstrdup(dst);
            S->fill++;
            break;
        }

        if (sl->hash == hv && sl->key == key) {
            break;
        }

        j = (j + 1) & mask;
    }

    pthread_mutex_unlock(&S->mu);
}

// ============================ Copy Engine ============================

typedef struct {
    const Options *opt;
    DigestMap *dmap;
    InoMap *imap;
    PhysMap *pmap;
    Stats *stats;
} IOContext;

static unsigned g_tmp_counter = 0;

static unsigned next_tmp_counter(void) {
    return __atomic_add_fetch(&g_tmp_counter, 1, __ATOMIC_RELAXED);
}

static void make_temp_name(const char *final_name, char *out, size_t outsz) {
    snprintf(
        out,
        outsz,
        ".%s%s.%ld.%u",
        final_name,
        TMP_SUFFIX,
        (long)getpid(),
        next_tmp_counter()
    );
}

static int create_dir_like(const Options *opt, const char *src_dir, const char *dst_dir, Stats *st) {
    struct stat sdst;

    /* Fast-path: destination already exists */
    if (lstat(dst_dir, &sdst) == 0) {
        if (!S_ISDIR(sdst.st_mode)) {
            warnx("destination exists and not a dir: %s", dst_dir);
            return -1;
        }

        if (!opt->dry_run) {
            /* Apply directory metadata per policy */
            if (opt->meta_mode == META_FULL) {
                if (copyfile(src_dir, dst_dir, NULL, COPYFILE_METADATA | COPYFILE_NOFOLLOW) != 0) {
                    vmsg("metadata(dir,full): %s -> %s: %s", src_dir, dst_dir, strerror(errno));
                }
            } else if (opt->meta_mode == META_SAFE) {
                /* SAFE: mode only (skip utimensat here to avoid extra syscalls; dirs will be dirtied anyway) */
                struct stat ssrc;
                if (lstat(src_dir, &ssrc) == 0) {
                    (void)chmod(dst_dir, ssrc.st_mode & 07777);
                    /* intentionally omit utimensat for dirs in SAFE mode */
                }
            } /* META_MINIMAL: do nothing */

            /* Durability */
            if (opt->dur_mode == DUR_STRICT) {
                int dfd = open(dst_dir, O_RDONLY | O_CLOEXEC);

                if (dfd >= 0) {
                    (void)fsync_full(dfd);
                    close(dfd);
                }

                char pdir[PATH_MAX];
                path_dirname(dst_dir, pdir, sizeof(pdir));

                int pdfd = open(pdir, O_RDONLY | O_CLOEXEC);
                if (pdfd >= 0) {
                    (void)fsync_full(pdfd);
                    close(pdfd);
                }
            } else {
                /* DUR_BATCH: count durability against the parent directory (reuse cached fd) */
                char pdir[PATH_MAX];
                path_dirname(dst_dir, pdir, sizeof(pdir));

                int pdfd = dirfd_cache_open(pdir);
                if (pdfd >= 0) {
                    dirb_note(pdfd, 0, opt); /* no open/close of dst_dir */
                }
            }
        } else {
            pmsg("PLAN   MKDIR %s (exists, metadata policy=%d)", dst_dir, (int)opt->meta_mode);
            (void)__atomic_add_fetch(&st->dirs_created, 1, __ATOMIC_RELAXED);
        }
        return 0;
    }

    if (opt->dry_run) {
        pmsg("PLAN   MKDIR %s", dst_dir);
        (void)__atomic_add_fetch(&st->dirs_created, 1, __ATOMIC_RELAXED);
        return 0;
    }

    /* Create just the final directory; if missing parents, ensure once and retry */
    if (mkdir(dst_dir, 0777) != 0) {
        if (errno == ENOENT) {
            int rc = ensure_parent_dirs_for(dst_dir);

            if (rc != 0) {
                int e = errno;

                if (is_perm_error(e) || rc == -2) {
                    warnx("mkdir: %s: %s (parents blocked)", dst_dir, strerror(e));

                    return -1; /* skip this dir subtree */
                }

                warnx("mkdir: %s: %s", dst_dir, strerror(e));

                return -1;
            }

            if (mkdir(dst_dir, 0777) != 0 && errno != EEXIST) {
                int e = errno;

                warnx("mkdir: %s: %s", dst_dir, strerror(e));

                return -1;
            }
        } else if (errno != EEXIST) {
            int e = errno;

            warnx("mkdir: %s: %s", dst_dir, strerror(e));

            return -1;
        }
    }

    (void)__atomic_add_fetch(&st->dirs_created, 1, __ATOMIC_RELAXED);

    /* Metadata per policy */
    if (opt->meta_mode == META_FULL) {
        if (copyfile(src_dir, dst_dir, NULL, COPYFILE_METADATA | COPYFILE_NOFOLLOW) != 0) {
            vmsg("metadata(dir,full): %s -> %s: %s", src_dir, dst_dir, strerror(errno));
        }
    } else if (opt->meta_mode == META_SAFE) {
        struct stat ssrc;
        if (lstat(src_dir, &ssrc) == 0) {
            (void)chmod(dst_dir, ssrc.st_mode & 07777);
            /* intentionally omit utimensat for dirs in SAFE mode */
        }
    } /* META_MINIMAL: do nothing */

    /* Durability handling */
    if (opt->dur_mode == DUR_STRICT) {
        int dfd = open(dst_dir, O_RDONLY | O_CLOEXEC);
        if (dfd >= 0) {
            (void)fsync_full(dfd);
            close(dfd);
        }

        char pdir[PATH_MAX];
        path_dirname(dst_dir, pdir, sizeof(pdir));

        int pdfd = open(pdir, O_RDONLY | O_CLOEXEC);
        if (pdfd >= 0) {
            (void)fsync_full(pdfd);
            close(pdfd);
        }
    } else {
        /* DUR_BATCH: account durability on the parent directory only */
        char pdir[PATH_MAX];
        path_dirname(dst_dir, pdir, sizeof(pdir));

        int pdfd = dirfd_cache_open(pdir);
        if (pdfd >= 0) {
            dirb_note(pdfd, 0, opt);
            close(pdfd);
        }
    }

    return 0;
}

static int maybe_should_overwrite(const Options *opt, const struct stat *st_src, const struct stat *st_dst) {
    if (opt->clobber == CLOBBER_OVERWRITE) {
        return 1;
    }

    if (opt->clobber == CLOBBER_UPDATE) {
        if (st_src->st_mtimespec.tv_sec > st_dst->st_mtimespec.tv_sec) {
            return 1;
        }

        if (st_src->st_mtimespec.tv_sec == st_dst->st_mtimespec.tv_sec &&
            st_src->st_mtimespec.tv_nsec > st_dst->st_mtimespec.tv_nsec) {
            return 1;
        }

        if (st_src->st_size != st_dst->st_size) {
            return 1;
        }

        return 0;
    }

    return 0;
}

static bool same_volume_path_dirfd(const char *src_path, int ddirfd) {
    struct stat s1, s2;

    if (lstat(src_path, &s1) != 0) {
        return false;
    }

    if (fstat(ddirfd, &s2) != 0) {
        return false;
    }

    return s1.st_dev == s2.st_dev;
}

static void copy_metadata_policy_fd(const Options *opt, int sfd, int dfd, const char *src_path,
                                    const char *dst_path) {
    (void)opt;

    copyfile_flags_t f = COPYFILE_METADATA | COPYFILE_NOFOLLOW;
    copyfile_state_t st = copyfile_state_alloc();

    if (st) {
        if (fcopyfile(sfd, dfd, st, f) != 0) {
            if (copyfile(src_path, dst_path, NULL, f) != 0) {
                warnx("metadata: %s -> %s: %s", src_path, dst_path, strerror(errno));
            }
        }
        copyfile_state_free(st);
    } else {
        if (copyfile(src_path, dst_path, NULL, f) != 0) {
            warnx("metadata: %s -> %s: %s", src_path, dst_path, strerror(errno));
        }
    }
}

static int rename_tmp_to_final(int ddirfd, const char *tmp, const char *final, const Options *opt) {
#if HAVE_RENAMEATX_NP
    unsigned flags = (opt->clobber == CLOBBER_NO) ? RENAME_EXCL : 0;

    return renameatx_np(ddirfd, tmp, ddirfd, final, flags);
#else
    if (opt->clobber == CLOBBER_NO) {
        struct stat st;

        if (fstatat(ddirfd, final, &st, AT_SYMLINK_NOFOLLOW) == 0) {
            errno = EEXIST;

            return -1;
        }
    }

    return renameat(ddirfd, tmp, ddirfd, final);
#endif
}

static int try_clone_to_tmp_fd(int sfd, int ddirfd, const char *tmpname) {
    (void)unlinkat(ddirfd, tmpname, 0);

    if (fclonefileat(sfd, ddirfd, tmpname, 0) == 0) {
        int tfd = openat(ddirfd, tmpname, O_RDONLY | O_NOFOLLOW);

        if (tfd >= 0) {
            (void)fsync_full(tfd);
            close(tfd);
        }

        return 0;
    }

    return -1;
}

static int try_clone_to_tmp_path(const char *src_path, int ddirfd, const char *tmpname) {
    int sfd = open(src_path, O_RDONLY | O_NOFOLLOW);

    if (sfd < 0) {
        return -1;
    }

    int rc = try_clone_to_tmp_fd(sfd, ddirfd, tmpname);

    close(sfd);

    return rc;
}

static int try_fcopyfile_kernel(int sfd, int dfd, bool try_clone) {
    copyfile_state_t st = copyfile_state_alloc();
    copyfile_flags_t f = COPYFILE_ALL | COPYFILE_NOFOLLOW;

    if (try_clone) {
        f |= COPYFILE_CLONE;
    }

    int rc = fcopyfile(sfd, dfd, st, f);

    copyfile_state_free(st);

    return rc;
}

static int copy_streaming_dense(const Options *opt, int sfd, int ddirfd, const char *tmpname, off_t total_sz,
                                Stats *stats, unsigned char *opt_digest) {
#ifdef F_NOCACHE
    (void)fcntl(sfd, F_NOCACHE, 1);
#endif
    int dfd = openat(ddirfd, tmpname, O_WRONLY | O_CREAT | O_EXCL, 0600);
    if (dfd < 0) {
        return -1;
    }
#ifdef F_NOCACHE
    (void)fcntl(dfd, F_NOCACHE, 1);
#endif
    if (total_sz > 0) {
        fstore_t fs = {0};
        fs.fst_flags = F_ALLOCATECONTIG;
        fs.fst_posmode = F_PEOFPOSMODE;
        fs.fst_length = total_sz;
        if (fcntl(dfd, F_PREALLOCATE, &fs) == -1) {
            fs.fst_flags = F_ALLOCATEALL;
            (void)fcntl(dfd, F_PREALLOCATE, &fs);
        }
        (void)ftruncate(dfd, total_sz);
    }
#ifdef F_RDAHEAD
    (void)fcntl(sfd, F_RDAHEAD, 1);
#endif
#ifdef F_RDADVISE
    struct radvisory ra = {.ra_offset = 0, .ra_count = (int)opt->rdadvise_window};
    (void)fcntl(sfd, F_RDADVISE, &ra);
#endif
    XXH3_state_t* state = NULL;

    if (opt_digest) {
        state = XXH3_createState();

        if (state) {
            XXH3_128bits_reset(state);
        }
    }

    BufNode *buf = bp_acquire();

    for (;;) {
        ssize_t r = read(sfd, buf->data, buf->sz);

        if (r < 0) {
            if (errno == EINTR) {
                continue;
            }

            bp_release(buf);
            close(dfd);

            if (state) {
                XXH3_freeState(state);
            }

            return -1;
        }

        if (r == 0) {
            break;
        }

        size_t off = 0;

        while (off < (size_t)r) {
            ssize_t w = write(dfd, buf->data + off, (size_t)r - off);

            if (w < 0) {
                if (errno == EINTR) {
                    continue;
                }

                if (errno == EAGAIN) {
                    (void)sched_yield();
                    continue;
                }

                bp_release(buf);
                close(dfd);

                if (state) {
                    XXH3_freeState(state);
                }

                return -1;
            }

            off += (size_t)w;

            /* Logical progress + physical I/O bytes */
            (void)__atomic_add_fetch(&stats->bytes_done,    (uint64_t)w, __ATOMIC_RELAXED);
            (void)__atomic_add_fetch(&stats->bytes_phys_io, (uint64_t)w, __ATOMIC_RELAXED);
        }

        if (state) {
            XXH3_128bits_update(state, buf->data, (size_t)r);
        }
    }

    if (opt->dur_mode == DUR_STRICT) {
        (void)fsync_full(dfd);
    }

    bp_release(buf);

    if (state) {
        XXH128_hash_t const hash = XXH3_128bits_digest(state);
        memcpy(opt_digest, &hash, sizeof(hash));
        XXH3_freeState(state);
    }

    close(dfd);

    return 0;
}

static inline bool looks_sparse(const struct stat *st) {
    uint64_t alloc = (uint64_t)st->st_blocks * 512ull;

    return S_ISREG(st->st_mode) && st->st_size > 0 && alloc < (uint64_t)st->st_size;
}

static int copy_streaming_sparse(const Options *opt, int sfd, int ddirfd, const char *tmpname, off_t total_sz,
                                 Stats *stats, unsigned char *opt_digest) {
#ifdef F_NOCACHE
    (void)fcntl(sfd, F_NOCACHE, 1);
#endif
    (void)unlinkat(ddirfd, tmpname, 0);
    int dfd = openat(ddirfd, tmpname, O_WRONLY | O_CREAT | O_EXCL, 0600);
    if (dfd < 0) {
        return -1;
    }
#ifdef F_NOCACHE
    (void)fcntl(dfd, F_NOCACHE, 1);
#endif
#ifdef F_RDAHEAD
    (void)fcntl(sfd, F_RDAHEAD, 1);
#endif
#ifdef F_RDADVISE
    struct radvisory ra = {.ra_offset = 0, .ra_count = (int)opt->rdadvise_window};
    (void)fcntl(sfd, F_RDADVISE, &ra);
#endif

    XXH3_state_t* state = NULL;

    if (opt_digest) {
        state = XXH3_createState();
        if (state) {
            XXH3_128bits_reset(state);
        }
    }

    errno = 0;
    off_t pos = lseek(sfd, 0, SEEK_DATA);

    bool have_holes = !(pos == (off_t)-1 && (errno == EINVAL || errno == ENXIO));

    if (!have_holes) {
        close(dfd);
        if (state) XXH3_freeState(state);
        return copy_streaming_dense(opt, sfd, ddirfd, tmpname, total_sz, stats, opt_digest);
    }

    BufNode *buf = bp_acquire();

    off_t off = 0;

    while (off < total_sz) {
        off_t data = lseek(sfd, off, SEEK_DATA);

        if (data == (off_t)-1) {
            if (errno == ENXIO) {
                if (off < total_sz) {
                    uint64_t hole = (uint64_t)(total_sz - off);

                    /* logical progress only for holes */
                    (void)__atomic_add_fetch(&stats->bytes_done, hole, __ATOMIC_RELAXED);
                }

                break;
            }

            bp_release(buf);
            close(dfd);

            if (state) {
                XXH3_freeState(state);
            }

            return -1;
        }

        if (off < data) {
            uint64_t hole = (uint64_t)(data - off);
            /* logical progress only for holes */
            (void)__atomic_add_fetch(&stats->bytes_done, hole, __ATOMIC_RELAXED);
        }

        off_t hole = lseek(sfd, data, SEEK_HOLE);

        if (hole == (off_t)-1) {
            bp_release(buf);
            close(dfd);

            if (state) {
                XXH3_freeState(state);
            }

            return -1;
        }

        if (hole > total_sz) {
            hole = total_sz;
        }

        if (lseek(sfd, data, SEEK_SET) == (off_t)-1) {
            bp_release(buf);
            close(dfd);

            if (state) {
                XXH3_freeState(state);
            }

            return -1;
        }

        if (lseek(dfd, data, SEEK_SET) == (off_t)-1) {
            bp_release(buf);
            close(dfd);

            if (state) {
                XXH3_freeState(state);
            }

            return -1;
        }

        off_t to_copy = hole - data;

        while (to_copy > 0) {
            size_t chunk = (size_t)((to_copy > (off_t)buf->sz) ? buf->sz : to_copy);

            ssize_t r = read(sfd, buf->data, chunk);

            if (r < 0) {
                if (errno == EINTR) {
                    continue;
                }

                bp_release(buf);
                close(dfd);

                if (state) {
                    XXH3_freeState(state);
                }

                return -1;
            }

            if (r == 0) {
                break;
            }

            size_t woff = 0;

            while (woff < (size_t)r) {
                ssize_t w = write(dfd, buf->data + woff, (size_t)r - woff);

                if (w < 0) {
                    if (errno == EINTR) {
                        continue;
                    }

                    if (errno == EAGAIN) {
                        (void)sched_yield();
                        continue;
                    }

                    bp_release(buf);
                    close(dfd);

                    if (state) {
                        XXH3_freeState(state);
                    }

                    return -1;
                }

                woff += (size_t)w;

                /* Logical progress + physical I/O bytes for data regions */
                (void)__atomic_add_fetch(&stats->bytes_done,    (uint64_t)w, __ATOMIC_RELAXED);
                (void)__atomic_add_fetch(&stats->bytes_phys_io, (uint64_t)w, __ATOMIC_RELAXED);
            }

            if (state) {
                XXH3_128bits_update(state, buf->data, (size_t)r);
            }

            to_copy -= r;
        }

        off = hole;
    }

    if (ftruncate(dfd, total_sz) != 0) {
        bp_release(buf);
        close(dfd);

        if (state) {
            XXH3_freeState(state);
        }

        return -1;
    }

    if (opt->dur_mode == DUR_STRICT) {
        (void)fsync_full(dfd);
    }

    bp_release(buf);

    if (state) {
        XXH128_hash_t const hash = XXH3_128bits_digest(state);

        memcpy(opt_digest, &hash, sizeof(hash));

        XXH3_freeState(state);
    }

    close(dfd);

    return 0;
}

static int copy_file_with_strategies(const IOContext *io, const Entry *e, const char *dst_final) {
    const Options *opt = io->opt;
    Stats         *st  = io->stats;

    if (!opt->dry_run) {
        int ec = ensure_parent_dirs_for(dst_final);

        if (ec != 0) {
            int err = errno;

            if (is_perm_error(err) || ec == -2) {
                if (g_verbose) {
                    pmsg("SKIP   %s (perm-denied parents: %s)", dst_final, strerror(err));
                }

                (void)__atomic_add_fetch(&st->files_skipped, 1, __ATOMIC_RELAXED);
                (void)__atomic_add_fetch(&st->bytes_skipped, (uint64_t)e->size, __ATOMIC_RELAXED);

                return 0; /* soft-skip */
            }

            /* Non-permission mkdir failure: mark as failed job (non-fatal overall) */
            warnx("mkdir parents for %s: %s", dst_final, strerror(err));

            return -1;
        }
    }

    struct stat st_dst, st_src;

    bool dst_exists = (lstat(dst_final, &st_dst) == 0);

    if (lstat(e->src_path, &st_src) != 0) {
        warnx("stat src: %s: %s", e->src_path, strerror(errno));
        return -1;
    }

    /* Refresh entry’s size so all downstream size uses are consistent. */
    ((Entry *)e)->size = st_src.st_size;

    /* --- capture live inode topology for hardlink decisions --- */
    dev_t   cur_dev   = st_src.st_dev;
    ino_t   cur_ino   = st_src.st_ino;
    nlink_t cur_nlink = st_src.st_nlink;
    off_t   cur_size  = st_src.st_size;

    /* Optionally also keep the Entry coherent (not strictly required) */
    ((Entry *)e)->dev   = cur_dev;
    ((Entry *)e)->ino   = cur_ino;
    ((Entry *)e)->nlink = cur_nlink;

    /* Invalidate any precomputed digest if the inode changed since enumeration. */
    if (e->hashed &&
        (e->size != st_src.st_size ||
         e->mtime.tv_sec  != st_src.st_mtimespec.tv_sec  ||
         e->mtime.tv_nsec != st_src.st_mtimespec.tv_nsec ||
         e->ctime.tv_sec  != st_src.st_ctimespec.tv_sec  ||
         e->ctime.tv_nsec != st_src.st_ctimespec.tv_nsec)) {
        ((Entry *)e)->hashed = false; /* stale pre-hash -> do not trust */
    }

    /* Existing destination policy */
    if (dst_exists && !maybe_should_overwrite(opt, &st_src, &st_dst)) {
        if (opt->delete_source_existing) {
            /* Attempt equality with logging of the reason under -v */
            DeqReason er = files_equal_with_reason(opt,
                                                   e->src_path, &st_src,
                                                   dst_final,  &st_dst,
                                                   /*verbose_log=*/(g_verbose > 0));
            if (er == DEQ_OK) {
                if (opt->dry_run) {
                    pmsg("PLAN   DELSRC %s (existing-equal)", e->src_path);
                    (void)__atomic_add_fetch(&st->files_skipped,     1,                 __ATOMIC_RELAXED);
                    (void)__atomic_add_fetch(&st->bytes_skipped,     (uint64_t)e->size, __ATOMIC_RELAXED);
                    (void)__atomic_add_fetch(&st->files_deleted_src, 1,                 __ATOMIC_RELAXED);
                    (void)__atomic_add_fetch(&st->bytes_deleted_src, (uint64_t)e->size, __ATOMIC_RELAXED);
                    (void)__atomic_add_fetch(&st->bytes_done,        (uint64_t)e->size, __ATOMIC_RELAXED);
                    return 0;
                }

                if (unlink(e->src_path) == 0) {
                    pmsg("DEL    %s (existing-equal)", e->src_path);
                    (void)__atomic_add_fetch(&st->files_skipped,     1,                 __ATOMIC_RELAXED);
                    (void)__atomic_add_fetch(&st->bytes_skipped,     (uint64_t)e->size, __ATOMIC_RELAXED);
                    (void)__atomic_add_fetch(&st->files_deleted_src, 1,                 __ATOMIC_RELAXED);
                    (void)__atomic_add_fetch(&st->bytes_deleted_src, (uint64_t)e->size, __ATOMIC_RELAXED);
                    (void)__atomic_add_fetch(&st->bytes_done,        (uint64_t)e->size, __ATOMIC_RELAXED);
                    return 0;
                }

                warnx("unlink: %s: %s", e->src_path, strerror(errno));
                (void)__atomic_add_fetch(&st->files_skipped, 1, __ATOMIC_RELAXED);
                (void)__atomic_add_fetch(&st->bytes_skipped, (uint64_t)e->size, __ATOMIC_RELAXED);
                (void)__atomic_add_fetch(&st->bytes_done,    (uint64_t)e->size, __ATOMIC_RELAXED);
                return 0;
            } else {
                /* When verbose, files_equal_with_reason already printed the concrete reason. */
                if (g_verbose) {
                    /* No further action; fall through to SKIP accounting. */
                }
            }
        }

        if (opt->dry_run) pmsg("PLAN   SKIP (exists) %s", dst_final);
        else              pmsg("SKIP   %s", dst_final);

        (void)__atomic_add_fetch(&st->files_skipped, 1, __ATOMIC_RELAXED);
        (void)__atomic_add_fetch(&st->bytes_skipped, (uint64_t)e->size, __ATOMIC_RELAXED);
        (void)__atomic_add_fetch(&st->bytes_done,    (uint64_t)e->size, __ATOMIC_RELAXED);
        return 0;
    }

    /* Resolve destination dir FD */
    char ddir_path[PATH_MAX];
    path_dirname(dst_final, ddir_path, sizeof(ddir_path));
    int ddirfd = dirfd_cache_open(ddir_path);
    if (ddirfd < 0) {
        warnx("open dir: %s: %s", ddir_path, strerror(errno));
        return -1;
    }

    const char *fname = path_basename(dst_final);

    /* ===================== HARDLINK QUICK-PATH ===================== */
    if (opt->preserve_hardlinks && cur_nlink > 1) {
        InoKey k = { .dev = cur_dev, .ino = cur_ino };
        char  *first = imap_get(io->imap, k);

        /* Sanity-check: if we have an anchor, it must match source size */
        if (first) {
            struct stat st_first;
            if (lstat(first, &st_first) != 0 || st_first.st_size != cur_size) {
                first = NULL; /* stale/mismatched anchor: disable hard-link path */
            }
        }

        if (first) {
            if (opt->dry_run) {
                pmsg("PLAN   HARDLINK %s <= %s", dst_final, first);
                (void)__atomic_add_fetch(&st->files_hardlinked, 1,                 __ATOMIC_RELAXED);
                (void)__atomic_add_fetch(&st->bytes_hardlinked, (uint64_t)e->size, __ATOMIC_RELAXED);
                (void)__atomic_add_fetch(&st->bytes_done,       (uint64_t)e->size, __ATOMIC_RELAXED);
                (void)verify_and_delete_source(opt, e, dst_final, st);
                return 0;
            }

            char tmpname[NAME_MAX + 128];
            make_temp_name(fname, tmpname, sizeof(tmpname));
            (void)unlinkat(ddirfd, tmpname, 0);

            char fdir[PATH_MAX];
            path_dirname(first, fdir, sizeof(fdir));
            int fdfd = dirfd_cache_open(fdir);

            if (fdfd >= 0 &&
                linkat(fdfd, path_basename(first), ddirfd, tmpname, 0) == 0)
            {
                if (rename_tmp_to_final(ddirfd, tmpname, fname, opt) != 0) {
                    warnx("rename after hardlink: %s -> %s: %s",
                         tmpname, fname, strerror(errno));
                    (void)unlinkat(ddirfd, tmpname, 0);
                    /* fall through to COPY/CLONE fallback */
                    goto COPY_FALLBACK;
                }

                /* --- verify that the newly linked dst matches live source size --- */
                struct stat st_newdst;
                bool link_ok = (fstatat(ddirfd, fname, &st_newdst, AT_SYMLINK_NOFOLLOW) == 0) &&
                               (st_newdst.st_size == cur_size);
                if (!link_ok) {
                    /* Mismatched hard-link target (likely stale anchor). Undo and fall back. */
                    (void)unlinkat(ddirfd, fname, 0);
                    goto COPY_FALLBACK;
                }

                if (opt->dur_mode == DUR_STRICT) (void)fsync_full(ddirfd);
                else                              dirb_note(ddirfd, (size_t)e->size, opt);

                if (copyfile(e->src_path, dst_final, NULL,
                             COPYFILE_METADATA | COPYFILE_NOFOLLOW) != 0)
                {
                    vmsg("metadata(min): %s -> %s: %s",
                         e->src_path, dst_final, strerror(errno));
                }

                pmsg("LINK   %s <= %s", dst_final, first);

                (void)__atomic_add_fetch(&st->files_hardlinked, 1,                 __ATOMIC_RELAXED);
                (void)__atomic_add_fetch(&st->bytes_hardlinked, (uint64_t)e->size, __ATOMIC_RELAXED);
                (void)__atomic_add_fetch(&st->bytes_done,       (uint64_t)e->size, __ATOMIC_RELAXED);

                /* Record live inode anchor for future members of this hard-link set */
                if (opt->preserve_hardlinks && cur_nlink > 1) {
                    InoKey kk = { .dev = cur_dev, .ino = cur_ino };
                    imap_put(io->imap, kk, dst_final);
                }

                (void)verify_and_delete_source(opt, e, dst_final, st);
                return 0;
            }

            /* If linkat/rename failed, fall through to COPY/CLONE fallback. */
        }
    }

COPY_FALLBACK:
    /* ===================== DEDUPE / CLONE / COPY ===================== */

    /* Autotune gating (disable clone paths when forced) */
    const bool disable_clone = (opt->force_streaming || opt->force_kernel_only);

    /* Dedupe via physical fingerprint → clone from anchor */
    if (!disable_clone && e->has_physfp && e->physfp != 0) {
        char *anchor = pmap_get(io->pmap, e->physfp);

        if (anchor && same_volume_path_dirfd(anchor, ddirfd)) {
            if (opt->dry_run) {
                pmsg("PLAN   CLONE(dedupe-phys) %s <= %s", dst_final, anchor);
                (void)__atomic_add_fetch(&st->files_cloned_dedupe, 1,                 __ATOMIC_RELAXED);
                (void)__atomic_add_fetch(&st->bytes_cloned_dedupe, (uint64_t)e->size, __ATOMIC_RELAXED);
                (void)__atomic_add_fetch(&st->bytes_done,          (uint64_t)e->size, __ATOMIC_RELAXED);
                (void)verify_and_delete_source(opt, e, dst_final, st);
                return 0;
            }

            char tmpname[NAME_MAX + 128];
            make_temp_name(fname, tmpname, sizeof(tmpname));

            if (try_clone_to_tmp_path(anchor, ddirfd, tmpname) == 0) {
                int tfd = openat(ddirfd, tmpname, O_RDONLY | O_NOFOLLOW);
                if (tfd >= 0) {
                    int sfd = open(e->src_path, O_RDONLY | O_NOFOLLOW);
                    if (sfd >= 0) {
                        copy_metadata_policy_fd(opt, sfd, tfd, e->src_path, dst_final);
                        close(sfd);
                    }
                    close(tfd);
                }

                if (rename_tmp_to_final(ddirfd, tmpname, fname, opt) != 0) {
                    warnx("rename tmp->dst: %s -> %s: %s", tmpname, fname, strerror(errno));
                    (void)unlinkat(ddirfd, tmpname, 0);
                    return -1;
                }

                if (opt->dur_mode == DUR_STRICT) (void)fsync_full(ddirfd);
                else                              dirb_note(ddirfd, (size_t)e->size, opt);

                pmsg("CLONE  %s <= %s (dedupe-phys)", dst_final, anchor);

                (void)__atomic_add_fetch(&st->files_cloned_dedupe, 1,                 __ATOMIC_RELAXED);
                (void)__atomic_add_fetch(&st->bytes_cloned_dedupe, (uint64_t)e->size, __ATOMIC_RELAXED);
                (void)__atomic_add_fetch(&st->bytes_done,          (uint64_t)e->size, __ATOMIC_RELAXED);

                if (e->has_physfp) {
                    pmap_put(io->pmap, e->physfp, dst_final);
                }

                /* seed hardlink map with live metadata if relevant */
                if (opt->preserve_hardlinks && cur_nlink > 1) {
                    InoKey kk = { .dev = cur_dev, .ino = cur_ino };
                    imap_put(io->imap, kk, dst_final);
                }

                (void)verify_and_delete_source(opt, e, dst_final, st);
                return 0;
            }
        }
    }

    /* Dedupe via content hash → clone from anchor */
    if (!disable_clone && e->hashed) {
        char *anchor = dmap_get(io->dmap, e->digest);

        if (anchor && same_volume_path_dirfd(anchor, ddirfd)) {
            if (opt->dry_run) {
                pmsg("PLAN   CLONE(dedupe) %s <= %s", dst_final, anchor);
                (void)__atomic_add_fetch(&st->files_cloned_dedupe, 1,                 __ATOMIC_RELAXED);
                (void)__atomic_add_fetch(&st->bytes_cloned_dedupe, (uint64_t)e->size, __ATOMIC_RELAXED);
                (void)__atomic_add_fetch(&st->bytes_done,          (uint64_t)e->size, __ATOMIC_RELAXED);
                (void)verify_and_delete_source(opt, e, dst_final, st);
                return 0;
            }

            char tmpname[NAME_MAX + 128];
            make_temp_name(fname, tmpname, sizeof(tmpname));

            if (try_clone_to_tmp_path(anchor, ddirfd, tmpname) == 0) {
                int tfd = openat(ddirfd, tmpname, O_RDONLY | O_NOFOLLOW);
                if (tfd >= 0) {
                    int sfd = open(e->src_path, O_RDONLY | O_NOFOLLOW);
                    if (sfd >= 0) {
                        copy_metadata_policy_fd(opt, sfd, tfd, e->src_path, dst_final);
                        close(sfd);
                    }
                    close(tfd);
                }

                if (rename_tmp_to_final(ddirfd, tmpname, fname, opt) != 0) {
                    warnx("rename tmp->dst: %s -> %s: %s", tmpname, fname, strerror(errno));
                    (void)unlinkat(ddirfd, tmpname, 0);
                    return -1;
                }

                if (opt->dur_mode == DUR_STRICT) (void)fsync_full(ddirfd);
                else                              dirb_note(ddirfd, (size_t)e->size, opt);

                pmsg("CLONE  %s <= %s (dedupe)", dst_final, anchor);

                (void)__atomic_add_fetch(&st->files_cloned_dedupe, 1,                 __ATOMIC_RELAXED);
                (void)__atomic_add_fetch(&st->bytes_cloned_dedupe, (uint64_t)e->size, __ATOMIC_RELAXED);
                (void)__atomic_add_fetch(&st->bytes_done,          (uint64_t)e->size, __ATOMIC_RELAXED);

                /* seed hardlink map with live metadata if relevant */
                if (opt->preserve_hardlinks && cur_nlink > 1) {
                    InoKey kk = { .dev = cur_dev, .ino = cur_ino };
                    imap_put(io->imap, kk, dst_final);
                }

                (void)verify_and_delete_source(opt, e, dst_final, st);
                return 0;
            }
        }
    }

    /* --------------------- Streaming/KERNEL copy section --------------------- */
    /* Reserve inflight window for streaming fallback */
    const off_t src_sz = st_src.st_size;

    size_t reserve = (size_t)((src_sz > 0 && src_sz < (off_t)SIZE_MAX)
                       ? src_sz : (off_t)(16 << 20));

    tb_reserve(reserve);

    char tmpname[NAME_MAX + 128];
    make_temp_name(fname, tmpname, sizeof(tmpname));

    bool success = false;

    /* Same-volume clone (unless disabled by autotune) */
    if (!disable_clone && same_volume_path_dirfd(e->src_path, ddirfd)) {
        if (try_clone_to_tmp_path(e->src_path, ddirfd, tmpname) == 0) {
            int tfd = openat(ddirfd, tmpname, O_RDONLY | O_NOFOLLOW);
            if (tfd >= 0) {
                int sfd = open(e->src_path, O_RDONLY | O_NOFOLLOW);
                if (sfd >= 0) {
                    copy_metadata_policy_fd(opt, sfd, tfd, e->src_path, dst_final);
                    close(sfd);
                }
                close(tfd);
            }

            if (rename_tmp_to_final(ddirfd, tmpname, fname, opt) != 0) {
                warnx("rename tmp->dst: %s -> %s: %s",
                     tmpname, fname, strerror(errno));
                (void)unlinkat(ddirfd, tmpname, 0);
                tb_release(reserve);
                return -1;
            }

            if (opt->dur_mode == DUR_STRICT) (void)fsync_full(ddirfd);
            else                              dirb_note(ddirfd, (size_t)e->size, opt);

            pmsg("CLONE  %s <= %s", dst_final, e->src_path);

            (void)__atomic_add_fetch(&st->files_cloned_src, 1,                 __ATOMIC_RELAXED);
            (void)__atomic_add_fetch(&st->bytes_cloned_src, (uint64_t)e->size, __ATOMIC_RELAXED);
            (void)__atomic_add_fetch(&st->bytes_done,       (uint64_t)e->size, __ATOMIC_RELAXED);

            success = true;
        }
    }

    int rc  = -1;
    int sfd = -1;

    if (!success) {
        sfd = open(e->src_path, O_RDONLY | O_NOFOLLOW);
        if (sfd < 0) {
            tb_release(reserve);
            warnx("open src: %s: %s", e->src_path, strerror(errno));
            return -1;
        }

        /* Kernel-assisted copy (optionally forced for benchmarking) */
        if (opt->use_kernel_copy || opt->force_kernel_only) {
            int dfd = openat(ddirfd, tmpname, O_WRONLY | O_CREAT | O_EXCL, 0600);
            if (dfd >= 0) {
                const bool try_clone_flag = (opt->use_kernel_copy && !opt->force_kernel_only);
                rc = try_fcopyfile_kernel(sfd, dfd, try_clone_flag);

                if (rc == 0) {
                    if (opt->dur_mode == DUR_STRICT) {
                        (void)fsync_full(dfd);
                    }
                    (void)__atomic_add_fetch(&st->bytes_done,    (uint64_t)e->size, __ATOMIC_RELAXED);
                    (void)__atomic_add_fetch(&st->bytes_phys_io, (uint64_t)e->size, __ATOMIC_RELAXED);
                }

                close(dfd);

                if (rc != 0) {
                    (void)unlinkat(ddirfd, tmpname, 0);
                }
            }

            if (opt->force_kernel_only && rc != 0) {
                close(sfd);
                tb_release(reserve);
                warnx("kernel-copy path failed for benchmarking");
                return -1;
            }
        }

        /* Streaming fallback */
        if (rc != 0 && !opt->force_kernel_only) {
            unsigned char digest_tmp[DIGEST_LEN];
            bool          want_digest = !e->hashed;

            if (opt->sparse_copy && looks_sparse(&st_src)) {
                rc = copy_streaming_sparse(opt, sfd, ddirfd, tmpname, src_sz, st, want_digest ? digest_tmp : NULL);
            } else {
                rc = copy_streaming_dense(opt, sfd, ddirfd, tmpname, src_sz, st, want_digest ? digest_tmp : NULL);
            }

            if (rc == 0 && want_digest) {
                ((Entry *)e)->hashed = true;
                memcpy(((Entry *)e)->digest, digest_tmp, DIGEST_LEN);

                HashCacheEnt ent = {
                    .dev        = e->dev,
                    .ino        = e->ino,
                    .size       = (uint64_t)e->size,
                    .mtime_sec  = e->mtime.tv_sec,
                    .mtime_nsec = e->mtime.tv_nsec,
                    .ctime_sec  = e->ctime.tv_sec,
                    .ctime_nsec = e->ctime.tv_nsec
                };
                memcpy(ent.digest, e->digest, DIGEST_LEN);
                hcache_append(&ent);
            }
        }

        if (sfd >= 0) close(sfd);

        if (rc != 0) {
            tb_release(reserve);
            warnx("copy: %s -> %s failed: %s", e->src_path, tmpname, strerror(errno));
            (void)unlinkat(ddirfd, tmpname, 0);
            return -1;
        }

        /* Finalize: metadata, rename, durability */
        int tfd = openat(ddirfd, tmpname, O_RDONLY | O_NOFOLLOW);
        if (tfd >= 0) {
            int sfd2 = open(e->src_path, O_RDONLY | O_NOFOLLOW);
            if (sfd2 >= 0) {
                copy_metadata_policy_fd(opt, sfd2, tfd, e->src_path, dst_final);
                close(sfd2);
            }
            close(tfd);
        }

        if (rename_tmp_to_final(ddirfd, tmpname, fname, opt) != 0) {
            warnx("rename tmp->dst: %s -> %s: %s", tmpname, fname, strerror(errno));
            (void)unlinkat(ddirfd, tmpname, 0);
            tb_release(reserve);
            return -1;
        }

        if (opt->dur_mode == DUR_STRICT) (void)fsync_full(ddirfd);
        else                              dirb_note(ddirfd, (size_t)e->size, opt);

        char hb[64];
        pmsg("COPY   %s (%s)", dst_final, human_bytes((double)e->size, hb, sizeof(hb)));

        (void)__atomic_add_fetch(&st->files_copied, 1,                 __ATOMIC_RELAXED);
        (void)__atomic_add_fetch(&st->bytes_copied, (uint64_t)e->size, __ATOMIC_RELAXED);
    }

    /* Record anchors for dedupe/hardlink preservation */
    if (e->hashed)     dmap_put(io->dmap, e->digest, dst_final);
    if (e->has_physfp) pmap_put(io->pmap, e->physfp, dst_final);

    if (opt->preserve_hardlinks && cur_nlink > 1) {
        InoKey k = { .dev = cur_dev, .ino = cur_ino };
        imap_put(io->imap, k, dst_final);
    }

    (void)verify_and_delete_source(opt, e, dst_final, st);

    tb_release(reserve);
    return 0;
}

// ============================ Parallel Execution ============================

typedef struct {
    VecEntry *ents;
    size_t *indices;
    size_t count;
    size_t next;
    pthread_mutex_t mu;
    volatile uint64_t done;
    int bufsize;
} WorkQueue;

typedef struct {
    size_t *indices;
    size_t start;
    size_t end;
    VecEntry *ents;
} SortWork;

typedef struct {
    size_t entry_idx;
    char dst_path[PATH_MAX];
} IOJob;

typedef struct {
    IOJob *queue;
    size_t cap;
    size_t count;
    size_t head;
    size_t tail;
    volatile sig_atomic_t done;
    pthread_mutex_t head_mu;
    pthread_cond_t cv_work;
} IOJobQueue;

typedef struct {
    IOJobQueue *q;
    VecEntry *ents;
    IOContext *ctx;
} IOWorkerCtx;

static void *sample_worker(void *arg) {
    WorkQueue *w = (WorkQueue *)arg;

    for (;;) {
        size_t idx;
        pthread_mutex_lock(&w->mu);

        if (w->next >= w->count) {
            pthread_mutex_unlock(&w->mu);
            break;
        }

        idx = w->indices[w->next++];

        pthread_mutex_unlock(&w->mu);

        Entry *e = &w->ents->v[idx];

        e->sample64 = sample_fingerprint(e->src_path, e->size);

        (void)__atomic_add_fetch(&w->done, 1, __ATOMIC_RELAXED);
    }
    return NULL;
}

static void *hash_worker(void *arg) {
    WorkQueue *w = (WorkQueue *)arg;

    for (;;) {
        size_t idx;
        pthread_mutex_lock(&w->mu);

        if (w->next >= w->count) {
            pthread_mutex_unlock(&w->mu);
            break;
        }

        idx = w->indices[w->next++];

        pthread_mutex_unlock(&w->mu);

        Entry *e = &w->ents->v[idx];

        if (e->type != T_FILE) {
            (void)__atomic_add_fetch(&w->done, 1, __ATOMIC_RELAXED);

            continue;
        }

        if (!e->hashed) {
            if (hcache_lookup(e->dev, e->ino, (uint64_t)e->size, e->mtime.tv_sec, e->mtime.tv_nsec,
                              e->ctime.tv_sec, e->ctime.tv_nsec, e->digest)) {
                e->hashed = true;

                (void)__atomic_add_fetch(&w->done, 1, __ATOMIC_RELAXED);

                continue;
            }
        }

        if (xxh128_file(e->src_path, e->digest, w->bufsize)) {
            e->hashed = true;

            HashCacheEnt ent = {.dev = e->dev,
                                .ino = e->ino,
                                .size = (uint64_t)e->size,
                                .mtime_sec = e->mtime.tv_sec,
                                .mtime_nsec = e->mtime.tv_nsec,
                                .ctime_sec = e->ctime.tv_sec,
                                .ctime_nsec = e->ctime.tv_nsec};

            memcpy(ent.digest, e->digest, DIGEST_LEN);

            hcache_append(&ent);
        } else {
            e->hashed = false;

            warnx("hash failed: %s: %s", e->src_path, strerror(errno));
        }

        (void)__atomic_add_fetch(&w->done, 1, __ATOMIC_RELAXED);
    }

    return NULL;
}

static void *phys_worker(void *arg) {
    WorkQueue *w = (WorkQueue *)arg;
    for (;;) {
        size_t idx;
        pthread_mutex_lock(&w->mu);
        if (w->next >= w->count) {
            pthread_mutex_unlock(&w->mu);
            break;
        }
        idx = w->indices[w->next++];
        pthread_mutex_unlock(&w->mu);

        Entry *e = &w->ents->v[idx];
        if (e->type == T_FILE && e->size >= PHYS_FP_MIN_SIZE) {
            uint64_t fp = phys_fingerprint_path(e->src_path, e->size);
            if (fp) {
                e->has_physfp = true;
                e->physfp = fp;
            }
        }
        (void)__atomic_add_fetch(&w->done, 1, __ATOMIC_RELAXED);
    }
    return NULL;
}

static void insertion_sort_by_sample64(size_t *indices, size_t start, size_t end, VecEntry *ents) {
    for (size_t i = start + 1; i < end; i++) {
        size_t key_idx = indices[i];
        uint64_t key_sample = ents->v[key_idx].sample64;

        size_t j = i;

        while (j > start && ents->v[indices[j - 1]].sample64 > key_sample) {
            indices[j] = indices[j - 1];
            j--;
        }

        indices[j] = key_idx;
    }
}

static void *sort_worker(void *arg) {
    SortWork *work = (SortWork *)arg;

    insertion_sort_by_sample64(
        work->indices,
        work->start,
        work->end,
        work->ents
    );

    return NULL;
}

static void io_queue_init(IOJobQueue *q, size_t initial_cap) {
    q->queue = (IOJob *)malloc(initial_cap * sizeof(IOJob));

    if (!q->queue) {
        die("OOM");
    }

    q->cap = initial_cap;
    q->count = 0;
    q->head = 0;
    q->tail = 0;
    q->done = 0;

    pthread_mutex_init(&q->head_mu, NULL);
    pthread_cond_init(&q->cv_work, NULL);
}

static void io_queue_grow_unlocked(IOJobQueue *q) {
    size_t new_cap = q->cap ? (q->cap << 1) : 4096;

    if (new_cap <= q->count + 1) {
        new_cap = q->count + 1 + (q->count >> 1) + 1;
    }

    IOJob *nq = (IOJob *)malloc(new_cap * sizeof(IOJob));

    if (!nq) {
        die("OOM");
    }

    if (q->count) {
        if (q->head < q->tail) {
            memcpy(nq, &q->queue[q->head], q->count * sizeof(IOJob));
        } else {
            size_t first = q->cap - q->head;
            memcpy(nq, &q->queue[q->head], first * sizeof(IOJob));
            memcpy(nq + first, &q->queue[0], q->tail * sizeof(IOJob));
        }
    }

    free(q->queue);

    q->queue = nq;
    q->cap = new_cap;
    q->head = 0;
    q->tail = q->count;
}

static void io_queue_push(IOJobQueue *q, IOJob job) {
    pthread_mutex_lock(&q->head_mu);

    if (q->count >= q->cap - 1) {
        io_queue_grow_unlocked(q);
    }

    q->queue[q->tail] = job;
    q->tail = (q->tail + 1) % q->cap;
    q->count++;

    pthread_cond_signal(&q->cv_work);
    pthread_mutex_unlock(&q->head_mu);
}

static bool io_queue_pop(IOJobQueue *q, IOJob *job) {
    pthread_mutex_lock(&q->head_mu);

    while (q->count == 0 && !q->done) {
        pthread_cond_wait(&q->cv_work, &q->head_mu);
    }

    if (q->count == 0) {
        pthread_mutex_unlock(&q->head_mu);

        return false;
    }

    *job = q->queue[q->head];

    q->head = (q->head + 1) % q->cap;

    (void)__atomic_fetch_sub(&q->count, 1, __ATOMIC_RELAXED);

    pthread_mutex_unlock(&q->head_mu);

    return true;
}

static void io_queue_finish(IOJobQueue *q) {
    q->done = 1;

    pthread_cond_broadcast(&q->cv_work);
}

static void *io_worker(void *arg) {
    IOWorkerCtx *wctx = (IOWorkerCtx *)arg;
    IOContext *io = wctx->ctx;

    const Options *opt = io->opt;

    Stats *stats = io->stats;

    IOJob job;
    while (io_queue_pop(wctx->q, &job)) {
        Entry *e = &wctx->ents->v[job.entry_idx];

        switch (e->type) {
            case T_DIR: {
                (void)create_dir_like(opt, e->src_path, job.dst_path, stats);
                /* increment directory job completion (counts both created and already-present dirs) */
                (void)__atomic_add_fetch(&stats->dirs_done_jobs, 1, __ATOMIC_RELAXED);
                break;
            }

            case T_SYMLINK: {
                /* Read source link body first */
                char *link_body = NULL;
                size_t link_len = 0;

                if (!readlink_malloc(e->src_path, &link_body, &link_len)) {
                    warnx("readlink: %s: %s", e->src_path, strerror(errno));
                    break;
                }

                /* Fast-path: if destination already exists and is an equal symlink */
                struct stat sdst;
                const bool dst_exists = (lstat(job.dst_path, &sdst) == 0);

                if (dst_exists && S_ISLNK(sdst.st_mode)) {
                    if (symlink_targets_equal_paths(e->src_path, job.dst_path)) {
                        /* Idempotent: optionally delete source when --delete-source-existing */
                        (void)verify_and_delete_symlink_existing_equal(io->opt, e, job.dst_path, io->stats);
                        free(link_body);
                        break;
                    }

                    /* Destination exists but differs: honor clobber policy */
                    if (io->opt->clobber == CLOBBER_NO) {
                        pmsg("SKIP   %s (exists, no-clobber symlink)", job.dst_path);
                        free(link_body);
                        break;
                    }
                } else if (dst_exists) {
                    /* Exists but not a symlink */
                    if (io->opt->clobber == CLOBBER_NO) {
                        pmsg("SKIP   %s (exists, no-clobber)", job.dst_path);
                        free(link_body);
                        break;
                    }
                }

                /* Ensure parent dir and create symlink via temp + atomic rename */
                if (ensure_parent_dirs_for(job.dst_path) != 0) {
                    int e = errno;
                    if (is_perm_error(e)) {
                        if (g_verbose) {
                            pmsg("SKIP   %s (perm-denied parents: %s)", job.dst_path, strerror(e));
                        }
                    } else {
                        warnx("mkdir parents for %s: %s", job.dst_path, strerror(e));
                    }

                    break; /* skip this symlink job, continue others */
                }

                char ddir[PATH_MAX];
                path_dirname(job.dst_path, ddir, sizeof(ddir));
                int ddirfd = dirfd_cache_open(ddir);
                if (ddirfd < 0) {
                    warnx("open dir: %s: %s", ddir, strerror(errno));
                    free(link_body);
                    break;
                }

                const char *fname = path_basename(job.dst_path);
                char tmpname[NAME_MAX + 128];
                make_temp_name(fname, tmpname, sizeof(tmpname));
                (void)unlinkat(ddirfd, tmpname, 0);

                if (symlinkat(link_body, ddirfd, tmpname) != 0) {
                    warnx("symlink: %s -> %s: %s", tmpname, link_body, strerror(errno));
                    (void)unlinkat(ddirfd, tmpname, 0);
                    free(link_body);
                    break;
                }

                if (rename_tmp_to_final(ddirfd, tmpname, fname, io->opt) != 0) {
                    warnx("rename symlink: %s -> %s: %s", tmpname, fname, strerror(errno));
                    (void)unlinkat(ddirfd, tmpname, 0);
                    free(link_body);
                    break;
                }

                /* Durability for the directory entry */
                if (io->opt->dur_mode == DUR_STRICT) {
                    (void)fsync_full(ddirfd);
                } else {
                    dirb_note(ddirfd, 0, io->opt);
                }

                /* Copy symlink metadata only (NOFOLLOW) */
                if (copyfile(e->src_path, job.dst_path, NULL, COPYFILE_METADATA | COPYFILE_NOFOLLOW) != 0) {
                    vmsg("metadata(min, symlink): %s -> %s: %s", e->src_path, job.dst_path, strerror(errno));
                }

                pmsg("SYMLINK %s -> %s", job.dst_path, link_body);
                (void)__atomic_add_fetch(&io->stats->symlinks_created, 1, __ATOMIC_RELAXED);

                /* Optional deletion of source symlink after we made an identical one */
                (void)verify_and_delete_source_symlink(io->opt, e, job.dst_path, io->stats);

                free(link_body);
                break;
            }

            case T_FILE:
                (void)copy_file_with_strategies(io, e, job.dst_path);

                (void)__atomic_add_fetch(&stats->files_done, 1, __ATOMIC_RELAXED);

                break;

            default:
                vmsg("skip special: %s", e->src_path);

                break;
        }
    }

    return NULL;
}

// ============================ Comparators ============================

static VecEntry *g_sort_ents = NULL;

static int cmp_size_then_rel_idx_asc(const void *a, const void *b) {
    size_t ia = *(const size_t *)a, ib = *(const size_t *)b;

    const Entry *ea = &g_sort_ents->v[ia], *eb = &g_sort_ents->v[ib];

    if (ea->size < eb->size) return -1;
    if (ea->size > eb->size) return 1;

    return strcmp(ea->rel_path, eb->rel_path);
}

static int cmp_size_desc_then_rel(const void *a, const void *b) {
    size_t ia = *(const size_t *)a, ib = *(const size_t *)b;

    const Entry *ea = &g_sort_ents->v[ia], *eb = &g_sort_ents->v[ib];

    if (ea->size > eb->size) return -1;
    if (ea->size < eb->size) return 1;

    return strcmp(ea->rel_path, eb->rel_path);
}

/* ==========================================================================
 * Benchmark Runner (internal)
 * ========================================================================== */

/* ---------- Size histogram for profile similarity (4 buckets) ---------- */

static inline int size_bucket(off_t sz) {
    if ((uint64_t)sz <= (64ull << 10)) return 0;          /* ≤ 64 KiB */
    if ((uint64_t)sz <= (1ull  << 20)) return 1;          /* ≤ 1 MiB  */
    if ((uint64_t)sz <= (64ull << 20)) return 2;          /* ≤ 64 MiB */
    return 3;
}

/* ===== Autotune model types (must appear before predictors that use them) ===== */

#ifndef DCOPY_PERFOBS_DEFINED
#define DCOPY_PERFOBS_DEFINED 1
typedef struct {
    double mibs_mean;     /* MiB/s (physical throughput) */
    double filesps_mean;  /* small-file rate (files/s)   */
    double majflt_rate;   /* major faults per repeat     */
} PerfObs;
#endif

/* ---------- Workload mix (bytes + small-file count) & time predictor ---------- */

typedef struct {
    uint64_t bytes_total;          /* total logical bytes across files */
    uint64_t files_small;          /* count of files <= small_thresh_bytes */
    double   small_thresh_bytes;   /* threshold used to define "small" (default: 1 MiB) */
} WorkloadMix;

static WorkloadMix compute_workload_mix(const VecEntry *all, double small_thresh_bytes)
{
    WorkloadMix wm = {0};
    wm.small_thresh_bytes = small_thresh_bytes;

    for (size_t i = 0; i < all->n; i++) {
        const Entry *e = &all->v[i];

        if (e->type != T_FILE) {
            continue;
        }

        wm.bytes_total += (uint64_t)e->size;

        if ((double)e->size <= small_thresh_bytes) {
            wm.files_small++;
        }
    }

    return wm;
}

/* Predict full-tree wall time for a combo measured as:
   - o->mibs     : MiB/s (PHYSICAL I/O throughput; we already compute this)
   - o->filesps  : files/s (small-file rate measured by the subset)
   Model: time = (bytes_total / MBps) + (files_small / filesps)
   Note: MBps = mibs (MiB/s), convert bytes_total to MiB.
*/
__attribute__((always_inline))
static double predict_time_seconds(const PerfObs *o, const WorkloadMix *wm)
{
    const double eps = 1e-9;
    const double mb   = (double)wm->bytes_total / (1024.0 * 1024.0);
    const double mbps = (o->mibs_mean > eps) ? o->mibs_mean : eps;
    const double fps  = (o->filesps_mean > eps) ? o->filesps_mean : eps;
    const double t_bytes = mb / mbps;
    const double t_files = (double)wm->files_small / fps;

    return t_bytes + t_files;
}

/* ---------- Environment profile key (src/dst APFS + mount flags + hw) ---------- */
typedef struct {
    dev_t   src_dev;
    dev_t   dst_dev;
    int     apfs_src;
    int     apfs_dst;
    uint32_t src_mnt_flags;
    uint32_t dst_mnt_flags;
    int     os_major, os_minor;
    int     ncpu;
    uint64_t ram_bytes;
} EnvKey;

/* ---------- Clone feasibility & cost probe ---------- */
typedef struct {
    int     clone_allowed;      /* any success on fclonefileat */
    double  clone_cost_sec;     /* median end-to-end temp->rename(+fsync) cost */
} CloneProbe;

/* Best-effort median of small vector */
__attribute__((always_inline))
static double
median_d(double *a, int n)
{
    if (n <= 0) return DBL_MAX; /* fast-math safe sentinel */

    /* insertion sort (small n) */
    for (int i = 1; i < n; i++) {
        double key = a[i];
        int j = i - 1;
        while (j >= 0 && a[j] > key) {
            a[j + 1] = a[j];
            j--;
        }
        a[j + 1] = key;
    }
    return a[n / 2];
}

__attribute__((always_inline))
static double
mad_d(const double *x, int n, double med)
{
    if (n <= 0) return DBL_MAX; /* fast-math safe sentinel */

    /* guard temporary size (matches outer-run cap of 50; buffer sized 64) */
    double dev[64];
    if (n > (int)(sizeof(dev) / sizeof(dev[0]))) return DBL_MAX;

    for (int i = 0; i < n; i++) dev[i] = fabs(x[i] - med);
    return median_d(dev, n);
}

static CloneProbe probe_clone(const VecEntry *all, const char *dst_root) {
    CloneProbe P = {0};

    char scratch[PATH_MAX];

    snprintf(scratch, sizeof(scratch), "%s/.dcopy-probe.%ld.%u",
            dst_root, (long)getpid(), (unsigned)arc4random());

    (void)ensure_dir_exists_recursive(scratch);

    /* Collect up to 6 representative files by size buckets */
    const int MAXP = 6;
    size_t picked[MAXP];

    int np = 0;
    int seen_b[4] = {0,0,0,0};

    for (size_t i = 0; i < all->n && np < MAXP; i++) {
        const Entry *e = &all->v[i];

        if (e->type != T_FILE || e->size <= 0) {
            continue;
        }

        int b = size_bucket(e->size);

        if (seen_b[b] >= 2) {
            /* at most two per bucket */
            continue;
        }

        picked[np++] = i;
        seen_b[b]++;
    }

    double costs[MAXP];
    int nc = 0;

    for (int k = 0; k < np; k++) {
        const Entry *e = &all->v[picked[k]];

        char tmpname[PATH_MAX];
        char finalname[PATH_MAX];

        snprintf(tmpname,  sizeof(tmpname),  "%s/tmp.%u",   scratch, (unsigned)arc4random());
        snprintf(finalname,sizeof(finalname),"%s/out.%u",   scratch, (unsigned)arc4random());

        ensure_parent_dirs_for(finalname);

        int ddirfd = dirfd_cache_open(scratch);

        if (ddirfd < 0) {
            continue;
        }

        double t0 = now_sec();

        /* Try cloning e->src_path into tmp within scratch */
        if (try_clone_to_tmp_path(e->src_path, ddirfd, path_basename(tmpname)) == 0) {
            /* finalize to 'finalname' for realistic rename cost */
            if (rename_tmp_to_final(ddirfd, path_basename(tmpname),
                                    path_basename(finalname), &(Options){ .clobber = CLOBBER_OVERWRITE }) == 0) {
                (void)fsync_full(ddirfd);

                double t1 = now_sec();
                costs[nc++] = (t1 - t0);

                P.clone_allowed = 1;

                (void)unlink(finalname);
            } else {
                (void)unlink(tmpname);
            }
        } else {
            (void)unlink(tmpname);
        }
    }

    if (nc > 0) {
        P.clone_cost_sec = median_d(costs, nc);
    } else {
        P.clone_cost_sec = 0.0;
    }

    remove_tree_recursive(scratch);

    return P;
}

/* ---------- Latin Hypercube-like seeding over discrete sets ---------- */
typedef struct {
    int io_threads;
    size_t buf_mib, inflight_mib, rdadv_mib;
    DurabilityMode dur;
    int pathmode; /* 0 streaming-only, 1 kernel-only */
} TuneCombo;

typedef struct {
    TuneCombo theta;
    double mibs, filesps, majflt_rate;
    double score;        /* now = -pred_secs */
    double eval_frac;    /* evaluation fraction (e.g., 0.25, 0.60, 1.00) */
    double pred_secs;    /* predicted full-tree time in seconds */
} Trial;

/* Fast-math-safe sentinels + measured predicate */
#ifndef DCOPY_SENTINELS_DEFINED
#define DCOPY_SENTINELS_DEFINED 1
#include <float.h>
#define DCOPY_PSECS_INF     (DBL_MAX)     /* unmeasured predicted seconds */
#define DCOPY_SCORE_NEG_INF (-(DBL_MAX))  /* unmeasured score */
static inline int trial_measured(const Trial *t) {
    return t->pred_secs < (DBL_MAX / 2.0);
}
#endif

static int combo_equal(const TuneCombo *a, const TuneCombo *b) {
    return a->io_threads   == b->io_threads   &&
           a->buf_mib      == b->buf_mib      &&
           a->inflight_mib == b->inflight_mib &&
           a->rdadv_mib    == b->rdadv_mib    &&
           a->dur          == b->dur          &&
           a->pathmode     == b->pathmode;
}

static int push_unique_trial(Trial *T, int *n, int cap, TuneCombo th) {
    for (int i = 0; i < *n; i++) {
        if (combo_equal(&T[i].theta, &th)) {
            return 0;
        }
    }

    if (*n >= cap) {
        return 0;
    }

    T[*n].theta = th;
    (*n)++;

    return 1;
}

static int seed_combos(Trial *out, int cap) {
    const int cpus = cpu_count();

    const int io_set[] = { (cpus < 2 ? 1 : 2),
                           (cpus < 4 ? cpus : 4),
                           (cpus < 8 ? cpus : 8) };

    const size_t buf_set[]  = {1,2,4,8,16};               /* MiB */
    const size_t infl_set[] = {256,512,1024,2048,4096};   /* MiB */
    const size_t rd_set[]   = {16,32,64,128};             /* MiB */

    const DurabilityMode dur_set[] = {DUR_STRICT, DUR_BATCH};

    const int path_set[] = {0,1}; /* 0 streaming-only, 1 kernel-only */

    int n = 0;

    (void)push_unique_trial(out, &n, cap, (TuneCombo){ io_set[0], 1,   256,  16, DUR_STRICT, 0 });
    (void)push_unique_trial(out, &n, cap, (TuneCombo){ io_set[1], 4,   512,  32, DUR_BATCH,  0 });
    (void)push_unique_trial(out, &n, cap, (TuneCombo){ io_set[2], 8,  1024,  64, DUR_BATCH,  1 });

    while (n < cap) {
        TuneCombo th = {
            .io_threads   = io_set[arc4random_uniform((uint32_t)(sizeof(io_set)/sizeof(io_set[0])))],
            .buf_mib      = buf_set[arc4random_uniform((uint32_t)(sizeof(buf_set)/sizeof(buf_set[0])))],
            .inflight_mib = infl_set[arc4random_uniform((uint32_t)(sizeof(infl_set)/sizeof(infl_set[0])))],
            .rdadv_mib    = rd_set[arc4random_uniform((uint32_t)(sizeof(rd_set)/sizeof(rd_set[0])))],
            .dur          = dur_set[arc4random_uniform((uint32_t)(sizeof(dur_set)/sizeof(dur_set[0])))],
            .pathmode     = path_set[arc4random_uniform((uint32_t)(sizeof(path_set)/sizeof(path_set[0])))]
        };

        (void)push_unique_trial(out, &n, cap, th);
    }

    return n;
}

/* ---------- Student-t 95%% critical (small n lookup) ---------- */
static double tcrit95_df(int df) {
    /* df = n-1; for n in [2..30], fallback to normal for df>30 */
    switch (df) {
        case 1:  return 12.706; case 2:  return 4.303;  case 3:  return 3.182;
        case 4:  return 2.776;  case 5:  return 2.571;  case 6:  return 2.447;
        case 7:  return 2.365;  case 8:  return 2.306;  case 9:  return 2.262;
        case 10: return 2.228;  case 11: return 2.201;  case 12: return 2.179;
        case 13: return 2.160;  case 14: return 2.145;  case 15: return 2.131;
        case 16: return 2.120;  case 17: return 2.110;  case 18: return 2.101;
        case 19: return 2.093;  case 20: return 2.086;  case 21: return 2.080;
        case 22: return 2.074;  case 23: return 2.069;  case 24: return 2.064;
        case 25: return 2.060;  case 26: return 2.056;  case 27: return 2.052;
        case 28: return 2.048;  case 29: return 2.045;  case 30: return 2.042;
        default: return 1.960;
    }
}

/* ---------- Winsorize small arrays (protect from outliers) ---------- */
static void winsorize_5pct(double *a, int n) {
    if (n < 3) {
        return;
    }

    /* Find min and max and clamp by next values */
    int imin = 0, imax = 0;

    for (int i = 1; i < n; i++) {
        if (a[i] < a[imin]) {
            imin = i;
        }

        if (a[i] > a[imax]) {
            imax = i;
        }
    }

    /* Replace min and max with neighbors (rough 5% for small n) */
    double avg = 0.0;

    for (int i = 0; i < n; i++) {
        avg += a[i];
    }

    avg /= n;

    a[imin] = avg;
    a[imax] = avg;
}

typedef struct {
    double   secs;
    uint64_t bytes;
    double   mibs;
    double   files_per_s_small;
} BenchMetrics;

/* ---------- Fractional subset runner (declared later) ---------- */
static int run_subset_pipeline_fraction(VecEntry *subset, const char *dst_root,
                                        Options *opt, double fraction,
                                        BenchMetrics *out);

/* ---------- Measure one combo with adaptive repeats and CI bounds ---------- */

static PerfObs measure_combo(const TuneCombo *th,
                             VecEntry *S,
                             const char *run_dir,          /* per-trial destination root */
                             Options base,
                             double frac,
                             double eps_mibs_pct,
                             double eps_files_pct,
                             int *combo_err)
{
    /* Build options for this combo (bench overrides included) */
    Options o = base;

    o.jobs_io         = th->io_threads;
    o.buf_size        = (size_t)th->buf_mib << 20;
    o.inflight_cap    = (size_t)th->inflight_mib << 20;
    o.rdadvise_window = (size_t)th->rdadv_mib   << 20;
    o.dur_mode        = th->dur;
    o.use_kernel_copy = (th->pathmode == 1);
    o.force_kernel_only = (th->pathmode == 1);
    o.force_streaming = (th->pathmode == 0);

    /* Critical for benchmarking: prevent EEXIST/skip noise and reduce metadata noise */
    o.clobber   = CLOBBER_OVERWRITE;
    o.meta_mode = META_MINIMAL;

    clamp_inflight_to_ram(&o);

    const int NMAX = 5;
    double mibs[NMAX], fps[NMAX];

    int n = 0;

    struct rusage ru0, ru1;
    (void)getrusage(RUSAGE_SELF, &ru0);
    long majflt_before = ru0.ru_majflt;

    while (n < NMAX) {
        /* Ensure we don't hold stale directory FDs from a prior repeat */
        dirfd_cache_close_all();

        /* Recreate the run directory (and parents) fresh for this repeat */
        (void)(void)ensure_dir_exists_recursive(run_dir);

        BenchMetrics m = {0};
        int rc = run_subset_pipeline_fraction(S, run_dir, &o, frac, &m);
        if (rc != 0) { if (combo_err) *combo_err = 1; break; }

        mibs[n] = m.mibs;
        fps[n]  = m.files_per_s_small;
        n++;

        if (n >= 2) {
            /* t-interval half-widths */
            double mean_m=0, mean_f=0, s_m=0, s_f=0;

            for (int i = 0; i < n; i++){
                mean_m += mibs[i];
                mean_f += fps[i];
            }

            mean_m/=n;
            mean_f/=n;

            for (int i = 0; i < n; i++){
                s_m += (mibs[i]-mean_m)*(mibs[i]-mean_m);
                s_f += (fps[i]-mean_f)*(fps[i]-mean_f);
            }

            s_m = (n>1) ? sqrt(s_m/(n-1)) : 0.0;
            s_f = (n>1) ? sqrt(s_f/(n-1)) : 0.0;

            double t = tcrit95_df(n-1);

            double hw_m = t * s_m / sqrt((double)n);
            double hw_f = t * s_f / sqrt((double)n);

            double rel_m = (mean_m > 0) ? (hw_m/mean_m*100.0) : 100.0;
            double rel_f = (mean_f > 0) ? (hw_f/mean_f*100.0) : 100.0;

            if (rel_m <= eps_mibs_pct && rel_f <= eps_files_pct) {
                /* Clean once more after final repeat */
                remove_tree_recursive(run_dir);
                dirfd_cache_close_all();

                break;
            }
        }

        /* Clean up the whole per-repeat run dir to start next repeat from a blank slate. */
        remove_tree_recursive(run_dir);

        /* Invalidate cached directory FDs that point to now-removed paths. */
        dirfd_cache_close_all();
    }

    (void)getrusage(RUSAGE_SELF, &ru1);
    long majflt_after = ru1.ru_majflt;

    /* Stabilize if we had 3+ samples */
    if (n >= 3) {
        winsorize_5pct(mibs, n);
        winsorize_5pct(fps, n);
    }

    double mean_m=0, mean_f=0;

    for (int i = 0; i < n; i++){
        mean_m += mibs[i];
        mean_f += fps[i];
    }

    if (n > 0){
        mean_m/=n;
        mean_f/=n;
    }

    PerfObs obs = { .mibs_mean = mean_m, .filesps_mean = mean_f };

    long df = (majflt_after > majflt_before) ? (majflt_after - majflt_before) : 0;
    obs.majflt_rate = (double)df / (double)(n>0? n : 1);

    /* Final safety: ensure no stale FDs leak into the next combo */
    dirfd_cache_close_all();

    return obs;
}

/* ---------- Score function ---------- */
static inline double score_from_pred_seconds(double secs) {
    /* Lower seconds is better -> higher score */
    return -secs;
}

static int run_subset_pipeline(VecEntry *subset,
                               const char *dst_root,
                               Options *opt,
                               BenchMetrics *out)
{
    /* Ensure inflight matches this run */
    g_inflight_cap = opt->inflight_cap;

    /* No progress noise for micro-runs */
    const int saved_quiet = g_quiet;
    const int saved_prog  = g_progress_enabled;
    g_progress_enabled    = 0;

    DigestMap dmap; dmap_init(&dmap);
    InoMap    imap; imap_init(&imap);
    PhysMap   pmap; pmap_init(&pmap);

    Stats stats = {0};
    uint64_t small_files_cnt = 0;

    for (size_t i = 0; i < subset->n; i++) {
        if (subset->v[i].type == T_FILE) {
            stats.files_total++;

            stats.bytes_total += (uint64_t)subset->v[i].size;

            if ((uint64_t)subset->v[i].size <= (1ull << 20)) {
                small_files_cnt++;
            }
        }
    }

    __atomic_store_n(&stats.copy_phase_started, 0, __ATOMIC_RELAXED);

    gettimeofday(&stats.start_time, NULL);

    IOJobQueue io_q;
    io_queue_init(&io_q, subset->n > 1024 ? subset->n : 1024);

    IOContext   io_ctx = { .opt = opt, .dmap = &dmap, .imap = &imap, .pmap = &pmap, .stats = &stats };
    IOWorkerCtx wctx   = { .q   = &io_q, .ents = subset, .ctx   = &io_ctx };

    int n_io = opt->jobs_io ? opt->jobs_io : (cpu_count() < 4 ? cpu_count() : 4);

    if (n_io < 1) {
        n_io = 1;
    }

    if (n_io > MAX_THREADS) {
        n_io = MAX_THREADS;
    }

    pthread_t *tids = (pthread_t *)calloc((size_t)n_io, sizeof(pthread_t));

    for (int t = 0; t < n_io; t++) {
        if (pthread_create(&tids[t], NULL, io_worker, &wctx) != 0) {
            die("pthread_create(io_worker)");
        }
    }

    /* 1) Directories first */
    for (size_t i = 0; i < subset->n; i++) {
        if (subset->v[i].type != T_DIR) continue;

        char dst_path[PATH_MAX];
        path_join2(dst_root, subset->v[i].rel_path, dst_path, sizeof dst_path);

        IOJob j = { .entry_idx = i };
        snprintf(j.dst_path, sizeof(j.dst_path), "%s", dst_path);

        io_queue_push(&io_q, j);
    }

    /* 2) Files by size descending */
    size_t nf = 0;

    for (size_t i = 0; i < subset->n; i++) {
        if (subset->v[i].type == T_FILE) {
            nf++;
        }
    }

    if (nf > 0) {
        size_t *ord = (size_t *)malloc(nf * sizeof(size_t));
        size_t   k  = 0;

        for (size_t i = 0; i < subset->n; i++) {
            if (subset->v[i].type == T_FILE) {
                ord[k++] = i;
            }
        }

        g_sort_ents = subset;

        qsort(ord, nf, sizeof(size_t), cmp_size_desc_then_rel);

        __atomic_store_n(&stats.copy_phase_started, 1, __ATOMIC_RELAXED);

        for (size_t z = 0; z < nf; z++) {
            char dst_path[PATH_MAX];
            path_join2(dst_root, subset->v[ord[z]].rel_path, dst_path, sizeof dst_path);

            IOJob j = { .entry_idx = ord[z] };
            snprintf(j.dst_path, sizeof(j.dst_path), "%s", dst_path);

            io_queue_push(&io_q, j);
        }

        free(ord);
    }

    /* 3) Symlinks/others last */
    for (size_t i = 0; i < subset->n; i++) {
        if (subset->v[i].type == T_FILE || subset->v[i].type == T_DIR) {
            continue;
        }

        char dst_path[PATH_MAX];
        path_join2(dst_root, subset->v[i].rel_path, dst_path, sizeof dst_path);

        IOJob j = { .entry_idx = i };
        snprintf(j.dst_path, sizeof(j.dst_path), "%s", dst_path);

        io_queue_push(&io_q, j);
    }

    io_queue_finish(&io_q);

    for (int t = 0; t < n_io; t++) {
        pthread_join(tids[t], NULL);
    }

    free(tids);

    if (opt->dur_mode == DUR_BATCH) {
        dirfd_cache_fsync_all();
    }

    struct timeval now;
    gettimeofday(&now, NULL);

    double secs = (double)(now.tv_sec  - stats.start_time.tv_sec) +
                  (double)(now.tv_usec - stats.start_time.tv_usec) / 1e6;

    if (secs <= 0) {
        secs = 1e-3;
    }

    /* Use PHYSICAL bytes for benchmark throughput to avoid clone/hole inflation. */
    out->secs            = secs;
    out->bytes           = stats.bytes_phys_io;
    out->mibs            = ((double)stats.bytes_phys_io) / secs / (1024.0 * 1024.0);
    out->files_per_s_small = (secs > 0.0) ? ((double)small_files_cnt / secs) : 0.0;

    dmap_free(&dmap);
    imap_free(&imap);
    pmap_free(&pmap);

    g_quiet            = saved_quiet;
    g_progress_enabled = saved_prog;

    return 0;
}

// ============================ Progress & Summary ============================

typedef struct {
    Stats *stats;
    volatile sig_atomic_t done;
} ProgressCtx;

static void update_copy_progress(Stats *stats);

static void *progress_updater_thread(void *arg) {
    ProgressCtx *ctx = (ProgressCtx *)arg;

    while (!ctx->done) {
        update_copy_progress(ctx->stats);

        struct timespec ts = {.tv_sec = 0, .tv_nsec = 100000000};

        (void)nanosleep(&ts, NULL);
    }

    update_copy_progress(ctx->stats);

    return NULL;
}

static void update_sample_progress(uint64_t done, uint64_t total) {
    if (g_quiet || !g_progress_enabled) {
        return;
    }

    static const char spinner[] = {'-', '\\', '|', '/'};

    static int spin_idx = 0;

    int percent = (total > 0) ? (int)((done * 100) / total) : 100;

    pthread_mutex_lock(&g_log_mu);

    fprintf(stderr, "\r[%c] Sampling files... %d%% (%llu/%llu)",
            spinner[spin_idx], percent,
            (unsigned long long)done, (unsigned long long)total);

    fflush(stderr);

    pthread_mutex_unlock(&g_log_mu);

    spin_idx = (spin_idx + 1) % (sizeof(spinner));

    if (done >= total) {
        pthread_mutex_lock(&g_log_mu);

        fprintf(stderr, "\r[+] Sampling files... Done.                       \n");
        fflush(stderr);

        pthread_mutex_unlock(&g_log_mu);
    }
}

static void update_hash_progress(uint64_t done, uint64_t total) {
    if (g_quiet || !g_progress_enabled) {
        return;
    }

    static const char spinner[] = {'-', '\\', '|', '/'};

    static int spin_idx = 0;

    int percent = (total > 0) ? (int)((done * 100) / total) : 100;

    pthread_mutex_lock(&g_log_mu);

    fprintf(stderr, "\r[%c] Hashing files... %d%% (%llu/%llu)",
            spinner[spin_idx], percent,
            (unsigned long long)done, (unsigned long long)total);
    fflush(stderr);

    pthread_mutex_unlock(&g_log_mu);

    spin_idx = (spin_idx + 1) % (sizeof(spinner));

    if (done >= total) {
        pthread_mutex_lock(&g_log_mu);

        fprintf(stderr, "\r[+] Hashing files... Done.                        \n");
        fflush(stderr);

        pthread_mutex_unlock(&g_log_mu);
    }
}

static void update_copy_progress(Stats *stats) {
    if (g_quiet || !g_progress_enabled) {
        return;
    }

    struct timeval now;
    gettimeofday(&now, NULL);

    double elapsed = (double)(now.tv_sec - stats->start_time.tv_sec) +
                     (double)(now.tv_usec - stats->start_time.tv_usec) / 1e6;

    uint64_t files_done  = __atomic_load_n(&stats->files_done,      __ATOMIC_RELAXED);
    uint64_t files_total = __atomic_load_n(&stats->files_total,     __ATOMIC_RELAXED);
    uint64_t bytes_done  = __atomic_load_n(&stats->bytes_done,      __ATOMIC_RELAXED);
    uint64_t bytes_total = __atomic_load_n(&stats->bytes_total,     __ATOMIC_RELAXED);

    uint64_t dirs_done   = __atomic_load_n(&stats->dirs_done_jobs,  __ATOMIC_RELAXED);
    uint64_t dirs_total  = __atomic_load_n(&stats->dirs_total_jobs, __ATOMIC_RELAXED);

    int phase_started    = __atomic_load_n(&stats->copy_phase_started, __ATOMIC_RELAXED);

    double rate = (elapsed > 0.1) ? (double)bytes_done / elapsed : 0.0;

    char rate_buf[32];
    (void)human_bytes(rate, rate_buf, sizeof(rate_buf));

    int dir_percent  = (dirs_total  > 0) ? (int)((dirs_done  * 100) / dirs_total ) : 100;
    int file_percent = (files_total > 0) ? (int)((files_done * 100) / files_total) : 100;

    pthread_mutex_lock(&g_log_mu);

    if (!phase_started) {
        fprintf(stderr,
                "\rPreparing directory structure: Dirs %llu/%llu (%3d%%)",
                (unsigned long long)dirs_done,
                (unsigned long long)dirs_total,
                dir_percent);

        if (files_total == 0 && dirs_total > 0 && dirs_done >= dirs_total) {
            fprintf(stderr, "\rPreparing directory structure: Complete.                              \n");
        }

        fflush(stderr);

        pthread_mutex_unlock(&g_log_mu);

        return;
    }

    int bar_width = 20;
    int bar_pos = 0;

    if (bytes_total > 0) {
        int percent_bytes = (int)((bytes_done * 100) / bytes_total);
        bar_pos = (percent_bytes * bar_width) / 100;

        if (bar_pos < 0) {
            bar_pos = 0;
        }

        if (bar_pos > bar_width) {
            bar_pos = bar_width;
        }
    } else {
        bar_pos = bar_width;
    }

    fprintf(stderr,
            "\rCopying: [%.*s%.*s] %3d%% | %llu/%llu files | %s/s | Dirs %llu/%llu (%3d%%)",
            bar_pos, "====================",
            bar_width - bar_pos, "                    ",
            file_percent,
            (unsigned long long)files_done,
            (unsigned long long)files_total,
            rate_buf,
            (unsigned long long)dirs_done,
            (unsigned long long)dirs_total,
            dir_percent);

    if (files_done >= files_total) {
        fprintf(stderr, "\rCopying: Complete.                                                              \n");
    }

    fflush(stderr);

    pthread_mutex_unlock(&g_log_mu);
}

static void print_summary(const Stats *stats, const Options *opt) {
    if (g_quiet) return;

    char total_buf[32], copied_buf[32], cloned_src_buf[32], cloned_dedupe_buf[32],
         linked_buf[32], skipped_buf[32], delsrc_buf[32];

    (void)human_bytes((double)stats->bytes_total,       total_buf,        sizeof(total_buf));
    (void)human_bytes((double)stats->bytes_copied,      copied_buf,       sizeof(copied_buf));
    (void)human_bytes((double)stats->bytes_cloned_src,  cloned_src_buf,   sizeof(cloned_src_buf));
    (void)human_bytes((double)stats->bytes_cloned_dedupe, cloned_dedupe_buf, sizeof(cloned_dedupe_buf));
    (void)human_bytes((double)stats->bytes_hardlinked,  linked_buf,       sizeof(linked_buf));
    (void)human_bytes((double)stats->bytes_skipped,     skipped_buf,      sizeof(skipped_buf));
    (void)human_bytes((double)stats->bytes_deleted_src, delsrc_buf,       sizeof(delsrc_buf));

    pthread_mutex_lock(&g_log_mu);
    fprintf(stderr, "\n--- Copy Summary ---\n");
    if (opt->dry_run) {
        fprintf(stderr, "Mode: Dry Run (no changes were made)\n");
    }

    fprintf(stderr,
            "Total objects processed: %llu files (%s), %llu dirs, %llu symlinks\n\n",
            (unsigned long long)stats->files_total,
            total_buf,
            (unsigned long long)stats->dirs_total_jobs,
            (unsigned long long)stats->symlinks_total);

    fprintf(stderr, "File Operations Breakdown:\n");
    fprintf(stderr, "  - Copied (full data):    %6llu files (%s)\n",
            (unsigned long long)stats->files_copied,        copied_buf);
    fprintf(stderr, "  - Cloned (from source):  %6llu files (%s)\n",
            (unsigned long long)stats->files_cloned_src,    cloned_src_buf);
    fprintf(stderr, "  - Cloned (deduplicated): %6llu files (%s)\n",
            (unsigned long long)stats->files_cloned_dedupe, cloned_dedupe_buf);
    fprintf(stderr, "  - Hardlinked:            %6llu files (%s)\n",
            (unsigned long long)stats->files_hardlinked,    linked_buf);
    fprintf(stderr, "  - Skipped (no-clobber):  %6llu files (%s)\n",
            (unsigned long long)stats->files_skipped,       skipped_buf);
    fprintf(stderr, "  - Deleted sources:       %6llu files (%s)\n",
            (unsigned long long)stats->files_deleted_src,   delsrc_buf);

    fprintf(stderr, "\nSymlink Operations:\n");
    fprintf(stderr, "  - Created at destination: %6llu\n",
            (unsigned long long)stats->symlinks_created);
    fprintf(stderr, "  - Deleted from source:    %6llu\n",
            (unsigned long long)stats->symlinks_deleted_src);

    fprintf(stderr, "--------------------\n");
    fflush(stderr);
    pthread_mutex_unlock(&g_log_mu);
}

/* ==========================================================================
 * Sampling & Autotune
 * ========================================================================== */

typedef struct {
    size_t *idx;
    size_t  n;
} IndexVec;

/* ============================================================================
 * Stratified subset builder (replaces fixed 25%% rule with bucketed sampling)
 * ========================================================================== */
static int
build_stratified_subset(VecEntry *all,
                        size_t cap_large,
                        size_t cap_small,
                        VecEntry *subset,
                        uint64_t byte_budget)
{
    subset->v   = NULL;
    subset->n   = 0;
    subset->cap = 0;

    /* Root bench/ directory */
    Entry d = (Entry){0};
    d.type      = T_DIR;
    d.src_path  = xstrdup(".");
    d.rel_path  = xstrdup("bench");
    ve_push(subset, d);

    /* Collect candidates by buckets */
    size_t idx_tiny_cap  = cap_small;
    size_t idx_small_cap = cap_small;
    size_t idx_med_cap   = cap_large;
    size_t idx_large_cap = cap_large;

    size_t tiny = 0, small = 0, med = 0, large = 0;

    uint64_t left = byte_budget;

    /* One pass, pick proportionally until caps and budget are hit */
    for (size_t i = 0; i < all->n && left > 0; i++) {
        const Entry *src = &all->v[i];
        if (src->type != T_FILE || src->size <= 0) continue;

        int b = size_bucket(src->size);
        bool take = false;
        switch (b) {
            case 0: if (tiny  < idx_tiny_cap)  { take = true; tiny++;  } break;
            case 1: if (small < idx_small_cap) { take = true; small++; } break;
            case 2: if (med   < idx_med_cap)   { take = true; med++;   } break;
            default:if (large < idx_large_cap) { take = true; large++; } break;
        }
        if (!take) continue;

        /* Ensure per-file does not wildly exceed remaining budget */
        if ((uint64_t)src->size > left && left < (uint64_t)src->size / 4) continue;

        const char *base = path_basename(src->rel_path);
        uint64_t h64 = XXH3_64bits(src->src_path, strlen(src->src_path));
        Entry e = *src;
        char nameonly[NAME_MAX + 96];
        snprintf(nameonly, sizeof(nameonly), "bench/%016llx__%s",
                 (unsigned long long)h64, base);
        e.rel_path = xstrdup(nameonly);
        e.src_path = xstrdup(src->src_path);
        ve_push(subset, e);

        if (left > (uint64_t)src->size) left -= (uint64_t)src->size;
        else left = 0;
    }

    return 0;
}

/* ============================================================================
 * Run only a fraction of a prepared subset (by bytes), preserving order.
 * ========================================================================== */
static int run_subset_pipeline_fraction(VecEntry *subset, const char *dst_root,
                                        Options *opt, double fraction,
                                        BenchMetrics *out)
{
    if (fraction <= 0.0) fraction = 0.25;
    if (fraction > 1.0)  fraction = 1.0;

    /* Compute total bytes of files in subset (ignore dirs) */
    uint64_t bytes_total = 0;
    for (size_t i = 0; i < subset->n; i++) {
        if (subset->v[i].type == T_FILE) bytes_total += (uint64_t)subset->v[i].size;
    }
    uint64_t byte_cap = (uint64_t)((double)bytes_total * fraction);

    VecEntry frac = {0};
    /* Always include bench/ directory entry */
    for (size_t i = 0; i < subset->n; i++) {
        if (subset->v[i].type == T_DIR && strcmp(subset->v[i].rel_path, "bench") == 0) {
            Entry d = {0};
            d.type      = T_DIR;
            d.src_path  = xstrdup(".");
            d.rel_path  = xstrdup("bench");
            ve_push(&frac, d);
            break;
        }
    }

    uint64_t left = byte_cap;
    for (size_t i = 0; i < subset->n && left > 0; i++) {
        const Entry *e = &subset->v[i];
        if (e->type != T_FILE) continue;

        if ((uint64_t)e->size > left && left < (uint64_t)e->size / 4) continue;

        Entry c = *e;
        c.src_path = xstrdup(e->src_path);
        c.rel_path = xstrdup(e->rel_path);
        ve_push(&frac, c);

        if (left > (uint64_t)e->size) left -= (uint64_t)e->size;
        else left = 0;
    }

    /* Execute full runner on fractional subset */
    int rc = run_subset_pipeline(&frac, dst_root, opt, out);
    ve_free(&frac);
    return rc;
}

/* --------------------------------------------------------------------------
 * Autotune legend & info — SINGLE-RUN TABLE
 * Matches columns:
 *   # | mode | tier | io | inflight | buf | rdadv | dur | throughput | t_pred(s) | score
 * -------------------------------------------------------------------------- */
static void
print_autotune_legend_single(const WorkloadMix *wm, double highest_tier_seen)
{
    char hb[64];
    human_bytes((double)wm->bytes_total, hb, sizeof(hb));

    pmsg("\nLegend / Info (single run)");
    pmsg("  mode       : 'streaming' = userland read/write; 'kernel' = fcopyfile(3) path.");
    pmsg("  tier       : evaluation fraction of the subset (0.25, 0.60, 1.00). Higher tiers are preferred.");
    pmsg("  io         : I/O worker threads (data path; distinct from hashing threads).");
    pmsg("  inflight   : global in-flight cap (MiB) to bound page-cache / memory pressure.");
    pmsg("  buf        : per-I/O-thread read buffer size (MiB).");
    pmsg("  rdadv      : read-ahead advisory window (MiB) via fcntl(F_RDADVISE).");
    pmsg("  dur        : durability policy — 'strict' (fsync each create+rename) vs 'batch' (coalesced fsync).");
    pmsg("  throughput : average *physical* I/O rate for the subset (humanized); excludes APFS clones & holes.");
    pmsg("  t_pred(s)  : predicted full-tree wall time (seconds), computed as:");
    pmsg("                 t_pred = (bytes_total / MBps) + (files_small / filesps)");
    pmsg("               where MBps and filesps come from measurements on the subset;");
    pmsg("               'small' means size ≤ %.0f KiB.", wm->small_thresh_bytes / 1024.0);
    pmsg("  score      : -t_pred(s) (higher score == lower predicted time).");

    pmsg("\nWorkload mix used for prediction");
    pmsg("  total bytes        : %s", hb);
    pmsg("  small-file cutoff  : %.0f KiB", wm->small_thresh_bytes / 1024.0);
    pmsg("  small files counted: %llu", (unsigned long long)wm->files_small);

    pmsg("\nSelection policy (single run)");
    pmsg("  1) Use the highest evaluated tier present (prefer 1.00; highest seen = %.2f).", highest_tier_seen);
    pmsg("  2) Within that tier, choose the row with lowest t_pred (highest score).");

    pmsg("\nNotes");
    pmsg("  • Autotune uses overwrite + minimal metadata to reduce per-combo noise.");
    pmsg("  • Directory-FD cache is flushed between repeats to avoid stale openat(2) failures.");
    pmsg("  • Inner repeats tighten 95%% CIs; results may still vary with background load/thermals.");
    pmsg("  • Table throughput is humanized; scoring uses exact MiB/s internally.");
}

/* --------------------------------------------------------------------------
 * Autotune legend & info — AGGREGATED TABLE (over --bench-runs K)
 * Matches columns:
 *   # | mode | io | inflight | buf | rdadv | dur | t_pred_med | t_MAD | t_mean | n
 * -------------------------------------------------------------------------- */
static void
print_autotune_legend_agg(const WorkloadMix *wm, int K_runs, double highest_tier_seen)
{
    char hb[64];
    human_bytes((double)wm->bytes_total, hb, sizeof(hb));

    pmsg("\nLegend / Info (aggregated over %d runs)", K_runs);
    pmsg("  mode       : 'streaming' = userland read/write; 'kernel' = fcopyfile(3) path.");
    pmsg("  io         : I/O worker threads (data path).");
    pmsg("  inflight   : global in-flight cap (MiB).");
    pmsg("  buf        : per-I/O-thread read buffer size (MiB).");
    pmsg("  rdadv      : read-ahead advisory window (MiB).");
    pmsg("  dur        : durability policy.");
    pmsg("  t_pred_med : median of predicted full-tree times (seconds) across runs.");
    pmsg("  t_MAD      : median absolute deviation of predicted times (robust dispersion).");
    pmsg("  t_mean     : arithmetic mean of predicted times.");
    pmsg("  n          : number of runs contributing to this row (≤ %d).", K_runs);

    pmsg("\nPrediction model (per run)");
    pmsg("  t_pred = (bytes_total / MBps) + (files_small / filesps)");
    pmsg("    • MBps and filesps measured on the subset; 'small' means size ≤ %.0f KiB.",
         wm->small_thresh_bytes / 1024.0);
    pmsg("    • bytes_total used here: %s; small files counted: %llu.",
         hb, (unsigned long long)wm->files_small);

    pmsg("\nSelection policy (aggregated)");
    pmsg("  1) Rank configs by lowest t_pred_med (robust to outliers).");
    pmsg("  2) Tie-break by smaller t_MAD, then smaller t_mean.");
    pmsg("  3) Preference is still given to configs observed at higher tiers (highest seen = %.2f).",
         highest_tier_seen);

    pmsg("\nNotes");
    pmsg("  • The aggregated table summarizes only the winning config from each run.");
    pmsg("  • Inner per-combo CI control remains active; outer aggregation reduces environmental noise.");
    pmsg("  • Throughput columns are omitted here by design; the decision metric is t_pred.");
}

/* ---------- Full-equality for TuneCombo ---------- */
static int combo_equal_full(const TuneCombo *a, const TuneCombo *b) {
    return a->io_threads   == b->io_threads   &&
           a->buf_mib      == b->buf_mib      &&
           a->inflight_mib == b->inflight_mib &&
           a->rdadv_mib    == b->rdadv_mib    &&
           a->dur          == b->dur          &&
           a->pathmode     == b->pathmode;
}

/* ---------- Aggregation rows ---------- */

typedef struct {
    TuneCombo theta;
    double    secs[64];   /* predicted seconds per outer run (max 50 used) */
    int       n;
} AggRow;

/* Compute best (lowest) median so far across populated AggRow slots. */
static double agg_best_median_so_far(const AggRow *agg, int rn)
{
    double best = DBL_MAX;
    double buf[64];

    for (int i = 0; i < rn; i++) {
        if (agg[i].n <= 0) {
            continue;
        }

        int n = agg[i].n;

        for (int j = 0; j < n; j++) {
            buf[j] = agg[i].secs[j];
        }

        double med = median_d(buf, n);

        if (med < best) {
            best = med;
        }
    }
    return best;
}

static int agg_find_or_add(AggRow *rows, int *rn, int cap, const TuneCombo *th) {
    for (int i = 0; i < *rn; i++) {
        if (combo_equal_full(&rows[i].theta, th)) {
            return i;
        }
    }

    if (*rn >= cap) {
        return -1;
    }

    rows[*rn].theta = *th;
    rows[*rn].n = 0;

    return (*rn)++;
}

/* --------------------------------------------------------------------------
 * Aggregation helpers: compute row statistics, sort, and print table
 * -------------------------------------------------------------------------- */

/* Per-config statistics over outer runs */
typedef struct {
    int    idx;     /* index into AggRow[] */
    double med;     /* median of predicted seconds */
    double mad;     /* median absolute deviation */
    double mean;    /* arithmetic mean of predicted seconds */
    int    n;       /* number of samples */
} AggStatRow;

/* Compute median, MAD, and mean for one aggregated row */
static AggStatRow
agg_compute_stats(const AggRow *ar)
{
    AggStatRow r = {0};
    r.idx = -1; /* caller sets idx */

    if (ar->n <= 0) {
        r.med  = DBL_MAX;
        r.mad  = DBL_MAX;
        r.mean = DBL_MAX;
        r.n    = 0;
        return r;
    }

    double buf[64];
    for (int j = 0; j < ar->n; j++) buf[j] = ar->secs[j];

    r.med  = median_d(buf, ar->n);
    r.mad  = mad_d(ar->secs, ar->n, r.med);

    double sum = 0.0;
    for (int j = 0; j < ar->n; j++) sum += ar->secs[j];
    r.mean = sum / (double)ar->n;
    r.n    = ar->n;

    return r;
}

/* Sort rows by (med asc, MAD asc, mean asc) */
static void
agg_sort_rows(AggStatRow *rows, int m)
{
    for (int i = 0; i < m; i++) {
        int best = i;

        for (int j = i + 1; j < m; j++) {
            const AggStatRow *A = &rows[best], *B = &rows[j];

            if (B->med < A->med - 1e-9 ||
                (fabs(B->med - A->med) <= 1e-9 && B->mad < A->mad - 1e-9) ||
                (fabs(B->med - A->med) <= 1e-9 && fabs(B->mad - A->mad) <= 1e-9 && B->mean < A->mean - 1e-9))
            {
                best = j;
            }
        }

        if (best != i) {
            AggStatRow tmp = rows[i];
            rows[i] = rows[best];
            rows[best] = tmp;
        }
    }
}

/* Print aggregated ranking table */
static void
agg_print_table(const AggRow *agg, const AggStatRow *rows, int m, int K)
{
    pmsg("\n--- Aggregated Results (median over %d runs) ---", K);
    pmsg("%-3s %-9s %-7s %-11s %-9s %-9s %-10s %-12s %-10s %-10s %-4s",
         "#", "mode", "io", "inflight", "buf", "rdadv", "dur",
         "t_pred_med", "t_MAD", "t_mean", "n");

    for (int r = 0; r < m; r++) {
        const AggRow    *ar = &agg[ rows[r].idx ];
        const TuneCombo *t  = &ar->theta;

        pmsg("%-3d %-9s %-7d %-11zu %-9zu %-9zu %-10s %-12.2f %-10.2f %-10.2f %-4d",
             r + 1,
             (t->pathmode == 1 ? "kernel" : "streaming"),
             t->io_threads,
             t->inflight_mib,
             t->buf_mib,
             t->rdadv_mib,
             (t->dur == DUR_BATCH ? "batch" : "strict"),
             rows[r].med, rows[r].mad, rows[r].mean, rows[r].n);
    }
}

typedef struct {
    Options     o;
    double      score;
    double      mibs;
    double      filesps;
    const char *label;     /* "streaming" | "kernel" */
} ComboRes;

/* Single-run autotune with lightweight, live progress output, final validation,
   and full table+legend. */
static Trial
run_autotune_once(VecEntry *all, const char *dst_root, Options base, int print_table, double *out_best_frac)
{
    /* Workload mix */
    WorkloadMix WM = compute_workload_mix(all, 1.0 * (1024.0 * 1024.0));

    /* Suppress noisy inner printing for table-less runs, but we still show status. */
    int saved_quiet = g_quiet, saved_prog = g_progress_enabled;

    if (!print_table) {
        g_quiet = 1;
        g_progress_enabled = 0;
    }


    /* Scratch */
    char bench_leaf[64];

    snprintf(bench_leaf, sizeof bench_leaf, ".dcopy-bench.%ld.%u",
             (long)getpid(), (unsigned)arc4random());

    char scratch[PATH_MAX];
    path_join2(dst_root, bench_leaf, scratch, sizeof scratch);
    (void)ensure_dir_exists_recursive(scratch);

    clamp_inflight_to_ram(&base);

    /* Probe (not used in prediction yet) */
    CloneProbe probe = probe_clone(all, scratch);
    (void)probe;

    /* Subset */
    VecEntry subset = (VecEntry){0};

    (void)build_stratified_subset(
        all,
        base.bench_large_cap,
        base.bench_small_cap,
        &subset,
        base.bench_byte_budget
    );

    /* Seed */
    enum { MAX_COMBOS = 36 };
    Trial pool[MAX_COMBOS];
    int pool_n = 0;

    pool_n = seed_combos(pool, MAX_COMBOS);

    for (int z = 0; z < pool_n; z++) {
        pool[z].pred_secs = DCOPY_PSECS_INF;   /* unmeasured */
        pool[z].score     = DCOPY_SCORE_NEG_INF;
        pool[z].eval_frac = 0.0;
    }

    const int    eta         = (base.bench_eta >= 2 ? base.bench_eta : 3);
    const double tier_frac[] = {0.25, 0.60, 1.00};
    const int    tier_count  = 3;

    const bool   exhaustive = base.bench_exhaustive;
    const size_t max_combos  = base.bench_max_combos; /* 0 => unlimited */
    const double t_start     = now_sec();
    const double t_budget    = (double)((base.bench_seconds > 0) ? base.bench_seconds : 0);

    int current_n = pool_n;
    int tier      = 0;
    int tested_total = 0;
    int global_trial_idx = 0;

    enum { STOP_NONE=0, STOP_TIME=1, STOP_MAX=2 } stop_reason = STOP_NONE;

    /* Initial status line */
    bench_statusf("Autotune run %d/%d | preparing subset...", g_bench_outer_idx, g_bench_outer_total);

    while (tier < tier_count) {
        /* Tier header status */
        bench_statusf("Autotune run %d/%d | tier %.2f | combos 0/%d",
                      g_bench_outer_idx, g_bench_outer_total, tier_frac[tier], current_n);

        for (int i = 0; i < current_n; i++) {
            if (!exhaustive && t_budget > 0.0 && (now_sec() - t_start) > t_budget) {
                stop_reason = STOP_TIME; break;
            }

            if (!exhaustive && max_combos > 0 && (size_t)tested_total >= max_combos) {
                stop_reason = STOP_MAX; break;
            }

            TuneCombo th = pool[i].theta;

            char run_leaf[32], run_dir[PATH_MAX];
            snprintf(run_leaf, sizeof run_leaf, "run%06d", global_trial_idx++);
            path_join2(scratch, run_leaf, run_dir, sizeof run_dir);

            int comb_err = 0;
            PerfObs obs = measure_combo(&th, &subset, run_dir,
                                        base, tier_frac[tier],
                                        base.bench_ci_mibs_pct, base.bench_ci_files_pct,
                                        &comb_err);

            const double secs = predict_time_seconds(&obs, &WM);
            pool[i].mibs        = obs.mibs_mean;
            pool[i].filesps     = obs.filesps_mean;
            pool[i].majflt_rate = obs.majflt_rate;
            pool[i].pred_secs   = secs;
            pool[i].score       = score_from_pred_seconds(secs);
            pool[i].eval_frac   = tier_frac[tier];

            tested_total++;

            /* Update per-tier status */
            bench_statusf("Autotune run %d/%d | tier %.2f | combos %d/%d",
                          g_bench_outer_idx, g_bench_outer_total, tier_frac[tier], i + 1, current_n);
        }

        if (stop_reason != STOP_NONE) break;

        /* Keep top K = ceil(n/eta) */
        int keep = (current_n + eta - 1) / eta;
        if (keep < 1) keep = 1;

        /* rank by score desc (i.e., t_pred asc) */
        for (int i = 0; i < keep; i++) {
            int best = i;

            for (int j = i + 1; j < current_n; j++) {
                if (pool[j].score > pool[best].score) {
                    best = j;
                }
            }

            if (best != i) {
                Trial tmp = pool[i];
                pool[i] = pool[best];
                pool[best] = tmp;
            }
        }

        current_n = keep;
        tier++;

        /* Endline after tier completes */
        bench_status_endline();

        if (current_n == 1) break;
    }

    /* -------- Final validation up-shift (re-measure top lower-tier configs at highest tier) -------- */
    {
        /* Find highest tier actually measured this run */
        double max_frac = 0.0;

        for (int i = 0; i < pool_n; i++) {
            if (trial_measured(&pool[i]) && pool[i].eval_frac > max_frac) {
                max_frac = pool[i].eval_frac;
            }
        }

        if (max_frac > 0.0) {
            /* Gather lower-tier candidates */
            int cand_idx[MAX_COMBOS]; int cn = 0;

            for (int i = 0; i < pool_n; i++) {
                if (trial_measured(&pool[i]) && pool[i].eval_frac < max_frac) {
                    cand_idx[cn++] = i;
                }
            }

            /* Sort candidates by current score desc */
            for (int a = 0; a < cn; a++) {
                int best = a;

                for (int b = a + 1; b < cn; b++) {
                    if (pool[cand_idx[b]].score > pool[cand_idx[best]].score) {
                        best = b;
                    }
                }

                if (best != a) {
                    int t = cand_idx[a];
                    cand_idx[a] = cand_idx[best];
                    cand_idx[best] = t;
                }
            }

            const int VALIDATE_TOP = 3;
            const int take = (cn < VALIDATE_TOP ? cn : VALIDATE_TOP);

            for (int k = 0; k < take; k++) {
                const int i = cand_idx[k];

                char vleaf[32], vdir[PATH_MAX];

                snprintf(vleaf, sizeof vleaf, "validate%06d", global_trial_idx++);
                path_join2(scratch, vleaf, vdir, sizeof vdir);

                bench_statusf("Autotune run %d/%d | validating @%.2f: io=%d inflight=%zu buf=%zu rdadv=%zu %s",
                              g_bench_outer_idx, g_bench_outer_total, max_frac,
                              pool[i].theta.io_threads, pool[i].theta.inflight_mib,
                              pool[i].theta.buf_mib, pool[i].theta.rdadv_mib,
                              (pool[i].theta.dur == DUR_BATCH ? "batch" : "strict"));

                int comb_err = 0;
                PerfObs obs = measure_combo(&pool[i].theta, &subset, vdir,
                                            base, max_frac,
                                            base.bench_ci_mibs_pct, base.bench_ci_files_pct,
                                            &comb_err);

                const double secs = predict_time_seconds(&obs, &WM);
                pool[i].mibs        = obs.mibs_mean;
                pool[i].filesps     = obs.filesps_mean;
                pool[i].majflt_rate = obs.majflt_rate;
                pool[i].pred_secs   = secs;
                pool[i].score       = score_from_pred_seconds(secs);
                pool[i].eval_frac   = max_frac;

                bench_status_endline();
            }
        }
    }
    /* ------------------------------------ end validation up-shift ----------------------------------- */

    /* Winner among highest tier (after validation) */
    double best_frac = 0.0;
    for (int i = 0; i < pool_n; i++) {
        if (trial_measured(&pool[i]) && pool[i].eval_frac > best_frac) {
            best_frac = pool[i].eval_frac;
        }
    }

    Trial winner = (Trial){0};
    double best_score = DCOPY_SCORE_NEG_INF;

    for (int i = 0; i < pool_n; i++) {
        if (!trial_measured(&pool[i])) continue;
        if (fabs(pool[i].eval_frac - best_frac) > 1e-9) continue;

        if (pool[i].score > best_score) {
            best_score = pool[i].score;
            winner = pool[i];
        }
    }

    if (out_best_frac) *out_best_frac = best_frac;

    /* Final status line for this run (winner) */
    bench_statusf("Autotune run %d/%d | winner: io=%d inflight=%zu buf=%zu rdadv=%zu %s | t_pred=%.2fs",
                  g_bench_outer_idx, g_bench_outer_total,
                  winner.theta.io_threads, winner.theta.inflight_mib, winner.theta.buf_mib, winner.theta.rdadv_mib,
                  (winner.theta.dur == DUR_BATCH ? "batch" : "strict"),
                  winner.pred_secs);

    bench_status_endline();

    /* Optional full table & legend for this run */
    if (print_table) {
        /* Winner summary */
        char rate_buf[48];

        const double bytes_per_sec = winner.mibs * 1024.0 * 1024.0;

        human_rate(bytes_per_sec, rate_buf, sizeof(rate_buf));

        pmsg("\n=== Autotune Result ===");
        pmsg("Recommended flags:");
        pmsg("  --io-threads %d --inflight %zu --buf-size %zu --rdadvise %zu --durability %s%s",
             winner.theta.io_threads,
             winner.theta.inflight_mib,
             winner.theta.buf_mib,
             winner.theta.rdadv_mib,
             (winner.theta.dur == DUR_BATCH ? "batch" : "strict"),
             (winner.theta.pathmode == 1 ? " --kernel-copy" : ""));
        pmsg("Estimated throughput on sample: %s  (small files: %.1f files/s)", rate_buf, winner.filesps);
        pmsg("Predicted full-tree time: %.1f s  (small<=%.0f KiB files counted: %llu)",
             winner.pred_secs,
             WM.small_thresh_bytes / 1024.0,
             (unsigned long long)WM.files_small);

        /* Ranked table (measured rows only) */
        {
            Trial rows[MAX_COMBOS]; int rn = 0;

            for (int i = 0; i < pool_n; i++) {
                if (trial_measured(&pool[i])) {
                    rows[rn++] = pool[i];
                }
            }

            /* Sort by score desc (i.e., t_pred asc) */
            for (int i = 0; i < rn; i++) {
                int b = i;

                for (int j = i + 1; j < rn; j++) {
                    if (rows[j].score > rows[b].score) {
                        b = j;
                    }
                }

                if (b != i) {
                    Trial t = rows[i];
                    rows[i] = rows[b];
                    rows[b] = t;
                }
            }

            pmsg("\n--- Autotune Results (ranked) ---");
            pmsg("%-3s %-9s %-4s %-7s %-11s %-9s %-9s %-10s %-18s %-14s %-8s",
                 "#", "mode", "tier", "io", "inflight", "buf", "rdadv", "dur",
                 "throughput", "t_pred(s)", "score");

            for (int i = 0; i < rn; i++) {
                char rb[48];

                const double bps = rows[i].mibs * 1024.0 * 1024.0;

                human_rate(bps, rb, sizeof(rb));

                pmsg("%-3d %-9s %-4.2f %-7d %-11zu %-9zu %-9zu %-10s %-18s %-14.1f %-8.2f",
                     i + 1,
                     (rows[i].theta.pathmode == 1 ? "kernel" : "streaming"),
                     rows[i].eval_frac,
                     rows[i].theta.io_threads,
                     rows[i].theta.inflight_mib,
                     rows[i].theta.buf_mib,
                     rows[i].theta.rdadv_mib,
                     (rows[i].theta.dur == DUR_BATCH ? "batch" : "strict"),
                     rb,
                     rows[i].pred_secs,
                     rows[i].score);
            }
        }

        /* Legend aligned to the table above */
        print_autotune_legend_single(&WM, best_frac);
    }

    /* Cleanup & restore */
    remove_tree_recursive(scratch);
    ve_free(&subset);

    if (!print_table) {
        g_quiet = saved_quiet;
        g_progress_enabled = saved_prog;
    }

    return winner;
}

/* --------------------------------------------------------------------------
 * Orchestrator: run autotune K times and aggregate statistically
 * Policy:
 *   - Run single-run tuner K = opt.bench_runs times.
 *   - For each winning config, store predicted seconds (t_pred) in AggRow.
 *   - Choose the best configuration by lowest median(t_pred),
 *     tie-breaking by MAD, then mean.
 *   - If K == 1, defer to single-run tuner with full output.
 * -------------------------------------------------------------------------- */
static void
run_autotune(VecEntry *all, const char *dst_root, Options opt)
{
    if (opt.bench_runs <= 1) {
        double best_frac = 0.0;
        (void)run_autotune_once(all, dst_root, opt, /*print_table=*/1, &best_frac);
        return;
    }

    const int K = opt.bench_runs;

    AggRow agg[64];
    int    rn = 0;
    double highest_tier = 0.0;

    for (int r = 0; r < K; r++) {
        /* Expose outer-run context for the status line */
        g_bench_outer_idx   = r + 1;    /* 1-based */
        g_bench_outer_total = K;

        const int print_table = (r == K - 1) ? 1 : 0;

        double run_best_frac = 0.0;
        Trial  win = run_autotune_once(all, dst_root, opt, print_table, &run_best_frac);

        if (run_best_frac > highest_tier) highest_tier = run_best_frac;

        int idx = agg_find_or_add(agg, &rn, (int)(sizeof(agg) / sizeof(agg[0])), &win.theta);
        if (idx >= 0 && agg[idx].n < (int)(sizeof(agg[idx].secs) / sizeof(agg[idx].secs[0]))) {
            agg[idx].secs[ agg[idx].n++ ] = win.pred_secs;
        }

        /* Outer-run progress with best-so-far median */
        double best_so_far = agg_best_median_so_far(agg, rn);
        if (best_so_far < DBL_MAX/2) {
            bench_statusf("Autotune runs: %d/%d complete | best median so far: %.2fs",
                        r + 1, K, best_so_far);
            bench_status_endline();
        } else {
            bench_statusf("Autotune runs: %d/%d complete", r + 1, K);
            bench_status_endline();
        }
    }

    /* Aggregate and report */
    AggStatRow rows[64]; int m = 0;
    for (int i = 0; i < rn; i++) {
        if (agg[i].n <= 0) continue;
        rows[m]     = agg_compute_stats(&agg[i]);
        rows[m].idx = i;
        m++;
    }
    if (m == 0) {
        pmsg("\n=== Autotune Result (aggregated over %d runs) ===", K);
        pmsg("No valid measurements collected.");
        return;
    }

    agg_sort_rows(rows, m);

    const AggStatRow *best = &rows[0];
    const TuneCombo  *th   = &agg[ best->idx ].theta;

    pmsg("\n=== Autotune Result (aggregated over %d runs) ===", K);
    pmsg("Recommended flags (robust median t_pred):");
    pmsg("  --io-threads %d --inflight %zu --buf-size %zu --rdadvise %zu --durability %s%s",
        th->io_threads, th->inflight_mib, th->buf_mib, th->rdadv_mib,
        (th->dur == DUR_BATCH ? "batch" : "strict"),
        (th->pathmode == 1 ? " --kernel-copy" : ""));
    pmsg("Predicted full-tree time (median): %.2f s  [MAD: %.2f s, mean: %.2f s]  (over %d runs)",
        best->med, best->mad, best->mean, best->n);
    pmsg("Highest evaluated tier across runs: %.2f", highest_tier);

    if (opt.bench_show_table) {
        agg_print_table(agg, rows, m, K);
    }

    WorkloadMix WM = compute_workload_mix(all, 1.0 * (1024.0 * 1024.0));
    print_autotune_legend_agg(&WM, opt.bench_runs, highest_tier);
}

// ============================ main ============================

static bool is_directory_path(const char *p) {
    struct stat st;
    if (lstat(p, &st) == 0) {
        return S_ISDIR(st.st_mode);
    }
    return false;
}

int main(int argc, char **argv)
{
    /* ----------------------------- Defaults ----------------------------- */
    Options opt = {0};

    /* Core behavior */
    opt.dry_run            = false;
    opt.follow_symlinks    = false;
    opt.preserve_hardlinks = true;
    opt.progress           = 1;
    opt.include_icloud     = false;
    opt.one_file_system    = false;
    opt.use_kernel_copy    = false;
    opt.sparse_copy        = true;

    /* Deletion defaults */
    opt.delete_source            = false;
    opt.delete_source_existing   = false;
    opt.delete_verify            = VERIFY_HASH;

    /* Tunables */
    opt.jobs_hash          = 0;
    opt.jobs_io            = 0;
    opt.clobber            = CLOBBER_NO;
    opt.meta_mode          = META_SAFE;
    opt.dur_mode           = DUR_STRICT;
    opt.enum_mode          = ENUM_BULK;
    opt.buf_size           = READ_BUFSZ_DEFAULT;
    opt.rdadvise_window    = (size_t)DEFAULT_RDADVISE_MB << 20;
    opt.inflight_cap       = (size_t)DEFAULT_INFLIGHT_MB << 20;
    opt.fsync_batch_files  = 128;
    opt.fsync_batch_bytes  = 128 << 20;

    /* Autotune defaults */
    opt.autotune           = false;
    opt.bench_seconds      = 35;
    opt.bench_byte_budget  = 512ull << 20;     /* 512 MiB per combo */
    opt.bench_large_cap    = 12;
    opt.bench_small_cap    = 1024;
    opt.bench_show_table   = false;
    opt.bench_exhaustive   = false;
    opt.bench_max_combos   = 0;                /* 0 = unlimited unless limited by time */
    opt.force_streaming    = false;
    opt.force_kernel_only  = false;

    /* Autotune noise/search control */
    opt.bench_ci_mibs_pct  = 3.0;
    opt.bench_ci_files_pct = 10.0;
    opt.bench_fast         = false;
    opt.bench_eta          = 3;
    opt.bench_runs         = 1;

    /* --------------------------- Parse options -------------------------- */
    int i = 1;
    while (i < argc) {
        /* basic */
        if (strcmp(argv[i], "-n") == 0) {
            opt.dry_run = true; i++; continue;
        }
        if (strcmp(argv[i], "-v") == 0) {
            g_verbose++; i++; continue;
        }
        if (strcmp(argv[i], "-q") == 0) {
            g_quiet = 1; i++; continue;
        }

        /* threads */
        if (strcmp(argv[i], "-j") == 0 && i + 1 < argc) {
            uint64_t v; if (!parse_u64(argv[i+1], &v) || v < 1 || v > 256) die("invalid -j");
            opt.jobs_hash = (int)v; i += 2; continue;
        }
        if (strcmp(argv[i], "--io-threads") == 0 && i + 1 < argc) {
            uint64_t v; if (!parse_u64(argv[i+1], &v) || v < 1 || v > 256) die("invalid --io-threads");
            opt.jobs_io = (int)v; i += 2; continue;
        }

        /* clobber/update */
        if (strcmp(argv[i], "--overwrite") == 0) {
            opt.clobber = CLOBBER_OVERWRITE; i++; continue;
        }
        if (strcmp(argv[i], "--update") == 0) {
            opt.clobber = CLOBBER_UPDATE; i++; continue;
        }

        /* deletion */
        if (strcmp(argv[i], "--delete-source") == 0) {
            opt.delete_source = true; i++; continue;
        }
        if (strcmp(argv[i], "--delete-source-existing") == 0) {
            opt.delete_source_existing = true; i++; continue;
        }
        if (strcmp(argv[i], "--delete-verify") == 0 && i + 1 < argc) {
            const char *m = argv[i+1];
            if      (strcmp(m, "size")   == 0) opt.delete_verify = VERIFY_SIZE;
            else if (strcmp(m, "sample") == 0) opt.delete_verify = VERIFY_SAMPLE;
            else if (strcmp(m, "hash")   == 0) opt.delete_verify = VERIFY_HASH;
            else die("invalid --delete-verify (use size|sample|hash)");
            i += 2; continue;
        }

        /* traversal flags */
        if (strcmp(argv[i], "--follow-symlinks") == 0) {
            opt.follow_symlinks = true; i++; continue;
        }
        if (strcmp(argv[i], "--no-hardlinks") == 0) {
            opt.preserve_hardlinks = false; i++; continue;
        }
        if (strcmp(argv[i], "--no-progress") == 0) {
            opt.progress = 0; i++; continue;
        }
        if (strcmp(argv[i], "--no-sparse") == 0) {
            opt.sparse_copy = false; i++; continue;
        }
        if (strcmp(argv[i], "--one-file-system") == 0) {
            opt.one_file_system = true; i++; continue;
        }
        if (strcmp(argv[i], "--include-icloud") == 0) {
            opt.include_icloud = true; i++; continue;
        }
        if (strcmp(argv[i], "--kernel-copy") == 0) {
            opt.use_kernel_copy = true; i++; continue;
        }

        /* policy */
        if (strcmp(argv[i], "--metadata") == 0 && i + 1 < argc) {
            const char *m = argv[i+1];
            if      (strcmp(m, "full")    == 0) opt.meta_mode = META_FULL;
            else if (strcmp(m, "safe")    == 0) opt.meta_mode = META_SAFE;
            else if (strcmp(m, "minimal") == 0) opt.meta_mode = META_MINIMAL;
            else die("invalid --metadata");
            i += 2; continue;
        }
        if (strcmp(argv[i], "--durability") == 0 && i + 1 < argc) {
            const char *m = argv[i+1];
            if      (strcmp(m, "strict") == 0) opt.dur_mode = DUR_STRICT;
            else if (strcmp(m, "batch")  == 0) opt.dur_mode = DUR_BATCH;
            else die("invalid --durability");
            i += 2; continue;
        }
        if (strcmp(argv[i], "--enum") == 0 && i + 1 < argc) {
            const char *m = argv[i+1];
            if      (strcmp(m, "bulk") == 0) opt.enum_mode = ENUM_BULK;
            else if (strcmp(m, "fts")  == 0) opt.enum_mode = ENUM_FTS;
            else die("invalid --enum (use bulk|fts)");
            i += 2; continue;
        }

        /* buffers & inflight */
        if (strcmp(argv[i], "--buf-size") == 0 && i + 1 < argc) {
            uint64_t mb; if (!parse_u64(argv[i+1], &mb) || mb < 1 || mb > 1024) die("invalid --buf-size");
            opt.buf_size = (size_t)mb << 20; i += 2; continue;
        }
        if (strcmp(argv[i], "--rdadvise") == 0 && i + 1 < argc) {
            uint64_t mb; if (!parse_u64(argv[i+1], &mb) || mb < 1 || mb > 4096) die("invalid --rdadvise");
            opt.rdadvise_window = (size_t)mb << 20; i += 2; continue;
        }
        if (strcmp(argv[i], "--inflight") == 0 && i + 1 < argc) {
            uint64_t mb; if (!parse_u64(argv[i+1], &mb) || mb < 16 || mb > 16384) die("invalid --inflight");
            opt.inflight_cap = (size_t)mb << 20; i += 2; continue;
        }

        /* batching for DUR_BATCH */
        if (strcmp(argv[i], "--fsync-batch") == 0 && i + 1 < argc) {
            uint64_t n; if (!parse_u64(argv[i+1], &n) || n < 1 || n > 1000000) die("invalid --fsync-batch");
            opt.fsync_batch_files = (size_t)n; i += 2; continue;
        }
        if (strcmp(argv[i], "--fsync-bytes") == 0 && i + 1 < argc) {
            uint64_t mb; if (!parse_u64(argv[i+1], &mb) || mb < 1 || mb > 1048576) die("invalid --fsync-bytes");
            opt.fsync_batch_bytes = (size_t)mb << 20; i += 2; continue;
        }

        /* help */
        if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
            usage(stdout); return 0;
        }

        /* autotune/bench */
        if (strcmp(argv[i], "--autotune") == 0) {
            opt.autotune = true; i++; continue;
        }
        if (strcmp(argv[i], "--bench-seconds") == 0 && i + 1 < argc) {
            uint64_t v; if (!parse_u64(argv[i+1], &v) || v < 5 || v > 600) die("invalid --bench-seconds");
            opt.bench_seconds = (int)v; i += 2; continue;
        }
        if (strcmp(argv[i], "--bench-max") == 0 && i + 1 < argc) {
            uint64_t v;
            if (!parse_u64(argv[i + 1], &v) || v > 1000000ull) die("invalid --bench-max");
            opt.bench_max_combos = (size_t)v; i += 2; continue;
        }
        if (strcmp(argv[i], "--bench-mib") == 0 && i + 1 < argc) {
            uint64_t v; if (!parse_u64(argv[i+1], &v) || v < 64 || v > 16384) die("invalid --bench-mib");
            opt.bench_byte_budget = v << 20; i += 2; continue;
        }
        if (strcmp(argv[i], "--bench-large") == 0 && i + 1 < argc) {
            uint64_t v; if (!parse_u64(argv[i+1], &v) || v < 1 || v > 256) die("invalid --bench-large");
            opt.bench_large_cap = (size_t)v; i += 2; continue;
        }
        if (strcmp(argv[i], "--bench-small") == 0 && i + 1 < argc) {
            uint64_t v; if (!parse_u64(argv[i+1], &v) || v < 1 || v > 32768) die("invalid --bench-small");
            opt.bench_small_cap = (size_t)v; i += 2; continue;
        }
        if (strcmp(argv[i], "--bench-table") == 0) {
            opt.bench_show_table = true; i++; continue;
        }
        if (strcmp(argv[i], "--bench-exhaustive") == 0) {
            opt.bench_exhaustive = true; i++; continue;
        }
        if (strcmp(argv[i], "--bench-ci-mibs") == 0 && i + 1 < argc) {
            uint64_t v; if (!parse_u64(argv[i+1], &v) || v < 1 || v > 100) die("invalid --bench-ci-mibs");
            opt.bench_ci_mibs_pct = (double)v; i += 2; continue;
        }
        if (strcmp(argv[i], "--bench-ci-files") == 0 && i + 1 < argc) {
            uint64_t v; if (!parse_u64(argv[i+1], &v) || v < 1 || v > 100) die("invalid --bench-ci-files");
            opt.bench_ci_files_pct = (double)v; i += 2; continue;
        }
        if (strcmp(argv[i], "--bench-fast") == 0) {
            opt.bench_fast = true; i++; continue;
        }
        if (strcmp(argv[i], "--bench-eta") == 0 && i + 1 < argc) {
            uint64_t v; if (!parse_u64(argv[i+1], &v) || v < 2 || v > 8) die("invalid --bench-eta");
            opt.bench_eta = (int)v; i += 2; continue;
        }
        if (strcmp(argv[i], "--bench-runs") == 0 && i + 1 < argc) {
            uint64_t v; if (!parse_u64(argv[i+1], &v) || v < 1 || v > 50) die("invalid --bench-runs (1..50)");
            opt.bench_runs = (int)v; i += 2; continue;
        }

        /* first non-option */
        break;
    }

    /* ------------------------------ Args ------------------------------- */
    g_progress_enabled = opt.progress && !g_quiet;

    const int n_paths = argc - i;
    if (n_paths < 2) {
        usage(stderr);
        return 2;
    }

    char *dst_root = argv[argc - 1];
    const int n_srcs = n_paths - 1;

    struct stat st_dst;
    const bool dst_exists = (lstat(dst_root, &st_dst) == 0);
    const bool dst_is_dir = dst_exists
                          ? S_ISDIR(st_dst.st_mode)
                          : (n_srcs > 1 || is_directory_path(argv[i]));

    if (!dst_exists && (n_srcs > 1 || is_directory_path(argv[i]))) {
        if (!opt.dry_run) {
            int rc = ensure_dir_exists_recursive(dst_root);

            if (rc != 0) {
                int e = errno;

                if (is_perm_error(e) || rc == -2) {
                    warnx("mkdir: %s: %s (blocked; proceeding, paths under this will be skipped)", dst_root, strerror(e));
                } else {
                    warnx("mkdir: %s: %s", dst_root, strerror(e));
                }
            }
        } else {
            pmsg("PLAN   MKDIR %s", dst_root);
        }
    } else if (dst_exists && !dst_is_dir && n_srcs > 1) {
        die("destination must be a directory with multiple sources");
    }

    /* -------------------------- Initialization ------------------------- */
    dirfd_cache_init();
    hcache_load(dst_root);
    g_inflight_cap = opt.inflight_cap;

    const int cpus   = cpu_count();
    int       n_hash = opt.jobs_hash ? opt.jobs_hash : cpus;
    int       n_io   = opt.jobs_io   ? opt.jobs_io   : (cpus < 4 ? cpus : 4);
    if (n_hash < 1) n_hash = 1; if (n_hash > MAX_THREADS) n_hash = MAX_THREADS;
    if (n_io   < 1) n_io   = 1; if (n_io   > MAX_THREADS) n_io   = MAX_THREADS;

    bp_init(opt.buf_size, n_io);

    RootPlan *rp = (RootPlan *)calloc((size_t)n_srcs, sizeof(RootPlan));
    if (!rp) die("OOM");

    if (n_srcs == 1) {
        struct stat st_src;
        if (lstat(argv[i], &st_src) != 0) die("stat src: %s: %s", argv[i], strerror(errno));
        rp[0].src_root    = xstrdup(argv[i]);
        rp[0].is_dir      = S_ISDIR(st_src.st_mode);
        rp[0].inject_base = xstrdup(rp[0].is_dir ? "" : path_basename(argv[i]));
    } else {
        for (int k = 0; k < n_srcs; k++) {
            struct stat st_src;
            if (lstat(argv[i + k], &st_src) != 0) die("stat src: %s: %s", argv[i + k], strerror(errno));
            rp[k].src_root    = xstrdup(argv[i + k]);
            rp[k].is_dir      = S_ISDIR(st_src.st_mode);
            rp[k].inject_base = xstrdup(path_basename(argv[i + k]));
        }
    }

    /* --------------------------- Enumeration --------------------------- */
    pmsg("Scanning paths and collecting file information...");

    VecEntry ents = (VecEntry){0};
    for (int k = 0; k < n_srcs; k++) {
        if (opt.enum_mode == ENUM_BULK) {
            collect_entries_bulk(&ents, &rp[k], &opt);
        } else {
            collect_entries_fts(&ents, &rp[k], &opt);
        }
    }

    /* Filter iCloud container prefixes unless explicitly included */
    {
        size_t w = 0;
        for (size_t z = 0; z < ents.n; z++) {
            const char *p = ents.v[z].src_path;
            if (!opt.include_icloud &&
                (strstr(p, "/Library/Mobile Documents/") ||
                 strstr(p, "/com~apple~CloudDocs/")))
            {
                if (g_verbose) vmsg("skip iCloud path prefix: %s", p);
                continue;
            }
            ents.v[w++] = ents.v[z];
        }
        ents.n = w;
    }

    pmsg("Collected %zu files and directories to process.", ents.n);

    /* ----------------------------- Autotune ---------------------------- */
    if (opt.autotune) {
        pmsg("Autotuning on representative samples (non-destructive)...");
        run_autotune(&ents, dst_root, opt);

        ve_free(&ents);
        for (int k = 0; k < n_srcs; k++) {
            free(rp[k].src_root);
            free(rp[k].inject_base);
        }
        free(rp);
        dirfd_cache_close_all();
        return 0;
    }

    /* -------------------- Pre-analysis (sampling/hash) ----------------- */
    const size_t N = ents.n;

    size_t *idx_by_size = (size_t *)malloc(N * sizeof(size_t));
    if (!idx_by_size) die("OOM");
    for (size_t z = 0; z < N; z++) idx_by_size[z] = z;
    g_sort_ents = &ents;
    qsort(idx_by_size, N, sizeof(size_t), cmp_size_then_rel_idx_asc);

    size_t   *sample_indices = NULL; size_t sn = 0, scap = 0;
    size_t   *hash_indices   = NULL; size_t hn = 0, hcap = 0;
    uint64_t  total_bytes    = 0;

    /* Sample pass for 3-point fingerprints and total bytes */
    size_t i0 = 0;
    while (i0 < N) {
        size_t j  = i0 + 1;
        off_t  sz = ents.v[idx_by_size[i0]].size;
        while (j < N && ents.v[idx_by_size[j]].size == sz) j++;
        const size_t cnt = j - i0;

        for (size_t k = i0; k < j; k++) {
            const size_t ix = idx_by_size[k];
            if (ents.v[ix].type != T_FILE) continue;
            total_bytes += (uint64_t)ents.v[ix].size;
            if (cnt > 1) {
                if (sn == scap) {
                    scap = scap ? (scap << 1) : 4096;
                    sample_indices = (size_t *)realloc(sample_indices, scap * sizeof(size_t));
                    if (!sample_indices) die("OOM");
                }
                sample_indices[sn++] = ix;
            }
        }
        i0 = j;
    }

    if (sn > 0) {
        WorkQueue sw = {
            .ents = &ents,
            .indices = sample_indices,
            .count = sn,
            .next = 0,
            .done = 0
        };

        pthread_mutex_init(&sw.mu, NULL);

        pthread_t *tids = (pthread_t *)calloc((size_t)opt.jobs_hash ? (size_t)opt.jobs_hash : (size_t)cpu_count(),
                                              sizeof(pthread_t));

        const int nth = (int)((tids) ? ((opt.jobs_hash) ? opt.jobs_hash : cpu_count()) : 0);

        if (!tids) {
            die("OOM");
        }

        for (int t = 0; t < nth; t++) {
            if (pthread_create(&tids[t], NULL, sample_worker, &sw) != 0) {
                die("pthread_create(sample)");
            }
        }

        while (true) {
            uint64_t done = __atomic_load_n(&sw.done, __ATOMIC_RELAXED);

            update_sample_progress(done, sw.count);

            if (done >= sw.count) {
                break;
            }

            struct timespec ts = {
                .tv_sec = 0,
                .tv_nsec = 100000000
            };

            (void)nanosleep(&ts, NULL);
        }

        for (int t = 0; t < nth; t++) {
            pthread_join(tids[t], NULL);
        }

        free(tids);

        pthread_mutex_destroy(&sw.mu);
    }

    free(sample_indices);

    /* Physical-fingerprint candidates (equal-size groups >=3, size >= threshold) */
    size_t *phys_indices = NULL;
    size_t pn = 0, pcap = 0;

    i0 = 0;

    while (i0 < N) {
        size_t j  = i0 + 1;
        off_t  sz = ents.v[idx_by_size[i0]].size;

        while (j < N && ents.v[idx_by_size[j]].size == sz) {
            j++;
        }

        const size_t cnt = j - i0;

        if (cnt >= 3 && sz >= PHYS_FP_MIN_SIZE) {
            for (size_t k = i0; k < j; k++) {
                const size_t ix = idx_by_size[k];

                if (ents.v[ix].type != T_FILE) {
                    continue;
                }

                if (pn == pcap) {
                    pcap = pcap ? (pcap << 1) : 4096;

                    phys_indices = (size_t *)realloc(phys_indices, pcap * sizeof(size_t));

                    if (!phys_indices) {
                        die("OOM");
                    }
                }

                phys_indices[pn++] = ix;
            }
        }

        i0 = j;
    }

    if (pn > 0) {
        WorkQueue pw = {
            .ents = &ents,
            .indices = phys_indices,
            .count = pn,
            .next = 0,
            .done = 0
        };

        pthread_mutex_init(&pw.mu, NULL);

        const int nth = opt.jobs_hash ? opt.jobs_hash : cpu_count();

        pthread_t *ptids = (pthread_t *)calloc((size_t)nth, sizeof(pthread_t));

        if (!ptids) {
            die("OOM");
        }

        for (int t = 0; t < nth; t++) {
            if (pthread_create(&ptids[t], NULL, phys_worker, &pw) != 0) {
                die("pthread_create(phys)");
            }
        }

        while (true) {
            uint64_t done = __atomic_load_n(&pw.done, __ATOMIC_RELAXED);

            update_sample_progress(done, pw.count);

            if (done >= pw.count) {
                break;
            }

            struct timespec ts = {
                .tv_sec = 0,
                .tv_nsec = 100000000
            };

            (void)nanosleep(&ts, NULL);
        }

        for (int t = 0; t < nth; t++) {
            pthread_join(ptids[t], NULL);
        }

        free(ptids);

        pthread_mutex_destroy(&pw.mu);
    }

    free(phys_indices);

    /* Sort each equal-size group by sample64 to cluster identicals (stable small sort) */
    pmsg("Analyzing file content (sorting samples)...");

    size_t sort_jobs_count = 0;

    SortWork *sort_jobs = (SortWork *)malloc(N * sizeof(SortWork));

    if (!sort_jobs) {
        die("OOM");
    }

    i0 = 0;

    while (i0 < N) {
        size_t j = i0 + 1;

        while (j < N && ents.v[idx_by_size[j]].size == ents.v[idx_by_size[i0]].size) {
            j++;
        }

        if (j - i0 > 1) {
            sort_jobs[sort_jobs_count++] = (SortWork){
                .indices = idx_by_size,
                .start   = i0,
                .end     = j,
                .ents    = &ents
            };
        }

        i0 = j;
    }

    if (sort_jobs_count > 0) {
        const int nth = opt.jobs_hash ? opt.jobs_hash : cpu_count();

        pthread_t *sort_tids = (pthread_t *)calloc((size_t)nth, sizeof(pthread_t));

        if (!sort_tids) {
            die("OOM");
        }

        for (size_t k = 0; k < sort_jobs_count; k++) {
            int thread_idx = (int)(k % (size_t)nth);

            if (k >= (size_t)nth) {
                pthread_join(sort_tids[thread_idx], NULL);
            }

            if (pthread_create(&sort_tids[thread_idx], NULL, sort_worker, &sort_jobs[k]) != 0) {
                die("pthread_create(sort)");
            }
        }

        for (int t = 0; t < nth; t++) {
            pthread_join(sort_tids[t], NULL);
        }

        free(sort_tids);
    }
    free(sort_jobs);

    /* Build list of files to hash fully (groups with same size+sample64) */
    i0 = 0;

    while (i0 < N) {
        size_t j = i0 + 1;

        while (j < N &&
               ents.v[idx_by_size[j]].size     == ents.v[idx_by_size[i0]].size &&
               ents.v[idx_by_size[j]].sample64 == ents.v[idx_by_size[i0]].sample64) {
            j++;
        }

        if (j - i0 > 1) {
            for (size_t k = i0; k < j; k++) {
                const size_t ix = idx_by_size[k];

                if (ents.v[ix].type != T_FILE) {
                    continue;
                }

                if (hn == hcap) {
                    hcap = hcap ? (hcap << 1) : 4096;
                    hash_indices = (size_t *)realloc(hash_indices, hcap * sizeof(size_t));

                    if (!hash_indices) {
                        die("OOM");
                    }
                }

                hash_indices[hn++] = ix;
            }
        }

        i0 = j;
    }

    if (hn > 0) {
        WorkQueue hw = {
            .ents    = &ents,
            .indices = hash_indices,
            .count   = hn,
            .next    = 0,
            .done    = 0,
            .bufsize = (int)opt.buf_size
        };

        pthread_mutex_init(&hw.mu, NULL);

        const int nth = opt.jobs_hash ? opt.jobs_hash : cpu_count();

        pthread_t *tids = (pthread_t *)calloc((size_t)nth, sizeof(pthread_t));

        if (!tids) {
            die("OOM");
        }

        for (int t = 0; t < nth; t++) {
            if (pthread_create(&tids[t], NULL, hash_worker, &hw) != 0) {
                die("pthread_create(hash)");
            }
        }

        while (true) {
            uint64_t done = __atomic_load_n(&hw.done, __ATOMIC_RELAXED);

            update_hash_progress(done, hw.count);

            if (done >= hw.count) {
                break;
            }

            struct timespec ts = { .tv_sec = 0, .tv_nsec = 100000000 };

            (void)nanosleep(&ts, NULL);
        }

        for (int t = 0; t < nth; t++) {
            pthread_join(tids[t], NULL);
        }

        free(tids);

        pthread_mutex_destroy(&hw.mu);
    }

    free(hash_indices);

    /* ---------------------------- IO Pipeline --------------------------- */
    Stats stats = {0};
    stats.bytes_total = total_bytes;

    /* Count totals for summary/progress (files contribute to bytes_total) */
    stats.files_total      = 0;
    stats.dirs_total_jobs  = 0;
    stats.symlinks_total   = 0;

    for (size_t z = 0; z < ents.n; z++) {
        switch (ents.v[z].type) {
            case T_FILE:
                stats.files_total++;
                break;
            case T_DIR:
                stats.dirs_total_jobs++;
                break;
            case T_SYMLINK:
                stats.symlinks_total++;
                break;
            default:
                break;
        }
    }

    __atomic_store_n(&stats.copy_phase_started, 0, __ATOMIC_RELAXED);

    gettimeofday(&stats.start_time, NULL);

    DigestMap dmap; dmap_init(&dmap);
    InoMap    imap; imap_init(&imap);
    PhysMap   pmap; pmap_init(&pmap);

    IOJobQueue io_q;
    io_queue_init(&io_q, ents.n > 4096 ? ents.n : 4096);

    IOContext  io_ctx = { .opt = &opt, .dmap = &dmap, .imap = &imap, .pmap = &pmap, .stats = &stats };
    IOWorkerCtx wctx  = { .q   = &io_q, .ents = &ents, .ctx  = &io_ctx };

    pthread_t *io_tids = (pthread_t *)calloc((size_t)n_io, sizeof(pthread_t));

    if (!io_tids) {
        die("OOM");
    }

    for (int t = 0; t < n_io; t++) {
        if (pthread_create(&io_tids[t], NULL, io_worker, &wctx) != 0) {
            die("pthread_create(io_worker)");
        }
    }

    pthread_t  progress_tid;
    ProgressCtx progress_ctx = { .stats = &stats, .done = 0 };

    if (g_progress_enabled) {
        if (pthread_create(&progress_tid, NULL, progress_updater_thread, &progress_ctx) != 0)
            die("failed to create progress thread");
    }

    /* Enqueue directories first */
    for (size_t zz = 0; zz < ents.n; zz++) {
        Entry *e = &ents.v[zz];

        if (e->type != T_DIR) {
            continue;
        }

        char dst_path[PATH_MAX];

        if (n_srcs == 1) {
            struct stat st_src0; lstat(argv[i], &st_src0);

            if (S_ISDIR(st_src0.st_mode)) {
                if (e->rel_path[0] == '\0') {
                    snprintf(dst_path, sizeof(dst_path), "%s", dst_root);
                } else {
                    path_join2(dst_root, e->rel_path, dst_path, sizeof(dst_path));
                }
            } else {
                if (dst_is_dir) {
                    path_join2(dst_root, e->rel_path, dst_path, sizeof(dst_path));
                } else {
                    snprintf(dst_path, sizeof(dst_path), "%s", dst_root);
                }
            }
        } else {
            path_join2(dst_root, e->rel_path, dst_path, sizeof(dst_path));
        }

        IOJob job = { .entry_idx = zz };

        snprintf(job.dst_path, sizeof(job.dst_path), "%s", dst_path);

        io_queue_push(&io_q, job);
    }

    /* Enqueue files (size-desc) */
    size_t n_files = 0;

    for (size_t zz = 0; zz < ents.n; zz++) {
        if (ents.v[zz].type == T_FILE) {
            n_files++;
        }
    }

    if (n_files > 0) {
        size_t *file_idx = (size_t *)malloc(n_files * sizeof(size_t));

        if (!file_idx) {
            die("OOM");
        }

        size_t pos = 0;

        for (size_t zz = 0; zz < ents.n; zz++) {
            if (ents.v[zz].type == T_FILE) {
                file_idx[pos++] = zz;
            }
        }

        g_sort_ents = &ents;

        qsort(file_idx, n_files, sizeof(size_t), cmp_size_desc_then_rel);

        for (size_t kf = 0; kf < n_files; kf++) {
            if (kf == 0) __atomic_store_n(&stats.copy_phase_started, 1, __ATOMIC_RELAXED);

            const size_t zz_idx = file_idx[kf];
            Entry *e = &ents.v[zz_idx];

            char dst_path[PATH_MAX];
            if (n_srcs == 1) {
                struct stat st_src0; lstat(argv[i], &st_src0);
                if (S_ISDIR(st_src0.st_mode)) {
                    if (e->rel_path[0] == '\0') snprintf(dst_path, sizeof(dst_path), "%s", dst_root);
                    else                         path_join2(dst_root, e->rel_path, dst_path, sizeof(dst_path));
                } else {
                    if (dst_is_dir) path_join2(dst_root, e->rel_path, dst_path, sizeof(dst_path));
                    else             snprintf(dst_path, sizeof(dst_path), "%s", dst_root);
                }
            } else {
                path_join2(dst_root, e->rel_path, dst_path, sizeof(dst_path));
            }

            IOJob job = { .entry_idx = zz_idx };
            snprintf(job.dst_path, sizeof(job.dst_path), "%s", dst_path);
            io_queue_push(&io_q, job);
        }
        free(file_idx);
    }

    /* Enqueue others (symlinks/special) */
    for (size_t zz = 0; zz < ents.n; zz++) {
        Entry *e = &ents.v[zz];
        if (e->type == T_FILE || e->type == T_DIR) continue;

        char dst_path[PATH_MAX];
        if (n_srcs == 1) {
            struct stat st_src0; lstat(argv[i], &st_src0);
            if (S_ISDIR(st_src0.st_mode)) {
                if (e->rel_path[0] == '\0') snprintf(dst_path, sizeof(dst_path), "%s", dst_root);
                else                         path_join2(dst_root, e->rel_path, dst_path, sizeof(dst_path));
            } else {
                if (dst_is_dir) path_join2(dst_root, e->rel_path, dst_path, sizeof(dst_path));
                else             snprintf(dst_path, sizeof(dst_path), "%s", dst_root);
            }
        } else {
            path_join2(dst_root, e->rel_path, dst_path, sizeof(dst_path));
        }

        IOJob job = { .entry_idx = zz };
        snprintf(job.dst_path, sizeof(job.dst_path), "%s", dst_path);
        io_queue_push(&io_q, job);
    }

    /* Finish queue and join workers */
    io_queue_finish(&io_q);

    for (int t = 0; t < n_io; t++) {
        pthread_join(io_tids[t], NULL);
    }

    free(io_tids);

    prune_empty_source_dirs(&opt, &ents);

    if (opt.dur_mode == DUR_BATCH) {
        dirfd_cache_fsync_all();
    }

    if (g_progress_enabled) {
        progress_ctx.done = 1;
        pthread_join(progress_tid, NULL);
    }

    /* ------------------------------ Summary ---------------------------- */
    print_summary(&stats, &opt);

    /* ------------------------------ Cleanup ---------------------------- */
    dmap_free(&dmap);
    imap_free(&imap);
    pmap_free(&pmap);
    ve_free(&ents);

    for (int k = 0; k < n_srcs; k++) {
        free(rp[k].src_root);
        free(rp[k].inject_base);
    }
    free(rp);

    free(idx_by_size);
    dirfd_cache_close_all();

    return 0;
}

xxhash.h

/*
 * xxHash - Extremely Fast Hash algorithm
 * Header File
 * Copyright (C) 2012-2023 Yann Collet
 *
 * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *    * Redistributions of source code must retain the above copyright
 *      notice, this list of conditions and the following disclaimer.
 *    * Redistributions in binary form must reproduce the above
 *      copyright notice, this list of conditions and the following disclaimer
 *      in the documentation and/or other materials provided with the
 *      distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * You can contact the author at:
 *   - xxHash homepage: https://www.xxhash.com
 *   - xxHash source repository: https://github.com/Cyan4973/xxHash
 */

/*!
 * @mainpage xxHash
 *
 * xxHash is an extremely fast non-cryptographic hash algorithm, working at RAM speed
 * limits.
 *
 * It is proposed in four flavors, in three families:
 * 1. @ref XXH32_family
 *   - Classic 32-bit hash function. Simple, compact, and runs on almost all
 *     32-bit and 64-bit systems.
 * 2. @ref XXH64_family
 *   - Classic 64-bit adaptation of XXH32. Just as simple, and runs well on most
 *     64-bit systems (but _not_ 32-bit systems).
 * 3. @ref XXH3_family
 *   - Modern 64-bit and 128-bit hash function family which features improved
 *     strength and performance across the board, especially on smaller data.
 *     It benefits greatly from SIMD and 64-bit without requiring it.
 *
 * Benchmarks
 * ---
 * The reference system uses an Intel i7-9700K CPU, and runs Ubuntu x64 20.04.
 * The open source benchmark program is compiled with clang v10.0 using -O3 flag.
 *
 * | Hash Name            | ISA ext | Width | Large Data Speed | Small Data Velocity |
 * | -------------------- | ------- | ----: | ---------------: | ------------------: |
 * | XXH3_64bits()        | @b AVX2 |    64 |        59.4 GB/s |               133.1 |
 * | MeowHash             | AES-NI  |   128 |        58.2 GB/s |                52.5 |
 * | XXH3_128bits()       | @b AVX2 |   128 |        57.9 GB/s |               118.1 |
 * | CLHash               | PCLMUL  |    64 |        37.1 GB/s |                58.1 |
 * | XXH3_64bits()        | @b SSE2 |    64 |        31.5 GB/s |               133.1 |
 * | XXH3_128bits()       | @b SSE2 |   128 |        29.6 GB/s |               118.1 |
 * | RAM sequential read  |         |   N/A |        28.0 GB/s |                 N/A |
 * | ahash                | AES-NI  |    64 |        22.5 GB/s |               107.2 |
 * | City64               |         |    64 |        22.0 GB/s |                76.6 |
 * | T1ha2                |         |    64 |        22.0 GB/s |                99.0 |
 * | City128              |         |   128 |        21.7 GB/s |                57.7 |
 * | FarmHash             | AES-NI  |    64 |        21.3 GB/s |                71.9 |
 * | XXH64()              |         |    64 |        19.4 GB/s |                71.0 |
 * | SpookyHash           |         |    64 |        19.3 GB/s |                53.2 |
 * | Mum                  |         |    64 |        18.0 GB/s |                67.0 |
 * | CRC32C               | SSE4.2  |    32 |        13.0 GB/s |                57.9 |
 * | XXH32()              |         |    32 |         9.7 GB/s |                71.9 |
 * | City32               |         |    32 |         9.1 GB/s |                66.0 |
 * | Blake3*              | @b AVX2 |   256 |         4.4 GB/s |                 8.1 |
 * | Murmur3              |         |    32 |         3.9 GB/s |                56.1 |
 * | SipHash*             |         |    64 |         3.0 GB/s |                43.2 |
 * | Blake3*              | @b SSE2 |   256 |         2.4 GB/s |                 8.1 |
 * | HighwayHash          |         |    64 |         1.4 GB/s |                 6.0 |
 * | FNV64                |         |    64 |         1.2 GB/s |                62.7 |
 * | Blake2*              |         |   256 |         1.1 GB/s |                 5.1 |
 * | SHA1*                |         |   160 |         0.8 GB/s |                 5.6 |
 * | MD5*                 |         |   128 |         0.6 GB/s |                 7.8 |
 * @note
 *   - Hashes which require a specific ISA extension are noted. SSE2 is also noted,
 *     even though it is mandatory on x64.
 *   - Hashes with an asterisk are cryptographic. Note that MD5 is non-cryptographic
 *     by modern standards.
 *   - Small data velocity is a rough average of algorithm's efficiency for small
 *     data. For more accurate information, see the wiki.
 *   - More benchmarks and strength tests are found on the wiki:
 *         https://github.com/Cyan4973/xxHash/wiki
 *
 * Usage
 * ------
 * All xxHash variants use a similar API. Changing the algorithm is a trivial
 * substitution.
 *
 * @pre
 *    For functions which take an input and length parameter, the following
 *    requirements are assumed:
 *    - The range from [`input`, `input + length`) is valid, readable memory.
 *      - The only exception is if the `length` is `0`, `input` may be `NULL`.
 *    - For C++, the objects must have the *TriviallyCopyable* property, as the
 *      functions access bytes directly as if it was an array of `unsigned char`.
 *
 * @anchor single_shot_example
 * **Single Shot**
 *
 * These functions are stateless functions which hash a contiguous block of memory,
 * immediately returning the result. They are the easiest and usually the fastest
 * option.
 *
 * XXH32(), XXH64(), XXH3_64bits(), XXH3_128bits()
 *
 * @code{.c}
 *   #include <string.h>
 *   #include "xxhash.h"
 *
 *   // Example for a function which hashes a null terminated string with XXH32().
 *   XXH32_hash_t hash_string(const char* string, XXH32_hash_t seed)
 *   {
 *       // NULL pointers are only valid if the length is zero
 *       size_t length = (string == NULL) ? 0 : strlen(string);
 *       return XXH32(string, length, seed);
 *   }
 * @endcode
 *
 *
 * @anchor streaming_example
 * **Streaming**
 *
 * These groups of functions allow incremental hashing of unknown size, even
 * more than what would fit in a size_t.
 *
 * XXH32_reset(), XXH64_reset(), XXH3_64bits_reset(), XXH3_128bits_reset()
 *
 * @code{.c}
 *   #include <stdio.h>
 *   #include <assert.h>
 *   #include "xxhash.h"
 *   // Example for a function which hashes a FILE incrementally with XXH3_64bits().
 *   XXH64_hash_t hashFile(FILE* f)
 *   {
 *       // Allocate a state struct. Do not just use malloc() or new.
 *       XXH3_state_t* state = XXH3_createState();
 *       assert(state != NULL && "Out of memory!");
 *       // Reset the state to start a new hashing session.
 *       XXH3_64bits_reset(state);
 *       char buffer[4096];
 *       size_t count;
 *       // Read the file in chunks
 *       while ((count = fread(buffer, 1, sizeof(buffer), f)) != 0) {
 *           // Run update() as many times as necessary to process the data
 *           XXH3_64bits_update(state, buffer, count);
 *       }
 *       // Retrieve the finalized hash. This will not change the state.
 *       XXH64_hash_t result = XXH3_64bits_digest(state);
 *       // Free the state. Do not use free().
 *       XXH3_freeState(state);
 *       return result;
 *   }
 * @endcode
 *
 * Streaming functions generate the xxHash value from an incremental input.
 * This method is slower than single-call functions, due to state management.
 * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized.
 *
 * An XXH state must first be allocated using `XXH*_createState()`.
 *
 * Start a new hash by initializing the state with a seed using `XXH*_reset()`.
 *
 * Then, feed the hash state by calling `XXH*_update()` as many times as necessary.
 *
 * The function returns an error code, with 0 meaning OK, and any other value
 * meaning there is an error.
 *
 * Finally, a hash value can be produced anytime, by using `XXH*_digest()`.
 * This function returns the nn-bits hash as an int or long long.
 *
 * It's still possible to continue inserting input into the hash state after a
 * digest, and generate new hash values later on by invoking `XXH*_digest()`.
 *
 * When done, release the state using `XXH*_freeState()`.
 *
 *
 * @anchor canonical_representation_example
 * **Canonical Representation**
 *
 * The default return values from XXH functions are unsigned 32, 64 and 128 bit
 * integers.
 * This the simplest and fastest format for further post-processing.
 *
 * However, this leaves open the question of what is the order on the byte level,
 * since little and big endian conventions will store the same number differently.
 *
 * The canonical representation settles this issue by mandating big-endian
 * convention, the same convention as human-readable numbers (large digits first).
 *
 * When writing hash values to storage, sending them over a network, or printing
 * them, it's highly recommended to use the canonical representation to ensure
 * portability across a wider range of systems, present and future.
 *
 * The following functions allow transformation of hash values to and from
 * canonical format.
 *
 * XXH32_canonicalFromHash(), XXH32_hashFromCanonical(),
 * XXH64_canonicalFromHash(), XXH64_hashFromCanonical(),
 * XXH128_canonicalFromHash(), XXH128_hashFromCanonical(),
 *
 * @code{.c}
 *   #include <stdio.h>
 *   #include "xxhash.h"
 *
 *   // Example for a function which prints XXH32_hash_t in human readable format
 *   void printXxh32(XXH32_hash_t hash)
 *   {
 *       XXH32_canonical_t cano;
 *       XXH32_canonicalFromHash(&cano, hash);
 *       size_t i;
 *       for(i = 0; i < sizeof(cano.digest); ++i) {
 *           printf("%02x", cano.digest[i]);
 *       }
 *       printf("\n");
 *   }
 *
 *   // Example for a function which converts XXH32_canonical_t to XXH32_hash_t
 *   XXH32_hash_t convertCanonicalToXxh32(XXH32_canonical_t cano)
 *   {
 *       XXH32_hash_t hash = XXH32_hashFromCanonical(&cano);
 *       return hash;
 *   }
 * @endcode
 *
 *
 * @file xxhash.h
 * xxHash prototypes and implementation
 */

#if defined(__cplusplus) && !defined(XXH_NO_EXTERNC_GUARD)
extern "C" {
#endif

/* ****************************
 *  INLINE mode
 ******************************/
/*!
 * @defgroup public Public API
 * Contains details on the public xxHash functions.
 * @{
 */
#ifdef XXH_DOXYGEN
/*!
 * @brief Gives access to internal state declaration, required for static allocation.
 *
 * Incompatible with dynamic linking, due to risks of ABI changes.
 *
 * Usage:
 * @code{.c}
 *     #define XXH_STATIC_LINKING_ONLY
 *     #include "xxhash.h"
 * @endcode
 */
#  define XXH_STATIC_LINKING_ONLY
/* Do not undef XXH_STATIC_LINKING_ONLY for Doxygen */

/*!
 * @brief Gives access to internal definitions.
 *
 * Usage:
 * @code{.c}
 *     #define XXH_STATIC_LINKING_ONLY
 *     #define XXH_IMPLEMENTATION
 *     #include "xxhash.h"
 * @endcode
 */
#  define XXH_IMPLEMENTATION
/* Do not undef XXH_IMPLEMENTATION for Doxygen */

/*!
 * @brief Exposes the implementation and marks all functions as `inline`.
 *
 * Use these build macros to inline xxhash into the target unit.
 * Inlining improves performance on small inputs, especially when the length is
 * expressed as a compile-time constant:
 *
 *  https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html
 *
 * It also keeps xxHash symbols private to the unit, so they are not exported.
 *
 * Usage:
 * @code{.c}
 *     #define XXH_INLINE_ALL
 *     #include "xxhash.h"
 * @endcode
 * Do not compile and link xxhash.o as a separate object, as it is not useful.
 */
#  define XXH_INLINE_ALL
#  undef XXH_INLINE_ALL
/*!
 * @brief Exposes the implementation without marking functions as inline.
 */
#  define XXH_PRIVATE_API
#  undef XXH_PRIVATE_API
/*!
 * @brief Emulate a namespace by transparently prefixing all symbols.
 *
 * If you want to include _and expose_ xxHash functions from within your own
 * library, but also want to avoid symbol collisions with other libraries which
 * may also include xxHash, you can use @ref XXH_NAMESPACE to automatically prefix
 * any public symbol from xxhash library with the value of @ref XXH_NAMESPACE
 * (therefore, avoid empty or numeric values).
 *
 * Note that no change is required within the calling program as long as it
 * includes `xxhash.h`: Regular symbol names will be automatically translated
 * by this header.
 */
#  define XXH_NAMESPACE /* YOUR NAME HERE */
#  undef XXH_NAMESPACE
#endif

#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \
    && !defined(XXH_INLINE_ALL_31684351384)
   /* this section should be traversed only once */
#  define XXH_INLINE_ALL_31684351384
   /* give access to the advanced API, required to compile implementations */
#  undef XXH_STATIC_LINKING_ONLY   /* avoid macro redef */
#  define XXH_STATIC_LINKING_ONLY
   /* make all functions private */
#  undef XXH_PUBLIC_API
#  if defined(__GNUC__)
#    define XXH_PUBLIC_API static __inline __attribute__((__unused__))
#  elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
#    define XXH_PUBLIC_API static inline
#  elif defined(_MSC_VER)
#    define XXH_PUBLIC_API static __inline
#  else
     /* note: this version may generate warnings for unused static functions */
#    define XXH_PUBLIC_API static
#  endif

   /*
    * This part deals with the special case where a unit wants to inline xxHash,
    * but "xxhash.h" has previously been included without XXH_INLINE_ALL,
    * such as part of some previously included *.h header file.
    * Without further action, the new include would just be ignored,
    * and functions would effectively _not_ be inlined (silent failure).
    * The following macros solve this situation by prefixing all inlined names,
    * avoiding naming collision with previous inclusions.
    */
   /* Before that, we unconditionally #undef all symbols,
    * in case they were already defined with XXH_NAMESPACE.
    * They will then be redefined for XXH_INLINE_ALL
    */
#  undef XXH_versionNumber
    /* XXH32 */
#  undef XXH32
#  undef XXH32_createState
#  undef XXH32_freeState
#  undef XXH32_reset
#  undef XXH32_update
#  undef XXH32_digest
#  undef XXH32_copyState
#  undef XXH32_canonicalFromHash
#  undef XXH32_hashFromCanonical
    /* XXH64 */
#  undef XXH64
#  undef XXH64_createState
#  undef XXH64_freeState
#  undef XXH64_reset
#  undef XXH64_update
#  undef XXH64_digest
#  undef XXH64_copyState
#  undef XXH64_canonicalFromHash
#  undef XXH64_hashFromCanonical
    /* XXH3_64bits */
#  undef XXH3_64bits
#  undef XXH3_64bits_withSecret
#  undef XXH3_64bits_withSeed
#  undef XXH3_64bits_withSecretandSeed
#  undef XXH3_createState
#  undef XXH3_freeState
#  undef XXH3_copyState
#  undef XXH3_64bits_reset
#  undef XXH3_64bits_reset_withSeed
#  undef XXH3_64bits_reset_withSecret
#  undef XXH3_64bits_update
#  undef XXH3_64bits_digest
#  undef XXH3_generateSecret
    /* XXH3_128bits */
#  undef XXH128
#  undef XXH3_128bits
#  undef XXH3_128bits_withSeed
#  undef XXH3_128bits_withSecret
#  undef XXH3_128bits_reset
#  undef XXH3_128bits_reset_withSeed
#  undef XXH3_128bits_reset_withSecret
#  undef XXH3_128bits_reset_withSecretandSeed
#  undef XXH3_128bits_update
#  undef XXH3_128bits_digest
#  undef XXH128_isEqual
#  undef XXH128_cmp
#  undef XXH128_canonicalFromHash
#  undef XXH128_hashFromCanonical
    /* Finally, free the namespace itself */
#  undef XXH_NAMESPACE

    /* employ the namespace for XXH_INLINE_ALL */
#  define XXH_NAMESPACE XXH_INLINE_
   /*
    * Some identifiers (enums, type names) are not symbols,
    * but they must nonetheless be renamed to avoid redeclaration.
    * Alternative solution: do not redeclare them.
    * However, this requires some #ifdefs, and has a more dispersed impact.
    * Meanwhile, renaming can be achieved in a single place.
    */
#  define XXH_IPREF(Id)   XXH_NAMESPACE ## Id
#  define XXH_OK XXH_IPREF(XXH_OK)
#  define XXH_ERROR XXH_IPREF(XXH_ERROR)
#  define XXH_errorcode XXH_IPREF(XXH_errorcode)
#  define XXH32_canonical_t  XXH_IPREF(XXH32_canonical_t)
#  define XXH64_canonical_t  XXH_IPREF(XXH64_canonical_t)
#  define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t)
#  define XXH32_state_s XXH_IPREF(XXH32_state_s)
#  define XXH32_state_t XXH_IPREF(XXH32_state_t)
#  define XXH64_state_s XXH_IPREF(XXH64_state_s)
#  define XXH64_state_t XXH_IPREF(XXH64_state_t)
#  define XXH3_state_s  XXH_IPREF(XXH3_state_s)
#  define XXH3_state_t  XXH_IPREF(XXH3_state_t)
#  define XXH128_hash_t XXH_IPREF(XXH128_hash_t)
   /* Ensure the header is parsed again, even if it was previously included */
#  undef XXHASH_H_5627135585666179
#  undef XXHASH_H_STATIC_13879238742
#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */

/* ****************************************************************
 *  Stable API
 *****************************************************************/
#ifndef XXHASH_H_5627135585666179
#define XXHASH_H_5627135585666179 1

/*! @brief Marks a global symbol. */
#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API)
#  if defined(_WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT))
#    ifdef XXH_EXPORT
#      define XXH_PUBLIC_API __declspec(dllexport)
#    elif XXH_IMPORT
#      define XXH_PUBLIC_API __declspec(dllimport)
#    endif
#  else
#    define XXH_PUBLIC_API   /* do nothing */
#  endif
#endif

#ifdef XXH_NAMESPACE
#  define XXH_CAT(A,B) A##B
#  define XXH_NAME2(A,B) XXH_CAT(A,B)
#  define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
/* XXH32 */
#  define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
#  define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
#  define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
#  define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
#  define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
#  define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
#  define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
#  define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
#  define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
/* XXH64 */
#  define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
#  define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
#  define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
#  define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
#  define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
#  define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
#  define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
#  define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
#  define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
/* XXH3_64bits */
#  define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits)
#  define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret)
#  define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed)
#  define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed)
#  define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState)
#  define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState)
#  define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState)
#  define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset)
#  define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed)
#  define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret)
#  define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed)
#  define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update)
#  define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest)
#  define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret)
#  define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed)
/* XXH3_128bits */
#  define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128)
#  define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits)
#  define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed)
#  define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret)
#  define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed)
#  define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset)
#  define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed)
#  define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret)
#  define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed)
#  define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update)
#  define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest)
#  define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual)
#  define XXH128_cmp     XXH_NAME2(XXH_NAMESPACE, XXH128_cmp)
#  define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash)
#  define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical)
#endif


/* *************************************
*  Compiler specifics
***************************************/

/* specific declaration modes for Windows */
#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API)
#  if defined(_WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT))
#    ifdef XXH_EXPORT
#      define XXH_PUBLIC_API __declspec(dllexport)
#    elif XXH_IMPORT
#      define XXH_PUBLIC_API __declspec(dllimport)
#    endif
#  else
#    define XXH_PUBLIC_API   /* do nothing */
#  endif
#endif

#if defined (__GNUC__)
# define XXH_CONSTF  __attribute__((__const__))
# define XXH_PUREF   __attribute__((__pure__))
# define XXH_MALLOCF __attribute__((__malloc__))
#else
# define XXH_CONSTF  /* disable */
# define XXH_PUREF
# define XXH_MALLOCF
#endif

/* *************************************
*  Version
***************************************/
#define XXH_VERSION_MAJOR    0
#define XXH_VERSION_MINOR    8
#define XXH_VERSION_RELEASE  3
/*! @brief Version number, encoded as two digits each */
#define XXH_VERSION_NUMBER  (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)

/*!
 * @brief Obtains the xxHash version.
 *
 * This is mostly useful when xxHash is compiled as a shared library,
 * since the returned value comes from the library, as opposed to header file.
 *
 * @return @ref XXH_VERSION_NUMBER of the invoked library.
 */
XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber (void);


/* ****************************
*  Common basic types
******************************/
#include <stddef.h>   /* size_t */
/*!
 * @brief Exit code for the streaming API.
 */
typedef enum {
    XXH_OK = 0, /*!< OK */
    XXH_ERROR   /*!< Error */
} XXH_errorcode;


/*-**********************************************************************
*  32-bit hash
************************************************************************/
#if defined(XXH_DOXYGEN) /* Don't show <stdint.h> include */
/*!
 * @brief An unsigned 32-bit integer.
 *
 * Not necessarily defined to `uint32_t` but functionally equivalent.
 */
typedef uint32_t XXH32_hash_t;

#elif !defined (__VMS) \
  && (defined (__cplusplus) \
  || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
#   ifdef _AIX
#     include <inttypes.h>
#   else
#     include <stdint.h>
#   endif
    typedef uint32_t XXH32_hash_t;

#else
#   include <limits.h>
#   if UINT_MAX == 0xFFFFFFFFUL
      typedef unsigned int XXH32_hash_t;
#   elif ULONG_MAX == 0xFFFFFFFFUL
      typedef unsigned long XXH32_hash_t;
#   else
#     error "unsupported platform: need a 32-bit type"
#   endif
#endif

/*!
 * @}
 *
 * @defgroup XXH32_family XXH32 family
 * @ingroup public
 * Contains functions used in the classic 32-bit xxHash algorithm.
 *
 * @note
 *   XXH32 is useful for older platforms, with no or poor 64-bit performance.
 *   Note that the @ref XXH3_family provides competitive speed for both 32-bit
 *   and 64-bit systems, and offers true 64/128 bit hash results.
 *
 * @see @ref XXH64_family, @ref XXH3_family : Other xxHash families
 * @see @ref XXH32_impl for implementation details
 * @{
 */

/*!
 * @brief Calculates the 32-bit hash of @p input using xxHash32.
 *
 * @param input The block of data to be hashed, at least @p length bytes in size.
 * @param length The length of @p input, in bytes.
 * @param seed The 32-bit seed to alter the hash's output predictably.
 *
 * @pre
 *   The memory between @p input and @p input + @p length must be valid,
 *   readable, contiguous memory. However, if @p length is `0`, @p input may be
 *   `NULL`. In C++, this also must be *TriviallyCopyable*.
 *
 * @return The calculated 32-bit xxHash32 value.
 *
 * @see @ref single_shot_example "Single Shot Example" for an example.
 */
XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed);

#ifndef XXH_NO_STREAM
/*!
 * @typedef struct XXH32_state_s XXH32_state_t
 * @brief The opaque state struct for the XXH32 streaming API.
 *
 * @see XXH32_state_s for details.
 * @see @ref streaming_example "Streaming Example"
 */
typedef struct XXH32_state_s XXH32_state_t;

/*!
 * @brief Allocates an @ref XXH32_state_t.
 *
 * @return An allocated pointer of @ref XXH32_state_t on success.
 * @return `NULL` on failure.
 *
 * @note Must be freed with XXH32_freeState().
 *
 * @see @ref streaming_example "Streaming Example"
 */
XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void);
/*!
 * @brief Frees an @ref XXH32_state_t.
 *
 * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState().
 *
 * @return @ref XXH_OK.
 *
 * @note @p statePtr must be allocated with XXH32_createState().
 *
 * @see @ref streaming_example "Streaming Example"
 *
 */
XXH_PUBLIC_API XXH_errorcode  XXH32_freeState(XXH32_state_t* statePtr);
/*!
 * @brief Copies one @ref XXH32_state_t to another.
 *
 * @param dst_state The state to copy to.
 * @param src_state The state to copy from.
 * @pre
 *   @p dst_state and @p src_state must not be `NULL` and must not overlap.
 */
XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state);

/*!
 * @brief Resets an @ref XXH32_state_t to begin a new hash.
 *
 * @param statePtr The state struct to reset.
 * @param seed The 32-bit seed to alter the hash result predictably.
 *
 * @pre
 *   @p statePtr must not be `NULL`.
 *
 * @return @ref XXH_OK on success.
 * @return @ref XXH_ERROR on failure.
 *
 * @note This function resets and seeds a state. Call it before @ref XXH32_update().
 *
 * @see @ref streaming_example "Streaming Example"
 */
XXH_PUBLIC_API XXH_errorcode XXH32_reset  (XXH32_state_t* statePtr, XXH32_hash_t seed);

/*!
 * @brief Consumes a block of @p input to an @ref XXH32_state_t.
 *
 * @param statePtr The state struct to update.
 * @param input The block of data to be hashed, at least @p length bytes in size.
 * @param length The length of @p input, in bytes.
 *
 * @pre
 *   @p statePtr must not be `NULL`.
 * @pre
 *   The memory between @p input and @p input + @p length must be valid,
 *   readable, contiguous memory. However, if @p length is `0`, @p input may be
 *   `NULL`. In C++, this also must be *TriviallyCopyable*.
 *
 * @return @ref XXH_OK on success.
 * @return @ref XXH_ERROR on failure.
 *
 * @note Call this to incrementally consume blocks of data.
 *
 * @see @ref streaming_example "Streaming Example"
 */
XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);

/*!
 * @brief Returns the calculated hash value from an @ref XXH32_state_t.
 *
 * @param statePtr The state struct to calculate the hash from.
 *
 * @pre
 *  @p statePtr must not be `NULL`.
 *
 * @return The calculated 32-bit xxHash32 value from that state.
 *
 * @note
 *   Calling XXH32_digest() will not affect @p statePtr, so you can update,
 *   digest, and update again.
 *
 * @see @ref streaming_example "Streaming Example"
 */
XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr);
#endif /* !XXH_NO_STREAM */

/*******   Canonical representation   *******/

/*!
 * @brief Canonical (big endian) representation of @ref XXH32_hash_t.
 */
typedef struct {
    unsigned char digest[4]; /*!< Hash bytes, big endian */
} XXH32_canonical_t;

/*!
 * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t.
 *
 * @param dst  The @ref XXH32_canonical_t pointer to be stored to.
 * @param hash The @ref XXH32_hash_t to be converted.
 *
 * @pre
 *   @p dst must not be `NULL`.
 *
 * @see @ref canonical_representation_example "Canonical Representation Example"
 */
XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);

/*!
 * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t.
 *
 * @param src The @ref XXH32_canonical_t to convert.
 *
 * @pre
 *   @p src must not be `NULL`.
 *
 * @return The converted hash.
 *
 * @see @ref canonical_representation_example "Canonical Representation Example"
 */
XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);


/*! @cond Doxygen ignores this part */
#ifdef __has_attribute
# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x)
#else
# define XXH_HAS_ATTRIBUTE(x) 0
#endif
/*! @endcond */

/*! @cond Doxygen ignores this part */
/* C-language Attributes are added in C23. */
#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 202311L) && defined(__has_c_attribute)
# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x)
#else
# define XXH_HAS_C_ATTRIBUTE(x) 0
#endif
/*! @endcond */

/*! @cond Doxygen ignores this part */
#if defined(__cplusplus) && defined(__has_cpp_attribute)
# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
#else
# define XXH_HAS_CPP_ATTRIBUTE(x) 0
#endif
/*! @endcond */

/*! @cond Doxygen ignores this part */
/*
 * Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute
 * introduced in CPP17 and C23.
 * CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough
 * C23   : https://en.cppreference.com/w/c/language/attributes/fallthrough
 */
#if XXH_HAS_C_ATTRIBUTE(fallthrough) || XXH_HAS_CPP_ATTRIBUTE(fallthrough)
# define XXH_FALLTHROUGH [[fallthrough]]
#elif XXH_HAS_ATTRIBUTE(__fallthrough__)
# define XXH_FALLTHROUGH __attribute__ ((__fallthrough__))
#else
# define XXH_FALLTHROUGH /* fallthrough */
#endif
/*! @endcond */

/*! @cond Doxygen ignores this part */
/*
 * Define XXH_NOESCAPE for annotated pointers in public API.
 * https://clang.llvm.org/docs/AttributeReference.html#noescape
 * As of writing this, only supported by clang.
 */
#if XXH_HAS_ATTRIBUTE(noescape)
# define XXH_NOESCAPE __attribute__((__noescape__))
#else
# define XXH_NOESCAPE
#endif
/*! @endcond */


/*!
 * @}
 * @ingroup public
 * @{
 */

#ifndef XXH_NO_LONG_LONG
/*-**********************************************************************
*  64-bit hash
************************************************************************/
#if defined(XXH_DOXYGEN) /* don't include <stdint.h> */
/*!
 * @brief An unsigned 64-bit integer.
 *
 * Not necessarily defined to `uint64_t` but functionally equivalent.
 */
typedef uint64_t XXH64_hash_t;
#elif !defined (__VMS) \
  && (defined (__cplusplus) \
  || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
#   ifdef _AIX
#     include <inttypes.h>
#   else
#     include <stdint.h>
#   endif
   typedef uint64_t XXH64_hash_t;
#else
#  include <limits.h>
#  if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL
     /* LP64 ABI says uint64_t is unsigned long */
     typedef unsigned long XXH64_hash_t;
#  else
     /* the following type must have a width of 64-bit */
     typedef unsigned long long XXH64_hash_t;
#  endif
#endif

/*!
 * @}
 *
 * @defgroup XXH64_family XXH64 family
 * @ingroup public
 * @{
 * Contains functions used in the classic 64-bit xxHash algorithm.
 *
 * @note
 *   XXH3 provides competitive speed for both 32-bit and 64-bit systems,
 *   and offers true 64/128 bit hash results.
 *   It provides better speed for systems with vector processing capabilities.
 */

/*!
 * @brief Calculates the 64-bit hash of @p input using xxHash64.
 *
 * @param input The block of data to be hashed, at least @p length bytes in size.
 * @param length The length of @p input, in bytes.
 * @param seed The 64-bit seed to alter the hash's output predictably.
 *
 * @pre
 *   The memory between @p input and @p input + @p length must be valid,
 *   readable, contiguous memory. However, if @p length is `0`, @p input may be
 *   `NULL`. In C++, this also must be *TriviallyCopyable*.
 *
 * @return The calculated 64-bit xxHash64 value.
 *
 * @see @ref single_shot_example "Single Shot Example" for an example.
 */
XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed);

/*******   Streaming   *******/
#ifndef XXH_NO_STREAM
/*!
 * @brief The opaque state struct for the XXH64 streaming API.
 *
 * @see XXH64_state_s for details.
 * @see @ref streaming_example "Streaming Example"
 */
typedef struct XXH64_state_s XXH64_state_t;   /* incomplete type */

/*!
 * @brief Allocates an @ref XXH64_state_t.
 *
 * @return An allocated pointer of @ref XXH64_state_t on success.
 * @return `NULL` on failure.
 *
 * @note Must be freed with XXH64_freeState().
 *
 * @see @ref streaming_example "Streaming Example"
 */
XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void);

/*!
 * @brief Frees an @ref XXH64_state_t.
 *
 * @param statePtr A pointer to an @ref XXH64_state_t allocated with @ref XXH64_createState().
 *
 * @return @ref XXH_OK.
 *
 * @note @p statePtr must be allocated with XXH64_createState().
 *
 * @see @ref streaming_example "Streaming Example"
 */
XXH_PUBLIC_API XXH_errorcode  XXH64_freeState(XXH64_state_t* statePtr);

/*!
 * @brief Copies one @ref XXH64_state_t to another.
 *
 * @param dst_state The state to copy to.
 * @param src_state The state to copy from.
 * @pre
 *   @p dst_state and @p src_state must not be `NULL` and must not overlap.
 */
XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dst_state, const XXH64_state_t* src_state);

/*!
 * @brief Resets an @ref XXH64_state_t to begin a new hash.
 *
 * @param statePtr The state struct to reset.
 * @param seed The 64-bit seed to alter the hash result predictably.
 *
 * @pre
 *   @p statePtr must not be `NULL`.
 *
 * @return @ref XXH_OK on success.
 * @return @ref XXH_ERROR on failure.
 *
 * @note This function resets and seeds a state. Call it before @ref XXH64_update().
 *
 * @see @ref streaming_example "Streaming Example"
 */
XXH_PUBLIC_API XXH_errorcode XXH64_reset  (XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed);

/*!
 * @brief Consumes a block of @p input to an @ref XXH64_state_t.
 *
 * @param statePtr The state struct to update.
 * @param input The block of data to be hashed, at least @p length bytes in size.
 * @param length The length of @p input, in bytes.
 *
 * @pre
 *   @p statePtr must not be `NULL`.
 * @pre
 *   The memory between @p input and @p input + @p length must be valid,
 *   readable, contiguous memory. However, if @p length is `0`, @p input may be
 *   `NULL`. In C++, this also must be *TriviallyCopyable*.
 *
 * @return @ref XXH_OK on success.
 * @return @ref XXH_ERROR on failure.
 *
 * @note Call this to incrementally consume blocks of data.
 *
 * @see @ref streaming_example "Streaming Example"
 */
XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length);

/*!
 * @brief Returns the calculated hash value from an @ref XXH64_state_t.
 *
 * @param statePtr The state struct to calculate the hash from.
 *
 * @pre
 *  @p statePtr must not be `NULL`.
 *
 * @return The calculated 64-bit xxHash64 value from that state.
 *
 * @note
 *   Calling XXH64_digest() will not affect @p statePtr, so you can update,
 *   digest, and update again.
 *
 * @see @ref streaming_example "Streaming Example"
 */
XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_digest (XXH_NOESCAPE const XXH64_state_t* statePtr);
#endif /* !XXH_NO_STREAM */
/*******   Canonical representation   *******/

/*!
 * @brief Canonical (big endian) representation of @ref XXH64_hash_t.
 */
typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t;

/*!
 * @brief Converts an @ref XXH64_hash_t to a big endian @ref XXH64_canonical_t.
 *
 * @param dst The @ref XXH64_canonical_t pointer to be stored to.
 * @param hash The @ref XXH64_hash_t to be converted.
 *
 * @pre
 *   @p dst must not be `NULL`.
 *
 * @see @ref canonical_representation_example "Canonical Representation Example"
 */
XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash);

/*!
 * @brief Converts an @ref XXH64_canonical_t to a native @ref XXH64_hash_t.
 *
 * @param src The @ref XXH64_canonical_t to convert.
 *
 * @pre
 *   @p src must not be `NULL`.
 *
 * @return The converted hash.
 *
 * @see @ref canonical_representation_example "Canonical Representation Example"
 */
XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src);

#ifndef XXH_NO_XXH3

/*!
 * @}
 * ************************************************************************
 * @defgroup XXH3_family XXH3 family
 * @ingroup public
 * @{
 *
 * XXH3 is a more recent hash algorithm featuring:
 *  - Improved speed for both small and large inputs
 *  - True 64-bit and 128-bit outputs
 *  - SIMD acceleration
 *  - Improved 32-bit viability
 *
 * Speed analysis methodology is explained here:
 *
 *    https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html
 *
 * Compared to XXH64, expect XXH3 to run approximately
 * ~2x faster on large inputs and >3x faster on small ones,
 * exact differences vary depending on platform.
 *
 * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic,
 * but does not require it.
 * Most 32-bit and 64-bit targets that can run XXH32 smoothly can run XXH3
 * at competitive speeds, even without vector support. Further details are
 * explained in the implementation.
 *
 * XXH3 has a fast scalar implementation, but it also includes accelerated SIMD
 * implementations for many common platforms:
 *   - AVX512
 *   - AVX2
 *   - SSE2
 *   - ARM NEON
 *   - WebAssembly SIMD128
 *   - POWER8 VSX
 *   - s390x ZVector
 * This can be controlled via the @ref XXH_VECTOR macro, but it automatically
 * selects the best version according to predefined macros. For the x86 family, an
 * automatic runtime dispatcher is included separately in @ref xxh_x86dispatch.c.
 *
 * XXH3 implementation is portable:
 * it has a generic C90 formulation that can be compiled on any platform,
 * all implementations generate exactly the same hash value on all platforms.
 * Starting from v0.8.0, it's also labelled "stable", meaning that
 * any future version will also generate the same hash value.
 *
 * XXH3 offers 2 variants, _64bits and _128bits.
 *
 * When only 64 bits are needed, prefer invoking the _64bits variant, as it
 * reduces the amount of mixing, resulting in faster speed on small inputs.
 * It's also generally simpler to manipulate a scalar return type than a struct.
 *
 * The API supports one-shot hashing, streaming mode, and custom secrets.
 */

/*!
 * @ingroup tuning
 * @brief Possible values for @ref XXH_VECTOR.
 *
 * Unless set explicitly, determined automatically.
 */
#  define XXH_SCALAR 0 /*!< Portable scalar version */
#  define XXH_SSE2   1 /*!< SSE2 for Pentium 4, Opteron, all x86_64. */
#  define XXH_AVX2   2 /*!< AVX2 for Haswell and Bulldozer */
#  define XXH_AVX512 3 /*!< AVX512 for Skylake and Icelake */
#  define XXH_NEON   4 /*!< NEON for most ARMv7-A, all AArch64, and WASM SIMD128 */
#  define XXH_VSX    5 /*!< VSX and ZVector for POWER8/z13 (64-bit) */
#  define XXH_SVE    6 /*!< SVE for some ARMv8-A and ARMv9-A */
#  define XXH_LSX    7 /*!< LSX (128-bit SIMD) for LoongArch64 */
#  define XXH_LASX   8 /*!< LASX (256-bit SIMD) for LoongArch64 */
#  define XXH_RVV    9 /*!< RVV (RISC-V Vector) for RISC-V */

/*-**********************************************************************
*  XXH3 64-bit variant
************************************************************************/

/*!
 * @brief Calculates 64-bit unseeded variant of XXH3 hash of @p input.
 *
 * @param input  The block of data to be hashed, at least @p length bytes in size.
 * @param length The length of @p input, in bytes.
 *
 * @pre
 *   The memory between @p input and @p input + @p length must be valid,
 *   readable, contiguous memory. However, if @p length is `0`, @p input may be
 *   `NULL`. In C++, this also must be *TriviallyCopyable*.
 *
 * @return The calculated 64-bit XXH3 hash value.
 *
 * @note
 *   This is equivalent to @ref XXH3_64bits_withSeed() with a seed of `0`, however
 *   it may have slightly better performance due to constant propagation of the
 *   defaults.
 *
 * @see
 *    XXH3_64bits_withSeed(), XXH3_64bits_withSecret(): other seeding variants
 * @see @ref single_shot_example "Single Shot Example" for an example.
 */
XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length);

/*!
 * @brief Calculates 64-bit seeded variant of XXH3 hash of @p input.
 *
 * @param input  The block of data to be hashed, at least @p length bytes in size.
 * @param length The length of @p input, in bytes.
 * @param seed   The 64-bit seed to alter the hash result predictably.
 *
 * @pre
 *   The memory between @p input and @p input + @p length must be valid,
 *   readable, contiguous memory. However, if @p length is `0`, @p input may be
 *   `NULL`. In C++, this also must be *TriviallyCopyable*.
 *
 * @return The calculated 64-bit XXH3 hash value.
 *
 * @note
 *    seed == 0 produces the same results as @ref XXH3_64bits().
 *
 * This variant generates a custom secret on the fly based on default secret
 * altered using the @p seed value.
 *
 * While this operation is decently fast, note that it's not completely free.
 *
 * @see @ref single_shot_example "Single Shot Example" for an example.
 */
XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed);

/*!
 * The bare minimum size for a custom secret.
 *
 * @see
 *  XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(),
 *  XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret().
 */
#define XXH3_SECRET_SIZE_MIN 136

/*!
 * @brief Calculates 64-bit variant of XXH3 with a custom "secret".
 *
 * @param data       The block of data to be hashed, at least @p len bytes in size.
 * @param len        The length of @p data, in bytes.
 * @param secret     The secret data.
 * @param secretSize The length of @p secret, in bytes.
 *
 * @return The calculated 64-bit XXH3 hash value.
 *
 * @pre
 *   The memory between @p data and @p data + @p len must be valid,
 *   readable, contiguous memory. However, if @p length is `0`, @p data may be
 *   `NULL`. In C++, this also must be *TriviallyCopyable*.
 *
 * It's possible to provide any blob of bytes as a "secret" to generate the hash.
 * This makes it more difficult for an external actor to prepare an intentional collision.
 * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN).
 * However, the quality of the secret impacts the dispersion of the hash algorithm.
 * Therefore, the secret _must_ look like a bunch of random bytes.
 * Avoid "trivial" or structured data such as repeated sequences or a text document.
 * Whenever in doubt about the "randomness" of the blob of bytes,
 * consider employing @ref XXH3_generateSecret() instead (see below).
 * It will generate a proper high entropy secret derived from the blob of bytes.
 * Another advantage of using XXH3_generateSecret() is that
 * it guarantees that all bits within the initial blob of bytes
 * will impact every bit of the output.
 * This is not necessarily the case when using the blob of bytes directly
 * because, when hashing _small_ inputs, only a portion of the secret is employed.
 *
 * @see @ref single_shot_example "Single Shot Example" for an example.
 */
XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize);


/*******   Streaming   *******/
#ifndef XXH_NO_STREAM
/*
 * Streaming requires state maintenance.
 * This operation costs memory and CPU.
 * As a consequence, streaming is slower than one-shot hashing.
 * For better performance, prefer one-shot functions whenever applicable.
 */

/*!
 * @brief The opaque state struct for the XXH3 streaming API.
 *
 * @see XXH3_state_s for details.
 * @see @ref streaming_example "Streaming Example"
 */
typedef struct XXH3_state_s XXH3_state_t;
XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t* XXH3_createState(void);
XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr);

/*!
 * @brief Copies one @ref XXH3_state_t to another.
 *
 * @param dst_state The state to copy to.
 * @param src_state The state to copy from.
 * @pre
 *   @p dst_state and @p src_state must not be `NULL` and must not overlap.
 */
XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state);

/*!
 * @brief Resets an @ref XXH3_state_t to begin a new hash.
 *
 * @param statePtr The state struct to reset.
 *
 * @pre
 *   @p statePtr must not be `NULL`.
 *
 * @return @ref XXH_OK on success.
 * @return @ref XXH_ERROR on failure.
 *
 * @note
 *   - This function resets `statePtr` and generate a secret with default parameters.
 *   - Call this function before @ref XXH3_64bits_update().
 *   - Digest will be equivalent to `XXH3_64bits()`.
 *
 * @see @ref streaming_example "Streaming Example"
 *
 */
XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr);

/*!
 * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash.
 *
 * @param statePtr The state struct to reset.
 * @param seed     The 64-bit seed to alter the hash result predictably.
 *
 * @pre
 *   @p statePtr must not be `NULL`.
 *
 * @return @ref XXH_OK on success.
 * @return @ref XXH_ERROR on failure.
 *
 * @note
 *   - This function resets `statePtr` and generate a secret from `seed`.
 *   - Call this function before @ref XXH3_64bits_update().
 *   - Digest will be equivalent to `XXH3_64bits_withSeed()`.
 *
 * @see @ref streaming_example "Streaming Example"
 *
 */
XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed);

/*!
 * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash.
 *
 * @param statePtr The state struct to reset.
 * @param secret     The secret data.
 * @param secretSize The length of @p secret, in bytes.
 *
 * @pre
 *   @p statePtr must not be `NULL`.
 *
 * @return @ref XXH_OK on success.
 * @return @ref XXH_ERROR on failure.
 *
 * @note
 *   `secret` is referenced, it _must outlive_ the hash streaming session.
 *
 * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN,
 * and the quality of produced hash values depends on secret's entropy
 * (secret's content should look like a bunch of random bytes).
 * When in doubt about the randomness of a candidate `secret`,
 * consider employing `XXH3_generateSecret()` instead (see below).
 *
 * @see @ref streaming_example "Streaming Example"
 */
XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize);

/*!
 * @brief Consumes a block of @p input to an @ref XXH3_state_t.
 *
 * @param statePtr The state struct to update.
 * @param input The block of data to be hashed, at least @p length bytes in size.
 * @param length The length of @p input, in bytes.
 *
 * @pre
 *   @p statePtr must not be `NULL`.
 * @pre
 *   The memory between @p input and @p input + @p length must be valid,
 *   readable, contiguous memory. However, if @p length is `0`, @p input may be
 *   `NULL`. In C++, this also must be *TriviallyCopyable*.
 *
 * @return @ref XXH_OK on success.
 * @return @ref XXH_ERROR on failure.
 *
 * @note Call this to incrementally consume blocks of data.
 *
 * @see @ref streaming_example "Streaming Example"
 */
XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length);

/*!
 * @brief Returns the calculated XXH3 64-bit hash value from an @ref XXH3_state_t.
 *
 * @param statePtr The state struct to calculate the hash from.
 *
 * @pre
 *  @p statePtr must not be `NULL`.
 *
 * @return The calculated XXH3 64-bit hash value from that state.
 *
 * @note
 *   Calling XXH3_64bits_digest() will not affect @p statePtr, so you can update,
 *   digest, and update again.
 *
 * @see @ref streaming_example "Streaming Example"
 */
XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr);
#endif /* !XXH_NO_STREAM */

/* note : canonical representation of XXH3 is the same as XXH64
 * since they both produce XXH64_hash_t values */


/*-**********************************************************************
*  XXH3 128-bit variant
************************************************************************/

/*!
 * @brief The return value from 128-bit hashes.
 *
 * Stored in little endian order, although the fields themselves are in native
 * endianness.
 */
typedef struct {
    XXH64_hash_t low64;   /*!< `value & 0xFFFFFFFFFFFFFFFF` */
    XXH64_hash_t high64;  /*!< `value >> 64` */
} XXH128_hash_t;

/*!
 * @brief Calculates 128-bit unseeded variant of XXH3 of @p data.
 *
 * @param data The block of data to be hashed, at least @p length bytes in size.
 * @param len  The length of @p data, in bytes.
 *
 * @return The calculated 128-bit variant of XXH3 value.
 *
 * The 128-bit variant of XXH3 has more strength, but it has a bit of overhead
 * for shorter inputs.
 *
 * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of `0`, however
 * it may have slightly better performance due to constant propagation of the
 * defaults.
 *
 * @see XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding variants
 * @see @ref single_shot_example "Single Shot Example" for an example.
 */
XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* data, size_t len);
/*! @brief Calculates 128-bit seeded variant of XXH3 hash of @p data.
 *
 * @param data The block of data to be hashed, at least @p length bytes in size.
 * @param len  The length of @p data, in bytes.
 * @param seed The 64-bit seed to alter the hash result predictably.
 *
 * @return The calculated 128-bit variant of XXH3 value.
 *
 * @note
 *    seed == 0 produces the same results as @ref XXH3_64bits().
 *
 * This variant generates a custom secret on the fly based on default secret
 * altered using the @p seed value.
 *
 * While this operation is decently fast, note that it's not completely free.
 *
 * @see XXH3_128bits(), XXH3_128bits_withSecret(): other seeding variants
 * @see @ref single_shot_example "Single Shot Example" for an example.
 */
XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSeed(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed);
/*!
 * @brief Calculates 128-bit variant of XXH3 with a custom "secret".
 *
 * @param data       The block of data to be hashed, at least @p len bytes in size.
 * @param len        The length of @p data, in bytes.
 * @param secret     The secret data.
 * @param secretSize The length of @p secret, in bytes.
 *
 * @return The calculated 128-bit variant of XXH3 value.
 *
 * It's possible to provide any blob of bytes as a "secret" to generate the hash.
 * This makes it more difficult for an external actor to prepare an intentional collision.
 * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN).
 * However, the quality of the secret impacts the dispersion of the hash algorithm.
 * Therefore, the secret _must_ look like a bunch of random bytes.
 * Avoid "trivial" or structured data such as repeated sequences or a text document.
 * Whenever in doubt about the "randomness" of the blob of bytes,
 * consider employing @ref XXH3_generateSecret() instead (see below).
 * It will generate a proper high entropy secret derived from the blob of bytes.
 * Another advantage of using XXH3_generateSecret() is that
 * it guarantees that all bits within the initial blob of bytes
 * will impact every bit of the output.
 * This is not necessarily the case when using the blob of bytes directly
 * because, when hashing _small_ inputs, only a portion of the secret is employed.
 *
 * @see @ref single_shot_example "Single Shot Example" for an example.
 */
XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize);

/*******   Streaming   *******/
#ifndef XXH_NO_STREAM
/*
 * Streaming requires state maintenance.
 * This operation costs memory and CPU.
 * As a consequence, streaming is slower than one-shot hashing.
 * For better performance, prefer one-shot functions whenever applicable.
 *
 * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits().
 * Use already declared XXH3_createState() and XXH3_freeState().
 *
 * All reset and streaming functions have same meaning as their 64-bit counterpart.
 */

/*!
 * @brief Resets an @ref XXH3_state_t to begin a new hash.
 *
 * @param statePtr The state struct to reset.
 *
 * @pre
 *   @p statePtr must not be `NULL`.
 *
 * @return @ref XXH_OK on success.
 * @return @ref XXH_ERROR on failure.
 *
 * @note
 *   - This function resets `statePtr` and generate a secret with default parameters.
 *   - Call it before @ref XXH3_128bits_update().
 *   - Digest will be equivalent to `XXH3_128bits()`.
 *
 * @see @ref streaming_example "Streaming Example"
 */
XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr);

/*!
 * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash.
 *
 * @param statePtr The state struct to reset.
 * @param seed     The 64-bit seed to alter the hash result predictably.
 *
 * @pre
 *   @p statePtr must not be `NULL`.
 *
 * @return @ref XXH_OK on success.
 * @return @ref XXH_ERROR on failure.
 *
 * @note
 *   - This function resets `statePtr` and generate a secret from `seed`.
 *   - Call it before @ref XXH3_128bits_update().
 *   - Digest will be equivalent to `XXH3_128bits_withSeed()`.
 *
 * @see @ref streaming_example "Streaming Example"
 */
XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed);
/*!
 * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash.
 *
 * @param statePtr   The state struct to reset.
 * @param secret     The secret data.
 * @param secretSize The length of @p secret, in bytes.
 *
 * @pre
 *   @p statePtr must not be `NULL`.
 *
 * @return @ref XXH_OK on success.
 * @return @ref XXH_ERROR on failure.
 *
 * `secret` is referenced, it _must outlive_ the hash streaming session.
 * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN,
 * and the quality of produced hash values depends on secret's entropy
 * (secret's content should look like a bunch of random bytes).
 * When in doubt about the randomness of a candidate `secret`,
 * consider employing `XXH3_generateSecret()` instead (see below).
 *
 * @see @ref streaming_example "Streaming Example"
 */
XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize);

/*!
 * @brief Consumes a block of @p input to an @ref XXH3_state_t.
 *
 * Call this to incrementally consume blocks of data.
 *
 * @param statePtr The state struct to update.
 * @param input The block of data to be hashed, at least @p length bytes in size.
 * @param length The length of @p input, in bytes.
 *
 * @pre
 *   @p statePtr must not be `NULL`.
 *
 * @return @ref XXH_OK on success.
 * @return @ref XXH_ERROR on failure.
 *
 * @note
 *   The memory between @p input and @p input + @p length must be valid,
 *   readable, contiguous memory. However, if @p length is `0`, @p input may be
 *   `NULL`. In C++, this also must be *TriviallyCopyable*.
 *
 */
XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length);

/*!
 * @brief Returns the calculated XXH3 128-bit hash value from an @ref XXH3_state_t.
 *
 * @param statePtr The state struct to calculate the hash from.
 *
 * @pre
 *  @p statePtr must not be `NULL`.
 *
 * @return The calculated XXH3 128-bit hash value from that state.
 *
 * @note
 *   Calling XXH3_128bits_digest() will not affect @p statePtr, so you can update,
 *   digest, and update again.
 *
 */
XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr);
#endif /* !XXH_NO_STREAM */

/* Following helper functions make it possible to compare XXH128_hast_t values.
 * Since XXH128_hash_t is a structure, this capability is not offered by the language.
 * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */

/*!
 * @brief Check equality of two XXH128_hash_t values
 *
 * @param h1 The 128-bit hash value.
 * @param h2 Another 128-bit hash value.
 *
 * @return `1` if `h1` and `h2` are equal.
 * @return `0` if they are not.
 */
XXH_PUBLIC_API XXH_PUREF int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2);

/*!
 * @brief Compares two @ref XXH128_hash_t
 *
 * This comparator is compatible with stdlib's `qsort()`/`bsearch()`.
 *
 * @param h128_1 Left-hand side value
 * @param h128_2 Right-hand side value
 *
 * @return >0 if @p h128_1  > @p h128_2
 * @return =0 if @p h128_1 == @p h128_2
 * @return <0 if @p h128_1  < @p h128_2
 */
XXH_PUBLIC_API XXH_PUREF int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2);


/*******   Canonical representation   *******/
typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t;


/*!
 * @brief Converts an @ref XXH128_hash_t to a big endian @ref XXH128_canonical_t.
 *
 * @param dst  The @ref XXH128_canonical_t pointer to be stored to.
 * @param hash The @ref XXH128_hash_t to be converted.
 *
 * @pre
 *   @p dst must not be `NULL`.
 * @see @ref canonical_representation_example "Canonical Representation Example"
 */
XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash);

/*!
 * @brief Converts an @ref XXH128_canonical_t to a native @ref XXH128_hash_t.
 *
 * @param src The @ref XXH128_canonical_t to convert.
 *
 * @pre
 *   @p src must not be `NULL`.
 *
 * @return The converted hash.
 * @see @ref canonical_representation_example "Canonical Representation Example"
 */
XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src);


#endif  /* !XXH_NO_XXH3 */
#endif  /* XXH_NO_LONG_LONG */

/*!
 * @}
 */
#endif /* XXHASH_H_5627135585666179 */



#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742)
#define XXHASH_H_STATIC_13879238742
/* ****************************************************************************
 * This section contains declarations which are not guaranteed to remain stable.
 * They may change in future versions, becoming incompatible with a different
 * version of the library.
 * These declarations should only be used with static linking.
 * Never use them in association with dynamic linking!
 ***************************************************************************** */

/*
 * These definitions are only present to allow static allocation
 * of XXH states, on stack or in a struct, for example.
 * Never **ever** access their members directly.
 */

/*!
 * @internal
 * @brief Structure for XXH32 streaming API.
 *
 * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
 * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is
 * an opaque type. This allows fields to safely be changed.
 *
 * Typedef'd to @ref XXH32_state_t.
 * Do not access the members of this struct directly.
 * @see XXH64_state_s, XXH3_state_s
 */
struct XXH32_state_s {
   XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */
   XXH32_hash_t large_len;    /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */
   XXH32_hash_t acc[4];       /*!< Accumulator lanes */
   unsigned char buffer[16];  /*!< Internal buffer for partial reads. */
   XXH32_hash_t bufferedSize; /*!< Amount of data in @ref buffer */
   XXH32_hash_t reserved;     /*!< Reserved field. Do not read nor write to it. */
};   /* typedef'd to XXH32_state_t */


#ifndef XXH_NO_LONG_LONG  /* defined when there is no 64-bit support */

/*!
 * @internal
 * @brief Structure for XXH64 streaming API.
 *
 * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
 * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is
 * an opaque type. This allows fields to safely be changed.
 *
 * Typedef'd to @ref XXH64_state_t.
 * Do not access the members of this struct directly.
 * @see XXH32_state_s, XXH3_state_s
 */
struct XXH64_state_s {
   XXH64_hash_t total_len;    /*!< Total length hashed. This is always 64-bit. */
   XXH64_hash_t acc[4];       /*!< Accumulator lanes */
   unsigned char buffer[32];  /*!< Internal buffer for partial reads.. */
   XXH32_hash_t bufferedSize; /*!< Amount of data in @ref buffer */
   XXH32_hash_t reserved32;   /*!< Reserved field, needed for padding anyways*/
   XXH64_hash_t reserved64;   /*!< Reserved field. Do not read or write to it. */
};   /* typedef'd to XXH64_state_t */

#ifndef XXH_NO_XXH3

#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */
#  define XXH_ALIGN(n)      _Alignas(n)
#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */
/* In C++ alignas() is a keyword */
#  define XXH_ALIGN(n)      alignas(n)
#elif defined(__GNUC__)
#  define XXH_ALIGN(n)      __attribute__ ((aligned(n)))
#elif defined(_MSC_VER)
#  define XXH_ALIGN(n)      __declspec(align(n))
#else
#  define XXH_ALIGN(n)   /* disabled */
#endif

/* Old GCC versions only accept the attribute after the type in structures. */
#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L))   /* C11+ */ \
    && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \
    && defined(__GNUC__)
#   define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align)
#else
#   define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type
#endif

/*!
 * @internal
 * @brief The size of the internal XXH3 buffer.
 *
 * This is the optimal update size for incremental hashing.
 *
 * @see XXH3_64b_update(), XXH3_128b_update().
 */
#define XXH3_INTERNALBUFFER_SIZE 256

/*!
 * @def XXH3_SECRET_DEFAULT_SIZE
 * @brief Default Secret's size
 *
 * This is the size of internal XXH3_kSecret
 * and is needed by XXH3_generateSecret_fromSeed().
 *
 * Not to be confused with @ref XXH3_SECRET_SIZE_MIN.
 */
#define XXH3_SECRET_DEFAULT_SIZE 192

/*!
 * @internal
 * @brief Structure for XXH3 streaming API.
 *
 * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
 * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined.
 * Otherwise it is an opaque type.
 * Never use this definition in combination with dynamic library.
 * This allows fields to safely be changed in the future.
 *
 * @note ** This structure has a strict alignment requirement of 64 bytes!! **
 * Do not allocate this with `malloc()` or `new`,
 * it will not be sufficiently aligned.
 * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation.
 *
 * Typedef'd to @ref XXH3_state_t.
 * Do never access the members of this struct directly.
 *
 * @see XXH3_INITSTATE() for stack initialization.
 * @see XXH3_createState(), XXH3_freeState().
 * @see XXH32_state_s, XXH64_state_s
 */
struct XXH3_state_s {
   XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]);
       /*!< The 8 accumulators. See @ref XXH32_state_s::acc and @ref XXH64_state_s::acc */
   XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]);
       /*!< Used to store a custom secret generated from a seed. */
   XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]);
       /*!< The internal buffer. @see XXH32_state_s::mem32 */
   XXH32_hash_t bufferedSize;
       /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */
   XXH32_hash_t useSeed;
       /*!< Reserved field. Needed for padding on 64-bit. */
   size_t nbStripesSoFar;
       /*!< Number or stripes processed. */
   XXH64_hash_t totalLen;
       /*!< Total length hashed. 64-bit even on 32-bit targets. */
   size_t nbStripesPerBlock;
       /*!< Number of stripes per block. */
   size_t secretLimit;
       /*!< Size of @ref customSecret or @ref extSecret */
   XXH64_hash_t seed;
       /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */
   XXH64_hash_t reserved64;
       /*!< Reserved field. */
   const unsigned char* extSecret;
       /*!< Reference to an external secret for the _withSecret variants, NULL
        *   for other variants. */
   /* note: there may be some padding at the end due to alignment on 64 bytes */
}; /* typedef'd to XXH3_state_t */

#undef XXH_ALIGN_MEMBER

/*!
 * @brief Initializes a stack-allocated `XXH3_state_s`.
 *
 * When the @ref XXH3_state_t structure is merely emplaced on stack,
 * it should be initialized with XXH3_INITSTATE() or a memset()
 * in case its first reset uses XXH3_NNbits_reset_withSeed().
 * This init can be omitted if the first reset uses default or _withSecret mode.
 * This operation isn't necessary when the state is created with XXH3_createState().
 * Note that this doesn't prepare the state for a streaming operation,
 * it's still necessary to use XXH3_NNbits_reset*() afterwards.
 */
#define XXH3_INITSTATE(XXH3_state_ptr)                       \
    do {                                                     \
        XXH3_state_t* tmp_xxh3_state_ptr = (XXH3_state_ptr); \
        tmp_xxh3_state_ptr->seed = 0;                        \
        tmp_xxh3_state_ptr->extSecret = NULL;                \
    } while(0)


/*!
 * @brief Calculates the 128-bit hash of @p data using XXH3.
 *
 * @param data The block of data to be hashed, at least @p len bytes in size.
 * @param len  The length of @p data, in bytes.
 * @param seed The 64-bit seed to alter the hash's output predictably.
 *
 * @pre
 *   The memory between @p data and @p data + @p len must be valid,
 *   readable, contiguous memory. However, if @p len is `0`, @p data may be
 *   `NULL`. In C++, this also must be *TriviallyCopyable*.
 *
 * @return The calculated 128-bit XXH3 value.
 *
 * @see @ref single_shot_example "Single Shot Example" for an example.
 */
XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed);


/* ===   Experimental API   === */
/* Symbols defined below must be considered tied to a specific library version. */

/*!
 * @brief Derive a high-entropy secret from any user-defined content, named customSeed.
 *
 * @param secretBuffer    A writable buffer for derived high-entropy secret data.
 * @param secretSize      Size of secretBuffer, in bytes.  Must be >= XXH3_SECRET_SIZE_MIN.
 * @param customSeed      A user-defined content.
 * @param customSeedSize  Size of customSeed, in bytes.
 *
 * @return @ref XXH_OK on success.
 * @return @ref XXH_ERROR on failure.
 *
 * The generated secret can be used in combination with `*_withSecret()` functions.
 * The `_withSecret()` variants are useful to provide a higher level of protection
 * than 64-bit seed, as it becomes much more difficult for an external actor to
 * guess how to impact the calculation logic.
 *
 * The function accepts as input a custom seed of any length and any content,
 * and derives from it a high-entropy secret of length @p secretSize into an
 * already allocated buffer @p secretBuffer.
 *
 * The generated secret can then be used with any `*_withSecret()` variant.
 * The functions @ref XXH3_128bits_withSecret(), @ref XXH3_64bits_withSecret(),
 * @ref XXH3_128bits_reset_withSecret() and @ref XXH3_64bits_reset_withSecret()
 * are part of this list. They all accept a `secret` parameter
 * which must be large enough for implementation reasons (>= @ref XXH3_SECRET_SIZE_MIN)
 * _and_ feature very high entropy (consist of random-looking bytes).
 * These conditions can be a high bar to meet, so @ref XXH3_generateSecret() can
 * be employed to ensure proper quality.
 *
 * @p customSeed can be anything. It can have any size, even small ones,
 * and its content can be anything, even "poor entropy" sources such as a bunch
 * of zeroes. The resulting `secret` will nonetheless provide all required qualities.
 *
 * @pre
 *   - @p secretSize must be >= @ref XXH3_SECRET_SIZE_MIN
 *   - When @p customSeedSize > 0, supplying NULL as customSeed is undefined behavior.
 *
 * Example code:
 * @code{.c}
 *    #include <stdio.h>
 *    #include <stdlib.h>
 *    #include <string.h>
 *    #define XXH_STATIC_LINKING_ONLY // expose unstable API
 *    #include "xxhash.h"
 *    // Hashes argv[2] using the entropy from argv[1].
 *    int main(int argc, char* argv[])
 *    {
 *        char secret[XXH3_SECRET_SIZE_MIN];
 *        if (argv != 3) { return 1; }
 *        XXH3_generateSecret(secret, sizeof(secret), argv[1], strlen(argv[1]));
 *        XXH64_hash_t h = XXH3_64bits_withSecret(
 *             argv[2], strlen(argv[2]),
 *             secret, sizeof(secret)
 *        );
 *        printf("%016llx\n", (unsigned long long) h);
 *    }
 * @endcode
 */
XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize);

/*!
 * @brief Generate the same secret as the _withSeed() variants.
 *
 * @param secretBuffer A writable buffer of @ref XXH3_SECRET_DEFAULT_SIZE bytes
 * @param seed         The 64-bit seed to alter the hash result predictably.
 *
 * The generated secret can be used in combination with
 *`*_withSecret()` and `_withSecretandSeed()` variants.
 *
 * Example C++ `std::string` hash class:
 * @code{.cpp}
 *    #include <string>
 *    #define XXH_STATIC_LINKING_ONLY // expose unstable API
 *    #include "xxhash.h"
 *    // Slow, seeds each time
 *    class HashSlow {
 *        XXH64_hash_t seed;
 *    public:
 *        HashSlow(XXH64_hash_t s) : seed{s} {}
 *        size_t operator()(const std::string& x) const {
 *            return size_t{XXH3_64bits_withSeed(x.c_str(), x.length(), seed)};
 *        }
 *    };
 *    // Fast, caches the seeded secret for future uses.
 *    class HashFast {
 *        unsigned char secret[XXH3_SECRET_DEFAULT_SIZE];
 *    public:
 *        HashFast(XXH64_hash_t s) {
 *            XXH3_generateSecret_fromSeed(secret, seed);
 *        }
 *        size_t operator()(const std::string& x) const {
 *            return size_t{
 *                XXH3_64bits_withSecret(x.c_str(), x.length(), secret, sizeof(secret))
 *            };
 *        }
 *    };
 * @endcode
 */
XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed);

/*!
 * @brief Maximum size of "short" key in bytes.
 */
#define XXH3_MIDSIZE_MAX 240

/*!
 * @brief Calculates 64/128-bit seeded variant of XXH3 hash of @p data.
 *
 * @param data       The block of data to be hashed, at least @p len bytes in size.
 * @param len        The length of @p data, in bytes.
 * @param secret     The secret data.
 * @param secretSize The length of @p secret, in bytes.
 * @param seed       The 64-bit seed to alter the hash result predictably.
 *
 * These variants generate hash values using either:
 * - @p seed for "short" keys (< @ref XXH3_MIDSIZE_MAX = 240 bytes)
 * - @p secret for "large" keys (>= @ref XXH3_MIDSIZE_MAX).
 *
 * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`.
 * `_withSeed()` has to generate the secret on the fly for "large" keys.
 * It's fast, but can be perceptible for "not so large" keys (< 1 KB).
 * `_withSecret()` has to generate the masks on the fly for "small" keys,
 * which requires more instructions than _withSeed() variants.
 * Therefore, _withSecretandSeed variant combines the best of both worlds.
 *
 * When @p secret has been generated by XXH3_generateSecret_fromSeed(),
 * this variant produces *exactly* the same results as `_withSeed()` variant,
 * hence offering only a pure speed benefit on "large" input,
 * by skipping the need to regenerate the secret for every large input.
 *
 * Another usage scenario is to hash the secret to a 64-bit hash value,
 * for example with XXH3_64bits(), which then becomes the seed,
 * and then employ both the seed and the secret in _withSecretandSeed().
 * On top of speed, an added benefit is that each bit in the secret
 * has a 50% chance to swap each bit in the output, via its impact to the seed.
 *
 * This is not guaranteed when using the secret directly in "small data" scenarios,
 * because only portions of the secret are employed for small data.
 */
XXH_PUBLIC_API XXH_PUREF XXH64_hash_t
XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* data, size_t len,
                              XXH_NOESCAPE const void* secret, size_t secretSize,
                              XXH64_hash_t seed);

/*!
 * @brief Calculates 128-bit seeded variant of XXH3 hash of @p data.
 *
 * @param input      The memory segment to be hashed, at least @p len bytes in size.
 * @param length     The length of @p data, in bytes.
 * @param secret     The secret used to alter hash result predictably.
 * @param secretSize The length of @p secret, in bytes (must be >= XXH3_SECRET_SIZE_MIN)
 * @param seed64     The 64-bit seed to alter the hash result predictably.
 *
 * @return @ref XXH_OK on success.
 * @return @ref XXH_ERROR on failure.
 *
 * @see XXH3_64bits_withSecretandSeed(): contract is the same.
 */
XXH_PUBLIC_API XXH_PUREF XXH128_hash_t
XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length,
                               XXH_NOESCAPE const void* secret, size_t secretSize,
                               XXH64_hash_t seed64);

#ifndef XXH_NO_STREAM
/*!
 * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash.
 *
 * @param statePtr   A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState().
 * @param secret     The secret data.
 * @param secretSize The length of @p secret, in bytes.
 * @param seed64     The 64-bit seed to alter the hash result predictably.
 *
 * @return @ref XXH_OK on success.
 * @return @ref XXH_ERROR on failure.
 *
 * @see XXH3_64bits_withSecretandSeed(). Contract is identical.
 */
XXH_PUBLIC_API XXH_errorcode
XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr,
                                    XXH_NOESCAPE const void* secret, size_t secretSize,
                                    XXH64_hash_t seed64);

/*!
 * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash.
 *
 * @param statePtr   A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState().
 * @param secret     The secret data.
 * @param secretSize The length of @p secret, in bytes.
 * @param seed64     The 64-bit seed to alter the hash result predictably.
 *
 * @return @ref XXH_OK on success.
 * @return @ref XXH_ERROR on failure.
 *
 * @see XXH3_64bits_withSecretandSeed(). Contract is identical.
 *
 * Note: there was a bug in an earlier version of this function (<= v0.8.2)
 * that would make it generate an incorrect hash value
 * when @p seed == 0 and @p length < XXH3_MIDSIZE_MAX
 * and @p secret is different from XXH3_generateSecret_fromSeed().
 * As stated in the contract, the correct hash result must be
 * the same as XXH3_128bits_withSeed() when @p length <= XXH3_MIDSIZE_MAX.
 * Results generated by this older version are wrong, hence not comparable.
 */
XXH_PUBLIC_API XXH_errorcode
XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr,
                                     XXH_NOESCAPE const void* secret, size_t secretSize,
                                     XXH64_hash_t seed64);

#endif /* !XXH_NO_STREAM */

#endif  /* !XXH_NO_XXH3 */
#endif  /* XXH_NO_LONG_LONG */
#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)
#  define XXH_IMPLEMENTATION
#endif

#endif  /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */


/* ======================================================================== */
/* ======================================================================== */
/* ======================================================================== */


/*-**********************************************************************
 * xxHash implementation
 *-**********************************************************************
 * xxHash's implementation used to be hosted inside xxhash.c.
 *
 * However, inlining requires implementation to be visible to the compiler,
 * hence be included alongside the header.
 * Previously, implementation was hosted inside xxhash.c,
 * which was then #included when inlining was activated.
 * This construction created issues with a few build and install systems,
 * as it required xxhash.c to be stored in /include directory.
 *
 * xxHash implementation is now directly integrated within xxhash.h.
 * As a consequence, xxhash.c is no longer needed in /include.
 *
 * xxhash.c is still available and is still useful.
 * In a "normal" setup, when xxhash is not inlined,
 * xxhash.h only exposes the prototypes and public symbols,
 * while xxhash.c can be built into an object file xxhash.o
 * which can then be linked into the final binary.
 ************************************************************************/

#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \
   || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387)
#  define XXH_IMPLEM_13a8737387

/* *************************************
*  Tuning parameters
***************************************/

/*!
 * @defgroup tuning Tuning parameters
 * @{
 *
 * Various macros to control xxHash's behavior.
 */
#ifdef XXH_DOXYGEN
/*!
 * @brief Define this to disable 64-bit code.
 *
 * Useful if only using the @ref XXH32_family and you have a strict C90 compiler.
 */
#  define XXH_NO_LONG_LONG
#  undef XXH_NO_LONG_LONG /* don't actually */
/*!
 * @brief Controls how unaligned memory is accessed.
 *
 * By default, access to unaligned memory is controlled by `memcpy()`, which is
 * safe and portable.
 *
 * Unfortunately, on some target/compiler combinations, the generated assembly
 * is sub-optimal.
 *
 * The below switch allow selection of a different access method
 * in the search for improved performance.
 *
 * @par Possible options:
 *
 *  - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy`
 *   @par
 *     Use `memcpy()`. Safe and portable. Note that most modern compilers will
 *     eliminate the function call and treat it as an unaligned access.
 *
 *  - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((aligned(1)))`
 *   @par
 *     Depends on compiler extensions and is therefore not portable.
 *     This method is safe _if_ your compiler supports it,
 *     and *generally* as fast or faster than `memcpy`.
 *
 *  - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast
 *  @par
 *     Casts directly and dereferences. This method doesn't depend on the
 *     compiler, but it violates the C standard as it directly dereferences an
 *     unaligned pointer. It can generate buggy code on targets which do not
 *     support unaligned memory accesses, but in some circumstances, it's the
 *     only known way to get the most performance.
 *
 *  - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift
 *  @par
 *     Also portable. This can generate the best code on old compilers which don't
 *     inline small `memcpy()` calls, and it might also be faster on big-endian
 *     systems which lack a native byteswap instruction. However, some compilers
 *     will emit literal byteshifts even if the target supports unaligned access.
 *
 *
 * @warning
 *   Methods 1 and 2 rely on implementation-defined behavior. Use these with
 *   care, as what works on one compiler/platform/optimization level may cause
 *   another to read garbage data or even crash.
 *
 * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details.
 *
 * Prefer these methods in priority order (0 > 3 > 1 > 2)
 */
#  define XXH_FORCE_MEMORY_ACCESS 0

/*!
 * @def XXH_SIZE_OPT
 * @brief Controls how much xxHash optimizes for size.
 *
 * xxHash, when compiled, tends to result in a rather large binary size. This
 * is mostly due to heavy usage to forced inlining and constant folding of the
 * @ref XXH3_family to increase performance.
 *
 * However, some developers prefer size over speed. This option can
 * significantly reduce the size of the generated code. When using the `-Os`
 * or `-Oz` options on GCC or Clang, this is defined to 1 by default,
 * otherwise it is defined to 0.
 *
 * Most of these size optimizations can be controlled manually.
 *
 * This is a number from 0-2.
 *  - `XXH_SIZE_OPT` == 0: Default. xxHash makes no size optimizations. Speed
 *    comes first.
 *  - `XXH_SIZE_OPT` == 1: Default for `-Os` and `-Oz`. xxHash is more
 *    conservative and disables hacks that increase code size. It implies the
 *    options @ref XXH_NO_INLINE_HINTS == 1, @ref XXH_FORCE_ALIGN_CHECK == 0,
 *    and @ref XXH3_NEON_LANES == 8 if they are not already defined.
 *  - `XXH_SIZE_OPT` == 2: xxHash tries to make itself as small as possible.
 *    Performance may cry. For example, the single shot functions just use the
 *    streaming API.
 */
#  define XXH_SIZE_OPT 0

/*!
 * @def XXH_FORCE_ALIGN_CHECK
 * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32()
 * and XXH64() only).
 *
 * This is an important performance trick for architectures without decent
 * unaligned memory access performance.
 *
 * It checks for input alignment, and when conditions are met, uses a "fast
 * path" employing direct 32-bit/64-bit reads, resulting in _dramatically
 * faster_ read speed.
 *
 * The check costs one initial branch per hash, which is generally negligible,
 * but not zero.
 *
 * Moreover, it's not useful to generate an additional code path if memory
 * access uses the same instruction for both aligned and unaligned
 * addresses (e.g. x86 and aarch64).
 *
 * In these cases, the alignment check can be removed by setting this macro to 0.
 * Then the code will always use unaligned memory access.
 * Align check is automatically disabled on x86, x64, ARM64, and some ARM chips
 * which are platforms known to offer good unaligned memory accesses performance.
 *
 * It is also disabled by default when @ref XXH_SIZE_OPT >= 1.
 *
 * This option does not affect XXH3 (only XXH32 and XXH64).
 */
#  define XXH_FORCE_ALIGN_CHECK 0

/*!
 * @def XXH_NO_INLINE_HINTS
 * @brief When non-zero, sets all functions to `static`.
 *
 * By default, xxHash tries to force the compiler to inline almost all internal
 * functions.
 *
 * This can usually improve performance due to reduced jumping and improved
 * constant folding, but significantly increases the size of the binary which
 * might not be favorable.
 *
 * Additionally, sometimes the forced inlining can be detrimental to performance,
 * depending on the architecture.
 *
 * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the
 * compiler full control on whether to inline or not.
 *
 * When not optimizing (-O0), using `-fno-inline` with GCC or Clang, or if
 * @ref XXH_SIZE_OPT >= 1, this will automatically be defined.
 */
#  define XXH_NO_INLINE_HINTS 0

/*!
 * @def XXH3_INLINE_SECRET
 * @brief Determines whether to inline the XXH3 withSecret code.
 *
 * When the secret size is known, the compiler can improve the performance
 * of XXH3_64bits_withSecret() and XXH3_128bits_withSecret().
 *
 * However, if the secret size is not known, it doesn't have any benefit. This
 * happens when xxHash is compiled into a global symbol. Therefore, if
 * @ref XXH_INLINE_ALL is *not* defined, this will be defined to 0.
 *
 * Additionally, this defaults to 0 on GCC 12+, which has an issue with function pointers
 * that are *sometimes* force inline on -Og, and it is impossible to automatically
 * detect this optimization level.
 */
#  define XXH3_INLINE_SECRET 0

/*!
 * @def XXH32_ENDJMP
 * @brief Whether to use a jump for `XXH32_finalize`.
 *
 * For performance, `XXH32_finalize` uses multiple branches in the finalizer.
 * This is generally preferable for performance,
 * but depending on exact architecture, a jmp may be preferable.
 *
 * This setting is only possibly making a difference for very small inputs.
 */
#  define XXH32_ENDJMP 0

/*!
 * @internal
 * @brief Redefines old internal names.
 *
 * For compatibility with code that uses xxHash's internals before the names
 * were changed to improve namespacing. There is no other reason to use this.
 */
#  define XXH_OLD_NAMES
#  undef XXH_OLD_NAMES /* don't actually use, it is ugly. */

/*!
 * @def XXH_NO_STREAM
 * @brief Disables the streaming API.
 *
 * When xxHash is not inlined and the streaming functions are not used, disabling
 * the streaming functions can improve code size significantly, especially with
 * the @ref XXH3_family which tends to make constant folded copies of itself.
 */
#  define XXH_NO_STREAM
#  undef XXH_NO_STREAM /* don't actually */
#endif /* XXH_DOXYGEN */
/*!
 * @}
 */

#ifndef XXH_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
   /* prefer __packed__ structures (method 1) for GCC
    * < ARMv7 with unaligned access (e.g. Raspbian armhf) still uses byte shifting, so we use memcpy
    * which for some reason does unaligned loads. */
#  if defined(__GNUC__) && !(defined(__ARM_ARCH) && __ARM_ARCH < 7 && defined(__ARM_FEATURE_UNALIGNED))
#    define XXH_FORCE_MEMORY_ACCESS 1
#  endif
#endif

#ifndef XXH_SIZE_OPT
   /* default to 1 for -Os or -Oz */
#  if (defined(__GNUC__) || defined(__clang__)) && defined(__OPTIMIZE_SIZE__)
#    define XXH_SIZE_OPT 1
#  else
#    define XXH_SIZE_OPT 0
#  endif
#endif

#ifndef XXH_FORCE_ALIGN_CHECK  /* can be defined externally */
   /* don't check on sizeopt, x86, aarch64, or arm when unaligned access is available */
#  if XXH_SIZE_OPT >= 1 || \
      defined(__i386)  || defined(__x86_64__) || defined(__aarch64__) || defined(__ARM_FEATURE_UNALIGNED) \
   || defined(_M_IX86) || defined(_M_X64)     || defined(_M_ARM64)    || defined(_M_ARM) /* visual */
#    define XXH_FORCE_ALIGN_CHECK 0
#  else
#    define XXH_FORCE_ALIGN_CHECK 1
#  endif
#endif

#ifndef XXH_NO_INLINE_HINTS
#  if XXH_SIZE_OPT >= 1 || defined(__NO_INLINE__)  /* -O0, -fno-inline */
#    define XXH_NO_INLINE_HINTS 1
#  else
#    define XXH_NO_INLINE_HINTS 0
#  endif
#endif

#ifndef XXH3_INLINE_SECRET
#  if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12) \
     || !defined(XXH_INLINE_ALL)
#    define XXH3_INLINE_SECRET 0
#  else
#    define XXH3_INLINE_SECRET 1
#  endif
#endif

#ifndef XXH32_ENDJMP
/* generally preferable for performance */
#  define XXH32_ENDJMP 0
#endif

/*!
 * @defgroup impl Implementation
 * @{
 */


/* *************************************
*  Includes & Memory related functions
***************************************/
#if defined(XXH_NO_STREAM)
/* nothing */
#elif defined(XXH_NO_STDLIB)

/* When requesting to disable any mention of stdlib,
 * the library loses the ability to invoked malloc / free.
 * In practice, it means that functions like `XXH*_createState()`
 * will always fail, and return NULL.
 * This flag is useful in situations where
 * xxhash.h is integrated into some kernel, embedded or limited environment
 * without access to dynamic allocation.
 */

static XXH_CONSTF void* XXH_malloc(size_t s) { (void)s; return NULL; }
static void XXH_free(void* p) { (void)p; }

#else

/*
 * Modify the local functions below should you wish to use
 * different memory routines for malloc() and free()
 */
#include <stdlib.h>

/*!
 * @internal
 * @brief Modify this function to use a different routine than malloc().
 */
static XXH_MALLOCF void* XXH_malloc(size_t s) { return malloc(s); }

/*!
 * @internal
 * @brief Modify this function to use a different routine than free().
 */
static void XXH_free(void* p) { free(p); }

#endif  /* XXH_NO_STDLIB */

#ifndef XXH_memcpy
/*!
 * @internal
 * @brief XXH_memcpy() macro can be redirected at compile time
 */
#  include <string.h>
#  define XXH_memcpy memcpy
#endif

#ifndef XXH_memset
/*!
 * @internal
 * @brief XXH_memset() macro can be redirected at compile time
 */
#  include <string.h>
#  define XXH_memset memset
#endif

#ifndef XXH_memcmp
/*!
 * @internal
 * @brief XXH_memcmp() macro can be redirected at compile time
 * Note: only needed by XXH128.
 */
#  include <string.h>
#  define XXH_memcmp memcmp
#endif



#include <limits.h>   /* ULLONG_MAX */


/* *************************************
*  Compiler Specific Options
***************************************/
#ifdef _MSC_VER /* Visual Studio warning fix */
#  pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#endif

#if XXH_NO_INLINE_HINTS  /* disable inlining hints */
#  if defined(__GNUC__) || defined(__clang__)
#    define XXH_FORCE_INLINE static __attribute__((__unused__))
#  else
#    define XXH_FORCE_INLINE static
#  endif
#  define XXH_NO_INLINE static
/* enable inlining hints */
#elif defined(__GNUC__) || defined(__clang__)
#  define XXH_FORCE_INLINE static __inline__ __attribute__((__always_inline__, __unused__))
#  define XXH_NO_INLINE static __attribute__((__noinline__))
#elif defined(_MSC_VER)  /* Visual Studio */
#  define XXH_FORCE_INLINE static __forceinline
#  define XXH_NO_INLINE static __declspec(noinline)
#elif defined (__cplusplus) \
  || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L))   /* C99 */
#  define XXH_FORCE_INLINE static inline
#  define XXH_NO_INLINE static
#else
#  define XXH_FORCE_INLINE static
#  define XXH_NO_INLINE static
#endif

#if defined(XXH_INLINE_ALL)
#  define XXH_STATIC XXH_FORCE_INLINE
#else
#  define XXH_STATIC static
#endif

#if XXH3_INLINE_SECRET
#  define XXH3_WITH_SECRET_INLINE XXH_FORCE_INLINE
#else
#  define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE
#endif

#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */
#  define XXH_RESTRICT   /* disable */
#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* >= C99 */
#  define XXH_RESTRICT   restrict
#elif (defined (__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) \
   || (defined (__clang__)) \
   || (defined (_MSC_VER) && (_MSC_VER >= 1400)) \
   || (defined (__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300))
/*
 * There are a LOT more compilers that recognize __restrict but this
 * covers the major ones.
 */
#  define XXH_RESTRICT   __restrict
#else
#  define XXH_RESTRICT   /* disable */
#endif

/* *************************************
*  Debug
***************************************/
/*!
 * @ingroup tuning
 * @def XXH_DEBUGLEVEL
 * @brief Sets the debugging level.
 *
 * XXH_DEBUGLEVEL is expected to be defined externally, typically via the
 * compiler's command line options. The value must be a number.
 */
#ifndef XXH_DEBUGLEVEL
#  ifdef DEBUGLEVEL /* backwards compat */
#    define XXH_DEBUGLEVEL DEBUGLEVEL
#  else
#    define XXH_DEBUGLEVEL 0
#  endif
#endif

#if (XXH_DEBUGLEVEL>=1)
#  include <assert.h>   /* note: can still be disabled with NDEBUG */
#  define XXH_ASSERT(c)   assert(c)
#else
#  if defined(__INTEL_COMPILER)
#    define XXH_ASSERT(c)   XXH_ASSUME((unsigned char) (c))
#  else
#    define XXH_ASSERT(c)   XXH_ASSUME(c)
#  endif
#endif

/* note: use after variable declarations */
#ifndef XXH_STATIC_ASSERT
#  if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)    /* C11 */
#    define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { _Static_assert((c),m); } while(0)
#  elif defined(__cplusplus) && (__cplusplus >= 201103L)            /* C++11 */
#    define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0)
#  else
#    define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0)
#  endif
#  define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c)
#endif

/*!
 * @internal
 * @def XXH_COMPILER_GUARD(var)
 * @brief Used to prevent unwanted optimizations for @p var.
 *
 * It uses an empty GCC inline assembly statement with a register constraint
 * which forces @p var into a general purpose register (eg eax, ebx, ecx
 * on x86) and marks it as modified.
 *
 * This is used in a few places to avoid unwanted autovectorization (e.g.
 * XXH32_round()). All vectorization we want is explicit via intrinsics,
 * and _usually_ isn't wanted elsewhere.
 *
 * We also use it to prevent unwanted constant folding for AArch64 in
 * XXH3_initCustomSecret_scalar().
 */
#if defined(__GNUC__) || defined(__clang__)
#  define XXH_COMPILER_GUARD(var) __asm__("" : "+r" (var))
#else
#  define XXH_COMPILER_GUARD(var) ((void)0)
#endif

/* Specifically for NEON vectors which use the "w" constraint, on
 * Clang. */
#if defined(__clang__) && defined(__ARM_ARCH) && !defined(__wasm__)
#  define XXH_COMPILER_GUARD_CLANG_NEON(var) __asm__("" : "+w" (var))
#else
#  define XXH_COMPILER_GUARD_CLANG_NEON(var) ((void)0)
#endif

/* *************************************
*  Basic Types
***************************************/
#if !defined (__VMS) \
 && (defined (__cplusplus) \
 || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
#   ifdef _AIX
#     include <inttypes.h>
#   else
#     include <stdint.h>
#   endif
    typedef uint8_t xxh_u8;
#else
    typedef unsigned char xxh_u8;
#endif
typedef XXH32_hash_t xxh_u32;

#ifdef XXH_OLD_NAMES
#  warning "XXH_OLD_NAMES is planned to be removed starting v0.9. If the program depends on it, consider moving away from it by employing newer type names directly"
#  define BYTE xxh_u8
#  define U8   xxh_u8
#  define U32  xxh_u32
#endif

/* ***   Memory access   *** */

/*!
 * @internal
 * @fn xxh_u32 XXH_read32(const void* ptr)
 * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness.
 *
 * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
 *
 * @param ptr The pointer to read from.
 * @return The 32-bit native endian integer from the bytes at @p ptr.
 */

/*!
 * @internal
 * @fn xxh_u32 XXH_readLE32(const void* ptr)
 * @brief Reads an unaligned 32-bit little endian integer from @p ptr.
 *
 * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
 *
 * @param ptr The pointer to read from.
 * @return The 32-bit little endian integer from the bytes at @p ptr.
 */

/*!
 * @internal
 * @fn xxh_u32 XXH_readBE32(const void* ptr)
 * @brief Reads an unaligned 32-bit big endian integer from @p ptr.
 *
 * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
 *
 * @param ptr The pointer to read from.
 * @return The 32-bit big endian integer from the bytes at @p ptr.
 */

/*!
 * @internal
 * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align)
 * @brief Like @ref XXH_readLE32(), but has an option for aligned reads.
 *
 * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
 * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is
 * always @ref XXH_alignment::XXH_unaligned.
 *
 * @param ptr The pointer to read from.
 * @param align Whether @p ptr is aligned.
 * @pre
 *   If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte
 *   aligned.
 * @return The 32-bit little endian integer from the bytes at @p ptr.
 */

#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
/*
 * Manual byteshift. Best for old compilers which don't inline memcpy.
 * We actually directly use XXH_readLE32 and XXH_readBE32.
 */
#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))

/*
 * Force direct memory access. Only works on CPU which support unaligned memory
 * access in hardware.
 */
static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; }

#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))

/*
 * __attribute__((aligned(1))) is supported by gcc and clang. Originally the
 * documentation claimed that it only increased the alignment, but actually it
 * can decrease it on gcc, clang, and icc:
 * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502,
 * https://gcc.godbolt.org/z/xYez1j67Y.
 */
#ifdef XXH_OLD_NAMES
typedef union { xxh_u32 u32; } __attribute__((__packed__)) unalign;
#endif
static xxh_u32 XXH_read32(const void* ptr)
{
    typedef __attribute__((__aligned__(1))) __attribute__((__may_alias__)) xxh_u32 xxh_unalign32;
    return *((const xxh_unalign32*)ptr);
}

#else

/*
 * Portable and safe solution. Generally efficient.
 * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
 */
static xxh_u32 XXH_read32(const void* memPtr)
{
    xxh_u32 val;
    XXH_memcpy(&val, memPtr, sizeof(val));
    return val;
}

#endif   /* XXH_FORCE_DIRECT_MEMORY_ACCESS */


/* ***   Endianness   *** */

/*!
 * @ingroup tuning
 * @def XXH_CPU_LITTLE_ENDIAN
 * @brief Whether the target is little endian.
 *
 * Defined to 1 if the target is little endian, or 0 if it is big endian.
 * It can be defined externally, for example on the compiler command line.
 *
 * If it is not defined,
 * a runtime check (which is usually constant folded) is used instead.
 *
 * @note
 *   This is not necessarily defined to an integer constant.
 *
 * @see XXH_isLittleEndian() for the runtime check.
 */
#ifndef XXH_CPU_LITTLE_ENDIAN
/*
 * Try to detect endianness automatically, to avoid the nonstandard behavior
 * in `XXH_isLittleEndian()`
 */
#  if defined(_WIN32) /* Windows is always little endian */ \
     || defined(__LITTLE_ENDIAN__) \
     || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
#    define XXH_CPU_LITTLE_ENDIAN 1
#  elif defined(__BIG_ENDIAN__) \
     || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
#    define XXH_CPU_LITTLE_ENDIAN 0
#  else
/*!
 * @internal
 * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN.
 *
 * Most compilers will constant fold this.
 */
static int XXH_isLittleEndian(void)
{
    /*
     * Portable and well-defined behavior.
     * Don't use static: it is detrimental to performance.
     */
    const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 };
    return one.c[0];
}
#   define XXH_CPU_LITTLE_ENDIAN   XXH_isLittleEndian()
#  endif
#endif




/* ****************************************
*  Compiler-specific Functions and Macros
******************************************/
#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)

#ifdef __has_builtin
#  define XXH_HAS_BUILTIN(x) __has_builtin(x)
#else
#  define XXH_HAS_BUILTIN(x) 0
#endif



/*
 * C23 and future versions have standard "unreachable()".
 * Once it has been implemented reliably we can add it as an
 * additional case:
 *
 * ```
 * #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 202311L)
 * #  include <stddef.h>
 * #  ifdef unreachable
 * #    define XXH_UNREACHABLE() unreachable()
 * #  endif
 * #endif
 * ```
 *
 * Note C++23 also has std::unreachable() which can be detected
 * as follows:
 * ```
 * #if defined(__cpp_lib_unreachable) && (__cpp_lib_unreachable >= 202202L)
 * #  include <utility>
 * #  define XXH_UNREACHABLE() std::unreachable()
 * #endif
 * ```
 * NB: `__cpp_lib_unreachable` is defined in the `<version>` header.
 * We don't use that as including `<utility>` in `extern "C"` blocks
 * doesn't work on GCC12
 */

#if XXH_HAS_BUILTIN(__builtin_unreachable)
#  define XXH_UNREACHABLE() __builtin_unreachable()

#elif defined(_MSC_VER)
#  define XXH_UNREACHABLE() __assume(0)

#else
#  define XXH_UNREACHABLE()
#endif

#if XXH_HAS_BUILTIN(__builtin_assume)
#  define XXH_ASSUME(c) __builtin_assume(c)
#else
#  define XXH_ASSUME(c) if (!(c)) { XXH_UNREACHABLE(); }
#endif

/*!
 * @internal
 * @def XXH_rotl32(x,r)
 * @brief 32-bit rotate left.
 *
 * @param x The 32-bit integer to be rotated.
 * @param r The number of bits to rotate.
 * @pre
 *   @p r > 0 && @p r < 32
 * @note
 *   @p x and @p r may be evaluated multiple times.
 * @return The rotated result.
 */
#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \
                               && XXH_HAS_BUILTIN(__builtin_rotateleft64)
#  define XXH_rotl32 __builtin_rotateleft32
#  define XXH_rotl64 __builtin_rotateleft64
#elif XXH_HAS_BUILTIN(__builtin_stdc_rotate_left)
#  define XXH_rotl32 __builtin_stdc_rotate_left
#  define XXH_rotl64 __builtin_stdc_rotate_left
/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */
#elif defined(_MSC_VER)
#  define XXH_rotl32(x,r) _rotl(x,r)
#  define XXH_rotl64(x,r) _rotl64(x,r)
#else
#  define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r))))
#  define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r))))
#endif

/*!
 * @internal
 * @fn xxh_u32 XXH_swap32(xxh_u32 x)
 * @brief A 32-bit byteswap.
 *
 * @param x The 32-bit integer to byteswap.
 * @return @p x, byteswapped.
 */
#if defined(_MSC_VER)     /* Visual Studio */
#  define XXH_swap32 _byteswap_ulong
#elif XXH_GCC_VERSION >= 403
#  define XXH_swap32 __builtin_bswap32
#else
static xxh_u32 XXH_swap32 (xxh_u32 x)
{
    return  ((x << 24) & 0xff000000 ) |
            ((x <<  8) & 0x00ff0000 ) |
            ((x >>  8) & 0x0000ff00 ) |
            ((x >> 24) & 0x000000ff );
}
#endif


/* ***************************
*  Memory reads
*****************************/

/*!
 * @internal
 * @brief Enum to indicate whether a pointer is aligned.
 */
typedef enum {
    XXH_aligned,  /*!< Aligned */
    XXH_unaligned /*!< Possibly unaligned */
} XXH_alignment;

/*
 * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load.
 *
 * This is ideal for older compilers which don't inline memcpy.
 */
#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))

XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr)
{
    const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
    return bytePtr[0]
         | ((xxh_u32)bytePtr[1] << 8)
         | ((xxh_u32)bytePtr[2] << 16)
         | ((xxh_u32)bytePtr[3] << 24);
}

XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr)
{
    const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
    return bytePtr[3]
         | ((xxh_u32)bytePtr[2] << 8)
         | ((xxh_u32)bytePtr[1] << 16)
         | ((xxh_u32)bytePtr[0] << 24);
}

#else
XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr)
{
    return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
}

static xxh_u32 XXH_readBE32(const void* ptr)
{
    return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
}
#endif

XXH_FORCE_INLINE xxh_u32
XXH_readLE32_align(const void* ptr, XXH_alignment align)
{
    if (align==XXH_unaligned) {
        return XXH_readLE32(ptr);
    } else {
        return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr);
    }
}


/* *************************************
*  Misc
***************************************/
/*! @ingroup public */
XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }


/* *******************************************************************
*  32-bit hash functions
*********************************************************************/
/*!
 * @}
 * @defgroup XXH32_impl XXH32 implementation
 * @ingroup impl
 *
 * Details on the XXH32 implementation.
 * @{
 */
 /* #define instead of static const, to be used as initializers */
#define XXH_PRIME32_1  0x9E3779B1U  /*!< 0b10011110001101110111100110110001 */
#define XXH_PRIME32_2  0x85EBCA77U  /*!< 0b10000101111010111100101001110111 */
#define XXH_PRIME32_3  0xC2B2AE3DU  /*!< 0b11000010101100101010111000111101 */
#define XXH_PRIME32_4  0x27D4EB2FU  /*!< 0b00100111110101001110101100101111 */
#define XXH_PRIME32_5  0x165667B1U  /*!< 0b00010110010101100110011110110001 */

#ifdef XXH_OLD_NAMES
#  define PRIME32_1 XXH_PRIME32_1
#  define PRIME32_2 XXH_PRIME32_2
#  define PRIME32_3 XXH_PRIME32_3
#  define PRIME32_4 XXH_PRIME32_4
#  define PRIME32_5 XXH_PRIME32_5
#endif

/*!
 * @internal
 * @brief Normal stripe processing routine.
 *
 * This shuffles the bits so that any bit from @p input impacts several bits in
 * @p acc.
 *
 * @param acc The accumulator lane.
 * @param input The stripe of input to mix.
 * @return The mixed accumulator lane.
 */
static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input)
{
    acc += input * XXH_PRIME32_2;
    acc  = XXH_rotl32(acc, 13);
    acc *= XXH_PRIME32_1;
#if (defined(__SSE4_1__) || defined(__aarch64__) || defined(__wasm_simd128__)) && !defined(XXH_ENABLE_AUTOVECTORIZE)
    /*
     * UGLY HACK:
     * A compiler fence is used to prevent GCC and Clang from
     * autovectorizing the XXH32 loop (pragmas and attributes don't work for some
     * reason) without globally disabling SSE4.1.
     *
     * The reason we want to avoid vectorization is because despite working on
     * 4 integers at a time, there are multiple factors slowing XXH32 down on
     * SSE4:
     * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on
     *   newer chips!) making it slightly slower to multiply four integers at
     *   once compared to four integers independently. Even when pmulld was
     *   fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE
     *   just to multiply unless doing a long operation.
     *
     * - Four instructions are required to rotate,
     *      movqda tmp,  v // not required with VEX encoding
     *      pslld  tmp, 13 // tmp <<= 13
     *      psrld  v,   19 // x >>= 19
     *      por    v,  tmp // x |= tmp
     *   compared to one for scalar:
     *      roll   v, 13    // reliably fast across the board
     *      shldl  v, v, 13 // Sandy Bridge and later prefer this for some reason
     *
     * - Instruction level parallelism is actually more beneficial here because
     *   the SIMD actually serializes this operation: While v1 is rotating, v2
     *   can load data, while v3 can multiply. SSE forces them to operate
     *   together.
     *
     * This is also enabled on AArch64, as Clang is *very aggressive* in vectorizing
     * the loop. NEON is only faster on the A53, and with the newer cores, it is less
     * than half the speed.
     *
     * Additionally, this is used on WASM SIMD128 because it JITs to the same
     * SIMD instructions and has the same issue.
     */
    XXH_COMPILER_GUARD(acc);
#endif
    return acc;
}

/*!
 * @internal
 * @brief Mixes all bits to finalize the hash.
 *
 * The final mix ensures that all input bits have a chance to impact any bit in
 * the output digest, resulting in an unbiased distribution.
 *
 * @param hash The hash to avalanche.
 * @return The avalanched hash.
 */
static xxh_u32 XXH32_avalanche(xxh_u32 hash)
{
    hash ^= hash >> 15;
    hash *= XXH_PRIME32_2;
    hash ^= hash >> 13;
    hash *= XXH_PRIME32_3;
    hash ^= hash >> 16;
    return hash;
}

#define XXH_get32bits(p) XXH_readLE32_align(p, align)

/*!
 * @internal
 * @brief Sets up the initial accumulator state for XXH32().
 */
XXH_FORCE_INLINE void
XXH32_initAccs(xxh_u32 *acc, xxh_u32 seed)
{
    XXH_ASSERT(acc != NULL);
    acc[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
    acc[1] = seed + XXH_PRIME32_2;
    acc[2] = seed + 0;
    acc[3] = seed - XXH_PRIME32_1;
}

/*!
 * @internal
 * @brief Consumes a block of data for XXH32().
 *
 * @return the end input pointer.
 */
XXH_FORCE_INLINE const xxh_u8 *
XXH32_consumeLong(
    xxh_u32 *XXH_RESTRICT acc,
    xxh_u8 const *XXH_RESTRICT input,
    size_t len,
    XXH_alignment align
)
{
    const xxh_u8* const bEnd = input + len;
    const xxh_u8* const limit = bEnd - 15;
    XXH_ASSERT(acc != NULL);
    XXH_ASSERT(input != NULL);
    XXH_ASSERT(len >= 16);
    do {
        acc[0] = XXH32_round(acc[0], XXH_get32bits(input)); input += 4;
        acc[1] = XXH32_round(acc[1], XXH_get32bits(input)); input += 4;
        acc[2] = XXH32_round(acc[2], XXH_get32bits(input)); input += 4;
        acc[3] = XXH32_round(acc[3], XXH_get32bits(input)); input += 4;
    } while (input < limit);

    return input;
}

/*!
 * @internal
 * @brief Merges the accumulator lanes together for XXH32()
 */
XXH_FORCE_INLINE XXH_PUREF xxh_u32
XXH32_mergeAccs(const xxh_u32 *acc)
{
    XXH_ASSERT(acc != NULL);
    return XXH_rotl32(acc[0], 1)  + XXH_rotl32(acc[1], 7)
         + XXH_rotl32(acc[2], 12) + XXH_rotl32(acc[3], 18);
}

/*!
 * @internal
 * @brief Processes the last 0-15 bytes of @p ptr.
 *
 * There may be up to 15 bytes remaining to consume from the input.
 * This final stage will digest them to ensure that all input bytes are present
 * in the final mix.
 *
 * @param hash The hash to finalize.
 * @param ptr The pointer to the remaining input.
 * @param len The remaining length, modulo 16.
 * @param align Whether @p ptr is aligned.
 * @return The finalized hash.
 * @see XXH64_finalize().
 */
static XXH_PUREF xxh_u32
XXH32_finalize(xxh_u32 hash, const xxh_u8* ptr, size_t len, XXH_alignment align)
{
#define XXH_PROCESS1 do {                             \
    hash += (*ptr++) * XXH_PRIME32_5;                 \
    hash = XXH_rotl32(hash, 11) * XXH_PRIME32_1;      \
} while (0)

#define XXH_PROCESS4 do {                             \
    hash += XXH_get32bits(ptr) * XXH_PRIME32_3;       \
    ptr += 4;                                         \
    hash  = XXH_rotl32(hash, 17) * XXH_PRIME32_4;     \
} while (0)

    if (ptr==NULL) XXH_ASSERT(len == 0);

    /* Compact rerolled version; generally faster */
    if (!XXH32_ENDJMP) {
        len &= 15;
        while (len >= 4) {
            XXH_PROCESS4;
            len -= 4;
        }
        while (len > 0) {
            XXH_PROCESS1;
            --len;
        }
        return XXH32_avalanche(hash);
    } else {
         switch(len&15) /* or switch(bEnd - p) */ {
           case 12:      XXH_PROCESS4;
                         XXH_FALLTHROUGH;  /* fallthrough */
           case 8:       XXH_PROCESS4;
                         XXH_FALLTHROUGH;  /* fallthrough */
           case 4:       XXH_PROCESS4;
                         return XXH32_avalanche(hash);

           case 13:      XXH_PROCESS4;
                         XXH_FALLTHROUGH;  /* fallthrough */
           case 9:       XXH_PROCESS4;
                         XXH_FALLTHROUGH;  /* fallthrough */
           case 5:       XXH_PROCESS4;
                         XXH_PROCESS1;
                         return XXH32_avalanche(hash);

           case 14:      XXH_PROCESS4;
                         XXH_FALLTHROUGH;  /* fallthrough */
           case 10:      XXH_PROCESS4;
                         XXH_FALLTHROUGH;  /* fallthrough */
           case 6:       XXH_PROCESS4;
                         XXH_PROCESS1;
                         XXH_PROCESS1;
                         return XXH32_avalanche(hash);

           case 15:      XXH_PROCESS4;
                         XXH_FALLTHROUGH;  /* fallthrough */
           case 11:      XXH_PROCESS4;
                         XXH_FALLTHROUGH;  /* fallthrough */
           case 7:       XXH_PROCESS4;
                         XXH_FALLTHROUGH;  /* fallthrough */
           case 3:       XXH_PROCESS1;
                         XXH_FALLTHROUGH;  /* fallthrough */
           case 2:       XXH_PROCESS1;
                         XXH_FALLTHROUGH;  /* fallthrough */
           case 1:       XXH_PROCESS1;
                         XXH_FALLTHROUGH;  /* fallthrough */
           case 0:       return XXH32_avalanche(hash);
        }
        XXH_ASSERT(0);
        return hash;   /* reaching this point is deemed impossible */
    }
}

#ifdef XXH_OLD_NAMES
#  define PROCESS1 XXH_PROCESS1
#  define PROCESS4 XXH_PROCESS4
#else
#  undef XXH_PROCESS1
#  undef XXH_PROCESS4
#endif

/*!
 * @internal
 * @brief The implementation for @ref XXH32().
 *
 * @param input , len , seed Directly passed from @ref XXH32().
 * @param align Whether @p input is aligned.
 * @return The calculated hash.
 */
XXH_FORCE_INLINE XXH_PUREF xxh_u32
XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align)
{
    xxh_u32 h32;

    if (input==NULL) XXH_ASSERT(len == 0);

    if (len>=16) {
        xxh_u32 acc[4];
        XXH32_initAccs(acc, seed);

        input = XXH32_consumeLong(acc, input, len, align);

        h32 = XXH32_mergeAccs(acc);
    } else {
        h32  = seed + XXH_PRIME32_5;
    }

    h32 += (xxh_u32)len;

    return XXH32_finalize(h32, input, len&15, align);
}

/*! @ingroup XXH32_family */
XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed)
{
#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2
    /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
    XXH32_state_t state;
    XXH32_reset(&state, seed);
    XXH32_update(&state, (const xxh_u8*)input, len);
    return XXH32_digest(&state);
#else
    if (XXH_FORCE_ALIGN_CHECK) {
        if ((((size_t)input) & 3) == 0) {   /* Input is 4-bytes aligned, leverage the speed benefit */
            return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned);
    }   }

    return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned);
#endif
}



/*******   Hash streaming   *******/
#ifndef XXH_NO_STREAM
/*! @ingroup XXH32_family */
XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
{
    return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
}
/*! @ingroup XXH32_family */
XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
{
    XXH_free(statePtr);
    return XXH_OK;
}

/*! @ingroup XXH32_family */
XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState)
{
    XXH_memcpy(dstState, srcState, sizeof(*dstState));
}

/*! @ingroup XXH32_family */
XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed)
{
    XXH_ASSERT(statePtr != NULL);
    XXH_memset(statePtr, 0, sizeof(*statePtr));
    XXH32_initAccs(statePtr->acc, seed);
    return XXH_OK;
}


/*! @ingroup XXH32_family */
XXH_PUBLIC_API XXH_errorcode
XXH32_update(XXH32_state_t* state, const void* input, size_t len)
{
    if (input==NULL) {
        XXH_ASSERT(len == 0);
        return XXH_OK;
    }

    state->total_len_32 += (XXH32_hash_t)len;
    state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16));

    XXH_ASSERT(state->bufferedSize < sizeof(state->buffer));
    if (len < sizeof(state->buffer) - state->bufferedSize)  {   /* fill in tmp buffer */
        XXH_memcpy(state->buffer + state->bufferedSize, input, len);
        state->bufferedSize += (XXH32_hash_t)len;
        return XXH_OK;
    }

    {   const xxh_u8* xinput = (const xxh_u8*)input;
        const xxh_u8* const bEnd = xinput + len;

        if (state->bufferedSize) {   /* non-empty buffer: complete first */
            XXH_memcpy(state->buffer + state->bufferedSize, xinput, sizeof(state->buffer) - state->bufferedSize);
            xinput += sizeof(state->buffer) - state->bufferedSize;
            /* then process one round */
            (void)XXH32_consumeLong(state->acc, state->buffer, sizeof(state->buffer), XXH_aligned);
            state->bufferedSize = 0;
        }

        XXH_ASSERT(xinput <= bEnd);
        if ((size_t)(bEnd - xinput) >= sizeof(state->buffer)) {
            /* Process the remaining data */
            xinput = XXH32_consumeLong(state->acc, xinput, (size_t)(bEnd - xinput), XXH_unaligned);
        }

        if (xinput < bEnd) {
            /* Copy the leftover to the tmp buffer */
            XXH_memcpy(state->buffer, xinput, (size_t)(bEnd-xinput));
            state->bufferedSize = (unsigned)(bEnd-xinput);
        }
    }

    return XXH_OK;
}


/*! @ingroup XXH32_family */
XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state)
{
    xxh_u32 h32;

    if (state->large_len) {
        h32 = XXH32_mergeAccs(state->acc);
    } else {
        h32 = state->acc[2] /* == seed */ + XXH_PRIME32_5;
    }

    h32 += state->total_len_32;

    return XXH32_finalize(h32, state->buffer, state->bufferedSize, XXH_aligned);
}
#endif /* !XXH_NO_STREAM */

/*******   Canonical representation   *******/

/*! @ingroup XXH32_family */
XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
{
    XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
    if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
    XXH_memcpy(dst, &hash, sizeof(*dst));
}
/*! @ingroup XXH32_family */
XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
{
    return XXH_readBE32(src);
}


#ifndef XXH_NO_LONG_LONG

/* *******************************************************************
*  64-bit hash functions
*********************************************************************/
/*!
 * @}
 * @ingroup impl
 * @{
 */
/*******   Memory access   *******/

typedef XXH64_hash_t xxh_u64;

#ifdef XXH_OLD_NAMES
#  define U64 xxh_u64
#endif

#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
/*
 * Manual byteshift. Best for old compilers which don't inline memcpy.
 * We actually directly use XXH_readLE64 and XXH_readBE64.
 */
#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))

/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
static xxh_u64 XXH_read64(const void* memPtr)
{
    return *(const xxh_u64*) memPtr;
}

#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))

/*
 * __attribute__((aligned(1))) is supported by gcc and clang. Originally the
 * documentation claimed that it only increased the alignment, but actually it
 * can decrease it on gcc, clang, and icc:
 * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502,
 * https://gcc.godbolt.org/z/xYez1j67Y.
 */
#ifdef XXH_OLD_NAMES
typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((__packed__)) unalign64;
#endif
static xxh_u64 XXH_read64(const void* ptr)
{
    typedef __attribute__((__aligned__(1))) __attribute__((__may_alias__)) xxh_u64 xxh_unalign64;
    return *((const xxh_unalign64*)ptr);
}

#else

/*
 * Portable and safe solution. Generally efficient.
 * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
 */
static xxh_u64 XXH_read64(const void* memPtr)
{
    xxh_u64 val;
    XXH_memcpy(&val, memPtr, sizeof(val));
    return val;
}

#endif   /* XXH_FORCE_DIRECT_MEMORY_ACCESS */

#if defined(_MSC_VER)     /* Visual Studio */
#  define XXH_swap64 _byteswap_uint64
#elif XXH_GCC_VERSION >= 403
#  define XXH_swap64 __builtin_bswap64
#else
static xxh_u64 XXH_swap64(xxh_u64 x)
{
    return  ((x << 56) & 0xff00000000000000ULL) |
            ((x << 40) & 0x00ff000000000000ULL) |
            ((x << 24) & 0x0000ff0000000000ULL) |
            ((x << 8)  & 0x000000ff00000000ULL) |
            ((x >> 8)  & 0x00000000ff000000ULL) |
            ((x >> 24) & 0x0000000000ff0000ULL) |
            ((x >> 40) & 0x000000000000ff00ULL) |
            ((x >> 56) & 0x00000000000000ffULL);
}
#endif


/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */
#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))

XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr)
{
    const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
    return bytePtr[0]
         | ((xxh_u64)bytePtr[1] << 8)
         | ((xxh_u64)bytePtr[2] << 16)
         | ((xxh_u64)bytePtr[3] << 24)
         | ((xxh_u64)bytePtr[4] << 32)
         | ((xxh_u64)bytePtr[5] << 40)
         | ((xxh_u64)bytePtr[6] << 48)
         | ((xxh_u64)bytePtr[7] << 56);
}

XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr)
{
    const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
    return bytePtr[7]
         | ((xxh_u64)bytePtr[6] << 8)
         | ((xxh_u64)bytePtr[5] << 16)
         | ((xxh_u64)bytePtr[4] << 24)
         | ((xxh_u64)bytePtr[3] << 32)
         | ((xxh_u64)bytePtr[2] << 40)
         | ((xxh_u64)bytePtr[1] << 48)
         | ((xxh_u64)bytePtr[0] << 56);
}

#else
XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr)
{
    return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
}

static xxh_u64 XXH_readBE64(const void* ptr)
{
    return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
}
#endif

XXH_FORCE_INLINE xxh_u64
XXH_readLE64_align(const void* ptr, XXH_alignment align)
{
    if (align==XXH_unaligned)
        return XXH_readLE64(ptr);
    else
        return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr);
}


/*******   xxh64   *******/
/*!
 * @}
 * @defgroup XXH64_impl XXH64 implementation
 * @ingroup impl
 *
 * Details on the XXH64 implementation.
 * @{
 */
/* #define rather that static const, to be used as initializers */
#define XXH_PRIME64_1  0x9E3779B185EBCA87ULL  /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */
#define XXH_PRIME64_2  0xC2B2AE3D27D4EB4FULL  /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */
#define XXH_PRIME64_3  0x165667B19E3779F9ULL  /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */
#define XXH_PRIME64_4  0x85EBCA77C2B2AE63ULL  /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */
#define XXH_PRIME64_5  0x27D4EB2F165667C5ULL  /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */

#ifdef XXH_OLD_NAMES
#  define PRIME64_1 XXH_PRIME64_1
#  define PRIME64_2 XXH_PRIME64_2
#  define PRIME64_3 XXH_PRIME64_3
#  define PRIME64_4 XXH_PRIME64_4
#  define PRIME64_5 XXH_PRIME64_5
#endif

/*! @copydoc XXH32_round */
static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input)
{
    acc += input * XXH_PRIME64_2;
    acc  = XXH_rotl64(acc, 31);
    acc *= XXH_PRIME64_1;
#if (defined(__AVX512F__)) && !defined(XXH_ENABLE_AUTOVECTORIZE)
    /*
     * DISABLE AUTOVECTORIZATION:
     * A compiler fence is used to prevent GCC and Clang from
     * autovectorizing the XXH64 loop (pragmas and attributes don't work for some
     * reason) without globally disabling AVX512.
     *
     * Autovectorization of XXH64 tends to be detrimental,
     * though the exact outcome may change depending on exact cpu and compiler version.
     * For information, it has been reported as detrimental for Skylake-X,
     * but possibly beneficial for Zen4.
     *
     * The default is to disable auto-vectorization,
     * but you can select to enable it instead using `XXH_ENABLE_AUTOVECTORIZE` build variable.
     */
    XXH_COMPILER_GUARD(acc);
#endif
    return acc;
}

static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val)
{
    val  = XXH64_round(0, val);
    acc ^= val;
    acc  = acc * XXH_PRIME64_1 + XXH_PRIME64_4;
    return acc;
}

/*! @copydoc XXH32_avalanche */
static xxh_u64 XXH64_avalanche(xxh_u64 hash)
{
    hash ^= hash >> 33;
    hash *= XXH_PRIME64_2;
    hash ^= hash >> 29;
    hash *= XXH_PRIME64_3;
    hash ^= hash >> 32;
    return hash;
}


#define XXH_get64bits(p) XXH_readLE64_align(p, align)

/*!
 * @internal
 * @brief Sets up the initial accumulator state for XXH64().
 */
XXH_FORCE_INLINE void
XXH64_initAccs(xxh_u64 *acc, xxh_u64 seed)
{
    XXH_ASSERT(acc != NULL);
    acc[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
    acc[1] = seed + XXH_PRIME64_2;
    acc[2] = seed + 0;
    acc[3] = seed - XXH_PRIME64_1;
}

/*!
 * @internal
 * @brief Consumes a block of data for XXH64().
 *
 * @return the end input pointer.
 */
XXH_FORCE_INLINE const xxh_u8 *
XXH64_consumeLong(
    xxh_u64 *XXH_RESTRICT acc,
    xxh_u8 const *XXH_RESTRICT input,
    size_t len,
    XXH_alignment align
)
{
    const xxh_u8* const bEnd = input + len;
    const xxh_u8* const limit = bEnd - 31;
    XXH_ASSERT(acc != NULL);
    XXH_ASSERT(input != NULL);
    XXH_ASSERT(len >= 32);
    do {
        /* reroll on 32-bit */
        if (sizeof(void *) < sizeof(xxh_u64)) {
            size_t i;
            for (i = 0; i < 4; i++) {
                acc[i] = XXH64_round(acc[i], XXH_get64bits(input));
                input += 8;
            }
        } else {
            acc[0] = XXH64_round(acc[0], XXH_get64bits(input)); input += 8;
            acc[1] = XXH64_round(acc[1], XXH_get64bits(input)); input += 8;
            acc[2] = XXH64_round(acc[2], XXH_get64bits(input)); input += 8;
            acc[3] = XXH64_round(acc[3], XXH_get64bits(input)); input += 8;
        }
    } while (input < limit);

    return input;
}

/*!
 * @internal
 * @brief Merges the accumulator lanes together for XXH64()
 */
XXH_FORCE_INLINE XXH_PUREF xxh_u64
XXH64_mergeAccs(const xxh_u64 *acc)
{
    XXH_ASSERT(acc != NULL);
    {
        xxh_u64 h64 = XXH_rotl64(acc[0], 1) + XXH_rotl64(acc[1], 7)
                    + XXH_rotl64(acc[2], 12) + XXH_rotl64(acc[3], 18);
        /* reroll on 32-bit */
        if (sizeof(void *) < sizeof(xxh_u64)) {
            size_t i;
            for (i = 0; i < 4; i++) {
                h64 = XXH64_mergeRound(h64, acc[i]);
            }
        } else {
            h64 = XXH64_mergeRound(h64, acc[0]);
            h64 = XXH64_mergeRound(h64, acc[1]);
            h64 = XXH64_mergeRound(h64, acc[2]);
            h64 = XXH64_mergeRound(h64, acc[3]);
        }
        return h64;
    }
}

/*!
 * @internal
 * @brief Processes the last 0-31 bytes of @p ptr.
 *
 * There may be up to 31 bytes remaining to consume from the input.
 * This final stage will digest them to ensure that all input bytes are present
 * in the final mix.
 *
 * @param hash The hash to finalize.
 * @param ptr The pointer to the remaining input.
 * @param len The remaining length, modulo 32.
 * @param align Whether @p ptr is aligned.
 * @return The finalized hash
 * @see XXH32_finalize().
 */
XXH_STATIC XXH_PUREF xxh_u64
XXH64_finalize(xxh_u64 hash, const xxh_u8* ptr, size_t len, XXH_alignment align)
{
    if (ptr==NULL) XXH_ASSERT(len == 0);
    len &= 31;
    while (len >= 8) {
        xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr));
        ptr += 8;
        hash ^= k1;
        hash  = XXH_rotl64(hash,27) * XXH_PRIME64_1 + XXH_PRIME64_4;
        len -= 8;
    }
    if (len >= 4) {
        hash ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1;
        ptr += 4;
        hash = XXH_rotl64(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3;
        len -= 4;
    }
    while (len > 0) {
        hash ^= (*ptr++) * XXH_PRIME64_5;
        hash = XXH_rotl64(hash, 11) * XXH_PRIME64_1;
        --len;
    }
    return  XXH64_avalanche(hash);
}

#ifdef XXH_OLD_NAMES
#  define PROCESS1_64 XXH_PROCESS1_64
#  define PROCESS4_64 XXH_PROCESS4_64
#  define PROCESS8_64 XXH_PROCESS8_64
#else
#  undef XXH_PROCESS1_64
#  undef XXH_PROCESS4_64
#  undef XXH_PROCESS8_64
#endif

/*!
 * @internal
 * @brief The implementation for @ref XXH64().
 *
 * @param input , len , seed Directly passed from @ref XXH64().
 * @param align Whether @p input is aligned.
 * @return The calculated hash.
 */
XXH_FORCE_INLINE XXH_PUREF xxh_u64
XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align)
{
    xxh_u64 h64;
    if (input==NULL) XXH_ASSERT(len == 0);

    if (len>=32) {  /* Process a large block of data */
        xxh_u64 acc[4];
        XXH64_initAccs(acc, seed);

        input = XXH64_consumeLong(acc, input, len, align);

        h64 = XXH64_mergeAccs(acc);
    } else {
        h64  = seed + XXH_PRIME64_5;
    }

    h64 += (xxh_u64) len;

    return XXH64_finalize(h64, input, len, align);
}


/*! @ingroup XXH64_family */
XXH_PUBLIC_API XXH64_hash_t XXH64 (XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed)
{
#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2
    /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
    XXH64_state_t state;
    XXH64_reset(&state, seed);
    XXH64_update(&state, (const xxh_u8*)input, len);
    return XXH64_digest(&state);
#else
    if (XXH_FORCE_ALIGN_CHECK) {
        if ((((size_t)input) & 7)==0) {  /* Input is aligned, let's leverage the speed advantage */
            return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned);
    }   }

    return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned);

#endif
}

/*******   Hash Streaming   *******/
#ifndef XXH_NO_STREAM
/*! @ingroup XXH64_family*/
XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
{
    return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
}
/*! @ingroup XXH64_family */
XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
{
    XXH_free(statePtr);
    return XXH_OK;
}

/*! @ingroup XXH64_family */
XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dstState, const XXH64_state_t* srcState)
{
    XXH_memcpy(dstState, srcState, sizeof(*dstState));
}

/*! @ingroup XXH64_family */
XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed)
{
    XXH_ASSERT(statePtr != NULL);
    XXH_memset(statePtr, 0, sizeof(*statePtr));
    XXH64_initAccs(statePtr->acc, seed);
    return XXH_OK;
}

/*! @ingroup XXH64_family */
XXH_PUBLIC_API XXH_errorcode
XXH64_update (XXH_NOESCAPE XXH64_state_t* state, XXH_NOESCAPE const void* input, size_t len)
{
    if (input==NULL) {
        XXH_ASSERT(len == 0);
        return XXH_OK;
    }

    state->total_len += len;

    XXH_ASSERT(state->bufferedSize <= sizeof(state->buffer));
    if (len < sizeof(state->buffer) - state->bufferedSize)  {   /* fill in tmp buffer */
        XXH_memcpy(state->buffer + state->bufferedSize, input, len);
        state->bufferedSize += (XXH32_hash_t)len;
        return XXH_OK;
    }

    {   const xxh_u8* xinput = (const xxh_u8*)input;
        const xxh_u8* const bEnd = xinput + len;

        if (state->bufferedSize) {   /* non-empty buffer => complete first */
            XXH_memcpy(state->buffer + state->bufferedSize, xinput, sizeof(state->buffer) - state->bufferedSize);
            xinput += sizeof(state->buffer) - state->bufferedSize;
            /* and process one round */
            (void)XXH64_consumeLong(state->acc, state->buffer, sizeof(state->buffer), XXH_aligned);
            state->bufferedSize = 0;
        }

        XXH_ASSERT(xinput <= bEnd);
        if ((size_t)(bEnd - xinput) >= sizeof(state->buffer)) {
            /* Process the remaining data */
            xinput = XXH64_consumeLong(state->acc, xinput, (size_t)(bEnd - xinput), XXH_unaligned);
        }

        if (xinput < bEnd) {
            /* Copy the leftover to the tmp buffer */
            XXH_memcpy(state->buffer, xinput, (size_t)(bEnd-xinput));
            state->bufferedSize = (unsigned)(bEnd-xinput);
        }
    }

    return XXH_OK;
}


/*! @ingroup XXH64_family */
XXH_PUBLIC_API XXH64_hash_t XXH64_digest(XXH_NOESCAPE const XXH64_state_t* state)
{
    xxh_u64 h64;

    if (state->total_len >= 32) {
        h64 = XXH64_mergeAccs(state->acc);
    } else {
        h64  = state->acc[2] /*seed*/ + XXH_PRIME64_5;
    }

    h64 += (xxh_u64) state->total_len;

    return XXH64_finalize(h64, state->buffer, (size_t)state->total_len, XXH_aligned);
}
#endif /* !XXH_NO_STREAM */

/******* Canonical representation   *******/

/*! @ingroup XXH64_family */
XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash)
{
    XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));
    if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
    XXH_memcpy(dst, &hash, sizeof(*dst));
}

/*! @ingroup XXH64_family */
XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src)
{
    return XXH_readBE64(src);
}

#ifndef XXH_NO_XXH3

/* *********************************************************************
*  XXH3
*  New generation hash designed for speed on small keys and vectorization
************************************************************************ */
/*!
 * @}
 * @defgroup XXH3_impl XXH3 implementation
 * @ingroup impl
 * @{
 */

/* ===   Compiler specifics   === */


#if (defined(__GNUC__) && (__GNUC__ >= 3))  \
  || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \
  || defined(__clang__)
#    define XXH_likely(x) __builtin_expect(x, 1)
#    define XXH_unlikely(x) __builtin_expect(x, 0)
#else
#    define XXH_likely(x) (x)
#    define XXH_unlikely(x) (x)
#endif

#ifndef XXH_HAS_INCLUDE
#  ifdef __has_include
/*
 * Not defined as XXH_HAS_INCLUDE(x) (function-like) because
 * this causes segfaults in Apple Clang 4.2 (on Mac OS X 10.7 Lion)
 */
#    define XXH_HAS_INCLUDE __has_include
#  else
#    define XXH_HAS_INCLUDE(x) 0
#  endif
#endif

#if defined(__GNUC__) || defined(__clang__)
#  if defined(__ARM_FEATURE_SVE)
#    include <arm_sve.h>
#  endif
#  if defined(__ARM_NEON__) || defined(__ARM_NEON) \
   || (defined(_M_ARM) && _M_ARM >= 7) \
   || defined(_M_ARM64) || defined(_M_ARM64EC) \
   || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE(<arm_neon.h>)) /* WASM SIMD128 via SIMDe */
#    define inline __inline__  /* circumvent a clang bug */
#    include <arm_neon.h>
#    undef inline
#  elif defined(__AVX2__)
#    include <immintrin.h>
#  elif defined(__SSE2__)
#    include <emmintrin.h>
#  elif defined(__loongarch_asx)
#    include <lasxintrin.h>
#    include <lsxintrin.h>
#  elif defined(__loongarch_sx)
#    include <lsxintrin.h>
#  elif defined(__riscv_vector)
#    include <riscv_vector.h>
#  endif
#endif

#if defined(_MSC_VER)
#  include <intrin.h>
#endif

/*
 * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while
 * remaining a true 64-bit/128-bit hash function.
 *
 * This is done by prioritizing a subset of 64-bit operations that can be
 * emulated without too many steps on the average 32-bit machine.
 *
 * For example, these two lines seem similar, and run equally fast on 64-bit:
 *
 *   xxh_u64 x;
 *   x ^= (x >> 47); // good
 *   x ^= (x >> 13); // bad
 *
 * However, to a 32-bit machine, there is a major difference.
 *
 * x ^= (x >> 47) looks like this:
 *
 *   x.lo ^= (x.hi >> (47 - 32));
 *
 * while x ^= (x >> 13) looks like this:
 *
 *   // note: funnel shifts are not usually cheap.
 *   x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13));
 *   x.hi ^= (x.hi >> 13);
 *
 * The first one is significantly faster than the second, simply because the
 * shift is larger than 32. This means:
 *  - All the bits we need are in the upper 32 bits, so we can ignore the lower
 *    32 bits in the shift.
 *  - The shift result will always fit in the lower 32 bits, and therefore,
 *    we can ignore the upper 32 bits in the xor.
 *
 * Thanks to this optimization, XXH3 only requires these features to be efficient:
 *
 *  - Usable unaligned access
 *  - A 32-bit or 64-bit ALU
 *      - If 32-bit, a decent ADC instruction
 *  - A 32 or 64-bit multiply with a 64-bit result
 *  - For the 128-bit variant, a decent byteswap helps short inputs.
 *
 * The first two are already required by XXH32, and almost all 32-bit and 64-bit
 * platforms which can run XXH32 can run XXH3 efficiently.
 *
 * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one
 * notable exception.
 *
 * First of all, Thumb-1 lacks support for the UMULL instruction which
 * performs the important long multiply. This means numerous __aeabi_lmul
 * calls.
 *
 * Second of all, the 8 functional registers are just not enough.
 * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need
 * Lo registers, and this shuffling results in thousands more MOVs than A32.
 *
 * A32 and T32 don't have this limitation. They can access all 14 registers,
 * do a 32->64 multiply with UMULL, and the flexible operand allowing free
 * shifts is helpful, too.
 *
 * Therefore, we do a quick sanity check.
 *
 * If compiling Thumb-1 for a target which supports ARM instructions, we will
 * emit a warning, as it is not a "sane" platform to compile for.
 *
 * Usually, if this happens, it is because of an accident and you probably need
 * to specify -march, as you likely meant to compile for a newer architecture.
 *
 * Credit: large sections of the vectorial and asm source code paths
 *         have been contributed by @easyaspi314
 */
#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM)
#   warning "XXH3 is highly inefficient without ARM or Thumb-2."
#endif

/* ==========================================
 * Vectorization detection
 * ========================================== */

#ifdef XXH_DOXYGEN
/*!
 * @ingroup tuning
 * @brief Overrides the vectorization implementation chosen for XXH3.
 *
 * Can be defined to 0 to disable SIMD,
 * or any other authorized value of @ref XXH_VECTOR.
 *
 * If this is not defined, it uses predefined macros to determine the best
 * implementation.
 */
#  define XXH_VECTOR XXH_SCALAR
/*!
 * @ingroup tuning
 * @brief Selects the minimum alignment for XXH3's accumulators.
 *
 * When using SIMD, this should match the alignment required for said vector
 * type, so, for example, 32 for AVX2.
 *
 * Default: Auto detected.
 */
#  define XXH_ACC_ALIGN 8
#endif

/* Actual definition */
#ifndef XXH_DOXYGEN
#endif

#ifndef XXH_VECTOR    /* can be defined on command line */
#  if ( \
        defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \
     || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \
     || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE(<arm_neon.h>)) /* wasm simd128 via SIMDe */ \
   ) && ( \
        defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \
    || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \
   )
#    define XXH_VECTOR XXH_NEON
#  elif defined(__ARM_FEATURE_SVE)
#    define XXH_VECTOR XXH_SVE
#  elif defined(__AVX512F__)
#    define XXH_VECTOR XXH_AVX512
#  elif defined(__AVX2__)
#    define XXH_VECTOR XXH_AVX2
#  elif defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2))
#    define XXH_VECTOR XXH_SSE2
#  elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \
     || (defined(__s390x__) && defined(__VEC__)) \
     && defined(__GNUC__) /* TODO: IBM XL */
#    define XXH_VECTOR XXH_VSX
#  elif defined(__loongarch_asx)
#    define XXH_VECTOR XXH_LASX
#  elif defined(__loongarch_sx)
#    define XXH_VECTOR XXH_LSX
#  elif defined(__riscv_vector)
#    define XXH_VECTOR XXH_RVV
#  else
#    define XXH_VECTOR XXH_SCALAR
#  endif
#endif

/* __ARM_FEATURE_SVE is only supported by GCC & Clang. */
#if (XXH_VECTOR == XXH_SVE) && !defined(__ARM_FEATURE_SVE)
#  ifdef _MSC_VER
#    pragma warning(once : 4606)
#  else
#    warning "__ARM_FEATURE_SVE isn't supported. Use SCALAR instead."
#  endif
#  undef XXH_VECTOR
#  define XXH_VECTOR XXH_SCALAR
#endif

/*
 * Controls the alignment of the accumulator,
 * for compatibility with aligned vector loads, which are usually faster.
 */
#ifndef XXH_ACC_ALIGN
#  if defined(XXH_X86DISPATCH)
#     define XXH_ACC_ALIGN 64  /* for compatibility with avx512 */
#  elif XXH_VECTOR == XXH_SCALAR  /* scalar */
#     define XXH_ACC_ALIGN 8
#  elif XXH_VECTOR == XXH_SSE2  /* sse2 */
#     define XXH_ACC_ALIGN 16
#  elif XXH_VECTOR == XXH_AVX2  /* avx2 */
#     define XXH_ACC_ALIGN 32
#  elif XXH_VECTOR == XXH_NEON  /* neon */
#     define XXH_ACC_ALIGN 16
#  elif XXH_VECTOR == XXH_VSX   /* vsx */
#     define XXH_ACC_ALIGN 16
#  elif XXH_VECTOR == XXH_AVX512  /* avx512 */
#     define XXH_ACC_ALIGN 64
#  elif XXH_VECTOR == XXH_SVE   /* sve */
#     define XXH_ACC_ALIGN 64
#  elif XXH_VECTOR == XXH_LASX   /* lasx */
#     define XXH_ACC_ALIGN 64
#  elif XXH_VECTOR == XXH_LSX   /* lsx */
#     define XXH_ACC_ALIGN 64
#  elif XXH_VECTOR == XXH_RVV   /* rvv */
#     define XXH_ACC_ALIGN 64   /* could be 8, but 64 may be faster */
#  endif
#endif

#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \
    || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512
#  define XXH_SEC_ALIGN XXH_ACC_ALIGN
#elif XXH_VECTOR == XXH_SVE
#  define XXH_SEC_ALIGN XXH_ACC_ALIGN
#elif XXH_VECTOR == XXH_RVV
#  define XXH_SEC_ALIGN XXH_ACC_ALIGN
#else
#  define XXH_SEC_ALIGN 8
#endif

#if defined(__GNUC__) || defined(__clang__)
#  define XXH_ALIASING __attribute__((__may_alias__))
#else
#  define XXH_ALIASING /* nothing */
#endif

/*
 * UGLY HACK:
 * GCC usually generates the best code with -O3 for xxHash.
 *
 * However, when targeting AVX2, it is overzealous in its unrolling resulting
 * in code roughly 3/4 the speed of Clang.
 *
 * There are other issues, such as GCC splitting _mm256_loadu_si256 into
 * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which
 * only applies to Sandy and Ivy Bridge... which don't even support AVX2.
 *
 * That is why when compiling the AVX2 version, it is recommended to use either
 *   -O2 -mavx2 -march=haswell
 * or
 *   -O2 -mavx2 -mno-avx256-split-unaligned-load
 * for decent performance, or to use Clang instead.
 *
 * Fortunately, we can control the first one with a pragma that forces GCC into
 * -O2, but the other one we can't control without "failed to inline always
 * inline function due to target mismatch" warnings.
 */
#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \
  && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
  && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */
#  pragma GCC push_options
#  pragma GCC optimize("-O2")
#endif

#if XXH_VECTOR == XXH_NEON

/*
 * UGLY HACK: While AArch64 GCC on Linux does not seem to care, on macOS, GCC -O3
 * optimizes out the entire hashLong loop because of the aliasing violation.
 *
 * However, GCC is also inefficient at load-store optimization with vld1q/vst1q,
 * so the only option is to mark it as aliasing.
 */
typedef uint64x2_t xxh_aliasing_uint64x2_t XXH_ALIASING;

/*!
 * @internal
 * @brief `vld1q_u64` but faster and alignment-safe.
 *
 * On AArch64, unaligned access is always safe, but on ARMv7-a, it is only
 * *conditionally* safe (`vld1` has an alignment bit like `movdq[ua]` in x86).
 *
 * GCC for AArch64 sees `vld1q_u8` as an intrinsic instead of a load, so it
 * prohibits load-store optimizations. Therefore, a direct dereference is used.
 *
 * Otherwise, `vld1q_u8` is used with `vreinterpretq_u8_u64` to do a safe
 * unaligned load.
 */
#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__)
XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) /* silence -Wcast-align */
{
    return *(xxh_aliasing_uint64x2_t const *)ptr;
}
#else
XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr)
{
    return vreinterpretq_u64_u8(vld1q_u8((uint8_t const*)ptr));
}
#endif

/*!
 * @internal
 * @brief `vmlal_u32` on low and high halves of a vector.
 *
 * This is a workaround for AArch64 GCC < 11 which implemented arm_neon.h with
 * inline assembly and were therefore incapable of merging the `vget_{low, high}_u32`
 * with `vmlal_u32`.
 */
#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 11
XXH_FORCE_INLINE uint64x2_t
XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
{
    /* Inline assembly is the only way */
    __asm__("umlal   %0.2d, %1.2s, %2.2s" : "+w" (acc) : "w" (lhs), "w" (rhs));
    return acc;
}
XXH_FORCE_INLINE uint64x2_t
XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
{
    /* This intrinsic works as expected */
    return vmlal_high_u32(acc, lhs, rhs);
}
#else
/* Portable intrinsic versions */
XXH_FORCE_INLINE uint64x2_t
XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
{
    return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs));
}
/*! @copydoc XXH_vmlal_low_u32
 * Assume the compiler converts this to vmlal_high_u32 on aarch64 */
XXH_FORCE_INLINE uint64x2_t
XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
{
    return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs));
}
#endif

/*!
 * @ingroup tuning
 * @brief Controls the NEON to scalar ratio for XXH3
 *
 * This can be set to 2, 4, 6, or 8.
 *
 * ARM Cortex CPUs are _very_ sensitive to how their pipelines are used.
 *
 * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but only 2 of those
 * can be NEON. If you are only using NEON instructions, you are only using 2/3 of the CPU
 * bandwidth.
 *
 * This is even more noticeable on the more advanced cores like the Cortex-A76 which
 * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once.
 *
 * Therefore, to make the most out of the pipeline, it is beneficial to run 6 NEON lanes
 * and 2 scalar lanes, which is chosen by default.
 *
 * This does not apply to Apple processors or 32-bit processors, which run better with
 * full NEON. These will default to 8. Additionally, size-optimized builds run 8 lanes.
 *
 * This change benefits CPUs with large micro-op buffers without negatively affecting
 * most other CPUs:
 *
 *  | Chipset               | Dispatch type       | NEON only | 6:2 hybrid | Diff. |
 *  |:----------------------|:--------------------|----------:|-----------:|------:|
 *  | Snapdragon 730 (A76)  | 2 NEON/8 micro-ops  |  8.8 GB/s |  10.1 GB/s |  ~16% |
 *  | Snapdragon 835 (A73)  | 2 NEON/3 micro-ops  |  5.1 GB/s |   5.3 GB/s |   ~5% |
 *  | Marvell PXA1928 (A53) | In-order dual-issue |  1.9 GB/s |   1.9 GB/s |    0% |
 *  | Apple M1              | 4 NEON/8 micro-ops  | 37.3 GB/s |  36.1 GB/s |  ~-3% |
 *
 * It also seems to fix some bad codegen on GCC, making it almost as fast as clang.
 *
 * When using WASM SIMD128, if this is 2 or 6, SIMDe will scalarize 2 of the lanes meaning
 * it effectively becomes worse 4.
 *
 * @see XXH3_accumulate_512_neon()
 */
# ifndef XXH3_NEON_LANES
#  if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \
   && !defined(__APPLE__) && XXH_SIZE_OPT <= 0
#   define XXH3_NEON_LANES 6
#  else
#   define XXH3_NEON_LANES XXH_ACC_NB
#  endif
# endif
#endif  /* XXH_VECTOR == XXH_NEON */

/*
 * VSX and Z Vector helpers.
 *
 * This is very messy, and any pull requests to clean this up are welcome.
 *
 * There are a lot of problems with supporting VSX and s390x, due to
 * inconsistent intrinsics, spotty coverage, and multiple endiannesses.
 */
#if XXH_VECTOR == XXH_VSX
/* Annoyingly, these headers _may_ define three macros: `bool`, `vector`,
 * and `pixel`. This is a problem for obvious reasons.
 *
 * These keywords are unnecessary; the spec literally says they are
 * equivalent to `__bool`, `__vector`, and `__pixel` and may be undef'd
 * after including the header.
 *
 * We use pragma push_macro/pop_macro to keep the namespace clean. */
#  pragma push_macro("bool")
#  pragma push_macro("vector")
#  pragma push_macro("pixel")
/* silence potential macro redefined warnings */
#  undef bool
#  undef vector
#  undef pixel

#  if defined(__s390x__)
#    include <s390intrin.h>
#  else
#    include <altivec.h>
#  endif

/* Restore the original macro values, if applicable. */
#  pragma pop_macro("pixel")
#  pragma pop_macro("vector")
#  pragma pop_macro("bool")

typedef __vector unsigned long long xxh_u64x2;
typedef __vector unsigned char xxh_u8x16;
typedef __vector unsigned xxh_u32x4;

/*
 * UGLY HACK: Similar to aarch64 macOS GCC, s390x GCC has the same aliasing issue.
 */
typedef xxh_u64x2 xxh_aliasing_u64x2 XXH_ALIASING;

# ifndef XXH_VSX_BE
#  if defined(__BIG_ENDIAN__) \
  || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
#    define XXH_VSX_BE 1
#  elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__
#    warning "-maltivec=be is not recommended. Please use native endianness."
#    define XXH_VSX_BE 1
#  else
#    define XXH_VSX_BE 0
#  endif
# endif /* !defined(XXH_VSX_BE) */

# if XXH_VSX_BE
#  if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__))
#    define XXH_vec_revb vec_revb
#  else
/*!
 * A polyfill for POWER9's vec_revb().
 */
XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val)
{
    xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00,
                                  0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 };
    return vec_perm(val, val, vByteSwap);
}
#  endif
# endif /* XXH_VSX_BE */

/*!
 * Performs an unaligned vector load and byte swaps it on big endian.
 */
XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr)
{
    xxh_u64x2 ret;
    XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2));
# if XXH_VSX_BE
    ret = XXH_vec_revb(ret);
# endif
    return ret;
}

/*
 * vec_mulo and vec_mule are very problematic intrinsics on PowerPC
 *
 * These intrinsics weren't added until GCC 8, despite existing for a while,
 * and they are endian dependent. Also, their meaning swap depending on version.
 * */
# if defined(__s390x__)
 /* s390x is always big endian, no issue on this platform */
#  define XXH_vec_mulo vec_mulo
#  define XXH_vec_mule vec_mule
# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) && !defined(__ibmxl__)
/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */
 /* The IBM XL Compiler (which defined __clang__) only implements the vec_* operations */
#  define XXH_vec_mulo __builtin_altivec_vmulouw
#  define XXH_vec_mule __builtin_altivec_vmuleuw
# else
/* gcc needs inline assembly */
/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */
XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b)
{
    xxh_u64x2 result;
    __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
    return result;
}
XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b)
{
    xxh_u64x2 result;
    __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
    return result;
}
# endif /* XXH_vec_mulo, XXH_vec_mule */
#endif /* XXH_VECTOR == XXH_VSX */

#if XXH_VECTOR == XXH_SVE
#define ACCRND(acc, offset) \
do { \
    svuint64_t input_vec = svld1_u64(mask, xinput + offset);         \
    svuint64_t secret_vec = svld1_u64(mask, xsecret + offset);       \
    svuint64_t mixed = sveor_u64_x(mask, secret_vec, input_vec);     \
    svuint64_t swapped = svtbl_u64(input_vec, kSwap);                \
    svuint64_t mixed_lo = svextw_u64_x(mask, mixed);                 \
    svuint64_t mixed_hi = svlsr_n_u64_x(mask, mixed, 32);            \
    svuint64_t mul = svmad_u64_x(mask, mixed_lo, mixed_hi, swapped); \
    acc = svadd_u64_x(mask, acc, mul);                               \
} while (0)
#endif /* XXH_VECTOR == XXH_SVE */

/* prefetch
 * can be disabled, by declaring XXH_NO_PREFETCH build macro */
#if defined(XXH_NO_PREFETCH)
#  define XXH_PREFETCH(ptr)  (void)(ptr)  /* disabled */
#else
#  if XXH_SIZE_OPT >= 1
#    define XXH_PREFETCH(ptr) (void)(ptr)
#  elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))  /* _mm_prefetch() not defined outside of x86/x64 */
#    include <mmintrin.h>   /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
#    define XXH_PREFETCH(ptr)  _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
#  elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
#    define XXH_PREFETCH(ptr)  __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
#  else
#    define XXH_PREFETCH(ptr) (void)(ptr)  /* disabled */
#  endif
#endif  /* XXH_NO_PREFETCH */


/* ==========================================
 * XXH3 default settings
 * ========================================== */

#define XXH_SECRET_DEFAULT_SIZE 192   /* minimum XXH3_SECRET_SIZE_MIN */

#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN)
#  error "default keyset is not large enough"
#endif

/*!
 * @internal
 * @def XXH3_kSecret
 * @brief Pseudorandom secret taken directly from FARSH. */
XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = {
    0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,
    0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,
    0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21,
    0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c,
    0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3,
    0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8,
    0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d,
    0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64,
    0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb,
    0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e,
    0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce,
    0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e,
};

static const xxh_u64 PRIME_MX1 = 0x165667919E3779F9ULL;  /*!< 0b0001011001010110011001111001000110011110001101110111100111111001 */
static const xxh_u64 PRIME_MX2 = 0x9FB21C651E98DF25ULL;  /*!< 0b1001111110110010000111000110010100011110100110001101111100100101 */

#ifdef XXH_OLD_NAMES
#  define kSecret XXH3_kSecret
#endif

#ifdef XXH_DOXYGEN
/*!
 * @brief Calculates a 32-bit to 64-bit long multiply.
 *
 * Implemented as a macro.
 *
 * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't
 * need to (but it shouldn't need to anyways, it is about 7 instructions to do
 * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we
 * use that instead of the normal method.
 *
 * If you are compiling for platforms like Thumb-1 and don't have a better option,
 * you may also want to write your own long multiply routine here.
 *
 * @param x, y Numbers to be multiplied
 * @return 64-bit product of the low 32 bits of @p x and @p y.
 */
XXH_FORCE_INLINE xxh_u64
XXH_mult32to64(xxh_u64 x, xxh_u64 y)
{
   return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF);
}
#elif defined(_MSC_VER) && defined(_M_IX86)
#    define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y))
#else
/*
 * Downcast + upcast is usually better than masking on older compilers like
 * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers.
 *
 * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands
 * and perform a full 64x64 multiply -- entirely redundant on 32-bit.
 */
#    define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y))
#endif

/*!
 * @brief Calculates a 64->128-bit long multiply.
 *
 * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar
 * version.
 *
 * @param lhs , rhs The 64-bit integers to be multiplied
 * @return The 128-bit result represented in an @ref XXH128_hash_t.
 */
static XXH128_hash_t
XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs)
{
    /*
     * GCC/Clang __uint128_t method.
     *
     * On most 64-bit targets, GCC and Clang define a __uint128_t type.
     * This is usually the best way as it usually uses a native long 64-bit
     * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64.
     *
     * Usually.
     *
     * Despite being a 32-bit platform, Clang (and emscripten) define this type
     * despite not having the arithmetic for it. This results in a laggy
     * compiler builtin call which calculates a full 128-bit multiply.
     * In that case it is best to use the portable one.
     * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677
     */
#if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \
    && defined(__SIZEOF_INT128__) \
    || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)

    __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs;
    XXH128_hash_t r128;
    r128.low64  = (xxh_u64)(product);
    r128.high64 = (xxh_u64)(product >> 64);
    return r128;

    /*
     * MSVC for x64's _umul128 method.
     *
     * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct);
     *
     * This compiles to single operand MUL on x64.
     */
#elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC)

#ifndef _MSC_VER
#   pragma intrinsic(_umul128)
#endif
    xxh_u64 product_high;
    xxh_u64 const product_low = _umul128(lhs, rhs, &product_high);
    XXH128_hash_t r128;
    r128.low64  = product_low;
    r128.high64 = product_high;
    return r128;

    /*
     * MSVC for ARM64's __umulh method.
     *
     * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method.
     */
#elif defined(_M_ARM64) || defined(_M_ARM64EC)

#ifndef _MSC_VER
#   pragma intrinsic(__umulh)
#endif
    XXH128_hash_t r128;
    r128.low64  = lhs * rhs;
    r128.high64 = __umulh(lhs, rhs);
    return r128;

#else
    /*
     * Portable scalar method. Optimized for 32-bit and 64-bit ALUs.
     *
     * This is a fast and simple grade school multiply, which is shown below
     * with base 10 arithmetic instead of base 0x100000000.
     *
     *           9 3 // D2 lhs = 93
     *         x 7 5 // D2 rhs = 75
     *     ----------
     *           1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15
     *         4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45
     *         2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21
     *     + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63
     *     ---------
     *         2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27
     *     + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67
     *     ---------
     *       6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975
     *
     * The reasons for adding the products like this are:
     *  1. It avoids manual carry tracking. Just like how
     *     (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX.
     *     This avoids a lot of complexity.
     *
     *  2. It hints for, and on Clang, compiles to, the powerful UMAAL
     *     instruction available in ARM's Digital Signal Processing extension
     *     in 32-bit ARMv6 and later, which is shown below:
     *
     *         void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm)
     *         {
     *             xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm;
     *             *RdLo = (xxh_u32)(product & 0xFFFFFFFF);
     *             *RdHi = (xxh_u32)(product >> 32);
     *         }
     *
     *     This instruction was designed for efficient long multiplication, and
     *     allows this to be calculated in only 4 instructions at speeds
     *     comparable to some 64-bit ALUs.
     *
     *  3. It isn't terrible on other platforms. Usually this will be a couple
     *     of 32-bit ADD/ADCs.
     */

    /* First calculate all of the cross products. */
    xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF);
    xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32,        rhs & 0xFFFFFFFF);
    xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32);
    xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32,        rhs >> 32);

    /* Now add the products together. These will never overflow. */
    xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi;
    xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32)        + hi_hi;
    xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF);

    XXH128_hash_t r128;
    r128.low64  = lower;
    r128.high64 = upper;
    return r128;
#endif
}

/*!
 * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it.
 *
 * The reason for the separate function is to prevent passing too many structs
 * around by value. This will hopefully inline the multiply, but we don't force it.
 *
 * @param lhs , rhs The 64-bit integers to multiply
 * @return The low 64 bits of the product XOR'd by the high 64 bits.
 * @see XXH_mult64to128()
 */
static xxh_u64
XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs)
{
    XXH128_hash_t product = XXH_mult64to128(lhs, rhs);
    return product.low64 ^ product.high64;
}

/*! Seems to produce slightly better code on GCC for some reason. */
XXH_FORCE_INLINE XXH_CONSTF xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift)
{
    XXH_ASSERT(0 <= shift && shift < 64);
    return v64 ^ (v64 >> shift);
}

/*
 * This is a fast avalanche stage,
 * suitable when input bits are already partially mixed
 */
static XXH64_hash_t XXH3_avalanche(xxh_u64 h64)
{
    h64 = XXH_xorshift64(h64, 37);
    h64 *= PRIME_MX1;
    h64 = XXH_xorshift64(h64, 32);
    return h64;
}

/*
 * This is a stronger avalanche,
 * inspired by Pelle Evensen's rrmxmx
 * preferable when input has not been previously mixed
 */
static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len)
{
    /* this mix is inspired by Pelle Evensen's rrmxmx */
    h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24);
    h64 *= PRIME_MX2;
    h64 ^= (h64 >> 35) + len ;
    h64 *= PRIME_MX2;
    return XXH_xorshift64(h64, 28);
}


/* ==========================================
 * Short keys
 * ==========================================
 * One of the shortcomings of XXH32 and XXH64 was that their performance was
 * sub-optimal on short lengths. It used an iterative algorithm which strongly
 * favored lengths that were a multiple of 4 or 8.
 *
 * Instead of iterating over individual inputs, we use a set of single shot
 * functions which piece together a range of lengths and operate in constant time.
 *
 * Additionally, the number of multiplies has been significantly reduced. This
 * reduces latency, especially when emulating 64-bit multiplies on 32-bit.
 *
 * Depending on the platform, this may or may not be faster than XXH32, but it
 * is almost guaranteed to be faster than XXH64.
 */

/*
 * At very short lengths, there isn't enough input to fully hide secrets, or use
 * the entire secret.
 *
 * There is also only a limited amount of mixing we can do before significantly
 * impacting performance.
 *
 * Therefore, we use different sections of the secret and always mix two secret
 * samples with an XOR. This should have no effect on performance on the
 * seedless or withSeed variants because everything _should_ be constant folded
 * by modern compilers.
 *
 * The XOR mixing hides individual parts of the secret and increases entropy.
 *
 * This adds an extra layer of strength for custom secrets.
 */
XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
{
    XXH_ASSERT(input != NULL);
    XXH_ASSERT(1 <= len && len <= 3);
    XXH_ASSERT(secret != NULL);
    /*
     * len = 1: combined = { input[0], 0x01, input[0], input[0] }
     * len = 2: combined = { input[1], 0x02, input[0], input[1] }
     * len = 3: combined = { input[2], 0x03, input[0], input[1] }
     */
    {   xxh_u8  const c1 = input[0];
        xxh_u8  const c2 = input[len >> 1];
        xxh_u8  const c3 = input[len - 1];
        xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2  << 24)
                               | ((xxh_u32)c3 <<  0) | ((xxh_u32)len << 8);
        xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed;
        xxh_u64 const keyed = (xxh_u64)combined ^ bitflip;
        return XXH64_avalanche(keyed);
    }
}

XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
{
    XXH_ASSERT(input != NULL);
    XXH_ASSERT(secret != NULL);
    XXH_ASSERT(4 <= len && len <= 8);
    seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32;
    {   xxh_u32 const input1 = XXH_readLE32(input);
        xxh_u32 const input2 = XXH_readLE32(input + len - 4);
        xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed;
        xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32);
        xxh_u64 const keyed = input64 ^ bitflip;
        return XXH3_rrmxmx(keyed, len);
    }
}

XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
{
    XXH_ASSERT(input != NULL);
    XXH_ASSERT(secret != NULL);
    XXH_ASSERT(9 <= len && len <= 16);
    {   xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed;
        xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed;
        xxh_u64 const input_lo = XXH_readLE64(input)           ^ bitflip1;
        xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2;
        xxh_u64 const acc = len
                          + XXH_swap64(input_lo) + input_hi
                          + XXH3_mul128_fold64(input_lo, input_hi);
        return XXH3_avalanche(acc);
    }
}

XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
{
    XXH_ASSERT(len <= 16);
    {   if (XXH_likely(len >  8)) return XXH3_len_9to16_64b(input, len, secret, seed);
        if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed);
        if (len) return XXH3_len_1to3_64b(input, len, secret, seed);
        return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64)));
    }
}

/*
 * DISCLAIMER: There are known *seed-dependent* multicollisions here due to
 * multiplication by zero, affecting hashes of lengths 17 to 240.
 *
 * However, they are very unlikely.
 *
 * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all
 * unseeded non-cryptographic hashes, it does not attempt to defend itself
 * against specially crafted inputs, only random inputs.
 *
 * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes
 * cancelling out the secret is taken an arbitrary number of times (addressed
 * in XXH3_accumulate_512), this collision is very unlikely with random inputs
 * and/or proper seeding:
 *
 * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a
 * function that is only called up to 16 times per hash with up to 240 bytes of
 * input.
 *
 * This is not too bad for a non-cryptographic hash function, especially with
 * only 64 bit outputs.
 *
 * The 128-bit variant (which trades some speed for strength) is NOT affected
 * by this, although it is always a good idea to use a proper seed if you care
 * about strength.
 */
XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input,
                                     const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64)
{
#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
  && defined(__i386__) && defined(__SSE2__)  /* x86 + SSE2 */ \
  && !defined(XXH_ENABLE_AUTOVECTORIZE)      /* Define to disable like XXH32 hack */
    /*
     * UGLY HACK:
     * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in
     * slower code.
     *
     * By forcing seed64 into a register, we disrupt the cost model and
     * cause it to scalarize. See `XXH32_round()`
     *
     * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600,
     * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on
     * GCC 9.2, despite both emitting scalar code.
     *
     * GCC generates much better scalar code than Clang for the rest of XXH3,
     * which is why finding a more optimal codepath is an interest.
     */
    XXH_COMPILER_GUARD(seed64);
#endif
    {   xxh_u64 const input_lo = XXH_readLE64(input);
        xxh_u64 const input_hi = XXH_readLE64(input+8);
        return XXH3_mul128_fold64(
            input_lo ^ (XXH_readLE64(secret)   + seed64),
            input_hi ^ (XXH_readLE64(secret+8) - seed64)
        );
    }
}

/* For mid range keys, XXH3 uses a Mum-hash variant. */
XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
                     const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
                     XXH64_hash_t seed)
{
    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
    XXH_ASSERT(16 < len && len <= 128);

    {   xxh_u64 acc = len * XXH_PRIME64_1;
#if XXH_SIZE_OPT >= 1
        /* Smaller and cleaner, but slightly slower. */
        unsigned int i = (unsigned int)(len - 1) / 32;
        do {
            acc += XXH3_mix16B(input+16 * i, secret+32*i, seed);
            acc += XXH3_mix16B(input+len-16*(i+1), secret+32*i+16, seed);
        } while (i-- != 0);
#else
        if (len > 32) {
            if (len > 64) {
                if (len > 96) {
                    acc += XXH3_mix16B(input+48, secret+96, seed);
                    acc += XXH3_mix16B(input+len-64, secret+112, seed);
                }
                acc += XXH3_mix16B(input+32, secret+64, seed);
                acc += XXH3_mix16B(input+len-48, secret+80, seed);
            }
            acc += XXH3_mix16B(input+16, secret+32, seed);
            acc += XXH3_mix16B(input+len-32, secret+48, seed);
        }
        acc += XXH3_mix16B(input+0, secret+0, seed);
        acc += XXH3_mix16B(input+len-16, secret+16, seed);
#endif
        return XXH3_avalanche(acc);
    }
}

XXH_NO_INLINE XXH_PUREF XXH64_hash_t
XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
                      const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
                      XXH64_hash_t seed)
{
    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
    XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);

    #define XXH3_MIDSIZE_STARTOFFSET 3
    #define XXH3_MIDSIZE_LASTOFFSET  17

    {   xxh_u64 acc = len * XXH_PRIME64_1;
        xxh_u64 acc_end;
        unsigned int const nbRounds = (unsigned int)len / 16;
        unsigned int i;
        XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
        for (i=0; i<8; i++) {
            acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed);
        }
        /* last bytes */
        acc_end = XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed);
        XXH_ASSERT(nbRounds >= 8);
        acc = XXH3_avalanche(acc);
#if defined(__clang__)                                /* Clang */ \
    && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \
    && !defined(XXH_ENABLE_AUTOVECTORIZE)             /* Define to disable */
        /*
         * UGLY HACK:
         * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86.
         * In everywhere else, it uses scalar code.
         *
         * For 64->128-bit multiplies, even if the NEON was 100% optimal, it
         * would still be slower than UMAAL (see XXH_mult64to128).
         *
         * Unfortunately, Clang doesn't handle the long multiplies properly and
         * converts them to the nonexistent "vmulq_u64" intrinsic, which is then
         * scalarized into an ugly mess of VMOV.32 instructions.
         *
         * This mess is difficult to avoid without turning autovectorization
         * off completely, but they are usually relatively minor and/or not
         * worth it to fix.
         *
         * This loop is the easiest to fix, as unlike XXH32, this pragma
         * _actually works_ because it is a loop vectorization instead of an
         * SLP vectorization.
         */
        #pragma clang loop vectorize(disable)
#endif
        for (i=8 ; i < nbRounds; i++) {
            /*
             * Prevents clang for unrolling the acc loop and interleaving with this one.
             */
            XXH_COMPILER_GUARD(acc);
            acc_end += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed);
        }
        return XXH3_avalanche(acc + acc_end);
    }
}


/* =======     Long Keys     ======= */

#define XXH_STRIPE_LEN 64
#define XXH_SECRET_CONSUME_RATE 8   /* nb of secret bytes consumed at each accumulation */
#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64))

#ifdef XXH_OLD_NAMES
#  define STRIPE_LEN XXH_STRIPE_LEN
#  define ACC_NB XXH_ACC_NB
#endif

#ifndef XXH_PREFETCH_DIST
#  ifdef __clang__
#    define XXH_PREFETCH_DIST 320
#  else
#    if (XXH_VECTOR == XXH_AVX512)
#      define XXH_PREFETCH_DIST 512
#    else
#      define XXH_PREFETCH_DIST 384
#    endif
#  endif  /* __clang__ */
#endif  /* XXH_PREFETCH_DIST */

/*
 * These macros are to generate an XXH3_accumulate() function.
 * The two arguments select the name suffix and target attribute.
 *
 * The name of this symbol is XXH3_accumulate_<name>() and it calls
 * XXH3_accumulate_512_<name>().
 *
 * It may be useful to hand implement this function if the compiler fails to
 * optimize the inline function.
 */
#define XXH3_ACCUMULATE_TEMPLATE(name)                      \
void                                                        \
XXH3_accumulate_##name(xxh_u64* XXH_RESTRICT acc,           \
                       const xxh_u8* XXH_RESTRICT input,    \
                       const xxh_u8* XXH_RESTRICT secret,   \
                       size_t nbStripes)                    \
{                                                           \
    size_t n;                                               \
    for (n = 0; n < nbStripes; n++ ) {                      \
        const xxh_u8* const in = input + n*XXH_STRIPE_LEN;  \
        XXH_PREFETCH(in + XXH_PREFETCH_DIST);               \
        XXH3_accumulate_512_##name(                         \
                 acc,                                       \
                 in,                                        \
                 secret + n*XXH_SECRET_CONSUME_RATE);       \
    }                                                       \
}


XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64)
{
    if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64);
    XXH_memcpy(dst, &v64, sizeof(v64));
}

/* Several intrinsic functions below are supposed to accept __int64 as argument,
 * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ .
 * However, several environments do not define __int64 type,
 * requiring a workaround.
 */
#if !defined (__VMS) \
  && (defined (__cplusplus) \
  || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
    typedef int64_t xxh_i64;
#else
    /* the following type must have a width of 64-bit */
    typedef long long xxh_i64;
#endif


/*
 * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized.
 *
 * It is a hardened version of UMAC, based off of FARSH's implementation.
 *
 * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD
 * implementations, and it is ridiculously fast.
 *
 * We harden it by mixing the original input to the accumulators as well as the product.
 *
 * This means that in the (relatively likely) case of a multiply by zero, the
 * original input is preserved.
 *
 * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve
 * cross-pollination, as otherwise the upper and lower halves would be
 * essentially independent.
 *
 * This doesn't matter on 64-bit hashes since they all get merged together in
 * the end, so we skip the extra step.
 *
 * Both XXH3_64bits and XXH3_128bits use this subroutine.
 */

#if (XXH_VECTOR == XXH_AVX512) \
     || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0)

#ifndef XXH_TARGET_AVX512
# define XXH_TARGET_AVX512  /* disable attribute target */
#endif

XXH_FORCE_INLINE XXH_TARGET_AVX512 void
XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc,
                     const void* XXH_RESTRICT input,
                     const void* XXH_RESTRICT secret)
{
    __m512i* const xacc = (__m512i *) acc;
    XXH_ASSERT((((size_t)acc) & 63) == 0);
    XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i));

    {
        /* data_vec    = input[0]; */
        __m512i const data_vec    = _mm512_loadu_si512   (input);
        /* key_vec     = secret[0]; */
        __m512i const key_vec     = _mm512_loadu_si512   (secret);
        /* data_key    = data_vec ^ key_vec; */
        __m512i const data_key    = _mm512_xor_si512     (data_vec, key_vec);
        /* data_key_lo = data_key >> 32; */
        __m512i const data_key_lo = _mm512_srli_epi64 (data_key, 32);
        /* product     = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
        __m512i const product     = _mm512_mul_epu32     (data_key, data_key_lo);
        /* xacc[0] += swap(data_vec); */
        __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2));
        __m512i const sum       = _mm512_add_epi64(*xacc, data_swap);
        /* xacc[0] += product; */
        *xacc = _mm512_add_epi64(product, sum);
    }
}
XXH_FORCE_INLINE XXH_TARGET_AVX512 XXH3_ACCUMULATE_TEMPLATE(avx512)

/*
 * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing.
 *
 * Multiplication isn't perfect, as explained by Google in HighwayHash:
 *
 *  // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to
 *  // varying degrees. In descending order of goodness, bytes
 *  // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32.
 *  // As expected, the upper and lower bytes are much worse.
 *
 * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291
 *
 * Since our algorithm uses a pseudorandom secret to add some variance into the
 * mix, we don't need to (or want to) mix as often or as much as HighwayHash does.
 *
 * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid
 * extraction.
 *
 * Both XXH3_64bits and XXH3_128bits use this subroutine.
 */

XXH_FORCE_INLINE XXH_TARGET_AVX512 void
XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
{
    XXH_ASSERT((((size_t)acc) & 63) == 0);
    XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i));
    {   __m512i* const xacc = (__m512i*) acc;
        const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1);

        /* xacc[0] ^= (xacc[0] >> 47) */
        __m512i const acc_vec     = *xacc;
        __m512i const shifted     = _mm512_srli_epi64    (acc_vec, 47);
        /* xacc[0] ^= secret; */
        __m512i const key_vec     = _mm512_loadu_si512   (secret);
        __m512i const data_key    = _mm512_ternarylogic_epi32(key_vec, acc_vec, shifted, 0x96 /* key_vec ^ acc_vec ^ shifted */);

        /* xacc[0] *= XXH_PRIME32_1; */
        __m512i const data_key_hi = _mm512_srli_epi64 (data_key, 32);
        __m512i const prod_lo     = _mm512_mul_epu32     (data_key, prime32);
        __m512i const prod_hi     = _mm512_mul_epu32     (data_key_hi, prime32);
        *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32));
    }
}

XXH_FORCE_INLINE XXH_TARGET_AVX512 void
XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
{
    XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0);
    XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64);
    XXH_ASSERT(((size_t)customSecret & 63) == 0);
    (void)(&XXH_writeLE64);
    {   int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i);
        __m512i const seed_pos = _mm512_set1_epi64((xxh_i64)seed64);
        __m512i const seed     = _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_set1_epi8(0), seed_pos);

        const __m512i* const src  = (const __m512i*) ((const void*) XXH3_kSecret);
              __m512i* const dest = (      __m512i*) customSecret;
        int i;
        XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */
        XXH_ASSERT(((size_t)dest & 63) == 0);
        for (i=0; i < nbRounds; ++i) {
            dest[i] = _mm512_add_epi64(_mm512_load_si512(src + i), seed);
    }   }
}

#endif

#if (XXH_VECTOR == XXH_AVX2) \
    || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0)

#ifndef XXH_TARGET_AVX2
# define XXH_TARGET_AVX2  /* disable attribute target */
#endif

XXH_FORCE_INLINE XXH_TARGET_AVX2 void
XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc,
                    const void* XXH_RESTRICT input,
                    const void* XXH_RESTRICT secret)
{
    XXH_ASSERT((((size_t)acc) & 31) == 0);
    {   __m256i* const xacc    =       (__m256i *) acc;
        /* Unaligned. This is mainly for pointer arithmetic, and because
         * _mm256_loadu_si256 requires  a const __m256i * pointer for some reason. */
        const         __m256i* const xinput  = (const __m256i *) input;
        /* Unaligned. This is mainly for pointer arithmetic, and because
         * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
        const         __m256i* const xsecret = (const __m256i *) secret;

        size_t i;
        for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) {
            /* data_vec    = xinput[i]; */
            __m256i const data_vec    = _mm256_loadu_si256    (xinput+i);
            /* key_vec     = xsecret[i]; */
            __m256i const key_vec     = _mm256_loadu_si256   (xsecret+i);
            /* data_key    = data_vec ^ key_vec; */
            __m256i const data_key    = _mm256_xor_si256     (data_vec, key_vec);
            /* data_key_lo = data_key >> 32; */
            __m256i const data_key_lo = _mm256_srli_epi64 (data_key, 32);
            /* product     = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
            __m256i const product     = _mm256_mul_epu32     (data_key, data_key_lo);
            /* xacc[i] += swap(data_vec); */
            __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2));
            __m256i const sum       = _mm256_add_epi64(xacc[i], data_swap);
            /* xacc[i] += product; */
            xacc[i] = _mm256_add_epi64(product, sum);
    }   }
}
XXH_FORCE_INLINE XXH_TARGET_AVX2 XXH3_ACCUMULATE_TEMPLATE(avx2)

XXH_FORCE_INLINE XXH_TARGET_AVX2 void
XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
{
    XXH_ASSERT((((size_t)acc) & 31) == 0);
    {   __m256i* const xacc = (__m256i*) acc;
        /* Unaligned. This is mainly for pointer arithmetic, and because
         * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
        const         __m256i* const xsecret = (const __m256i *) secret;
        const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1);

        size_t i;
        for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) {
            /* xacc[i] ^= (xacc[i] >> 47) */
            __m256i const acc_vec     = xacc[i];
            __m256i const shifted     = _mm256_srli_epi64    (acc_vec, 47);
            __m256i const data_vec    = _mm256_xor_si256     (acc_vec, shifted);
            /* xacc[i] ^= xsecret; */
            __m256i const key_vec     = _mm256_loadu_si256   (xsecret+i);
            __m256i const data_key    = _mm256_xor_si256     (data_vec, key_vec);

            /* xacc[i] *= XXH_PRIME32_1; */
            __m256i const data_key_hi = _mm256_srli_epi64 (data_key, 32);
            __m256i const prod_lo     = _mm256_mul_epu32     (data_key, prime32);
            __m256i const prod_hi     = _mm256_mul_epu32     (data_key_hi, prime32);
            xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32));
        }
    }
}

XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
{
    XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0);
    XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6);
    XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64);
    (void)(&XXH_writeLE64);
    XXH_PREFETCH(customSecret);
    {   __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64);

        const __m256i* const src  = (const __m256i*) ((const void*) XXH3_kSecret);
              __m256i*       dest = (      __m256i*) customSecret;

#       if defined(__GNUC__) || defined(__clang__)
        /*
         * On GCC & Clang, marking 'dest' as modified will cause the compiler:
         *   - do not extract the secret from sse registers in the internal loop
         *   - use less common registers, and avoid pushing these reg into stack
         */
        XXH_COMPILER_GUARD(dest);
#       endif
        XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */
        XXH_ASSERT(((size_t)dest & 31) == 0);

        /* GCC -O2 need unroll loop manually */
        dest[0] = _mm256_add_epi64(_mm256_load_si256(src+0), seed);
        dest[1] = _mm256_add_epi64(_mm256_load_si256(src+1), seed);
        dest[2] = _mm256_add_epi64(_mm256_load_si256(src+2), seed);
        dest[3] = _mm256_add_epi64(_mm256_load_si256(src+3), seed);
        dest[4] = _mm256_add_epi64(_mm256_load_si256(src+4), seed);
        dest[5] = _mm256_add_epi64(_mm256_load_si256(src+5), seed);
    }
}

#endif

/* x86dispatch always generates SSE2 */
#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH)

#ifndef XXH_TARGET_SSE2
# define XXH_TARGET_SSE2  /* disable attribute target */
#endif

XXH_FORCE_INLINE XXH_TARGET_SSE2 void
XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc,
                    const void* XXH_RESTRICT input,
                    const void* XXH_RESTRICT secret)
{
    /* SSE2 is just a half-scale version of the AVX2 version. */
    XXH_ASSERT((((size_t)acc) & 15) == 0);
    {   __m128i* const xacc    =       (__m128i *) acc;
        /* Unaligned. This is mainly for pointer arithmetic, and because
         * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
        const         __m128i* const xinput  = (const __m128i *) input;
        /* Unaligned. This is mainly for pointer arithmetic, and because
         * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
        const         __m128i* const xsecret = (const __m128i *) secret;

        size_t i;
        for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) {
            /* data_vec    = xinput[i]; */
            __m128i const data_vec    = _mm_loadu_si128   (xinput+i);
            /* key_vec     = xsecret[i]; */
            __m128i const key_vec     = _mm_loadu_si128   (xsecret+i);
            /* data_key    = data_vec ^ key_vec; */
            __m128i const data_key    = _mm_xor_si128     (data_vec, key_vec);
            /* data_key_lo = data_key >> 32; */
            __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
            /* product     = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
            __m128i const product     = _mm_mul_epu32     (data_key, data_key_lo);
            /* xacc[i] += swap(data_vec); */
            __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2));
            __m128i const sum       = _mm_add_epi64(xacc[i], data_swap);
            /* xacc[i] += product; */
            xacc[i] = _mm_add_epi64(product, sum);
    }   }
}
XXH_FORCE_INLINE XXH_TARGET_SSE2 XXH3_ACCUMULATE_TEMPLATE(sse2)

XXH_FORCE_INLINE XXH_TARGET_SSE2 void
XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
{
    XXH_ASSERT((((size_t)acc) & 15) == 0);
    {   __m128i* const xacc = (__m128i*) acc;
        /* Unaligned. This is mainly for pointer arithmetic, and because
         * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
        const         __m128i* const xsecret = (const __m128i *) secret;
        const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1);

        size_t i;
        for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) {
            /* xacc[i] ^= (xacc[i] >> 47) */
            __m128i const acc_vec     = xacc[i];
            __m128i const shifted     = _mm_srli_epi64    (acc_vec, 47);
            __m128i const data_vec    = _mm_xor_si128     (acc_vec, shifted);
            /* xacc[i] ^= xsecret[i]; */
            __m128i const key_vec     = _mm_loadu_si128   (xsecret+i);
            __m128i const data_key    = _mm_xor_si128     (data_vec, key_vec);

            /* xacc[i] *= XXH_PRIME32_1; */
            __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
            __m128i const prod_lo     = _mm_mul_epu32     (data_key, prime32);
            __m128i const prod_hi     = _mm_mul_epu32     (data_key_hi, prime32);
            xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32));
        }
    }
}

XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
{
    XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0);
    (void)(&XXH_writeLE64);
    {   int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i);

#       if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER <= 1900
        /* MSVC 32bit mode does not support _mm_set_epi64x before 2015
         * and some specific variants of 2015 may also lack it */
        /* Cast to unsigned 64-bit first to avoid signed arithmetic issues */
        xxh_u64 const seed64_unsigned = (xxh_u64)seed64;
        xxh_u64 const neg_seed64 = (xxh_u64)(0ULL - seed64_unsigned);
        __m128i const seed = _mm_set_epi32(
            (int)(neg_seed64 >> 32),      /* high 32 bits of negated seed */
            (int)(neg_seed64),            /* low 32 bits of negated seed */
            (int)(seed64_unsigned >> 32), /* high 32 bits of original seed */
            (int)(seed64_unsigned)        /* low 32 bits of original seed */
        );
#       else
        __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64);
#       endif
        int i;

        const void* const src16 = XXH3_kSecret;
        __m128i* dst16 = (__m128i*) customSecret;
#       if defined(__GNUC__) || defined(__clang__)
        /*
         * On GCC & Clang, marking 'dest' as modified will cause the compiler:
         *   - do not extract the secret from sse registers in the internal loop
         *   - use less common registers, and avoid pushing these reg into stack
         */
        XXH_COMPILER_GUARD(dst16);
#       endif
        XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */
        XXH_ASSERT(((size_t)dst16 & 15) == 0);

        for (i=0; i < nbRounds; ++i) {
            dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed);
    }   }
}

#endif

#if (XXH_VECTOR == XXH_NEON)

/* forward declarations for the scalar routines */
XXH_FORCE_INLINE void
XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input,
                 void const* XXH_RESTRICT secret, size_t lane);

XXH_FORCE_INLINE void
XXH3_scalarScrambleRound(void* XXH_RESTRICT acc,
                         void const* XXH_RESTRICT secret, size_t lane);

/*!
 * @internal
 * @brief The bulk processing loop for NEON and WASM SIMD128.
 *
 * The NEON code path is actually partially scalar when running on AArch64. This
 * is to optimize the pipelining and can have up to 15% speedup depending on the
 * CPU, and it also mitigates some GCC codegen issues.
 *
 * @see XXH3_NEON_LANES for configuring this and details about this optimization.
 *
 * NEON's 32-bit to 64-bit long multiply takes a half vector of 32-bit
 * integers instead of the other platforms which mask full 64-bit vectors,
 * so the setup is more complicated than just shifting right.
 *
 * Additionally, there is an optimization for 4 lanes at once noted below.
 *
 * Since, as stated, the most optimal amount of lanes for Cortexes is 6,
 * there needs to be *three* versions of the accumulate operation used
 * for the remaining 2 lanes.
 *
 * WASM's SIMD128 uses SIMDe's arm_neon.h polyfill because the intrinsics overlap
 * nearly perfectly.
 */

XXH_FORCE_INLINE void
XXH3_accumulate_512_neon( void* XXH_RESTRICT acc,
                    const void* XXH_RESTRICT input,
                    const void* XXH_RESTRICT secret)
{
    XXH_ASSERT((((size_t)acc) & 15) == 0);
    XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0);
    {   /* GCC for darwin arm64 does not like aliasing here */
        xxh_aliasing_uint64x2_t* const xacc = (xxh_aliasing_uint64x2_t*) acc;
        /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */
        uint8_t const* xinput = (const uint8_t *) input;
        uint8_t const* xsecret  = (const uint8_t *) secret;

        size_t i;
#ifdef __wasm_simd128__
        /*
         * On WASM SIMD128, Clang emits direct address loads when XXH3_kSecret
         * is constant propagated, which results in it converting it to this
         * inside the loop:
         *
         *    a = v128.load(XXH3_kSecret +  0 + $secret_offset, offset = 0)
         *    b = v128.load(XXH3_kSecret + 16 + $secret_offset, offset = 0)
         *    ...
         *
         * This requires a full 32-bit address immediate (and therefore a 6 byte
         * instruction) as well as an add for each offset.
         *
         * Putting an asm guard prevents it from folding (at the cost of losing
         * the alignment hint), and uses the free offset in `v128.load` instead
         * of adding secret_offset each time which overall reduces code size by
         * about a kilobyte and improves performance.
         */
        XXH_COMPILER_GUARD(xsecret);
#endif
        /* Scalar lanes use the normal scalarRound routine */
        for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) {
            XXH3_scalarRound(acc, input, secret, i);
        }
        i = 0;
        /* 4 NEON lanes at a time. */
        for (; i+1 < XXH3_NEON_LANES / 2; i+=2) {
            /* data_vec = xinput[i]; */
            uint64x2_t data_vec_1 = XXH_vld1q_u64(xinput  + (i * 16));
            uint64x2_t data_vec_2 = XXH_vld1q_u64(xinput  + ((i+1) * 16));
            /* key_vec  = xsecret[i];  */
            uint64x2_t key_vec_1  = XXH_vld1q_u64(xsecret + (i * 16));
            uint64x2_t key_vec_2  = XXH_vld1q_u64(xsecret + ((i+1) * 16));
            /* data_swap = swap(data_vec) */
            uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1);
            uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1);
            /* data_key = data_vec ^ key_vec; */
            uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1);
            uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2);

            /*
             * If we reinterpret the 64x2 vectors as 32x4 vectors, we can use a
             * de-interleave operation for 4 lanes in 1 step with `vuzpq_u32` to
             * get one vector with the low 32 bits of each lane, and one vector
             * with the high 32 bits of each lane.
             *
             * The intrinsic returns a double vector because the original ARMv7-a
             * instruction modified both arguments in place. AArch64 and SIMD128 emit
             * two instructions from this intrinsic.
             *
             *  [ dk11L | dk11H | dk12L | dk12H ] -> [ dk11L | dk12L | dk21L | dk22L ]
             *  [ dk21L | dk21H | dk22L | dk22H ] -> [ dk11H | dk12H | dk21H | dk22H ]
             */
            uint32x4x2_t unzipped = vuzpq_u32(
                vreinterpretq_u32_u64(data_key_1),
                vreinterpretq_u32_u64(data_key_2)
            );
            /* data_key_lo = data_key & 0xFFFFFFFF */
            uint32x4_t data_key_lo = unzipped.val[0];
            /* data_key_hi = data_key >> 32 */
            uint32x4_t data_key_hi = unzipped.val[1];
            /*
             * Then, we can split the vectors horizontally and multiply which, as for most
             * widening intrinsics, have a variant that works on both high half vectors
             * for free on AArch64. A similar instruction is available on SIMD128.
             *
             * sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi
             */
            uint64x2_t sum_1 = XXH_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi);
            uint64x2_t sum_2 = XXH_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi);
            /*
             * Clang reorders
             *    a += b * c;     // umlal   swap.2d, dkl.2s, dkh.2s
             *    c += a;         // add     acc.2d, acc.2d, swap.2d
             * to
             *    c += a;         // add     acc.2d, acc.2d, swap.2d
             *    c += b * c;     // umlal   acc.2d, dkl.2s, dkh.2s
             *
             * While it would make sense in theory since the addition is faster,
             * for reasons likely related to umlal being limited to certain NEON
             * pipelines, this is worse. A compiler guard fixes this.
             */
            XXH_COMPILER_GUARD_CLANG_NEON(sum_1);
            XXH_COMPILER_GUARD_CLANG_NEON(sum_2);
            /* xacc[i] = acc_vec + sum; */
            xacc[i]   = vaddq_u64(xacc[i], sum_1);
            xacc[i+1] = vaddq_u64(xacc[i+1], sum_2);
        }
        /* Operate on the remaining NEON lanes 2 at a time. */
        for (; i < XXH3_NEON_LANES / 2; i++) {
            /* data_vec = xinput[i]; */
            uint64x2_t data_vec = XXH_vld1q_u64(xinput  + (i * 16));
            /* key_vec  = xsecret[i];  */
            uint64x2_t key_vec  = XXH_vld1q_u64(xsecret + (i * 16));
            /* acc_vec_2 = swap(data_vec) */
            uint64x2_t data_swap = vextq_u64(data_vec, data_vec, 1);
            /* data_key = data_vec ^ key_vec; */
            uint64x2_t data_key = veorq_u64(data_vec, key_vec);
            /* For two lanes, just use VMOVN and VSHRN. */
            /* data_key_lo = data_key & 0xFFFFFFFF; */
            uint32x2_t data_key_lo = vmovn_u64(data_key);
            /* data_key_hi = data_key >> 32; */
            uint32x2_t data_key_hi = vshrn_n_u64(data_key, 32);
            /* sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi; */
            uint64x2_t sum = vmlal_u32(data_swap, data_key_lo, data_key_hi);
            /* Same Clang workaround as before */
            XXH_COMPILER_GUARD_CLANG_NEON(sum);
            /* xacc[i] = acc_vec + sum; */
            xacc[i] = vaddq_u64 (xacc[i], sum);
        }
    }
}
XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(neon)

XXH_FORCE_INLINE void
XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
{
    XXH_ASSERT((((size_t)acc) & 15) == 0);

    {   xxh_aliasing_uint64x2_t* xacc       = (xxh_aliasing_uint64x2_t*) acc;
        uint8_t const* xsecret = (uint8_t const*) secret;

        size_t i;
        /* WASM uses operator overloads and doesn't need these. */
#ifndef __wasm_simd128__
        /* { prime32_1, prime32_1 } */
        uint32x2_t const kPrimeLo = vdup_n_u32(XXH_PRIME32_1);
        /* { 0, prime32_1, 0, prime32_1 } */
        uint32x4_t const kPrimeHi = vreinterpretq_u32_u64(vdupq_n_u64((xxh_u64)XXH_PRIME32_1 << 32));
#endif

        /* AArch64 uses both scalar and neon at the same time */
        for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) {
            XXH3_scalarScrambleRound(acc, secret, i);
        }
        for (i=0; i < XXH3_NEON_LANES / 2; i++) {
            /* xacc[i] ^= (xacc[i] >> 47); */
            uint64x2_t acc_vec  = xacc[i];
            uint64x2_t shifted  = vshrq_n_u64(acc_vec, 47);
            uint64x2_t data_vec = veorq_u64(acc_vec, shifted);

            /* xacc[i] ^= xsecret[i]; */
            uint64x2_t key_vec  = XXH_vld1q_u64(xsecret + (i * 16));
            uint64x2_t data_key = veorq_u64(data_vec, key_vec);
            /* xacc[i] *= XXH_PRIME32_1 */
#ifdef __wasm_simd128__
            /* SIMD128 has multiply by u64x2, use it instead of expanding and scalarizing */
            xacc[i] = data_key * XXH_PRIME32_1;
#else
            /*
             * Expanded version with portable NEON intrinsics
             *
             *    lo(x) * lo(y) + (hi(x) * lo(y) << 32)
             *
             * prod_hi = hi(data_key) * lo(prime) << 32
             *
             * Since we only need 32 bits of this multiply a trick can be used, reinterpreting the vector
             * as a uint32x4_t and multiplying by { 0, prime, 0, prime } to cancel out the unwanted bits
             * and avoid the shift.
             */
            uint32x4_t prod_hi = vmulq_u32 (vreinterpretq_u32_u64(data_key), kPrimeHi);
            /* Extract low bits for vmlal_u32  */
            uint32x2_t data_key_lo = vmovn_u64(data_key);
            /* xacc[i] = prod_hi + lo(data_key) * XXH_PRIME32_1; */
            xacc[i] = vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, kPrimeLo);
#endif
        }
    }
}
#endif

#if (XXH_VECTOR == XXH_VSX)

XXH_FORCE_INLINE void
XXH3_accumulate_512_vsx(  void* XXH_RESTRICT acc,
                    const void* XXH_RESTRICT input,
                    const void* XXH_RESTRICT secret)
{
    /* presumed aligned */
    xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc;
    xxh_u8 const* const xinput   = (xxh_u8 const*) input;   /* no alignment restriction */
    xxh_u8 const* const xsecret  = (xxh_u8 const*) secret;    /* no alignment restriction */
    xxh_u64x2 const v32 = { 32, 32 };
    size_t i;
    for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) {
        /* data_vec = xinput[i]; */
        xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + 16*i);
        /* key_vec = xsecret[i]; */
        xxh_u64x2 const key_vec  = XXH_vec_loadu(xsecret + 16*i);
        xxh_u64x2 const data_key = data_vec ^ key_vec;
        /* shuffled = (data_key << 32) | (data_key >> 32); */
        xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32);
        /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */
        xxh_u64x2 const product  = XXH_vec_mulo((xxh_u32x4)data_key, shuffled);
        /* acc_vec = xacc[i]; */
        xxh_u64x2 acc_vec        = xacc[i];
        acc_vec += product;

        /* swap high and low halves */
#ifdef __s390x__
        acc_vec += vec_permi(data_vec, data_vec, 2);
#else
        acc_vec += vec_xxpermdi(data_vec, data_vec, 2);
#endif
        xacc[i] = acc_vec;
    }
}
XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(vsx)

XXH_FORCE_INLINE void
XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
{
    XXH_ASSERT((((size_t)acc) & 15) == 0);

    {   xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc;
        const xxh_u8* const xsecret = (const xxh_u8*) secret;
        /* constants */
        xxh_u64x2 const v32  = { 32, 32 };
        xxh_u64x2 const v47 = { 47, 47 };
        xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 };
        size_t i;
        for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) {
            /* xacc[i] ^= (xacc[i] >> 47); */
            xxh_u64x2 const acc_vec  = xacc[i];
            xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47);

            /* xacc[i] ^= xsecret[i]; */
            xxh_u64x2 const key_vec  = XXH_vec_loadu(xsecret + 16*i);
            xxh_u64x2 const data_key = data_vec ^ key_vec;

            /* xacc[i] *= XXH_PRIME32_1 */
            /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF);  */
            xxh_u64x2 const prod_even  = XXH_vec_mule((xxh_u32x4)data_key, prime);
            /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32);  */
            xxh_u64x2 const prod_odd  = XXH_vec_mulo((xxh_u32x4)data_key, prime);
            xacc[i] = prod_odd + (prod_even << v32);
    }   }
}

#endif

#if (XXH_VECTOR == XXH_SVE)

XXH_FORCE_INLINE void
XXH3_accumulate_512_sve( void* XXH_RESTRICT acc,
                   const void* XXH_RESTRICT input,
                   const void* XXH_RESTRICT secret)
{
    uint64_t *xacc = (uint64_t *)acc;
    const uint64_t *xinput = (const uint64_t *)(const void *)input;
    const uint64_t *xsecret = (const uint64_t *)(const void *)secret;
    svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1);
    uint64_t element_count = svcntd();
    if (element_count >= 8) {
        svbool_t mask = svptrue_pat_b64(SV_VL8);
        svuint64_t vacc = svld1_u64(mask, xacc);
        ACCRND(vacc, 0);
        svst1_u64(mask, xacc, vacc);
    } else if (element_count == 2) {   /* sve128 */
        svbool_t mask = svptrue_pat_b64(SV_VL2);
        svuint64_t acc0 = svld1_u64(mask, xacc + 0);
        svuint64_t acc1 = svld1_u64(mask, xacc + 2);
        svuint64_t acc2 = svld1_u64(mask, xacc + 4);
        svuint64_t acc3 = svld1_u64(mask, xacc + 6);
        ACCRND(acc0, 0);
        ACCRND(acc1, 2);
        ACCRND(acc2, 4);
        ACCRND(acc3, 6);
        svst1_u64(mask, xacc + 0, acc0);
        svst1_u64(mask, xacc + 2, acc1);
        svst1_u64(mask, xacc + 4, acc2);
        svst1_u64(mask, xacc + 6, acc3);
    } else {
        svbool_t mask = svptrue_pat_b64(SV_VL4);
        svuint64_t acc0 = svld1_u64(mask, xacc + 0);
        svuint64_t acc1 = svld1_u64(mask, xacc + 4);
        ACCRND(acc0, 0);
        ACCRND(acc1, 4);
        svst1_u64(mask, xacc + 0, acc0);
        svst1_u64(mask, xacc + 4, acc1);
    }
}

XXH_FORCE_INLINE void
XXH3_accumulate_sve(xxh_u64* XXH_RESTRICT acc,
               const xxh_u8* XXH_RESTRICT input,
               const xxh_u8* XXH_RESTRICT secret,
               size_t nbStripes)
{
    if (nbStripes != 0) {
        uint64_t *xacc = (uint64_t *)acc;
        const uint64_t *xinput = (const uint64_t *)(const void *)input;
        const uint64_t *xsecret = (const uint64_t *)(const void *)secret;
        svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1);
        uint64_t element_count = svcntd();
        if (element_count >= 8) {
            svbool_t mask = svptrue_pat_b64(SV_VL8);
            svuint64_t vacc = svld1_u64(mask, xacc + 0);
            do {
                /* svprfd(svbool_t, void *, enum svfprop); */
                svprfd(mask, xinput + 128, SV_PLDL1STRM);
                ACCRND(vacc, 0);
                xinput += 8;
                xsecret += 1;
                nbStripes--;
           } while (nbStripes != 0);

           svst1_u64(mask, xacc + 0, vacc);
        } else if (element_count == 2) { /* sve128 */
            svbool_t mask = svptrue_pat_b64(SV_VL2);
            svuint64_t acc0 = svld1_u64(mask, xacc + 0);
            svuint64_t acc1 = svld1_u64(mask, xacc + 2);
            svuint64_t acc2 = svld1_u64(mask, xacc + 4);
            svuint64_t acc3 = svld1_u64(mask, xacc + 6);
            do {
                svprfd(mask, xinput + 128, SV_PLDL1STRM);
                ACCRND(acc0, 0);
                ACCRND(acc1, 2);
                ACCRND(acc2, 4);
                ACCRND(acc3, 6);
                xinput += 8;
                xsecret += 1;
                nbStripes--;
           } while (nbStripes != 0);

           svst1_u64(mask, xacc + 0, acc0);
           svst1_u64(mask, xacc + 2, acc1);
           svst1_u64(mask, xacc + 4, acc2);
           svst1_u64(mask, xacc + 6, acc3);
        } else {
            svbool_t mask = svptrue_pat_b64(SV_VL4);
            svuint64_t acc0 = svld1_u64(mask, xacc + 0);
            svuint64_t acc1 = svld1_u64(mask, xacc + 4);
            do {
                svprfd(mask, xinput + 128, SV_PLDL1STRM);
                ACCRND(acc0, 0);
                ACCRND(acc1, 4);
                xinput += 8;
                xsecret += 1;
                nbStripes--;
           } while (nbStripes != 0);

           svst1_u64(mask, xacc + 0, acc0);
           svst1_u64(mask, xacc + 4, acc1);
       }
    }
}

#endif

#if (XXH_VECTOR == XXH_LSX)
#define _LSX_SHUFFLE(z, y, x, w) (((z) << 6) | ((y) << 4) | ((x) << 2) | (w))

XXH_FORCE_INLINE void
XXH3_accumulate_512_lsx( void* XXH_RESTRICT acc,
                    const void* XXH_RESTRICT input,
                    const void* XXH_RESTRICT secret)
{
    XXH_ASSERT((((size_t)acc) & 15) == 0);
    {
        __m128i* const xacc    =       (__m128i *) acc;
        const __m128i* const xinput  = (const __m128i *) input;
        const __m128i* const xsecret = (const __m128i *) secret;
        size_t i;

        for (i = 0; i < XXH_STRIPE_LEN / sizeof(__m128i); i++) {
            /* data_vec = xinput[i]; */
            __m128i const data_vec = __lsx_vld(xinput + i, 0);
            /* key_vec = xsecret[i]; */
            __m128i const key_vec = __lsx_vld(xsecret + i, 0);
            /* data_key = data_vec ^ key_vec; */
            __m128i const data_key = __lsx_vxor_v(data_vec, key_vec);
            /* data_key_lo = data_key >> 32; */
            __m128i const data_key_lo = __lsx_vsrli_d(data_key, 32);
            // __m128i const data_key_lo = __lsx_vsrli_d(data_key, 32);
            /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
            __m128i const product = __lsx_vmulwev_d_wu(data_key, data_key_lo);
            /* xacc[i] += swap(data_vec); */
            __m128i const data_swap = __lsx_vshuf4i_w(data_vec, _LSX_SHUFFLE(1, 0, 3, 2));
            __m128i const sum = __lsx_vadd_d(xacc[i], data_swap);
            /* xacc[i] += product; */
            xacc[i] = __lsx_vadd_d(product, sum);
        }
    }
}
XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(lsx)

XXH_FORCE_INLINE void
XXH3_scrambleAcc_lsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
{
    XXH_ASSERT((((size_t)acc) & 15) == 0);
    {
        __m128i* const xacc = (__m128i*) acc;
        const __m128i* const xsecret = (const __m128i *) secret;
        const __m128i prime32 = __lsx_vreplgr2vr_d(XXH_PRIME32_1);
        size_t i;

        for (i = 0; i < XXH_STRIPE_LEN / sizeof(__m128i); i++) {
            /* xacc[i] ^= (xacc[i] >> 47) */
            __m128i const acc_vec = xacc[i];
            __m128i const shifted = __lsx_vsrli_d(acc_vec, 47);
            __m128i const data_vec = __lsx_vxor_v(acc_vec, shifted);
            /* xacc[i] ^= xsecret[i]; */
            __m128i const key_vec = __lsx_vld(xsecret + i, 0);
            __m128i const data_key = __lsx_vxor_v(data_vec, key_vec);

            /* xacc[i] *= XXH_PRIME32_1; */
            xacc[i] = __lsx_vmul_d(data_key, prime32);
        }
    }
}

#endif

#if (XXH_VECTOR == XXH_LASX)
#define _LASX_SHUFFLE(z, y, x, w) (((z) << 6) | ((y) << 4) | ((x) << 2) | (w))

XXH_FORCE_INLINE void
XXH3_accumulate_512_lasx( void* XXH_RESTRICT acc,
                    const void* XXH_RESTRICT input,
                    const void* XXH_RESTRICT secret)
{
    XXH_ASSERT((((size_t)acc) & 31) == 0);
    {
        size_t i;
        __m256i* const xacc    =       (__m256i *) acc;
        const __m256i* const xinput  = (const __m256i *) input;
        const __m256i* const xsecret = (const __m256i *) secret;

        for (i = 0; i < XXH_STRIPE_LEN / sizeof(__m256i); i++) {
            /* data_vec = xinput[i]; */
            __m256i const data_vec = __lasx_xvld(xinput + i, 0);
            /* key_vec = xsecret[i]; */
            __m256i const key_vec = __lasx_xvld(xsecret + i, 0);
            /* data_key = data_vec ^ key_vec; */
            __m256i const data_key = __lasx_xvxor_v(data_vec, key_vec);
            /* data_key_lo = data_key >> 32; */
            __m256i const data_key_lo = __lasx_xvsrli_d(data_key, 32);
            // __m256i const data_key_lo = __lasx_xvsrli_d(data_key, 32);
            /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
            __m256i const product = __lasx_xvmulwev_d_wu(data_key, data_key_lo);
            /* xacc[i] += swap(data_vec); */
            __m256i const data_swap = __lasx_xvshuf4i_w(data_vec, _LASX_SHUFFLE(1, 0, 3, 2));
            __m256i const sum = __lasx_xvadd_d(xacc[i], data_swap);
            /* xacc[i] += product; */
            xacc[i] = __lasx_xvadd_d(product, sum);
        }
    }
}
XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(lasx)

XXH_FORCE_INLINE void
XXH3_scrambleAcc_lasx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
{
    XXH_ASSERT((((size_t)acc) & 31) == 0);
    {
        __m256i* const xacc = (__m256i*) acc;
        const __m256i* const xsecret = (const __m256i *) secret;
        const __m256i prime32 = __lasx_xvreplgr2vr_d(XXH_PRIME32_1);
        size_t i;

        for (i = 0; i < XXH_STRIPE_LEN / sizeof(__m256i); i++) {
            /* xacc[i] ^= (xacc[i] >> 47) */
            __m256i const acc_vec = xacc[i];
            __m256i const shifted = __lasx_xvsrli_d(acc_vec, 47);
            __m256i const data_vec = __lasx_xvxor_v(acc_vec, shifted);
            /* xacc[i] ^= xsecret[i]; */
            __m256i const key_vec = __lasx_xvld(xsecret + i, 0);
            __m256i const data_key = __lasx_xvxor_v(data_vec, key_vec);

            /* xacc[i] *= XXH_PRIME32_1; */
            xacc[i] = __lasx_xvmul_d(data_key, prime32);
        }
    }
}

#endif

#if (XXH_VECTOR == XXH_RVV)
    #define XXH_CONCAT2(X, Y) X ## Y
    #define XXH_CONCAT(X, Y) XXH_CONCAT2(X, Y)
#if ((defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 13) || \
        (defined(__clang__) && __clang_major__ < 16))
    #define XXH_RVOP(op) op
    #define XXH_RVCAST(op) XXH_CONCAT(vreinterpret_v_, op)
#else
    #define XXH_RVOP(op) XXH_CONCAT(__riscv_, op)
    #define XXH_RVCAST(op) XXH_CONCAT(__riscv_vreinterpret_v_, op)
#endif
XXH_FORCE_INLINE void
XXH3_accumulate_512_rvv(  void* XXH_RESTRICT acc,
                    const void* XXH_RESTRICT input,
                    const void* XXH_RESTRICT secret)
{
    XXH_ASSERT((((size_t)acc) & 63) == 0);
    {
        // Try to set vector lenght to 512 bits.
        // If this length is unavailable, then maximum available will be used
        size_t vl = XXH_RVOP(vsetvl_e64m2)(8);

        uint64_t*       xacc    = (uint64_t*) acc;
        const uint64_t* xinput  = (const uint64_t*) input;
        const uint64_t* xsecret = (const uint64_t*) secret;
        static const uint64_t swap_mask[16] = {1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14};
        vuint64m2_t xswap_mask = XXH_RVOP(vle64_v_u64m2)(swap_mask, vl);

        size_t i;
        for (i = 0; i < XXH_STRIPE_LEN/8; i += vl) {
            /* data_vec = xinput[i]; */
            vuint64m2_t data_vec = XXH_RVCAST(u8m2_u64m2)(XXH_RVOP(vle8_v_u8m2)((const uint8_t*)(xinput + i), vl * 8));
            /* key_vec = xsecret[i]; */
            vuint64m2_t key_vec = XXH_RVCAST(u8m2_u64m2)(XXH_RVOP(vle8_v_u8m2)((const uint8_t*)(xsecret + i), vl * 8));
            /* acc_vec = xacc[i]; */
            vuint64m2_t acc_vec = XXH_RVOP(vle64_v_u64m2)(xacc + i, vl);
            /* data_key = data_vec ^ key_vec; */
            vuint64m2_t data_key = XXH_RVOP(vxor_vv_u64m2)(data_vec, key_vec, vl);
            /* data_key_hi = data_key >> 32; */
            vuint64m2_t data_key_hi = XXH_RVOP(vsrl_vx_u64m2)(data_key, 32, vl);
            /* data_key_lo = data_key & 0xffffffff; */
            vuint64m2_t data_key_lo = XXH_RVOP(vand_vx_u64m2)(data_key, 0xffffffff, vl);
            /* swap high and low halves */
            vuint64m2_t data_swap = XXH_RVOP(vrgather_vv_u64m2)(data_vec, xswap_mask, vl);
            /* acc_vec += data_key_lo * data_key_hi; */
            acc_vec = XXH_RVOP(vmacc_vv_u64m2)(acc_vec, data_key_lo, data_key_hi, vl);
            /* acc_vec += data_swap; */
            acc_vec = XXH_RVOP(vadd_vv_u64m2)(acc_vec, data_swap, vl);
            /* xacc[i] = acc_vec; */
            XXH_RVOP(vse64_v_u64m2)(xacc + i, acc_vec, vl);
        }
    }
}

XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(rvv)

XXH_FORCE_INLINE void
XXH3_scrambleAcc_rvv(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
{
    XXH_ASSERT((((size_t)acc) & 15) == 0);
    {
        size_t count = XXH_STRIPE_LEN/8;
        uint64_t* xacc = (uint64_t*)acc;
        const uint8_t* xsecret = (const uint8_t *)secret;
        size_t vl;
        for (; count > 0; count -= vl, xacc += vl, xsecret += vl*8) {
            vl = XXH_RVOP(vsetvl_e64m2)(count);
            {
                /* key_vec = xsecret[i]; */
                vuint64m2_t key_vec = XXH_RVCAST(u8m2_u64m2)(XXH_RVOP(vle8_v_u8m2)(xsecret, vl*8));
                /* acc_vec = xacc[i]; */
                vuint64m2_t acc_vec = XXH_RVOP(vle64_v_u64m2)(xacc, vl);
                /* acc_vec ^= acc_vec >> 47; */
                vuint64m2_t vsrl = XXH_RVOP(vsrl_vx_u64m2)(acc_vec, 47, vl);
                acc_vec = XXH_RVOP(vxor_vv_u64m2)(acc_vec, vsrl, vl);
                /* acc_vec ^= key_vec; */
                acc_vec = XXH_RVOP(vxor_vv_u64m2)(acc_vec, key_vec, vl);
                /* acc_vec *= XXH_PRIME32_1; */
                acc_vec = XXH_RVOP(vmul_vx_u64m2)(acc_vec, XXH_PRIME32_1, vl);
                /* xacc[i] *= acc_vec; */
                XXH_RVOP(vse64_v_u64m2)(xacc, acc_vec, vl);
            }
        }
    }
}

XXH_FORCE_INLINE void
XXH3_initCustomSecret_rvv(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
{
    XXH_STATIC_ASSERT(XXH_SEC_ALIGN >= 8);
    XXH_ASSERT(((size_t)customSecret & 7) == 0);
    (void)(&XXH_writeLE64);
    {
        size_t count = XXH_SECRET_DEFAULT_SIZE/8;
        size_t vl;
        size_t VLMAX = XXH_RVOP(vsetvlmax_e64m2)();
        int64_t* cSecret = (int64_t*)customSecret;
        const int64_t* kSecret = (const int64_t*)(const void*)XXH3_kSecret;

#if __riscv_v_intrinsic >= 1000000
        // ratified v1.0 intrinics version
        vbool32_t mneg = XXH_RVCAST(u8m1_b32)(
                         XXH_RVOP(vmv_v_x_u8m1)(0xaa, XXH_RVOP(vsetvlmax_e8m1)()));
#else
        // support pre-ratification intrinics, which lack mask to vector casts
        size_t vlmax = XXH_RVOP(vsetvlmax_e8m1)();
        vbool32_t mneg = XXH_RVOP(vmseq_vx_u8mf4_b32)(
                         XXH_RVOP(vand_vx_u8mf4)(
                         XXH_RVOP(vid_v_u8mf4)(vlmax), 1, vlmax), 1, vlmax);
#endif
        vint64m2_t seed = XXH_RVOP(vmv_v_x_i64m2)((int64_t)seed64, VLMAX);
        seed = XXH_RVOP(vneg_v_i64m2_mu)(mneg, seed, seed, VLMAX);

        for (; count > 0; count -= vl, cSecret += vl, kSecret += vl) {
            /* make sure vl=VLMAX until last iteration */
            vl = XXH_RVOP(vsetvl_e64m2)(count < VLMAX ? count : VLMAX);
            {
                vint64m2_t src = XXH_RVOP(vle64_v_i64m2)(kSecret, vl);
                vint64m2_t res = XXH_RVOP(vadd_vv_i64m2)(src, seed, vl);
                XXH_RVOP(vse64_v_i64m2)(cSecret, res, vl);
            }
        }
    }
}
#endif


/* scalar variants - universal */

#if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__))
/*
 * In XXH3_scalarRound(), GCC and Clang have a similar codegen issue, where they
 * emit an excess mask and a full 64-bit multiply-add (MADD X-form).
 *
 * While this might not seem like much, as AArch64 is a 64-bit architecture, only
 * big Cortex designs have a full 64-bit multiplier.
 *
 * On the little cores, the smaller 32-bit multiplier is used, and full 64-bit
 * multiplies expand to 2-3 multiplies in microcode. This has a major penalty
 * of up to 4 latency cycles and 2 stall cycles in the multiply pipeline.
 *
 * Thankfully, AArch64 still provides the 32-bit long multiply-add (UMADDL) which does
 * not have this penalty and does the mask automatically.
 */
XXH_FORCE_INLINE xxh_u64
XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc)
{
    xxh_u64 ret;
    /* note: %x = 64-bit register, %w = 32-bit register */
    __asm__("umaddl %x0, %w1, %w2, %x3" : "=r" (ret) : "r" (lhs), "r" (rhs), "r" (acc));
    return ret;
}
#else
XXH_FORCE_INLINE xxh_u64
XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc)
{
    return XXH_mult32to64((xxh_u32)lhs, (xxh_u32)rhs) + acc;
}
#endif

/*!
 * @internal
 * @brief Scalar round for @ref XXH3_accumulate_512_scalar().
 *
 * This is extracted to its own function because the NEON path uses a combination
 * of NEON and scalar.
 */
XXH_FORCE_INLINE void
XXH3_scalarRound(void* XXH_RESTRICT acc,
                 void const* XXH_RESTRICT input,
                 void const* XXH_RESTRICT secret,
                 size_t lane)
{
    xxh_u64* xacc = (xxh_u64*) acc;
    xxh_u8 const* xinput  = (xxh_u8 const*) input;
    xxh_u8 const* xsecret = (xxh_u8 const*) secret;
    XXH_ASSERT(lane < XXH_ACC_NB);
    XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0);
    {
        xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8);
        xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8);
        xacc[lane ^ 1] += data_val; /* swap adjacent lanes */
        xacc[lane] = XXH_mult32to64_add64(data_key /* & 0xFFFFFFFF */, data_key >> 32, xacc[lane]);
    }
}

/*!
 * @internal
 * @brief Processes a 64 byte block of data using the scalar path.
 */
XXH_FORCE_INLINE void
XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc,
                     const void* XXH_RESTRICT input,
                     const void* XXH_RESTRICT secret)
{
    size_t i;
    /* ARM GCC refuses to unroll this loop, resulting in a 24% slowdown on ARMv6. */
#if defined(__GNUC__) && !defined(__clang__) \
  && (defined(__arm__) || defined(__thumb2__)) \
  && defined(__ARM_FEATURE_UNALIGNED) /* no unaligned access just wastes bytes */ \
  && XXH_SIZE_OPT <= 0
#  pragma GCC unroll 8
#endif
    for (i=0; i < XXH_ACC_NB; i++) {
        XXH3_scalarRound(acc, input, secret, i);
    }
}
XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(scalar)

/*!
 * @internal
 * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar().
 *
 * This is extracted to its own function because the NEON path uses a combination
 * of NEON and scalar.
 */
XXH_FORCE_INLINE void
XXH3_scalarScrambleRound(void* XXH_RESTRICT acc,
                         void const* XXH_RESTRICT secret,
                         size_t lane)
{
    xxh_u64* const xacc = (xxh_u64*) acc;   /* presumed aligned */
    const xxh_u8* const xsecret = (const xxh_u8*) secret;   /* no alignment restriction */
    XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0);
    XXH_ASSERT(lane < XXH_ACC_NB);
    {
        xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8);
        xxh_u64 acc64 = xacc[lane];
        acc64 = XXH_xorshift64(acc64, 47);
        acc64 ^= key64;
        acc64 *= XXH_PRIME32_1;
        xacc[lane] = acc64;
    }
}

/*!
 * @internal
 * @brief Scrambles the accumulators after a large chunk has been read
 */
XXH_FORCE_INLINE void
XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
{
    size_t i;
    for (i=0; i < XXH_ACC_NB; i++) {
        XXH3_scalarScrambleRound(acc, secret, i);
    }
}

XXH_FORCE_INLINE void
XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
{
    /*
     * We need a separate pointer for the hack below,
     * which requires a non-const pointer.
     * Any decent compiler will optimize this out otherwise.
     */
    const xxh_u8* kSecretPtr = XXH3_kSecret;
    XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0);

#if defined(__GNUC__) && defined(__aarch64__)
    /*
     * UGLY HACK:
     * GCC and Clang generate a bunch of MOV/MOVK pairs for aarch64, and they are
     * placed sequentially, in order, at the top of the unrolled loop.
     *
     * While MOVK is great for generating constants (2 cycles for a 64-bit
     * constant compared to 4 cycles for LDR), it fights for bandwidth with
     * the arithmetic instructions.
     *
     *   I   L   S
     * MOVK
     * MOVK
     * MOVK
     * MOVK
     * ADD
     * SUB      STR
     *          STR
     * By forcing loads from memory (as the asm line causes the compiler to assume
     * that XXH3_kSecretPtr has been changed), the pipelines are used more
     * efficiently:
     *   I   L   S
     *      LDR
     *  ADD LDR
     *  SUB     STR
     *          STR
     *
     * See XXH3_NEON_LANES for details on the pipeline.
     *
     * XXH3_64bits_withSeed, len == 256, Snapdragon 835
     *   without hack: 2654.4 MB/s
     *   with hack:    3202.9 MB/s
     */
    XXH_COMPILER_GUARD(kSecretPtr);
#endif
    {   int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16;
        int i;
        for (i=0; i < nbRounds; i++) {
            /*
             * The asm hack causes the compiler to assume that kSecretPtr aliases with
             * customSecret, and on aarch64, this prevented LDP from merging two
             * loads together for free. Putting the loads together before the stores
             * properly generates LDP.
             */
            xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i)     + seed64;
            xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64;
            XXH_writeLE64((xxh_u8*)customSecret + 16*i,     lo);
            XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi);
    }   }
}


typedef void (*XXH3_f_accumulate)(xxh_u64* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, size_t);
typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*);
typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64);


#if (XXH_VECTOR == XXH_AVX512)

#define XXH3_accumulate_512 XXH3_accumulate_512_avx512
#define XXH3_accumulate     XXH3_accumulate_avx512
#define XXH3_scrambleAcc    XXH3_scrambleAcc_avx512
#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512

#elif (XXH_VECTOR == XXH_AVX2)

#define XXH3_accumulate_512 XXH3_accumulate_512_avx2
#define XXH3_accumulate     XXH3_accumulate_avx2
#define XXH3_scrambleAcc    XXH3_scrambleAcc_avx2
#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2

#elif (XXH_VECTOR == XXH_SSE2)

#define XXH3_accumulate_512 XXH3_accumulate_512_sse2
#define XXH3_accumulate     XXH3_accumulate_sse2
#define XXH3_scrambleAcc    XXH3_scrambleAcc_sse2
#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2

#elif (XXH_VECTOR == XXH_NEON)

#define XXH3_accumulate_512 XXH3_accumulate_512_neon
#define XXH3_accumulate     XXH3_accumulate_neon
#define XXH3_scrambleAcc    XXH3_scrambleAcc_neon
#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar

#elif (XXH_VECTOR == XXH_VSX)

#define XXH3_accumulate_512 XXH3_accumulate_512_vsx
#define XXH3_accumulate     XXH3_accumulate_vsx
#define XXH3_scrambleAcc    XXH3_scrambleAcc_vsx
#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar

#elif (XXH_VECTOR == XXH_SVE)
#define XXH3_accumulate_512 XXH3_accumulate_512_sve
#define XXH3_accumulate     XXH3_accumulate_sve
#define XXH3_scrambleAcc    XXH3_scrambleAcc_scalar
#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar

#elif (XXH_VECTOR == XXH_LASX)
#define XXH3_accumulate_512 XXH3_accumulate_512_lasx
#define XXH3_accumulate     XXH3_accumulate_lasx
#define XXH3_scrambleAcc    XXH3_scrambleAcc_lasx
#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar

#elif (XXH_VECTOR == XXH_LSX)
#define XXH3_accumulate_512 XXH3_accumulate_512_lsx
#define XXH3_accumulate     XXH3_accumulate_lsx
#define XXH3_scrambleAcc    XXH3_scrambleAcc_lsx
#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar

#elif (XXH_VECTOR == XXH_RVV)
#define XXH3_accumulate_512 XXH3_accumulate_512_rvv
#define XXH3_accumulate     XXH3_accumulate_rvv
#define XXH3_scrambleAcc    XXH3_scrambleAcc_rvv
#define XXH3_initCustomSecret XXH3_initCustomSecret_rvv

#else /* scalar */

#define XXH3_accumulate_512 XXH3_accumulate_512_scalar
#define XXH3_accumulate     XXH3_accumulate_scalar
#define XXH3_scrambleAcc    XXH3_scrambleAcc_scalar
#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar

#endif

#if XXH_SIZE_OPT >= 1 /* don't do SIMD for initialization */
#  undef XXH3_initCustomSecret
#  define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
#endif

XXH_FORCE_INLINE void
XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc,
                      const xxh_u8* XXH_RESTRICT input, size_t len,
                      const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
                            XXH3_f_accumulate f_acc,
                            XXH3_f_scrambleAcc f_scramble)
{
    size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE;
    size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock;
    size_t const nb_blocks = (len - 1) / block_len;

    size_t n;

    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);

    for (n = 0; n < nb_blocks; n++) {
        f_acc(acc, input + n*block_len, secret, nbStripesPerBlock);
        f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN);
    }

    /* last partial block */
    XXH_ASSERT(len > XXH_STRIPE_LEN);
    {   size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN;
        XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE));
        f_acc(acc, input + nb_blocks*block_len, secret, nbStripes);

        /* last stripe */
        {   const xxh_u8* const p = input + len - XXH_STRIPE_LEN;
#define XXH_SECRET_LASTACC_START 7  /* not aligned on 8, last secret is different from acc & scrambler */
            XXH3_accumulate_512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START);
    }   }
}

XXH_FORCE_INLINE xxh_u64
XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret)
{
    return XXH3_mul128_fold64(
               acc[0] ^ XXH_readLE64(secret),
               acc[1] ^ XXH_readLE64(secret+8) );
}

static XXH_PUREF XXH64_hash_t
XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start)
{
    xxh_u64 result64 = start;
    size_t i = 0;

    for (i = 0; i < 4; i++) {
        result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i);
#if defined(__clang__)                                /* Clang */ \
    && (defined(__arm__) || defined(__thumb__))       /* ARMv7 */ \
    && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */  \
    && !defined(XXH_ENABLE_AUTOVECTORIZE)             /* Define to disable */
        /*
         * UGLY HACK:
         * Prevent autovectorization on Clang ARMv7-a. Exact same problem as
         * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b.
         * XXH3_64bits, len == 256, Snapdragon 835:
         *   without hack: 2063.7 MB/s
         *   with hack:    2560.7 MB/s
         */
        XXH_COMPILER_GUARD(result64);
#endif
    }

    return XXH3_avalanche(result64);
}

/* do not align on 8, so that the secret is different from the accumulator */
#define XXH_SECRET_MERGEACCS_START 11

static XXH_PUREF XXH64_hash_t
XXH3_finalizeLong_64b(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 len)
{
    return XXH3_mergeAccs(acc, secret + XXH_SECRET_MERGEACCS_START, len * XXH_PRIME64_1);
}

#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \
                        XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 }

XXH_FORCE_INLINE XXH64_hash_t
XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len,
                           const void* XXH_RESTRICT secret, size_t secretSize,
                           XXH3_f_accumulate f_acc,
                           XXH3_f_scrambleAcc f_scramble)
{
    XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC;

    XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc, f_scramble);

    /* converge into final hash */
    XXH_STATIC_ASSERT(sizeof(acc) == 64);
    XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
    return XXH3_finalizeLong_64b(acc, (const xxh_u8*)secret, (xxh_u64)len);
}

/*
 * It's important for performance to transmit secret's size (when it's static)
 * so that the compiler can properly optimize the vectorized loop.
 * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set.
 * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE
 * breaks -Og, this is XXH_NO_INLINE.
 */
XXH3_WITH_SECRET_INLINE XXH64_hash_t
XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len,
                             XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
{
    (void)seed64;
    return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate, XXH3_scrambleAcc);
}

/*
 * It's preferable for performance that XXH3_hashLong is not inlined,
 * as it results in a smaller function for small data, easier to the instruction cache.
 * Note that inside this no_inline function, we do inline the internal loop,
 * and provide a statically defined secret size to allow optimization of vector loop.
 */
XXH_NO_INLINE XXH_PUREF XXH64_hash_t
XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len,
                          XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
{
    (void)seed64; (void)secret; (void)secretLen;
    return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate, XXH3_scrambleAcc);
}

/*
 * XXH3_hashLong_64b_withSeed():
 * Generate a custom key based on alteration of default XXH3_kSecret with the seed,
 * and then use this key for long mode hashing.
 *
 * This operation is decently fast but nonetheless costs a little bit of time.
 * Try to avoid it whenever possible (typically when seed==0).
 *
 * It's important for performance that XXH3_hashLong is not inlined. Not sure
 * why (uop cache maybe?), but the difference is large and easily measurable.
 */
XXH_FORCE_INLINE XXH64_hash_t
XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len,
                                    XXH64_hash_t seed,
                                    XXH3_f_accumulate f_acc,
                                    XXH3_f_scrambleAcc f_scramble,
                                    XXH3_f_initCustomSecret f_initSec)
{
#if XXH_SIZE_OPT <= 0
    if (seed == 0)
        return XXH3_hashLong_64b_internal(input, len,
                                          XXH3_kSecret, sizeof(XXH3_kSecret),
                                          f_acc, f_scramble);
#endif
    {   XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
        f_initSec(secret, seed);
        return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret),
                                          f_acc, f_scramble);
    }
}

/*
 * It's important for performance that XXH3_hashLong is not inlined.
 */
XXH_NO_INLINE XXH64_hash_t
XXH3_hashLong_64b_withSeed(const void* XXH_RESTRICT input, size_t len,
                           XXH64_hash_t seed, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
{
    (void)secret; (void)secretLen;
    return XXH3_hashLong_64b_withSeed_internal(input, len, seed,
                XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret);
}


typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t,
                                          XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t);

XXH_FORCE_INLINE XXH64_hash_t
XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len,
                     XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen,
                     XXH3_hashLong64_f f_hashLong)
{
    XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN);
    /*
     * If an action is to be taken if `secretLen` condition is not respected,
     * it should be done here.
     * For now, it's a contract pre-condition.
     * Adding a check and a branch here would cost performance at every hash.
     * Also, note that function signature doesn't offer room to return an error.
     */
    if (len <= 16)
        return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64);
    if (len <= 128)
        return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
    if (len <= XXH3_MIDSIZE_MAX)
        return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
    return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen);
}


/* ===   Public entry point   === */

/*! @ingroup XXH3_family */
XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length)
{
    return XXH3_64bits_internal(input, length, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default);
}

/*! @ingroup XXH3_family */
XXH_PUBLIC_API XXH64_hash_t
XXH3_64bits_withSecret(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize)
{
    return XXH3_64bits_internal(input, length, 0, secret, secretSize, XXH3_hashLong_64b_withSecret);
}

/*! @ingroup XXH3_family */
XXH_PUBLIC_API XXH64_hash_t
XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed)
{
    return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed);
}

XXH_PUBLIC_API XXH64_hash_t
XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed)
{
    if (length <= XXH3_MIDSIZE_MAX)
        return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL);
    return XXH3_hashLong_64b_withSecret(input, length, seed, (const xxh_u8*)secret, secretSize);
}


/* ===   XXH3 streaming   === */
#ifndef XXH_NO_STREAM
/*
 * Malloc's a pointer that is always aligned to @align.
 *
 * This must be freed with `XXH_alignedFree()`.
 *
 * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte
 * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2
 * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON.
 *
 * This underalignment previously caused a rather obvious crash which went
 * completely unnoticed due to XXH3_createState() not actually being tested.
 * Credit to RedSpah for noticing this bug.
 *
 * The alignment is done manually: Functions like posix_memalign or _mm_malloc
 * are avoided: To maintain portability, we would have to write a fallback
 * like this anyways, and besides, testing for the existence of library
 * functions without relying on external build tools is impossible.
 *
 * The method is simple: Overallocate, manually align, and store the offset
 * to the original behind the returned pointer.
 *
 * Align must be a power of 2 and 8 <= align <= 128.
 */
static XXH_MALLOCF void* XXH_alignedMalloc(size_t s, size_t align)
{
    XXH_ASSERT(align <= 128 && align >= 8); /* range check */
    XXH_ASSERT((align & (align-1)) == 0);   /* power of 2 */
    XXH_ASSERT(s != 0 && s < (s + align));  /* empty/overflow */
    {   /* Overallocate to make room for manual realignment and an offset byte */
        xxh_u8* base = (xxh_u8*)XXH_malloc(s + align);
        if (base != NULL) {
            /*
             * Get the offset needed to align this pointer.
             *
             * Even if the returned pointer is aligned, there will always be
             * at least one byte to store the offset to the original pointer.
             */
            size_t offset = align - ((size_t)base & (align - 1)); /* base % align */
            /* Add the offset for the now-aligned pointer */
            xxh_u8* ptr = base + offset;

            XXH_ASSERT((size_t)ptr % align == 0);

            /* Store the offset immediately before the returned pointer. */
            ptr[-1] = (xxh_u8)offset;
            return ptr;
        }
        return NULL;
    }
}
/*
 * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass
 * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout.
 */
static void XXH_alignedFree(void* p)
{
    if (p != NULL) {
        xxh_u8* ptr = (xxh_u8*)p;
        /* Get the offset byte we added in XXH_malloc. */
        xxh_u8 offset = ptr[-1];
        /* Free the original malloc'd pointer */
        xxh_u8* base = ptr - offset;
        XXH_free(base);
    }
}
/*! @ingroup XXH3_family */
/*!
 * @brief Allocate an @ref XXH3_state_t.
 *
 * @return An allocated pointer of @ref XXH3_state_t on success.
 * @return `NULL` on failure.
 *
 * @note Must be freed with XXH3_freeState().
 *
 * @see @ref streaming_example "Streaming Example"
 */
XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void)
{
    XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64);
    if (state==NULL) return NULL;
    XXH3_INITSTATE(state);
    return state;
}

/*! @ingroup XXH3_family */
/*!
 * @brief Frees an @ref XXH3_state_t.
 *
 * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState().
 *
 * @return @ref XXH_OK.
 *
 * @note Must be allocated with XXH3_createState().
 *
 * @see @ref streaming_example "Streaming Example"
 */
XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr)
{
    XXH_alignedFree(statePtr);
    return XXH_OK;
}

/*! @ingroup XXH3_family */
XXH_PUBLIC_API void
XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state)
{
    XXH_memcpy(dst_state, src_state, sizeof(*dst_state));
}

static void
XXH3_reset_internal(XXH3_state_t* statePtr,
                    XXH64_hash_t seed,
                    const void* secret, size_t secretSize)
{
    size_t const initStart = offsetof(XXH3_state_t, bufferedSize);
    size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart;
    XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart);
    XXH_ASSERT(statePtr != NULL);
    /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */
    XXH_memset((char*)statePtr + initStart, 0, initLength);
    statePtr->acc[0] = XXH_PRIME32_3;
    statePtr->acc[1] = XXH_PRIME64_1;
    statePtr->acc[2] = XXH_PRIME64_2;
    statePtr->acc[3] = XXH_PRIME64_3;
    statePtr->acc[4] = XXH_PRIME64_4;
    statePtr->acc[5] = XXH_PRIME32_2;
    statePtr->acc[6] = XXH_PRIME64_5;
    statePtr->acc[7] = XXH_PRIME32_1;
    statePtr->seed = seed;
    statePtr->useSeed = (seed != 0);
    statePtr->extSecret = (const unsigned char*)secret;
    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
    statePtr->secretLimit = secretSize - XXH_STRIPE_LEN;
    statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE;
}

/*! @ingroup XXH3_family */
XXH_PUBLIC_API XXH_errorcode
XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr)
{
    if (statePtr == NULL) return XXH_ERROR;
    XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE);
    return XXH_OK;
}

/*! @ingroup XXH3_family */
XXH_PUBLIC_API XXH_errorcode
XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize)
{
    if (statePtr == NULL) return XXH_ERROR;
    XXH3_reset_internal(statePtr, 0, secret, secretSize);
    if (secret == NULL) return XXH_ERROR;
    if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
    return XXH_OK;
}

/*! @ingroup XXH3_family */
XXH_PUBLIC_API XXH_errorcode
XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed)
{
    if (statePtr == NULL) return XXH_ERROR;
    if (seed==0) return XXH3_64bits_reset(statePtr);
    if ((seed != statePtr->seed) || (statePtr->extSecret != NULL))
        XXH3_initCustomSecret(statePtr->customSecret, seed);
    XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE);
    return XXH_OK;
}

/*! @ingroup XXH3_family */
XXH_PUBLIC_API XXH_errorcode
XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed64)
{
    if (statePtr == NULL) return XXH_ERROR;
    if (secret == NULL) return XXH_ERROR;
    if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
    XXH3_reset_internal(statePtr, seed64, secret, secretSize);
    statePtr->useSeed = 1; /* always, even if seed64==0 */
    return XXH_OK;
}

/*!
 * @internal
 * @brief Processes a large input for XXH3_update() and XXH3_digest_long().
 *
 * Unlike XXH3_hashLong_internal_loop(), this can process data that overlaps a block.
 *
 * @param acc                Pointer to the 8 accumulator lanes
 * @param nbStripesSoFarPtr  In/out pointer to the number of leftover stripes in the block*
 * @param nbStripesPerBlock  Number of stripes in a block
 * @param input              Input pointer
 * @param nbStripes          Number of stripes to process
 * @param secret             Secret pointer
 * @param secretLimit        Offset of the last block in @p secret
 * @param f_acc              Pointer to an XXH3_accumulate implementation
 * @param f_scramble         Pointer to an XXH3_scrambleAcc implementation
 * @return                   Pointer past the end of @p input after processing
 */
XXH_FORCE_INLINE const xxh_u8 *
XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc,
                    size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock,
                    const xxh_u8* XXH_RESTRICT input, size_t nbStripes,
                    const xxh_u8* XXH_RESTRICT secret, size_t secretLimit,
                    XXH3_f_accumulate f_acc,
                    XXH3_f_scrambleAcc f_scramble)
{
    const xxh_u8* initialSecret = secret + *nbStripesSoFarPtr * XXH_SECRET_CONSUME_RATE;
    /* Process full blocks */
    if (nbStripes >= (nbStripesPerBlock - *nbStripesSoFarPtr)) {
        /* Process the initial partial block... */
        size_t nbStripesThisIter = nbStripesPerBlock - *nbStripesSoFarPtr;

        do {
            /* Accumulate and scramble */
            f_acc(acc, input, initialSecret, nbStripesThisIter);
            f_scramble(acc, secret + secretLimit);
            input += nbStripesThisIter * XXH_STRIPE_LEN;
            nbStripes -= nbStripesThisIter;
            /* Then continue the loop with the full block size */
            nbStripesThisIter = nbStripesPerBlock;
            initialSecret = secret;
        } while (nbStripes >= nbStripesPerBlock);
        *nbStripesSoFarPtr = 0;
    }
    /* Process a partial block */
    if (nbStripes > 0) {
        f_acc(acc, input, initialSecret, nbStripes);
        input += nbStripes * XXH_STRIPE_LEN;
        *nbStripesSoFarPtr += nbStripes;
    }
    /* Return end pointer */
    return input;
}

#ifndef XXH3_STREAM_USE_STACK
# if XXH_SIZE_OPT <= 0 && !defined(__clang__) /* clang doesn't need additional stack space */
#   define XXH3_STREAM_USE_STACK 1
# endif
#endif
/* This function accepts f_acc and f_scramble as function pointers,
 * making it possible to implement multiple variants with different acc & scramble stages.
 * This is notably useful to implement multiple vector variants with different intrinsics.
 */
XXH_FORCE_INLINE XXH_errorcode
XXH3_update(XXH3_state_t* XXH_RESTRICT const state,
            const xxh_u8* XXH_RESTRICT input, size_t len,
            XXH3_f_accumulate f_acc,
            XXH3_f_scrambleAcc f_scramble)
{
    if (input==NULL) {
        XXH_ASSERT(len == 0);
        return XXH_OK;
    }

    XXH_ASSERT(state != NULL);
    state->totalLen += len;

    /* small input : just fill in tmp buffer */
    XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE);
    if (len <= XXH3_INTERNALBUFFER_SIZE - state->bufferedSize) {
        XXH_memcpy(state->buffer + state->bufferedSize, input, len);
        state->bufferedSize += (XXH32_hash_t)len;
        return XXH_OK;
    }

    {   const xxh_u8* const bEnd = input + len;
        const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1
        /* For some reason, gcc and MSVC seem to suffer greatly
         * when operating accumulators directly into state.
         * Operating into stack space seems to enable proper optimization.
         * clang, on the other hand, doesn't seem to need this trick */
        XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8];
        XXH_memcpy(acc, state->acc, sizeof(acc));
#else
        xxh_u64* XXH_RESTRICT const acc = state->acc;
#endif

        /* total input is now > XXH3_INTERNALBUFFER_SIZE */
        #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN)
        XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0);   /* clean multiple */

        /*
         * Internal buffer is partially filled (always, except at beginning)
         * Complete it, then consume it.
         */
        if (state->bufferedSize) {
            size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize;
            XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize);
            input += loadSize;
            XXH3_consumeStripes(acc,
                               &state->nbStripesSoFar, state->nbStripesPerBlock,
                                state->buffer, XXH3_INTERNALBUFFER_STRIPES,
                                secret, state->secretLimit,
                                f_acc, f_scramble);
            state->bufferedSize = 0;
        }
        XXH_ASSERT(input < bEnd);
        if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) {
            size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN;
            input = XXH3_consumeStripes(acc,
                                       &state->nbStripesSoFar, state->nbStripesPerBlock,
                                       input, nbStripes,
                                       secret, state->secretLimit,
                                       f_acc, f_scramble);
            XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN);

        }
        /* Some remaining input (always) : buffer it */
        XXH_ASSERT(input < bEnd);
        XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE);
        XXH_ASSERT(state->bufferedSize == 0);
        XXH_memcpy(state->buffer, input, (size_t)(bEnd-input));
        state->bufferedSize = (XXH32_hash_t)(bEnd-input);
#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1
        /* save stack accumulators into state */
        XXH_memcpy(state->acc, acc, sizeof(acc));
#endif
    }

    return XXH_OK;
}

/*
 * Both XXH3_64bits_update and XXH3_128bits_update use this routine.
 */
XXH_NO_INLINE XXH_errorcode
XXH3_update_regular(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len)
{
    return XXH3_update(state, (const xxh_u8*)input, len,
                       XXH3_accumulate, XXH3_scrambleAcc);
}

/*! @ingroup XXH3_family */
XXH_PUBLIC_API XXH_errorcode
XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len)
{
    return XXH3_update_regular(state, input, len);
}


XXH_FORCE_INLINE void
XXH3_digest_long (XXH64_hash_t* acc,
                  const XXH3_state_t* state,
                  const unsigned char* secret)
{
    xxh_u8 lastStripe[XXH_STRIPE_LEN];
    const xxh_u8* lastStripePtr;

    /*
     * Digest on a local copy. This way, the state remains unaltered, and it can
     * continue ingesting more input afterwards.
     */
    XXH_memcpy(acc, state->acc, sizeof(state->acc));
    if (state->bufferedSize >= XXH_STRIPE_LEN) {
        /* Consume remaining stripes then point to remaining data in buffer */
        size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN;
        size_t nbStripesSoFar = state->nbStripesSoFar;
        XXH3_consumeStripes(acc,
                           &nbStripesSoFar, state->nbStripesPerBlock,
                            state->buffer, nbStripes,
                            secret, state->secretLimit,
                            XXH3_accumulate, XXH3_scrambleAcc);
        lastStripePtr = state->buffer + state->bufferedSize - XXH_STRIPE_LEN;
    } else {  /* bufferedSize < XXH_STRIPE_LEN */
        /* Copy to temp buffer */
        size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize;
        XXH_ASSERT(state->bufferedSize > 0);  /* there is always some input buffered */
        XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize);
        XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize);
        lastStripePtr = lastStripe;
    }
    /* Last stripe */
    XXH3_accumulate_512(acc,
                        lastStripePtr,
                        secret + state->secretLimit - XXH_SECRET_LASTACC_START);
}

/*! @ingroup XXH3_family */
XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* state)
{
    const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
    if (state->totalLen > XXH3_MIDSIZE_MAX) {
        XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB];
        XXH3_digest_long(acc, state, secret);
        return XXH3_finalizeLong_64b(acc, secret, (xxh_u64)state->totalLen);
    }
    /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */
    if (state->useSeed)
        return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed);
    return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen),
                                  secret, state->secretLimit + XXH_STRIPE_LEN);
}
#endif /* !XXH_NO_STREAM */


/* ==========================================
 * XXH3 128 bits (a.k.a XXH128)
 * ==========================================
 * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant,
 * even without counting the significantly larger output size.
 *
 * For example, extra steps are taken to avoid the seed-dependent collisions
 * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B).
 *
 * This strength naturally comes at the cost of some speed, especially on short
 * lengths. Note that longer hashes are about as fast as the 64-bit version
 * due to it using only a slight modification of the 64-bit loop.
 *
 * XXH128 is also more oriented towards 64-bit machines. It is still extremely
 * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64).
 */

XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
{
    /* A doubled version of 1to3_64b with different constants. */
    XXH_ASSERT(input != NULL);
    XXH_ASSERT(1 <= len && len <= 3);
    XXH_ASSERT(secret != NULL);
    /*
     * len = 1: combinedl = { input[0], 0x01, input[0], input[0] }
     * len = 2: combinedl = { input[1], 0x02, input[0], input[1] }
     * len = 3: combinedl = { input[2], 0x03, input[0], input[1] }
     */
    {   xxh_u8 const c1 = input[0];
        xxh_u8 const c2 = input[len >> 1];
        xxh_u8 const c3 = input[len - 1];
        xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24)
                                | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8);
        xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13);
        xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed;
        xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed;
        xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl;
        xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph;
        XXH128_hash_t h128;
        h128.low64  = XXH64_avalanche(keyed_lo);
        h128.high64 = XXH64_avalanche(keyed_hi);
        return h128;
    }
}

XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
{
    XXH_ASSERT(input != NULL);
    XXH_ASSERT(secret != NULL);
    XXH_ASSERT(4 <= len && len <= 8);
    seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32;
    {   xxh_u32 const input_lo = XXH_readLE32(input);
        xxh_u32 const input_hi = XXH_readLE32(input + len - 4);
        xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32);
        xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed;
        xxh_u64 const keyed = input_64 ^ bitflip;

        /* Shift len to the left to ensure it is even, this avoids even multiplies. */
        XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2));

        m128.high64 += (m128.low64 << 1);
        m128.low64  ^= (m128.high64 >> 3);

        m128.low64   = XXH_xorshift64(m128.low64, 35);
        m128.low64  *= PRIME_MX2;
        m128.low64   = XXH_xorshift64(m128.low64, 28);
        m128.high64  = XXH3_avalanche(m128.high64);
        return m128;
    }
}

XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
{
    XXH_ASSERT(input != NULL);
    XXH_ASSERT(secret != NULL);
    XXH_ASSERT(9 <= len && len <= 16);
    {   xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed;
        xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed;
        xxh_u64 const input_lo = XXH_readLE64(input);
        xxh_u64       input_hi = XXH_readLE64(input + len - 8);
        XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1);
        /*
         * Put len in the middle of m128 to ensure that the length gets mixed to
         * both the low and high bits in the 128x64 multiply below.
         */
        m128.low64 += (xxh_u64)(len - 1) << 54;
        input_hi   ^= bitfliph;
        /*
         * Add the high 32 bits of input_hi to the high 32 bits of m128, then
         * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to
         * the high 64 bits of m128.
         *
         * The best approach to this operation is different on 32-bit and 64-bit.
         */
        if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */
            /*
             * 32-bit optimized version, which is more readable.
             *
             * On 32-bit, it removes an ADC and delays a dependency between the two
             * halves of m128.high64, but it generates an extra mask on 64-bit.
             */
            m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2);
        } else {
            /*
             * 64-bit optimized (albeit more confusing) version.
             *
             * Uses some properties of addition and multiplication to remove the mask:
             *
             * Let:
             *    a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF)
             *    b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000)
             *    c = XXH_PRIME32_2
             *
             *    a + (b * c)
             * Inverse Property: x + y - x == y
             *    a + (b * (1 + c - 1))
             * Distributive Property: x * (y + z) == (x * y) + (x * z)
             *    a + (b * 1) + (b * (c - 1))
             * Identity Property: x * 1 == x
             *    a + b + (b * (c - 1))
             *
             * Substitute a, b, and c:
             *    input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1))
             *
             * Since input_hi.hi + input_hi.lo == input_hi, we get this:
             *    input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1))
             */
            m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1);
        }
        /* m128 ^= XXH_swap64(m128 >> 64); */
        m128.low64  ^= XXH_swap64(m128.high64);

        {   /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */
            XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2);
            h128.high64 += m128.high64 * XXH_PRIME64_2;

            h128.low64   = XXH3_avalanche(h128.low64);
            h128.high64  = XXH3_avalanche(h128.high64);
            return h128;
    }   }
}

/*
 * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN
 */
XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
{
    XXH_ASSERT(len <= 16);
    {   if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed);
        if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed);
        if (len) return XXH3_len_1to3_128b(input, len, secret, seed);
        {   XXH128_hash_t h128;
            xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72);
            xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88);
            h128.low64 = XXH64_avalanche(seed ^ bitflipl);
            h128.high64 = XXH64_avalanche( seed ^ bitfliph);
            return h128;
    }   }
}

/*
 * A bit slower than XXH3_mix16B, but handles multiply by zero better.
 */
XXH_FORCE_INLINE XXH128_hash_t
XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2,
              const xxh_u8* secret, XXH64_hash_t seed)
{
    acc.low64  += XXH3_mix16B (input_1, secret+0, seed);
    acc.low64  ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8);
    acc.high64 += XXH3_mix16B (input_2, secret+16, seed);
    acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8);
    return acc;
}


XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len,
                      const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
                      XXH64_hash_t seed)
{
    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
    XXH_ASSERT(16 < len && len <= 128);

    {   XXH128_hash_t acc;
        acc.low64 = len * XXH_PRIME64_1;
        acc.high64 = 0;

#if XXH_SIZE_OPT >= 1
        {
            /* Smaller, but slightly slower. */
            unsigned int i = (unsigned int)(len - 1) / 32;
            do {
                acc = XXH128_mix32B(acc, input+16*i, input+len-16*(i+1), secret+32*i, seed);
            } while (i-- != 0);
        }
#else
        if (len > 32) {
            if (len > 64) {
                if (len > 96) {
                    acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed);
                }
                acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed);
            }
            acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed);
        }
        acc = XXH128_mix32B(acc, input, input+len-16, secret, seed);
#endif
        {   XXH128_hash_t h128;
            h128.low64  = acc.low64 + acc.high64;
            h128.high64 = (acc.low64    * XXH_PRIME64_1)
                        + (acc.high64   * XXH_PRIME64_4)
                        + ((len - seed) * XXH_PRIME64_2);
            h128.low64  = XXH3_avalanche(h128.low64);
            h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64);
            return h128;
        }
    }
}

XXH_NO_INLINE XXH_PUREF XXH128_hash_t
XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len,
                       const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
                       XXH64_hash_t seed)
{
    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
    XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);

    {   XXH128_hash_t acc;
        unsigned i;
        acc.low64 = len * XXH_PRIME64_1;
        acc.high64 = 0;
        /*
         *  We set as `i` as offset + 32. We do this so that unchanged
         * `len` can be used as upper bound. This reaches a sweet spot
         * where both x86 and aarch64 get simple agen and good codegen
         * for the loop.
         */
        for (i = 32; i < 160; i += 32) {
            acc = XXH128_mix32B(acc,
                                input  + i - 32,
                                input  + i - 16,
                                secret + i - 32,
                                seed);
        }
        acc.low64 = XXH3_avalanche(acc.low64);
        acc.high64 = XXH3_avalanche(acc.high64);
        /*
         * NB: `i <= len` will duplicate the last 32-bytes if
         * len % 32 was zero. This is an unfortunate necessity to keep
         * the hash result stable.
         */
        for (i=160; i <= len; i += 32) {
            acc = XXH128_mix32B(acc,
                                input + i - 32,
                                input + i - 16,
                                secret + XXH3_MIDSIZE_STARTOFFSET + i - 160,
                                seed);
        }
        /* last bytes */
        acc = XXH128_mix32B(acc,
                            input + len - 16,
                            input + len - 32,
                            secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16,
                            (XXH64_hash_t)0 - seed);

        {   XXH128_hash_t h128;
            h128.low64  = acc.low64 + acc.high64;
            h128.high64 = (acc.low64    * XXH_PRIME64_1)
                        + (acc.high64   * XXH_PRIME64_4)
                        + ((len - seed) * XXH_PRIME64_2);
            h128.low64  = XXH3_avalanche(h128.low64);
            h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64);
            return h128;
        }
    }
}

static XXH_PUREF XXH128_hash_t
XXH3_finalizeLong_128b(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, size_t secretSize, xxh_u64 len)
{
    XXH128_hash_t h128;
    h128.low64 = XXH3_finalizeLong_64b(acc, secret, len);
    h128.high64 = XXH3_mergeAccs(acc, secret + secretSize
                                             - XXH_STRIPE_LEN - XXH_SECRET_MERGEACCS_START,
                                             ~(len * XXH_PRIME64_2));
    return h128;
}

XXH_FORCE_INLINE XXH128_hash_t
XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len,
                            const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
                            XXH3_f_accumulate f_acc,
                            XXH3_f_scrambleAcc f_scramble)
{
    XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC;

    XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc, f_scramble);

    /* converge into final hash */
    XXH_STATIC_ASSERT(sizeof(acc) == 64);
    XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
    return XXH3_finalizeLong_128b(acc, secret, secretSize, (xxh_u64)len);
}

/*
 * It's important for performance that XXH3_hashLong() is not inlined.
 */
XXH_NO_INLINE XXH_PUREF XXH128_hash_t
XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len,
                           XXH64_hash_t seed64,
                           const void* XXH_RESTRICT secret, size_t secretLen)
{
    (void)seed64; (void)secret; (void)secretLen;
    return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret),
                                       XXH3_accumulate, XXH3_scrambleAcc);
}

/*
 * It's important for performance to pass @p secretLen (when it's static)
 * to the compiler, so that it can properly optimize the vectorized loop.
 *
 * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE
 * breaks -Og, this is XXH_NO_INLINE.
 */
XXH3_WITH_SECRET_INLINE XXH128_hash_t
XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len,
                              XXH64_hash_t seed64,
                              const void* XXH_RESTRICT secret, size_t secretLen)
{
    (void)seed64;
    return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen,
                                       XXH3_accumulate, XXH3_scrambleAcc);
}

XXH_FORCE_INLINE XXH128_hash_t
XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len,
                                XXH64_hash_t seed64,
                                XXH3_f_accumulate f_acc,
                                XXH3_f_scrambleAcc f_scramble,
                                XXH3_f_initCustomSecret f_initSec)
{
    if (seed64 == 0)
        return XXH3_hashLong_128b_internal(input, len,
                                           XXH3_kSecret, sizeof(XXH3_kSecret),
                                           f_acc, f_scramble);
    {   XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
        f_initSec(secret, seed64);
        return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret),
                                           f_acc, f_scramble);
    }
}

/*
 * It's important for performance that XXH3_hashLong is not inlined.
 */
XXH_NO_INLINE XXH128_hash_t
XXH3_hashLong_128b_withSeed(const void* input, size_t len,
                            XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen)
{
    (void)secret; (void)secretLen;
    return XXH3_hashLong_128b_withSeed_internal(input, len, seed64,
                XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret);
}

typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t,
                                            XXH64_hash_t, const void* XXH_RESTRICT, size_t);

XXH_FORCE_INLINE XXH128_hash_t
XXH3_128bits_internal(const void* input, size_t len,
                      XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen,
                      XXH3_hashLong128_f f_hl128)
{
    XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN);
    /*
     * If an action is to be taken if `secret` conditions are not respected,
     * it should be done here.
     * For now, it's a contract pre-condition.
     * Adding a check and a branch here would cost performance at every hash.
     */
    if (len <= 16)
        return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64);
    if (len <= 128)
        return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
    if (len <= XXH3_MIDSIZE_MAX)
        return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
    return f_hl128(input, len, seed64, secret, secretLen);
}


/* ===   Public XXH128 API   === */

/*! @ingroup XXH3_family */
XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* input, size_t len)
{
    return XXH3_128bits_internal(input, len, 0,
                                 XXH3_kSecret, sizeof(XXH3_kSecret),
                                 XXH3_hashLong_128b_default);
}

/*! @ingroup XXH3_family */
XXH_PUBLIC_API XXH128_hash_t
XXH3_128bits_withSecret(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize)
{
    return XXH3_128bits_internal(input, len, 0,
                                 (const xxh_u8*)secret, secretSize,
                                 XXH3_hashLong_128b_withSecret);
}

/*! @ingroup XXH3_family */
XXH_PUBLIC_API XXH128_hash_t
XXH3_128bits_withSeed(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed)
{
    return XXH3_128bits_internal(input, len, seed,
                                 XXH3_kSecret, sizeof(XXH3_kSecret),
                                 XXH3_hashLong_128b_withSeed);
}

/*! @ingroup XXH3_family */
XXH_PUBLIC_API XXH128_hash_t
XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed)
{
    if (len <= XXH3_MIDSIZE_MAX)
        return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL);
    return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize);
}

/*! @ingroup XXH3_family */
XXH_PUBLIC_API XXH128_hash_t
XXH128(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed)
{
    return XXH3_128bits_withSeed(input, len, seed);
}


/* ===   XXH3 128-bit streaming   === */
#ifndef XXH_NO_STREAM
/*
 * All initialization and update functions are identical to 64-bit streaming variant.
 * The only difference is the finalization routine.
 */

/*! @ingroup XXH3_family */
XXH_PUBLIC_API XXH_errorcode
XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr)
{
    return XXH3_64bits_reset(statePtr);
}

/*! @ingroup XXH3_family */
XXH_PUBLIC_API XXH_errorcode
XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize)
{
    return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize);
}

/*! @ingroup XXH3_family */
XXH_PUBLIC_API XXH_errorcode
XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed)
{
    return XXH3_64bits_reset_withSeed(statePtr, seed);
}

/*! @ingroup XXH3_family */
XXH_PUBLIC_API XXH_errorcode
XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed)
{
    return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed);
}

/*! @ingroup XXH3_family */
XXH_PUBLIC_API XXH_errorcode
XXH3_128bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len)
{
    return XXH3_update_regular(state, input, len);
}

/*! @ingroup XXH3_family */
XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* state)
{
    const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
    if (state->totalLen > XXH3_MIDSIZE_MAX) {
        XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB];
        XXH3_digest_long(acc, state, secret);
        XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
        return XXH3_finalizeLong_128b(acc, secret, state->secretLimit + XXH_STRIPE_LEN,  (xxh_u64)state->totalLen);
    }
    /* len <= XXH3_MIDSIZE_MAX : short code */
    if (state->useSeed)
        return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed);
    return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen),
                                   secret, state->secretLimit + XXH_STRIPE_LEN);
}
#endif /* !XXH_NO_STREAM */
/* 128-bit utility functions */

/* return : 1 is equal, 0 if different */
/*! @ingroup XXH3_family */
XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2)
{
    /* note : XXH128_hash_t is compact, it has no padding byte */
    return !(XXH_memcmp(&h1, &h2, sizeof(h1)));
}

/* This prototype is compatible with stdlib's qsort().
 * @return : >0 if *h128_1  > *h128_2
 *           <0 if *h128_1  < *h128_2
 *           =0 if *h128_1 == *h128_2  */
/*! @ingroup XXH3_family */
XXH_PUBLIC_API int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2)
{
    XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1;
    XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2;
    int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64);
    /* note : bets that, in most cases, hash values are different */
    if (hcmp) return hcmp;
    return (h1.low64 > h2.low64) - (h2.low64 > h1.low64);
}


/*======   Canonical representation   ======*/
/*! @ingroup XXH3_family */
XXH_PUBLIC_API void
XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash)
{
    XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t));
    if (XXH_CPU_LITTLE_ENDIAN) {
        hash.high64 = XXH_swap64(hash.high64);
        hash.low64  = XXH_swap64(hash.low64);
    }
    XXH_memcpy(dst, &hash.high64, sizeof(hash.high64));
    XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64));
}

/*! @ingroup XXH3_family */
XXH_PUBLIC_API XXH128_hash_t
XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src)
{
    XXH128_hash_t h;
    h.high64 = XXH_readBE64(src);
    h.low64  = XXH_readBE64(src->digest + 8);
    return h;
}



/* ==========================================
 * Secret generators
 * ==========================================
 */
#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x))

XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128)
{
    XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 );
    XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 );
}

/*! @ingroup XXH3_family */
XXH_PUBLIC_API XXH_errorcode
XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize)
{
#if (XXH_DEBUGLEVEL >= 1)
    XXH_ASSERT(secretBuffer != NULL);
    XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
#else
    /* production mode, assert() are disabled */
    if (secretBuffer == NULL) return XXH_ERROR;
    if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
#endif

    if (customSeedSize == 0) {
        customSeed = XXH3_kSecret;
        customSeedSize = XXH_SECRET_DEFAULT_SIZE;
    }
#if (XXH_DEBUGLEVEL >= 1)
    XXH_ASSERT(customSeed != NULL);
#else
    if (customSeed == NULL) return XXH_ERROR;
#endif

    /* Fill secretBuffer with a copy of customSeed - repeat as needed */
    {   size_t pos = 0;
        while (pos < secretSize) {
            size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize);
            XXH_memcpy((char*)secretBuffer + pos, customSeed, toCopy);
            pos += toCopy;
    }   }

    {   size_t const nbSeg16 = secretSize / 16;
        size_t n;
        XXH128_canonical_t scrambler;
        XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0));
        for (n=0; n<nbSeg16; n++) {
            XXH128_hash_t const h128 = XXH128(&scrambler, sizeof(scrambler), n);
            XXH3_combine16((char*)secretBuffer + n*16, h128);
        }
        /* last segment */
        XXH3_combine16((char*)secretBuffer + secretSize - 16, XXH128_hashFromCanonical(&scrambler));
    }
    return XXH_OK;
}

/*! @ingroup XXH3_family */
XXH_PUBLIC_API void
XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed)
{
    XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
    XXH3_initCustomSecret(secret, seed);
    XXH_ASSERT(secretBuffer != NULL);
    XXH_memcpy(secretBuffer, secret, XXH_SECRET_DEFAULT_SIZE);
}



/* Pop our optimization override from above */
#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \
  && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
  && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */
#  pragma GCC pop_options
#endif

#endif  /* XXH_NO_LONG_LONG */

#endif  /* XXH_NO_XXH3 */

/*!
 * @}
 */
#endif  /* XXH_IMPLEMENTATION */


#if defined (__cplusplus) && !defined(XXH_NO_EXTERNC_GUARD)
} /* extern "C" */
#endif