D 1B rows generator

main.d

import std.parallelism, std.stdio;
import core.cpuid, core.sync.mutex;

void main(string[] args) {
    auto f = File("measurements.txt", "w");
    auto mutex = new shared Mutex();
    scope(exit) f.close();


    writeln("generating measurements");
    const totalRows = 1_000_000_000;
    const chunkSize = 5_000_000;
    const numChunks = totalRows / chunkSize;

    foreach (i; parallel(iota(numChunks))) {
        auto ap = appender!(char[]);
        ap.reserve(chunkSize * 25);

        auto rnd = Xorshift(unpredictableSeed);
        foreach(j; iota(chunkSize)) {
            auto randomStation = stationNames[uniform(0, stationNames.length, rnd)];
            auto measurement = uniform(-50f, 50f, rnd);
            formattedWrite(ap, "%s;%.1f\n", randomStation, measurement);
        }

        mutex.lock();
        f.rawWrite(ap.data);
        mutex.unlock();

        if (i % 10 == 0) {
            writefln("chunk %d/%d generated", i, numChunks);
        }
    }

    writeln("done");
}