Benchmark encoding many small strs vs. one large string

README.md

This assumes it is running in the root of https://github.com/richvdh/json_benchmark

Run as python3 benchmark_encode.py.

benchmark_encode.py

#!/usr/bin/env python
import json
import timeit

# number of runs over the data for each repeat
N_RUNS = 2000000

# number of times to repeat the runs. We'll take the fastest run.
N_REPEATS = 5


def benchmark_encode_once(objs):
    for obj in objs:
        "".join(obj).encoder("utf-8")


def benchmark_encode_each(objs):
    for obj in objs:
        b"".join([part.encode("utf-8") for part in obj])


def run_benchmarks():
    encoder = json.JSONEncoder()

    with open('data/large.json') as f:
        large_obj_data = f.read()
    large_obj = list(encoder.iterencode(json.loads(large_obj_data)))

    with open('data/one-json-per-line.txt') as f:
        small_objs_data = f.readlines()
    small_objs = [list(encoder.iterencode(line)) for line in small_objs_data]

    benchmarks = [
        ('encode once (large obj)', lambda: benchmark_encode_once([large_obj])),
        ('encode once (small objs)', lambda: benchmark_encode_once(small_objs)),
        ('encode each (large obj)', lambda: benchmark_encode_each([large_obj])),
        ('encode each (small objs)', lambda: benchmark_encode_each(small_objs)),
    ]

    print('Running benchmarks...')

    for benchmark_name, fn in benchmarks:
        print('   %s...' % benchmark_name)
        time = timeit.timeit(lambda: fn, number=1)
        print('      first run: %f' % time)

        times = timeit.repeat(lambda: fn, number=N_RUNS, repeat=N_REPEATS)
        best = min(times)
        print('      %i loops, best of %i: %f sec per loop (best total %f)' % (
            N_RUNS, len(times), best / N_RUNS, best
        ))


if __name__ == '__main__':
    run_benchmarks()