add-field-to-eiq-log.py

import json
from datetime import datetime

TRACE_ID='45b893611ffd06a3cfb3cbfe93e1f601'
RUN_ID='579c4210-5dc3-4562-b3a6-916e090f6d81'

def get_timestamp(timestamp: str) -> datetime:
    return datetime.strptime(timestamp,"%Y-%m-%dT%H:%M:%S.%fZ")

def main():
    with open("data/logfile.ndjson") as fp:
        _data = fp.readlines()

    data = []
    for line in _data:
        if TRACE_ID in line or RUN_ID in line:
            line = json.loads(line)
            data.append(line)

    out = []
    for i,line in enumerate(data):
        if line.get("timestamp", None) is None:
            continue
        if i != 0:
            line["time_taken"] = (get_timestamp(line["timestamp"]) - get_timestamp(data[i-1]["timestamp"])).total_seconds()
            out.append(json.dumps(line))
    print(out[1])

    with open("output.ndjson","w") as fp:
        for i in out:
            fp.write(i + '\n')



if __name__ == "__main__":
    main()