single_thread_arrow_parquet_example.py

import pyarrow.parquet as pq
import pyarrow as pa
import pyarrow.dataset as ds
import numpy as np
import pandas as pd
import pathlib
from rich import print
import tqdm

total_iters = 10000
with tqdm.tqdm(total=total_iters) as pbar:
    for i in range(total_iters):
        if pathlib.Path("test.parquet").exists() and i == 0:
            pathlib.Path("test.parquet").unlink()

        table_entry = pa.table(
            [[i**2], [i**3], [i**4]], names=["square", "cube", "fourth_power"]
        )
        if pathlib.Path("test.parquet").exists():
            table = pq.read_table("test.parquet")
            table = pa.concat_tables(
                [
                    table,
                    table_entry,
                ]
            )
        else:
            table = table_entry

        pq.write_table(table, "test.parquet")
        pbar.update(1)
        pbar.set_description(f"{len(table)} entries")