Gist for Ubuntu and Debian users
nginx 1.16.1 + push stream module latest
apt-get update -y \
iyvinjose
/ data_loading_utils.py
Last active
March 27, 2025 14:12
Read large files line by line without loading entire file to memory. Supports files of GB size
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| def read_lines_from_file_as_data_chunks(file_name, chunk_size, callback, return_whole_chunk=False): | |
| """ | |
| read file line by line regardless of its size | |
| :param file_name: absolute path of file to read | |
| :param chunk_size: size of data to be read at at time | |
| :param callback: callback method, prototype ----> def callback(data, eof, file_name) | |
| :return: | |
| """ | |
| def read_in_chunks(file_obj, chunk_size=5000): |