chunk_file_saver.py

import tempfile

class ChunkWriter:
    pass

class CSVChunkWriter:
    def __init__(self, filename=None):
        self.filename = filename
        if filename:
            self.file_ = open(filename, 'wb')
        else:
            self.file_ = tempfile.NamedTemporaryFile()
        
    def write_chunk(self, df):
        if self.first_write:
            df.to_csv(self.file_, mode='a', encoding='utf-8')
        else:
            df.to_csv(self.file_, mode='a', encoding='utf-8', header=False)
            
    def finish(self):
        pass

class JSONChunkWriter:

    def __init__(self, filename=None):
        self.filename = filename
        if filename:
            self.file_ = open(filename, 'w')
        else:
            self.file_ = tempfile.NamedTemporaryFile()
        self.file_.write('[')
            
    def write_chunk(self, json_string):
        """store_chunk

        :param json_string: output from `json.dumps` function
        :type json_string: str
        """
        json_string_formatted = ', {}'.format(json_string[:-1])  # last character is ']'
        self.file_.write(json_string_formatted)
      
    def finish(self):
        self.file_.write(']')