pymarc_working.py

#!/usr/bin/python3
import os
import logging
from pymarc import MARCReader
from tqdm import tqdm

def main():
    # clean-up
    try:
        os.system('cls')
        os.system('del testing.log')
    except:
        pass

    # logging
    logging.basicConfig(filename='testing.log',level=logging.DEBUG)

    # progressbar
    #total_records = 3715163
    total_records = 3715163
    pbar = tqdm(total=total_records)

    # open marc file and read contents
    # change the slice to limit files
    for filename in os.listdir('.//data'): # <------- check this limiter
        counter = 0
        with open(f'data/{filename}', 'rb') as fh:
            reader = MARCReader(fh, to_unicode=False, utf8_handling='ignore')

            # progressbar
            pbar.set_description(f"Reading {filename}")

            # loop through records
            for record in wrapper(reader):
                #try:
                counter += 1
                pbar.update(1)

                #except Exception as error:
                #    logging.info(error)
                #    pbar.update(1)
                #    counter += 1
                #    next(reader)

        # log totals
        logging.info(f'{filename} total: {counter}')

    # finish
    pbar.set_description(f"Finished")

    print(counter)


# functions
def wrapper(gen):
    while True:
        try:
            yield next(gen)
        except StopIteration:
            break
        except Exception as error:
            logging.info(error)
            pass

# top-level
if __name__ == "__main__":
    main()

https://stackoverflow.com/questions/13653783/how-to-catch-an-exception-in-the-for-loop-iterator