Elasticsearch scrolling query fuction

scrolling_queries.py

import elasticsearch


def scroll_query(es, index, query):
    """

    :param es: an elasticsearch client object created with elasticsearch.Elasticsearch()
    :param index: the name of the log to query (e.g. 'conn', 'timetohello', etc.)
    :param query: elasticsearch query (docs on elastic.co)
    :return: iterator containing results of elastic query
    """
    page = es.search(index=index, scroll='2m', size=10, body=query)
    sid = page['_scroll_id']
    scroll_size = page['hits']['total']

    while (scroll_size > 0):
        page = es.scroll(scroll_id=sid, scroll='2m')
        sid = page['_scroll_id']
        scroll_size = len(page['hits']['hits'])

        for res in page['hits']['hits']:
            yield res['_source']
     

# Example usage
if __name__ == '__main__':
    es_client = elasticsearch.Elasticsearch([ELASTIC_ENDPOINT]) #Replace with your elastic endpoint
    
    lookup = {
        'query': {
            'term': {'id_resp_h': '8.8.8.8'}
        }
    }

    for res in scroll_query(es_client, 'conn', lookup):
        print(res)