Phil Snyder philerooski

## verify_load_snapshot.py
"""
Verify snapshot data load integrity.

This script checks:
1. LOAD_LOG table for any errors or anomalies
2. Compares tables between PROD_576 and PROD_568 schemas
3. Validates record counts (PROD_576 should have more records)

Usage:
    python verify_snapshot_load.py --database SYNAPSE_RDS_SNAPSHOT \\

## download_all_forms.py
#!/usr/bin/env python3
"""
Download all form data from a specific form group to local directory.
"""

import argparse
import sys
import json
import tempfile
import shutil

## load_snapshot_data.py
"""
Load RDS snapshot data from S3 via Snowflake external stage into tables.

This script supports two modes:
1. Bootstrap mode (--bootstrap-stack): Creates a new schema, external stage,
   file format, and grants privileges before loading data.
2. Manual mode: Loads data into an existing schema with pre-configured stage.

The script dynamically discovers all data types from the S3 stage URL,
creates tables using INFER_SCHEMA from Parquet files, loads the data,

## analyze_load_errors.py
"""
Analyze errors from the LOAD_LOG table.

This script queries the LOAD_LOG table for failed operations,
categorizes errors by type, and groups data types by error category.

This is a complementary script to https://gist.github.com/philerooski/a740b25f066f1ad205344637160aa969
"""

import snowflake.connector

## load_snapshot_data.py
"""
Load snapshot data from Snowflake stage into tables.

This script processes prefixes from PREFIX_LIST table, derives table names,
creates tables using INFER_SCHEMA, and logs all operations to LOAD_LOG.

See `--help` for optional parameter `--only-affected`
"""

import snowflake.connector

## create_confluence_pages.py
import os
import random
import logging
import argparse
import toml
import snowflake.connector
from atlassian import Confluence

# Configure logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')

## gx_basic.py
### Does not work

expectation_suite_name = "my_expectation_suite"
checkpoint_name = "my_checkpoint"

context = gx.get_context()

# # Initialize expectation suite
def init_expectation_suite():
    expectation_suite = context.add_expectation_suite(

## convert_to_parquet.py
"""
Run this script from within the unzipped directory `JMV_fitbit_dta`

Download zipped data here: https://www.synapse.org/Synapse:syn62667431
"""
import pandas as pd
import json
import os


## fitbitsleeplogs_demo.py
"""
A script which uploads validation results and a data validation
report to S3 for the FitbitSleepLogs data type. This was run in
Glue 4.0 while specifying --additional-python-modules great_expectations==0.18.11,boto3==1.24.70
"""
import json
import logging
import os
import subprocess
import sys

## Great_Expectations_Prototype.ipynb

      
              1 file
            
          
              0 forks
            
          
                0 comments
              
            
              0 stars
            
          
                philerooski
                / Great_Expectations_Prototype.ipynb
            
            
              Last active
              September 21, 2023 22:21
            
          
      Loading

      Sorry, something went wrong. Reload?
      Sorry, we cannot display this file.
      Sorry, this file is invalid so it cannot be displayed.
      
          Viewer requires iframe.
	"""
	Verify snapshot data load integrity.

	This script checks:
	1. LOAD_LOG table for any errors or anomalies
	2. Compares tables between PROD_576 and PROD_568 schemas
	3. Validates record counts (PROD_576 should have more records)

	Usage:
	python verify_snapshot_load.py --database SYNAPSE_RDS_SNAPSHOT \\
	#!/usr/bin/env python3
	"""
	Download all form data from a specific form group to local directory.
	"""

	import argparse
	import sys
	import json
	import tempfile
	import shutil
	"""
	Load RDS snapshot data from S3 via Snowflake external stage into tables.

	This script supports two modes:
	1. Bootstrap mode (--bootstrap-stack): Creates a new schema, external stage,
	file format, and grants privileges before loading data.
	2. Manual mode: Loads data into an existing schema with pre-configured stage.

	The script dynamically discovers all data types from the S3 stage URL,
	creates tables using INFER_SCHEMA from Parquet files, loads the data,
	"""
	Analyze errors from the LOAD_LOG table.

	This script queries the LOAD_LOG table for failed operations,
	categorizes errors by type, and groups data types by error category.

	This is a complementary script to https://gist.github.com/philerooski/a740b25f066f1ad205344637160aa969
	"""

	import snowflake.connector
	"""
	Load snapshot data from Snowflake stage into tables.

	This script processes prefixes from PREFIX_LIST table, derives table names,
	creates tables using INFER_SCHEMA, and logs all operations to LOAD_LOG.

	See `--help` for optional parameter `--only-affected`
	"""

	import snowflake.connector
	import os
	import random
	import logging
	import argparse
	import toml
	import snowflake.connector
	from atlassian import Confluence

	# Configure logging
	logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
	### Does not work

	expectation_suite_name = "my_expectation_suite"
	checkpoint_name = "my_checkpoint"

	context = gx.get_context()

	# # Initialize expectation suite
	def init_expectation_suite():
	expectation_suite = context.add_expectation_suite(
	"""
	Run this script from within the unzipped directory `JMV_fitbit_dta`

	Download zipped data here: https://www.synapse.org/Synapse:syn62667431
	"""
	import pandas as pd
	import json
	import os
	"""
	A script which uploads validation results and a data validation
	report to S3 for the FitbitSleepLogs data type. This was run in
	Glue 4.0 while specifying --additional-python-modules great_expectations==0.18.11,boto3==1.24.70
	"""
	import json
	import logging
	import os
	import subprocess
	import sys