Created
January 10, 2026 11:10
-
-
Save mdrakiburrahman/2a547f44bade247a624b301f84774b48 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "eventTime":"2026-01-09T23:18:22.634Z", | |
| "producer":"https://github.com/OpenLineage/OpenLineage/tree/1.23.0/integration/spark", | |
| "schemaURL":"https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent", | |
| "eventType":"START", | |
| "run":{ | |
| "runId":"019ba50d-c8ab-798b-86ae-e437366a5a3f", | |
| "facets":{ | |
| "parent":{ | |
| "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.23.0/integration/spark", | |
| "_schemaURL":"https://openlineage.io/spec/facets/1-0-1/ParentRunFacet.json#/$defs/ParentRunFacet", | |
| "run":{ | |
| "runId":"019ba50d-6576-72ec-9c89-0fc6243db44e" | |
| }, | |
| "job":{ | |
| "namespace":"default", | |
| "name":"local_session" | |
| } | |
| }, | |
| "spark_properties":{ | |
| "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.23.0/integration/spark", | |
| "_schemaURL":"https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", | |
| "properties":{ | |
| "spark.master":"local[*]", | |
| "spark.app.name":"local-session" | |
| } | |
| }, | |
| "processing_engine":{ | |
| "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.23.0/integration/spark", | |
| "_schemaURL":"https://openlineage.io/spec/facets/1-1-1/ProcessingEngineRunFacet.json#/$defs/ProcessingEngineRunFacet", | |
| "version":"3.5.1", | |
| "name":"spark", | |
| "openlineageAdapterVersion":"1.23.0" | |
| }, | |
| "environment-properties":{ | |
| "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.23.0/integration/spark", | |
| "_schemaURL":"https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", | |
| "environment-properties":{ | |
| } | |
| } | |
| } | |
| }, | |
| "job":{ | |
| "namespace":"default", | |
| "name":"local_session.append_data_exec_v1.spark_catalog_product_sales_summary", | |
| "facets":{ | |
| "jobType":{ | |
| "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.23.0/integration/spark", | |
| "_schemaURL":"https://openlineage.io/spec/facets/2-0-3/JobTypeJobFacet.json#/$defs/JobTypeJobFacet", | |
| "processingType":"BATCH", | |
| "integration":"SPARK", | |
| "jobType":"SQL_JOB" | |
| }, | |
| "sql":{ | |
| "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.23.0/integration/spark", | |
| "_schemaURL":"https://openlineage.io/spec/facets/1-0-1/SQLJobFacet.json#/$defs/SQLJobFacet", | |
| "query":"\n SELECT\n p.productName,\n COUNT(s.OrderID) as total_sales,\n SUM(s.Quantity) as total_quantity,\n ROUND(AVG(s.Quantity), 2) as avg_quantity_per_sale\n FROM demo_etl.products p\n LEFT JOIN demo_etl.sales s ON p.productID = s.productID\n GROUP BY p.productName\n ORDER BY total_sales DESC\n " | |
| } | |
| } | |
| }, | |
| "inputs":[ | |
| { | |
| "namespace":"file", | |
| "name":"/workspaces/spark-sandbox/projects/spark-scala/warehouse/demo_etl.db/products", | |
| "facets":{ | |
| "dataSource":{ | |
| "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.23.0/integration/spark", | |
| "_schemaURL":"https://openlineage.io/spec/facets/1-0-1/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet", | |
| "name":"file", | |
| "uri":"file" | |
| }, | |
| "version":{ | |
| "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.23.0/integration/spark", | |
| "_schemaURL":"https://openlineage.io/spec/facets/1-0-1/DatasetVersionDatasetFacet.json#/$defs/DatasetVersionDatasetFacet", | |
| "datasetVersion":"1" | |
| }, | |
| "schema":{ | |
| "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.23.0/integration/spark", | |
| "_schemaURL":"https://openlineage.io/spec/facets/1-1-1/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet", | |
| "fields":[ | |
| { | |
| "name":"ProductID", | |
| "type":"string" | |
| }, | |
| { | |
| "name":"ProductName", | |
| "type":"string" | |
| }, | |
| { | |
| "name":"Price", | |
| "type":"integer" | |
| } | |
| ] | |
| }, | |
| "symlinks":{ | |
| "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.23.0/integration/spark", | |
| "_schemaURL":"https://openlineage.io/spec/facets/1-0-1/SymlinksDatasetFacet.json#/$defs/SymlinksDatasetFacet", | |
| "identifiers":[ | |
| { | |
| "namespace":"file:/workspaces/spark-sandbox/projects/spark-scala/warehouse", | |
| "name":"demo_etl.products", | |
| "type":"TABLE" | |
| } | |
| ] | |
| } | |
| }, | |
| "inputFacets":{ | |
| } | |
| }, | |
| { | |
| "namespace":"file", | |
| "name":"/workspaces/spark-sandbox/projects/spark-scala/warehouse/demo_etl.db/sales", | |
| "facets":{ | |
| "dataSource":{ | |
| "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.23.0/integration/spark", | |
| "_schemaURL":"https://openlineage.io/spec/facets/1-0-1/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet", | |
| "name":"file", | |
| "uri":"file" | |
| }, | |
| "version":{ | |
| "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.23.0/integration/spark", | |
| "_schemaURL":"https://openlineage.io/spec/facets/1-0-1/DatasetVersionDatasetFacet.json#/$defs/DatasetVersionDatasetFacet", | |
| "datasetVersion":"1" | |
| }, | |
| "schema":{ | |
| "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.23.0/integration/spark", | |
| "_schemaURL":"https://openlineage.io/spec/facets/1-1-1/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet", | |
| "fields":[ | |
| { | |
| "name":"OrderID", | |
| "type":"integer" | |
| }, | |
| { | |
| "name":"ProductID", | |
| "type":"string" | |
| }, | |
| { | |
| "name":"Quantity", | |
| "type":"integer" | |
| } | |
| ] | |
| }, | |
| "symlinks":{ | |
| "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.23.0/integration/spark", | |
| "_schemaURL":"https://openlineage.io/spec/facets/1-0-1/SymlinksDatasetFacet.json#/$defs/SymlinksDatasetFacet", | |
| "identifiers":[ | |
| { | |
| "namespace":"file:/workspaces/spark-sandbox/projects/spark-scala/warehouse", | |
| "name":"demo_etl.sales", | |
| "type":"TABLE" | |
| } | |
| ] | |
| } | |
| }, | |
| "inputFacets":{ | |
| } | |
| } | |
| ], | |
| "outputs":[ | |
| { | |
| "namespace":"file", | |
| "name":"/workspaces/spark-sandbox/projects/spark-scala/warehouse/demo_etl.db/product_sales_summary", | |
| "facets":{ | |
| "dataSource":{ | |
| "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.23.0/integration/spark", | |
| "_schemaURL":"https://openlineage.io/spec/facets/1-0-1/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet", | |
| "name":"file", | |
| "uri":"file" | |
| }, | |
| "schema":{ | |
| "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.23.0/integration/spark", | |
| "_schemaURL":"https://openlineage.io/spec/facets/1-1-1/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet", | |
| "fields":[ | |
| { | |
| "name":"productName", | |
| "type":"string" | |
| }, | |
| { | |
| "name":"total_sales", | |
| "type":"long" | |
| }, | |
| { | |
| "name":"total_quantity", | |
| "type":"long" | |
| }, | |
| { | |
| "name":"avg_quantity_per_sale", | |
| "type":"double" | |
| } | |
| ] | |
| }, | |
| "storage":{ | |
| "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.23.0/integration/spark", | |
| "_schemaURL":"https://openlineage.io/spec/facets/1-0-1/StorageDatasetFacet.json#/$defs/StorageDatasetFacet", | |
| "storageLayer":"delta", | |
| "fileFormat":"parquet" | |
| }, | |
| "columnLineage":{ | |
| "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.23.0/integration/spark", | |
| "_schemaURL":"https://openlineage.io/spec/facets/1-2-0/ColumnLineageDatasetFacet.json#/$defs/ColumnLineageDatasetFacet", | |
| "fields":{ | |
| "productName":{ | |
| "inputFields":[ | |
| { | |
| "namespace":"file", | |
| "name":"/workspaces/spark-sandbox/projects/spark-scala/warehouse/demo_etl.db/products", | |
| "field":"ProductName", | |
| "transformations":[ | |
| { | |
| "type":"DIRECT", | |
| "subtype":"IDENTITY", | |
| "description":"", | |
| "masking":false | |
| }, | |
| { | |
| "type":"INDIRECT", | |
| "subtype":"GROUP_BY", | |
| "description":"", | |
| "masking":false | |
| } | |
| ] | |
| }, | |
| { | |
| "namespace":"file", | |
| "name":"/workspaces/spark-sandbox/projects/spark-scala/warehouse/demo_etl.db/sales", | |
| "field":"ProductID", | |
| "transformations":[ | |
| { | |
| "type":"INDIRECT", | |
| "subtype":"JOIN", | |
| "description":"", | |
| "masking":false | |
| }, | |
| { | |
| "type":"INDIRECT", | |
| "subtype":"FILTER", | |
| "description":"", | |
| "masking":false | |
| } | |
| ] | |
| }, | |
| { | |
| "namespace":"file", | |
| "name":"/workspaces/spark-sandbox/projects/spark-scala/warehouse/demo_etl.db/sales", | |
| "field":"OrderID", | |
| "transformations":[ | |
| { | |
| "type":"INDIRECT", | |
| "subtype":"SORT", | |
| "description":"", | |
| "masking":true | |
| } | |
| ] | |
| }, | |
| { | |
| "namespace":"file", | |
| "name":"/workspaces/spark-sandbox/projects/spark-scala/warehouse/demo_etl.db/products", | |
| "field":"ProductID", | |
| "transformations":[ | |
| { | |
| "type":"INDIRECT", | |
| "subtype":"JOIN", | |
| "description":"", | |
| "masking":false | |
| } | |
| ] | |
| } | |
| ] | |
| }, | |
| "total_sales":{ | |
| "inputFields":[ | |
| { | |
| "namespace":"file", | |
| "name":"/workspaces/spark-sandbox/projects/spark-scala/warehouse/demo_etl.db/products", | |
| "field":"ProductName", | |
| "transformations":[ | |
| { | |
| "type":"INDIRECT", | |
| "subtype":"GROUP_BY", | |
| "description":"", | |
| "masking":false | |
| } | |
| ] | |
| }, | |
| { | |
| "namespace":"file", | |
| "name":"/workspaces/spark-sandbox/projects/spark-scala/warehouse/demo_etl.db/sales", | |
| "field":"ProductID", | |
| "transformations":[ | |
| { | |
| "type":"INDIRECT", | |
| "subtype":"JOIN", | |
| "description":"", | |
| "masking":false | |
| }, | |
| { | |
| "type":"INDIRECT", | |
| "subtype":"FILTER", | |
| "description":"", | |
| "masking":false | |
| } | |
| ] | |
| }, | |
| { | |
| "namespace":"file", | |
| "name":"/workspaces/spark-sandbox/projects/spark-scala/warehouse/demo_etl.db/sales", | |
| "field":"OrderID", | |
| "transformations":[ | |
| { | |
| "type":"DIRECT", | |
| "subtype":"AGGREGATION", | |
| "description":"", | |
| "masking":true | |
| }, | |
| { | |
| "type":"INDIRECT", | |
| "subtype":"SORT", | |
| "description":"", | |
| "masking":true | |
| } | |
| ] | |
| }, | |
| { | |
| "namespace":"file", | |
| "name":"/workspaces/spark-sandbox/projects/spark-scala/warehouse/demo_etl.db/products", | |
| "field":"ProductID", | |
| "transformations":[ | |
| { | |
| "type":"INDIRECT", | |
| "subtype":"JOIN", | |
| "description":"", | |
| "masking":false | |
| } | |
| ] | |
| } | |
| ] | |
| }, | |
| "total_quantity":{ | |
| "inputFields":[ | |
| { | |
| "namespace":"file", | |
| "name":"/workspaces/spark-sandbox/projects/spark-scala/warehouse/demo_etl.db/sales", | |
| "field":"Quantity", | |
| "transformations":[ | |
| { | |
| "type":"DIRECT", | |
| "subtype":"AGGREGATION", | |
| "description":"", | |
| "masking":false | |
| } | |
| ] | |
| }, | |
| { | |
| "namespace":"file", | |
| "name":"/workspaces/spark-sandbox/projects/spark-scala/warehouse/demo_etl.db/products", | |
| "field":"ProductName", | |
| "transformations":[ | |
| { | |
| "type":"INDIRECT", | |
| "subtype":"GROUP_BY", | |
| "description":"", | |
| "masking":false | |
| } | |
| ] | |
| }, | |
| { | |
| "namespace":"file", | |
| "name":"/workspaces/spark-sandbox/projects/spark-scala/warehouse/demo_etl.db/sales", | |
| "field":"ProductID", | |
| "transformations":[ | |
| { | |
| "type":"INDIRECT", | |
| "subtype":"JOIN", | |
| "description":"", | |
| "masking":false | |
| }, | |
| { | |
| "type":"INDIRECT", | |
| "subtype":"FILTER", | |
| "description":"", | |
| "masking":false | |
| } | |
| ] | |
| }, | |
| { | |
| "namespace":"file", | |
| "name":"/workspaces/spark-sandbox/projects/spark-scala/warehouse/demo_etl.db/sales", | |
| "field":"OrderID", | |
| "transformations":[ | |
| { | |
| "type":"INDIRECT", | |
| "subtype":"SORT", | |
| "description":"", | |
| "masking":true | |
| } | |
| ] | |
| }, | |
| { | |
| "namespace":"file", | |
| "name":"/workspaces/spark-sandbox/projects/spark-scala/warehouse/demo_etl.db/products", | |
| "field":"ProductID", | |
| "transformations":[ | |
| { | |
| "type":"INDIRECT", | |
| "subtype":"JOIN", | |
| "description":"", | |
| "masking":false | |
| } | |
| ] | |
| } | |
| ] | |
| }, | |
| "avg_quantity_per_sale":{ | |
| "inputFields":[ | |
| { | |
| "namespace":"file", | |
| "name":"/workspaces/spark-sandbox/projects/spark-scala/warehouse/demo_etl.db/sales", | |
| "field":"Quantity", | |
| "transformations":[ | |
| { | |
| "type":"DIRECT", | |
| "subtype":"AGGREGATION", | |
| "description":"", | |
| "masking":false | |
| } | |
| ] | |
| }, | |
| { | |
| "namespace":"file", | |
| "name":"/workspaces/spark-sandbox/projects/spark-scala/warehouse/demo_etl.db/products", | |
| "field":"ProductName", | |
| "transformations":[ | |
| { | |
| "type":"INDIRECT", | |
| "subtype":"GROUP_BY", | |
| "description":"", | |
| "masking":false | |
| } | |
| ] | |
| }, | |
| { | |
| "namespace":"file", | |
| "name":"/workspaces/spark-sandbox/projects/spark-scala/warehouse/demo_etl.db/sales", | |
| "field":"ProductID", | |
| "transformations":[ | |
| { | |
| "type":"INDIRECT", | |
| "subtype":"JOIN", | |
| "description":"", | |
| "masking":false | |
| }, | |
| { | |
| "type":"INDIRECT", | |
| "subtype":"FILTER", | |
| "description":"", | |
| "masking":false | |
| } | |
| ] | |
| }, | |
| { | |
| "namespace":"file", | |
| "name":"/workspaces/spark-sandbox/projects/spark-scala/warehouse/demo_etl.db/sales", | |
| "field":"OrderID", | |
| "transformations":[ | |
| { | |
| "type":"INDIRECT", | |
| "subtype":"SORT", | |
| "description":"", | |
| "masking":true | |
| } | |
| ] | |
| }, | |
| { | |
| "namespace":"file", | |
| "name":"/workspaces/spark-sandbox/projects/spark-scala/warehouse/demo_etl.db/products", | |
| "field":"ProductID", | |
| "transformations":[ | |
| { | |
| "type":"INDIRECT", | |
| "subtype":"JOIN", | |
| "description":"", | |
| "masking":false | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| } | |
| }, | |
| "symlinks":{ | |
| "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.23.0/integration/spark", | |
| "_schemaURL":"https://openlineage.io/spec/facets/1-0-1/SymlinksDatasetFacet.json#/$defs/SymlinksDatasetFacet", | |
| "identifiers":[ | |
| { | |
| "namespace":"file:/workspaces/spark-sandbox/projects/spark-scala/warehouse", | |
| "name":"demo_etl.product_sales_summary", | |
| "type":"TABLE" | |
| } | |
| ] | |
| } | |
| }, | |
| "outputFacets":{ | |
| } | |
| } | |
| ] | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment