Skip to content

Instantly share code, notes, and snippets.

@rsignell

rsignell/CORDEX_create_icechunk_s3.ipynb

Created November 5, 2025 13:35
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save rsignell/aa8cbd04ffaec1eb2a767f341b8d22f4 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save rsignell/aa8cbd04ffaec1eb2a767f341b8d22f4 to your computer and use it in GitHub Desktop.
Download ZIP
CORDEX_create_icechunk_s3.ipynb
Display the source blob
Display the rendered blob
Raw
CORDEX_create_icechunk_s3.ipynb
{
"cells": [
{
"cell_type": "markdown",
"id": "2bb23cda-d2dc-4ce0-a773-b0936451aa9d",
"metadata": {},
"source": [
"# Create Icechunk virtual dataset from CORDEX NetCDF files\n",
"* Originally on azure, now on s3\n",
"* The native NetCDF file chunks are tiny (**60kiB**): `Float:(1, 133, 116)`"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "8af6b6a8-728e-4f4c-adbe-4bc1045dc9ce",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<donfig.config_obj.ConfigSet at 0x7ad54ce68e10>"
]
},
"execution_count": 1,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import zarr\n",
"zarr.config.set({'async.concurrency': 128})"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "27632d73-826b-4e48-befb-1a5cfaa5c471",
"metadata": {},
"outputs": [],
"source": [
"import warnings\n",
"import os\n",
"import fsspec\n",
"import icechunk\n",
"import xarray as xr\n",
"from obstore.store import from_url\n",
"\n",
"from virtualizarr import open_virtual_dataset\n",
"from virtualizarr.parsers import HDFParser\n",
"from virtualizarr.registry import ObjectStoreRegistry\n",
"\n",
"warnings.filterwarnings(\"ignore\", category=UserWarning)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "bccbad9f-3380-4c57-a3b0-8b9a81173dbc",
"metadata": {},
"outputs": [],
"source": [
"# load AWS credentials for writing to OSN as environment vars\n",
"from dotenv import load_dotenv\n",
"_ = load_dotenv(f'{os.environ['HOME']}/dotenv/osn-esip.env')\n",
"\n",
"# Define icechunk storage\n",
"storage_endpoint = 'https://usgs.osn.mghpcc.org'\n",
"storage_bucket = 'esip'\n",
"storage_name = 'cordex-arctic'\n",
"\n",
"fs = fsspec.filesystem('s3', anon=False, endpoint_url=storage_endpoint)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "0121616e-38e8-4bd2-b581-de25c69797e1",
"metadata": {},
"outputs": [],
"source": [
"flist = fs.glob(f's3://{storage_bucket}/rsignell/cordex/arctic/*.nc')\n",
"flist = [f's3://{f}' for f in flist]"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "72ff27fa-5b44-41e3-b327-1132c9ce3cf6",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"19\n",
"s3://esip/rsignell/cordex/arctic/tasmax_ARC-44_ICHEC-EC-EARTH_rcp85_r12i1p1_SMHI-RCA4_v1_day_20960101-21001231.nc\n"
]
}
],
"source": [
"print(len(flist))\n",
"print(flist[-1])"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "8cd91ecf-fd2e-462c-aee5-7e8e8d5b0922",
"metadata": {},
"outputs": [],
"source": [
"bucket = \"s3://esip\"\n",
"store = from_url(bucket, region=\"not-used\", endpoint=storage_endpoint)\n",
"registry = ObjectStoreRegistry({bucket: store})\n",
"parser = HDFParser()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "77c8e6a3-4c7e-4ad5-81d8-53ebb2f5d2a0",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/srv/conda/envs/notebook/lib/python3.13/site-packages/zarr/core/dtype/npy/bytes.py:383: UnstableSpecificationWarning: The data type (NullTerminatedBytes(length=1)) does not have a Zarr V3 specification. That means that the representation of arrays saved with this data type may change without warning in a future version of Zarr Python. Arrays stored with this data type may be unreadable by other Zarr libraries. Use this data type at your own risk! Check https://github.com/zarr-developers/zarr-extensions/tree/main/data-types for the status of data type specifications for Zarr V3.\n",
" v3_unstable_dtype_warning(self)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 39 s, sys: 18.8 s, total: 57.9 s\n",
"Wall time: 9min 47s\n"
]
}
],
"source": [
"%%time\n",
"ds_list = [\n",
" open_virtual_dataset(\n",
" url=url,\n",
" parser=parser,\n",
" registry=registry,\n",
" loadable_variables=[\"time\"],\n",
" )\n",
" for url in flist\n",
"]"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "28b5788b-fcac-4164-8b3f-a21ec6808eab",
"metadata": {},
"outputs": [],
"source": [
"def fix_ds(ds):\n",
" return ds.drop_vars(['time_bnds', 'rotated_pole'])"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "30acc863-3ad7-4685-b91f-16bb508a2680",
"metadata": {},
"outputs": [],
"source": [
"ds_list_fixed = [fix_ds(ds) for ds in ds_list]"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "8ea502e7-ba50-429d-aae6-defa29491b71",
"metadata": {},
"outputs": [],
"source": [
"combined_ds = xr.concat(\n",
" ds_list_fixed,\n",
" dim=\"time\",\n",
" coords=\"minimal\",\n",
" compat=\"override\",\n",
" combine_attrs=\"override\",\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "4ee2d54e-113b-489d-8e31-8686ec0ccadc",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div><svg style=\"position: absolute; width: 0; height: 0; overflow: hidden\">\n",
"<defs>\n",
"<symbol id=\"icon-database\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M16 0c-8.837 0-16 2.239-16 5v4c0 2.761 7.163 5 16 5s16-2.239 16-5v-4c0-2.761-7.163-5-16-5z\"></path>\n",
"<path d=\"M16 17c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"<path d=\"M16 26c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"</symbol>\n",
"<symbol id=\"icon-file-text2\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M28.681 7.159c-0.694-0.947-1.662-2.053-2.724-3.116s-2.169-2.030-3.116-2.724c-1.612-1.182-2.393-1.319-2.841-1.319h-15.5c-1.378 0-2.5 1.121-2.5 2.5v27c0 1.378 1.122 2.5 2.5 2.5h23c1.378 0 2.5-1.122 2.5-2.5v-19.5c0-0.448-0.137-1.23-1.319-2.841zM24.543 5.457c0.959 0.959 1.712 1.825 2.268 2.543h-4.811v-4.811c0.718 0.556 1.584 1.309 2.543 2.268zM28 29.5c0 0.271-0.229 0.5-0.5 0.5h-23c-0.271 0-0.5-0.229-0.5-0.5v-27c0-0.271 0.229-0.5 0.5-0.5 0 0 15.499-0 15.5 0v7c0 0.552 0.448 1 1 1h7v19.5z\"></path>\n",
"<path d=\"M23 26h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 22h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 18h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"</symbol>\n",
"</defs>\n",
"</svg>\n",
"<style>/* CSS stylesheet for displaying xarray objects in notebooks */\n",
"\n",
":root {\n",
" --xr-font-color0: var(\n",
" --jp-content-font-color0,\n",
" var(--pst-color-text-base rgba(0, 0, 0, 1))\n",
" );\n",
" --xr-font-color2: var(\n",
" --jp-content-font-color2,\n",
" var(--pst-color-text-base, rgba(0, 0, 0, 0.54))\n",
" );\n",
" --xr-font-color3: var(\n",
" --jp-content-font-color3,\n",
" var(--pst-color-text-base, rgba(0, 0, 0, 0.38))\n",
" );\n",
" --xr-border-color: var(\n",
" --jp-border-color2,\n",
" hsl(from var(--pst-color-on-background, white) h s calc(l - 10))\n",
" );\n",
" --xr-disabled-color: var(\n",
" --jp-layout-color3,\n",
" hsl(from var(--pst-color-on-background, white) h s calc(l - 40))\n",
" );\n",
" --xr-background-color: var(\n",
" --jp-layout-color0,\n",
" var(--pst-color-on-background, white)\n",
" );\n",
" --xr-background-color-row-even: var(\n",
" --jp-layout-color1,\n",
" hsl(from var(--pst-color-on-background, white) h s calc(l - 5))\n",
" );\n",
" --xr-background-color-row-odd: var(\n",
" --jp-layout-color2,\n",
" hsl(from var(--pst-color-on-background, white) h s calc(l - 15))\n",
" );\n",
"}\n",
"\n",
"html[theme=\"dark\"],\n",
"html[data-theme=\"dark\"],\n",
"body[data-theme=\"dark\"],\n",
"body.vscode-dark {\n",
" --xr-font-color0: var(\n",
" --jp-content-font-color0,\n",
" var(--pst-color-text-base, rgba(255, 255, 255, 1))\n",
" );\n",
" --xr-font-color2: var(\n",
" --jp-content-font-color2,\n",
" var(--pst-color-text-base, rgba(255, 255, 255, 0.54))\n",
" );\n",
" --xr-font-color3: var(\n",
" --jp-content-font-color3,\n",
" var(--pst-color-text-base, rgba(255, 255, 255, 0.38))\n",
" );\n",
" --xr-border-color: var(\n",
" --jp-border-color2,\n",
" hsl(from var(--pst-color-on-background, #111111) h s calc(l + 10))\n",
" );\n",
" --xr-disabled-color: var(\n",
" --jp-layout-color3,\n",
" hsl(from var(--pst-color-on-background, #111111) h s calc(l + 40))\n",
" );\n",
" --xr-background-color: var(\n",
" --jp-layout-color0,\n",
" var(--pst-color-on-background, #111111)\n",
" );\n",
" --xr-background-color-row-even: var(\n",
" --jp-layout-color1,\n",
" hsl(from var(--pst-color-on-background, #111111) h s calc(l + 5))\n",
" );\n",
" --xr-background-color-row-odd: var(\n",
" --jp-layout-color2,\n",
" hsl(from var(--pst-color-on-background, #111111) h s calc(l + 15))\n",
" );\n",
"}\n",
"\n",
".xr-wrap {\n",
" display: block !important;\n",
" min-width: 300px;\n",
" max-width: 700px;\n",
" line-height: 1.6;\n",
"}\n",
"\n",
".xr-text-repr-fallback {\n",
" /* fallback to plain text repr when CSS is not injected (untrusted notebook) */\n",
" display: none;\n",
"}\n",
"\n",
".xr-header {\n",
" padding-top: 6px;\n",
" padding-bottom: 6px;\n",
" margin-bottom: 4px;\n",
" border-bottom: solid 1px var(--xr-border-color);\n",
"}\n",
"\n",
".xr-header > div,\n",
".xr-header > ul {\n",
" display: inline;\n",
" margin-top: 0;\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-obj-type,\n",
".xr-obj-name,\n",
".xr-group-name {\n",
" margin-left: 2px;\n",
" margin-right: 10px;\n",
"}\n",
"\n",
".xr-group-name::before {\n",
" content: \"📁\";\n",
" padding-right: 0.3em;\n",
"}\n",
"\n",
".xr-group-name,\n",
".xr-obj-type {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-sections {\n",
" padding-left: 0 !important;\n",
" display: grid;\n",
" grid-template-columns: 150px auto auto 1fr 0 20px 0 20px;\n",
" margin-block-start: 0;\n",
" margin-block-end: 0;\n",
"}\n",
"\n",
".xr-section-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-section-item input {\n",
" display: inline-block;\n",
" opacity: 0;\n",
" height: 0;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-section-item input + label {\n",
" color: var(--xr-disabled-color);\n",
" border: 2px solid transparent !important;\n",
"}\n",
"\n",
".xr-section-item input:enabled + label {\n",
" cursor: pointer;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-item input:focus + label {\n",
" border: 2px solid var(--xr-font-color0) !important;\n",
"}\n",
"\n",
".xr-section-item input:enabled + label:hover {\n",
" color: var(--xr-font-color0);\n",
"}\n",
"\n",
".xr-section-summary {\n",
" grid-column: 1;\n",
" color: var(--xr-font-color2);\n",
" font-weight: 500;\n",
"}\n",
"\n",
".xr-section-summary > span {\n",
" display: inline-block;\n",
" padding-left: 0.5em;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-summary-in + label:before {\n",
" display: inline-block;\n",
" content: \"►\";\n",
" font-size: 11px;\n",
" width: 15px;\n",
" text-align: center;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label:before {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label:before {\n",
" content: \"▼\";\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label > span {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-summary,\n",
".xr-section-inline-details {\n",
" padding-top: 4px;\n",
"}\n",
"\n",
".xr-section-inline-details {\n",
" grid-column: 2 / -1;\n",
"}\n",
"\n",
".xr-section-details {\n",
" display: none;\n",
" grid-column: 1 / -1;\n",
" margin-top: 4px;\n",
" margin-bottom: 5px;\n",
"}\n",
"\n",
".xr-section-summary-in:checked ~ .xr-section-details {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-group-box {\n",
" display: inline-grid;\n",
" grid-template-columns: 0px 20px auto;\n",
" width: 100%;\n",
"}\n",
"\n",
".xr-group-box-vline {\n",
" grid-column-start: 1;\n",
" border-right: 0.2em solid;\n",
" border-color: var(--xr-border-color);\n",
" width: 0px;\n",
"}\n",
"\n",
".xr-group-box-hline {\n",
" grid-column-start: 2;\n",
" grid-row-start: 1;\n",
" height: 1em;\n",
" width: 20px;\n",
" border-bottom: 0.2em solid;\n",
" border-color: var(--xr-border-color);\n",
"}\n",
"\n",
".xr-group-box-contents {\n",
" grid-column-start: 3;\n",
"}\n",
"\n",
".xr-array-wrap {\n",
" grid-column: 1 / -1;\n",
" display: grid;\n",
" grid-template-columns: 20px auto;\n",
"}\n",
"\n",
".xr-array-wrap > label {\n",
" grid-column: 1;\n",
" vertical-align: top;\n",
"}\n",
"\n",
".xr-preview {\n",
" color: var(--xr-font-color3);\n",
"}\n",
"\n",
".xr-array-preview,\n",
".xr-array-data {\n",
" padding: 0 5px !important;\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-array-data,\n",
".xr-array-in:checked ~ .xr-array-preview {\n",
" display: none;\n",
"}\n",
"\n",
".xr-array-in:checked ~ .xr-array-data,\n",
".xr-array-preview {\n",
" display: inline-block;\n",
"}\n",
"\n",
".xr-dim-list {\n",
" display: inline-block !important;\n",
" list-style: none;\n",
" padding: 0 !important;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list li {\n",
" display: inline-block;\n",
" padding: 0;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list:before {\n",
" content: \"(\";\n",
"}\n",
"\n",
".xr-dim-list:after {\n",
" content: \")\";\n",
"}\n",
"\n",
".xr-dim-list li:not(:last-child):after {\n",
" content: \",\";\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-has-index {\n",
" font-weight: bold;\n",
"}\n",
"\n",
".xr-var-list,\n",
".xr-var-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-var-item > div,\n",
".xr-var-item label,\n",
".xr-var-item > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-even);\n",
" border-color: var(--xr-background-color-row-odd);\n",
" margin-bottom: 0;\n",
" padding-top: 2px;\n",
"}\n",
"\n",
".xr-var-item > .xr-var-name:hover span {\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-var-list > li:nth-child(odd) > div,\n",
".xr-var-list > li:nth-child(odd) > label,\n",
".xr-var-list > li:nth-child(odd) > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-odd);\n",
" border-color: var(--xr-background-color-row-even);\n",
"}\n",
"\n",
".xr-var-name {\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-var-dims {\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-var-dtype {\n",
" grid-column: 3;\n",
" text-align: right;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-preview {\n",
" grid-column: 4;\n",
"}\n",
"\n",
".xr-index-preview {\n",
" grid-column: 2 / 5;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-name,\n",
".xr-var-dims,\n",
".xr-var-dtype,\n",
".xr-preview,\n",
".xr-attrs dt {\n",
" white-space: nowrap;\n",
" overflow: hidden;\n",
" text-overflow: ellipsis;\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-var-name:hover,\n",
".xr-var-dims:hover,\n",
".xr-var-dtype:hover,\n",
".xr-attrs dt:hover {\n",
" overflow: visible;\n",
" width: auto;\n",
" z-index: 1;\n",
"}\n",
"\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" display: none;\n",
" border-top: 2px dotted var(--xr-background-color);\n",
" padding-bottom: 20px !important;\n",
" padding-top: 10px !important;\n",
"}\n",
"\n",
".xr-var-attrs-in + label,\n",
".xr-var-data-in + label,\n",
".xr-index-data-in + label {\n",
" padding: 0 1px;\n",
"}\n",
"\n",
".xr-var-attrs-in:checked ~ .xr-var-attrs,\n",
".xr-var-data-in:checked ~ .xr-var-data,\n",
".xr-index-data-in:checked ~ .xr-index-data {\n",
" display: block;\n",
"}\n",
"\n",
".xr-var-data > table {\n",
" float: right;\n",
"}\n",
"\n",
".xr-var-data > pre,\n",
".xr-index-data > pre,\n",
".xr-var-data > table > tbody > tr {\n",
" background-color: transparent !important;\n",
"}\n",
"\n",
".xr-var-name span,\n",
".xr-var-data,\n",
".xr-index-name div,\n",
".xr-index-data,\n",
".xr-attrs {\n",
" padding-left: 25px !important;\n",
"}\n",
"\n",
".xr-attrs,\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" grid-column: 1 / -1;\n",
"}\n",
"\n",
"dl.xr-attrs {\n",
" padding: 0;\n",
" margin: 0;\n",
" display: grid;\n",
" grid-template-columns: 125px auto;\n",
"}\n",
"\n",
".xr-attrs dt,\n",
".xr-attrs dd {\n",
" padding: 0;\n",
" margin: 0;\n",
" float: left;\n",
" padding-right: 10px;\n",
" width: auto;\n",
"}\n",
"\n",
".xr-attrs dt {\n",
" font-weight: normal;\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-attrs dt:hover span {\n",
" display: inline-block;\n",
" background: var(--xr-background-color);\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-attrs dd {\n",
" grid-column: 2;\n",
" white-space: pre-wrap;\n",
" word-break: break-all;\n",
"}\n",
"\n",
".xr-icon-database,\n",
".xr-icon-file-text2,\n",
".xr-no-icon {\n",
" display: inline-block;\n",
" vertical-align: middle;\n",
" width: 1em;\n",
" height: 1.5em !important;\n",
" stroke-width: 0;\n",
" stroke: currentColor;\n",
" fill: currentColor;\n",
"}\n",
"\n",
".xr-var-attrs-in:checked + label > .xr-icon-file-text2,\n",
".xr-var-data-in:checked + label > .xr-icon-database,\n",
".xr-index-data-in:checked + label > .xr-icon-database {\n",
" color: var(--xr-font-color0);\n",
" filter: drop-shadow(1px 1px 5px var(--xr-font-color2));\n",
" stroke-width: 0.8px;\n",
"}\n",
"</style><pre class='xr-text-repr-fallback'>&lt;xarray.Dataset&gt; Size: 2GB\n",
"Dimensions: (time: 34698, rlat: 133, rlon: 116)\n",
"Coordinates:\n",
" * time (time) datetime64[ns] 278kB 2006-01-01T12:00:00 ... 2100-12-31T1...\n",
" rlat (rlat) float64 1kB ManifestArray&lt;shape=(133,), dtype=float64, ch...\n",
" rlon (rlon) float64 928B ManifestArray&lt;shape=(116,), dtype=float64, c...\n",
" height float64 8B ManifestArray&lt;shape=(), dtype=float64, chunks=()&gt;\n",
" lon (rlat, rlon) float64 123kB ManifestArray&lt;shape=(133, 116), dtype...\n",
" lat (rlat, rlon) float64 123kB ManifestArray&lt;shape=(133, 116), dtype...\n",
"Data variables:\n",
" tasmax (time, rlat, rlon) float32 2GB ManifestArray&lt;shape=(34698, 133, ...\n",
"Attributes: (12/22)\n",
" Conventions: CF-1.4\n",
" contact: rossby.cordex@smhi.se\n",
" creation_date: 2013-06-20-T22:16:58Z\n",
" experiment: RCP8.5\n",
" experiment_id: rcp85\n",
" driving_experiment: ICHEC-EC-EARTH, rcp85, r12i1p1\n",
" ... ...\n",
" product: output\n",
" references: http://www.smhi.se/en/Research/Research-d...\n",
" tracking_id: 3f965fbb-e4b1-4835-a0e3-c0150e715af9\n",
" rossby_comment: 201307: CORDEX Arctic 0.44 deg | RCA4 v1 ...\n",
" rossby_run_id: 201307\n",
" rossby_grib_path: /nobackup/rossby16/rossby/joint_exp/corde...</pre><div class='xr-wrap' style='display:none'><div class='xr-header'><div class='xr-obj-type'>xarray.Dataset</div></div><ul class='xr-sections'><li class='xr-section-item'><input id='section-d14b7447-d3df-412e-9f1c-e84ff389ab04' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-d14b7447-d3df-412e-9f1c-e84ff389ab04' class='xr-section-summary' title='Expand/collapse section'>Dimensions:</label><div class='xr-section-inline-details'><ul class='xr-dim-list'><li><span class='xr-has-index'>time</span>: 34698</li><li><span>rlat</span>: 133</li><li><span>rlon</span>: 116</li></ul></div><div class='xr-section-details'></div></li><li class='xr-section-item'><input id='section-03a1b9be-6e58-43d3-ace5-53690c45afc5' class='xr-section-summary-in' type='checkbox' checked><label for='section-03a1b9be-6e58-43d3-ace5-53690c45afc5' class='xr-section-summary' >Coordinates: <span>(6)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>time</span></div><div class='xr-var-dims'>(time)</div><div class='xr-var-dtype'>datetime64[ns]</div><div class='xr-var-preview xr-preview'>2006-01-01T12:00:00 ... 2100-12-...</div><input id='attrs-7cbbaf70-bbcf-4a9b-b182-40a13f92c5da' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-7cbbaf70-bbcf-4a9b-b182-40a13f92c5da' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-cc6e5b1a-f0cc-4736-8e71-2b1a1aa6dd12' class='xr-var-data-in' type='checkbox'><label for='data-cc6e5b1a-f0cc-4736-8e71-2b1a1aa6dd12' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>standard_name :</span></dt><dd>time</dd><dt><span>long_name :</span></dt><dd>time</dd><dt><span>bounds :</span></dt><dd>time_bnds</dd><dt><span>axis :</span></dt><dd>T</dd></dl></div><div class='xr-var-data'><pre>array([&#x27;2006-01-01T12:00:00.000000000&#x27;, &#x27;2006-01-02T12:00:00.000000000&#x27;,\n",
" &#x27;2006-01-03T12:00:00.000000000&#x27;, ..., &#x27;2100-12-29T12:00:00.000000000&#x27;,\n",
" &#x27;2100-12-30T12:00:00.000000000&#x27;, &#x27;2100-12-31T12:00:00.000000000&#x27;],\n",
" shape=(34698,), dtype=&#x27;datetime64[ns]&#x27;)</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span>rlat</span></div><div class='xr-var-dims'>(rlat)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>ManifestArray&lt;shape=(133,), dtyp...</div><input id='attrs-ed7ab6fe-bbf4-4860-a7d7-0347e3bfa623' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-ed7ab6fe-bbf4-4860-a7d7-0347e3bfa623' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-4c77b6b3-3f0a-4fd3-be91-6d82bd96754a' class='xr-var-data-in' type='checkbox'><label for='data-4c77b6b3-3f0a-4fd3-be91-6d82bd96754a' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>standard_name :</span></dt><dd>grid_latitude</dd><dt><span>long_name :</span></dt><dd>latitude in rotated pole grid</dd><dt><span>units :</span></dt><dd>degrees</dd><dt><span>axis :</span></dt><dd>Y</dd></dl></div><div class='xr-var-data'><pre>ManifestArray&lt;shape=(133,), dtype=float64, chunks=(133,)&gt;</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span>rlon</span></div><div class='xr-var-dims'>(rlon)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>ManifestArray&lt;shape=(116,), dtyp...</div><input id='attrs-31b459d1-d4c8-4a4e-b39c-401782735d58' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-31b459d1-d4c8-4a4e-b39c-401782735d58' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-127c44f6-20d1-4563-9578-e5697f294dde' class='xr-var-data-in' type='checkbox'><label for='data-127c44f6-20d1-4563-9578-e5697f294dde' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>standard_name :</span></dt><dd>grid_longitude</dd><dt><span>long_name :</span></dt><dd>longitude in rotated pole grid</dd><dt><span>units :</span></dt><dd>degrees</dd><dt><span>axis :</span></dt><dd>X</dd></dl></div><div class='xr-var-data'><pre>ManifestArray&lt;shape=(116,), dtype=float64, chunks=(116,)&gt;</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span>height</span></div><div class='xr-var-dims'>()</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>ManifestArray&lt;shape=(), dtype=fl...</div><input id='attrs-92424bde-7314-49bf-b1a0-e8143942c456' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-92424bde-7314-49bf-b1a0-e8143942c456' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-bfd96f49-8428-464a-920b-31cfc867cd7f' class='xr-var-data-in' type='checkbox'><label for='data-bfd96f49-8428-464a-920b-31cfc867cd7f' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>axis :</span></dt><dd>Z</dd><dt><span>long_name :</span></dt><dd>height</dd><dt><span>positive :</span></dt><dd>up</dd><dt><span>standard_name :</span></dt><dd>height</dd><dt><span>units :</span></dt><dd>m</dd></dl></div><div class='xr-var-data'><pre>ManifestArray&lt;shape=(), dtype=float64, chunks=()&gt;</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span>lon</span></div><div class='xr-var-dims'>(rlat, rlon)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>ManifestArray&lt;shape=(133, 116), ...</div><input id='attrs-893e0b4b-6887-467c-a6dc-ca47be03ddd5' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-893e0b4b-6887-467c-a6dc-ca47be03ddd5' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-d77d5d7d-c7c5-478a-bab5-6b34c50ac7b9' class='xr-var-data-in' type='checkbox'><label for='data-d77d5d7d-c7c5-478a-bab5-6b34c50ac7b9' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>standard_name :</span></dt><dd>longitude</dd><dt><span>long_name :</span></dt><dd>longitude</dd><dt><span>units :</span></dt><dd>degrees_east</dd></dl></div><div class='xr-var-data'><pre>ManifestArray&lt;shape=(133, 116), dtype=float64, chunks=(133, 116)&gt;</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span>lat</span></div><div class='xr-var-dims'>(rlat, rlon)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>ManifestArray&lt;shape=(133, 116), ...</div><input id='attrs-9d67363a-2d1c-4358-9922-932ae5437ba8' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-9d67363a-2d1c-4358-9922-932ae5437ba8' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-f4c3d470-35e9-4ab9-a71c-2c2f985bb664' class='xr-var-data-in' type='checkbox'><label for='data-f4c3d470-35e9-4ab9-a71c-2c2f985bb664' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>standard_name :</span></dt><dd>latitude</dd><dt><span>long_name :</span></dt><dd>latitude</dd><dt><span>units :</span></dt><dd>degrees_north</dd></dl></div><div class='xr-var-data'><pre>ManifestArray&lt;shape=(133, 116), dtype=float64, chunks=(133, 116)&gt;</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-77db2e2c-7775-47d7-910b-caafe3ca70d6' class='xr-section-summary-in' type='checkbox' checked><label for='section-77db2e2c-7775-47d7-910b-caafe3ca70d6' class='xr-section-summary' >Data variables: <span>(1)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span>tasmax</span></div><div class='xr-var-dims'>(time, rlat, rlon)</div><div class='xr-var-dtype'>float32</div><div class='xr-var-preview xr-preview'>ManifestArray&lt;shape=(34698, 133,...</div><input id='attrs-a9480191-b1cf-4812-afde-e36668f14844' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-a9480191-b1cf-4812-afde-e36668f14844' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-c5d44c7e-f79f-4b44-88c4-9a9ba1f66e30' class='xr-var-data-in' type='checkbox'><label for='data-c5d44c7e-f79f-4b44-88c4-9a9ba1f66e30' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>grid_mapping :</span></dt><dd>rotated_pole</dd><dt><span>_FillValue :</span></dt><dd>AAAAgB2vFUQ=</dd><dt><span>missing_value :</span></dt><dd>1.0000000200408773e+20</dd><dt><span>standard_name :</span></dt><dd>air_temperature</dd><dt><span>long_name :</span></dt><dd>Daily Maximum Near-Surface Air Temperature</dd><dt><span>units :</span></dt><dd>K</dd><dt><span>coordinates :</span></dt><dd>lon lat height</dd><dt><span>cell_methods :</span></dt><dd>time: maximum</dd></dl></div><div class='xr-var-data'><pre>ManifestArray&lt;shape=(34698, 133, 116), dtype=float32, chunks=(1, 133, 116)&gt;</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-8fd9ebdb-27b7-4849-af1a-39eedf322d9f' class='xr-section-summary-in' type='checkbox' ><label for='section-8fd9ebdb-27b7-4849-af1a-39eedf322d9f' class='xr-section-summary' >Attributes: <span>(22)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><dl class='xr-attrs'><dt><span>Conventions :</span></dt><dd>CF-1.4</dd><dt><span>contact :</span></dt><dd>rossby.cordex@smhi.se</dd><dt><span>creation_date :</span></dt><dd>2013-06-20-T22:16:58Z</dd><dt><span>experiment :</span></dt><dd>RCP8.5</dd><dt><span>experiment_id :</span></dt><dd>rcp85</dd><dt><span>driving_experiment :</span></dt><dd>ICHEC-EC-EARTH, rcp85, r12i1p1</dd><dt><span>driving_model_id :</span></dt><dd>ICHEC-EC-EARTH</dd><dt><span>driving_model_ensemble_member :</span></dt><dd>r12i1p1</dd><dt><span>driving_experiment_name :</span></dt><dd>rcp85</dd><dt><span>frequency :</span></dt><dd>day</dd><dt><span>institution :</span></dt><dd>Swedish Meteorological and Hydrological Institute, Rossby Centre</dd><dt><span>institute_id :</span></dt><dd>SMHI</dd><dt><span>model_id :</span></dt><dd>SMHI-RCA4</dd><dt><span>rcm_version_id :</span></dt><dd>v1</dd><dt><span>project_id :</span></dt><dd>CORDEX</dd><dt><span>CORDEX_domain :</span></dt><dd>ARC-44</dd><dt><span>product :</span></dt><dd>output</dd><dt><span>references :</span></dt><dd>http://www.smhi.se/en/Research/Research-departments/climate-research-rossby-centre</dd><dt><span>tracking_id :</span></dt><dd>3f965fbb-e4b1-4835-a0e3-c0150e715af9</dd><dt><span>rossby_comment :</span></dt><dd>201307: CORDEX Arctic 0.44 deg | RCA4 v1 | ICHEC-EC-EARTH | r12i1p1 | rcp85 | L40</dd><dt><span>rossby_run_id :</span></dt><dd>201307</dd><dt><span>rossby_grib_path :</span></dt><dd>/nobackup/rossby16/rossby/joint_exp/cordex/201307/raw/</dd></dl></div></li></ul></div></div>"
],
"text/plain": [
"<xarray.Dataset> Size: 2GB\n",
"Dimensions: (time: 34698, rlat: 133, rlon: 116)\n",
"Coordinates:\n",
" * time (time) datetime64[ns] 278kB 2006-01-01T12:00:00 ... 2100-12-31T1...\n",
" rlat (rlat) float64 1kB ManifestArray<shape=(133,), dtype=float64, ch...\n",
" rlon (rlon) float64 928B ManifestArray<shape=(116,), dtype=float64, c...\n",
" height float64 8B ManifestArray<shape=(), dtype=float64, chunks=()>\n",
" lon (rlat, rlon) float64 123kB ManifestArray<shape=(133, 116), dtype...\n",
" lat (rlat, rlon) float64 123kB ManifestArray<shape=(133, 116), dtype...\n",
"Data variables:\n",
" tasmax (time, rlat, rlon) float32 2GB ManifestArray<shape=(34698, 133, ...\n",
"Attributes: (12/22)\n",
" Conventions: CF-1.4\n",
" contact: rossby.cordex@smhi.se\n",
" creation_date: 2013-06-20-T22:16:58Z\n",
" experiment: RCP8.5\n",
" experiment_id: rcp85\n",
" driving_experiment: ICHEC-EC-EARTH, rcp85, r12i1p1\n",
" ... ...\n",
" product: output\n",
" references: http://www.smhi.se/en/Research/Research-d...\n",
" tracking_id: 3f965fbb-e4b1-4835-a0e3-c0150e715af9\n",
" rossby_comment: 201307: CORDEX Arctic 0.44 deg | RCA4 v1 ...\n",
" rossby_run_id: 201307\n",
" rossby_grib_path: /nobackup/rossby16/rossby/joint_exp/corde..."
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"combined_ds"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "8d354f6d-27dc-4142-aa71-3b25d6945eb1",
"metadata": {},
"outputs": [],
"source": [
"# remove old existing icechunk storage with this name\n",
"try:\n",
" # Use the same prefix as the storage\n",
" fs.rm(f's3://{storage_bucket}/icechunk/{storage_name}', recursive=True)\n",
"except:\n",
" pass"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "b188d383-0bdd-49b3-8dea-7b22415e38bc",
"metadata": {},
"outputs": [],
"source": [
"storage = icechunk.s3_storage(\n",
" bucket=storage_bucket,\n",
" prefix=f\"icechunk/{storage_name}\",\n",
" from_env=True,\n",
" endpoint_url=storage_endpoint,\n",
" region='not-used', # N/A for Pangeo-EOSC bucket, but required param\n",
" force_path_style=True)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "b55dd170-f73c-4228-8275-98cb48ad08b7",
"metadata": {},
"outputs": [],
"source": [
"config = icechunk.RepositoryConfig.default()\n",
"\n",
"config.set_virtual_chunk_container(\n",
" icechunk.VirtualChunkContainer(\n",
" url_prefix=f\"s3://{storage_bucket}/\",\n",
" store=icechunk.s3_store(region=\"not-used\", anonymous=False, s3_compatible=True, \n",
" force_path_style=True, endpoint_url=storage_endpoint),\n",
" ),\n",
")\n",
"\n",
"repo = icechunk.Repository.create(storage, config)\n",
"session = repo.writable_session(\"main\")"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "539e70f4-581d-4271-aca1-d428047e3a23",
"metadata": {},
"outputs": [],
"source": [
"combined_ds.virtualize.to_icechunk(session.store)"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "ca845726-0b39-41ba-a465-f5311a521b67",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'Q6JB8A1Y9YF40J01NCXG'"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"session.commit(\"initial commit of CORDEX data\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.13.9"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment