taranto-icechunk-FMRC.ipynb

taranto-icechunk-FMRC.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3f7d557d-79c9-4f8c-8875-4ef924e1cc44",
   "metadata": {},
   "source": [
    "# Access the virtual Icechunk Repo FMRC using Rolodex \n",
    "This SYFEM coastal model is pushing a netCDF file to object storage each day, and also running the virtualizarr2 append_to_icechunk to update the repo. This notebook opens the up-to-date forecast model run collection and uses Rolodex to extract BestEstimate time series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef8a999d-8fb7-4e1a-b06a-ec763f0e2f68",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=UserWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28adc8ce-720b-46dc-b2d7-d6dd6219a161",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load AWS credentials for Pangeo-EOSC storage as environment vars\n",
    "\n",
    "import icechunk\n",
    "import xarray as xr\n",
    "from obstore.store import from_url\n",
    "from dotenv import load_dotenv\n",
    "_ = load_dotenv('/home/jovyan/dotenv/rsignell4.env')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ca88cd4-b90b-4435-bd22-ea36f0dd6500",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define storage\n",
    "storage_endpoint = 'https://pangeo-eosc-minioapi.vm.fedcloud.eu'\n",
    "storage_bucket = 'rsignell4-protocoast'\n",
    "storage_name = 'taranto-icechunk'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b7a0d5a-c206-45eb-b319-32ad53fc9a56",
   "metadata": {},
   "outputs": [],
   "source": [
    "bucket = \"s3://rsignell4-protocoast\"\n",
    "store = from_url(bucket, region=\"not-used\", endpoint=storage_endpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c08e03a8-eed2-41ce-bab2-59b45f09e834",
   "metadata": {},
   "outputs": [],
   "source": [
    "storage = icechunk.s3_storage(\n",
    "    bucket=storage_bucket,\n",
    "    prefix=f\"icechunk/{storage_name}\",\n",
    "    from_env=True,\n",
    "    endpoint_url=storage_endpoint,\n",
    "    region='not-used',   # N/A for Pangeo-EOSC bucket, but required param\n",
    "    force_path_style=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2bb552df-8b3a-4355-a7b0-017c981cb90f",
   "metadata": {},
   "outputs": [],
   "source": [
    "config = icechunk.RepositoryConfig.default()\n",
    "\n",
    "config.set_virtual_chunk_container(\n",
    "    icechunk.VirtualChunkContainer(\n",
    "        url_prefix=f\"s3://{storage_bucket}/\",\n",
    "        store=icechunk.s3_store(region=\"not-used\", anonymous=False, s3_compatible=True, \n",
    "                                force_path_style=True, endpoint_url=storage_endpoint),\n",
    "    ),\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a871f9f1-0407-456b-8268-e2a6d29be7a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "credentials = icechunk.containers_credentials({f\"s3://{storage_bucket}/\": icechunk.s3_credentials(anonymous=False)})\n",
    "\n",
    "read_repo = icechunk.Repository.open(storage, config, authorize_virtual_chunk_access=credentials)\n",
    "\n",
    "read_session = read_repo.readonly_session(\"main\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50ef0510-c660-4d09-9540-50f053fabb7d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "ds = xr.open_zarr(read_session.store, consolidated=False, zarr_format=3, chunks={})\n",
    "ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "256e5cfa-fe0f-4fa0-8282-6b8d42569be5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import rolodex.forecast\n",
    "from rolodex.forecast import (\n",
    "    BestEstimate,\n",
    "    ConstantForecast,\n",
    "    ConstantOffset,\n",
    "    ForecastIndex,\n",
    "    Model,\n",
    "    ModelRun,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a8126b6-af3b-4a4f-8d34-b3395fccf46d",
   "metadata": {},
   "outputs": [],
   "source": [
    "ds.coords[\"valid_time\"] = rolodex.forecast.create_lazy_valid_time_variable(\n",
    "    reference_time=ds.time, period=ds.step\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc0e0825-7cb2-48dc-95f3-bbe821fcc06d",
   "metadata": {},
   "outputs": [],
   "source": [
    "newds = ds.drop_indexes([\"time\", \"step\"]).set_xindex(\n",
    "    [\"time\", \"step\", \"valid_time\"], ForecastIndex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54e061a7-d3e7-4a46-a7a0-a2bbb380cbad",
   "metadata": {},
   "outputs": [],
   "source": [
    "ds_best = newds.sel(time=slice('2026-01-01','2026-02-03')).sel(valid_time=BestEstimate(offset=7))  # start at forecast hour 2 instead of 0 (analysis time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b20a2851-914f-45e8-bbf8-963d720b5377",
   "metadata": {},
   "outputs": [],
   "source": [
    "ds_best = newds.sel(time=slice('2026-02-05','2026-02-11')).sel(valid_time=BestEstimate(offset=7))  # start at forecast hour 2 instead of 0 (analysis time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c734f433-e593-4cf9-a897-8cbadcddb4c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "ds_best = newds.sel(valid_time=BestEstimate(offset=7))  # start at forecast hour 2 instead of 0 (analysis time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2db14a51-7e19-444f-8cc4-ab8704a57a80",
   "metadata": {},
   "outputs": [],
   "source": [
    "import hvplot.xarray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "932477eb-f927-4e69-8617-13cbb23023fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "da = ds_best['temperature'][:,100,0].compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d908be83-1536-451a-9edf-adf1f6e35605",
   "metadata": {},
   "outputs": [],
   "source": [
    "da.hvplot(x='valid_time', grid=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}