Skip to content

Instantly share code, notes, and snippets.

@matt-long

matt-long/cesm-le-toxarray-issue.ipynb

Last active July 15, 2019 19:20
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save matt-long/bf4ee82ad797906599fd054c5f56e63e to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save matt-long/bf4ee82ad797906599fd054c5f56e63e to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
cesm-le-toxarray-issue.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Trouble loading CESM-LE atmospheric data with `intake-esm`"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/glade/work/mclong/miniconda3/envs/analysis/lib/python3.7/site-packages/tqdm/autonotebook/__init__.py:18: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
" \" (e.g. in jupyter console)\", TqdmExperimentalWarning)\n"
]
}
],
"source": [
"import intake\n",
"import intake_esm"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Confirm database and cache directories"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"data-cache-directory: /glade/scratch/mclong/tmp/intake_esm/data_cache\n",
"database-directory: /glade/u/home/mclong/.intake_esm/collections\n"
]
}
],
"source": [
"for key in ['data-cache-directory', 'database-directory']:\n",
" print(f'{key}: {intake_esm.config.get(key)}')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Open collection"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>resource</th>\n",
" <th>resource_type</th>\n",
" <th>direct_access</th>\n",
" <th>experiment</th>\n",
" <th>case</th>\n",
" <th>component</th>\n",
" <th>stream</th>\n",
" <th>variable</th>\n",
" <th>date_range</th>\n",
" <th>member_id</th>\n",
" <th>file_fullpath</th>\n",
" <th>file_basename</th>\n",
" <th>file_dirname</th>\n",
" <th>ctrl_branch_year</th>\n",
" <th>year_offset</th>\n",
" <th>sequence_order</th>\n",
" <th>has_ocean_bgc</th>\n",
" <th>grid</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>CTRL:GLADE:posix:/glade/collections/cdg/data/c...</td>\n",
" <td>posix</td>\n",
" <td>True</td>\n",
" <td>CTRL</td>\n",
" <td>b.e11.B1850C5CN.f09_g16.005</td>\n",
" <td>atm</td>\n",
" <td>cam.h1</td>\n",
" <td>T200</td>\n",
" <td>18000101-18991231</td>\n",
" <td>0</td>\n",
" <td>/glade/collections/cdg/data/cesmLE/CESM-CAM5-B...</td>\n",
" <td>b.e11.B1850C5CN.f09_g16.005.cam.h1.T200.180001...</td>\n",
" <td>/glade/collections/cdg/data/cesmLE/CESM-CAM5-B...</td>\n",
" <td>NaN</td>\n",
" <td>1448.0</td>\n",
" <td>0</td>\n",
" <td>True</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>CTRL:GLADE:posix:/glade/collections/cdg/data/c...</td>\n",
" <td>posix</td>\n",
" <td>True</td>\n",
" <td>CTRL</td>\n",
" <td>b.e11.B1850C5CN.f09_g16.005</td>\n",
" <td>atm</td>\n",
" <td>cam.h1</td>\n",
" <td>T200</td>\n",
" <td>08000101-08991231</td>\n",
" <td>0</td>\n",
" <td>/glade/collections/cdg/data/cesmLE/CESM-CAM5-B...</td>\n",
" <td>b.e11.B1850C5CN.f09_g16.005.cam.h1.T200.080001...</td>\n",
" <td>/glade/collections/cdg/data/cesmLE/CESM-CAM5-B...</td>\n",
" <td>NaN</td>\n",
" <td>1448.0</td>\n",
" <td>0</td>\n",
" <td>True</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>CTRL:GLADE:posix:/glade/collections/cdg/data/c...</td>\n",
" <td>posix</td>\n",
" <td>True</td>\n",
" <td>CTRL</td>\n",
" <td>b.e11.B1850C5CN.f09_g16.005</td>\n",
" <td>atm</td>\n",
" <td>cam.h1</td>\n",
" <td>T200</td>\n",
" <td>04020101-04991231</td>\n",
" <td>0</td>\n",
" <td>/glade/collections/cdg/data/cesmLE/CESM-CAM5-B...</td>\n",
" <td>b.e11.B1850C5CN.f09_g16.005.cam.h1.T200.040201...</td>\n",
" <td>/glade/collections/cdg/data/cesmLE/CESM-CAM5-B...</td>\n",
" <td>NaN</td>\n",
" <td>1448.0</td>\n",
" <td>0</td>\n",
" <td>True</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>CTRL:GLADE:posix:/glade/collections/cdg/data/c...</td>\n",
" <td>posix</td>\n",
" <td>True</td>\n",
" <td>CTRL</td>\n",
" <td>b.e11.B1850C5CN.f09_g16.005</td>\n",
" <td>atm</td>\n",
" <td>cam.h1</td>\n",
" <td>T200</td>\n",
" <td>10000101-10991231</td>\n",
" <td>0</td>\n",
" <td>/glade/collections/cdg/data/cesmLE/CESM-CAM5-B...</td>\n",
" <td>b.e11.B1850C5CN.f09_g16.005.cam.h1.T200.100001...</td>\n",
" <td>/glade/collections/cdg/data/cesmLE/CESM-CAM5-B...</td>\n",
" <td>NaN</td>\n",
" <td>1448.0</td>\n",
" <td>0</td>\n",
" <td>True</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>CTRL:GLADE:posix:/glade/collections/cdg/data/c...</td>\n",
" <td>posix</td>\n",
" <td>True</td>\n",
" <td>CTRL</td>\n",
" <td>b.e11.B1850C5CN.f09_g16.005</td>\n",
" <td>atm</td>\n",
" <td>cam.h1</td>\n",
" <td>T200</td>\n",
" <td>11000101-11991231</td>\n",
" <td>0</td>\n",
" <td>/glade/collections/cdg/data/cesmLE/CESM-CAM5-B...</td>\n",
" <td>b.e11.B1850C5CN.f09_g16.005.cam.h1.T200.110001...</td>\n",
" <td>/glade/collections/cdg/data/cesmLE/CESM-CAM5-B...</td>\n",
" <td>NaN</td>\n",
" <td>1448.0</td>\n",
" <td>0</td>\n",
" <td>True</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" resource resource_type \\\n",
"0 CTRL:GLADE:posix:/glade/collections/cdg/data/c... posix \n",
"1 CTRL:GLADE:posix:/glade/collections/cdg/data/c... posix \n",
"2 CTRL:GLADE:posix:/glade/collections/cdg/data/c... posix \n",
"3 CTRL:GLADE:posix:/glade/collections/cdg/data/c... posix \n",
"4 CTRL:GLADE:posix:/glade/collections/cdg/data/c... posix \n",
"\n",
" direct_access experiment case component stream \\\n",
"0 True CTRL b.e11.B1850C5CN.f09_g16.005 atm cam.h1 \n",
"1 True CTRL b.e11.B1850C5CN.f09_g16.005 atm cam.h1 \n",
"2 True CTRL b.e11.B1850C5CN.f09_g16.005 atm cam.h1 \n",
"3 True CTRL b.e11.B1850C5CN.f09_g16.005 atm cam.h1 \n",
"4 True CTRL b.e11.B1850C5CN.f09_g16.005 atm cam.h1 \n",
"\n",
" variable date_range member_id \\\n",
"0 T200 18000101-18991231 0 \n",
"1 T200 08000101-08991231 0 \n",
"2 T200 04020101-04991231 0 \n",
"3 T200 10000101-10991231 0 \n",
"4 T200 11000101-11991231 0 \n",
"\n",
" file_fullpath \\\n",
"0 /glade/collections/cdg/data/cesmLE/CESM-CAM5-B... \n",
"1 /glade/collections/cdg/data/cesmLE/CESM-CAM5-B... \n",
"2 /glade/collections/cdg/data/cesmLE/CESM-CAM5-B... \n",
"3 /glade/collections/cdg/data/cesmLE/CESM-CAM5-B... \n",
"4 /glade/collections/cdg/data/cesmLE/CESM-CAM5-B... \n",
"\n",
" file_basename \\\n",
"0 b.e11.B1850C5CN.f09_g16.005.cam.h1.T200.180001... \n",
"1 b.e11.B1850C5CN.f09_g16.005.cam.h1.T200.080001... \n",
"2 b.e11.B1850C5CN.f09_g16.005.cam.h1.T200.040201... \n",
"3 b.e11.B1850C5CN.f09_g16.005.cam.h1.T200.100001... \n",
"4 b.e11.B1850C5CN.f09_g16.005.cam.h1.T200.110001... \n",
"\n",
" file_dirname ctrl_branch_year \\\n",
"0 /glade/collections/cdg/data/cesmLE/CESM-CAM5-B... NaN \n",
"1 /glade/collections/cdg/data/cesmLE/CESM-CAM5-B... NaN \n",
"2 /glade/collections/cdg/data/cesmLE/CESM-CAM5-B... NaN \n",
"3 /glade/collections/cdg/data/cesmLE/CESM-CAM5-B... NaN \n",
"4 /glade/collections/cdg/data/cesmLE/CESM-CAM5-B... NaN \n",
"\n",
" year_offset sequence_order has_ocean_bgc grid \n",
"0 1448.0 0 True NaN \n",
"1 1448.0 0 True NaN \n",
"2 1448.0 0 True NaN \n",
"3 1448.0 0 True NaN \n",
"4 1448.0 0 True NaN "
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"col = intake.open_esm_metadatastore(collection_name='CESM1-LE')\n",
"col.df.head() "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Query for variable `TS` from atmosphere component"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>resource</th>\n",
" <th>resource_type</th>\n",
" <th>direct_access</th>\n",
" <th>experiment</th>\n",
" <th>case</th>\n",
" <th>component</th>\n",
" <th>stream</th>\n",
" <th>variable</th>\n",
" <th>date_range</th>\n",
" <th>member_id</th>\n",
" <th>file_fullpath</th>\n",
" <th>file_basename</th>\n",
" <th>file_dirname</th>\n",
" <th>ctrl_branch_year</th>\n",
" <th>year_offset</th>\n",
" <th>sequence_order</th>\n",
" <th>has_ocean_bgc</th>\n",
" <th>grid</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>110059</th>\n",
" <td>20C:HPSS:hsi:/CCSM/csm/CESM-CAM5-BGC-LE</td>\n",
" <td>hsi</td>\n",
" <td>False</td>\n",
" <td>20C</td>\n",
" <td>b.e11.B20TRC5CNBDRD.f09_g16.001</td>\n",
" <td>atm</td>\n",
" <td>cam.h0</td>\n",
" <td>TS</td>\n",
" <td>185001-200512</td>\n",
" <td>1</td>\n",
" <td>/CCSM/csm/CESM-CAM5-BGC-LE/atm/proc/tseries/mo...</td>\n",
" <td>b.e11.B20TRC5CNBDRD.f09_g16.001.cam.h0.TS.1850...</td>\n",
" <td>/CCSM/csm/CESM-CAM5-BGC-LE/atm/proc/tseries/mo...</td>\n",
" <td>402.0</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>True</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>108863</th>\n",
" <td>20C:HPSS:hsi:/CCSM/csm/CESM-CAM5-BGC-LE</td>\n",
" <td>hsi</td>\n",
" <td>False</td>\n",
" <td>20C</td>\n",
" <td>b.e11.B20TRC5CNBDRD.f09_g16.002</td>\n",
" <td>atm</td>\n",
" <td>cam.h0</td>\n",
" <td>TS</td>\n",
" <td>192001-200512</td>\n",
" <td>2</td>\n",
" <td>/CCSM/csm/CESM-CAM5-BGC-LE/atm/proc/tseries/mo...</td>\n",
" <td>b.e11.B20TRC5CNBDRD.f09_g16.002.cam.h0.TS.1920...</td>\n",
" <td>/CCSM/csm/CESM-CAM5-BGC-LE/atm/proc/tseries/mo...</td>\n",
" <td>402.0</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>True</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>107969</th>\n",
" <td>20C:HPSS:hsi:/CCSM/csm/CESM-CAM5-BGC-LE</td>\n",
" <td>hsi</td>\n",
" <td>False</td>\n",
" <td>20C</td>\n",
" <td>b.e11.B20TRC5CNBDRD.f09_g16.003</td>\n",
" <td>atm</td>\n",
" <td>cam.h0</td>\n",
" <td>TS</td>\n",
" <td>192001-200512</td>\n",
" <td>3</td>\n",
" <td>/CCSM/csm/CESM-CAM5-BGC-LE/atm/proc/tseries/mo...</td>\n",
" <td>b.e11.B20TRC5CNBDRD.f09_g16.003.cam.h0.TS.1920...</td>\n",
" <td>/CCSM/csm/CESM-CAM5-BGC-LE/atm/proc/tseries/mo...</td>\n",
" <td>402.0</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>False</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>107081</th>\n",
" <td>20C:HPSS:hsi:/CCSM/csm/CESM-CAM5-BGC-LE</td>\n",
" <td>hsi</td>\n",
" <td>False</td>\n",
" <td>20C</td>\n",
" <td>b.e11.B20TRC5CNBDRD.f09_g16.004</td>\n",
" <td>atm</td>\n",
" <td>cam.h0</td>\n",
" <td>TS</td>\n",
" <td>192001-200512</td>\n",
" <td>4</td>\n",
" <td>/CCSM/csm/CESM-CAM5-BGC-LE/atm/proc/tseries/mo...</td>\n",
" <td>b.e11.B20TRC5CNBDRD.f09_g16.004.cam.h0.TS.1920...</td>\n",
" <td>/CCSM/csm/CESM-CAM5-BGC-LE/atm/proc/tseries/mo...</td>\n",
" <td>402.0</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>False</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>106193</th>\n",
" <td>20C:HPSS:hsi:/CCSM/csm/CESM-CAM5-BGC-LE</td>\n",
" <td>hsi</td>\n",
" <td>False</td>\n",
" <td>20C</td>\n",
" <td>b.e11.B20TRC5CNBDRD.f09_g16.005</td>\n",
" <td>atm</td>\n",
" <td>cam.h0</td>\n",
" <td>TS</td>\n",
" <td>192001-200512</td>\n",
" <td>5</td>\n",
" <td>/CCSM/csm/CESM-CAM5-BGC-LE/atm/proc/tseries/mo...</td>\n",
" <td>b.e11.B20TRC5CNBDRD.f09_g16.005.cam.h0.TS.1920...</td>\n",
" <td>/CCSM/csm/CESM-CAM5-BGC-LE/atm/proc/tseries/mo...</td>\n",
" <td>402.0</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>False</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" resource resource_type direct_access \\\n",
"110059 20C:HPSS:hsi:/CCSM/csm/CESM-CAM5-BGC-LE hsi False \n",
"108863 20C:HPSS:hsi:/CCSM/csm/CESM-CAM5-BGC-LE hsi False \n",
"107969 20C:HPSS:hsi:/CCSM/csm/CESM-CAM5-BGC-LE hsi False \n",
"107081 20C:HPSS:hsi:/CCSM/csm/CESM-CAM5-BGC-LE hsi False \n",
"106193 20C:HPSS:hsi:/CCSM/csm/CESM-CAM5-BGC-LE hsi False \n",
"\n",
" experiment case component stream variable \\\n",
"110059 20C b.e11.B20TRC5CNBDRD.f09_g16.001 atm cam.h0 TS \n",
"108863 20C b.e11.B20TRC5CNBDRD.f09_g16.002 atm cam.h0 TS \n",
"107969 20C b.e11.B20TRC5CNBDRD.f09_g16.003 atm cam.h0 TS \n",
"107081 20C b.e11.B20TRC5CNBDRD.f09_g16.004 atm cam.h0 TS \n",
"106193 20C b.e11.B20TRC5CNBDRD.f09_g16.005 atm cam.h0 TS \n",
"\n",
" date_range member_id \\\n",
"110059 185001-200512 1 \n",
"108863 192001-200512 2 \n",
"107969 192001-200512 3 \n",
"107081 192001-200512 4 \n",
"106193 192001-200512 5 \n",
"\n",
" file_fullpath \\\n",
"110059 /CCSM/csm/CESM-CAM5-BGC-LE/atm/proc/tseries/mo... \n",
"108863 /CCSM/csm/CESM-CAM5-BGC-LE/atm/proc/tseries/mo... \n",
"107969 /CCSM/csm/CESM-CAM5-BGC-LE/atm/proc/tseries/mo... \n",
"107081 /CCSM/csm/CESM-CAM5-BGC-LE/atm/proc/tseries/mo... \n",
"106193 /CCSM/csm/CESM-CAM5-BGC-LE/atm/proc/tseries/mo... \n",
"\n",
" file_basename \\\n",
"110059 b.e11.B20TRC5CNBDRD.f09_g16.001.cam.h0.TS.1850... \n",
"108863 b.e11.B20TRC5CNBDRD.f09_g16.002.cam.h0.TS.1920... \n",
"107969 b.e11.B20TRC5CNBDRD.f09_g16.003.cam.h0.TS.1920... \n",
"107081 b.e11.B20TRC5CNBDRD.f09_g16.004.cam.h0.TS.1920... \n",
"106193 b.e11.B20TRC5CNBDRD.f09_g16.005.cam.h0.TS.1920... \n",
"\n",
" file_dirname ctrl_branch_year \\\n",
"110059 /CCSM/csm/CESM-CAM5-BGC-LE/atm/proc/tseries/mo... 402.0 \n",
"108863 /CCSM/csm/CESM-CAM5-BGC-LE/atm/proc/tseries/mo... 402.0 \n",
"107969 /CCSM/csm/CESM-CAM5-BGC-LE/atm/proc/tseries/mo... 402.0 \n",
"107081 /CCSM/csm/CESM-CAM5-BGC-LE/atm/proc/tseries/mo... 402.0 \n",
"106193 /CCSM/csm/CESM-CAM5-BGC-LE/atm/proc/tseries/mo... 402.0 \n",
"\n",
" year_offset sequence_order has_ocean_bgc grid \n",
"110059 NaN 0 True NaN \n",
"108863 NaN 0 True NaN \n",
"107969 NaN 0 False NaN \n",
"107081 NaN 0 False NaN \n",
"106193 NaN 0 False NaN "
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cat = col.search(experiment=['20C', 'RCP85'], \n",
" variable=['TS'], stream='cam.h0',\n",
" member_id=[i for i in range(1, 30)])\n",
"cat.query_results.head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 1st Error\n",
"\n",
"Using the `to_xarray` method on this query returns an error: `time` index has duplicate values."
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "70b820838a1d4c3687184dcc6a4c7219",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, description='dataset', max=1, style=ProgressStyle(description_width='initi…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "3ac4a5a4a7524c3e97a43ee8efeb320b",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, description='member', max=29, style=ProgressStyle(description_width='initi…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
},
{
"ename": "ValueError",
"evalue": "cannot reindex or align along dimension 'time' because the index has duplicate values",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-5-edd6840210bc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 3\u001b[0m member_id=[i for i in range(1, 30)])\n\u001b[1;32m 4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mdq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_xarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'time'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m12\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjoin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'inner'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m/glade/work/mclong/miniconda3/envs/analysis/lib/python3.7/site-packages/intake_esm/source.py\u001b[0m in \u001b[0;36mto_xarray\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m 117\u001b[0m \u001b[0m_kwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 118\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_kwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 119\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_dask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 120\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 121\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_get_schema\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/glade/work/mclong/miniconda3/envs/analysis/lib/python3.7/site-packages/intake_xarray/base.py\u001b[0m in \u001b[0;36mto_dask\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 68\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mto_dask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 69\u001b[0m \u001b[0;34m\"\"\"Return xarray object where variables are dask arrays\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 70\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_chunked\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 71\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 72\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/glade/work/mclong/miniconda3/envs/analysis/lib/python3.7/site-packages/intake_xarray/base.py\u001b[0m in \u001b[0;36mread_chunked\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 43\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mread_chunked\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 44\u001b[0m \u001b[0;34m\"\"\"Return xarray object (which will have chunks)\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_load_metadata\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 46\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ds\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/glade/work/mclong/miniconda3/envs/analysis/lib/python3.7/site-packages/intake/source/base.py\u001b[0m in \u001b[0;36m_load_metadata\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 115\u001b[0m \u001b[0;34m\"\"\"load metadata only if needed\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 116\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_schema\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 117\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_schema\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_schema\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 118\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatashape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_schema\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatashape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 119\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_schema\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/glade/work/mclong/miniconda3/envs/analysis/lib/python3.7/site-packages/intake_esm/source.py\u001b[0m in \u001b[0;36m_get_schema\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 126\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 127\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ds\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 128\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_open_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 129\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 130\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/glade/work/mclong/miniconda3/envs/analysis/lib/python3.7/site-packages/intake_esm/cesm.py\u001b[0m in \u001b[0;36m_open_dataset\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 147\u001b[0m \u001b[0mmember_column_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'member_id'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 148\u001b[0m \u001b[0mvariable_column_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'variable'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 149\u001b[0;31m \u001b[0mfile_fullpath_column_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'file_fullpath'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 150\u001b[0m )\n",
"\u001b[0;32m/glade/work/mclong/miniconda3/envs/analysis/lib/python3.7/site-packages/intake_esm/source.py\u001b[0m in \u001b[0;36m_open_dataset_groups\u001b[0;34m(self, dataset_fields, member_column_name, variable_column_name, file_fullpath_column_name, file_basename_column_name)\u001b[0m\n\u001b[1;32m 104\u001b[0m \u001b[0mmember_dsets\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmember_dset_i\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 105\u001b[0m _ds = aggregate.concat_ensembles(\n\u001b[0;32m--> 106\u001b[0;31m \u001b[0mmember_dsets\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmember_ids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmember_ids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjoin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'join'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 107\u001b[0m )\n\u001b[1;32m 108\u001b[0m \u001b[0mall_dsets\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdset_id\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_ds\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/glade/work/mclong/miniconda3/envs/analysis/lib/python3.7/site-packages/intake_esm/aggregate.py\u001b[0m in \u001b[0;36mconcat_ensembles\u001b[0;34m(dsets, member_ids, join, ensemble_dim_name, time_coord_name_default)\u001b[0m\n\u001b[1;32m 165\u001b[0m \u001b[0;31m# inner join keeps only overlapping segments of each ensemble\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 166\u001b[0m \u001b[0;31m# outer join gives us the longest possible record\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 167\u001b[0;31m \u001b[0mdsets_aligned\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mxr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0malign\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mdsets\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjoin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 168\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 169\u001b[0m \u001b[0;31m# use coords and static_vars from first dataset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/glade/work/mclong/miniconda3/envs/analysis/lib/python3.7/site-packages/xarray/core/alignment.py\u001b[0m in \u001b[0;36malign\u001b[0;34m(join, copy, indexes, exclude, fill_value, *objects)\u001b[0m\n\u001b[1;32m 156\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 157\u001b[0m new_obj = obj.reindex(copy=copy, fill_value=fill_value,\n\u001b[0;32m--> 158\u001b[0;31m **valid_indexers)\n\u001b[0m\u001b[1;32m 159\u001b[0m \u001b[0mnew_obj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencoding\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencoding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 160\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_obj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/glade/work/mclong/miniconda3/envs/analysis/lib/python3.7/site-packages/xarray/core/dataset.py\u001b[0m in \u001b[0;36mreindex\u001b[0;34m(self, indexers, method, tolerance, copy, fill_value, **indexers_kwargs)\u001b[0m\n\u001b[1;32m 2105\u001b[0m variables, indexes = alignment.reindex_variables(\n\u001b[1;32m 2106\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvariables\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msizes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindexes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindexers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2107\u001b[0;31m tolerance, copy=copy, fill_value=fill_value)\n\u001b[0m\u001b[1;32m 2108\u001b[0m \u001b[0mcoord_names\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_coord_names\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2109\u001b[0m \u001b[0mcoord_names\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/glade/work/mclong/miniconda3/envs/analysis/lib/python3.7/site-packages/xarray/core/alignment.py\u001b[0m in \u001b[0;36mreindex_variables\u001b[0;34m(variables, sizes, indexes, indexers, method, tolerance, copy, fill_value)\u001b[0m\n\u001b[1;32m 339\u001b[0m raise ValueError(\n\u001b[1;32m 340\u001b[0m \u001b[0;34m'cannot reindex or align along dimension %r because the '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 341\u001b[0;31m 'index has duplicate values' % dim)\n\u001b[0m\u001b[1;32m 342\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 343\u001b[0m \u001b[0mint_indexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_indexer_nd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtolerance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mValueError\u001b[0m: cannot reindex or align along dimension 'time' because the index has duplicate values"
]
}
],
"source": [
"cat = col.search(experiment=['20C', 'RCP85'], \n",
" variable=['TS'], stream='cam.h0',\n",
" member_id=[i for i in range(1, 30)])\n",
"\n",
"dq = cat.to_xarray(chunks={'time': 12}, join='inner')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"If I do a similar query for an ocean variable, there's no problem."
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "04326730499242d1bcbf2bfe8cd6c7b0",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, description='dataset', max=1, style=ProgressStyle(description_width='initi…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "4adadddaab054c2eb97b71fbdfeba743",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, description='member', max=29, style=ProgressStyle(description_width='initi…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
},
{
"data": {
"text/plain": [
"<xarray.Dataset>\n",
"Dimensions: (d2: 2, lat_aux_grid: 395, member_id: 29, moc_comp: 3, moc_z: 61, nlat: 384, nlon: 320, time: 2172, transport_comp: 5, transport_reg: 2, z_t: 1, z_t_150m: 15, z_w: 60, z_w_bot: 60, z_w_top: 60)\n",
"Coordinates:\n",
" * time (time) object 1920-02-01 00:00:00 ... 2101-01-01 00:00:00\n",
" * lat_aux_grid (lat_aux_grid) float64 -79.49 -78.95 ... 89.47 90.0\n",
" * moc_z (moc_z) float64 0.0 1e+03 2e+03 ... 5.25e+05 5.5e+05\n",
" * z_t (z_t) float64 500.0\n",
" * z_t_150m (z_t_150m) float64 500.0 1.5e+03 ... 1.35e+04 1.45e+04\n",
" * z_w (z_w) float64 0.0 1e+03 2e+03 ... 5e+05 5.25e+05\n",
" * z_w_bot (z_w_bot) float64 1e+03 2e+03 ... 5.25e+05 5.5e+05\n",
" * z_w_top (z_w_top) float64 0.0 1e+03 2e+03 ... 5e+05 5.25e+05\n",
" * member_id (member_id) int64 1 2 3 4 5 6 7 ... 24 25 26 27 28 29\n",
"Dimensions without coordinates: d2, moc_comp, nlat, nlon, transport_comp, transport_reg\n",
"Data variables:\n",
" ANGLE (nlat, nlon) float64 dask.array<shape=(384, 320), chunksize=(384, 320)>\n",
" ANGLET (nlat, nlon) float64 dask.array<shape=(384, 320), chunksize=(384, 320)>\n",
" DXT (nlat, nlon) float64 dask.array<shape=(384, 320), chunksize=(384, 320)>\n",
" DXU (nlat, nlon) float64 dask.array<shape=(384, 320), chunksize=(384, 320)>\n",
" DYT (nlat, nlon) float64 dask.array<shape=(384, 320), chunksize=(384, 320)>\n",
" DYU (nlat, nlon) float64 dask.array<shape=(384, 320), chunksize=(384, 320)>\n",
" HT (nlat, nlon) float64 dask.array<shape=(384, 320), chunksize=(384, 320)>\n",
" HTE (nlat, nlon) float64 dask.array<shape=(384, 320), chunksize=(384, 320)>\n",
" HTN (nlat, nlon) float64 dask.array<shape=(384, 320), chunksize=(384, 320)>\n",
" HU (nlat, nlon) float64 dask.array<shape=(384, 320), chunksize=(384, 320)>\n",
" HUS (nlat, nlon) float64 dask.array<shape=(384, 320), chunksize=(384, 320)>\n",
" HUW (nlat, nlon) float64 dask.array<shape=(384, 320), chunksize=(384, 320)>\n",
" KMT (nlat, nlon) float64 dask.array<shape=(384, 320), chunksize=(384, 320)>\n",
" KMU (nlat, nlon) float64 dask.array<shape=(384, 320), chunksize=(384, 320)>\n",
" REGION_MASK (nlat, nlon) float64 dask.array<shape=(384, 320), chunksize=(384, 320)>\n",
" T0_Kelvin float64 273.1\n",
" TAREA (nlat, nlon) float64 dask.array<shape=(384, 320), chunksize=(384, 320)>\n",
" TLAT (nlat, nlon) float64 dask.array<shape=(384, 320), chunksize=(384, 320)>\n",
" TLONG (nlat, nlon) float64 dask.array<shape=(384, 320), chunksize=(384, 320)>\n",
" UAREA (nlat, nlon) float64 dask.array<shape=(384, 320), chunksize=(384, 320)>\n",
" ULAT (nlat, nlon) float64 dask.array<shape=(384, 320), chunksize=(384, 320)>\n",
" ULONG (nlat, nlon) float64 dask.array<shape=(384, 320), chunksize=(384, 320)>\n",
" cp_air float64 1.005e+03\n",
" cp_sw float64 3.996e+07\n",
" days_in_norm_year timedelta64[ns] 365 days\n",
" dz (z_t) float32 dask.array<shape=(1,), chunksize=(1,)>\n",
" dzw (z_w) float32 dask.array<shape=(60,), chunksize=(60,)>\n",
" fwflux_factor float64 0.0001\n",
" grav float64 980.6\n",
" heat_to_PW float64 4.186e-15\n",
" hflux_factor float64 2.439e-05\n",
" latent_heat_fusion float64 3.337e+09\n",
" latent_heat_vapor float64 2.501e+06\n",
" mass_to_Sv float64 1e-12\n",
" moc_components (moc_comp) |S256 dask.array<shape=(3,), chunksize=(3,)>\n",
" momentum_factor float64 10.0\n",
" nsurface_t float64 8.621e+04\n",
" nsurface_u float64 8.305e+04\n",
" ocn_ref_salinity float64 34.7\n",
" omega float64 7.292e-05\n",
" ppt_to_salt float64 0.001\n",
" radius float64 6.371e+08\n",
" rho_air float64 1.292\n",
" rho_fw float64 1.0\n",
" rho_sw float64 1.026\n",
" salinity_factor float64 -0.00347\n",
" salt_to_Svppt float64 1e-09\n",
" salt_to_mmday float64 3.154e+05\n",
" salt_to_ppt float64 1e+03\n",
" sea_ice_salinity float64 4.0\n",
" sflux_factor float64 0.1\n",
" sound float64 1.5e+05\n",
" stefan_boltzmann float64 5.67e-08\n",
" transport_components (transport_comp) |S256 dask.array<shape=(5,), chunksize=(5,)>\n",
" transport_regions (transport_reg) |S256 dask.array<shape=(2,), chunksize=(2,)>\n",
" vonkar float64 0.4\n",
" time_bound (time, d2) object dask.array<shape=(2172, 2), chunksize=(12, 2)>\n",
" SST (member_id, time, z_t, nlat, nlon) float32 dask.array<shape=(29, 2172, 1, 384, 320), chunksize=(1, 12, 1, 384, 320)>\n",
"Attributes:\n",
" calendar: All years have exactly 365 days.\n",
" NCO: 4.3.7\n",
" revision: $Id: tavg.F90 41939 2012-11-14 16:37:23Z mlevy...\n",
" cell_methods: cell_methods = time: mean ==> the variable val...\n",
" history: 2019-07-15 12:47:59.584979 xarray.open_dataset...\n",
" source: CCSM POP2, the CCSM Ocean Component\n",
" title: b.e11.BRCP85C5CNBDRD.f09_g16.029\n",
" start_time: This dataset was created on 2014-05-29 at 21:3...\n",
" Conventions: CF-1.0; http://www.cgd.ucar.edu/cms/eaton/netc...\n",
" contents: Diagnostic and Prognostic Variables\n",
" nco_openmp_thread_number: 1"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cat = col.search(experiment=['20C', 'RCP85'], \n",
" variable=['SST'], stream='pop.h',\n",
" member_id=[i for i in range(1, 30)])\n",
"\n",
"dq = cat.to_xarray(chunks={'time': 12}, join='inner')\n",
"_, ds = dq.popitem()\n",
"ds"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"The problem is apparently with ensemble member 1; if I remove this run from the list, it works."
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "4f1917062e7349beba647c64f8413fc5",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, description='dataset', max=1, style=ProgressStyle(description_width='initi…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "ba55d08266084007a1c39188fc3d9fda",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, description='member', max=28, style=ProgressStyle(description_width='initi…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
},
{
"data": {
"text/plain": [
"<xarray.Dataset>\n",
"Dimensions: (ilev: 31, lat: 192, lev: 30, lon: 288, member_id: 28, nbnd: 2, slat: 191, slon: 288, time: 2172)\n",
"Coordinates:\n",
" * ilev (ilev) float64 2.255 5.032 10.16 18.56 ... 967.5 985.1 1e+03\n",
" * lat (lat) float64 -90.0 -89.06 -88.12 -87.17 ... 88.12 89.06 90.0\n",
" * lev (lev) float64 3.643 7.595 14.36 24.61 ... 957.5 976.3 992.6\n",
" * lon (lon) float64 0.0 1.25 2.5 3.75 ... 355.0 356.2 357.5 358.8\n",
" * slat (slat) float64 -89.53 -88.59 -87.64 ... 87.64 88.59 89.53\n",
" * slon (slon) float64 -0.625 0.625 1.875 3.125 ... 355.6 356.9 358.1\n",
" * time (time) object 1920-02-01 00:00:00 ... 2101-01-01 00:00:00\n",
" * member_id (member_id) int64 2 3 4 5 6 7 8 9 ... 22 23 24 25 26 27 28 29\n",
"Dimensions without coordinates: nbnd\n",
"Data variables:\n",
" P0 float64 1e+05\n",
" gw (lat) float64 dask.array<shape=(192,), chunksize=(192,)>\n",
" hyai (ilev) float64 dask.array<shape=(31,), chunksize=(31,)>\n",
" hyam (lev) float64 dask.array<shape=(30,), chunksize=(30,)>\n",
" hybi (ilev) float64 dask.array<shape=(31,), chunksize=(31,)>\n",
" hybm (lev) float64 dask.array<shape=(30,), chunksize=(30,)>\n",
" mdt int32 1800\n",
" nbdate int32 19200101\n",
" nbsec int32 0\n",
" ndbase int32 0\n",
" nlon (lat) int32 dask.array<shape=(192,), chunksize=(192,)>\n",
" nsbase int32 0\n",
" ntrk int32 1\n",
" ntrm int32 1\n",
" ntrn int32 1\n",
" w_stag (slat) float64 dask.array<shape=(191,), chunksize=(191,)>\n",
" wnummax (lat) int32 dask.array<shape=(192,), chunksize=(192,)>\n",
" ch4vmr (time) float64 dask.array<shape=(2172,), chunksize=(12,)>\n",
" co2vmr (time) float64 dask.array<shape=(2172,), chunksize=(12,)>\n",
" date (time) int32 dask.array<shape=(2172,), chunksize=(12,)>\n",
" date_written (time) |S8 dask.array<shape=(2172,), chunksize=(12,)>\n",
" datesec (time) int32 dask.array<shape=(2172,), chunksize=(12,)>\n",
" f11vmr (time) float64 dask.array<shape=(2172,), chunksize=(12,)>\n",
" f12vmr (time) float64 dask.array<shape=(2172,), chunksize=(12,)>\n",
" n2ovmr (time) float64 dask.array<shape=(2172,), chunksize=(12,)>\n",
" ndcur (time) int32 dask.array<shape=(2172,), chunksize=(12,)>\n",
" nscur (time) int32 dask.array<shape=(2172,), chunksize=(12,)>\n",
" nsteph (time) int32 dask.array<shape=(2172,), chunksize=(12,)>\n",
" sol_tsi (time) float64 dask.array<shape=(2172,), chunksize=(12,)>\n",
" time_bnds (time, nbnd) object dask.array<shape=(2172, 2), chunksize=(12, 2)>\n",
" time_written (time) |S8 dask.array<shape=(2172,), chunksize=(12,)>\n",
" LANDFRAC (member_id, time, lat, lon) float32 dask.array<shape=(28, 2172, 192, 288), chunksize=(1, 12, 192, 288)>\n",
" TS (member_id, time, lat, lon) float32 dask.array<shape=(28, 2172, 192, 288), chunksize=(1, 12, 192, 288)>\n",
"Attributes:\n",
" revision_Id: $Id$\n",
" NCO: 4.3.4\n",
" history: 2019-07-15 12:47:06.623309 xarray.open_dataset...\n",
" source: CAM\n",
" title: UNSET\n",
" initial_file: /glade/p/cesmdata/cseg/inputdata/atm/cam/inic/...\n",
" Conventions: CF-1.0\n",
" topography_file: /glade/p/cesmdata/cseg/inputdata/atm/cam/topo/...\n",
" Version: $Name$\n",
" logname: mai\n",
" nco_openmp_thread_number: 1"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cat = col.search(experiment=['20C', 'RCP85'], \n",
" variable=['TS', 'LANDFRAC'], stream='cam.h0', \n",
" member_id=[i for i in range(2, 30)])\n",
"dq = cat.to_xarray(chunks={'time': 12}, join='inner')\n",
"_, ds = dq.popitem()\n",
"ds"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"If I restrict the experiments to RCP8.5, it works."
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "732b1e43414e4181bac1384e2ce5aecc",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, description='dataset', max=1, style=ProgressStyle(description_width='initi…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "6164c48057c64cbc91d6219369bf216e",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, description='member', max=29, style=ProgressStyle(description_width='initi…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
},
{
"data": {
"text/plain": [
"<xarray.Dataset>\n",
"Dimensions: (ilev: 31, lat: 192, lev: 30, lon: 288, member_id: 29, nbnd: 2, slat: 191, slon: 288, time: 1140)\n",
"Coordinates:\n",
" * ilev (ilev) float64 2.255 5.032 10.16 18.56 ... 967.5 985.1 1e+03\n",
" * lat (lat) float64 -90.0 -89.06 -88.12 -87.17 ... 88.12 89.06 90.0\n",
" * lev (lev) float64 3.643 7.595 14.36 24.61 ... 957.5 976.3 992.6\n",
" * lon (lon) float64 0.0 1.25 2.5 3.75 ... 355.0 356.2 357.5 358.8\n",
" * slat (slat) float64 -89.53 -88.59 -87.64 ... 87.64 88.59 89.53\n",
" * slon (slon) float64 -0.625 0.625 1.875 3.125 ... 355.6 356.9 358.1\n",
" * time (time) object 2006-02-01 00:00:00 ... 2101-01-01 00:00:00\n",
" * member_id (member_id) int64 1 2 3 4 5 6 7 8 ... 22 23 24 25 26 27 28 29\n",
"Dimensions without coordinates: nbnd\n",
"Data variables:\n",
" P0 float64 1e+05\n",
" gw (lat) float64 dask.array<shape=(192,), chunksize=(192,)>\n",
" hyai (ilev) float64 dask.array<shape=(31,), chunksize=(31,)>\n",
" hyam (lev) float64 dask.array<shape=(30,), chunksize=(30,)>\n",
" hybi (ilev) float64 dask.array<shape=(31,), chunksize=(31,)>\n",
" hybm (lev) float64 dask.array<shape=(30,), chunksize=(30,)>\n",
" mdt int32 1800\n",
" nbdate int32 20060101\n",
" nbsec int32 0\n",
" ndbase int32 0\n",
" nlon (lat) int32 dask.array<shape=(192,), chunksize=(192,)>\n",
" nsbase int32 0\n",
" ntrk int32 1\n",
" ntrm int32 1\n",
" ntrn int32 1\n",
" w_stag (slat) float64 dask.array<shape=(191,), chunksize=(191,)>\n",
" wnummax (lat) int32 dask.array<shape=(192,), chunksize=(192,)>\n",
" ch4vmr (time) float64 dask.array<shape=(1140,), chunksize=(12,)>\n",
" co2vmr (time) float64 dask.array<shape=(1140,), chunksize=(12,)>\n",
" date (time) int32 dask.array<shape=(1140,), chunksize=(12,)>\n",
" date_written (time) |S8 dask.array<shape=(1140,), chunksize=(12,)>\n",
" datesec (time) int32 dask.array<shape=(1140,), chunksize=(12,)>\n",
" f11vmr (time) float64 dask.array<shape=(1140,), chunksize=(12,)>\n",
" f12vmr (time) float64 dask.array<shape=(1140,), chunksize=(12,)>\n",
" n2ovmr (time) float64 dask.array<shape=(1140,), chunksize=(12,)>\n",
" ndcur (time) int32 dask.array<shape=(1140,), chunksize=(12,)>\n",
" nscur (time) int32 dask.array<shape=(1140,), chunksize=(12,)>\n",
" nsteph (time) int32 dask.array<shape=(1140,), chunksize=(12,)>\n",
" sol_tsi (time) float64 dask.array<shape=(1140,), chunksize=(12,)>\n",
" time_bnds (time, nbnd) object dask.array<shape=(1140, 2), chunksize=(12, 2)>\n",
" time_written (time) |S8 dask.array<shape=(1140,), chunksize=(12,)>\n",
" LANDFRAC (member_id, time, lat, lon) float32 dask.array<shape=(29, 1140, 192, 288), chunksize=(1, 12, 192, 288)>\n",
" TS (member_id, time, lat, lon) float32 dask.array<shape=(29, 1140, 192, 288), chunksize=(1, 12, 192, 288)>\n",
"Attributes:\n",
" revision_Id: $Id$\n",
" NCO: 4.3.4\n",
" history: 2019-07-15 12:52:01.871666 xarray.open_dataset...\n",
" source: CAM\n",
" title: UNSET\n",
" Conventions: CF-1.0\n",
" topography_file: /glade/p/cesmdata/cseg/inputdata/atm/cam/topo/...\n",
" Version: $Name$\n",
" logname: mai\n",
" case: b.e11.BRCP85C5CNBDRD.f09_g16.029\n",
" nco_openmp_thread_number: 1"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cat = col.search(experiment=['RCP85'], \n",
" variable=['TS', 'LANDFRAC'], stream='cam.h0', \n",
" member_id=[i for i in range(1, 30)])\n",
"dq = cat.to_xarray(chunks={'time': 12}, join='inner')\n",
"_, ds = dq.popitem()\n",
"ds"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Here's the problem: for the TS variable, there's an extra file: 192001-200512. It's not present for several other variables."
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['/glade/collections/cdg/data/cesmLE/CESM-CAM5-BGC-LE/atm/proc/tseries/monthly/LANDFRAC/b.e11.B20TRC5CNBDRD.f09_g16.001.cam.h0.LANDFRAC.185001-200512.nc',\n",
" '/glade/collections/cdg/data/cesmLE/CESM-CAM5-BGC-LE/atm/proc/tseries/monthly/TS/b.e11.B20TRC5CNBDRD.f09_g16.001.cam.h0.TS.185001-200512.nc',\n",
" '/glade/collections/cdg/data/cesmLE/CESM-CAM5-BGC-LE/atm/proc/tseries/monthly/TS/b.e11.B20TRC5CNBDRD.f09_g16.001.cam.h0.TS.192001-200512.nc',\n",
" '/glade/collections/cdg/data/cesmLE/CESM-CAM5-BGC-LE/atm/proc/tseries/monthly/U/b.e11.B20TRC5CNBDRD.f09_g16.001.cam.h0.U.185001-200512.nc']"
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cat = col.search(experiment=['20C'], \n",
" variable=['TS', 'LANDFRAC', 'U'], stream='cam.h0', \n",
" direct_access=True,\n",
" member_id=[1])\n",
"df = cat.query_results\n",
"\n",
"df.file_fullpath.tolist()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Does this exist for any other variables? Nope."
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>resource</th>\n",
" <th>resource_type</th>\n",
" <th>direct_access</th>\n",
" <th>experiment</th>\n",
" <th>case</th>\n",
" <th>component</th>\n",
" <th>stream</th>\n",
" <th>variable</th>\n",
" <th>date_range</th>\n",
" <th>member_id</th>\n",
" <th>file_fullpath</th>\n",
" <th>file_basename</th>\n",
" <th>file_dirname</th>\n",
" <th>ctrl_branch_year</th>\n",
" <th>year_offset</th>\n",
" <th>sequence_order</th>\n",
" <th>has_ocean_bgc</th>\n",
" <th>grid</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>67451</th>\n",
" <td>20C:GLADE:posix:/glade/collections/cdg/data/ce...</td>\n",
" <td>posix</td>\n",
" <td>True</td>\n",
" <td>20C</td>\n",
" <td>b.e11.B20TRC5CNBDRD.f09_g16.001</td>\n",
" <td>atm</td>\n",
" <td>cam.h0</td>\n",
" <td>TS</td>\n",
" <td>192001-200512</td>\n",
" <td>1</td>\n",
" <td>/glade/collections/cdg/data/cesmLE/CESM-CAM5-B...</td>\n",
" <td>b.e11.B20TRC5CNBDRD.f09_g16.001.cam.h0.TS.1920...</td>\n",
" <td>/glade/collections/cdg/data/cesmLE/CESM-CAM5-B...</td>\n",
" <td>402.0</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>True</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" resource resource_type \\\n",
"67451 20C:GLADE:posix:/glade/collections/cdg/data/ce... posix \n",
"\n",
" direct_access experiment case component \\\n",
"67451 True 20C b.e11.B20TRC5CNBDRD.f09_g16.001 atm \n",
"\n",
" stream variable date_range member_id \\\n",
"67451 cam.h0 TS 192001-200512 1 \n",
"\n",
" file_fullpath \\\n",
"67451 /glade/collections/cdg/data/cesmLE/CESM-CAM5-B... \n",
"\n",
" file_basename \\\n",
"67451 b.e11.B20TRC5CNBDRD.f09_g16.001.cam.h0.TS.1920... \n",
"\n",
" file_dirname ctrl_branch_year \\\n",
"67451 /glade/collections/cdg/data/cesmLE/CESM-CAM5-B... 402.0 \n",
"\n",
" year_offset sequence_order has_ocean_bgc grid \n",
"67451 NaN 0 True NaN "
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cat = col.search(experiment=['20C'], stream='cam.h0', \n",
" date_range='192001-200512',\n",
" direct_access=True,\n",
" member_id=[1])\n",
"df = cat.query_results\n",
"df"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 2nd Error\n",
"\n",
"If I expand my query to include ensemble members >30, I get a different error. This results from the fact that some of the later ensemble members included additional variables in the files. `xarray` is throwing an error on the ensemble concatenation."
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "48c8f8b8338743e3b7761106a3aeeabb",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, description='dataset', max=1, style=ProgressStyle(description_width='initi…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "2e659a677ecc4105b324a489880ffc8b",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, description='member', max=38, style=ProgressStyle(description_width='initi…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
},
{
"ename": "ValueError",
"evalue": "encountered unexpected variable 'cosp_prs'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-7-481bda08fd9f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0mvariable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'TS'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstream\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'cam.h0'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m member_id=[i for i in range(2, 105)])\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mdq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_xarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'time'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m12\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjoin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'inner'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 5\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpopitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0mds\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/glade/work/mclong/miniconda3/envs/analysis/lib/python3.7/site-packages/intake_esm/source.py\u001b[0m in \u001b[0;36mto_xarray\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m 117\u001b[0m \u001b[0m_kwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 118\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_kwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 119\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_dask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 120\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 121\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_get_schema\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/glade/work/mclong/miniconda3/envs/analysis/lib/python3.7/site-packages/intake_xarray/base.py\u001b[0m in \u001b[0;36mto_dask\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 68\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mto_dask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 69\u001b[0m \u001b[0;34m\"\"\"Return xarray object where variables are dask arrays\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 70\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_chunked\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 71\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 72\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/glade/work/mclong/miniconda3/envs/analysis/lib/python3.7/site-packages/intake_xarray/base.py\u001b[0m in \u001b[0;36mread_chunked\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 43\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mread_chunked\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 44\u001b[0m \u001b[0;34m\"\"\"Return xarray object (which will have chunks)\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_load_metadata\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 46\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ds\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/glade/work/mclong/miniconda3/envs/analysis/lib/python3.7/site-packages/intake/source/base.py\u001b[0m in \u001b[0;36m_load_metadata\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 115\u001b[0m \u001b[0;34m\"\"\"load metadata only if needed\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 116\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_schema\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 117\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_schema\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_schema\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 118\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatashape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_schema\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatashape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 119\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_schema\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/glade/work/mclong/miniconda3/envs/analysis/lib/python3.7/site-packages/intake_esm/source.py\u001b[0m in \u001b[0;36m_get_schema\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 126\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 127\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ds\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 128\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_open_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 129\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 130\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/glade/work/mclong/miniconda3/envs/analysis/lib/python3.7/site-packages/intake_esm/cesm.py\u001b[0m in \u001b[0;36m_open_dataset\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 147\u001b[0m \u001b[0mmember_column_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'member_id'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 148\u001b[0m \u001b[0mvariable_column_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'variable'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 149\u001b[0;31m \u001b[0mfile_fullpath_column_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'file_fullpath'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 150\u001b[0m )\n",
"\u001b[0;32m/glade/work/mclong/miniconda3/envs/analysis/lib/python3.7/site-packages/intake_esm/source.py\u001b[0m in \u001b[0;36m_open_dataset_groups\u001b[0;34m(self, dataset_fields, member_column_name, variable_column_name, file_fullpath_column_name, file_basename_column_name)\u001b[0m\n\u001b[1;32m 104\u001b[0m \u001b[0mmember_dsets\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmember_dset_i\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 105\u001b[0m _ds = aggregate.concat_ensembles(\n\u001b[0;32m--> 106\u001b[0;31m \u001b[0mmember_dsets\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmember_ids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmember_ids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjoin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'join'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 107\u001b[0m )\n\u001b[1;32m 108\u001b[0m \u001b[0mall_dsets\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdset_id\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_ds\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/glade/work/mclong/miniconda3/envs/analysis/lib/python3.7/site-packages/intake_esm/aggregate.py\u001b[0m in \u001b[0;36mconcat_ensembles\u001b[0;34m(dsets, member_ids, join, ensemble_dim_name, time_coord_name_default)\u001b[0m\n\u001b[1;32m 173\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 174\u001b[0m \u001b[0mensemble_dim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mxr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataArray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmember_ids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdims\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mensemble_dim_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mensemble_dim_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 175\u001b[0;31m \u001b[0mds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mxr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobjs_to_concat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mensemble_dim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcoords\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'minimal'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 176\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 177\u001b[0m \u001b[0mds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_restore_non_dim_coords\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/glade/work/mclong/miniconda3/envs/analysis/lib/python3.7/site-packages/xarray/core/concat.py\u001b[0m in \u001b[0;36mconcat\u001b[0;34m(objs, dim, data_vars, coords, compat, positions, indexers, mode, concat_over, fill_value)\u001b[0m\n\u001b[1;32m 117\u001b[0m raise TypeError('can only concatenate xarray Dataset and DataArray '\n\u001b[1;32m 118\u001b[0m 'objects, got %s' % type(first_obj))\n\u001b[0;32m--> 119\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobjs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata_vars\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcoords\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpositions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfill_value\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 120\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 121\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/glade/work/mclong/miniconda3/envs/analysis/lib/python3.7/site-packages/xarray/core/concat.py\u001b[0m in \u001b[0;36m_dataset_concat\u001b[0;34m(datasets, dim, data_vars, coords, compat, positions, fill_value)\u001b[0m\n\u001b[1;32m 255\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvariables\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 256\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mk\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mresult_vars\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mk\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mconcat_over\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 257\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'encountered unexpected variable %r'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 258\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mresult_coord_names\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcoords\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 259\u001b[0m raise ValueError('%r is a coordinate in some datasets but not '\n",
"\u001b[0;31mValueError\u001b[0m: encountered unexpected variable 'cosp_prs'"
]
}
],
"source": [
"cat = col.search(experiment=['20C', 'RCP85'], \n",
" variable=['TS'], stream='cam.h0', \n",
" member_id=[i for i in range(2, 105)])\n",
"dq = cat.to_xarray(chunks={'time': 12}, join='inner')\n",
"_, ds = dq.popitem()\n",
"ds"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,\n",
" 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27,\n",
" 28, 29, 30, 31, 32, 33, 34, 35, 101, 102, 103, 104])"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cat = col.search(experiment=['20C', 'RCP85'], \n",
" variable=['TS'], stream='cam.h0', \n",
" member_id=[i for i in range(2, 105)])\n",
"\n",
"df = cat.query_results\n",
"member_ids = df.member_id.unique()\n",
"member_ids"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Here's one of the problematic files."
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"netcdf b.e11.B20TRC5CNBDRD.f09_g16.035.cam.h0.TS.192001-200512 {\n",
"dimensions:\n",
"\tlat = 192 ;\n",
"\tlon = 288 ;\n",
"\tslat = 191 ;\n",
"\tslon = 288 ;\n",
"\ttime = UNLIMITED ; // (1032 currently)\n",
"\tnbnd = 2 ;\n",
"\tchars = 8 ;\n",
"\tlev = 30 ;\n",
"\tilev = 31 ;\n",
"\tcosp_prs = 7 ;\n",
"\tcosp_tau = 7 ;\n",
"\tcosp_scol = 10 ;\n",
"\tcosp_ht = 40 ;\n",
"\tcosp_sr = 15 ;\n",
"\tcosp_sza = 5 ;\n",
"\tcosp_htmisr = 16 ;\n",
"\tcosp_tau_modis = 6 ;\n",
"variables:\n",
"\tdouble lev(lev) ;\n",
"\t\tlev:long_name = \"hybrid level at midpoints (1000*(A+B))\" ;\n",
"\t\tlev:units = \"level\" ;\n",
"\t\tlev:positive = \"down\" ;\n",
"\t\tlev:standard_name = \"atmosphere_hybrid_sigma_pressure_coordinate\" ;\n",
"\t\tlev:formula_terms = \"a: hyam b: hybm p0: P0 ps: PS\" ;\n",
"\tdouble hyam(lev) ;\n",
"\t\thyam:long_name = \"hybrid A coefficient at layer midpoints\" ;\n",
"\tdouble hybm(lev) ;\n",
"\t\thybm:long_name = \"hybrid B coefficient at layer midpoints\" ;\n",
"\tdouble ilev(ilev) ;\n",
"\t\tilev:long_name = \"hybrid level at interfaces (1000*(A+B))\" ;\n",
"\t\tilev:units = \"level\" ;\n",
"\t\tilev:positive = \"down\" ;\n",
"\t\tilev:standard_name = \"atmosphere_hybrid_sigma_pressure_coordinate\" ;\n",
"\t\tilev:formula_terms = \"a: hyai b: hybi p0: P0 ps: PS\" ;\n",
"\tdouble hyai(ilev) ;\n",
"\t\thyai:long_name = \"hybrid A coefficient at layer interfaces\" ;\n",
"\tdouble hybi(ilev) ;\n",
"\t\thybi:long_name = \"hybrid B coefficient at layer interfaces\" ;\n",
"\tdouble cosp_prs(cosp_prs) ;\n",
"\t\tcosp_prs:long_name = \"COSP Mean ISCCP pressure\" ;\n",
"\t\tcosp_prs:units = \"mb\" ;\n",
"\t\tcosp_prs:bounds = \"cosp_prs_bnds\" ;\n",
"\tdouble cosp_prs_bnds(cosp_prs, nbnd) ;\n",
"\tdouble cosp_tau(cosp_tau) ;\n",
"\t\tcosp_tau:long_name = \"COSP Mean ISCCP optical depth\" ;\n",
"\t\tcosp_tau:units = \"unitless\" ;\n",
"\t\tcosp_tau:bounds = \"cosp_tau_bnds\" ;\n",
"\tdouble cosp_tau_bnds(cosp_tau, nbnd) ;\n",
"\tdouble cosp_scol(cosp_scol) ;\n",
"\t\tcosp_scol:long_name = \"COSP subcolumn\" ;\n",
"\t\tcosp_scol:units = \"number\" ;\n",
"\tdouble cosp_ht(cosp_ht) ;\n",
"\t\tcosp_ht:long_name = \"COSP Mean Height for lidar and radar simulator outputs\" ;\n",
"\t\tcosp_ht:units = \"m\" ;\n",
"\t\tcosp_ht:bounds = \"cosp_ht_bnds\" ;\n",
"\tdouble cosp_ht_bnds(cosp_ht, nbnd) ;\n",
"\tdouble cosp_sr(cosp_sr) ;\n",
"\t\tcosp_sr:long_name = \"COSP Mean Scattering Ratio for lidar simulator CFAD output\" ;\n",
"\t\tcosp_sr:units = \"1\" ;\n",
"\t\tcosp_sr:bounds = \"cosp_sr_bnds\" ;\n",
"\tdouble cosp_sr_bnds(cosp_sr, nbnd) ;\n",
"\tdouble cosp_sza(cosp_sza) ;\n",
"\t\tcosp_sza:long_name = \"COSP Parasol SZA\" ;\n",
"\t\tcosp_sza:units = \"degrees\" ;\n",
"\tdouble cosp_htmisr(cosp_htmisr) ;\n",
"\t\tcosp_htmisr:long_name = \"COSP MISR height\" ;\n",
"\t\tcosp_htmisr:units = \"km\" ;\n",
"\t\tcosp_htmisr:bounds = \"cosp_htmisr_bnds\" ;\n",
"\tdouble cosp_htmisr_bnds(cosp_htmisr, nbnd) ;\n",
"\tdouble cosp_tau_modis(cosp_tau_modis) ;\n",
"\t\tcosp_tau_modis:long_name = \"COSP Mean MODIS optical depth\" ;\n",
"\t\tcosp_tau_modis:units = \"unitless\" ;\n",
"\t\tcosp_tau_modis:bounds = \"cosp_tau_modis_bnds\" ;\n",
"\tdouble cosp_tau_modis_bnds(cosp_tau_modis, nbnd) ;\n",
"\tdouble P0 ;\n",
"\t\tP0:long_name = \"reference pressure\" ;\n",
"\t\tP0:units = \"Pa\" ;\n",
"\tdouble time(time) ;\n",
"\t\ttime:long_name = \"time\" ;\n",
"\t\ttime:units = \"days since 1920-01-01 00:00:00\" ;\n",
"\t\ttime:calendar = \"noleap\" ;\n",
"\t\ttime:bounds = \"time_bnds\" ;\n",
"\tint date(time) ;\n",
"\t\tdate:long_name = \"current date (YYYYMMDD)\" ;\n",
"\tint datesec(time) ;\n",
"\t\tdatesec:long_name = \"current seconds of current date\" ;\n",
"\tdouble lat(lat) ;\n",
"\t\tlat:long_name = \"latitude\" ;\n",
"\t\tlat:units = \"degrees_north\" ;\n",
"\tdouble lon(lon) ;\n",
"\t\tlon:long_name = \"longitude\" ;\n",
"\t\tlon:units = \"degrees_east\" ;\n",
"\tdouble slat(slat) ;\n",
"\t\tslat:long_name = \"staggered latitude\" ;\n",
"\t\tslat:units = \"degrees_north\" ;\n",
"\tdouble slon(slon) ;\n",
"\t\tslon:long_name = \"staggered longitude\" ;\n",
"\t\tslon:units = \"degrees_east\" ;\n",
"\tdouble w_stag(slat) ;\n",
"\t\tw_stag:long_name = \"staggered latitude weights\" ;\n",
"\tdouble time_bnds(time, nbnd) ;\n",
"\t\ttime_bnds:long_name = \"time interval endpoints\" ;\n",
"\tchar date_written(time, chars) ;\n",
"\tchar time_written(time, chars) ;\n",
"\tint ntrm ;\n",
"\t\tntrm:long_name = \"spectral truncation parameter M\" ;\n",
"\tint ntrn ;\n",
"\t\tntrn:long_name = \"spectral truncation parameter N\" ;\n",
"\tint ntrk ;\n",
"\t\tntrk:long_name = \"spectral truncation parameter K\" ;\n",
"\tint ndbase ;\n",
"\t\tndbase:long_name = \"base day\" ;\n",
"\tint nsbase ;\n",
"\t\tnsbase:long_name = \"seconds of base day\" ;\n",
"\tint nbdate ;\n",
"\t\tnbdate:long_name = \"base date (YYYYMMDD)\" ;\n",
"\tint nbsec ;\n",
"\t\tnbsec:long_name = \"seconds of base date\" ;\n",
"\tint mdt ;\n",
"\t\tmdt:long_name = \"timestep\" ;\n",
"\t\tmdt:units = \"s\" ;\n",
"\tint nlon(lat) ;\n",
"\t\tnlon:long_name = \"number of longitudes\" ;\n",
"\tint wnummax(lat) ;\n",
"\t\twnummax:long_name = \"cutoff Fourier wavenumber\" ;\n",
"\tdouble gw(lat) ;\n",
"\t\tgw:long_name = \"gauss weights\" ;\n",
"\tint ndcur(time) ;\n",
"\t\tndcur:long_name = \"current day (from base day)\" ;\n",
"\tint nscur(time) ;\n",
"\t\tnscur:long_name = \"current seconds of current day\" ;\n",
"\tdouble co2vmr(time) ;\n",
"\t\tco2vmr:long_name = \"co2 volume mixing ratio\" ;\n",
"\tdouble ch4vmr(time) ;\n",
"\t\tch4vmr:long_name = \"ch4 volume mixing ratio\" ;\n",
"\tdouble n2ovmr(time) ;\n",
"\t\tn2ovmr:long_name = \"n2o volume mixing ratio\" ;\n",
"\tdouble f11vmr(time) ;\n",
"\t\tf11vmr:long_name = \"f11 volume mixing ratio\" ;\n",
"\tdouble f12vmr(time) ;\n",
"\t\tf12vmr:long_name = \"f12 volume mixing ratio\" ;\n",
"\tdouble sol_tsi(time) ;\n",
"\t\tsol_tsi:long_name = \"total solar irradiance\" ;\n",
"\t\tsol_tsi:units = \"W/m2\" ;\n",
"\tint nsteph(time) ;\n",
"\t\tnsteph:long_name = \"current timestep\" ;\n",
"\tfloat TS(time, lat, lon) ;\n",
"\t\tTS:units = \"K\" ;\n",
"\t\tTS:long_name = \"Surface temperature (radiative)\" ;\n",
"\t\tTS:cell_methods = \"time: mean\" ;\n",
"\n",
"// global attributes:\n",
"\t\t:Conventions = \"CF-1.0\" ;\n",
"\t\t:source = \"CAM\" ;\n",
"\t\t:case = \"b.e11.B20TRC5CNBDRD.f09_g16.035\" ;\n",
"\t\t:title = \"UNSET\" ;\n",
"\t\t:logname = \"adrianne\" ;\n",
"\t\t:host = \"ys5848\" ;\n",
"\t\t:Version = \"$Name$\" ;\n",
"\t\t:revision_Id = \"$Id$\" ;\n",
"\t\t:initial_file = \"b.e11.B20TRC5CNBDRD.f09_g16.001.cam.i.1920-01-01-00000.nc\" ;\n",
"\t\t:topography_file = \"/glade/p/cesmdata/cseg/inputdata/atm/cam/topo/USGS-gtopo30_0.9x1.25_remap_c051027.nc\" ;\n",
"}\n"
]
}
],
"source": [
"!ncdump -h /glade/collections/cdg/data/cesmLE/CESM-CAM5-BGC-LE/atm/proc/tseries/monthly/TS/b.e11.B20TRC5CNBDRD.f09_g16.035.cam.h0.TS.192001-200512.nc"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python [conda env:analysis]",
"language": "python",
"name": "conda-env-analysis-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment