Merge pull request #1604 from ESMValGroup/cmorize_CT2019

Add CMORizer for CT2019

doc/sphinx/source/input.rst

@@ -92,6 +92,8 @@ A list of the datasets for which a cmorizers is available is provided in the fol
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | CRU                          | tas, pr (Amon)                                                                                       |   2  | Python          |
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
+| CT2019                       | co2s (Amon)                                                                                          |   2  | Python          |
++------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | Duveiller2018                | albDiffiTr13                                                                                         |   2  | Python          |
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | Eppley-VGPM-MODIS            | intpp (Omon)                                                                                         |   2  | Python          |

esmvaltool/cmorizers/obs/cmor_config/CT2019.yml

@@ -0,0 +1,26 @@
+---
+# FTP server used for downloading the data
+ftp_host: 'aftp.cmdl.noaa.gov'
+data_dir: 'products/carbontracker/co2/CT2019/molefractions/co2_total_monthly/'
+
+# Pattern of input files
+input_file_pattern: 'CT2019.molefrac_glb3x2_*.nc'
+
+# Common global attributes for Cmorizer output
+attributes:
+  dataset_id: CT2019
+  version: '2019'
+  tier: 2
+  modeling_realm: reanaly
+  project_id: OBS6
+  source: 'https://www.esrl.noaa.gov/gmd/ccgg/carbontracker/index.php'
+  reference: 'ct2019'
+  comment: ''
+
+# Variables to cmorize
+variables:
+  co2s:
+    mip: Amon
+    add_aux_coords:
+      air_pressure: [0, 1, 2, 3]
+    raw_long_name: mole_fraction_of_carbon_dioxide_in_air

esmvaltool/cmorizers/obs/cmorize_obs_ct2019.py

@@ -0,0 +1,192 @@
+"""ESMValTool CMORizer for CT2019 data.
+
+Tier
+    Tier 2: other freely-available dataset.
+
+Source
+    https://www.esrl.noaa.gov/gmd/ccgg/carbontracker/index.php
+
+Last access
+    20200323
+
+Download and processing instructions
+    Create a new empty directory ``$RAWOBSPATH/Tier2/CT2019`` (where
+    ``$RAWOBSPATH`` is given by your user configuration file) where the raw
+    data will be stored. The download of the data is automatically handled by
+    this script. If data is already present in this directory, the download is
+    skipped (to force a new download delete your old files).
+
+"""
+
+import fnmatch
+import glob
+import logging
+import os
+import warnings
+from ftplib import FTP
+from pprint import pformat
+
+import dask.array as da
+import iris
+
+from . import utilities as utils
+
+logger = logging.getLogger(__name__)
+
+
+def _add_aux_coords(cube, input_files, coords_to_add):
+    """Add additional auxiliary coordinates to cube."""
+    for (coord_name, coord_dims) in coords_to_add.items():
+        logger.info("Adding auxiliary coordinate '%s' to '%s'", coord_name,
+                    cube.var_name)
+        coord_cube = _load_cube(input_files, coord_name)
+        utils.fix_coords(coord_cube)
+        dim_coords = [c.name() for c in coord_cube.coords(dim_coords=True)]
+        if 'boundary' in dim_coords:
+            (points, bounds) = _interpolate_center(coord_cube)
+            attributes = {
+                'comment': 'Coordinate points where estimated as arithmetic '
+                           'mean from given coordinate bounds',
+            }
+        else:
+            points = coord_cube.core_data()
+            bounds = None
+            attributes = {}
+        if coord_cube.long_name == 'air_pressure':
+            coord_cube.long_name = 'pressure'
+            coord_cube.standard_name = 'air_pressure'
+            coord_cube.var_name = 'plev'
+        aux_coord = iris.coords.AuxCoord(
+            points,
+            bounds=bounds,
+            var_name=coord_cube.var_name,
+            standard_name=coord_cube.standard_name,
+            long_name=coord_cube.long_name,
+            units=coord_cube.units,
+            attributes=attributes,
+        )
+        cube.add_aux_coord(aux_coord, coord_dims)
+
+
+def _download_files(in_dir, cfg):
+    """Download input files using FTP."""
+    logger.info("Downloading data from FTP server %s", cfg['ftp_host'])
+    logger.info("Looking for files matching %s", os.path.join(
+        cfg['data_dir'], cfg['input_file_pattern']))
+    input_files = []
+    with FTP(cfg['ftp_host']) as ftp_client:
+        logger.info(ftp_client.getwelcome())
+        ftp_client.login()
+        ftp_client.cwd(cfg['data_dir'])
+        files_to_download = fnmatch.filter(ftp_client.nlst(),
+                                           cfg['input_file_pattern'])
+        for filename in files_to_download:
+            logger.info("Downloading %s", filename)
+            new_path = os.path.join(in_dir, filename)
+            with open(new_path, mode='wb') as outfile:
+                ftp_client.retrbinary(f'RETR {filename}', outfile.write)
+            input_files.append(new_path)
+    return input_files
+
+
+def _get_input_files(in_dir, cfg):
+    """Get input files."""
+    pattern = os.path.join(in_dir, cfg['input_file_pattern'])
+    input_files = glob.glob(pattern)
+    if not input_files:
+        input_files = _download_files(in_dir, cfg)
+    logger.debug("Found input files:\n%s", pformat(input_files))
+    return input_files
+
+
+def _interpolate_center(cube, axis=1):
+    """Interpolate center value for grid cells when only boundary is given."""
+    indices = [slice(None)] * cube.ndim
+    idx_all_but_first = indices.copy()
+    idx_all_but_first[axis] = slice(1, None, None)
+    idx_all_but_last = indices.copy()
+    idx_all_but_last[axis] = slice(None, -1, None)
+    data_all_but_first = cube.core_data()[tuple(idx_all_but_first)]
+    data_all_but_last = cube.core_data()[tuple(idx_all_but_last)]
+    points = (data_all_but_first + data_all_but_last) / 2.0
+    bounds = da.stack((data_all_but_last, data_all_but_first), axis=-1)
+    return (points, bounds)
+
+
+def _remove_attributes(cubes):
+    """Remove attributes from cubes that prevent concatenation."""
+    for cube in cubes:
+        cube.attributes.pop('history', None)
+        cube.attributes.pop('nco_input_file_list', None)
+        cube.attributes.pop('nco_input_file_number', None)
+        cube.attributes.pop('nco_openmp_thread_number', None)
+        cube.attributes.pop('NCO', None)
+        cube.attributes.pop('version', None)
+
+
+def _load_cube(input_files, constraints):
+    """Load single :class:`iris.cube.Cube`."""
+    with warnings.catch_warnings():
+        warnings.filterwarnings(
+            'ignore',
+            message='Ignoring netCDF variable',
+            category=UserWarning,
+            module='iris',
+        )
+        cubes = iris.load(input_files, constraints)
+    _remove_attributes(cubes)
+    try:
+        cube = cubes.concatenate_cube()
+    except iris.exceptions.ConcatenateError:
+        if cubes[0].coords('time'):
+            raise
+        cube = cubes[0]
+    return cube
+
+
+def _extract_variable(short_name, var, cfg, input_files, out_dir):
+    """Extract variable."""
+    cmor_info = cfg['cmor_table'].get_variable(var['mip'], short_name)
+
+    # Extract data
+    constraint = var.get('raw_long_name', cmor_info.standard_name)
+    cube = _load_cube(input_files, constraint)
+    cube.var_name = short_name
+
+    # Add auxiliary variables
+    _add_aux_coords(cube, input_files, var.get('add_aux_coords', {}))
+
+    # Variable specific operations
+    if short_name == 'co2s':
+        cube = cube[:, 0, :, :]
+        cube.remove_coord('level')
+
+    # Fix units
+    cube.convert_units(cmor_info.units)
+    utils.convert_timeunits(cube, 1950)
+
+    # Fix coordinates
+    utils.fix_coords(cube)
+
+    # Fix metadata
+    attrs = cfg['attributes']
+    attrs['mip'] = var['mip']
+    utils.fix_var_metadata(cube, cmor_info)
+    utils.set_global_atts(cube, attrs)
+
+    # Save variable
+    utils.save_variable(cube,
+                        short_name,
+                        out_dir,
+                        attrs,
+                        unlimited_dimensions=['time'])
+
+
+def cmorization(in_dir, out_dir, cfg, _):
+    """Cmorization func call."""
+    input_files = _get_input_files(in_dir, cfg)
+
+    # Run the cmorization
+    for (short_name, var) in cfg['variables'].items():
+        logger.info("CMORizing variable '%s'", short_name)
+        _extract_variable(short_name, var, cfg, input_files, out_dir)

esmvaltool/recipes/examples/recipe_check_obs.yml

@@ -89,6 +89,16 @@ diagnostics:
     scripts: null
 
 
+  CT2019:
+    description: CT2019 check
+    variables:
+      co2s:
+    additional_datasets:
+      - {dataset: CT2019, project: OBS6, mip: Amon, tier: 2,
+         type: reanaly, version: 2019, start_year: 2000, end_year: 2018}
+    scripts: null
+
+
   Duveiller2018:
     description: Duveiller2018 check
     variables:

esmvaltool/references/ct2019.bibtex

@@ -0,0 +1,9 @@
+@misc{ct2019,
+  doi = {10.25925/39M3-6069},
+  url = {https://www.esrl.noaa.gov/gmd/ccgg/carbontracker/CT2019/},
+  author = {Jacobson,  Andrew R. and Schuldt,  Kenneth N. and Miller,  John B. and Oda,  Tomohiro and Tans,  Pieter and {Arlyn Andrews} and Mund,  John and Ott,  Lesley and Collatz,  George J. and Aalto,  Tuula and Afshar,  Sara and Aikin,  Ken and Aoki,  Shuji and Apadula,  Francesco and Baier,  Bianca and Bergamaschi,  Peter and Beyersdorf,  Andreas and Biraud,  Sebastien C. and Bollenbacher,  Alane and Bowling,  David and Brailsford,  Gordon and Abshire,  James Brice and Chen,  Gao and {Huilin Chen} and {Lukasz Chmura} and {Sites Climadat} and Colomb,  Aurelie and Conil,  Sébastien and Cox,  Adam and Cristofanelli,  Paolo and Cuevas,  Emilio and Curcoll,  Roger and Sloop,  Christopher D. and Davis,  Ken and Wekker,  Stephan De and Delmotte,  Marc and DiGangi,  Joshua P. and Dlugokencky,  Ed and Ehleringer,  Jim and Elkins,  James W. and Emmenegger,  Lukas and Fischer,  Marc L. and Forster,  Grant and Frumau,  Arnoud and Galkowski,  Michal and Gatti,  Luciana V. and Gloor,  Emanuel and Griffis,  Tim and Hammer,  Samuel and Haszpra,  László and Hatakka,  Juha and Heliasz,  Michal and Hensen,  Arjan and Hermanssen,  Ove and Hintsa,  Eric and Holst,  Jutta and Jaffe,  Dan and Karion,  Anna and Kawa,  Stephan Randolph and Keeling,  Ralph and Keronen,  Petri and Kolari,  Pasi and Kominkova,  Katerina and Kort,  Eric and Krummel,  Paul and Kubistin,  Dagmar and Labuschagne,  Casper and Langenfelds,  Ray and Laurent,  Olivier and Laurila,  Tuomas and Lauvaux,  Thomas and Law,  Bev and Lee,  John and Lehner,  Irene and Leuenberger,  Markus and Levin,  Ingeborg and Levula,  Janne and Lin,  John and Lindauer,  Matthias and Loh,  Zoe and Lopez,  Morgan and Myhre,  Cathrine Lund and Machida,  Toshinobu and Mammarella,  Ivan and Manca,  Giovanni and Manning,  Alistair and Manning,  Andrew and Marek,  Michal V. and Marklund,  Per and Martin,  Melissa Yang and Matsueda,  Hidekazu and McKain,  Kathryn and Meijer,  Harro and Meinhardt,  Frank and Miles,  Natasha and Miller,  Charles E. and M\"{o}lder,  Meelis and Montzka,  Stephen and Moore,  Fred and {Josep-Anton Morgui} and Morimoto,  Shinji and Munger,  Bill and {Jaroslaw Necki} and Newman,  Sally and Nichol,  Sylvia and Niwa,  Yosuke and O'Doherty,  Simon and {Mikaell Ottosson-L\"{o}fvenius} and Paplawsky,  Bill and Peischl,  Jeff and Peltola,  Olli and {Jean-Marc Pichon} and Piper,  Steve and Plass-D\"{o}lmer,  Christian and Ramonet,  Michel and Reyes-Sanchez,  Enrique and Richardson,  Scott and Riris,  Haris and Ryerson,  Thomas and Saito,  Kazuyuki and Sargent,  Maryann and Sawa,  Yousuke and Say,  Daniel and Scheeren,  Bert and Schmidt,  Martina and Schmidt,  Andres and Schumacher,  Marcus and Shepson,  Paul and Shook,  Michael and Stanley,  Kieran and Steinbacher,  Martin and Stephens,  Britton and Sweeney,  Colm and Thoning,  Kirk and Torn,  Margaret and Turnbull,  Jocelyn and Tørseth,  Kjetil and Bulk,  Pim Van Den and Laan-Luijkx,  Ingrid T. Van Der and Dinther,  Danielle Van and Vermeulen,  Alex and Viner,  Brian and Vitkova,  Gabriela and Walker,  Stephen and Weyrauch,  Dietmar and Wofsy,  Steve and Worthy,  Doug and {Dickon Young} and {Miroslaw Zimnoch}},
+  title = {CarbonTracker CT2019},
+  publisher = {NOAA Earth System Research Laboratory,  Global Monitoring Division},
+  year = {2020}
+}
+