Updates needed to get more definitions to work and to meet the specif…

…ications.

scripts/iconform

@@ -16,6 +16,9 @@ import datetime
 from dreqPy import dreq
 import uuid
 
+
+version = 'v'+str(datetime.datetime.now().year)+str(datetime.datetime.now().month).zfill(2)+str(datetime.datetime.now().day).zfill(2)
+
 # Map netcdf types to python types
 #data_types = {'char': 'char', 'byte': 'int8', 'short': 'int16', 'int': 'int32',
 #              'float': 'float32', 'real': 'float32', 'double': 'float64', 
@@ -152,7 +155,7 @@ def fill_missing_glob_attributes(attr, table, v, grids):
             if "parent_source_id" in attr.keys():
                 attr["parent_source_id"] = attr["source_id"]
             if "parent_time_units" in attr.keys():
-                attr["parent_time_units"] = "days since 0000-01-01 00:00:00"
+                attr["parent_time_units"] = "days since 0001-01-01 00:00:00"
     else:
         if "branch_time_in_child" in attr.keys():
             attr["branch_time_in_child"] = "none"
@@ -167,13 +170,13 @@ def fill_missing_glob_attributes(attr, table, v, grids):
 
     if "variant_label" in attr.keys():
         pre = attr["variant_label"].split('r')[1]
-        attr["realization_index"] = int(pre.split('i')[0])
+        attr["realization_index"] = (pre.split('i')[0])
         pre = pre.split('i')[1]
-        attr["initialization_index"] = int(pre.split('p')[0])
+        attr["initialization_index"] = (pre.split('p')[0])
         pre = pre.split('p')[1]
-        attr["physics_index"] = int(pre.split('f')[0])
+        attr["physics_index"] = (pre.split('f')[0])
         pre = int(pre.split('f')[1])
-        attr["forcing_index"] = pre
+        attr["forcing_index"] = str(pre)
 
     if "further_info_url" in attr.keys():
         if "__FILL__" in attr["further_info_url"]:       
@@ -260,7 +263,7 @@ def defineVar(v, varName, attr, table_info, definition, ig, experiment, out_dir)
         source_id = attributes['source_id']
     else:
         source_id = ''
-    if 'grid_labels' in attributes.keys():
+    if 'grid_label' in attributes.keys():
         grid = attributes['grid_label']
     else:
         grid = ''
@@ -298,11 +301,10 @@ def defineVar(v, varName, attr, table_info, definition, ig, experiment, out_dir)
         dst = date_strings[v["frequency"]]
     else:
         dst = ''
-    f_name = ("{0}/{1}/{2}/{3}/{4}/{5}/{6}/{7}/{8}/{9}/{10}_{11}_{12}_{13}_{14}_{15}{16}.nc".format(
+    f_name = ("{0}/{1}/{2}/{3}/{4}/{5}/{6}/{7}/{8}/{9}/{10}/{11}_{12}_{13}_{14}_{15}_{16}_{17}.nc".format(
                out_dir, mip_era, activity_id, institution_id, source_id, experiment, ripf, mipTable,
-               varName, grid, 
+               varName, grid, version, 
                varName, mipTable, source_id, experiment, ripf, grid, dst))
-
     var = {}
 
     # put together the dictionary entry for this variable    
@@ -364,7 +366,7 @@ def defineAxes(v, name):
             v2.pop(key,None)
     # Hardcode this value in for time.  Not ideal, but the request has it listed as "days since ?" and this will fail.
     if 'time' in name:
-        v2["units"] = "days since 0000-01-01 00:00:00"
+        v2["units"] = "days since 0001-01-01 00:00:00"
 
     # put everything into a variable dictionary
     var["attributes"] = v2

scripts/vardeps

@@ -59,7 +59,7 @@ def variable_search(obj, vars=None):
 #=========================================================================
 # print_columnar
 #=========================================================================
-def print_columnar(x, textwidth=100, indent=0, header=''):
+def print_columnar(x, textwidth=10000000, indent=0, header=''):
     hrstrp = '{}  '.format(str(header).rstrip())
     if len(hrstrp) > indent:
         indent = len(hrstrp)

source/pyconform/functions.py

@@ -9,6 +9,7 @@
 from pyconform.physarray import PhysArray, UnitsError
 from numpy.ma import sqrt, where
 from cf_units import Unit
+import numpy as np
 
 #=======================================================================================================================
 # is_constant - Determine if an argument is a constant (number or string)
@@ -309,6 +310,91 @@ def __getitem__(self, index):
         return data.mean(dimensions=indims)
 
 
+#===================================================================================================
+# SumFunction
+#===================================================================================================
+class SumFunction(Function):
+    key = 'sum'
+
+    def __init__(self, data, *dimensions):
+        super(SumFunction, self).__init__(data, *dimensions)
+        self.add_sumlike_dimensions(*dimensions)
+        data_info = data if is_constant(data) else data[None]
+        if not isinstance(data_info, PhysArray):
+            raise TypeError('sum: Data must be a PhysArray')
+        if not all(isinstance(d, basestring) for d in dimensions):
+            raise TypeError('sum: Dimensions must be strings')
+
+    def __getitem__(self, index):
+        data = self.arguments[0][index]
+        dimensions = self.arguments[1:]
+        indims = []
+        for d in dimensions:
+             print d,'in',data.dimensions,'?'
+             if d in data.dimensions:
+                 print 'will append ',data.dimensions.index(d)
+                 indims.append(data.dimensions.index(d))
+        return np.sum(data, indims[0])
+
+#===================================================================================================
+# MinFunction
+#===================================================================================================
+class MinFunction(Function):
+    key = 'min'
+
+    def __init__(self, data, *dimensions):
+        super(MinFunction, self).__init__(data, *dimensions)
+        self.add_sumlike_dimensions(*dimensions)
+        data_info = data if is_constant(data) else data[None]
+        if not isinstance(data_info, PhysArray):
+            raise TypeError('min: Data must be a PhysArray')
+        if not all(isinstance(d, basestring) for d in dimensions):
+            raise TypeError('min: Dimensions must be strings')
+
+    def __getitem__(self, index):
+        data = self.arguments[0][index]
+        dimensions = self.arguments[1:]
+        indims = []
+        if index is None:
+            return PhysArray(np.zeros((0,0,0)), dimensions=[data.dimensions[0],data.dimensions[2],data.dimensions[3]])
+        for d in dimensions:
+             if d in data.dimensions:
+                 indims.append(data.dimensions.index(d))
+        new_name='min({},{})'.format(data.name,dimensions)
+        m = np.amin(data, axis=indims[0])
+        return PhysArray(m, name=new_name, positive=data.positive, units=data.units, dimensions=[data.dimensions[0],data.dimensions[2],data.dimensions[3]])
+
+
+#===================================================================================================
+# MaxFunction
+#===================================================================================================
+class MaxFunction(Function):
+    key = 'max'
+
+    def __init__(self, data, *dimensions):
+        super(MaxFunction, self).__init__(data, *dimensions)
+        self.add_sumlike_dimensions(*dimensions)
+        data_info = data if is_constant(data) else data[None]
+        if not isinstance(data_info, PhysArray):
+            raise TypeError('max: Data must be a PhysArray')
+        if not all(isinstance(d, basestring) for d in dimensions):
+            raise TypeError('max: Dimensions must be strings')
+
+    def __getitem__(self, index):
+        data = self.arguments[0][index]
+        dimensions = self.arguments[1:]
+        indims = []
+        if index is None:
+            return PhysArray(np.zeros((0,0,0)), units=data.units, dimensions=[data.dimensions[0],data.dimensions[2],data.dimensions[3]])
+        for d in dimensions:
+             if d in data.dimensions:
+                 indims.append(data.dimensions.index(d))
+        new_name='max({},{})'.format(data.name,dimensions[0])
+        m = np.amax(data, axis=indims[0])
+        return PhysArray(m, name=new_name, positive=data.positive, units=data.units, dimensions=[data.dimensions[0],data.dimensions[2],data.dimensions[3]])
+
+
+
 #===================================================================================================
 # PositiveUpFunction
 #===================================================================================================

source/pyconform/miptableparser.py

@@ -418,7 +418,7 @@ def parse_table(self,exp,mips,tables,v_list,table_var_fields,table_axes_fields,t
                     if hasattr(c_var,'mipTable'):
                         var['mipTable']=c_var.mipTable
                         if c_var.mipTable in tables or '--ALL--' in tables: 
-                            var["_FillValue"] = "9.96921e+36"
+                            var["_FillValue"] = "1e+20"
                             if hasattr(c_var,'deflate'):
                                 var['deflate']= c_var.deflate
                             if hasattr(c_var,'deflate_level'):

source/pyconform/modules/CLM_landunit_to_CMIP6_Lut.py

@@ -21,20 +21,38 @@ def __init__(self, EFLX_LH_TOT, ntim, nlat, nlon, grid1d_ixy, grid1d_jxy, grid1d
 
     def __getitem__(self, index):
 
-        EFLX_LH_TOT = self.arguments[0] if is_constant(self.arguments[0]) else self.arguments[0][index]
-        ntim = self.arguments[1] if is_constant(self.arguments[1]) else self.arguments[1][index]
-        nlat = self.arguments[2] if is_constant(self.arguments[2]) else self.arguments[2][index]
-        nlon = self.arguments[3] if is_constant(self.arguments[3]) else self.arguments[3][index]
-        grid1d_ixy = self.arguments[4] if is_constant(self.arguments[4]) else self.arguments[4][index]
-        grid1d_jxy  = self.arguments[5] if is_constant(self.arguments[5]) else self.arguments[5][index]
-        grid1d_lon = self.arguments[6] if is_constant(self.arguments[6]) else self.arguments[6][index]
-        grid1d_lat = self.arguments[7] if is_constant(self.arguments[7]) else self.arguments[7][index]
-        land1d_lon = self.arguments[8] if is_constant(self.arguments[8]) else self.arguments[8][index]
-        land1d_lat = self.arguments[9] if is_constant(self.arguments[9]) else self.arguments[9][index]
-        land1d_ityplunit = self.arguments[10] if is_constant(self.arguments[10]) else self.arguments[10][index]
-        land1d_active = self.arguments[11] if is_constant(self.arguments[11]) else self.arguments[11][index]
-        land1d_wtgcell = self.arguments[12] if is_constant(self.arguments[12]) else self.arguments[12][index]
-
+        pEFLX_LH_TOT = self.arguments[0][index]
+        pntim = self.arguments[1][index]
+        pnlat = self.arguments[2][index]
+        pnlon = self.arguments[3][index]
+        pgrid1d_ixy = self.arguments[4][index]
+        pgrid1d_jxy  = self.arguments[5][index]
+        pgrid1d_lon = self.arguments[6][index]
+        pgrid1d_lat = self.arguments[7][index]
+        pland1d_lon = self.arguments[8][index]
+        pland1d_lat = self.arguments[9][index]
+        pland1d_ityplunit = self.arguments[10][index]
+        pland1d_active = self.arguments[11][index]
+        pland1d_wtgcell = self.arguments[12][index]
+
+        if index is None:
+            return PhysArray(np.zeros((0,0,0)), dimensions=[pntim.dimensions[0],pnlat.dimensions[0],pnlon.dimensions[0]])
+
+        EFLX_LH_TOT = pEFLX_LH_TOT.data 
+        ntim = pntim.data
+        nlat = pnlat.data
+        nlon = pnlon.data
+        grid1d_ixy = pgrid1d_ixy.data
+        grid1d_jxy  = pgrid1d_jxy.data
+        grid1d_lon = pgrid1d_lon.data
+        grid1d_lat = pgrid1d_lat.data
+        land1d_lon = pland1d_lon.data
+        land1d_lat = pland1d_lat.data
+        land1d_ityplunit = pland1d_ityplunit.data
+        land1d_active = pland1d_active.data
+        land1d_wtgcell = pland1d_wtgcell.data
+
+        missing = 1e+20
 
 	long_name = "latent heat flux on land use tile (lut=0:natveg, =1:crop, =2:pasture, =3:urban)"
 	nlut    = 4
@@ -47,9 +65,10 @@ def __getitem__(self, index):
 	eps = 1.e-5
 
 	# Will contain landunit variables for veg, crop, pasture, and urban on 2d grid
-	varo_lut = np.full([len(ntim),4,len(nlat),len(nlon)],fill_value=1.e36)
+	varo_lut_temp = np.full([len(ntim),4,len(nlat),len(nlon)],fill_value=missing)
+        varo_lut = np.ma.masked_values(varo_lut_temp, missing)
 	# Set pasture to fill value
-	varo_lut[:,pasture,:,:] = 1.e36
+	varo_lut[:,pasture,:,:] = missing
 
 	# If 1, landunit is active
 	active_lunit = 1
@@ -104,13 +123,13 @@ def __getitem__(self, index):
 		    if landunit_indx_veg.size > 0:
 			varo_lut[:,veg,jxy,ixy] = EFLX_LH_TOT[:,landunit_indx_veg].squeeze()
 		    else:
-			varo_lut[:,veg,jxy,ixy] = 1.e36
+			varo_lut[:,veg,jxy,ixy] = missing
 
 		    # Check for valid crop landunit
 		    if landunit_indx_crop.size > 0:
 			varo_lut[:,crop,jxy,ixy] = EFLX_LH_TOT[:,landunit_indx_crop].squeeze()
 		    else:
-			varo_lut[:,crop,jxy,ixy] = 1.e36
+			varo_lut[:,crop,jxy,ixy] = missing
 
 		    # Check for valid urban landunit and compute weighted-average
 		    if landunit_indx_urban.size > 0:
@@ -122,12 +141,18 @@ def __getitem__(self, index):
 			    sys.exit(-1)
 			varo_lut[:,urban,jxy,ixy] = np.sum(dum * weights)
 		    else:
-			varo_lut[:,urban,jxy,ixy] = 1.e36
+			varo_lut[:,urban,jxy,ixy] = missing
+
+        mvaro_lut = np.ma.mean(varo_lut, axis=1)
+
+        new_name = 'CLM_landunit_to_CMIP6_Lut({}{}{}{}{}{}{}{}{}{}{}{}{})'.format(pEFLX_LH_TOT.name, 
+                              pntim.name, pnlat.name, pnlon.name, pgrid1d_ixy.name, pgrid1d_jxy.name, pgrid1d_lon.name,
+                              pgrid1d_lat.name, pland1d_lon.name, pland1d_lat.name, pland1d_ityplunit.name,
+                              pland1d_active.name, pland1d_wtgcell.name) 
+
+        #mvaro_lut[mvaro_lut>=1e+16] = 1e+20
+        #ma_mvaro_lut = np.ma.masked_values(mvaro_lut, 1e+20) 
 
-        new_name = 'CLM_landunit_to_CMIP6_Lut({}{}{}{}{}{}{}{}{}{}{}{}{})'.format(EFLX_LH_TOT.name, 
-                              ntim.name, nlat.name, nlon.name, grid1d_ixy.name, grid1d_jxy.name, grid1d_lon.name,
-                              grid1d_lat.name, land1d_lon.name, land1d_lat.name, land1d_ityplunit.name,
-                              land1d_active.name, land1d_wtgcell.name) 
-	return PhysArray(varo_lut,  name=new_name)
+	return PhysArray(mvaro_lut,  name=new_name, units=pEFLX_LH_TOT.units)