#!/usr/bin/env python """ This script has been built to alter the cmor tables, in order to allow non-standard conversions """ import argparse import getpass import json import sys import os from collections import OrderedDict def add_vartab_entries(varfile, cccmatabdir, cmortabdir="cmip6-cmor-tables/Tables", source_id="CanESM5"): """Take in the variable information from the specified vartab, and add it to the relevant cccma variable tables, along with the cmor variable table Parameters ---------- varfile : string (required) the json file containing the user's variable information cccmatabdir : string (required) the path to the CCCma variable table directory, with subdirectories for each supported source ID cmortabdir : string (optional) the path to the cmor table directory. Defaults to "cmip6-cmor-tables/Tables" source_id : string (optional) specifies what set of cccma variable tables to modify. Defaults to "CanESM5" """ # define required/optional fields in the user enty #-- fields that the user has to define required_cccma_user_fields = [ "CCCma TS var name", "CCCma diag file"] required_cmor_user_fields = [ "frequency", "modeling_realm", "standard_name", "units", "cell_methods", "cell_measures", "comment", "dimensions", "out_name", "type", "positive", "valid_min", "valid_max", "ok_min_mean_abs", "ok_max_mean_abs"] #-- fields that the user can supply, but if not, defaults will be set optional_cccma_user_fields = [ ("CCCma check", getpass.getuser()), ("CCCma optional deck", None), ("CCCma comment", None), ("CCCma NetCDF comment", None), ("CCCma target vertical grid", None), ("CCCma target horizontal grid", None), ("Include only for", None), ("Exclude for", None), ("priority", 0)] # load user file if it exists if not os.path.isfile(varfile): raise Exception("alter_cmor_tabs-vartab : unable to locate {}".format(varfile)) else: try: with open(varfile,'r') as f: user_vardict = json.load(f,object_pairs_hook=OrderedDict) except Exception: raise Exception("Failed to parse {}! Does it have correct json syntax?".format(varfile)) # confirm that the given cmor/cccma table directories exist if not os.path.isdir(cccmatabdir): raise Exception("unable to locate the given CCCma variable table directory!\n{}".format(cccmatabdir)) if not os.path.isdir(cmortabdir): raise Exception("unable to locate the given CMOR variable table directory!\n{}".format(cmortabdir)) # confirm that the given source id actually has a sub directory in the CCCma directory source_vartab_dir = os.path.join(cccmatabdir,source_id) if not os.path.isdir(source_vartab_dir): excep_str = "unable to find a '{}' dir in '{}!".format(source_id,cccmatabdir) excep_str += " Is source_id set correctly?" raise Exception(excep_str) # loop over all variable entries in the user file, and add them to the necessary variable files for var in user_vardict: # extract the desired cmor table, and delete the key if not "CMOR table" in user_vardict[var]: excep_str = "CMOR table not found in {} entry in {}".format(var,varfile) raise Exception(excep_str) CMOR_table = user_vardict[var]["CMOR table"] del user_vardict[var]["CMOR table"] # confirm that the required fields are present for field in required_cccma_user_fields: if not field in user_vardict[var]: raise Exception("unable to find the required cccma field '{}' in the user variable entry for '{}'".format(field,var)) for field in required_cmor_user_fields: if not field in user_vardict[var]: raise Exception("unable to find the required cmor field '{}' in the user variable entry for '{}'".format(field,var)) # create a separate dictionary for both the cccma and cmor tables cccma_var_entry = user_vardict[var].copy() cmor_var_entry = user_vardict[var].copy() # if necessary, add cccma specific fields to the cccma_vardict, and delete from cmor_vardict for field in required_cccma_user_fields: del cmor_var_entry[field] for field,default in optional_cccma_user_fields: if field in user_vardict[var]: # field is present -> delete from cmor dictionary del cmor_var_entry[field] else: # field isn't present -> add to cccma dictionary cccma_var_entry[field] = default # add 'CMOR Name' to cccma entry to be consistent with how our var tables are written cccma_var_entry['CMOR Name'] = var # open the cccma/cmor variable tables cccma_vartab_file = os.path.join(source_vartab_dir,"CMIP6_{}.json".format(CMOR_table)) cmor_vartab_file = os.path.join(cmortabdir,"CMIP6_{}.json".format(CMOR_table)) if not os.path.isfile(cccma_vartab_file): raise Exception("unable to find {}!".format(cccma_vartab_file)) if not os.path.isfile(cmor_vartab_file): raise Exception("unable to find {}!".format(cmor_vartab_file)) with open(cccma_vartab_file,'r') as f: cccma_vartab = json.load(f,object_pairs_hook=OrderedDict) with open(cmor_vartab_file,'r') as f: cmor_vartab = json.load(f,object_pairs_hook=OrderedDict) # add entries to the relevant table cccma_vartab[var] = cccma_var_entry cmor_vartab['variable_entry'][var] = cmor_var_entry # write the new variable table files # - Note the different indent levels are required to match how the files were originally written with open(cccma_vartab_file,'w') as f: json.dump(cccma_vartab,f,indent=2,separators=(', ',': ')) with open(cmor_vartab_file,'w') as f: json.dump(cmor_vartab,f,indent=4,separators=(', ',': ')) def add_CV_entries(field_type, source_file, cv_file="cmip6-cmor-tables/Tables/CMIP6_CV.json"): """Takes in the provided json file and attempts to add the contained entries into the cmor CVs Parameters ---------- field_type : string (required) what type of entires are contained in the source file. One of "source_id", "activity_id", "experiment_id" or "sub_experiment_id" source_file : string (required) the file containing the entries to add cv_file : str (optional) the CV file to modify - defaults to "cmip6-cmor-tables/Tables/CMIP6_CV.json" """ # open/load the CV if not os.path.isfile(cv_file): raise Exception("unable to find controlled vocabulary file : {}".format(cv_file)) with open(cv_file,'r') as f: CV_dict = json.load(f,object_pairs_hook=OrderedDict) # check that the file exists if not os.path.isfile(source_file): raise Exception("unable to find the specified json file : {}".format(source)) # open file and load data into dictionary try: with open(source_file,'r') as f: user_entries = json.load(f) except Exception: raise Exception("Failed to parse {}! Does it have correct json syntax?".format(source_file)) # loop over all the entries and add them to the CV for entry in user_entries: # add entry to CV and write new file CV_dict['CV'][field_type][entry] = user_entries[entry] # write updated CV with open(cv_file,'w') as f: json.dump(CV_dict,f,indent=4,separators=(',',':')) def copy_CV_entry(new_entry, entry2copy, field_type, cv_file="cmip6-cmor-tables/Tables/CMIP6_CV.json"): """Copy an existing entry in the cmor CVs with a new key Parameters ---------- new_entry : string name to use for the new key in the CV entry2copy : string name of the entry to copy field_type : type of field to copy in the CV. One of "source_id", "activity_id", "experiment_id" or "sub_experiment_id" cv_file : str (optional) the CV file to modify - defaults to "cmip6-cmor-tables/Tables/CMIP6_CV.json" """ # open/load the CV if not os.path.isfile(cv_file): raise Exception("unable to find controlled vocabulary file : {}".format(cv_file)) with open(cv_file,'r') as f: CV_dict = json.load(f,object_pairs_hook=OrderedDict) # initialize the user entry by copying the original try: # assumes its a dictionary, so we must explicity copy user_entry = CV_dict['CV'][field_type][entry2copy].copy() except AttributeError: # the entry isn't a dictionary, no need to copy user_entry = CV_dict['CV'][field_type][entry2copy] except KeyError: raise Exception("the specified entry to copy ({} => {}) does not exist!".format(field_type,entry2copy)) # update the nested entry for 'field_type' if its present if field_type in user_entry: user_entry[field_type] = new_entry # add entry to CV and write to file CV_dict['CV'][field_type][new_entry] = user_entry with open(cv_file,'w') as f: json.dump(CV_dict,f,indent=4,separators=(',',':')) def process_args(args): """Translate given arguments into dictionary and check consistency Parameters ---------- args : argparse.Namespace object Returns ------- arg_dict : dictionary dictionary containing the processed argument list """ # initiate dict arg_dict = {} if args.target == "add2cv": # determine the field type if args.source_id: arg_dict['field_type'] = "source_id" elif args.activity_id: arg_dict['field_type'] = "activity_id" elif args.experiment_id: arg_dict['field_type'] = "experiment_id" elif args.subexperiment_id: arg_dict['field_type'] = "sub_experiment_id" # set source file arg_dict['source_file'] = args.jsonfile # set cv file if given if args.cv_file: arg_dict['cv_file'] = args.cv_file elif args.target == "add2vartabs": arg_dict['varfile'] = args.varfile arg_dict['source_id'] = args.source_id arg_dict['cmortabdir'] = args.cmortabdir arg_dict['cccmatabdir'] = args.cccmatabdir elif args.target == "cpCVent": arg_dict['new_entry'] = args.new_entry arg_dict['entry2copy'] = args.entry2copy # determine field type if args.source_id: arg_dict['field_type'] = "source_id" elif args.activity_id: arg_dict['field_type'] = "activity_id" elif args.experiment_id: arg_dict['field_type'] = "experiment_id" elif args.subexperiment_id: arg_dict['field_type'] = "sub_experiment_id" # set cv file if given if args.cv_file: arg_dict['cv_file'] = args.cv_file return arg_dict if __name__ == "__main__": #=============== # define parsers #=============== prog_desc = "Alter cmor tables" parser = argparse.ArgumentParser(prog = "alter_cmor_tabs", description = prog_desc) subparsers = parser.add_subparsers() #--------------------- # Add to CV subcommand add2cv_parser = subparsers.add_parser('add2cv',help="Add entries to the controlled vocabulary (CV)") add2cv_parser.set_defaults(target="add2cv") # the json file containing the desired entries add2cv_parser.add_argument("jsonfile", metavar="JSONFILE", type=str, help="The json file containing the desired entries") # flag to define what type of entries will be added add2cv_group = add2cv_parser.add_mutually_exclusive_group(required=True) add2cv_group.add_argument("--src", dest="source_id", action="store_true", help="Add Source IDs (i.e. CanESM5, CanESM5-CanOE) from the given file") add2cv_group.add_argument("--act", dest="activity_id", action="store_true", help="Add Activity IDs (i.e. CMIP, AMIP) from the given file") add2cv_group.add_argument("--exp", dest="experiment_id", action="store_true", help="Add Experiment IDs (i.e. historical, piControl) from the given file") add2cv_group.add_argument("--subexp", dest="subexperiment_id", action="store_true", help="Add Subexperiment IDs from the given file") # (optional) path to CV add2cv_parser.add_argument("--CV", metavar="CV FILE", type=str, dest="cv_file", default="cmip6-cmor-tables/Tables/CMIP6_CV.json", help=("File containing the controlled vocabulary. ")+ ("defaults to cmip6-cmor-tables/Tables/CMIP6_CV.json")) #-------------------------- # Copy CV entry subcommand cpCVent_parser = subparsers.add_parser('cpCVent',help="Copy existing CV entry with new key") cpCVent_parser.set_defaults(target="cpCVent") # the name of the new, desired entry cpCVent_parser.add_argument("new_entry",metavar="NEW_ENTRY",type=str, help="The name of the new entry") # the name of the entry to be copied cpCVent_parser.add_argument("entry2copy",metavar="ENTRY_TO_COPY",type=str, help="The name of the existing entry, to be copied") # flag to define what type of entry it is cpCVent_group = cpCVent_parser.add_mutually_exclusive_group(required=True) cpCVent_group.add_argument("-s","--src", dest="source_id", action="store_true", help="Copy a Source ID (i.e. CanESM5, CanESM5-CanOE)") cpCVent_group.add_argument("-a","--act", dest="activity_id", action="store_true", help="Copy an Activity ID (i.e. CMIP, AMIP)") cpCVent_group.add_argument("-e","--exp", dest="experiment_id", action="store_true", help="Copy Experiment ID (i.e. historical, piControl)") cpCVent_group.add_argument("-u","--subexp", dest="subexperiment_id", action="store_true", help="Copy Subexperiment ID") # (optional) path to CV cpCVent_parser.add_argument("--CV", metavar="CV FILE", type=str, dest="cv_file", default="cmip6-cmor-tables/Tables/CMIP6_CV.json", help=("File containing the controlled vocabulary. ")+ ("defaults to cmip6-cmor-tables/Tables/CMIP6_CV.json")) #---------------------------------- # Add to variable tables subcommand add2vartabs_parser = subparsers.add_parser('add2vartabs', help="Add variable information to the cmor variable tables, and equivalent CCCma tables") add2vartabs_parser.set_defaults(target="add2vartabs") # required args add2vartabs_parser.add_argument("varfile", type=str, help="json file containing the additional variable info") add2vartabs_parser.add_argument("source_id", type=str, help="relevant source ID, which is used to determine the relevant CCCma variable tables") # optional args that define where the CMOR/CCCma tables are contained add2vartabs_parser.add_argument("-C", "--cmortabdir", metavar="CMOR TAB DIR", type=str, dest="cmortabdir", default="cmip6-cmor-tables/Tables", help="path to the cmor table directory. Defaults to 'cmip6-cmor-tables/Tables'") add2vartabs_parser.add_argument("-c", "--cccmatabdir", metavar="CCCMA TAB DIR", type=str, dest="cccmatabdir", required=True, help="path to the cccma table directory") #================= # ingest arguments #================= args = parser.parse_args() processed_args = process_args(args) #============= # alter tables #============= if args.target == "add2cv": add_CV_entries(**processed_args) elif args.target == "cpCVent": copy_CV_entry(**processed_args) elif args.target == "add2vartabs": add_vartab_entries(**processed_args)