"""Class for generating component namelists."""
# Typically ignore this.
# pylint: disable=invalid-name
# Disable these because this is our standard setup
# pylint: disable=wildcard-import,unused-wildcard-import
import datetime
import re
import hashlib
from CIME.XML.standard_module_setup import *
from CIME.namelist import Namelist, parse, \
character_literal_to_string, string_to_character_literal, \
expand_literal_list, compress_literal_list, merge_literal_lists
from CIME.XML.namelist_definition import NamelistDefinition
from CIME.utils import expect, safe_copy
from CIME.XML.stream import Stream
logger = logging.getLogger(__name__)
_var_ref_re = re.compile(r"\$(\{)?(?P<name>\w+)(?(1)\})")
_ymd_re = re.compile(r"%(?P<digits>[1-9][0-9]*)?y(?P<month>m(?P<day>d)?)?")
_stream_file_template = """<?xml version="1.0"?>
<file id="stream" version="1.0">
<dataSource>
GENERIC
</dataSource>
<domainInfo>
<variableNames>
{domain_varnames}
</variableNames>
<filePath>
{domain_filepath}
</filePath>
<fileNames>
{domain_filenames}
</fileNames>
</domainInfo>
<fieldInfo>
<variableNames>
{data_varnames}
</variableNames>
<filePath>
{data_filepath}
</filePath>
<fileNames>
{data_filenames}
</fileNames>
<offset>
{offset}
</offset>
</fieldInfo>
</file>
"""
[docs]class NamelistGenerator(object):
"""Utility class for generating namelists for a given component."""
_streams_variables = []
#pylint:disable=too-many-arguments
def __init__(self, case, definition_files, files=None):
"""Construct a namelist generator.
Arguments:
`case` - `Case` object corresponding to the current case.
`infiles` - List of files with user namelist options.
`definition_files` - List of XML files containing namelist definitions.
`config` - A dictionary of attributes for matching defaults.
"""
# Save off important information from inputs.
self._case = case
self._din_loc_root = case.get_value('DIN_LOC_ROOT')
# Create definition object - this will validate the xml schema in the definition file
self._definition = NamelistDefinition(definition_files[0], files=files)
# Determine array of _stream_variables from definition object
# This is only applicable to data models
self._streams_namelists = {"streams": []}
self._streams_variables = self._definition.get_per_stream_entries()
for variable in self._streams_variables:
self._streams_namelists[variable] = []
# Create namelist object.
self._namelist = Namelist()
# Define __enter__ and __exit__ so that we can use this as a context manager
def __enter__(self):
return self
def __exit__(self, *_):
return False
[docs] def init_defaults(self, infiles, config, skip_groups=None, skip_entry_loop=None ):
"""Return array of names of all definition nodes
"""
# first clean out any settings left over from previous calls
self.new_instance()
self._definition.set_nodes(skip_groups=skip_groups)
# Determine the array of entry nodes that will be acted upon
entry_nodes = self._definition.set_nodes(skip_groups=skip_groups)
# Add attributes to definition object
self._definition.add_attributes(config)
# Parse the infile and create namelist settings for the contents of infile
# this will override all other settings in add_defaults
for file_ in infiles:
# Parse settings in "groupless" mode.
nml_dict = parse(in_file=file_, groupless=True)
# Add groups using the namelist definition.
new_namelist = self._definition.dict_to_namelist(nml_dict, filename=file_)
# Make sure that the input is actually valid.
self._definition.validate(new_namelist, filename=file_)
# Merge into existing settings (earlier settings have precedence
# over later settings).
self._namelist.merge_nl(new_namelist)
if not skip_entry_loop:
for entry in entry_nodes:
self.add_default(self._definition.get(entry, "id"))
else:
return [self._definition.get(entry, "id") for entry in entry_nodes]
[docs] @staticmethod
def quote_string(string):
"""Convert a string to a quoted Fortran literal.
Does nothing if the string appears to be quoted already.
"""
if string == "" or \
(string[0] not in ('"', "'") or string[0] != string[-1]):
string = string_to_character_literal(string)
return string
def _to_python_value(self, name, literals):
"""Transform a literal list as needed for `get_value`."""
var_type, _, var_size, = self._definition.split_type_string(name)
if len(literals) > 0:
value = expand_literal_list(literals)
else:
value = ''
return value
for i, scalar in enumerate(value):
if scalar == '':
value[i] = None
elif var_type == 'character':
value[i] = character_literal_to_string(scalar)
if var_size == 1:
return value[0]
else:
return value
def _to_namelist_literals(self, name, value):
"""Transform a literal list as needed for `set_value`.
This is the inverse of `_to_python_value`, except that many of the
changes have potentially already been performed.
"""
var_type, _, var_size, = self._definition.split_type_string(name)
if var_size == 1 and not isinstance(value, list):
value = [value]
for i, scalar in enumerate(value):
if scalar is None:
value[i] = ""
elif var_type == 'character':
expect(not isinstance(scalar, list), name)
value[i] = self.quote_string(scalar)
return compress_literal_list(value)
[docs] def get_value(self, name):
"""Get the current value of a given namelist variable.
Note that the return value of this function is always a string or a list
of strings. E.g. the scalar logical value .false. will be returned as
`".false."`, while an array of two .false. values will be returned as
`[".false.", ".false."]`. Whether or not a value is scalar is determined
by checking the array size in the namelist definition file.
Null values are converted to `None`, and repeated values are expanded,
e.g. `['2*3']` is converted to `['3', '3', '3']`.
For character variables, the value is converted to a Python string (e.g.
quotation marks are removed).
All other literals are returned as the raw string values that will be
written to the namelist.
"""
return self._to_python_value(name, self._namelist.get_value(name))
[docs] def set_value(self, name, value):
"""Set the current value of a given namelist variable.
Usually, you should use `add_default` instead of this function.
The `name` argument is the name of the variable to set, and the `value`
is a list of strings to use as settings. If the variable is scalar, the
list is optional; i.e. a scalar logical can be set using either
`value='.false.'` or `value=['.false.']`. If the variable is of type
character, and the input is missing quotes, quotes will be added
automatically. If `None` is provided in place of a string, this will be
translated to a null value.
Note that this function will overwrite the current value, which may hold
a user-specified setting. Even if `value` is (or contains) a null value,
the old setting for the variable will be thrown out completely.
"""
var_group = self._definition.get_group(name)
literals = self._to_namelist_literals(name, value)
_, _, var_size, = self._definition.split_type_string(name)
self._namelist.set_variable_value(var_group, name, literals, var_size)
[docs] def get_default(self, name, config=None, allow_none=False):
"""Get the value of a variable from the namelist definition file.
The `config` argument is passed through to the underlying
`NamelistDefaults.get_value` call as the `attribute` argument.
The return value of this function is a list of values that were found in
the defaults file. If there is no matching default, this function
returns `None` if `allow_none=True` is passed, otherwise an error is
raised.
Note that we perform some translation of the values, since there are a
few differences between Fortran namelist literals and values in the
defaults file:
1) In the defaults file, whitespace is ignored except within strings, so
the output of this function strips out most whitespace. (This implies
that commas are the only way to separate array elements in the
defaults file.)
2) In the defaults file, quotes around character literals (strings) are
optional, as long as the literal does not contain whitespace, commas,
or (single or double) quotes. If a setting for a character variable
does not seem to have quotes (and is not a null value), this function
will add them.
3) Default values may refer to variables in a case's `env_*.xml` files.
This function replaces references of the form `$VAR` or `${VAR}` with
the value of the variable `VAR` in an env file, if that variable
exists. This behavior is suppressed within single-quoted strings
(similar to parameter expansion in shell scripts).
"""
default = self._definition.get_value_match(name, attributes=config, exact_match=False)
if default is None:
expect(allow_none, "No default value found for {}.".format(name))
return None
default = expand_literal_list(default)
var_type,_,_ = self._definition.split_type_string(name)
for i, scalar in enumerate(default):
# Skip single-quoted strings.
if var_type == 'character' and scalar != '' and \
scalar[0] == scalar[-1] == "'":
continue
match = _var_ref_re.search(scalar)
while match:
env_val = self._case.get_value(match.group('name'))
expect(env_val is not None,
"Namelist default for variable {} refers to unknown XML variable {}.".format(name, match.group('name')))
scalar = scalar.replace(match.group(0), str(env_val), 1)
match = _var_ref_re.search(scalar)
default[i] = scalar
# Deal with missing quotes.
if var_type == 'character':
for i, scalar in enumerate(default):
# Preserve null values.
if scalar != '':
default[i] = self.quote_string(scalar)
default = self._to_python_value(name, default)
return default
[docs] def get_streams(self):
"""Get a list of all streams used for the current data model mode."""
return self.get_default("streamslist")
[docs] def clean_streams(self):
for variable in self._streams_variables:
self._streams_namelists[variable] = []
self._streams_namelists["streams"] = []
[docs] def new_instance(self):
""" Clean the object just enough to introduce a new instance """
self.clean_streams()
self._namelist.clean_groups()
def _sub_fields(self, varnames):
"""Substitute indicators with given values in a list of fields.
Replace any instance of the following substring indicators with the
appropriate values:
%glc = two-digit GLC elevation class from 00 through glc_nec
The difference between this function and `_sub_paths` is that this
function is intended to be used for variable names (especially from the
`strm_datvar` defaults), whereas `_sub_paths` is intended for use on
input data file paths.
Returns a string.
Example: If `_sub_fields` is called with an array containing two
elements, each of which contains two strings, and glc_nec=3:
foo bar
s2x_Ss_tsrf%glc tsrf%glc
then the returned array will be:
foo bar
s2x_Ss_tsrf00 tsrf00
s2x_Ss_tsrf01 tsrf01
s2x_Ss_tsrf02 tsrf02
s2x_Ss_tsrf03 tsrf03
"""
lines = varnames.split("\n")
new_lines = []
for line in lines:
if not line:
continue
if "%glc" in line:
if self._case.get_value('GLC_NEC') == 0:
glc_nec_indices = []
else:
glc_nec_indices = range(self._case.get_value('GLC_NEC')+1)
for i in glc_nec_indices:
new_lines.append(line.replace("%glc", "{:02d}".format(i)))
else:
new_lines.append(line)
return "\n".join(new_lines)
@staticmethod
def _days_in_month(month, year=1):
"""Number of days in the given month (specified as an int, 1-12).
The `year` argument gives the year for which to request the number of
days, in a Gregorian calendar. Defaults to `1` (not a leap year).
"""
month_start = datetime.date(year, month, 1)
if month == 12:
next_year = year+1
next_month = 1
else:
next_year = year
next_month = month + 1
next_month_start = datetime.date(next_year, next_month, 1)
return (next_month_start - month_start).days
def _sub_paths(self, filenames, year_start, year_end):
"""Substitute indicators with given values in a list of filenames.
Replace any instance of the following substring indicators with the
appropriate values:
%y = year from the range year_start to year_end
%ym = year-month from the range year_start to year_end with all 12
months
%ymd = year-month-day from the range year_start to year_end with
all 12 months
For the date indicators, the year may be prefixed with a number of
digits to use (the default is 4). E.g. `%2ymd` can be used to change the
number of year digits from 4 to 2.
Note that we assume that there is no mixing and matching of date
indicators, i.e. you cannot use `%4ymd` and `%2y` in the same line. Note
also that we use a no-leap calendar, i.e. every month has the same
number of days every year.
The difference between this function and `_sub_fields` is that this
function is intended to be used for file names (especially from the
`strm_datfil` defaults), whereas `_sub_fields` is intended for use on
variable names.
Returns a string (filenames separated by newlines).
"""
lines = [line for line in filenames.split("\n") if line]
new_lines = []
for line in lines:
match = _ymd_re.search(filenames)
if match is None:
new_lines.append(line)
continue
if match.group('digits'):
year_format = "{:0"+match.group('digits')+"d}"
else:
year_format = "{:04d}"
for year in range(year_start, year_end+1):
if match.group('day'):
for month in range(1, 13):
days = self._days_in_month(month)
for day in range(1, days+1):
date_string = (year_format + "-{:02d}-{:02d}").format(year, month, day)
new_line = line.replace(match.group(0), date_string)
new_lines.append(new_line)
elif match.group('month'):
for month in range(1, 13):
date_string = (year_format + "-{:02d}").format(year, month)
new_line = line.replace(match.group(0), date_string)
new_lines.append(new_line)
else:
date_string = year_format.format(year)
new_line = line.replace(match.group(0), date_string)
new_lines.append(new_line)
return "\n".join(new_lines)
[docs] def create_stream_file_and_update_shr_strdata_nml(self, config, caseroot, #pylint:disable=too-many-locals
stream, stream_path, data_list_path):
"""Write the pseudo-XML file corresponding to a given stream.
Arguments:
`config` - Used to look up namelist defaults. This is used *in addition*
to the `config` used to construct the namelist generator. The
main reason to supply additional configuration options here
is to specify stream-specific settings.
`stream` - Name of the stream.
`stream_path` - Path to write the stream file to.
`data_list_path` - Path of file to append input data information to.
"""
if os.path.exists(stream_path):
os.unlink(stream_path)
user_stream_path = os.path.join(caseroot, "user_"+os.path.basename(stream_path))
# Use the user's stream file, or create one if necessary.
config = config.copy()
config["stream"] = stream
# Stream-specific configuration.
if os.path.exists(user_stream_path):
safe_copy(user_stream_path, stream_path)
strmobj = Stream(infile=stream_path)
domain_filepath = strmobj.get_value("domainInfo/filePath")
data_filepath = strmobj.get_value("fieldInfo/filePath")
domain_filenames = strmobj.get_value("domainInfo/fileNames")
data_filenames = strmobj.get_value("fieldInfo/fileNames")
else:
# Figure out the details of this stream.
if stream in ("prescribed", "copyall"):
# Assume only one file for prescribed mode!
grid_file = self.get_default("strm_grid_file", config)
domain_filepath, domain_filenames = os.path.split(grid_file)
data_file = self.get_default("strm_data_file", config)
data_filepath, data_filenames = os.path.split(data_file)
else:
domain_filepath = self.get_default("strm_domdir", config)
domain_filenames = self.get_default("strm_domfil", config)
data_filepath = self.get_default("strm_datdir", config)
data_filenames = self.get_default("strm_datfil", config)
domain_varnames = self._sub_fields(self.get_default("strm_domvar", config))
data_varnames = self._sub_fields(self.get_default("strm_datvar", config))
offset = self.get_default("strm_offset", config)
year_start = int(self.get_default("strm_year_start", config))
year_end = int(self.get_default("strm_year_end", config))
data_filenames = self._sub_paths(data_filenames, year_start, year_end)
domain_filenames = self._sub_paths(domain_filenames, year_start, year_end)
# Overwrite domain_file if should be set from stream data
if domain_filenames == 'null':
domain_filepath = data_filepath
domain_filenames = data_filenames.splitlines()[0]
stream_file_text = _stream_file_template.format(
domain_varnames=domain_varnames,
domain_filepath=domain_filepath,
domain_filenames=domain_filenames,
data_varnames=data_varnames,
data_filepath=data_filepath,
data_filenames=data_filenames,
offset=offset,
)
with open(stream_path, 'w') as stream_file:
stream_file.write(stream_file_text)
lines_hash = self._get_input_file_hash(data_list_path)
with open(data_list_path, 'a') as input_data_list:
for i, filename in enumerate(domain_filenames.split("\n")):
if filename.strip() == '':
continue
filepath, filename = os.path.split(filename)
if not filepath:
filepath = os.path.join(domain_filepath, filename.strip())
string = "domain{:d} = {}\n".format(i+1, filepath)
hashValue = hashlib.md5(string.rstrip().encode('utf-8')).hexdigest()
if hashValue not in lines_hash:
input_data_list.write(string)
for i, filename in enumerate(data_filenames.split("\n")):
if filename.strip() == '':
continue
filepath = os.path.join(data_filepath, filename.strip())
string = "file{:d} = {}\n".format(i+1, filepath)
hashValue = hashlib.md5(string.rstrip().encode('utf-8')).hexdigest()
if hashValue not in lines_hash:
input_data_list.write(string)
self.update_shr_strdata_nml(config, stream, stream_path)
[docs] def update_shr_strdata_nml(self, config, stream, stream_path):
"""Updates values for the `shr_strdata_nml` namelist group.
This should be done once per stream, and it shouldn't usually be called
directly, since `create_stream_file` calls this method itself.
"""
assert config['stream'] == stream, \
"config stream is {}, but input stream is {}".format(config['stream'], stream)
# Double-check the years for sanity.
year_start = int(self.get_default("strm_year_start", config))
year_end = int(self.get_default("strm_year_end", config))
year_align = int(self.get_default("strm_year_align", config))
expect(year_end >= year_start,
"Stream {} starts at year {:d}, but ends at earlier year {:d}.".format(stream, year_start, year_end))
# Add to streams file.
stream_string = "{} {:d} {:d} {:d}".format(os.path.basename(stream_path),
year_align, year_start,
year_end)
self._streams_namelists["streams"].append(stream_string)
for variable in self._streams_variables:
default = self.get_default(variable, config)
expect(len(default) == 1,
"Stream {} had multiple settings for variable {}.".format(stream, variable))
self._streams_namelists[variable].append(default[0])
[docs] def set_abs_file_path(self, file_path):
"""If `file_path` is relative, make it absolute using `DIN_LOC_ROOT`.
If an absolute path is input, it is returned unchanged.
"""
if os.path.isabs(file_path):
return file_path
else:
fullpath = os.path.join(self._din_loc_root, file_path)
return fullpath
[docs] def add_default(self, name, value=None, ignore_abs_path=None):
"""Add a value for the specified variable to the namelist.
If the specified variable is already defined in the object, the existing
value is preserved. Otherwise, the `value` argument, if provided, will
be used to set the value. If no such value is found, the defaults file
will be consulted. If null values are present in any of the above, the
result will be a merged array of values.
If no value for the variable is found via any of the above, this method
will raise an exception.
"""
# pylint: disable=protected-access
group = self._definition.get_group(name)
# Use this to see if we need to raise an error when nothing is found.
have_value = False
# Check for existing value.
current_literals = self._namelist.get_variable_value(group, name)
# Check for input argument.
if value is not None:
have_value = True
# if compression were to occur, this is where it does
literals = self._to_namelist_literals(name, value)
current_literals = merge_literal_lists(literals, current_literals)
# Check for default value.
default = self.get_default(name, allow_none=True)
if default is not None:
have_value = True
default_literals = self._to_namelist_literals(name, default)
current_literals = merge_literal_lists(default_literals, current_literals)
expect(have_value, "No default value found for {}.".format(name))
# Go through file names and prepend input data root directory for
# absolute pathnames.
var_type, _, var_size = self._definition.split_type_string(name)
if var_type == "character" and ignore_abs_path is None:
var_input_pathname = self._definition.get_input_pathname(name)
if var_input_pathname == 'abs':
current_literals = expand_literal_list(current_literals)
for i, literal in enumerate(current_literals):
if literal == '':
continue
file_path = character_literal_to_string(literal)
# NOTE - these are hard-coded here and a better way is to make these extensible
if file_path == 'UNSET' or file_path == 'idmap' or file_path == 'idmap_ignore':
continue
if file_path == 'null':
continue
file_path = self.set_abs_file_path(file_path)
if not os.path.exists(file_path):
logger.warning("File not found: {} = {}, will attempt to download in check_input_data phase".format(name, literal))
current_literals[i] = string_to_character_literal(file_path)
current_literals = compress_literal_list(current_literals)
# Set the new value.
self._namelist.set_variable_value(group, name, current_literals, var_size)
[docs] def create_shr_strdata_nml(self):
"""Set defaults for `shr_strdata_nml` variables other than the variable domainfile """
self.add_default("datamode")
if self.get_value("datamode") != 'NULL':
self.add_default("streams",
value=self._streams_namelists["streams"])
for variable in self._streams_variables:
self.add_default(variable,
value=self._streams_namelists[variable])
[docs] def get_group_variables(self, group_name):
return self._namelist.get_group_variables(group_name)
def _get_input_file_hash(self, data_list_path):
lines_hash = set()
if os.path.isfile(data_list_path):
with open(data_list_path, "r") as input_data_list:
for line in input_data_list:
hashValue = hashlib.md5(line.rstrip().encode('utf-8')).hexdigest()
logger.debug( "Found line {} with hash {}".format(line,hashValue))
lines_hash.add(hashValue)
return lines_hash
def _write_input_files(self, data_list_path):
"""Write input data files to list."""
# append to input_data_list file
lines_hash = self._get_input_file_hash(data_list_path)
with open(data_list_path, "a") as input_data_list:
for group_name in self._namelist.get_group_names():
for variable_name in self._namelist.get_variable_names(group_name):
input_pathname = self._definition.get_node_element_info(variable_name, "input_pathname")
if input_pathname is not None:
# This is where we end up for all variables that are paths
# to input data files.
literals = self._namelist.get_variable_value(group_name, variable_name)
for literal in literals:
file_path = character_literal_to_string(literal)
# NOTE - these are hard-coded here and a better way is to make these extensible
if file_path == 'UNSET' or file_path == 'idmap' or file_path == 'idmap_ignore':
continue
if input_pathname == 'abs':
# No further mangling needed for absolute paths.
# At this point, there are overwrites that should be ignored
if not os.path.isabs(file_path):
continue
else:
pass
elif input_pathname.startswith('rel:'):
# The part past "rel" is the name of a variable that
# this variable specifies its path relative to.
root_var = input_pathname[4:]
root_dir = self.get_value(root_var)
file_path = os.path.join(root_dir, file_path)
else:
expect(False,
"Bad input_pathname value: {}.".format(input_pathname))
# Write to the input data list.
string = "{} = {}".format(variable_name, file_path)
hashValue = hashlib.md5(string.rstrip().encode('utf-8')).hexdigest()
if hashValue not in lines_hash:
logger.debug("Adding line {} with hash {}".format(string,hashValue))
input_data_list.write(string+"\n")
else:
logger.debug("Line already in file {}".format(string))
[docs] def write_output_file(self, namelist_file, data_list_path=None, groups=None, sorted_groups=True):
"""Write out the namelists and input data files.
The `namelist_file` and `modelio_file` are the locations to which the
component and modelio namelists will be written, respectively. The
`data_list_path` argument is the location of the `*.input_data_list`
file, which will have the input data files added to it.
"""
self._definition.validate(self._namelist)
if groups is None:
groups = self._namelist.get_group_names()
# remove groups that are never in namelist file
if "modelio" in groups:
groups.remove("modelio")
if "seq_maps" in groups:
groups.remove("seq_maps")
# write namelist file
self._namelist.write(namelist_file, groups=groups, sorted_groups=sorted_groups)
if data_list_path is not None:
self._write_input_files(data_list_path)
[docs] def add_nmlcontents(self, filename, group, append=True, format_="nmlcontents", sorted_groups=True):
""" Write only contents of nml group """
self._namelist.write(filename, groups=[group], append=append, format_=format_, sorted_groups=sorted_groups)
[docs] def write_seq_maps(self, filename):
""" Write out seq_maps.rc"""
self._namelist.write(filename, groups=["seq_maps"], format_="rc")
[docs] def write_modelio_file(self, filename):
""" Write component modelio files"""
self._namelist.write(filename, groups=["modelio", "pio_inparm"], format_="nml")