import CIME.compare_namelists, CIME.simple_compare
from CIME.utils import expect, append_status, EnvironmentContext
from CIME.test_status import *
from CIME.hist_utils import compare_baseline
from CIME.case import Case
import os, glob, logging
###############################################################################
[docs]def append_status_cprnc_log(msg, logfile_name, test_dir):
###############################################################################
try:
append_status(msg, logfile_name, caseroot=test_dir)
except IOError:
pass
###############################################################################
[docs]def compare_namelists(case, baseline_name, baseline_root, logfile_name):
###############################################################################
log_lvl = logging.getLogger().getEffectiveLevel()
logging.disable(logging.CRITICAL)
success = case.case_cmpgen_namelists(compare=True, compare_name=baseline_name, baseline_root=baseline_root, logfile_name=logfile_name)
logging.getLogger().setLevel(log_lvl)
return success
###############################################################################
[docs]def compare_history(case, baseline_name, baseline_root, log_id):
###############################################################################
real_user = case.get_value("REALUSER")
with EnvironmentContext(USER=real_user):
baseline_full_dir = os.path.join(baseline_root, baseline_name, case.get_value("CASEBASEID"))
outfile_suffix = "{}.{}".format(baseline_name, log_id)
try:
result, comments = compare_baseline(case, baseline_dir=baseline_full_dir,
outfile_suffix=outfile_suffix)
except IOError:
result, comments = compare_baseline(case, baseline_dir=baseline_full_dir,
outfile_suffix=None)
return result, comments
###############################################################################
[docs]def compare_test_results(baseline_name, baseline_root, test_root, compiler, test_id=None, compare_tests=None, namelists_only=False, hist_only=False):
###############################################################################
"""
Compares with baselines for all matching tests
Outputs results for each test to stdout (one line per test); possible status
codes are: PASS, FAIL, SKIP. (A SKIP denotes a test that did not make it to
the run phase or a test for which the run phase did not pass: we skip
baseline comparisons in this case.)
In addition, creates files named compare.log.BASELINE_NAME.TIMESTAMP in each
test directory, which contain more detailed output. Also creates
*.cprnc.out.BASELINE_NAME.TIMESTAMP files in each run directory.
Returns True if all tests generated either PASS or SKIP results, False if
there was at least one FAIL result.
"""
test_id_glob = "*{}*".format(compiler) if test_id is None else "*{}".format(test_id)
test_status_files = glob.glob("{}/{}/{}".format(test_root, test_id_glob, TEST_STATUS_FILENAME))
expect(test_status_files, "No matching test cases found in for {}/{}/{}".format(test_root, test_id_glob, TEST_STATUS_FILENAME))
# ID to use in the log file names, to avoid file name collisions with
# earlier files that may exist.
log_id = CIME.utils.get_timestamp()
all_pass_or_skip = True
for test_status_file in test_status_files:
test_dir = os.path.dirname(test_status_file)
ts = TestStatus(test_dir=test_dir)
test_name = ts.get_name()
if (compare_tests in [[], None] or CIME.utils.match_any(test_name, compare_tests)):
if (not hist_only):
nl_compare_result = None
nl_compare_comment = ""
nl_result = ts.get_status(SETUP_PHASE)
if (nl_result is None):
nl_compare_result = "SKIP"
nl_compare_comment = "Test did not make it to setup phase"
nl_do_compare = False
else:
nl_do_compare = True
else:
nl_do_compare = False
detailed_comments = ""
if (not namelists_only):
compare_result = None
compare_comment = ""
run_result = ts.get_status(RUN_PHASE)
if (run_result is None):
compare_result = "SKIP"
compare_comment = "Test did not make it to run phase"
do_compare = False
elif (run_result != TEST_PASS_STATUS):
compare_result = "SKIP"
compare_comment = "Run phase did not pass"
do_compare = False
else:
do_compare = True
else:
do_compare = False
if nl_do_compare or do_compare:
with Case(test_dir) as case:
if baseline_name is None:
baseline_name = case.get_value("BASELINE_NAME_CMP")
if not baseline_name:
baseline_name = CIME.utils.get_current_branch(repo=CIME.utils.get_cime_root())
if baseline_root is None:
baseline_root = case.get_value("BASELINE_ROOT")
logfile_name = "compare.log.{}.{}".format(baseline_name.replace("/", "_"), log_id)
append_status_cprnc_log(
"Comparing against baseline with compare_test_results:\n"
"Baseline: {}\n In baseline_root: {}".format(baseline_name, baseline_root),
logfile_name,
test_dir)
if nl_do_compare:
nl_success = compare_namelists(case, baseline_name, baseline_root, logfile_name)
if nl_success:
nl_compare_result = TEST_PASS_STATUS
nl_compare_comment = ""
else:
nl_compare_result = TEST_FAIL_STATUS
nl_compare_comment = "See {}/{}".format(test_dir, logfile_name)
all_pass_or_skip = False
if do_compare:
success, detailed_comments = compare_history(case, baseline_name, baseline_root, log_id)
if success:
compare_result = TEST_PASS_STATUS
else:
compare_result = TEST_FAIL_STATUS
all_pass_or_skip = False
# Following the logic in SystemTestsCommon._compare_baseline:
# We'll print the comment if it's a brief one-liner; otherwise
# the comment will only appear in the log file
if "\n" not in detailed_comments:
compare_comment = detailed_comments
brief_result = ""
if not hist_only:
brief_result += "{} {} {} {}\n".format(nl_compare_result, test_name, NAMELIST_PHASE, nl_compare_comment)
if not namelists_only:
brief_result += "{} {} {}".format(compare_result, test_name, BASELINE_PHASE)
if compare_comment:
brief_result += " {}".format(compare_comment)
brief_result += "\n"
print(brief_result)
append_status_cprnc_log(brief_result, logfile_name, test_dir)
if detailed_comments:
append_status_cprnc_log("Detailed comments:\n" + detailed_comments, logfile_name, test_dir)
return all_pass_or_skip