Source code for CIME.SystemTests.system_tests_compare_two

"""
Base class for CIME system tests that involve doing two runs and comparing their
output.

NOTE: Below is the flow of a multisubmit test.
Non-batch:
case_submit -> case_run     # PHASE 1
            -> case_run     # PHASE 2

batch:
case_submit -> case_run     # PHASE 1
case_run    -> case_submit
case_submit -> case_run     # PHASE 2

In the __init__ method for your test, you MUST call
    SystemTestsCompareTwo.__init__
See the documentation of that method for details.

Classes that inherit from this are REQUIRED to implement the following methods:

(1) _case_one_setup
    This method will be called to set up case 1, the "base" case

(2) _case_two_setup
    This method will be called to set up case 2, the "test" case

In addition, they MAY require the following methods:

(1) _common_setup
    This method will be called to set up both cases. It should contain any setup
    that's needed in both cases. This is called before _case_one_setup or
    _case_two_setup.

(2) _case_one_custom_prerun_action(self):
    Use this to do arbitrary actions immediately before running case one

(3) _case_two_custom_prerun_action(self):
    Use this to do arbitrary actions immediately before running case two

(4) _case_one_custom_postrun_action(self):
    Use this to do arbitrary actions immediately after running case one

(5) _case_two_custom_postrun_action(self):
    Use this to do arbitrary actions immediately after running case two
"""

from CIME.XML.standard_module_setup import *
from CIME.SystemTests.system_tests_common import SystemTestsCommon, fix_single_exe_case
from CIME.case import Case
from CIME.config import Config
from CIME.test_status import *

import shutil, os, glob

logger = logging.getLogger(__name__)


[docs] class SystemTestsCompareTwo(SystemTestsCommon): def __init__( self, case, separate_builds=False, run_two_suffix="test", run_one_description="", run_two_description="", multisubmit=False, ignore_fieldlist_diffs=False, case_two_keep_init_generated_files=False, dry_run=False, **kwargs ): """ Initialize a SystemTestsCompareTwo object. Individual test cases that inherit from SystemTestsCompareTwo MUST call this __init__ method. Args: case: case object passsed to __init__ method of individual test. This is the main case associated with the test. separate_builds (bool): Whether separate builds are needed for the two cases. If False, case2 uses the case1 executable. run_two_suffix (str, optional): Suffix appended to the case name for the second run. Defaults to 'test'. This can be anything other than 'base'. run_one_description (str, optional): Description printed to log file when starting the first run. Defaults to ''. run_two_description (str, optional): Description printed to log file when starting the second run. Defaults to ''. multisubmit (bool): Do first and second runs as different submissions. Designed for tests with RESUBMIT=1 ignore_fieldlist_diffs (bool): If True, then: If the two cases differ only in their field lists (i.e., all shared fields are bit-for-bit, but one case has some diagnostic fields that are missing from the other case), treat the two cases as identical. (This is needed for tests where one case exercises an option that produces extra diagnostic fields.) case_two_keep_init_generated_files (bool): If True, then do NOT remove the init_generated_files subdirectory of the case2 run directory before running case2. This should typically be kept at its default (False) so that rerunning a test gives the same behavior as in the initial run rather than reusing init_generated_files in the second run. However, this option is provided for the sake of specific tests, e.g., a test of the behavior of running with init_generated_files in place. """ SystemTestsCommon.__init__(self, case, **kwargs) self._separate_builds = separate_builds self._ignore_fieldlist_diffs = ignore_fieldlist_diffs self._case_two_keep_init_generated_files = case_two_keep_init_generated_files # run_one_suffix is just used as the suffix for the netcdf files # produced by the first case; we may eventually remove this, but for now # it is needed by the various component_*.sh scripts. run_two_suffix is # also used as the suffix for netcdf files, but more importantly is used # to create the case name for the clone case. # # NOTE(wjs, 2016-08-03) It is currently CRITICAL for run_one_suffix to # be 'base', because this is assumed for baseline comparison and # generation. Once that assumption is relaxed, then run_one_suffix can # be set in the call to the constructor just like run_two_suffix # currently is. Or, if these tools are rewritten to work without any # suffix, then run_one_suffix can be removed entirely. self._run_one_suffix = "base" self._run_two_suffix = run_two_suffix.rstrip() expect( self._run_two_suffix != self._run_one_suffix, "ERROR: Must have different suffixes for run one and run two", ) self._run_one_description = run_one_description self._run_two_description = run_two_description # Save case for first run so we can return to it if we switch self._case # to point to self._case2 self._case1 = self._case self._caseroot1 = self._get_caseroot() self._caseroot2 = self._get_caseroot2() # Initialize self._case2; it will get set to its true value in # _setup_cases_if_not_yet_done self._case2 = None # Prevent additional setup_case calls when detecting support for `--single-exe` if not dry_run: self._setup_cases_if_not_yet_done() self._multisubmit = ( multisubmit and self._case1.get_value("BATCH_SYSTEM") != "none" ) # ======================================================================== # Methods that MUST be implemented by specific tests that inherit from this # base class # ======================================================================== def _case_one_setup(self): """ This method will be called to set up case 1, the "base" case. This should be written to refer to self._case: this object will point to case1 at the point that this is called. """ raise NotImplementedError def _case_two_setup(self): """ This method will be called to set up case 2, the "test" case This should be written to refer to self._case: this object will point to case2 at the point that this is called. """ raise NotImplementedError # ======================================================================== # Methods that MAY be implemented by specific tests that inherit from this # base class, if they have any work to do in these methods # ======================================================================== def _common_setup(self): """ This method will be called to set up both cases. It should contain any setup that's needed in both cases. This is called before _case_one_setup or _case_two_setup. This should be written to refer to self._case: It will be called once with self._case pointing to case1, and once with self._case pointing to case2. """ def _case_one_custom_prerun_action(self): """ Use to do arbitrary actions immediately before running case one """ def _case_two_custom_prerun_action(self): """ Use to do arbitrary actions immediately before running case two """ def _case_one_custom_postrun_action(self): """ Use to do arbitrary actions immediately after running case one """ def _case_two_custom_postrun_action(self): """ Use to do arbitrary actions immediately after running case two """ # ======================================================================== # Main public methods # ========================================================================
[docs] def build_phase(self, sharedlib_only=False, model_only=False): # Subtle issue: case1 is already in a writeable state since it tends to be opened # with a with statement in all the API entrances in CIME. case2 was created via clone, # not a with statement, so it's not in a writeable state, so we need to use a with # statement here to put it in a writeable state. with self._case2: if self._separate_builds: self._activate_case1() self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only) self._activate_case2() # Although we're doing separate builds, it still makes sense # to share the sharedlibroot area with case1 so we can reuse # pieces of the build from there. if Config.instance().common_sharedlibroot: # We need to turn off this change for E3SM because it breaks # the MPAS build system self._case2.set_value( "SHAREDLIBROOT", self._case1.get_value("SHAREDLIBROOT") ) self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only) else: self._activate_case1() self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only) # pio_typename may be changed during the build if the default is not a # valid value for this build, update case2 to reflect this change for comp in self._case1.get_values("COMP_CLASSES"): comp_pio_typename = "{}_PIO_TYPENAME".format(comp) self._case2.set_value( comp_pio_typename, self._case1.get_value(comp_pio_typename) ) # The following is needed when _case_two_setup has a case_setup call # despite sharing the build (e.g., to change NTHRDS) self._case2.set_value("BUILD_COMPLETE", True)
[docs] def run_phase(self, success_change=False): # pylint: disable=arguments-differ """ Runs both phases of the two-phase test and compares their results If success_change is True, success requires some files to be different """ is_first_run = self._case1.get_value("IS_FIRST_RUN") compare_phase_name = "{}_{}_{}".format( COMPARE_PHASE, self._run_one_suffix, self._run_two_suffix ) # On a batch system with a multisubmit test "RESUBMIT" is used to track # which phase is being ran. By the end of the test it equals 0. If the # the test fails in a way where the RUN_PHASE is PEND then "RESUBMIT" # does not get reset to 1 on a rerun and the first phase is skiped # causing the COMPARE_PHASE to fail. This ensures that "RESUBMIT" will # get reset if the test state is not correct for a rerun. # NOTE: "IS_FIRST_RUN" is reset in "case_submit.py" if ( is_first_run and self._multisubmit and self._case1.get_value("RESUBMIT") == 0 ): self._resetup_case(RUN_PHASE, reset=True) first_phase = ( self._case1.get_value("RESUBMIT") == 1 ) # Only relevant for multi-submit tests run_type = self._case1.get_value("RUN_TYPE") logger.info( "_multisubmit {} first phase {}".format(self._multisubmit, first_phase) ) # First run if not self._multisubmit or first_phase: logger.info("Doing first run: " + self._run_one_description) # Add a PENDing compare phase so that we'll notice if the second part of compare two # doesn't run. with self._test_status: self._test_status.set_status(compare_phase_name, TEST_PEND_STATUS) self._activate_case1() self._case_one_custom_prerun_action() self.run_indv(suffix=self._run_one_suffix) self._case_one_custom_postrun_action() # Second run if not self._multisubmit or not first_phase: # Subtle issue: case1 is already in a writeable state since it tends to be opened # with a with statement in all the API entrances in CIME. case2 was created via clone, # not a with statement, so it's not in a writeable state, so we need to use a with # statement here to put it in a writeable state. with self._case2: logger.info("Doing second run: " + self._run_two_description) self._activate_case2() # This assures that case two namelists are populated self._skip_pnl = False # we need to make sure run2 is properly staged. if run_type != "startup": self._case2.check_case() self._case_two_custom_prerun_action() self.run_indv( suffix=self._run_two_suffix, keep_init_generated_files=self._case_two_keep_init_generated_files, ) self._case_two_custom_postrun_action() # Compare results # Case1 is the "main" case, and we need to do the comparisons from there self._activate_case1() self._link_to_case2_output() self._component_compare_test( self._run_one_suffix, self._run_two_suffix, success_change=success_change, ignore_fieldlist_diffs=self._ignore_fieldlist_diffs, )
[docs] def copy_case1_restarts_to_case2(self): """ Makes a copy (or symlink) of restart files and related files (necessary history files, rpointer files) from case1 to case2. This is not done automatically, but can be called by individual tests where case2 does a continue_run using case1's restart files. """ rundir2 = self._case2.get_value("RUNDIR") self._case1.archive_last_restarts( archive_restdir=rundir2, rundir=self._case1.get_value("RUNDIR"), link_to_restart_files=True, )
# ======================================================================== # Private methods # ======================================================================== def _get_caseroot2(self): """ Determines and returns caseroot for case2 Assumes that self._case1 is already set to point to the case1 object """ casename2 = self._case1.get_value("CASE") caseroot1 = self._case1.get_value("CASEROOT") # Nest the case directory for case2 inside the case directory for case1 caseroot2 = os.path.join(caseroot1, "case2", casename2) return caseroot2 def _get_output_root2(self): """ Determines and returns cime_output_root for case2 Assumes that self._case1 is already set to point to the case1 object """ # Since case2 has the same name as case1, its CIME_OUTPUT_ROOT # must also be different, so that anything put in # $CIME_OUTPUT_ROOT/$CASE/ is not accidentally shared between # case1 and case2. (Currently nothing is placed here, but this # helps prevent future problems.) output_root2 = os.path.join( self._case1.get_value("CIME_OUTPUT_ROOT"), self._case1.get_value("CASE"), "case2_output_root", ) return output_root2 def _get_case2_exeroot(self): """ Gets exeroot for case2. Returns None if we should use the default value of exeroot. """ if self._separate_builds: # case2's EXEROOT needs to be somewhere that (1) is unique # to this case (considering that case1 and case2 have the # same case name), and (2) does not have too long of a path # name (because too-long paths can make some compilers # fail). case1_exeroot = self._case1.get_value("EXEROOT") case2_exeroot = os.path.join(case1_exeroot, "case2bld") else: # Use default exeroot case2_exeroot = None return case2_exeroot def _get_case2_rundir(self): """ Gets rundir for case2. """ # case2's RUNDIR needs to be somewhere that is unique to this # case (considering that case1 and case2 have the same case # name). Note that the location below is symmetrical to the # location of case2's EXEROOT set in _get_case2_exeroot. case1_rundir = self._case1.get_value("RUNDIR") case2_rundir = os.path.join(case1_rundir, "case2run") return case2_rundir def _setup_cases_if_not_yet_done(self): """ Determines if case2 already exists on disk. If it does, this method creates the self._case2 object pointing to the case directory. If it doesn't exist, then this method creates case2 as a clone of case1, and sets the self._case2 object appropriately. This also does the setup for both case1 and case2. Assumes that the following variables are already set in self: _caseroot1 _caseroot2 _case1 Sets self._case2 """ # Use the existence of the case2 directory to signal whether we have # done the necessary test setup for this test: When we initially create # the case2 directory, we set up both test cases; then, if we find that # the case2 directory already exists, we assume that the setup has # already been done. (In some cases it could be problematic to redo the # test setup when it's not needed - e.g., by appending things to user_nl # files multiple times. This is why we want to make sure to just do the # test setup once.) if os.path.exists(self._caseroot2): self._case2 = self._case_from_existing_caseroot(self._caseroot2) else: try: self._case2 = self._case1.create_clone( self._caseroot2, keepexe=not self._separate_builds, cime_output_root=self._get_output_root2(), exeroot=self._get_case2_exeroot(), rundir=self._get_case2_rundir(), ) self._write_info_to_case2_output_root() self._setup_cases() except BaseException: # If a problem occurred in setting up the test cases, it's # important to remove the case2 directory: If it's kept around, # that would signal that test setup was done successfully, and # thus doesn't need to be redone - which is not the case. Of # course, we'll likely be left in an inconsistent state in this # case, but if we didn't remove the case2 directory, the next # re-build of the test would think, "okay, setup is done, I can # move on to the build", which would be wrong. if os.path.isdir(self._caseroot2): shutil.rmtree(self._caseroot2) self._activate_case1() logger.warning( "WARNING: Test case setup failed. Case2 has been removed, " "but the main case may be in an inconsistent state. " "If you want to rerun this test, you should create " "a new test rather than trying to rerun this one." ) raise def _case_from_existing_caseroot(self, caseroot): """ Returns a Case object from an existing caseroot directory Args: caseroot (str): path to existing caseroot """ return Case(case_root=caseroot, read_only=False) def _activate_case1(self): """ Make case 1 active for upcoming calls """ os.chdir(self._caseroot1) self._set_active_case(self._case1) def _activate_case2(self): """ Make case 2 active for upcoming calls """ os.chdir(self._caseroot2) self._set_active_case(self._case2) def _write_info_to_case2_output_root(self): """ Writes a file with some helpful information to case2's output_root. The motivation here is two-fold: (1) Currently, case2's output_root directory is empty. This could be confusing. (2) For users who don't know where to look, it could be hard to find case2's bld and run directories. It is somewhat easier to stumble upon case2's output_root, so we put a file there pointing them to the right place. """ readme_path = os.path.join(self._get_output_root2(), "README") try: with open(readme_path, "w") as fd: fd.write("This directory is typically empty.\n\n") fd.write( "case2's run dir is here: {}\n\n".format( self._case2.get_value("RUNDIR") ) ) fd.write( "case2's bld dir is here: {}\n".format( self._case2.get_value("EXEROOT") ) ) except IOError: # It's not a big deal if we can't write the README file # (e.g., because the directory doesn't exist or isn't # writeable; note that the former may be the case in unit # tests). So just continue merrily on our way if there was a # problem. pass def _setup_cases(self): """ Does all test-specific set up for the two test cases. """ # Set up case 1 self._activate_case1() self._common_setup() self._case_one_setup() # Flush the case so that, if errors occur later, then at least case 1 is # in a correct, post-setup state. This is important because the mere # existence of a case 2 directory signals that setup is done. So if the # build fails and the user rebuilds, setup won't be redone - so it's # important to ensure that the results of setup are flushed to disk. # # Note that case 1 will be in its post-setup state even if case 2 setup # fails. Putting the case1 flush after case 2 setup doesn't seem to help # with that (presumably some flush is called automatically), and anyway # wouldn't help with things like appending to user_nl files (which don't # rely on flush). So we just have to live with that possibility (but # note that we print a warning to the log file if that happens, in the # caller of this method). self._case.flush() # This assures that case one namelists are populated # and creates the case.test script self._case.case_setup(test_mode=False, reset=True) fix_single_exe_case(self._case) # Set up case 2 with self._case2: self._activate_case2() self._common_setup() self._case_two_setup() fix_single_exe_case(self._case2) # Go back to case 1 to ensure that's where we are for any following code self._activate_case1() def _link_to_case2_output(self): """ Looks for all files in rundir2 matching the pattern casename2*.nc.run2suffix For each file found, makes a link in rundir1 pointing to this file; the link is renamed so that the original occurrence of casename2 is replaced with casename1. For example: /glade/scratch/sacks/somecase/run/somecase.clm2.h0.nc.run2 -> /glade/scratch/sacks/somecase.run2/run/somecase.run2.clm2.h0.nc.run2 If the destination link already exists and points to the correct location, it is maintained as is. However, an exception will be raised if the destination link is not exactly as it should be: we avoid overwriting some existing file or link. """ casename1 = self._case1.get_value("CASE") casename2 = self._case2.get_value("CASE") rundir1 = self._case1.get_value("RUNDIR") rundir2 = self._case2.get_value("RUNDIR") run2suffix = self._run_two_suffix pattern = "{}*.nc.{}".format(casename2, run2suffix) case2_files = glob.glob(os.path.join(rundir2, pattern)) for one_file in case2_files: file_basename = os.path.basename(one_file) modified_basename = file_basename.replace(casename2, casename1, 1) one_link = os.path.join(rundir1, modified_basename) if os.path.islink(one_link) and os.readlink(one_link) == one_file: # Link is already set up correctly: do nothing # (os.symlink raises an exception if you try to replace an # existing file) pass else: os.symlink(one_file, one_link)