"""
This module contains a class that extends unittest.TestCase, adding custom assertions that
can be used when testing TestStatus.
"""
from CIME.XML.standard_module_setup import *
import unittest
import re
from CIME import test_status
[docs]
class CustomAssertionsTestStatus(unittest.TestCase):
[docs]
def assert_status_of_phase(self, output, status, phase, test_name, xfail=None):
"""Asserts that 'output' contains a line showing the given
status for the given phase for the given test_name.
'xfail' should have one of the following values:
- None (the default): assertion passes regardless of whether there is an
EXPECTED/UNEXPECTED string
- 'no': The line should end with the phase, with no additional text after that
- 'expected': After the phase, the line should contain '(EXPECTED FAILURE)'
- 'unexpected': After the phase, the line should contain '(UNEXPECTED'
"""
expected = r"^ *{} +".format(
re.escape(status)
) + self._test_name_and_phase_regex(test_name, phase)
if xfail == "no":
# There should be no other text after the testname and phase regex
expected += r" *$"
elif xfail == "expected":
expected += r" *{}".format(
re.escape(test_status.TEST_EXPECTED_FAILURE_COMMENT)
)
elif xfail == "unexpected":
expected += r" *{}".format(
re.escape(test_status.TEST_UNEXPECTED_FAILURE_COMMENT_START)
)
else:
expect(xfail is None, "Unhandled value of xfail argument")
expected_re = re.compile(expected, flags=re.MULTILINE)
self.assertRegex(output, expected_re)
[docs]
def assert_phase_absent(self, output, phase, test_name):
"""Asserts that 'output' does not contain a status line for the
given phase and test_name"""
expected = re.compile(
r"^.* +" + self._test_name_and_phase_regex(test_name, phase),
flags=re.MULTILINE,
)
self.assertNotRegex(output, expected)
[docs]
def assert_core_phases(self, output, test_name, fails):
"""Asserts that 'output' contains a line for each of the core test
phases for the given test_name. All results should be PASS
except those given by the fails list, which should be FAILS.
"""
for phase in test_status.CORE_PHASES:
if phase in fails:
status = test_status.TEST_FAIL_STATUS
else:
status = test_status.TEST_PASS_STATUS
self.assert_status_of_phase(
output=output, status=status, phase=phase, test_name=test_name
)
[docs]
def assert_num_expected_unexpected_fails(
self, output, num_expected, num_unexpected
):
"""Asserts that the number of occurrences of expected and unexpected fails in
'output' matches the given numbers"""
self.assertEqual(
output.count(test_status.TEST_EXPECTED_FAILURE_COMMENT), num_expected
)
self.assertEqual(
output.count(test_status.TEST_UNEXPECTED_FAILURE_COMMENT_START),
num_unexpected,
)
@staticmethod
def _test_name_and_phase_regex(test_name, phase):
"""Returns a regex matching the portion of a TestStatus line
containing the test name and phase"""
# The main purpose of extracting this into a shared method is:
# assert_phase_absent could wrongly pass if the format of the
# TestStatus output changed without that method's regex
# changing. By making its regex shared as much as possible with
# the regex in assert_status_of_phase, we decrease the chances
# of these false passes.
return r"{} +{}".format(re.escape(test_name), re.escape(phase))