Source code for bugzoo.core.test

__all__ = ['TestCase', 'TestSuite', 'TestOutcome']

from typing import Sequence, Dict, Iterator, Optional, List, Any
import attr
import warnings

from ..cmd import ExecResponse

[docs]@attr.s(frozen=True) class TestCaseOracle(object): """ Used to determine whether the outcome of a test case execution should be considered as a success or failure. """ code = attr.ib(type=int, default=0) output_contains = attr.ib(type=Optional[str], default=None) @staticmethod def from_dict(d: dict) -> 'TestCaseOracle': code = d.get('code', 0) if not isinstance(code, int): raise ValueError("failed to unpack test oracle: expected 'code' to be an int.") # noqa: pycodestyle if 'output' in d and 'contains' in d['output']: output_contains = d['output']['contains'] if not isinstance(output_contains, str): raise ValueError("failed to unpack test oracle: expected 'contains' to be a string.") # noqa: pycodestyle if output_contains == '': raise ValueError("failed to unpack test oracle: 'contains' must be a non-empty string.") # noqa: pycodestyle else: output_contains = None return TestCaseOracle(code, output_contains) def to_dict(self) -> Dict[str, Any]: d = {} # type: Dict[str, Any] d['code'] = self.code if self.output_contains is not None: d['output'] = {'contains': self.output_contains} return d
[docs] def check(self, response: ExecResponse) -> bool: """ Determines whether the raw command output from a test execution satisfies this oracle. """ if response.code != self.code: return False if self.output_contains and self.output_contains not in response.output: # noqa: pycodestyle return False return True
[docs]@attr.s(frozen=True) class TestCase(object): """ Describes an individual test case for a particular snapshot. """ name = attr.ib(type=str) time_limit = attr.ib(type=int) command = attr.ib(type=str) context = attr.ib(type=str) expected_outcome = attr.ib(type=Optional[bool]) oracle = attr.ib(type=TestCaseOracle) kill_after = attr.ib(type=int, default=1) def to_dict(self) -> Dict[str, Any]: return {'name':, 'time-limit': self.time_limit, 'command': self.command, 'context': self.context, 'expected-outcome': self.expected_outcome, 'oracle':, 'kill-after': self.kill_after}
[docs]@attr.s(frozen=True) class TestOutcome(object): """ Describes the outcome of a test execution. """ response = attr.ib(type=ExecResponse) passed = attr.ib(type=bool) @staticmethod def from_dict(d: dict) -> 'TestOutcome': response = ExecResponse.from_dict(d['response']) passed = d['passed'] return TestOutcome(response, passed) @property def duration(self) -> float: """ The duration of the test execution, measured in seconds. """ return self.response.duration def to_dict(self) -> dict: return {'passed': self.passed, 'response': self.response.to_dict()}
[docs]@attr.s(frozen=True) class TestSuite(object): """ Describes the test suite for a particular snapshot. Test suites are composed of a set of uniquely named individual test cases. """ _tests = attr.ib(type=Dict[str, TestCase], converter=lambda ts: { t for t in ts}) # type: ignore @staticmethod def from_dict(d: dict) -> 'TestSuite': command_template = d.get('command', './ __ID__') # type: str default_time_limit = d.get('time-limit', 60) # type: int command_context = d.get('context', '/experiment') # type: str default_oracle = TestCaseOracle() default_kill_after = 1 d_tests = d.get('tests', []) if d.get('type') == 'empty': return TestSuite([]) if d.get('type') == 'genprog': for i in range(1, d['failing'] + 1): d_tests.append({'name': "n{}".format(i), 'expected-outcome': False}) for i in range(1, d['passing'] + 1): d_tests.append({'name': "p{}".format(i), 'expected-outcome': True}) # build the tests tests = [] # type: List[TestCase] for test_desc in d_tests: test_oracle = default_oracle # type: TestCaseOracle test_kill_after = 1 # type: int # FIXME add expected outcome if isinstance(test_desc, str): test_name = test_desc test_time_limit = default_time_limit test_context = command_context test_command = \ command_template.replace('__ID__', test_name) test_expected_outcome = None elif isinstance(test_desc, dict): if 'name' in test_desc: test_name = test_desc['name'] else: test_name = test_desc['command'] if 'command' in test_desc: test_command = test_desc['command'] else: test_command = \ command_template.replace('__ID__', test_name) if 'oracle' in test_desc: test_oracle = TestCaseOracle.from_dict(test_desc['oracle']) if 'kill-after' in test_desc: test_kill_after = test_desc['kill-after'] test_context = test_desc.get('context', command_context) test_time_limit = \ test_desc.get('time-limit', default_time_limit) test_expected_outcome = test_desc.get('expected-outcome') test = TestCase(name=test_name, time_limit=test_time_limit, command=test_command, context=test_context, expected_outcome=test_expected_outcome, oracle=test_oracle, kill_after=test_kill_after) tests.append(test) return TestSuite(tests) @property def tests(self) -> Iterator[TestCase]: warnings.warn("'tests' property will be removed in v2.2", DeprecationWarning) yield from self._tests.values()
[docs] def __iter__(self) -> Iterator[TestCase]: """ Returns an iterator over the test cases contained within this test suite. """ yield from self._tests.values()
[docs] def __getitem__(self, name: str) -> TestCase: """ Attempts to fetch a test case from this test suite by its name. Raises: KeyError: if no test case with the given name belongs to this test suite. """ return self._tests[name]
def to_dict(self) -> Dict[str, Any]: return {'tests': [t.to_dict() for t in self.tests]}