Add e2e tests; fix identified problems

- move scripts/demo.py to examples/demo.py - add tests/* with pytest scripts - fix ArtifactEmitter to take optionality into account; fix low-level model Diagnosis.message to have the correct ty.Optional annotation - export the JSON type alias from ocptv.output since it's useful in tests Signed-off-by: mimir-d <mimir-d@users.noreply.github.com>
opencomputeproject · Jan 18, 2023 · d2c7c89 · d2c7c89
1 parent d6b6d7d
commit d2c7c89
Show file tree

Hide file tree

Showing 10 changed files with 255 additions and 10 deletions.
diff --git a/apis/python/scripts/demo.py → apis/python/examples/demo.py b/apis/python/scripts/demo.py → apis/python/examples/demo.py
diff --git a/apis/python/ocptv/objects.py b/apis/python/ocptv/objects.py
@@ -91,8 +91,7 @@ class Diagnosis:
         },
     )
 
-    message: str = dc.field(
-        default="",
+    message: ty.Optional[str] = dc.field(
         metadata={"spec_field": "message"},
     )
 

diff --git a/apis/python/ocptv/output.py b/apis/python/ocptv/output.py
@@ -7,7 +7,6 @@
 import dataclasses as dc
 import json
 import typing as ty
-from numbers import Number
 from enum import Enum
 
 from .objects import ArtifactType, Root, SchemaVersion, RootArtifactType
@@ -38,7 +37,8 @@ def configOutput(writer: Writer):
     _writer = writer
 
 
-_JSON = dict[str, "_JSON"] | list["_JSON"] | Number | str | None
+Primitive = float | int | bool | str | None
+JSON = dict[str, "JSON"] | list["JSON"] | Primitive
 
 
 class ArtifactEmitter:
@@ -48,14 +48,28 @@ def __init__(self):
 
     @staticmethod
     def _serialize(artifact: ArtifactType):
+        def is_optional(field: ty.Type):
+            # type hackery incoming
+            # ty.Optional[T] == ty.Union[T, None]
+            # since ty.Union[ty.Union[T,U]] = ty.Union[T,U] we can the
+            # following transitiveness to check that type is optional
+            return field == ty.Optional[field]
+
         def visit(
-            value: ArtifactType | dict | list | Number | str,
+            value: ArtifactType | dict | list | Primitive,
             formatter: ty.Optional[ty.Callable[[ty.Any], str]] = None,
-        ) -> _JSON:
+        ) -> JSON:
             if dc.is_dataclass(value):
-                obj: dict[str, _JSON] = {}
+                obj: dict[str, JSON] = {}
                 for field in dc.fields(value):
                     val = getattr(value, field.name)
+
+                    if val is None:
+                        if not is_optional(field.type):
+                            # TODO: fix exception text/type
+                            raise RuntimeError("unacceptable none where not optional")
+                        continue
+
                     spec_field: ty.Optional[str] = field.metadata.get(
                         "spec_field", None
                     )
@@ -78,7 +92,7 @@ def visit(
                 return [visit(k) for k in value]
             elif isinstance(value, dict):
                 return {k: visit(v) for k, v in value.items()}
-            elif isinstance(value, (str, Number, Enum)):
+            elif isinstance(value, (str, float, int, bool, Enum)):
                 # primitive types get here
                 if formatter is not None:
                     return formatter(value)

diff --git a/apis/python/ocptv/run.py b/apis/python/ocptv/run.py
@@ -50,7 +50,7 @@ def __init__(
         else:
             self._cmdline = self._get_cmdline()
 
-        self._params = parameters
+        self._params = parameters or {}
         self._emitter = ArtifactEmitter()
 
         self._stepId = 0
@@ -70,6 +70,7 @@ def start(self, *, duts: ty.Optional[list[Dut]] = None):
             name=self.name,
             version=self._version,
             command_line=self.command_line,
+            parameters=self._params,
             dut_info=[x.info for x in duts],
         )
         self._emitter.emit(RunArtifact(impl=start))

diff --git a/apis/python/ocptv/step.py b/apis/python/ocptv/step.py
@@ -65,7 +65,7 @@ def add_diagnosis(
         diagnosis_type: DiagnosisType,
         *,
         verdict: str,
-        message: str = "",
+        message: ty.Optional[str] = None,
         hardware_info_id: ty.Optional[str] = None,
         subcomponent=None,
     ):

diff --git a/apis/python/pyproject.toml b/apis/python/pyproject.toml
@@ -0,0 +1,4 @@
+[tool.pytest.ini_options]
+pythonpath = [
+    "."
+]
diff --git a/apis/python/tests/__init__.py b/apis/python/tests/__init__.py
diff --git a/apis/python/tests/mocks.py b/apis/python/tests/mocks.py
@@ -0,0 +1,28 @@
+import json
+
+from ocptv.output import Writer, JSON
+
+
+class MockWriter(Writer):
+    def __init__(self):
+        self.lines: list[str] = []
+
+    def write(self, buffer: str):
+        self.lines.append(buffer)
+
+    def decoded_obj(self, index: int) -> dict[str, JSON]:
+        """Decode an expected object from given output index"""
+        return json.loads(self.lines[index])
+
+
+def assert_json(line: str, expected: JSON):
+    actual = json.loads(line)
+
+    # remove timestamps, cant easily compare
+    if isinstance(expected, dict) and "timestamp" in actual:
+        del actual["timestamp"]
+
+    if isinstance(expected, dict) and "timestamp" in expected:
+        del expected["timestamp"]
+
+    assert actual == expected
diff --git a/apis/python/tests/test_runs.py b/apis/python/tests/test_runs.py
@@ -0,0 +1,170 @@
+import pytest
+import typing as ty
+
+import ocptv
+from ocptv import LogSeverity, TestStatus, TestResult, DiagnosisType
+from ocptv.output import JSON
+
+TestStatus.__test__ = False  # type: ignore
+TestResult.__test__ = False  # type: ignore
+
+from .mocks import MockWriter, assert_json
+
+
+@pytest.fixture
+def writer():
+    w = MockWriter()
+    ocptv.configOutput(w)
+    return w
+
+
+def test_simple_run(writer: MockWriter):
+    run = ocptv.TestRun(
+        name="test",
+        version="1.0",
+        command_line="cl",
+        parameters={"param": "test"},
+    )
+    run.start()
+    run.end(status=TestStatus.COMPLETE, result=TestResult.PASS)
+
+    assert len(writer.lines) == 3
+    assert_json(
+        writer.lines[1],
+        {
+            "testRunArtifact": {
+                "testRunStart": {
+                    "name": "test",
+                    "version": "1.0",
+                    "commandLine": "cl",
+                    "parameters": {
+                        "param": "test",
+                    },
+                    "dutInfo": [],
+                },
+            },
+            "sequenceNumber": 1,
+        },
+    )
+
+    assert_json(
+        writer.lines[2],
+        {
+            "testRunArtifact": {
+                "testRunEnd": {
+                    "status": "COMPLETE",
+                    "result": "PASS",
+                },
+            },
+            "sequenceNumber": 2,
+        },
+    )
+
+
+def test_run_scope(writer: MockWriter):
+    run = ocptv.TestRun(name="test", version="1.0")
+    with run.scope():
+        pass
+
+    assert len(writer.lines) == 3
+    assert_json(
+        writer.lines[2],
+        {
+            "testRunArtifact": {
+                "testRunEnd": {
+                    "status": "COMPLETE",
+                    "result": "PASS",
+                },
+            },
+            "sequenceNumber": 2,
+        },
+    )
+
+
+def test_run_skip_by_exception(writer: MockWriter):
+    run = ocptv.TestRun(name="run_skip", version="1.0")
+    with run.scope():
+        raise ocptv.TestRunError(
+            status=TestStatus.SKIP, result=TestResult.NOT_APPLICABLE
+        )
+
+    assert len(writer.lines) == 3
+    assert_json(
+        writer.lines[2],
+        {
+            "testRunArtifact": {
+                "testRunEnd": {
+                    "status": "SKIP",
+                    "result": "NOT_APPLICABLE",
+                },
+            },
+            "sequenceNumber": 2,
+        },
+    )
+
+
+def test_run_with_diagnosis(writer: MockWriter):
+    class Verdict:
+        PASS = "pass-default"
+
+    run = ocptv.TestRun(name="run_with_diagnosis", version="1.0")
+    with run.scope():
+        run.add_log(LogSeverity.INFO, "run info")
+        step = run.add_step("step0")
+        with step.scope():
+            step.add_diagnosis(DiagnosisType.PASS, verdict=Verdict.PASS)
+
+    assert len(writer.lines) == 7
+    assert "schemaVersion" in writer.decoded_obj(0)
+
+    assert "testRunArtifact" in writer.decoded_obj(1)
+    artifact = ty.cast(dict[str, JSON], writer.decoded_obj(1)["testRunArtifact"])
+    assert "testRunStart" in artifact
+
+    assert "testRunArtifact" in writer.decoded_obj(2)
+    artifact = ty.cast(dict[str, JSON], writer.decoded_obj(2)["testRunArtifact"])
+    assert "log" in artifact
+
+    assert_json(
+        writer.lines[3],
+        {
+            "testStepArtifact": {
+                "testStepStart": {
+                    "name": "step0",
+                },
+                "testStepId": "0",
+            },
+            "sequenceNumber": 3,
+        },
+    )
+
+    assert_json(
+        writer.lines[4],
+        {
+            "testStepArtifact": {
+                "diagnosis": {
+                    "type": "PASS",
+                    "verdict": Verdict.PASS,
+                },
+                "testStepId": "0",
+            },
+            "sequenceNumber": 4,
+        },
+    )
+
+    assert_json(
+        writer.lines[5],
+        {
+            "testStepArtifact": {
+                "testStepEnd": {
+                    "status": "COMPLETE",
+                },
+                "testStepId": "0",
+            },
+            "sequenceNumber": 5,
+        },
+    )
+
+    assert "testRunArtifact" in writer.decoded_obj(6)
+    artifact = ty.cast(dict[str, JSON], writer.decoded_obj(6)["testRunArtifact"])
+    assert "testRunEnd" in artifact
diff --git a/apis/python/tests/test_schema_version.py b/apis/python/tests/test_schema_version.py
@@ -0,0 +1,29 @@
+import pytest
+
+import ocptv
+
+from .mocks import MockWriter, assert_json
+
+
+@pytest.fixture
+def writer():
+    w = MockWriter()
+    ocptv.configOutput(w)
+    return w
+
+
+def test_schema_version_is_emitted(writer: MockWriter):
+    run = ocptv.TestRun(name="test", version="1.0")
+    run.start()
+
+    assert len(writer.lines) == 2
+    assert_json(
+        writer.lines[0],
+        {
+            "schemaVersion": {
+                "major": ocptv.OCPVersion.VERSION_2_0.value[0],
+                "minor": ocptv.OCPVersion.VERSION_2_0.value[1],
+            },
+            "sequenceNumber": 0,
+        },
+    )