Merge pull request #6 from TranslatorSRI/test_env

Use specified test case environments and move label up to run
TranslatorSRI · Feb 14, 2024 · 15b1cf3 · 15b1cf3
2 parents 4e7af9d + 45a7453
commit 15b1cf3
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 11 deletions.
diff --git a/setup.py b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name="sri-test-harness",
-    version="0.1.0",
+    version="0.1.1",
     author="Max Wang",
     author_email="max@covar.com",
     url="https://github.com/TranslatorSRI/TestHarness",

diff --git a/test_harness/main.py b/test_harness/main.py
@@ -46,7 +46,7 @@ async def main(args):
         logger=logger,
     )
     await reporter.get_auth()
-    await reporter.create_test_run()
+    await reporter.create_test_run(next(iter(tests.values())))
     slacker = Slacker()
     report = await run_tests(reporter, slacker, tests, logger)
 

diff --git a/test_harness/reporter.py b/test_harness/reporter.py
@@ -43,14 +43,17 @@ async def get_auth(self):
             }
         )
 
-    async def create_test_run(self):
+    async def create_test_run(self, test: TestCase):
         """Create a test run in the IR."""
         res = await self.authenticated_client.post(
             url=f"{self.base_path}/api/reporting/v1/test-runs",
             json={
                 "name": f"Test Harness Automated Tests: {datetime.now().strftime('%Y_%m_%d_%H_%M')}",
                 "startedAt": datetime.now().astimezone().isoformat(),
                 "framework": "Translator Automated Testing",
+                "config": {
+                    "environment": test.test_env,
+                },
             },
         )
         res.raise_for_status()
@@ -77,10 +80,6 @@ async def create_test(self, test: TestCase, asset: TestAsset):
                         "key": "TestAsset",
                         "value": asset.id,
                     },
-                    {
-                        "key": "Environment",
-                        "value": test.test_env,
-                    },
                 ],
             },
         )

diff --git a/test_harness/run.py b/test_harness/run.py
@@ -58,8 +58,7 @@ async def run_tests(
                 try:
                     test_input = json.dumps(
                         {
-                            # "environment": test.test_env,
-                            "environment": "test",
+                            "environment": test.test_env,
                             "predicate": test.test_case_predicate_name,
                             "runner_settings": test.test_case_runner_settings,
                             "expected_output": asset.expected_output,
@@ -82,8 +81,7 @@ async def run_tests(
             output_ids = [asset.output_id for asset in assets]
             expected_outputs = [asset.expected_output for asset in assets]
             test_inputs = [
-                # test.test_env,
-                "test",
+                test.test_env,
                 test.test_case_predicate_name,
                 test.test_case_runner_settings,
                 expected_outputs,