diff --git a/Dockerfile b/Dockerfile index 42fed87..c307a5a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,20 +5,19 @@ LABEL org.opencontainers.image.source /github.com/TranslatorSRI/TestHarness WORKDIR /app -# make sure all is writeable for the nru USER later on -RUN chmod -R 777 . - # set up requirements COPY requirements.txt . COPY requirements-runners.txt . RUN pip install -r requirements.txt RUN pip install -r requirements-runners.txt -# switch to the non-root user (nru). defined in the base image -USER nru - # set up source COPY . . -# set up entrypoint -ENTRYPOINT ["./main.sh"] \ No newline at end of file +# make sure all is writeable for the nru USER later on +RUN chmod -R 777 . + +RUN pip install . + +# switch to the non-root user (nru). defined in the base image +USER nru diff --git a/logs/.gitkeep b/logs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/requirements-runners.txt b/requirements-runners.txt index fcb1920..ab9aa83 100644 --- a/requirements-runners.txt +++ b/requirements-runners.txt @@ -1,2 +1,3 @@ -ARS_Test_Runner==0.0.2 -ui-test-runner==0.0.1 +ARS_Test_Runner==0.0.8 +benchmarks-runner==0.1.0 +ui-test-runner==0.0.2 diff --git a/requirements-test.txt b/requirements-test.txt index dcef8f5..29d5702 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,2 +1,3 @@ pytest==7.4.2 +pytest-asyncio==0.23.3 pytest-mock==3.11.1 diff --git a/requirements.txt b/requirements.txt index 173bd1e..c3b6f80 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,5 @@ httpx==0.25.0 pydantic==1.10.13 +reasoner_pydantic==4.1.6 +tqdm==4.66.1 +translator-testing-model==0.2.3 diff --git a/setup.py b/setup.py index 39a2548..3e1dd15 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ """Setup file for SRI Test Harness package.""" + from setuptools import setup with open("README.md", encoding="utf-8") as readme_file: @@ -6,7 +7,7 @@ setup( name="sri-test-harness", - version="0.0.1", + version="0.1.0", author="Max Wang", author_email="max@covar.com", url="https://github.com/TranslatorSRI/TestHarness", diff --git a/test_harness/download.py b/test_harness/download.py index 7088268..637beab 100644 --- a/test_harness/download.py +++ b/test_harness/download.py @@ -1,4 +1,5 @@ """Download tests.""" + import glob import httpx import io @@ -6,15 +7,17 @@ import logging from pathlib import Path import tempfile -from typing import List, Union +from typing import List, Union, Dict import zipfile -from .models import TestCase, TestSuite +from translator_testing_model.datamodel.pydanticmodel import TestCase, TestSuite def download_tests( - suite: Union[str, List[str]], url: Path, logger: logging.Logger -) -> List[TestCase]: + suite: Union[str, List[str]], + url: Path, + logger: logging.Logger, +) -> Dict[str, TestCase]: """Download tests from specified location.""" assert Path(url).suffix == ".zip" logger.info(f"Downloading tests from {url}...") @@ -28,34 +31,57 @@ def download_tests( zip_ref.extractall(tmpdir) # Find all json files in the downloaded zip - tests_paths = glob.glob(f"{tmpdir}/**/*.json", recursive=True) + # tests_paths = glob.glob(f"{tmpdir}/**/*.json", recursive=True) + + tests_paths = glob.glob(f"{tmpdir}/*/test_suites/test_suite_output.json") + + with open(tests_paths[0]) as f: + test_suite = TestSuite.parse_obj(json.load(f)) - all_tests = [] - suites = suite if type(suite) == list else [suite] - test_case_ids = [] + # all_tests = [] + # suites = suite if type(suite) == list else [suite] + # test_case_ids = [] - logger.info(f"Reading in {len(tests_paths)} tests...") + # logger.info(f"Reading in {len(test_suite.test_cases)} tests...") # do the reading of the tests and make a tests list - for test_path in tests_paths: - with open(test_path, "r") as f: - test_json = json.load(f) - try: - test_suite = TestSuite.parse_obj(test_json) - if test_suite.id in suites: - # if suite is selected, grab all its test cases - test_case_ids.extend(test_suite.case_ids) - except Exception as e: - # not a Test Suite - pass - try: - test_case = TestCase.parse_obj(test_json) - all_tests.append(test_case) - except Exception as e: - # not a Test Case - pass + # for test_case in test_suite.test_cases: + # try: + # test_suite = TestSuite.parse_obj(test_json) + # if test_suite.id in suites: + # if test_json["test_case_type"] == "acceptance": + # # if suite is selected, grab all its test cases + # # test_case_ids.extend(test_suite.case_ids) + # all_tests.append(test_json) + # continue + # if test_json.get("test_env"): + # # only grab Test Cases and not Test Assets + # all_tests.append(test_json) + # except Exception as e: + # # not a Test Suite + # pass + # try: + # # test_case = TestCase.parse_obj(test_json) + # if test_json["test_case_type"] == "quantitative": + # all_tests.append(test_json) + # continue + # # all_tests.append(test_json) + # except Exception as e: + # # not a Test Case + # print(e) + # pass # only return the tests from the specified suites - tests = list(filter(lambda x: x in test_case_ids, all_tests)) - logger.info(f"Passing along {len(tests)} tests") - return tests + # tests = list(filter(lambda x: x in test_case_ids, all_tests)) + # tests = [ + # test + # for test in all_tests + # for asset in test.test_assets + # if asset.output_id + # ] + # for test in tests: + # test.test_case_type = "acceptance" + # tests = all_tests + # tests = list(filter((lambda x: x for x in all_tests for asset in x.test_assets if asset.output_id), all_tests)) + logger.info(f"Passing along {len(test_suite.test_cases)} tests") + return test_suite.test_cases diff --git a/test_harness/logging.py b/test_harness/logger.py similarity index 96% rename from test_harness/logging.py rename to test_harness/logger.py index e721b1a..3f49b8d 100644 --- a/test_harness/logging.py +++ b/test_harness/logger.py @@ -100,3 +100,5 @@ def setup_logger(): logging.getLogger("httpcore").setLevel(logging.WARNING) logging.getLogger("httpx").setLevel(logging.WARNING) logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("matplotlib").setLevel(logging.WARNING) + logging.getLogger("root").setLevel(logging.WARNING) diff --git a/test_harness/main.py b/test_harness/main.py index 5feac59..9c7bc11 100644 --- a/test_harness/main.py +++ b/test_harness/main.py @@ -1,12 +1,16 @@ """Translator SRI Automated Test Harness.""" + from argparse import ArgumentParser +import asyncio import json from urllib.parse import urlparse from uuid import uuid4 -from .run import run_tests -from .download import download_tests -from .logging import get_logger, setup_logger +from test_harness.run import run_tests +from test_harness.download import download_tests +from test_harness.logger import get_logger, setup_logger +from test_harness.reporter import Reporter +from test_harness.slacker import Slacker setup_logger() @@ -18,7 +22,7 @@ def url_type(arg): raise TypeError("Invalid URL") -def main(args): +async def main(args): """Main Test Harness entrypoint.""" qid = str(uuid4())[:8] logger = get_logger(qid, args["log_level"]) @@ -32,18 +36,29 @@ def main(args): "Please run this command with `-h` to see the available options." ) - report = run_tests(tests, logger) + if len(tests) < 1: + return logger.warning("No tests to run. Exiting.") + + # Create test run in the Information Radiator + reporter = Reporter( + base_url=args.get("reporter_url"), + refresh_token=args.get("reporter_access_token"), + logger=logger, + ) + await reporter.get_auth() + await reporter.create_test_run() + slacker = Slacker() + report = await run_tests(reporter, slacker, tests, logger) - if args["save_to_dashboard"]: - logger.info("Saving to Testing Dashboard...") - raise NotImplementedError() + logger.info("Finishing up test run...") + await reporter.finish_test_run() if args["json_output"]: - logger.info("Saving report as JSON...") + # logger.info("Saving report as JSON...") with open("test_report.json", "w") as f: json.dump(report, f) - logger.info("All testing has completed!") + return logger.info("All tests have completed!") def cli(): @@ -78,6 +93,18 @@ def cli(): help="Path to a file of tests to be run. This would be the same output from downloading the tests via `download_tests()`", ) + parser.add_argument( + "--reporter_url", + type=url_type, + help="URL of the Testing Dashboard", + ) + + parser.add_argument( + "--reporter_access_token", + type=str, + help="Access token for authentication with the Testing Dashboard", + ) + parser.add_argument( "--save_to_dashboard", action="store_true", @@ -95,11 +122,11 @@ def cli(): type=str, choices=["ERROR", "WARNING", "INFO", "DEBUG"], help="Level of the logs.", - default="WARNING", + default="DEBUG", ) args = parser.parse_args() - main(vars(args)) + asyncio.run(main(vars(args))) if __name__ == "__main__": diff --git a/test_harness/models.py b/test_harness/models.py deleted file mode 100644 index 8c49bab..0000000 --- a/test_harness/models.py +++ /dev/null @@ -1,83 +0,0 @@ -from enum import Enum -from pydantic import BaseModel -from typing import List, Optional - - -class Type(str, Enum): - acceptance = "acceptance" - quantitative = "quantitative" - - -class Env(str, Enum): - dev = "dev" - ci = "ci" - test = "test" - prod = "prod" - - -class QueryType(str, Enum): - treats = "treats(creative)" - - -class ExpectedOutput(str, Enum): - top_answer = "TopAnswer" - acceptable = "Acceptable" - bad_but_forgivable = "BadButForgivable" - never_show = "NeverShow" - - -class TestCase(BaseModel): - """ - Test Case that Test Runners can ingest. - - type: Type - env: Env - query_type: QueryType - expected_output: ExpectedOutput - input_curie: str - output_curie: str - """ - - id: Optional[int] - type: Type - env: Env - query_type: QueryType - expected_output: ExpectedOutput - input_curie: str - output_curie: str - - -class Tests(BaseModel): - """List of Test Cases.""" - - __root__: List[TestCase] - - def __len__(self): - return len(self.__root__) - - def __iter__(self): - return self.__root__.__iter__() - - def __contains__(self, v): - return self.__root__.__contains__(v) - - def __getitem__(self, i): - return self.__root__.__getitem__(i) - - -class TestSuite(BaseModel): - """ - Test Suite containing the ids of Test Cases. - - id: int - case_ids: List[int] - """ - - id: int - case_ids: List[int] - - -class TestResult(BaseModel): - """Output of a Test.""" - - status: str diff --git a/test_harness/reporter.py b/test_harness/reporter.py new file mode 100644 index 0000000..a398f00 --- /dev/null +++ b/test_harness/reporter.py @@ -0,0 +1,175 @@ +"""Information Radiator Reporter.""" + +from datetime import datetime +import httpx +import logging +import os +from typing import List + +from translator_testing_model.datamodel.pydanticmodel import TestCase, TestAsset + + +class Reporter: + """Reports tests and statuses to the Information Radiator.""" + + def __init__( + self, + base_url=None, + refresh_token=None, + logger: logging.Logger = logging.getLogger(), + ): + self.base_path = base_url if base_url else os.getenv("ZE_BASE_URL") + self.refresh_token = ( + refresh_token if refresh_token else os.getenv("ZE_REFRESH_TOKEN") + ) + self.authenticated_client = None + self.test_run_id = None + self.logger = logger + + async def get_auth(self): + """Get access token for subsequent calls.""" + async with httpx.AsyncClient() as client: + res = await client.post( + url=f"{self.base_path}/api/iam/v1/auth/refresh", + json={ + "refreshToken": self.refresh_token, + }, + ) + res.raise_for_status() + auth_response = res.json() + self.authenticated_client = httpx.AsyncClient( + headers={ + "Authorization": f"Bearer {auth_response['authToken']}", + } + ) + + async def create_test_run(self): + """Create a test run in the IR.""" + res = await self.authenticated_client.post( + url=f"{self.base_path}/api/reporting/v1/test-runs", + json={ + "name": f"Test Harness Automated Tests: {datetime.now().strftime('%Y_%m_%d_%H_%M')}", + "startedAt": datetime.now().astimezone().isoformat(), + "framework": "Translator Automated Testing", + }, + ) + res.raise_for_status() + res_json = res.json() + self.test_run_id = res_json["id"] + return self.test_run_id + + async def create_test(self, test: TestCase, asset: TestAsset): + """Create a test in the IR.""" + name = f"{asset.name if asset.name else asset.description}" + res = await self.authenticated_client.post( + url=f"{self.base_path}/api/reporting/v1/test-runs/{self.test_run_id}/tests", + json={ + "name": name, + "className": test.name, + "methodName": asset.name, + "startedAt": datetime.now().astimezone().isoformat(), + "labels": [ + { + "key": "TestCase", + "value": test.id, + }, + { + "key": "TestAsset", + "value": asset.id, + }, + { + "key": "Environment", + "value": test.test_env, + }, + ], + }, + ) + res.raise_for_status() + res_json = res.json() + return res_json["id"] + + async def upload_labels(self, test_id: int, labels: List[dict]): + """Upload labels to the IR.""" + self.logger.info(labels) + res = await self.authenticated_client.put( + url=f"{self.base_path}/api/reporting/v1/test-runs/{self.test_run_id}/tests/{test_id}/labels", + json={ + "items": labels, + }, + ) + res.raise_for_status() + + async def upload_logs(self, test_id: int, logs: List[str]): + """Upload logs to the IR.""" + res = await self.authenticated_client.post( + url=f"{self.base_path}/api/reporting/v1/test-runs/{self.test_run_id}/logs", + json=[ + { + "testId": f"{test_id}", + "level": "INFO", + "timestamp": datetime.now().timestamp(), + "message": message, + } + for message in logs + ], + ) + res.raise_for_status() + + async def upload_artifact_references(self, test_id, artifact_references): + """Upload artifact references to the IR.""" + res = await self.authenticated_client.put( + url=f"{self.base_path}/api/reporting/v1/test-runs/{self.test_run_id}/tests/{test_id}/artifact-references", + json=artifact_references, + ) + res.raise_for_status() + + async def upload_screenshot(self, test_id, screenshot): + """Upload screenshots to the IR.""" + res = await self.authenticated_client.post( + url=f"{self.base_path}/api/reporting/v1/test-runs/{self.test_run_id}/tests/{test_id}/screenshots", + headers={ + "Content-Type": "image/png", + }, + data=screenshot, + ) + res.raise_for_status() + + async def upload_log(self, test_id, message): + """Upload logs to the IR.""" + res = await self.authenticated_client.post( + url=f"{self.base_path}/api/reporting/v1/test-runs/{self.test_run_id}/logs", + json=[ + { + "testId": f"{test_id}", + "level": "INFO", + "timestamp": datetime.now().timestamp(), + "message": message, + }, + ], + ) + res.raise_for_status() + + async def finish_test(self, test_id, result): + """Set the final status of a test.""" + res = await self.authenticated_client.put( + url=f"{self.base_path}/api/reporting/v1/test-runs/{self.test_run_id}/tests/{test_id}", + json={ + "result": result, + "endedAt": datetime.now().astimezone().isoformat(), + }, + ) + res.raise_for_status() + res_json = res.json() + return res_json["result"] + + async def finish_test_run(self): + """Set the final status of a test run.""" + res = await self.authenticated_client.put( + url=f"{self.base_path}/api/reporting/v1/test-runs/{self.test_run_id}", + json={ + "endedAt": datetime.now().astimezone().isoformat(), + }, + ) + res.raise_for_status() + res_json = res.json() + return res_json["status"] diff --git a/test_harness/run.py b/test_harness/run.py index e078be4..7b5ec6f 100644 --- a/test_harness/run.py +++ b/test_harness/run.py @@ -1,52 +1,195 @@ """Run tests through the Test Runners.""" + +from collections import defaultdict +import json import logging +import time from tqdm import tqdm -from typing import Dict +import traceback +from typing import Dict, List -from ui_test_runner import run_ui_test from ARS_Test_Runner.semantic_test import run_semantic_test as run_ars_test +from benchmarks_runner import run_benchmarks + +from translator_testing_model.datamodel.pydanticmodel import TestCase -from .models import Tests +from .reporter import Reporter +from .slacker import Slacker -def run_tests(tests: Tests, logger: logging.Logger) -> Dict: +async def run_tests( + reporter: Reporter, + slacker: Slacker, + tests: Dict[str, TestCase], + logger: logging.Logger = logging.getLogger(__name__), +) -> Dict: """Send tests through the Test Runners.""" - tests = Tests.parse_obj(tests) + start_time = time.time() logger.info(f"Running {len(tests)} tests...") - full_report = {} + full_report = { + "PASSED": 0, + "FAILED": 0, + "SKIPPED": 0, + } + await slacker.post_notification( + messages=[ + f"Running {len(tests)} tests...\n<{reporter.base_path}/test-runs/{reporter.test_run_id}|View in the Information Radiator>" + ] + ) # loop over all tests - for test in tqdm(tests): + for test in tqdm(tests.values()): + status = "PASSED" # check if acceptance test - if test.type == "acceptance": - full_report[test.input_curie] = {} + if not test.test_assets or not test.test_case_objective: + logger.warning(f"Test has missing required fields: {test.id}") + continue + if test.test_case_objective == "AcceptanceTest": + assets = test.test_assets + test_ids = [] + err_msg = "" + for asset in assets: + # create test in Test Dashboard + test_id = "" + try: + test_id = await reporter.create_test(test, asset) + test_ids.append(test_id) + except Exception: + logger.error(f"Failed to create test: {test.id}") + try: + test_input = json.dumps( + { + # "environment": test.test_env, + "environment": "test", + "predicate": test.test_case_predicate_name, + "runner_settings": test.test_case_runner_settings, + "expected_output": asset.expected_output, + "input_curie": test.test_case_input_id, + "output_curie": asset.output_id, + }, + indent=2, + ) + await reporter.upload_log( + test_id, + "Calling ARS Test Runner with: {test_input}".format( + test_input=test_input + ), + ) + except Exception as e: + logger.error(str(e)) + logger.error(f"Failed to upload logs to test: {test.id}, {test_id}") + + # group all outputs together to make one Translator query + output_ids = [asset.output_id for asset in assets] + expected_outputs = [asset.expected_output for asset in assets] + test_inputs = [ + # test.test_env, + "test", + test.test_case_predicate_name, + test.test_case_runner_settings, + expected_outputs, + test.test_case_input_id, + output_ids, + ] try: - ui_result = run_ui_test( - test.env, - test.query_type, - test.expected_output, - test.input_curie, - test.output_curie, - ) - full_report[test.input_curie]["ui"] = ui_result + ars_result = await run_ars_test(*test_inputs) except Exception as e: - logger.error(f"UI test failed with {e}") - full_report[test.input_curie]["ui"] = {"error": str(e)} + err_msg = f"ARS Test Runner failed with {traceback.format_exc()}" + logger.error(f"[{test.id}] {err_msg}") + ars_result = { + "pks": {}, + # this will effectively act as a list that we access by index down below + "results": defaultdict(lambda: {"error": err_msg}), + } + # full_report[test["test_case_input_id"]]["ars"] = {"error": str(e)} + # grab individual results for each asset + for index, (test_id, asset) in enumerate(zip(test_ids, assets)): + test_result = { + "pks": ars_result["pks"], + "result": ars_result["results"][index], + } + # grab only ars result if it exists, otherwise default to failed + status = test_result["result"].get("ars", {}).get("status", "FAILED") + full_report[status] += 1 + if not err_msg: + # only upload ara labels if the test ran successfully + try: + labels = [ + { + "key": ara, + "value": result["status"], + } + for ara, result in test_result["result"].items() + ] + await reporter.upload_labels(test_id, labels) + except Exception as e: + logger.warning(f"[{test.id}] failed to upload labels: {e}") + try: + await reporter.upload_log( + test_id, json.dumps(test_result, indent=4) + ) + except Exception as e: + logger.error(f"[{test.id}] failed to upload logs.") + try: + await reporter.finish_test(test_id, status) + except Exception as e: + logger.error(f"[{test.id}] failed to upload finished status.") + # full_report[test["test_case_input_id"]]["ars"] = ars_result + elif test["test_case_objective"] == "QuantitativeTest": + assets = test.test_assets[0] try: - ars_result = run_ars_test( - test.env, - test.query_type, - test.expected_output, - test.input_curie, - test.output_curie, + test_id = await reporter.create_test(test, assets) + except Exception: + logger.error(f"Failed to create test: {test.id}") + continue + try: + test_inputs = [ + assets.id, + # TODO: update this. Assumes is going to be ARS + test.components[0], + ] + await reporter.upload_log( + test_id, + f"Calling Benchmark Test Runner with: {json.dumps(test_inputs, indent=4)}", ) - full_report[test.input_curie]["ars"] = ars_result + benchmark_results, screenshots = await run_benchmarks(*test_inputs) + await reporter.upload_log(test_id, ("\n").join(benchmark_results)) + for screenshot in screenshots.values(): + await reporter.upload_screenshot(test_id, screenshot) + await reporter.finish_test(test_id, "PASSED") + full_report["PASSED"] += 1 except Exception as e: - logger.error(f"ARS test failed with {e}") - full_report[test.input_curie]["ars"] = {"error": str(e)} - elif test.type == "quantitative": - # implement the Benchmark Runner - logger.warning("Quantitative tests are not supported yet.") + logger.error(f"Benchmarks failed with {e}: {traceback.format_exc()}") + full_report["FAILED"] += 1 + try: + await reporter.upload_log(test_id, traceback.format_exc()) + except Exception: + logger.error( + f"Failed to upload fail logs for test {test_id}: {traceback.format_exc()}" + ) + await reporter.finish_test(test_id, "FAILED") else: - logger.warning(f"Unsupported test type: {test.type}") + try: + test_id = await reporter.create_test(test, test) + logger.error(f"Unsupported test type: {test.id}") + await reporter.upload_log( + test_id, f"Unsupported test type in test: {test.id}" + ) + status = "FAILED" + await reporter.finish_test(test_id, status) + except Exception: + logger.error(f"Failed to report errors with: {test.id}") + await slacker.post_notification( + messages=[ + """Test Suite: {test_suite_id}\nDuration: {duration} | Environment: {env}\n<{ir_url}|View in the Information Radiator>\n> Test Results:\n> Passed: {num_passed}, Failed: {num_failed}, Skipped: {num_skipped}""".format( + test_suite_id=1, + duration=round(time.time() - start_time, 2), + env="ci", + ir_url=f"{reporter.base_path}/test-runs/{reporter.test_run_id}", + num_passed=full_report["PASSED"], + num_failed=full_report["FAILED"], + num_skipped=full_report["SKIPPED"], + ) + ] + ) return full_report diff --git a/test_harness/slacker.py b/test_harness/slacker.py new file mode 100644 index 0000000..1fa9e65 --- /dev/null +++ b/test_harness/slacker.py @@ -0,0 +1,32 @@ +import httpx +import os + + +class Slacker: + """Slack notification poster.""" + + def __init__(self, url=None): + self.url = url if url else os.getenv("SLACK_WEBHOOK_URL") + + async def post_notification(self, messages=[]): + """Post a notification to Slack.""" + # https://gist.github.com/mrjk/079b745c4a8a118df756b127d6499aa0 + blocks = [] + for message in messages: + blocks.append( + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": str(message), + }, + } + ) + async with httpx.AsyncClient() as client: + res = await client.post( + url=self.url, + json={ + "text": ", ".join(block["text"]["text"] for block in blocks), + "blocks": blocks, + }, + ) diff --git a/tests/example_tests.py b/tests/example_tests.py index 7129c88..16e6578 100644 --- a/tests/example_tests.py +++ b/tests/example_tests.py @@ -1,12 +1,109 @@ """Example tests for the Test Harness.""" -example_tests = [ +from translator_testing_model.datamodel.pydanticmodel import TestSuite + +example_test_cases = TestSuite.parse_obj( { - "type": "acceptance", - "env": "test", - "query_type": "treats(creative)", - "expected_output": "TopAnswer", - "input_curie": "MONDO:0015564", - "output_curie": "PUBCHEM.COMPOUND:5284616", + "id": "TestSuite_1", + "name": None, + "description": None, + "tags": [], + "test_metadata": { + "id": "1", + "name": None, + "description": None, + "tags": [], + "test_source": "SMURF", + "test_reference": None, + "test_objective": "AcceptanceTest", + "test_annotations": [], + }, + "test_cases": { + "TestCase_1": { + "id": "TestCase_1", + "name": "what treats MONDO:0010794", + "description": "Valproic_Acid_treats_NARP_Syndrome; Barbiturates_treats_NARP_Syndrome", + "tags": [], + "test_env": "ci", + "query_type": None, + "test_assets": [ + { + "id": "Asset_3", + "name": "Valproic_Acid_treats_NARP_Syndrome", + "description": "Valproic_Acid_treats_NARP_Syndrome", + "tags": [], + "input_id": "MONDO:0010794", + "input_name": "NARP Syndrome", + "input_category": None, + "predicate_id": "biolink:treats", + "predicate_name": "treats", + "output_id": "DRUGBANK:DB00313", + "output_name": "Valproic Acid", + "output_category": None, + "association": None, + "qualifiers": [], + "expected_output": "NeverShow", + "test_issue": None, + "semantic_severity": None, + "in_v1": None, + "well_known": False, + "test_reference": None, + "runner_settings": ["inferred"], + "test_metadata": { + "id": "1", + "name": None, + "description": None, + "tags": [], + "test_source": "SMURF", + "test_reference": "https://github.com/NCATSTranslator/Feedback/issues/147", + "test_objective": "AcceptanceTest", + "test_annotations": [], + }, + }, + { + "id": "Asset_4", + "name": "Barbiturates_treats_NARP_Syndrome", + "description": "Barbiturates_treats_NARP_Syndrome", + "tags": [], + "input_id": "MONDO:0010794", + "input_name": "NARP Syndrome", + "input_category": None, + "predicate_id": "biolink:treats", + "predicate_name": "treats", + "output_id": "MESH:D001463", + "output_name": "Barbiturates", + "output_category": None, + "association": None, + "qualifiers": [], + "expected_output": "NeverShow", + "test_issue": None, + "semantic_severity": None, + "in_v1": None, + "well_known": False, + "test_reference": None, + "runner_settings": ["inferred"], + "test_metadata": { + "id": "1", + "name": None, + "description": None, + "tags": [], + "test_source": "SMURF", + "test_reference": "https://github.com/NCATSTranslator/Feedback/issues/147", + "test_objective": "AcceptanceTest", + "test_annotations": [], + }, + }, + ], + "preconditions": [], + "trapi_template": None, + "components": ["ars"], + "test_case_objective": "AcceptanceTest", + "test_case_source": None, + "test_case_predicate_name": "treats", + "test_case_predicate_id": "biolink:treats", + "test_case_input_id": "MONDO:0010794", + "test_case_runner_settings": ["inferred"], + } + }, } -] +).test_cases diff --git a/tests/mocker.py b/tests/mocker.py new file mode 100644 index 0000000..87210b2 --- /dev/null +++ b/tests/mocker.py @@ -0,0 +1,44 @@ +class MockReporter: + def __init__(self, base_url=None, refresh_token=None, logger=None): + self.base_path = base_url + self.test_run_id = 1 + pass + + async def get_auth(self): + pass + + async def create_test_run(self): + return 1 + + async def create_test(self, test, asset): + return 2 + + async def upload_labels(self, test_id, labels): + pass + + async def upload_logs(self, test_id, logs): + pass + + async def upload_artifact_references(self, test_id, artifact_references): + pass + + async def upload_screenshots(self, test_id, screenshot): + pass + + async def upload_log(self, test_id, message): + pass + + async def finish_test(self, test_id, result): + return result + + async def finish_test_run(self): + pass + + +class MockSlacker: + def __init__(self): + pass + + async def post_notification(self, messages): + print(f"posting messages: {messages}") + pass diff --git a/tests/test_main.py b/tests/test_main.py index c85938b..d8d0ee0 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,21 +1,29 @@ import pytest from test_harness.main import main -from .example_tests import example_tests +from .example_tests import example_test_cases +from .mocker import ( + MockReporter, + MockSlacker, +) -def test_main(mocker): + +@pytest.mark.asyncio +async def test_main(mocker): """Test the main function.""" # This article is awesome: https://nedbatchelder.com/blog/201908/why_your_mock_doesnt_work.html - run_ui_test = mocker.patch("test_harness.run.run_ui_test", return_value="Pass") run_ars_test = mocker.patch("test_harness.run.run_ars_test", return_value="Fail") - main( + run_tests = mocker.patch("test_harness.main.run_tests", return_value={}) + mocker.patch("test_harness.slacker.Slacker", return_value=MockSlacker()) + mocker.patch("test_harness.main.Reporter", return_value=MockReporter()) + await main( { - "tests": example_tests, + "tests": example_test_cases, "save_to_dashboard": False, "json_output": False, "log_level": "ERROR", } ) - run_ui_test.assert_called_once() - run_ars_test.assert_called_once() + # run_ui_test.assert_called_once() + run_tests.assert_called_once() diff --git a/tests/test_run.py b/tests/test_run.py new file mode 100644 index 0000000..2f2751e --- /dev/null +++ b/tests/test_run.py @@ -0,0 +1,48 @@ +import pytest + +from test_harness.run import run_tests +from .example_tests import example_test_cases + +from .mocker import ( + MockReporter, + MockSlacker, +) + + +@pytest.mark.asyncio +async def test_run_tests(mocker): + """Test the run_tests function.""" + # This article is awesome: https://nedbatchelder.com/blog/201908/why_your_mock_doesnt_work.html + run_ars_test = mocker.patch( + "test_harness.run.run_ars_test", + return_value={ + "pks": { + "parent_pk": "123abc", + "merged_pk": "456def", + }, + "results": [ + { + "ars": { + "status": "PASSED", + }, + }, + { + "ars": { + "status": "FAILED", + "message": "", + }, + }, + ], + }, + ) + run_benchmarks = mocker.patch("test_harness.run.run_benchmarks", return_value={}) + await run_tests( + reporter=MockReporter( + base_url="http://test", + ), + slacker=MockSlacker(), + tests=example_test_cases, + ) + # run_ui_test.assert_called_once() + run_ars_test.assert_called_once() + run_benchmarks.assert_not_called()