From bcccc8a3f6898189d7ddb92a94183a3872a76ee3 Mon Sep 17 00:00:00 2001 From: Lars Falk-Petersen Date: Fri, 27 Sep 2024 15:28:26 +0200 Subject: [PATCH] Able to run a few checks in cli hook mode: test_edr_landingpage, test_edr_conformance --- .vscode/launch.json | 10 ++++- run.sh | 8 +++- sedr/edreq11.py | 6 +-- sedr/hooks.py | 101 ++++++++++++++++++++++++++++++++++++++------ 4 files changed, 106 insertions(+), 19 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 972a17b..740639f 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -10,9 +10,15 @@ "request": "launch", "program": "venv/bin/st", "console": "integratedTerminal", - "args": ["run", "--experimental=openapi-3.1", "https://edrisobaric.k8s.met.no/api"], + "args": ["run", + // "--show-trace", + "--experimental=openapi-3.1", + "--checks", "all", + "--workers", "2", + // "--include-path-regex", "'/collections$'", + "--hypothesis-suppress-health-check=too_slow", + "https://edrisobaric.k8s.met.no/api"], "env": {"SCHEMATHESIS_HOOKS": "sedr.hooks"} - } ] } \ No newline at end of file diff --git a/run.sh b/run.sh index 60f8aa4..8cd4606 100644 --- a/run.sh +++ b/run.sh @@ -2,4 +2,10 @@ # export PYTHONPATH=$(pwd) export SCHEMATHESIS_HOOKS=sedr.hooks -st run --experimental=openapi-3.1 https://edrisobaric.k8s.met.no/api +st run \ + --experimental=openapi-3.1 \ + --checks all \ + --workers 8 \ + --include-path-regex '/collections$' \ + --hypothesis-suppress-health-check=too_slow \ + https://edrisobaric.k8s.met.no/api diff --git a/sedr/edreq11.py b/sedr/edreq11.py index e0995af..c7ac446 100644 --- a/sedr/edreq11.py +++ b/sedr/edreq11.py @@ -1,6 +1,6 @@ """EDR requirements.""" -import util +import sedr.util as util conformance_urls = [ "http://www.opengis.net/spec/ogcapi-edr-1/1.1/conf/core", @@ -44,7 +44,7 @@ def requirementA11_1(jsondata: str) -> tuple[bool, str]: for url in jsondata: if url in openapi_conformance_urls: - if ( + if util.args and ( util.args.openapi_version == "3.1" and "oas31" in url or util.args.openapi_version == "3.0" @@ -53,7 +53,7 @@ def requirementA11_1(jsondata: str) -> tuple[bool, str]: return True, url return ( False, - f"OpenAPI version {util.args.openapi_version} and version in conformance {url} doesn't match. See <{spec_url}> for more info.", + f"OpenAPI version {util.args.openapi_version if util.args else "unknown"} and version in conformance {url} doesn't match. See <{spec_url}> for more info.", ) return ( diff --git a/sedr/hooks.py b/sedr/hooks.py index 212185e..75e1186 100644 --- a/sedr/hooks.py +++ b/sedr/hooks.py @@ -6,20 +6,95 @@ from schemathesis.runner import events from typing import List import pprint +import json +import sedr.util as util +import sedr.edreq11 as edreq -class SimpleHandler(EventHandler): - def handle_event(self, context, event): - # pprint.pp(event) - if isinstance(event, events.Finished): - print("Done!") - pprint.pp(event) +@schemathesis.hook +def after_call(context, case, response): + """Hook runs after any call to the API.""" + if response.request: + # Log calls with status + # print( + # f"after_call {'OK' if response.ok else 'ERR'} " + # + f"{response.request.path_url} {response.text[0:150]}" + # ) + if response.request.path_url == "/": + # test_edr_landingpage + """Test that the landing page contains required elements.""" + spec_ref = "https://docs.ogc.org/is/19-072/19-072.html#_7c772474-7037-41c9-88ca-5c7e95235389" + try: + landingpage_json = response.json() + landing, landing_message = util.parse_landing_json(landingpage_json) + if not landing: + raise AssertionError( + f"Landing page is missing required elements. See <{spec_ref}> for more info. {landing_message}" + ) -@schemathesis.hook -def after_init_cli_run_handlers( - context: HookContext, - handlers: List[EventHandler], - execution_context: ExecutionContext, -) -> None: - handlers[:] = [SimpleHandler()] + print("Landingpage %s tested OK", response.url) + except json.decoder.JSONDecodeError as e: + print( + f"Landing page is not valid JSON, other formats are not tested yet., {e}" + ) + + + if response.request.path_url == "/conformance": + # def test_edr_conformance(case): + """Test /conformance endpoint.""" + conformance_json = response.json() + + if "conformsTo" not in conformance_json: + spec_ref = "https://docs.ogc.org/is/19-072/19-072.html#_4129e3d3-9428-4e91-9bfc-645405ed2369" + raise AssertionError( + f"Conformance page /conformance does not contain a conformsTo attribute. See {spec_ref} for more info." + ) + + resolves, resolves_message = util.test_conformance_links( + jsondata=conformance_json["conformsTo"] + ) + if not resolves: + raise AssertionError(resolves_message) + + requirementA2_2_A5, requirementA2_2_A5_message = edreq.requirementA2_2_A5( + jsondata=conformance_json["conformsTo"] + ) + if not requirementA2_2_A5: + raise AssertionError(requirementA2_2_A5_message) + + requirementA2_2_A7, requirementA2_2_A7_message = edreq.requirementA2_2_A7( + response.raw.version + ) + if not requirementA2_2_A7: + raise AssertionError(requirementA2_2_A7_message) + + requirementA11_1, requirementA11_1_message = edreq.requirementA11_1( + jsondata=conformance_json["conformsTo"] + ) + if not requirementA11_1: + raise AssertionError(requirementA11_1_message) + + print("Conformance %s tested OK", response.url) + + +# class SimpleHandler(EventHandler): +# def handle_event(self, context, event): +# print(event.__class__) + +# if isinstance(event, events.AfterProbing): +# print("AfterProbing event:") +# pprint.pp(event) + +# if isinstance(event, events.Finished): +# print("Finished event:") +# pprint.pp(event) + + +# @schemathesis.hook +# def after_init_cli_run_handlers( +# context: HookContext, +# handlers: List[EventHandler], +# execution_context: ExecutionContext, +# ) -> None: +# handlers[:] = [SimpleHandler()]