From fd5e9b745436b06f45e99b1e60847ee4033d8973 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Wed, 5 Feb 2025 10:04:30 -0500 Subject: [PATCH 01/21] feat: add basic endpoint to allow url checking OCD-4765 --- .../OnDemandUrlCheckerController.java | 45 +++++++++++ .../healthit/chpl/datadog/OnDemandUrl.java | 18 +++++ .../datadog/OnDemandUrlCheckerManager.java | 74 +++++++++++++++++++ .../DatadogSyntheticsTestResultService.java | 22 +++++- .../DatadogSyntheticsTestService.java | 6 +- 5 files changed, 161 insertions(+), 4 deletions(-) create mode 100644 chpl/chpl-api/src/main/java/gov/healthit/chpl/web/controller/OnDemandUrlCheckerController.java create mode 100644 chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrl.java create mode 100644 chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java diff --git a/chpl/chpl-api/src/main/java/gov/healthit/chpl/web/controller/OnDemandUrlCheckerController.java b/chpl/chpl-api/src/main/java/gov/healthit/chpl/web/controller/OnDemandUrlCheckerController.java new file mode 100644 index 0000000000..d560f1dee1 --- /dev/null +++ b/chpl/chpl-api/src/main/java/gov/healthit/chpl/web/controller/OnDemandUrlCheckerController.java @@ -0,0 +1,45 @@ +package gov.healthit.chpl.web.controller; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import com.datadog.api.client.ApiException; +import com.datadog.api.client.v1.model.SyntheticsAPITestResultFull; + +import gov.healthit.chpl.datadog.OnDemandUrl; +import gov.healthit.chpl.datadog.OnDemandUrlCheckerManager; +import gov.healthit.chpl.util.SwaggerSecurityRequirement; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.extern.log4j.Log4j2; + +@Log4j2 +@Tag(name = "urls", description = "") +@RestController +@RequestMapping("/urls") +public class OnDemandUrlCheckerController { + + private OnDemandUrlCheckerManager onDemandUrlCheckerManager; + + @Autowired + public OnDemandUrlCheckerController(OnDemandUrlCheckerManager onDemandUrlCheckerManager) { + this.onDemandUrlCheckerManager = onDemandUrlCheckerManager; + } + + @Operation(summary = "", + description = "Security Restrictions: ROLE_ADMIN or ROLE_ONC", + security = { + @SecurityRequirement(name = SwaggerSecurityRequirement.API_KEY), + @SecurityRequirement(name = SwaggerSecurityRequirement.BEARER) + }) + @RequestMapping(value = "", method = RequestMethod.POST, consumes = MediaType.APPLICATION_JSON_VALUE, produces = "application/json; charset=utf-8") + public SyntheticsAPITestResultFull checkUrl(@RequestBody OnDemandUrl url) throws InterruptedException, ApiException { + return onDemandUrlCheckerManager.checkUrl(url.getUrl()); + } + +} diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrl.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrl.java new file mode 100644 index 0000000000..6e9dc8de34 --- /dev/null +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrl.java @@ -0,0 +1,18 @@ +package gov.healthit.chpl.datadog; + +import java.io.Serializable; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class OnDemandUrl implements Serializable { + private static final long serialVersionUID = -3009297190983937267L; + + private String url; +} diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java new file mode 100644 index 0000000000..feb1d098cb --- /dev/null +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java @@ -0,0 +1,74 @@ +package gov.healthit.chpl.datadog; + +import java.util.Collections; +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.datadog.api.client.ApiException; +import com.datadog.api.client.v1.model.SyntheticsAPITest; +import com.datadog.api.client.v1.model.SyntheticsAPITestResultFull; +import com.datadog.api.client.v1.model.SyntheticsGetAPITestLatestResultsResponse; +import com.datadog.api.client.v1.model.SyntheticsTriggerBody; +import com.datadog.api.client.v1.model.SyntheticsTriggerTest; + +import gov.healthit.chpl.scheduler.job.urluptime.DatadogSyntheticsTestResultService; +import gov.healthit.chpl.scheduler.job.urluptime.DatadogSyntheticsTestService; +import lombok.extern.log4j.Log4j2; + +@Log4j2 +@Component +public class OnDemandUrlCheckerManager { + private static final Long TEMP_DEVELOPER_ID = -99L; + + private DatadogSyntheticsTestService datadogSyntheticsTestService; + private DatadogSyntheticsTestResultService datadogSyntheticsTestResultService; + + @Autowired + public OnDemandUrlCheckerManager(DatadogSyntheticsTestService datadogSyntheticsTestService, DatadogSyntheticsTestResultService datadogSyntheticsTestResultService) { + this.datadogSyntheticsTestService = datadogSyntheticsTestService; + this.datadogSyntheticsTestResultService = datadogSyntheticsTestResultService; + } + + public SyntheticsAPITestResultFull checkUrl(String url) throws InterruptedException, ApiException { + + SyntheticsAPITest test = datadogSyntheticsTestService.createSyntheticsTest(url, List.of(TEMP_DEVELOPER_ID)); + + LOGGER.info(test.getPublicId()); + + SyntheticsTriggerBody body = new SyntheticsTriggerBody() + .tests( + Collections.singletonList( + new SyntheticsTriggerTest().publicId(test.getPublicId()))); + + datadogSyntheticsTestService.getApiProvider().getApiInstance().triggerTests(body); + + SyntheticsGetAPITestLatestResultsResponse result = null; + Integer attempts = 0; + while ((result == null || result.getResults().size() == 0) && attempts < 45) { + Thread.sleep(1000); + attempts++; + LOGGER.info("Attempt: {}", attempts); + result = datadogSyntheticsTestResultService.getSyntheticsTestResults(test.getPublicId()); + LOGGER.info("Result: {}", result); + } + + SyntheticsAPITestResultFull fullTestResults = null; + if (result != null && result.getResults().size() > 0) { + LOGGER.info("Getting detailed results"); + fullTestResults = datadogSyntheticsTestResultService.getDetailedTestResult(test.getPublicId(), result.getResults().get(0).getResultId()); + LOGGER.info("Detailed results: {}", fullTestResults.getResult().getHttpStatusCode()); + LOGGER.info("Deleting test"); + datadogSyntheticsTestService.deleteSyntheticsTests(List.of(test.getPublicId())); + } else { + LOGGER.info("Deleting test"); + datadogSyntheticsTestService.deleteSyntheticsTests(List.of(test.getPublicId())); + LOGGER.info("No results found"); + throw new ApiException("No results found for test " + test.getPublicId()); + } + + return fullTestResults; + } + +} diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogSyntheticsTestResultService.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogSyntheticsTestResultService.java index aeb79db028..c7b4f9a45b 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogSyntheticsTestResultService.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogSyntheticsTestResultService.java @@ -24,7 +24,6 @@ public class DatadogSyntheticsTestResultService { private DatadogSyntheticsTestApiProvider apiProvider; - @Autowired public DatadogSyntheticsTestResultService(DatadogSyntheticsTestApiProvider apiProvider) { this.apiProvider = apiProvider; @@ -58,6 +57,27 @@ public List getSyntheticsTestResults(String public return testResults; } + public SyntheticsGetAPITestLatestResultsResponse getSyntheticsTestResults(String publicTestKey) { + // SyntheticsGetAPITestLatestResultsResponse response; + // List testResults = new + // ArrayList(); + try { + return apiProvider.getApiInstance().getAPITestLatestResults(publicTestKey); + // while (response.getResults().size() > 1) { + // testResults.addAll(response.getResults()); + // Long ts = getMostRecentTimestamp(response.getResults()); + // response = + // apiProvider.getApiInstance().getAPITestLatestResults(publicTestKey); + // } + // LOGGER.info("Found {} tests for monitor {}", testResults.size(), + // publicTestKey); + + } catch (ApiException e) { + LOGGER.error("Could not retrieve results for test key: {}", publicTestKey, e); + return null; + } + } + public SyntheticsAPITestResultFull getDetailedTestResult(String publicTestKey, String resultId) { try { return apiProvider.getApiInstance().getAPITestResult(publicTestKey, resultId); diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogSyntheticsTestService.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogSyntheticsTestService.java index befdbe58fa..a4da1782e4 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogSyntheticsTestService.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogSyntheticsTestService.java @@ -61,7 +61,7 @@ public DatadogSyntheticsTestService(DatadogSyntheticsTestApiProvider apiProvider this.datadogTestLocation = datadogTestLocation; } - protected DatadogSyntheticsTestApiProvider getApiProvider() { + public DatadogSyntheticsTestApiProvider getApiProvider() { return apiProvider; } @@ -114,8 +114,8 @@ public SyntheticsAPITest createSyntheticsTest(String url, List developerId .target(NOT_EMPTY_REGEX) .type(SyntheticsAssertionType.BODY)))) .request(new SyntheticsTestRequest() - .url(url) - .method(HTTP_METHOD_GET))) + .url(url) + .method(HTTP_METHOD_GET))) .options(new SyntheticsTestOptions() .httpVersion(SyntheticsTestOptionsHTTPVersion.ANY) .minFailureDuration(0L) From 4bf502bb5cb8663d4186e6a6d93e813569a48ba2 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Thu, 6 Feb 2025 18:38:25 -0500 Subject: [PATCH 02/21] feat: convert results into more usable format OCD-4765 --- .../OnDemandUrlCheckerController.java | 6 +-- .../OnDemandUrlCheckerAssertionResult.java | 11 ++++ .../datadog/OnDemandUrlCheckerManager.java | 50 +++++++++++++++---- .../datadog/OnDemandUrlCheckerResponse.java | 13 +++++ ...DemandUrl.java => OnDemandUrlRequest.java} | 2 +- 5 files changed, 69 insertions(+), 13 deletions(-) create mode 100644 chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerAssertionResult.java create mode 100644 chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerResponse.java rename chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/{OnDemandUrl.java => OnDemandUrlRequest.java} (85%) diff --git a/chpl/chpl-api/src/main/java/gov/healthit/chpl/web/controller/OnDemandUrlCheckerController.java b/chpl/chpl-api/src/main/java/gov/healthit/chpl/web/controller/OnDemandUrlCheckerController.java index d560f1dee1..d9b37ea45f 100644 --- a/chpl/chpl-api/src/main/java/gov/healthit/chpl/web/controller/OnDemandUrlCheckerController.java +++ b/chpl/chpl-api/src/main/java/gov/healthit/chpl/web/controller/OnDemandUrlCheckerController.java @@ -8,10 +8,10 @@ import org.springframework.web.bind.annotation.RestController; import com.datadog.api.client.ApiException; -import com.datadog.api.client.v1.model.SyntheticsAPITestResultFull; -import gov.healthit.chpl.datadog.OnDemandUrl; import gov.healthit.chpl.datadog.OnDemandUrlCheckerManager; +import gov.healthit.chpl.datadog.OnDemandUrlCheckerResponse; +import gov.healthit.chpl.datadog.OnDemandUrlRequest; import gov.healthit.chpl.util.SwaggerSecurityRequirement; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.security.SecurityRequirement; @@ -38,7 +38,7 @@ public OnDemandUrlCheckerController(OnDemandUrlCheckerManager onDemandUrlChecker @SecurityRequirement(name = SwaggerSecurityRequirement.BEARER) }) @RequestMapping(value = "", method = RequestMethod.POST, consumes = MediaType.APPLICATION_JSON_VALUE, produces = "application/json; charset=utf-8") - public SyntheticsAPITestResultFull checkUrl(@RequestBody OnDemandUrl url) throws InterruptedException, ApiException { + public OnDemandUrlCheckerResponse checkUrl(@RequestBody OnDemandUrlRequest url) throws InterruptedException, ApiException { return onDemandUrlCheckerManager.checkUrl(url.getUrl()); } diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerAssertionResult.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerAssertionResult.java new file mode 100644 index 0000000000..b5a3e1cccf --- /dev/null +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerAssertionResult.java @@ -0,0 +1,11 @@ +package gov.healthit.chpl.datadog; + +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class OnDemandUrlCheckerAssertionResult { + private Boolean passed; + private String actualValue; +} diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java index feb1d098cb..1a5bfc947e 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java @@ -2,6 +2,7 @@ import java.util.Collections; import java.util.List; +import java.util.Map; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -21,6 +22,13 @@ @Component public class OnDemandUrlCheckerManager { private static final Long TEMP_DEVELOPER_ID = -99L; + private static final String ASSERTION_RESULTS_KEY = "assertionResults"; + private static final String TYPE_KEY = "type"; + private static final String ACTUAL_KEY = "actual"; + private static final String VALID_KEY = "valid"; + private static final String TYPE_VALUE_STATUS_CODE = "statusCode"; + private static final String TYPE_VALUE_BODY = "body"; + private static final String TYPE_VALUE_RESPONSE_TIME = "responseTime"; private DatadogSyntheticsTestService datadogSyntheticsTestService; private DatadogSyntheticsTestResultService datadogSyntheticsTestResultService; @@ -31,7 +39,7 @@ public OnDemandUrlCheckerManager(DatadogSyntheticsTestService datadogSyntheticsT this.datadogSyntheticsTestResultService = datadogSyntheticsTestResultService; } - public SyntheticsAPITestResultFull checkUrl(String url) throws InterruptedException, ApiException { + public OnDemandUrlCheckerResponse checkUrl(String url) throws InterruptedException, ApiException { SyntheticsAPITest test = datadogSyntheticsTestService.createSyntheticsTest(url, List.of(TEMP_DEVELOPER_ID)); @@ -49,26 +57,50 @@ public SyntheticsAPITestResultFull checkUrl(String url) throws InterruptedExcept while ((result == null || result.getResults().size() == 0) && attempts < 45) { Thread.sleep(1000); attempts++; - LOGGER.info("Attempt: {}", attempts); result = datadogSyntheticsTestResultService.getSyntheticsTestResults(test.getPublicId()); - LOGGER.info("Result: {}", result); } SyntheticsAPITestResultFull fullTestResults = null; if (result != null && result.getResults().size() > 0) { - LOGGER.info("Getting detailed results"); fullTestResults = datadogSyntheticsTestResultService.getDetailedTestResult(test.getPublicId(), result.getResults().get(0).getResultId()); - LOGGER.info("Detailed results: {}", fullTestResults.getResult().getHttpStatusCode()); - LOGGER.info("Deleting test"); datadogSyntheticsTestService.deleteSyntheticsTests(List.of(test.getPublicId())); } else { - LOGGER.info("Deleting test"); datadogSyntheticsTestService.deleteSyntheticsTests(List.of(test.getPublicId())); - LOGGER.info("No results found"); throw new ApiException("No results found for test " + test.getPublicId()); } + LOGGER.info("Results: " + fullTestResults.toString()); + return convertToResponse(url, fullTestResults); + } - return fullTestResults; + private OnDemandUrlCheckerResponse convertToResponse(String url, SyntheticsAPITestResultFull fullTestResults) { + return OnDemandUrlCheckerResponse.builder() + .httpResponseAssertion(getAssertionResult(fullTestResults.getResult().getAdditionalProperties(), TYPE_VALUE_STATUS_CODE)) + .responseTimeAssertion(getAssertionResult(fullTestResults.getResult().getAdditionalProperties(), TYPE_VALUE_RESPONSE_TIME)) + .bodyNotEmptyAssertion(getAssertionResult(fullTestResults.getResult().getAdditionalProperties(), TYPE_VALUE_BODY)) + .url(url) + .build(); } + private OnDemandUrlCheckerAssertionResult getAssertionResult(Map results, String value) { + if (results.containsKey(ASSERTION_RESULTS_KEY) + && results.get(ASSERTION_RESULTS_KEY) instanceof List) { + + return ((List) results.get(ASSERTION_RESULTS_KEY)).stream() + .filter(map -> map instanceof Map) + .filter(map -> ((Map) map).containsKey(TYPE_KEY) + && ((Map) map).containsKey(VALID_KEY) + && ((String) ((Map) map).get(TYPE_KEY)).equals(value)) + + .findAny() + .map(map -> OnDemandUrlCheckerAssertionResult.builder() + .passed((Boolean) ((Map) map).get(VALID_KEY)) + .actualValue(((Map) map).get(ACTUAL_KEY).toString()) + .build()) + .orElse(OnDemandUrlCheckerAssertionResult.builder() + .passed(false) + .actualValue("Unknown") + .build()); + } + return null; + } } diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerResponse.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerResponse.java new file mode 100644 index 0000000000..5099b2d047 --- /dev/null +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerResponse.java @@ -0,0 +1,13 @@ +package gov.healthit.chpl.datadog; + +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class OnDemandUrlCheckerResponse { + private String url; + private OnDemandUrlCheckerAssertionResult responseTimeAssertion; + private OnDemandUrlCheckerAssertionResult bodyNotEmptyAssertion; + private OnDemandUrlCheckerAssertionResult httpResponseAssertion; +} diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrl.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlRequest.java similarity index 85% rename from chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrl.java rename to chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlRequest.java index 6e9dc8de34..3d0c0b7ccc 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrl.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlRequest.java @@ -11,7 +11,7 @@ @Builder @AllArgsConstructor @NoArgsConstructor -public class OnDemandUrl implements Serializable { +public class OnDemandUrlRequest implements Serializable { private static final long serialVersionUID = -3009297190983937267L; private String url; From ffd4f5bbf21e83f78bb4f169e9108bed8a0e923a Mon Sep 17 00:00:00 2001 From: Todd Young Date: Tue, 11 Feb 2025 10:04:23 -0500 Subject: [PATCH 03/21] feat: add handling for use cases where the URL is not valid OCD-4765 --- .../datadog/OnDemandUrlCheckerManager.java | 39 +++++++++++++------ .../datadog/OnDemandUrlCheckerResponse.java | 2 + 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java index 1a5bfc947e..bc6f44454a 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java @@ -61,24 +61,36 @@ public OnDemandUrlCheckerResponse checkUrl(String url) throws InterruptedExcepti } SyntheticsAPITestResultFull fullTestResults = null; - if (result != null && result.getResults().size() > 0) { + OnDemandUrlCheckerResponse response = null; + if (result != null + && result.getResults().size() > 0) { fullTestResults = datadogSyntheticsTestResultService.getDetailedTestResult(test.getPublicId(), result.getResults().get(0).getResultId()); - datadogSyntheticsTestService.deleteSyntheticsTests(List.of(test.getPublicId())); + response = convertToResponse(url, fullTestResults); + response.setPassed(result.getResults().get(0).getResult().getPassed()); + if (!result.getResults().get(0).getResult().getPassed()) { + response.setErrorMessage(fullTestResults.getResult().getFailure().getMessage()); + } } else { - datadogSyntheticsTestService.deleteSyntheticsTests(List.of(test.getPublicId())); throw new ApiException("No results found for test " + test.getPublicId()); } + datadogSyntheticsTestService.deleteSyntheticsTests(List.of(test.getPublicId())); LOGGER.info("Results: " + fullTestResults.toString()); - return convertToResponse(url, fullTestResults); + return response; } private OnDemandUrlCheckerResponse convertToResponse(String url, SyntheticsAPITestResultFull fullTestResults) { - return OnDemandUrlCheckerResponse.builder() - .httpResponseAssertion(getAssertionResult(fullTestResults.getResult().getAdditionalProperties(), TYPE_VALUE_STATUS_CODE)) - .responseTimeAssertion(getAssertionResult(fullTestResults.getResult().getAdditionalProperties(), TYPE_VALUE_RESPONSE_TIME)) - .bodyNotEmptyAssertion(getAssertionResult(fullTestResults.getResult().getAdditionalProperties(), TYPE_VALUE_BODY)) - .url(url) - .build(); + if (doAdditionalPropertiesExist(fullTestResults)) { + return OnDemandUrlCheckerResponse.builder() + .httpResponseAssertion(getAssertionResult(fullTestResults.getResult().getAdditionalProperties(), TYPE_VALUE_STATUS_CODE)) + .responseTimeAssertion(getAssertionResult(fullTestResults.getResult().getAdditionalProperties(), TYPE_VALUE_RESPONSE_TIME)) + .bodyNotEmptyAssertion(getAssertionResult(fullTestResults.getResult().getAdditionalProperties(), TYPE_VALUE_BODY)) + .url(url) + .build(); + } else { + return OnDemandUrlCheckerResponse.builder() + .url(url) + .build(); + } } private OnDemandUrlCheckerAssertionResult getAssertionResult(Map results, String value) { @@ -94,7 +106,7 @@ private OnDemandUrlCheckerAssertionResult getAssertionResult(Map .findAny() .map(map -> OnDemandUrlCheckerAssertionResult.builder() .passed((Boolean) ((Map) map).get(VALID_KEY)) - .actualValue(((Map) map).get(ACTUAL_KEY).toString()) + .actualValue(((Map) map).containsKey(ACTUAL_KEY) ? ((Map) map).get(ACTUAL_KEY).toString() : "") .build()) .orElse(OnDemandUrlCheckerAssertionResult.builder() .passed(false) @@ -103,4 +115,9 @@ private OnDemandUrlCheckerAssertionResult getAssertionResult(Map } return null; } + + private Boolean doAdditionalPropertiesExist(SyntheticsAPITestResultFull fullTestResults) { + return fullTestResults.getResult().getAdditionalProperties().containsKey(ASSERTION_RESULTS_KEY) + && fullTestResults.getResult().getAdditionalProperties().get(ASSERTION_RESULTS_KEY) instanceof List; + } } diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerResponse.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerResponse.java index 5099b2d047..1fd973b2d9 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerResponse.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerResponse.java @@ -7,6 +7,8 @@ @Builder public class OnDemandUrlCheckerResponse { private String url; + private String errorMessage; + private Boolean passed; private OnDemandUrlCheckerAssertionResult responseTimeAssertion; private OnDemandUrlCheckerAssertionResult bodyNotEmptyAssertion; private OnDemandUrlCheckerAssertionResult httpResponseAssertion; From 5c84e56c4275ece5bdfef223839e4300c24690ec Mon Sep 17 00:00:00 2001 From: Todd Young Date: Wed, 12 Feb 2025 10:13:57 -0500 Subject: [PATCH 04/21] refactor: attempt to remove thread.sleep OCD-4765 --- .../datadog/OnDemandUrlCheckerManager.java | 69 ++++++++++++++----- 1 file changed, 52 insertions(+), 17 deletions(-) diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java index bc6f44454a..64f515aeda 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java @@ -1,10 +1,12 @@ package gov.healthit.chpl.datadog; -import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.stereotype.Component; import com.datadog.api.client.ApiException; @@ -39,33 +41,68 @@ public OnDemandUrlCheckerManager(DatadogSyntheticsTestService datadogSyntheticsT this.datadogSyntheticsTestResultService = datadogSyntheticsTestResultService; } - public OnDemandUrlCheckerResponse checkUrl(String url) throws InterruptedException, ApiException { - - SyntheticsAPITest test = datadogSyntheticsTestService.createSyntheticsTest(url, List.of(TEMP_DEVELOPER_ID)); + @PreAuthorize("@permissions.hasAccess(T(gov.healthit.chpl.permissions.Permissions).URL_CHECKER, " + + "T(gov.healthit.chpl.permissions.domains.UrlCheckerDomainPermissions).CHECK)") + public OnDemandUrlCheckerResponse checkUrl(String url) throws ApiException { + SyntheticsAPITest test = null; + try { + test = createTest(url); + triggerTest(test); + SyntheticsGetAPITestLatestResultsResponse result = awaitTestResults(test); + OnDemandUrlCheckerResponse response = analyzeTestResults(result, test); + return response; + + // Integer attempts = 0; + // while ((result == null || result.getResults().size() == 0) && + // attempts < 45) { + // Thread.sleep(1000); + // attempts++; + // result = + // datadogSyntheticsTestResultService.getSyntheticsTestResults(test.getPublicId()); + // } + + } finally { + if (test != null) { + datadogSyntheticsTestService.deleteSyntheticsTests(List.of(test.getPublicId())); + } + } + } - LOGGER.info(test.getPublicId()); + private SyntheticsAPITest createTest(String url) { + return datadogSyntheticsTestService.createSyntheticsTest(url, List.of(TEMP_DEVELOPER_ID)); + } + private void triggerTest(SyntheticsAPITest test) throws ApiException { SyntheticsTriggerBody body = new SyntheticsTriggerBody() - .tests( - Collections.singletonList( - new SyntheticsTriggerTest().publicId(test.getPublicId()))); + .tests(List.of(new SyntheticsTriggerTest().publicId(test.getPublicId()))); datadogSyntheticsTestService.getApiProvider().getApiInstance().triggerTests(body); + } + + private SyntheticsGetAPITestLatestResultsResponse awaitTestResults(SyntheticsAPITest test) throws ApiException { + SyntheticsGetAPITestLatestResultsResponse result; + CompletableFuture future = CompletableFuture.supplyAsync(() -> { + return datadogSyntheticsTestResultService.getSyntheticsTestResults(test.getPublicId()); + }); - SyntheticsGetAPITestLatestResultsResponse result = null; - Integer attempts = 0; - while ((result == null || result.getResults().size() == 0) && attempts < 45) { - Thread.sleep(1000); - attempts++; - result = datadogSyntheticsTestResultService.getSyntheticsTestResults(test.getPublicId()); + try { + result = future.get(45, TimeUnit.SECONDS); + } catch (Exception e) { + LOGGER.error("Error getting test results: " + e.getMessage()); + throw new ApiException("No results found for test " + test.getPublicId()); + } finally { + datadogSyntheticsTestService.deleteSyntheticsTests(List.of(test.getPublicId())); } + return result; + } + private OnDemandUrlCheckerResponse analyzeTestResults(SyntheticsGetAPITestLatestResultsResponse result, SyntheticsAPITest test) throws ApiException { SyntheticsAPITestResultFull fullTestResults = null; OnDemandUrlCheckerResponse response = null; if (result != null && result.getResults().size() > 0) { fullTestResults = datadogSyntheticsTestResultService.getDetailedTestResult(test.getPublicId(), result.getResults().get(0).getResultId()); - response = convertToResponse(url, fullTestResults); + response = convertToResponse(test.getConfig().getRequest().getUrl(), fullTestResults); response.setPassed(result.getResults().get(0).getResult().getPassed()); if (!result.getResults().get(0).getResult().getPassed()) { response.setErrorMessage(fullTestResults.getResult().getFailure().getMessage()); @@ -73,8 +110,6 @@ public OnDemandUrlCheckerResponse checkUrl(String url) throws InterruptedExcepti } else { throw new ApiException("No results found for test " + test.getPublicId()); } - datadogSyntheticsTestService.deleteSyntheticsTests(List.of(test.getPublicId())); - LOGGER.info("Results: " + fullTestResults.toString()); return response; } From 9130a76dd52c92f1242eddcc5c20b4fd73c9cbe4 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Wed, 12 Feb 2025 10:15:28 -0500 Subject: [PATCH 05/21] feat: add security to url checker OCD-4765 --- .../chpl/permissions/Permissions.java | 6 +++++- .../domains/UrlCheckerDomainPermissions.java | 18 ++++++++++++++++ .../urlchecker/CheckActionPermissions.java | 21 +++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 chpl/chpl-service/src/main/java/gov/healthit/chpl/permissions/domains/UrlCheckerDomainPermissions.java create mode 100644 chpl/chpl-service/src/main/java/gov/healthit/chpl/permissions/domains/testinglab/urlchecker/CheckActionPermissions.java diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/permissions/Permissions.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/permissions/Permissions.java index f766a307c1..d3bf7a2e40 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/permissions/Permissions.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/permissions/Permissions.java @@ -39,6 +39,7 @@ import gov.healthit.chpl.permissions.domains.TestToolDomainPermissions; import gov.healthit.chpl.permissions.domains.TestingLabDomainPermissions; import gov.healthit.chpl.permissions.domains.UcdProcessDomainPermissions; +import gov.healthit.chpl.permissions.domains.UrlCheckerDomainPermissions; import gov.healthit.chpl.permissions.domains.UserPermissionsDomainPermissions; @Component @@ -79,6 +80,7 @@ public class Permissions { public static final String STANDARD = "STANDARD"; public static final String CODE_SET = "CODE_SET"; public static final String API_KEY = "API_KEY"; + public static final String CHECK_URL = "CHECK_URL"; private Map domainPermissions = new HashMap(); @@ -116,7 +118,8 @@ public Permissions(CertificationResultsDomainPermissions certificationResultsDom FunctionalityTestedDomainPermissions functionalityTestedDomainPermissions, StandardDomainPermissions standardDomainPermissions, CodeSetDomainPermissions codeSetPermissions, - ApiKeyDomainPermissions apiKeyPermissions) { + ApiKeyDomainPermissions apiKeyPermissions, + UrlCheckerDomainPermissions urlCheckerDomainPermissions) { domainPermissions.put(ACCESSIBILITY_STANDARD, accessibilityStandardDomainPermissions); domainPermissions.put(ACTIVITY, activityDomainPermissions); @@ -151,6 +154,7 @@ public Permissions(CertificationResultsDomainPermissions certificationResultsDom domainPermissions.put(UCD_PROCESS, ucdProcessDomainPermissions); domainPermissions.put(USER_PERMISSIONS, userPermissionsDomainPermissions); domainPermissions.put(API_KEY, apiKeyPermissions); + domainPermissions.put(CHECK_URL, urlCheckerDomainPermissions); } public boolean hasAccess(final String domain, final String action) { diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/permissions/domains/UrlCheckerDomainPermissions.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/permissions/domains/UrlCheckerDomainPermissions.java new file mode 100644 index 0000000000..c06f4690aa --- /dev/null +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/permissions/domains/UrlCheckerDomainPermissions.java @@ -0,0 +1,18 @@ +package gov.healthit.chpl.permissions.domains; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; + +import gov.healthit.chpl.permissions.domains.testinglab.urlchecker.CheckActionPermissions; + +@Component +public class UrlCheckerDomainPermissions extends DomainPermissions { + public static final String CHECK = "CHECK"; + + @Autowired + public UrlCheckerDomainPermissions( + @Qualifier("urlCheckerCheckActionPermissions") CheckActionPermissions checkActionPermissions) { + getActionPermissions().put(CHECK, checkActionPermissions); + } +} diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/permissions/domains/testinglab/urlchecker/CheckActionPermissions.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/permissions/domains/testinglab/urlchecker/CheckActionPermissions.java new file mode 100644 index 0000000000..ab7109085d --- /dev/null +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/permissions/domains/testinglab/urlchecker/CheckActionPermissions.java @@ -0,0 +1,21 @@ +package gov.healthit.chpl.permissions.domains.testinglab.urlchecker; + +import org.springframework.stereotype.Component; + +import gov.healthit.chpl.permissions.domains.ActionPermissions; + +@Component("urlCheckerCheckActionPermissions") +public class CheckActionPermissions extends ActionPermissions { + + @Override + public boolean hasAccess() { + return getResourcePermissions().isUserRoleAdmin() + || getResourcePermissions().isUserRoleOnc() + || getResourcePermissions().isUserRoleAcbAdmin(); + } + + @Override + public boolean hasAccess(Object obj) { + return false; + } +} From 401d04ff243f3a4a505286ec79994b6638bf04f0 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Wed, 12 Feb 2025 12:00:27 -0500 Subject: [PATCH 06/21] feat: replace thread.sleep with Failsafe library OCD-4765 --- chpl/chpl-service/pom.xml | 7 +++ .../datadog/OnDemandUrlCheckerManager.java | 50 +++++++++++++------ 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/chpl/chpl-service/pom.xml b/chpl/chpl-service/pom.xml index c42a0084b2..3dfb8c6e15 100644 --- a/chpl/chpl-service/pom.xml +++ b/chpl/chpl-service/pom.xml @@ -418,6 +418,13 @@ jwks-rsa 0.4.0 + + + + dev.failsafe + failsafe + 3.3.2 + diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java index 64f515aeda..7ea6e1eb51 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java @@ -1,9 +1,8 @@ package gov.healthit.chpl.datadog; +import java.time.Duration; import java.util.List; import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.access.prepost.PreAuthorize; @@ -16,6 +15,8 @@ import com.datadog.api.client.v1.model.SyntheticsTriggerBody; import com.datadog.api.client.v1.model.SyntheticsTriggerTest; +import dev.failsafe.Failsafe; +import dev.failsafe.RetryPolicy; import gov.healthit.chpl.scheduler.job.urluptime.DatadogSyntheticsTestResultService; import gov.healthit.chpl.scheduler.job.urluptime.DatadogSyntheticsTestService; import lombok.extern.log4j.Log4j2; @@ -80,20 +81,37 @@ private void triggerTest(SyntheticsAPITest test) throws ApiException { } private SyntheticsGetAPITestLatestResultsResponse awaitTestResults(SyntheticsAPITest test) throws ApiException { - SyntheticsGetAPITestLatestResultsResponse result; - CompletableFuture future = CompletableFuture.supplyAsync(() -> { - return datadogSyntheticsTestResultService.getSyntheticsTestResults(test.getPublicId()); - }); - - try { - result = future.get(45, TimeUnit.SECONDS); - } catch (Exception e) { - LOGGER.error("Error getting test results: " + e.getMessage()); - throw new ApiException("No results found for test " + test.getPublicId()); - } finally { - datadogSyntheticsTestService.deleteSyntheticsTests(List.of(test.getPublicId())); - } - return result; + // SyntheticsGetAPITestLatestResultsResponse result = null; + + // CompletableFuture future = + // CompletableFuture.supplyAsync(() -> { + // return + // datadogSyntheticsTestResultService.getSyntheticsTestResults(test.getPublicId()); + // }); + // + // try { + // result = future.get(45, TimeUnit.SECONDS); + // } catch (Exception e) { + // LOGGER.error("Error getting test results: " + e.getMessage()); + // throw new ApiException("No results found for test " + + // test.getPublicId()); + // } finally { + // datadogSyntheticsTestService.deleteSyntheticsTests(List.of(test.getPublicId())); + // } + + RetryPolicy retryPolicy = RetryPolicy. builder() + .withMaxAttempts(45) + .withDelay(Duration.ofSeconds(1)) + .withMaxDuration(Duration.ofSeconds(45)) + .onRetry(e -> LOGGER.info("Failure #{}. Retrying.", e.getAttemptCount())) + .onSuccess(e -> LOGGER.info("Success #{}.", e.getAttemptCount())) + .handleResultIf(res -> res == null || res.getResults().size() == 0) + .build(); + + return Failsafe.with(retryPolicy) + .get(() -> datadogSyntheticsTestResultService.getSyntheticsTestResults(test.getPublicId())); + + // return result; } private OnDemandUrlCheckerResponse analyzeTestResults(SyntheticsGetAPITestLatestResultsResponse result, SyntheticsAPITest test) throws ApiException { From f586f3f45593bee687e7c47bebd46199a848ce63 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Wed, 12 Feb 2025 13:47:10 -0500 Subject: [PATCH 07/21] refactor: remove commented code OCD-4765 --- chpl/chpl-service/pom.xml | 4 ++-- .../datadog/OnDemandUrlCheckerManager.java | 20 ------------------- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/chpl/chpl-service/pom.xml b/chpl/chpl-service/pom.xml index 3dfb8c6e15..447a0a7345 100644 --- a/chpl/chpl-service/pom.xml +++ b/chpl/chpl-service/pom.xml @@ -425,8 +425,8 @@ failsafe 3.3.2 - - + + org.junit.jupiter junit-jupiter-engine diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java index 7ea6e1eb51..21f9dacb1c 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java @@ -81,24 +81,6 @@ private void triggerTest(SyntheticsAPITest test) throws ApiException { } private SyntheticsGetAPITestLatestResultsResponse awaitTestResults(SyntheticsAPITest test) throws ApiException { - // SyntheticsGetAPITestLatestResultsResponse result = null; - - // CompletableFuture future = - // CompletableFuture.supplyAsync(() -> { - // return - // datadogSyntheticsTestResultService.getSyntheticsTestResults(test.getPublicId()); - // }); - // - // try { - // result = future.get(45, TimeUnit.SECONDS); - // } catch (Exception e) { - // LOGGER.error("Error getting test results: " + e.getMessage()); - // throw new ApiException("No results found for test " + - // test.getPublicId()); - // } finally { - // datadogSyntheticsTestService.deleteSyntheticsTests(List.of(test.getPublicId())); - // } - RetryPolicy retryPolicy = RetryPolicy. builder() .withMaxAttempts(45) .withDelay(Duration.ofSeconds(1)) @@ -110,8 +92,6 @@ private SyntheticsGetAPITestLatestResultsResponse awaitTestResults(SyntheticsAPI return Failsafe.with(retryPolicy) .get(() -> datadogSyntheticsTestResultService.getSyntheticsTestResults(test.getPublicId())); - - // return result; } private OnDemandUrlCheckerResponse analyzeTestResults(SyntheticsGetAPITestLatestResultsResponse result, SyntheticsAPITest test) throws ApiException { From 43057623755e4e176682686921c9c0fb1fcadeb6 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Thu, 13 Feb 2025 11:03:12 -0500 Subject: [PATCH 08/21] fix: update security constant OCD-4765 --- .gitignore | 2 ++ .../main/java/gov/healthit/chpl/permissions/Permissions.java | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 6834c137ea..e62d2d6f34 100644 --- a/.gitignore +++ b/.gitignore @@ -278,3 +278,5 @@ ready-for-integration.sh update-pr.sh /chpl/chpl-api/e2e/env/*.postman_environment.json newman/ +/Servers + diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/permissions/Permissions.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/permissions/Permissions.java index d3bf7a2e40..9b0eddcb16 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/permissions/Permissions.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/permissions/Permissions.java @@ -80,7 +80,7 @@ public class Permissions { public static final String STANDARD = "STANDARD"; public static final String CODE_SET = "CODE_SET"; public static final String API_KEY = "API_KEY"; - public static final String CHECK_URL = "CHECK_URL"; + public static final String URL_CHECKER = "URL_CHECKER"; private Map domainPermissions = new HashMap(); @@ -154,7 +154,7 @@ public Permissions(CertificationResultsDomainPermissions certificationResultsDom domainPermissions.put(UCD_PROCESS, ucdProcessDomainPermissions); domainPermissions.put(USER_PERMISSIONS, userPermissionsDomainPermissions); domainPermissions.put(API_KEY, apiKeyPermissions); - domainPermissions.put(CHECK_URL, urlCheckerDomainPermissions); + domainPermissions.put(URL_CHECKER, urlCheckerDomainPermissions); } public boolean hasAccess(final String domain, final String action) { From af58496353d30b2313044c0815c4d1e9ddf42900 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Mon, 17 Feb 2025 21:29:17 -0500 Subject: [PATCH 09/21] refactor: replace literals with constants OCD-4765 --- .../datadog/OnDemandUrlCheckerManager.java | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java index 21f9dacb1c..b0ecc3714e 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java @@ -25,6 +25,8 @@ @Component public class OnDemandUrlCheckerManager { private static final Long TEMP_DEVELOPER_ID = -99L; + private static final Integer MAX_ATTEMPTS = 45; + private static final Integer MAX_SECONDS = 45; private static final String ASSERTION_RESULTS_KEY = "assertionResults"; private static final String TYPE_KEY = "type"; private static final String ACTUAL_KEY = "actual"; @@ -52,28 +54,22 @@ public OnDemandUrlCheckerResponse checkUrl(String url) throws ApiException { SyntheticsGetAPITestLatestResultsResponse result = awaitTestResults(test); OnDemandUrlCheckerResponse response = analyzeTestResults(result, test); return response; - - // Integer attempts = 0; - // while ((result == null || result.getResults().size() == 0) && - // attempts < 45) { - // Thread.sleep(1000); - // attempts++; - // result = - // datadogSyntheticsTestResultService.getSyntheticsTestResults(test.getPublicId()); - // } - } finally { + LOGGER.info("Completed On Demand URL Check"); if (test != null) { + LOGGER.info("Deleting On Demand URL Check"); datadogSyntheticsTestService.deleteSyntheticsTests(List.of(test.getPublicId())); } } } private SyntheticsAPITest createTest(String url) { + LOGGER.info("Creating On Demand URL Check"); return datadogSyntheticsTestService.createSyntheticsTest(url, List.of(TEMP_DEVELOPER_ID)); } private void triggerTest(SyntheticsAPITest test) throws ApiException { + LOGGER.info("Triggering On Demand URL Check"); SyntheticsTriggerBody body = new SyntheticsTriggerBody() .tests(List.of(new SyntheticsTriggerTest().publicId(test.getPublicId()))); @@ -81,10 +77,11 @@ private void triggerTest(SyntheticsAPITest test) throws ApiException { } private SyntheticsGetAPITestLatestResultsResponse awaitTestResults(SyntheticsAPITest test) throws ApiException { - RetryPolicy retryPolicy = RetryPolicy. builder() - .withMaxAttempts(45) + LOGGER.info("Awaiting On Demand URL Check"); + RetryPolicy retryPolicy = RetryPolicy.builder() + .withMaxAttempts(MAX_ATTEMPTS) .withDelay(Duration.ofSeconds(1)) - .withMaxDuration(Duration.ofSeconds(45)) + .withMaxDuration(Duration.ofSeconds(MAX_SECONDS)) .onRetry(e -> LOGGER.info("Failure #{}. Retrying.", e.getAttemptCount())) .onSuccess(e -> LOGGER.info("Success #{}.", e.getAttemptCount())) .handleResultIf(res -> res == null || res.getResults().size() == 0) From 99291af3e1648fc4ec408a5698e210643cd842eb Mon Sep 17 00:00:00 2001 From: Todd Young Date: Tue, 18 Feb 2025 11:34:23 -0500 Subject: [PATCH 10/21] feat: do not use datadog.syntheticsTest.readOnly when creating temp synth test OCD-4765 --- .../job/urluptime/DatadogSyntheticsTestService.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogSyntheticsTestService.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogSyntheticsTestService.java index a4da1782e4..0259b9dbc5 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogSyntheticsTestService.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogSyntheticsTestService.java @@ -150,13 +150,8 @@ public SyntheticsAPITest createSyntheticsTest(String url, List developerId .tags(developerIdsToTags(developerIds)); try { - if (datadogIsReadOnly) { - LOGGER.info("Not updating datadog (due to environment setting) with Developers: {} and URL: {}", developerIds, url); - return body; - } else { - LOGGER.info("Adding Synthetics Test for URL: {}, with Developers: {}", url, developerIds); - return apiProvider.getApiInstance().createSyntheticsAPITest(body); - } + LOGGER.info("Adding Synthetics Test for URL: {}, with Developers: {}", url, developerIds); + return apiProvider.getApiInstance().createSyntheticsAPITest(body); } catch (ApiException e) { LOGGER.error("Could not create Synthetics Test for URL: {}", url, e); return null; From 6aa90e5be6ec50983a157f2285479578212ba55e Mon Sep 17 00:00:00 2001 From: Todd Young Date: Tue, 18 Feb 2025 13:25:33 -0500 Subject: [PATCH 11/21] feat: validate the url OCD-4765 --- .../OnDemandUrlCheckerController.java | 3 ++- .../src/main/resources/errors.properties | 2 ++ .../datadog/OnDemandUrlCheckerManager.java | 20 +++++++++++++++++-- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/chpl/chpl-api/src/main/java/gov/healthit/chpl/web/controller/OnDemandUrlCheckerController.java b/chpl/chpl-api/src/main/java/gov/healthit/chpl/web/controller/OnDemandUrlCheckerController.java index d9b37ea45f..09bc9de5f2 100644 --- a/chpl/chpl-api/src/main/java/gov/healthit/chpl/web/controller/OnDemandUrlCheckerController.java +++ b/chpl/chpl-api/src/main/java/gov/healthit/chpl/web/controller/OnDemandUrlCheckerController.java @@ -12,6 +12,7 @@ import gov.healthit.chpl.datadog.OnDemandUrlCheckerManager; import gov.healthit.chpl.datadog.OnDemandUrlCheckerResponse; import gov.healthit.chpl.datadog.OnDemandUrlRequest; +import gov.healthit.chpl.exception.ValidationException; import gov.healthit.chpl.util.SwaggerSecurityRequirement; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.security.SecurityRequirement; @@ -38,7 +39,7 @@ public OnDemandUrlCheckerController(OnDemandUrlCheckerManager onDemandUrlChecker @SecurityRequirement(name = SwaggerSecurityRequirement.BEARER) }) @RequestMapping(value = "", method = RequestMethod.POST, consumes = MediaType.APPLICATION_JSON_VALUE, produces = "application/json; charset=utf-8") - public OnDemandUrlCheckerResponse checkUrl(@RequestBody OnDemandUrlRequest url) throws InterruptedException, ApiException { + public OnDemandUrlCheckerResponse checkUrl(@RequestBody OnDemandUrlRequest url) throws InterruptedException, ApiException, ValidationException { return onDemandUrlCheckerManager.checkUrl(url.getUrl()); } diff --git a/chpl/chpl-resources/src/main/resources/errors.properties b/chpl/chpl-resources/src/main/resources/errors.properties index 25c2a7c326..c54eb463fb 100644 --- a/chpl/chpl-resources/src/main/resources/errors.properties +++ b/chpl/chpl-resources/src/main/resources/errors.properties @@ -649,6 +649,8 @@ listing.realWorldTesting.eligibilityYearNotUpdatable=Real World Eligibility Year listing.svap.url.invalid=SVAP Notice URL is not a well formed URL. +onDemandUrlTest.invalidUrl=URL to test is not a well formed URL. + job.missingRequiredData=%s must be specified for every job. job.doesNotExist=The job with ID %s does not exist. job.exists=A job with ID %s already exists. diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java index b0ecc3714e..8bce8a1675 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java @@ -17,8 +17,12 @@ import dev.failsafe.Failsafe; import dev.failsafe.RetryPolicy; +import gov.healthit.chpl.exception.ValidationException; import gov.healthit.chpl.scheduler.job.urluptime.DatadogSyntheticsTestResultService; import gov.healthit.chpl.scheduler.job.urluptime.DatadogSyntheticsTestService; +import gov.healthit.chpl.util.ErrorMessageUtil; +import gov.healthit.chpl.util.Util; +import gov.healthit.chpl.util.ValidationUtils; import lombok.extern.log4j.Log4j2; @Log4j2 @@ -37,17 +41,23 @@ public class OnDemandUrlCheckerManager { private DatadogSyntheticsTestService datadogSyntheticsTestService; private DatadogSyntheticsTestResultService datadogSyntheticsTestResultService; + private ValidationUtils validationUtils; + private ErrorMessageUtil errorMessageUtil; @Autowired - public OnDemandUrlCheckerManager(DatadogSyntheticsTestService datadogSyntheticsTestService, DatadogSyntheticsTestResultService datadogSyntheticsTestResultService) { + public OnDemandUrlCheckerManager(DatadogSyntheticsTestService datadogSyntheticsTestService, DatadogSyntheticsTestResultService datadogSyntheticsTestResultService, + ValidationUtils validationUtils, ErrorMessageUtil errorMessageUtil) { this.datadogSyntheticsTestService = datadogSyntheticsTestService; this.datadogSyntheticsTestResultService = datadogSyntheticsTestResultService; + this.validationUtils = validationUtils; + this.errorMessageUtil = errorMessageUtil; } @PreAuthorize("@permissions.hasAccess(T(gov.healthit.chpl.permissions.Permissions).URL_CHECKER, " + "T(gov.healthit.chpl.permissions.domains.UrlCheckerDomainPermissions).CHECK)") - public OnDemandUrlCheckerResponse checkUrl(String url) throws ApiException { + public OnDemandUrlCheckerResponse checkUrl(String url) throws ApiException, ValidationException { SyntheticsAPITest test = null; + validateUrl(url); try { test = createTest(url); triggerTest(test); @@ -63,6 +73,12 @@ public OnDemandUrlCheckerResponse checkUrl(String url) throws ApiException { } } + private void validateUrl(String url) throws ValidationException { + if (!validationUtils.isWellFormedUrl(url)) { + throw new ValidationException(errorMessageUtil.getMessage("onDemandUrlTest.invalidUrl")); + } + } + private SyntheticsAPITest createTest(String url) { LOGGER.info("Creating On Demand URL Check"); return datadogSyntheticsTestService.createSyntheticsTest(url, List.of(TEMP_DEVELOPER_ID)); From 9d5879701eb978a39d65c3353c6019c74442a522 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Fri, 28 Feb 2025 13:29:49 -0500 Subject: [PATCH 12/21] feat: ignore Datadog URL tests where the developer is -99 OCD-4765 --- .../datadog/OnDemandUrlCheckerManager.java | 38 ++++++++++++++++--- .../DatadogUrlUptimeSynchonizer.java | 16 +++++--- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java index 8bce8a1675..5c3f9f4eea 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java @@ -11,6 +11,7 @@ import com.datadog.api.client.ApiException; import com.datadog.api.client.v1.model.SyntheticsAPITest; import com.datadog.api.client.v1.model.SyntheticsAPITestResultFull; +import com.datadog.api.client.v1.model.SyntheticsApiTestFailureCode; import com.datadog.api.client.v1.model.SyntheticsGetAPITestLatestResultsResponse; import com.datadog.api.client.v1.model.SyntheticsTriggerBody; import com.datadog.api.client.v1.model.SyntheticsTriggerTest; @@ -28,7 +29,7 @@ @Log4j2 @Component public class OnDemandUrlCheckerManager { - private static final Long TEMP_DEVELOPER_ID = -99L; + public static final Long TEMP_DEVELOPER_ID = -99L; private static final Integer MAX_ATTEMPTS = 45; private static final Integer MAX_SECONDS = 45; private static final String ASSERTION_RESULTS_KEY = "assertionResults"; @@ -38,7 +39,9 @@ public class OnDemandUrlCheckerManager { private static final String TYPE_VALUE_STATUS_CODE = "statusCode"; private static final String TYPE_VALUE_BODY = "body"; private static final String TYPE_VALUE_RESPONSE_TIME = "responseTime"; - + + private List errorsToIgnore = List.of("BODY_TOO_LARGE_TO_PROCESS"); + private DatadogSyntheticsTestService datadogSyntheticsTestService; private DatadogSyntheticsTestResultService datadogSyntheticsTestResultService; private ValidationUtils validationUtils; @@ -80,7 +83,7 @@ private void validateUrl(String url) throws ValidationException { } private SyntheticsAPITest createTest(String url) { - LOGGER.info("Creating On Demand URL Check"); + LOGGER.info("Creating On Demand URL Check for: {}", url); return datadogSyntheticsTestService.createSyntheticsTest(url, List.of(TEMP_DEVELOPER_ID)); } @@ -114,9 +117,15 @@ private OnDemandUrlCheckerResponse analyzeTestResults(SyntheticsGetAPITestLatest && result.getResults().size() > 0) { fullTestResults = datadogSyntheticsTestResultService.getDetailedTestResult(test.getPublicId(), result.getResults().get(0).getResultId()); response = convertToResponse(test.getConfig().getRequest().getUrl(), fullTestResults); - response.setPassed(result.getResults().get(0).getResult().getPassed()); - if (!result.getResults().get(0).getResult().getPassed()) { - response.setErrorMessage(fullTestResults.getResult().getFailure().getMessage()); + if (fullTestResults.getResult().getFailure() != null + && isErrorIgnorable(fullTestResults.getResult().getFailure().getCode())) { + response.setPassed(true); + response.setErrorMessage(""); + } else { + response.setPassed(result.getResults().get(0).getResult().getPassed()); + if (!result.getResults().get(0).getResult().getPassed()) { + response.setErrorMessage(fullTestResults.getResult().getFailure().getMessage()); + } } } else { throw new ApiException("No results found for test " + test.getPublicId()); @@ -126,6 +135,15 @@ private OnDemandUrlCheckerResponse analyzeTestResults(SyntheticsGetAPITestLatest private OnDemandUrlCheckerResponse convertToResponse(String url, SyntheticsAPITestResultFull fullTestResults) { if (doAdditionalPropertiesExist(fullTestResults)) { + if (fullTestResults.getResult().getFailure() != null + && isErrorIgnorable(fullTestResults.getResult().getFailure().getCode())) { + return OnDemandUrlCheckerResponse.builder() + .httpResponseAssertion(OnDemandUrlCheckerAssertionResult.builder().passed(true).actualValue("").build()) + .responseTimeAssertion(OnDemandUrlCheckerAssertionResult.builder().passed(true).actualValue("").build()) + .bodyNotEmptyAssertion(OnDemandUrlCheckerAssertionResult.builder().passed(true).actualValue("").build()) + .url(url) + .build(); + } return OnDemandUrlCheckerResponse.builder() .httpResponseAssertion(getAssertionResult(fullTestResults.getResult().getAdditionalProperties(), TYPE_VALUE_STATUS_CODE)) .responseTimeAssertion(getAssertionResult(fullTestResults.getResult().getAdditionalProperties(), TYPE_VALUE_RESPONSE_TIME)) @@ -166,4 +184,12 @@ private Boolean doAdditionalPropertiesExist(SyntheticsAPITestResultFull fullTest return fullTestResults.getResult().getAdditionalProperties().containsKey(ASSERTION_RESULTS_KEY) && fullTestResults.getResult().getAdditionalProperties().get(ASSERTION_RESULTS_KEY) instanceof List; } + + private boolean isErrorIgnorable(SyntheticsApiTestFailureCode errorCode) { + return errorsToIgnore.stream() + .filter(code -> code.equals(errorCode.getValue())) + .findAny() + .isPresent(); + } + } diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogUrlUptimeSynchonizer.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogUrlUptimeSynchonizer.java index 11480bfd8e..940b6f3fab 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogUrlUptimeSynchonizer.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogUrlUptimeSynchonizer.java @@ -22,6 +22,7 @@ import com.datadog.api.client.v1.model.SyntheticsApiTestFailureCode; import com.datadog.api.client.v1.model.SyntheticsTestDetails; +import gov.healthit.chpl.datadog.OnDemandUrlCheckerManager; import gov.healthit.chpl.domain.Developer; import lombok.extern.log4j.Log4j2; @@ -160,7 +161,7 @@ private void synchronizeUrlUptimeMonitorsWithDatadogSyntheticsTests() { urlUptimeMonitors = urlUptimeMonitorDAO.getAll(); removeOutdatedUrlMonitors(urlUptimeMonitors, expectedUrlUptimeMonitors); } - + private void addMissingUrlMonitors(List existing, List expected) { expected.stream() .filter(uum -> !contains(existing, uum)) @@ -204,11 +205,14 @@ public static Predicate distinctByKey(Function keyExtr } private void addUrlUptimeMonitor(UrlUptimeMonitor urlUptimeMonitor) { - try { - LOGGER.info("Adding the following URL to url_uptime_monitor table: {}, {}", urlUptimeMonitor.getUrl(), urlUptimeMonitor.getDeveloper().getId()); - urlUptimeMonitorDAO.create(urlUptimeMonitor); - } catch (Exception e) { - LOGGER.error("Could not add the following URL to url_uptime_monitor table: {}", urlUptimeMonitor.getUrl(), e); + // Ignore monitors URL monitors with a developer of -99. These are from the On Demand Checker. + if (urlUptimeMonitor.getDeveloper().getId() != OnDemandUrlCheckerManager.TEMP_DEVELOPER_ID) { + try { + LOGGER.info("Adding the following URL to url_uptime_monitor table: {}, {}", urlUptimeMonitor.getUrl(), urlUptimeMonitor.getDeveloper().getId()); + urlUptimeMonitorDAO.create(urlUptimeMonitor); + } catch (Exception e) { + LOGGER.error("Could not add the following URL to url_uptime_monitor table: {}", urlUptimeMonitor.getUrl(), e); + } } } From 197157d14cc6c201ede5de6f4b3aa4e8b7bd8025 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Fri, 28 Feb 2025 13:57:20 -0500 Subject: [PATCH 13/21] feat: add logging for debugging purposes OCD-4765 --- .../chpl/datadog/OnDemandUrlCheckerManager.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java index 5c3f9f4eea..ac6d421dff 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java @@ -60,7 +60,7 @@ public OnDemandUrlCheckerManager(DatadogSyntheticsTestService datadogSyntheticsT + "T(gov.healthit.chpl.permissions.domains.UrlCheckerDomainPermissions).CHECK)") public OnDemandUrlCheckerResponse checkUrl(String url) throws ApiException, ValidationException { SyntheticsAPITest test = null; - validateUrl(url); + validateUrlWellFormed(url); try { test = createTest(url); triggerTest(test); @@ -70,13 +70,14 @@ public OnDemandUrlCheckerResponse checkUrl(String url) throws ApiException, Vali } finally { LOGGER.info("Completed On Demand URL Check"); if (test != null) { - LOGGER.info("Deleting On Demand URL Check"); + LOGGER.info("Deleting On Demand URL Check: {}", test.getPublicId()); datadogSyntheticsTestService.deleteSyntheticsTests(List.of(test.getPublicId())); + LOGGER.info("Deleted On Demand URL Check: {}", test.getPublicId()); } } } - private void validateUrl(String url) throws ValidationException { + private void validateUrlWellFormed(String url) throws ValidationException { if (!validationUtils.isWellFormedUrl(url)) { throw new ValidationException(errorMessageUtil.getMessage("onDemandUrlTest.invalidUrl")); } @@ -84,7 +85,9 @@ private void validateUrl(String url) throws ValidationException { private SyntheticsAPITest createTest(String url) { LOGGER.info("Creating On Demand URL Check for: {}", url); - return datadogSyntheticsTestService.createSyntheticsTest(url, List.of(TEMP_DEVELOPER_ID)); + var x = datadogSyntheticsTestService.createSyntheticsTest(url, List.of(TEMP_DEVELOPER_ID)); + LOGGER.info("Created On Demand URL Check: {}", x.getPublicId()); + return x; } private void triggerTest(SyntheticsAPITest test) throws ApiException { From f8eb6b67dafa5a2f8b1a1f3372c84d9dac654a35 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Fri, 28 Feb 2025 14:20:44 -0500 Subject: [PATCH 14/21] feat: add more logging for debugging purposes OCD-4765 --- .../chpl/datadog/OnDemandUrlCheckerManager.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java index ac6d421dff..85bf2391c1 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java @@ -68,12 +68,17 @@ public OnDemandUrlCheckerResponse checkUrl(String url) throws ApiException, Vali OnDemandUrlCheckerResponse response = analyzeTestResults(result, test); return response; } finally { - LOGGER.info("Completed On Demand URL Check"); - if (test != null) { - LOGGER.info("Deleting On Demand URL Check: {}", test.getPublicId()); - datadogSyntheticsTestService.deleteSyntheticsTests(List.of(test.getPublicId())); - LOGGER.info("Deleted On Demand URL Check: {}", test.getPublicId()); + try { + LOGGER.info("Completed On Demand URL Check"); + if (test != null) { + LOGGER.info("Deleting On Demand URL Check: {}", test.getPublicId()); + datadogSyntheticsTestService.deleteSyntheticsTests(List.of(test.getPublicId())); + LOGGER.info("Deleted On Demand URL Check: {}", test.getPublicId()); + } + } catch (Exception e) { + LOGGER.error("Failed to delete On Demand URL Check", e); } + } } From e616f1b8b910068c8a2f9eda113faf78abd510b2 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Fri, 28 Feb 2025 20:01:34 -0500 Subject: [PATCH 15/21] feat: remove readonly check when deleting a synthetic test OCD-4765 --- .../job/urluptime/DatadogSyntheticsTestService.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogSyntheticsTestService.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogSyntheticsTestService.java index 0259b9dbc5..e1e07235b5 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogSyntheticsTestService.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogSyntheticsTestService.java @@ -86,12 +86,7 @@ public SyntheticsAPITest getSyntheticsTest(String publicId) { public void deleteSyntheticsTests(List publicIds) { try { - if (datadogIsReadOnly) { - LOGGER.info("Not deleting from datadog (due to environment setting) with Public Ids: {}", publicIds); - } else { - apiProvider.getApiInstance().deleteTests(new SyntheticsDeleteTestsPayload().publicIds(publicIds)); - } - + apiProvider.getApiInstance().deleteTests(new SyntheticsDeleteTestsPayload().publicIds(publicIds)); } catch (ApiException e) { LOGGER.error("Could not delete Synthetic Tests from Datadog: {}", publicIds, e); } From 8d524f0df808b5de8c67ee3eebac08512c1e8b97 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Mon, 3 Mar 2025 09:26:02 -0500 Subject: [PATCH 16/21] feat: completely remove read only check from service OCD-4765 --- .../job/urluptime/DatadogSyntheticsTestService.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogSyntheticsTestService.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogSyntheticsTestService.java index e1e07235b5..98ff6d0209 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogSyntheticsTestService.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogSyntheticsTestService.java @@ -38,7 +38,6 @@ public class DatadogSyntheticsTestService { private static final Long SECONDS_IN_A_MINUTE = 60L; private DatadogSyntheticsTestApiProvider apiProvider; - private Boolean datadogIsReadOnly; private String datadogTestStartTime; private String datadogTestEndTime; private Long datadogCheckEveryMinutes; @@ -46,14 +45,12 @@ public class DatadogSyntheticsTestService { private String datadogTestLocation; public DatadogSyntheticsTestService(DatadogSyntheticsTestApiProvider apiProvider, - @Value("${datadog.syntheticsTest.readOnly}") Boolean datadogIsReadOnly, @Value("${datadog.syntheticsTest.startTime}") String datadogTestStartTime, @Value("${datadog.syntheticsTest.endTime}") String datadogTestEndTime, @Value("${datadog.syntheticsTest.checkEveryMinutes}") Long datadogCheckEveryMinutes, @Value("${datadog.syntheticsTest.timeout}") Integer datadogTestTimeout, @Value("${datadog.syntheticsTest.location}") String datadogTestLocation) { this.apiProvider = apiProvider; - this.datadogIsReadOnly = datadogIsReadOnly; this.datadogTestStartTime = datadogTestStartTime; this.datadogTestEndTime = datadogTestEndTime; this.datadogCheckEveryMinutes = datadogCheckEveryMinutes; @@ -164,11 +161,7 @@ public void setApplicableDevelopersForTest(String syntheticsApiTestPublicId, Lis .path("/tags") .value(test.getTags())); try { - if (datadogIsReadOnly) { - LOGGER.info("Not adding Developer(s) (due to environment setting) to existing Synthetics Test {}", developerIds); - } else { - apiProvider.getApiInstance().patchTest(syntheticsApiTestPublicId, body); - } + apiProvider.getApiInstance().patchTest(syntheticsApiTestPublicId, body); } catch (ApiException e) { LOGGER.error("Could not add Developer(s) to existing Synthetics Test {}", developerIds, e); } From bf6f8c5eec2ef36f7450d3946ebd4163e7a7939d Mon Sep 17 00:00:00 2001 From: Todd Young Date: Mon, 3 Mar 2025 11:15:38 -0500 Subject: [PATCH 17/21] feat: update url for endpoint OCD-4765 --- .../chpl/web/controller/OnDemandUrlCheckerController.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/chpl/chpl-api/src/main/java/gov/healthit/chpl/web/controller/OnDemandUrlCheckerController.java b/chpl/chpl-api/src/main/java/gov/healthit/chpl/web/controller/OnDemandUrlCheckerController.java index 09bc9de5f2..5f002c7de3 100644 --- a/chpl/chpl-api/src/main/java/gov/healthit/chpl/web/controller/OnDemandUrlCheckerController.java +++ b/chpl/chpl-api/src/main/java/gov/healthit/chpl/web/controller/OnDemandUrlCheckerController.java @@ -32,13 +32,13 @@ public OnDemandUrlCheckerController(OnDemandUrlCheckerManager onDemandUrlChecker this.onDemandUrlCheckerManager = onDemandUrlCheckerManager; } - @Operation(summary = "", - description = "Security Restrictions: ROLE_ADMIN or ROLE_ONC", + @Operation(summary = "Validates a URL. Three checks are performed: 1) HTTP Status code is 200, 2) Response time is less than 30 seconds, and 3) Response body is not empty.", + description = "Security Restrictions: chpl-admin, chpl-onc, or chpl-onc-acb", security = { @SecurityRequirement(name = SwaggerSecurityRequirement.API_KEY), @SecurityRequirement(name = SwaggerSecurityRequirement.BEARER) - }) - @RequestMapping(value = "", method = RequestMethod.POST, consumes = MediaType.APPLICATION_JSON_VALUE, produces = "application/json; charset=utf-8") + }) + @RequestMapping(value = "/validate", method = RequestMethod.POST, consumes = MediaType.APPLICATION_JSON_VALUE, produces = "application/json; charset=utf-8") public OnDemandUrlCheckerResponse checkUrl(@RequestBody OnDemandUrlRequest url) throws InterruptedException, ApiException, ValidationException { return onDemandUrlCheckerManager.checkUrl(url.getUrl()); } From 8658a662c9419910c33a503e1b6099d1596ee657 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Mon, 3 Mar 2025 11:34:14 -0500 Subject: [PATCH 18/21] refactor: remove commented code OCD-4765 --- .../DatadogSyntheticsTestResultService.java | 14 +------------- .../job/urluptime/DatadogUrlUptimeSynchonizer.java | 2 +- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogSyntheticsTestResultService.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogSyntheticsTestResultService.java index c7b4f9a45b..a4bd8a5a53 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogSyntheticsTestResultService.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogSyntheticsTestResultService.java @@ -58,20 +58,8 @@ public List getSyntheticsTestResults(String public } public SyntheticsGetAPITestLatestResultsResponse getSyntheticsTestResults(String publicTestKey) { - // SyntheticsGetAPITestLatestResultsResponse response; - // List testResults = new - // ArrayList(); try { - return apiProvider.getApiInstance().getAPITestLatestResults(publicTestKey); - // while (response.getResults().size() > 1) { - // testResults.addAll(response.getResults()); - // Long ts = getMostRecentTimestamp(response.getResults()); - // response = - // apiProvider.getApiInstance().getAPITestLatestResults(publicTestKey); - // } - // LOGGER.info("Found {} tests for monitor {}", testResults.size(), - // publicTestKey); - + return apiProvider.getApiInstance().getAPITestLatestResults(publicTestKey); } catch (ApiException e) { LOGGER.error("Could not retrieve results for test key: {}", publicTestKey, e); return null; diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogUrlUptimeSynchonizer.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogUrlUptimeSynchonizer.java index 940b6f3fab..fef186f3d0 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogUrlUptimeSynchonizer.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/scheduler/job/urluptime/DatadogUrlUptimeSynchonizer.java @@ -161,7 +161,7 @@ private void synchronizeUrlUptimeMonitorsWithDatadogSyntheticsTests() { urlUptimeMonitors = urlUptimeMonitorDAO.getAll(); removeOutdatedUrlMonitors(urlUptimeMonitors, expectedUrlUptimeMonitors); } - + private void addMissingUrlMonitors(List existing, List expected) { expected.stream() .filter(uum -> !contains(existing, uum)) From 74dc35b80dd8df21c4bb628b0d18897e73262646 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Wed, 5 Mar 2025 12:55:32 -0500 Subject: [PATCH 19/21] refactor: remove unused imports OCD-4765 --- chpl/chpl-service/pom.xml | 6 +++--- .../healthit/chpl/datadog/OnDemandUrlCheckerManager.java | 9 ++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/chpl/chpl-service/pom.xml b/chpl/chpl-service/pom.xml index 447a0a7345..1f569f81d4 100644 --- a/chpl/chpl-service/pom.xml +++ b/chpl/chpl-service/pom.xml @@ -362,7 +362,7 @@ ${redisson.version} - + org.htmlunit htmlunit 4.4.0 @@ -383,7 +383,7 @@ provided - + org.springframework.boot @@ -419,7 +419,7 @@ 0.4.0 - + dev.failsafe failsafe diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java index 85bf2391c1..522f721668 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java @@ -22,7 +22,6 @@ import gov.healthit.chpl.scheduler.job.urluptime.DatadogSyntheticsTestResultService; import gov.healthit.chpl.scheduler.job.urluptime.DatadogSyntheticsTestService; import gov.healthit.chpl.util.ErrorMessageUtil; -import gov.healthit.chpl.util.Util; import gov.healthit.chpl.util.ValidationUtils; import lombok.extern.log4j.Log4j2; @@ -39,9 +38,9 @@ public class OnDemandUrlCheckerManager { private static final String TYPE_VALUE_STATUS_CODE = "statusCode"; private static final String TYPE_VALUE_BODY = "body"; private static final String TYPE_VALUE_RESPONSE_TIME = "responseTime"; - - private List errorsToIgnore = List.of("BODY_TOO_LARGE_TO_PROCESS"); - + + private List errorsToIgnore = List.of("BODY_TOO_LARGE_TO_PROCESS"); + private DatadogSyntheticsTestService datadogSyntheticsTestService; private DatadogSyntheticsTestResultService datadogSyntheticsTestResultService; private ValidationUtils validationUtils; @@ -78,7 +77,7 @@ public OnDemandUrlCheckerResponse checkUrl(String url) throws ApiException, Vali } catch (Exception e) { LOGGER.error("Failed to delete On Demand URL Check", e); } - + } } From 075c4936782e0481bd1ec52c3648f3875b79106d Mon Sep 17 00:00:00 2001 From: Todd Young Date: Fri, 7 Mar 2025 10:34:45 -0500 Subject: [PATCH 20/21] fix: fix checkstyle issues OCD-4765 --- chpl/chpl-service/pom.xml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/chpl/chpl-service/pom.xml b/chpl/chpl-service/pom.xml index 1f569f81d4..7617f84e61 100644 --- a/chpl/chpl-service/pom.xml +++ b/chpl/chpl-service/pom.xml @@ -362,7 +362,7 @@ ${redisson.version} - + org.htmlunit htmlunit 4.4.0 @@ -418,14 +418,14 @@ jwks-rsa 0.4.0 - - - - dev.failsafe - failsafe - 3.3.2 - - + + + + dev.failsafe + failsafe + 3.3.2 + + org.junit.jupiter From 5a2b043f46efa72081b59bc86695127a2c3fa1e5 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Fri, 7 Mar 2025 10:38:28 -0500 Subject: [PATCH 21/21] fix: fix checkstyle issues OCD-4765 --- chpl/chpl-service/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chpl/chpl-service/pom.xml b/chpl/chpl-service/pom.xml index 7617f84e61..5290e30f04 100644 --- a/chpl/chpl-service/pom.xml +++ b/chpl/chpl-service/pom.xml @@ -426,7 +426,7 @@ 3.3.2 - + org.junit.jupiter junit-jupiter-engine