OCD-4765 - Service Base URL Real-Time Validator

.gitignore

@@ -278,4 +278,5 @@ ready-for-integration.sh
 update-pr.sh
 /chpl/chpl-api/e2e/env/*.postman_environment.json
 newman/
+/Servers
 Servers/

chpl/chpl-api/src/main/java/gov/healthit/chpl/web/controller/OnDemandUrlCheckerController.java

@@ -0,0 +1,46 @@
+package gov.healthit.chpl.web.controller;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RestController;
+
+import com.datadog.api.client.ApiException;
+
+import gov.healthit.chpl.datadog.OnDemandUrlCheckerManager;
+import gov.healthit.chpl.datadog.OnDemandUrlCheckerResponse;
+import gov.healthit.chpl.datadog.OnDemandUrlRequest;
+import gov.healthit.chpl.exception.ValidationException;
+import gov.healthit.chpl.util.SwaggerSecurityRequirement;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.security.SecurityRequirement;
+import io.swagger.v3.oas.annotations.tags.Tag;
+import lombok.extern.log4j.Log4j2;
+
+@Log4j2
+@Tag(name = "urls", description = "")
+@RestController
+@RequestMapping("/urls")
+public class OnDemandUrlCheckerController {
+
+    private OnDemandUrlCheckerManager onDemandUrlCheckerManager;
+
+    @Autowired
+    public OnDemandUrlCheckerController(OnDemandUrlCheckerManager onDemandUrlCheckerManager) {
+        this.onDemandUrlCheckerManager = onDemandUrlCheckerManager;
+    }
+
+    @Operation(summary = "Validates a URL.  Three checks are performed: 1) HTTP Status code is 200, 2) Response time is less than 30 seconds, and 3) Response body is not empty.",
+            description = "Security Restrictions: chpl-admin, chpl-onc, or chpl-onc-acb",
+            security = {
+                    @SecurityRequirement(name = SwaggerSecurityRequirement.API_KEY),
+                    @SecurityRequirement(name = SwaggerSecurityRequirement.BEARER)
+    })
+    @RequestMapping(value = "/validate", method = RequestMethod.POST, consumes = MediaType.APPLICATION_JSON_VALUE, produces = "application/json; charset=utf-8")
+    public OnDemandUrlCheckerResponse checkUrl(@RequestBody OnDemandUrlRequest url) throws InterruptedException, ApiException, ValidationException {
+        return onDemandUrlCheckerManager.checkUrl(url.getUrl());
+    }
+
+}

chpl/chpl-resources/src/main/resources/errors.properties

@@ -649,6 +649,8 @@ listing.realWorldTesting.eligibilityYearNotUpdatable=Real World Eligibility Year
 
 listing.svap.url.invalid=SVAP Notice URL is not a well formed URL.
 
+onDemandUrlTest.invalidUrl=URL to test is not a well formed URL.
+
 job.missingRequiredData=%s must be specified for every job.
 job.doesNotExist=The job with ID %s does not exist.
 job.exists=A job with ID %s already exists.

chpl/chpl-service/pom.xml

@@ -383,7 +383,7 @@
             <scope>provided</scope>
         </dependency>
 
-        <!-- Sprng Doc (Swagger) -->
+        <!-- Spring Doc (Swagger) -->
 
         <dependency>
             <groupId>org.springframework.boot</groupId>
@@ -419,6 +419,13 @@
             <version>0.4.0</version>
         </dependency>
 
+        <!-- https://failsafe.dev/ -->
+        <dependency>
+            <groupId>dev.failsafe</groupId>
+            <artifactId>failsafe</artifactId>
+            <version>3.3.2</version>
+        </dependency>
+
         <!--  needed for unit tests -->
         <dependency>
             <groupId>org.junit.jupiter</groupId>

chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerAssertionResult.java

@@ -0,0 +1,11 @@
+package gov.healthit.chpl.datadog;
+
+import lombok.Builder;
+import lombok.Data;
+
+@Data
+@Builder
+public class OnDemandUrlCheckerAssertionResult {
+    private Boolean passed;
+    private String actualValue;
+}

chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerManager.java

@@ -0,0 +1,202 @@
+package gov.healthit.chpl.datadog;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.stereotype.Component;
+
+import com.datadog.api.client.ApiException;
+import com.datadog.api.client.v1.model.SyntheticsAPITest;
+import com.datadog.api.client.v1.model.SyntheticsAPITestResultFull;
+import com.datadog.api.client.v1.model.SyntheticsApiTestFailureCode;
+import com.datadog.api.client.v1.model.SyntheticsGetAPITestLatestResultsResponse;
+import com.datadog.api.client.v1.model.SyntheticsTriggerBody;
+import com.datadog.api.client.v1.model.SyntheticsTriggerTest;
+
+import dev.failsafe.Failsafe;
+import dev.failsafe.RetryPolicy;
+import gov.healthit.chpl.exception.ValidationException;
+import gov.healthit.chpl.scheduler.job.urluptime.DatadogSyntheticsTestResultService;
+import gov.healthit.chpl.scheduler.job.urluptime.DatadogSyntheticsTestService;
+import gov.healthit.chpl.util.ErrorMessageUtil;
+import gov.healthit.chpl.util.ValidationUtils;
+import lombok.extern.log4j.Log4j2;
+
+@Log4j2
+@Component
+public class OnDemandUrlCheckerManager {
+    public static final Long TEMP_DEVELOPER_ID = -99L;
+    private static final Integer MAX_ATTEMPTS = 45;
+    private static final Integer MAX_SECONDS = 45;
+    private static final String ASSERTION_RESULTS_KEY = "assertionResults";
+    private static final String TYPE_KEY = "type";
+    private static final String ACTUAL_KEY = "actual";
+    private static final String VALID_KEY = "valid";
+    private static final String TYPE_VALUE_STATUS_CODE = "statusCode";
+    private static final String TYPE_VALUE_BODY = "body";
+    private static final String TYPE_VALUE_RESPONSE_TIME = "responseTime";
+
+    private List<String> errorsToIgnore = List.of("BODY_TOO_LARGE_TO_PROCESS");
+
+    private DatadogSyntheticsTestService datadogSyntheticsTestService;
+    private DatadogSyntheticsTestResultService datadogSyntheticsTestResultService;
+    private ValidationUtils validationUtils;
+    private ErrorMessageUtil errorMessageUtil;
+
+    @Autowired
+    public OnDemandUrlCheckerManager(DatadogSyntheticsTestService datadogSyntheticsTestService, DatadogSyntheticsTestResultService datadogSyntheticsTestResultService,
+            ValidationUtils validationUtils, ErrorMessageUtil errorMessageUtil) {
+        this.datadogSyntheticsTestService = datadogSyntheticsTestService;
+        this.datadogSyntheticsTestResultService = datadogSyntheticsTestResultService;
+        this.validationUtils = validationUtils;
+        this.errorMessageUtil = errorMessageUtil;
+    }
+
+    @PreAuthorize("@permissions.hasAccess(T(gov.healthit.chpl.permissions.Permissions).URL_CHECKER, "
+            + "T(gov.healthit.chpl.permissions.domains.UrlCheckerDomainPermissions).CHECK)")
+    public OnDemandUrlCheckerResponse checkUrl(String url) throws ApiException, ValidationException {
+        SyntheticsAPITest test = null;
+        validateUrlWellFormed(url);
+        try {
+            test = createTest(url);
+            triggerTest(test);
+            SyntheticsGetAPITestLatestResultsResponse result = awaitTestResults(test);
+            OnDemandUrlCheckerResponse response = analyzeTestResults(result, test);
+            return response;
+        } finally {
+            try {
+                LOGGER.info("Completed On Demand URL Check");
+                if (test != null) {
+                    LOGGER.info("Deleting On Demand URL Check: {}", test.getPublicId());
+                    datadogSyntheticsTestService.deleteSyntheticsTests(List.of(test.getPublicId()));
+                    LOGGER.info("Deleted On Demand URL Check: {}", test.getPublicId());
+                }
+            } catch (Exception e) {
+                LOGGER.error("Failed to delete On Demand URL Check", e);
+            }
+
+        }
+    }
+
+    private void validateUrlWellFormed(String url) throws ValidationException {
+        if (!validationUtils.isWellFormedUrl(url)) {
+            throw new ValidationException(errorMessageUtil.getMessage("onDemandUrlTest.invalidUrl"));
+        }
+    }
+
+    private SyntheticsAPITest createTest(String url) {
+        LOGGER.info("Creating On Demand URL Check for: {}", url);
+        var x =  datadogSyntheticsTestService.createSyntheticsTest(url, List.of(TEMP_DEVELOPER_ID));
+        LOGGER.info("Created On Demand URL Check: {}", x.getPublicId());
+        return x;
+    }
+
+    private void triggerTest(SyntheticsAPITest test) throws ApiException {
+        LOGGER.info("Triggering On Demand URL Check");
+        SyntheticsTriggerBody body = new SyntheticsTriggerBody()
+                .tests(List.of(new SyntheticsTriggerTest().publicId(test.getPublicId())));
+
+        datadogSyntheticsTestService.getApiProvider().getApiInstance().triggerTests(body);
+    }
+
+    private SyntheticsGetAPITestLatestResultsResponse awaitTestResults(SyntheticsAPITest test) throws ApiException {
+        LOGGER.info("Awaiting On Demand URL Check");
+        RetryPolicy<SyntheticsGetAPITestLatestResultsResponse> retryPolicy = RetryPolicy.<SyntheticsGetAPITestLatestResultsResponse>builder()
+                .withMaxAttempts(MAX_ATTEMPTS)
+                .withDelay(Duration.ofSeconds(1))
+                .withMaxDuration(Duration.ofSeconds(MAX_SECONDS))
+                .onRetry(e -> LOGGER.info("Failure #{}. Retrying.", e.getAttemptCount()))
+                .onSuccess(e -> LOGGER.info("Success #{}.", e.getAttemptCount()))
+                .handleResultIf(res -> res == null || res.getResults().size() == 0)
+                .build();
+
+        return Failsafe.with(retryPolicy)
+                .get(() -> datadogSyntheticsTestResultService.getSyntheticsTestResults(test.getPublicId()));
+    }
+
+    private OnDemandUrlCheckerResponse analyzeTestResults(SyntheticsGetAPITestLatestResultsResponse result, SyntheticsAPITest test) throws ApiException {
+        SyntheticsAPITestResultFull fullTestResults = null;
+        OnDemandUrlCheckerResponse response = null;
+        if (result != null
+                && result.getResults().size() > 0) {
+            fullTestResults = datadogSyntheticsTestResultService.getDetailedTestResult(test.getPublicId(), result.getResults().get(0).getResultId());
+            response = convertToResponse(test.getConfig().getRequest().getUrl(), fullTestResults);
+            if (fullTestResults.getResult().getFailure() != null
+                    && isErrorIgnorable(fullTestResults.getResult().getFailure().getCode())) {
+                response.setPassed(true);
+                response.setErrorMessage("");
+            } else {
+                response.setPassed(result.getResults().get(0).getResult().getPassed());
+                if (!result.getResults().get(0).getResult().getPassed()) {
+                    response.setErrorMessage(fullTestResults.getResult().getFailure().getMessage());
+                }
+            }
+        } else {
+            throw new ApiException("No results found for test " + test.getPublicId());
+        }
+        return response;
+    }
+
+    private OnDemandUrlCheckerResponse convertToResponse(String url, SyntheticsAPITestResultFull fullTestResults) {
+        if (doAdditionalPropertiesExist(fullTestResults)) {
+            if (fullTestResults.getResult().getFailure() != null
+                    && isErrorIgnorable(fullTestResults.getResult().getFailure().getCode())) {
+                return OnDemandUrlCheckerResponse.builder()
+                        .httpResponseAssertion(OnDemandUrlCheckerAssertionResult.builder().passed(true).actualValue("").build())
+                        .responseTimeAssertion(OnDemandUrlCheckerAssertionResult.builder().passed(true).actualValue("").build())
+                        .bodyNotEmptyAssertion(OnDemandUrlCheckerAssertionResult.builder().passed(true).actualValue("").build())
+                        .url(url)
+                        .build();
+            }
+            return OnDemandUrlCheckerResponse.builder()
+                    .httpResponseAssertion(getAssertionResult(fullTestResults.getResult().getAdditionalProperties(), TYPE_VALUE_STATUS_CODE))
+                    .responseTimeAssertion(getAssertionResult(fullTestResults.getResult().getAdditionalProperties(), TYPE_VALUE_RESPONSE_TIME))
+                    .bodyNotEmptyAssertion(getAssertionResult(fullTestResults.getResult().getAdditionalProperties(), TYPE_VALUE_BODY))
+                    .url(url)
+                    .build();
+        } else {
+            return OnDemandUrlCheckerResponse.builder()
+                    .url(url)
+                    .build();
+        }
+    }
+
+    private OnDemandUrlCheckerAssertionResult getAssertionResult(Map<String, Object> results, String value) {
+        if (results.containsKey(ASSERTION_RESULTS_KEY)
+                && results.get(ASSERTION_RESULTS_KEY) instanceof List<?>) {
+
+            return ((List<?>) results.get(ASSERTION_RESULTS_KEY)).stream()
+                    .filter(map -> map instanceof Map<?, ?>)
+                    .filter(map -> ((Map<?, ?>) map).containsKey(TYPE_KEY)
+                            && ((Map<?, ?>) map).containsKey(VALID_KEY)
+                            && ((String) ((Map<?, ?>) map).get(TYPE_KEY)).equals(value))
+
+                    .findAny()
+                    .map(map -> OnDemandUrlCheckerAssertionResult.builder()
+                            .passed((Boolean) ((Map<?, ?>) map).get(VALID_KEY))
+                            .actualValue(((Map<?, ?>) map).containsKey(ACTUAL_KEY) ? ((Map<?, ?>) map).get(ACTUAL_KEY).toString() : "")
+                            .build())
+                    .orElse(OnDemandUrlCheckerAssertionResult.builder()
+                            .passed(false)
+                            .actualValue("Unknown")
+                            .build());
+        }
+        return null;
+    }
+
+    private Boolean doAdditionalPropertiesExist(SyntheticsAPITestResultFull fullTestResults) {
+        return fullTestResults.getResult().getAdditionalProperties().containsKey(ASSERTION_RESULTS_KEY)
+                && fullTestResults.getResult().getAdditionalProperties().get(ASSERTION_RESULTS_KEY) instanceof List<?>;
+    }
+
+    private boolean isErrorIgnorable(SyntheticsApiTestFailureCode errorCode) {
+        return errorsToIgnore.stream()
+                .filter(code -> code.equals(errorCode.getValue()))
+                .findAny()
+                .isPresent();
+    }
+
+}

chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlCheckerResponse.java

@@ -0,0 +1,15 @@
+package gov.healthit.chpl.datadog;
+
+import lombok.Builder;
+import lombok.Data;
+
+@Data
+@Builder
+public class OnDemandUrlCheckerResponse {
+    private String url;
+    private String errorMessage;
+    private Boolean passed;
+    private OnDemandUrlCheckerAssertionResult responseTimeAssertion;
+    private OnDemandUrlCheckerAssertionResult bodyNotEmptyAssertion;
+    private OnDemandUrlCheckerAssertionResult httpResponseAssertion;
+}

chpl/chpl-service/src/main/java/gov/healthit/chpl/datadog/OnDemandUrlRequest.java

@@ -0,0 +1,18 @@
+package gov.healthit.chpl.datadog;
+
+import java.io.Serializable;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class OnDemandUrlRequest implements Serializable {
+    private static final long serialVersionUID = -3009297190983937267L;
+
+    private String url;
+}

chpl/chpl-service/src/main/java/gov/healthit/chpl/permissions/Permissions.java

@@ -39,6 +39,7 @@
 import gov.healthit.chpl.permissions.domains.TestToolDomainPermissions;
 import gov.healthit.chpl.permissions.domains.TestingLabDomainPermissions;
 import gov.healthit.chpl.permissions.domains.UcdProcessDomainPermissions;
+import gov.healthit.chpl.permissions.domains.UrlCheckerDomainPermissions;
 import gov.healthit.chpl.permissions.domains.UserPermissionsDomainPermissions;
 
 @Component
@@ -79,6 +80,7 @@ public class Permissions {
     public static final String STANDARD = "STANDARD";
     public static final String CODE_SET = "CODE_SET";
     public static final String API_KEY = "API_KEY";
+    public static final String URL_CHECKER = "URL_CHECKER";
 
     private Map<String, DomainPermissions> domainPermissions = new HashMap<String, DomainPermissions>();
 
@@ -116,7 +118,8 @@ public Permissions(CertificationResultsDomainPermissions certificationResultsDom
             FunctionalityTestedDomainPermissions functionalityTestedDomainPermissions,
             StandardDomainPermissions standardDomainPermissions,
             CodeSetDomainPermissions codeSetPermissions,
-            ApiKeyDomainPermissions apiKeyPermissions) {
+            ApiKeyDomainPermissions apiKeyPermissions,
+            UrlCheckerDomainPermissions urlCheckerDomainPermissions) {
 
         domainPermissions.put(ACCESSIBILITY_STANDARD, accessibilityStandardDomainPermissions);
         domainPermissions.put(ACTIVITY, activityDomainPermissions);
@@ -151,6 +154,7 @@ public Permissions(CertificationResultsDomainPermissions certificationResultsDom
         domainPermissions.put(UCD_PROCESS, ucdProcessDomainPermissions);
         domainPermissions.put(USER_PERMISSIONS, userPermissionsDomainPermissions);
         domainPermissions.put(API_KEY, apiKeyPermissions);
+        domainPermissions.put(URL_CHECKER, urlCheckerDomainPermissions);
     }
 
     public boolean hasAccess(final String domain, final String action) {

chpl/chpl-service/src/main/java/gov/healthit/chpl/permissions/domains/UrlCheckerDomainPermissions.java

@@ -0,0 +1,18 @@
+package gov.healthit.chpl.permissions.domains;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.stereotype.Component;
+
+import gov.healthit.chpl.permissions.domains.testinglab.urlchecker.CheckActionPermissions;
+
+@Component
+public class UrlCheckerDomainPermissions extends DomainPermissions {
+    public static final String CHECK = "CHECK";
+
+    @Autowired
+    public UrlCheckerDomainPermissions(
+            @Qualifier("urlCheckerCheckActionPermissions") CheckActionPermissions checkActionPermissions) {
+        getActionPermissions().put(CHECK, checkActionPermissions);
+    }
+}