From 0a6632188a81d3868d32b5a734492d121ecb4ac4 Mon Sep 17 00:00:00 2001 From: Bella Luz Quintero Date: Wed, 15 Jan 2025 11:01:14 -0700 Subject: [PATCH] modify isInDateFolder method to include added folders (goldencopy/assertion) Co-authored-by: Sylvie Co-authored-by: saquino0827 Co-authored-by: James Herr Co-authored-by: basiliskus <541149+basiliskus@users.noreply.github.com> --- .../rse2e/external/azure/AzureBlobHelper.java | 6 ++++-- .../external/azure/AzureBlobOrganizer.java | 20 +++++++++++-------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/external/azure/AzureBlobHelper.java b/rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/external/azure/AzureBlobHelper.java index 5423e1bdd..049e71112 100644 --- a/rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/external/azure/AzureBlobHelper.java +++ b/rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/external/azure/AzureBlobHelper.java @@ -18,8 +18,10 @@ public static String createDateBasedPath(LocalDate date, String originalName) { return buildDatePathPrefix(date) + originalName; } - public static boolean isInDateFolder(String blobPath, LocalDate creationDate) { - String expectedPath = buildDatePathPrefix(creationDate); + public static boolean isInDateFolder(String testType, String blobPath, LocalDate creationDate) { + // /GOLDEN_COPY/2021/09/01/ + // /ASSERTION/2021/09/01/ + String expectedPath = testType + buildDatePathPrefix(creationDate); return blobPath.startsWith(expectedPath); } } diff --git a/rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/external/azure/AzureBlobOrganizer.java b/rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/external/azure/AzureBlobOrganizer.java index 2d274b365..fa550f77c 100644 --- a/rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/external/azure/AzureBlobOrganizer.java +++ b/rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/external/azure/AzureBlobOrganizer.java @@ -25,7 +25,9 @@ public AzureBlobOrganizer(BlobContainerClient blobContainerClient) { this.blobContainerClient = blobContainerClient; } - private void deleteOldBlobs(String testType, ZoneId timeZone) { + // Delete the current date folder's blobs to prevent buildup when ran manually + private void deleteOldBlobsFromToday(String testType, ZoneId timeZone) { + // Check folder structures: YEAR/MONTH/DAY/Assertion_OR_GoldenCopy/ String prefix = LocalDate.now(timeZone).format(DateTimeFormatter.ofPattern("yyyy/MM/dd")) + "/" @@ -41,8 +43,8 @@ private void deleteOldBlobs(String testType, ZoneId timeZone) { // Organize blob into folder structure: YEAR/MONTH/DAY/Assertion_OR_GoldenCopy/SOURCE_NAME public void organizeAndCleanupBlobsByDate(int retentionDays, ZoneId timeZone) { - deleteOldBlobs(GOLDEN_COPY, timeZone); - deleteOldBlobs(ASSERTION, timeZone); + deleteOldBlobsFromToday(GOLDEN_COPY, timeZone); + deleteOldBlobsFromToday(ASSERTION, timeZone); for (BlobItem blobItem : blobContainerClient.listBlobs()) { String sourceName = blobItem.getName(); try { @@ -62,15 +64,17 @@ public void organizeAndCleanupBlobsByDate(int retentionDays, ZoneId timeZone) { continue; } - if (AzureBlobHelper.isInDateFolder(sourceName, sourceCreationDate)) { - continue; - } - - String testTypeAndSourceName = ASSERTION + sourceName; + String testType = ASSERTION; + String testTypeAndSourceName = testType + sourceName; if (sourceBlob.getBlobName().contains("GOLDEN-COPY")) { + testType = GOLDEN_COPY; testTypeAndSourceName = GOLDEN_COPY + sourceName; } + if (AzureBlobHelper.isInDateFolder(testType, sourceName, sourceCreationDate)) { + continue; + } + System.out.println("Organizing blob: " + testTypeAndSourceName); String destinationName =