Destination S3-V2: Maybe fix metadata issue (#52703)

airbytehq · Feb 3, 2025 · 5d2ba55 · 5d2ba55
1 parent e021e47
commit 5d2ba55
Show file tree

Hide file tree

Showing 4 changed files with 4 additions and 3 deletions.
diff --git a/...n/kotlin/io/airbyte/cdk/load/state/object_storage/ObjectStorageDestinationStateManager.kt b/...n/kotlin/io/airbyte/cdk/load/state/object_storage/ObjectStorageDestinationStateManager.kt
@@ -56,6 +56,7 @@ class ObjectStorageDestinationState(
         return client
             .list(pathFactory.getLongestStreamConstantPrefix(stream, isStaging = false))
             .filter { matcher.match(it.key) != null }
+            .toList() // Force the list call to complete before initiating metadata calls
             .mapNotNull { obj ->
                 val generationId =
                     client.getMetadata(obj.key)[METADATA_GENERATION_ID_KEY]?.toLongOrNull() ?: 0L
@@ -65,7 +66,6 @@ class ObjectStorageDestinationState(
                     null
                 }
             }
-            .toList()
     }
 
     /**

diff --git a/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt b/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt
@@ -68,7 +68,7 @@ class S3Client(
                 lastKey = obj.key
                 emit(S3Object(obj.key!!, bucketConfig))
             } // null contents => empty list, not error
-            if (client.listObjects(request).isTruncated == false) {
+            if (response.isTruncated == false) {
                 break
             }
             request = request.copy { marker = lastKey }

diff --git a/airbyte-integrations/connectors/destination-s3/metadata.yaml b/airbyte-integrations/connectors/destination-s3/metadata.yaml
@@ -2,7 +2,7 @@ data:
   connectorSubtype: file
   connectorType: destination
   definitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362
-  dockerImageTag: 1.5.0-rc.17
+  dockerImageTag: 1.5.0-rc.18
   dockerRepository: airbyte/destination-s3
   githubIssueLabel: destination-s3
   icon: s3.svg

diff --git a/docs/integrations/destinations/s3.md b/docs/integrations/destinations/s3.md
@@ -544,6 +544,7 @@ To see connector limitations, or troubleshoot your S3 connector, see more [in ou
 
 | Version     | Date       | Pull Request                                               | Subject                                                                                                                                              |
 |:------------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------|
+| 1.5.0-rc.18 | 2025-01-29 | [52703](https://github.com/airbytehq/airbyte/pull/52703)   | Force list call evaluation before making head calls                                                                                                  |
 | 1.5.0-rc.17 | 2025-01-29 | [52610](https://github.com/airbytehq/airbyte/pull/52642)   | Pin CDK 0.296                                                                                                                                        |
 | 1.5.0-rc.16 | 2025-01-29 | [52610](https://github.com/airbytehq/airbyte/pull/52610)   | Fix assume role behavior                                                                                                                             |
 | 1.5.0-rc.15 | 2025-01-23 | [52103](https://github.com/airbytehq/airbyte/pull/52103)   | Make the connector use our non root base image.                                                                                                      |