From 34cb68323828f6922ebdd27998b5dab9b2175e79 Mon Sep 17 00:00:00 2001
From: Jip van de Vijfeijke <jip.vandevijfeijke@altran.com>
Date: Tue, 31 May 2022 17:27:11 +0200
Subject: [PATCH] feat: S3 pagination

---
 graph-generator/s3-download.ts | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/graph-generator/s3-download.ts b/graph-generator/s3-download.ts
index c471e93..3d90559 100644
--- a/graph-generator/s3-download.ts
+++ b/graph-generator/s3-download.ts
@@ -28,13 +28,28 @@ export const downloadAndPrepareLogs = async (): Promise<string> => {
     return mergedFileName;
 }
 
+async function getAllKeys(s3, params, allKeys = []) {
+    const response = await s3.listObjectsV2(params).promise();
+    response.Contents.forEach(obj => allKeys.push(obj));
+
+    if (response.NextContinuationToken) {
+        params.ContinuationToken = response.NextContinuationToken;
+        await getAllKeys(s3, params, allKeys);
+    }
+
+    return allKeys;
+}
+
 const downloadLogs = async (): Promise<string[]> => {
     const bucket = process.env.AWS_S3_BUCKET_NAME;
 
-    const s3 = new AWS.S3();
-    const objects = await s3.listObjects({ Bucket: bucket }).promise();
+    const s3 = new AWS.S3()
+
+    const allObjects = await getAllKeys(s3,{ Bucket: bucket });
+    const selectedObjects = allObjects.slice(-2500);
+
 
-    const filePaths: string[] = await Promise.all((objects.Contents || []).map(async object => {
+    const filePaths: string[] = await Promise.all((selectedObjects || []).map(async object => {
         const data = await s3.getObject({ Key: object.Key, Bucket: bucket }).promise();
 
         const fullPath = `data/${object.Key}`;