diff --git a/lib/s3.js b/lib/s3.js index c8f2e2c..7a9e61d 100644 --- a/lib/s3.js +++ b/lib/s3.js @@ -17,8 +17,11 @@ const { createWriteStream, readdir, ensureDir, + stat, } = require("fs-extra"); +const AWS = require("aws-sdk"); const path = require("path"); +const MB = 1024 * 1024; // https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html const maxFileNameLength = 1024; @@ -69,6 +72,7 @@ class Bucket { }) { let configuration = { forcePathStyle, + s3ForcePathStyle: forcePathStyle, }; if (!bucket) { @@ -84,6 +88,7 @@ class Bucket { } if (region) configuration.region = region; this.client = new S3Client(configuration); + this.configuration = configuration; } async stat({ path }) { @@ -137,23 +142,64 @@ class Bucket { ); } - const uploader = new Upload({ - client: this.client, - queueSize: 4, // optional concurrency configuration - leavePartsOnError: false, // optional manually handle dropped parts + let chunkSize = 5 * MB; + if (localPath) { + let fileStat = await stat(localPath); + const desiredChunkSize = await Math.ceil(fileStat.size / 10000); + // at least 5MB per request, at most 10k requests + const minChunkSize = Math.max(5 * MB, Math.ceil(fileStat.size / 10000)); + chunkSize = Math.max(desiredChunkSize, minChunkSize); + } + + let uploader = V2MultipartUpload.bind(this); + let response = await uploader({ params: uploadParams, + partSize: chunkSize, }); + return response; + + async function V3SinglePartUpload({ params }) { + // Straight up V3 upload - not multipart + const command = new PutObjectCommand(uploadParams); + let response = await this.client.send(command); + return response.$metadata; + } + + async function V3MultipartUpload({ params, partSize }) { + // V3 multipart uploader - doesn't work + // https://github.com/aws/aws-sdk-js-v3/issues/1814 + const uploader = new Upload({ + client: this.client, + partSize, + queueSize: 4, // optional concurrency configuration + leavePartsOnError: false, // optional manually handle dropped parts + params, + }); - // uploader.on("httpUploadProgress", (progress) => { - // console.log(progress); - // }); + uploader.on("httpUploadProgress", (progress) => { + console.log(progress); + }); - const response = await uploader.done(); + const response = await uploader.done(); + return response.$metadata; + } - // const command = new PutObjectCommand(uploadParams); - // let response = await this.client.send(command); + async function V2MultipartUpload({ params, partSize }) { + // V2 multipart uploader + AWS.config = this.configuration; + const uploader = new AWS.S3.ManagedUpload({ + partSize, + params, + }); + // uploader.on("httpUploadProgress", (progress) => { + // console.log(progress); + // }); - return response.$metadata; + let response = await uploader.promise(); + return { + httpStatusCode: 200, + }; + } } async download({ target, localPath }) { diff --git a/package-lock.json b/package-lock.json index 9afa1b7..c172091 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@aws-sdk/client-s3": "^3.9.0", "@aws-sdk/lib-storage": "^3.9.0", "@aws-sdk/s3-request-presigner": "^3.9.0", + "aws-sdk": "^2.872.0", "fs-extra": "^9.1.0", "hasha": "^5.2.2", "lodash": "^4.17.21", @@ -4084,6 +4085,67 @@ "node": ">= 4.5.0" } }, + "node_modules/aws-sdk": { + "version": "2.872.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.872.0.tgz", + "integrity": "sha512-hI1/iwR1uPbuulvWZmCCmLKN1Oiv+Beutwcn+7ZOsWAEtsgsXiHmuRDS/ZdWiBRNQkfZgUhcCwLz7nOrWKpb8w==", + "dependencies": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/aws-sdk/node_modules/buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "dependencies": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "node_modules/aws-sdk/node_modules/ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, + "node_modules/aws-sdk/node_modules/punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + }, + "node_modules/aws-sdk/node_modules/sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, + "node_modules/aws-sdk/node_modules/url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "dependencies": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "node_modules/aws-sdk/node_modules/uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", + "bin": { + "uuid": "bin/uuid" + } + }, "node_modules/aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", @@ -5412,6 +5474,14 @@ "node": ">=6" } }, + "node_modules/events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=", + "engines": { + "node": ">=0.4.x" + } + }, "node_modules/exec-sh": { "version": "0.3.4", "resolved": "https://registry.npmjs.org/exec-sh/-/exec-sh-0.3.4.tgz", @@ -7332,6 +7402,14 @@ "jetify": "bin/jetify" } }, + "node_modules/jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=", + "engines": { + "node": ">= 0.6.0" + } + }, "node_modules/joi": { "version": "17.4.0", "resolved": "https://registry.npmjs.org/joi/-/joi-17.4.0.tgz", @@ -9883,8 +9961,7 @@ "node_modules/sax": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", - "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==", - "peer": true + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, "node_modules/saxes": { "version": "5.0.1", @@ -11393,11 +11470,19 @@ "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==", "dev": true }, + "node_modules/xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, "node_modules/xmlbuilder": { "version": "9.0.7", "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=", - "peer": true, "engines": { "node": ">=4.0" } @@ -15089,6 +15174,63 @@ "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==" }, + "aws-sdk": { + "version": "2.872.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.872.0.tgz", + "integrity": "sha512-hI1/iwR1uPbuulvWZmCCmLKN1Oiv+Beutwcn+7ZOsWAEtsgsXiHmuRDS/ZdWiBRNQkfZgUhcCwLz7nOrWKpb8w==", + "requires": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + }, + "dependencies": { + "buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + }, + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + } + } + }, "aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", @@ -16145,6 +16287,11 @@ "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", "peer": true }, + "events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" + }, "exec-sh": { "version": "0.3.4", "resolved": "https://registry.npmjs.org/exec-sh/-/exec-sh-0.3.4.tgz", @@ -17648,6 +17795,11 @@ "integrity": "sha512-JNAkmPeB/GS2tCRqUzRPsTOHpGDah7xP18vGJfIjZC+W2sxEHbxgJxetIjIqhjQ3yYbYNEELkM/spKLtwoOSUQ==", "peer": true }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" + }, "joi": { "version": "17.4.0", "resolved": "https://registry.npmjs.org/joi/-/joi-17.4.0.tgz", @@ -19770,8 +19922,7 @@ "sax": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", - "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==", - "peer": true + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, "saxes": { "version": "5.0.1", @@ -21002,11 +21153,19 @@ "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==", "dev": true }, + "xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, "xmlbuilder": { "version": "9.0.7", "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", - "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=", - "peer": true + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" }, "xmlchars": { "version": "2.2.0", diff --git a/package.json b/package.json index 88badcb..dc39a34 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "@aws-sdk/client-s3": "^3.9.0", "@aws-sdk/lib-storage": "^3.9.0", "@aws-sdk/s3-request-presigner": "^3.9.0", + "aws-sdk": "^2.872.0", "fs-extra": "^9.1.0", "hasha": "^5.2.2", "lodash": "^4.17.21",