From 19b1e4fd3dde9c8c60139c7f82e5e075e743b14d Mon Sep 17 00:00:00 2001 From: Brendan Bond Date: Thu, 12 Oct 2023 12:51:23 -0500 Subject: [PATCH] FIO 7239: support for AWS S3 Multipart Upload (#5356) * add multipart checkbox to file component editForm * refactor xhr in storage and add support for s3 multipart uploads * refactor to async/await * add support for midstream cancel * minor * refactor withRetries into util * begin tests * minor --- src/Formio.js | 4 +- src/components/file/File.js | 18 ++ .../file/editForm/File.edit.file.js | 40 +++ src/providers/storage/azure.js | 46 ++-- src/providers/storage/base64.js | 56 +++-- src/providers/storage/dropbox.js | 104 ++++---- src/providers/storage/googleDrive.js | 102 ++++---- src/providers/storage/indexeddb.js | 230 +++++++++--------- src/providers/storage/s3.js | 167 ++++++++++--- src/providers/storage/s3.unit.js | 80 ++++++ src/providers/storage/url.js | 4 +- src/providers/storage/util.js | 6 + src/providers/storage/xhr.js | 150 ++++++------ 13 files changed, 634 insertions(+), 373 deletions(-) create mode 100644 src/providers/storage/s3.unit.js create mode 100644 src/providers/storage/util.js diff --git a/src/Formio.js b/src/Formio.js index e7a9613080..339b8a1715 100644 --- a/src/Formio.js +++ b/src/Formio.js @@ -6,7 +6,7 @@ Formio.Providers = Providers; Formio.version = 'FORMIO_VERSION'; const isNil = (val) => val === null || val === undefined; -Formio.prototype.uploadFile = function(storage, file, fileName, dir, progressCallback, url, options, fileKey, groupPermissions, groupId, uploadStartCallback, abortCallback) { +Formio.prototype.uploadFile = function(storage, file, fileName, dir, progressCallback, url, options, fileKey, groupPermissions, groupId, uploadStartCallback, abortCallback, multipartOptions) { const requestArgs = { provider: storage, method: 'upload', @@ -26,7 +26,7 @@ Formio.prototype.uploadFile = function(storage, file, fileName, dir, progressCal if (uploadStartCallback) { uploadStartCallback(); } - return provider.uploadFile(file, fileName, dir, progressCallback, url, options, fileKey, groupPermissions, groupId, abortCallback); + return provider.uploadFile(file, fileName, dir, progressCallback, url, options, fileKey, groupPermissions, groupId, abortCallback, multipartOptions); } else { throw ('Storage provider not found'); diff --git a/src/components/file/File.js b/src/components/file/File.js index d08bf62274..609d595471 100644 --- a/src/components/file/File.js +++ b/src/components/file/File.js @@ -789,6 +789,23 @@ export default class FileComponent extends Field { } } + // Prep for a potential multipart upload + let count = 0; + const multipartOptions = this.component.useMultipartUpload && this.component.multipart ? { + ...this.component.multipart, + progressCallback: (total) => { + count++; + fileUpload.status = 'progress'; + fileUpload.progress = parseInt(100 * count / total); + delete fileUpload.message; + this.redraw(); + }, + changeMessage: (message) => { + fileUpload.message = message; + this.redraw(); + }, + } : false; + fileUpload.message = this.t('Starting upload...'); this.redraw(); @@ -814,6 +831,7 @@ export default class FileComponent extends Field { this.emit('fileUploadingStart', filePromise); }, (abort) => fileUpload.abort = abort, + multipartOptions ).then((fileInfo) => { const index = this.statuses.indexOf(fileUpload); if (index !== -1) { diff --git a/src/components/file/editForm/File.edit.file.js b/src/components/file/editForm/File.edit.file.js index cc369f0754..e76dba3a0d 100644 --- a/src/components/file/editForm/File.edit.file.js +++ b/src/components/file/editForm/File.edit.file.js @@ -21,6 +21,46 @@ export default [ } } }, + { + type: 'checkbox', + input: true, + key: 'useMultipartUpload', + label: 'Use the S3 Multipart Upload API', + tooltip: "The S3 Multipart Upload API is designed to improve the upload experience for larger objects (> 5GB).", + conditional: { + json: { '===': [{ var: 'data.storage' }, 's3'] } + }, + }, + { + label: 'Multipart Upload', + tableView: false, + key: 'multipart', + type: 'container', + input: true, + components: [ + { + label: 'Part Size (MB)', + applyMaskOn: 'change', + mask: false, + tableView: false, + delimiter: false, + requireDecimal: false, + inputFormat: 'plain', + truncateMultipleSpaces: false, + validate: { + min: 5, + max: 5000, + }, + key: 'partSize', + type: 'number', + input: true, + defaultValue: 500, + }, + ], + conditional: { + json: { '===': [{ var: 'data.useMultipartUpload' }, true] } + }, + }, { type: 'textfield', input: true, diff --git a/src/providers/storage/azure.js b/src/providers/storage/azure.js index 9eb301f0ac..60249bce75 100644 --- a/src/providers/storage/azure.js +++ b/src/providers/storage/azure.js @@ -1,26 +1,28 @@ import XHR from './xhr'; -const azure = (formio) => ({ - uploadFile(file, fileName, dir, progressCallback, url, options, fileKey, groupPermissions, groupId, abortCallback) { - return XHR.upload(formio, 'azure', (xhr, response) => { - xhr.openAndSetHeaders('PUT', response.url); - xhr.setRequestHeader('Content-Type', file.type); - xhr.setRequestHeader('x-ms-blob-type', 'BlockBlob'); - return file; - }, file, fileName, dir, progressCallback, groupPermissions, groupId, abortCallback).then(() => { - return { - storage: 'azure', - name: XHR.path([dir, fileName]), - size: file.size, - type: file.type, - groupPermissions, - groupId, - }; - }); - }, - downloadFile(file) { - return formio.makeRequest('file', `${formio.formUrl}/storage/azure?name=${XHR.trim(file.name)}`, 'GET'); - } -}); +function azure(formio) { + return { + uploadFile(file, fileName, dir, progressCallback, url, options, fileKey, groupPermissions, groupId, abortCallback) { + return XHR.upload(formio, 'azure', (xhr, response) => { + xhr.openAndSetHeaders('PUT', response.url); + xhr.setRequestHeader('Content-Type', file.type); + xhr.setRequestHeader('x-ms-blob-type', 'BlockBlob'); + return file; + }, file, fileName, dir, progressCallback, groupPermissions, groupId, abortCallback).then(() => { + return { + storage: 'azure', + name: XHR.path([dir, fileName]), + size: file.size, + type: file.type, + groupPermissions, + groupId, + }; + }); + }, + downloadFile(file) { + return formio.makeRequest('file', `${formio.formUrl}/storage/azure?name=${XHR.trim(file.name)}`, 'GET'); + } + }; +} azure.title = 'Azure File Services'; export default azure; diff --git a/src/providers/storage/base64.js b/src/providers/storage/base64.js index 9e8a080295..5369f8da84 100644 --- a/src/providers/storage/base64.js +++ b/src/providers/storage/base64.js @@ -1,33 +1,35 @@ -const base64 = () => ({ - title: 'Base64', - name: 'base64', - uploadFile(file, fileName) { - const reader = new FileReader(); +function base64() { + return { + title: 'Base64', + name: 'base64', + uploadFile(file, fileName) { + const reader = new FileReader(); - return new Promise((resolve, reject) => { - reader.onload = (event) => { - const url = event.target.result; - resolve({ - storage: 'base64', - name: fileName, - url: url, - size: file.size, - type: file.type, - }); - }; + return new Promise((resolve, reject) => { + reader.onload = (event) => { + const url = event.target.result; + resolve({ + storage: 'base64', + name: fileName, + url: url, + size: file.size, + type: file.type, + }); + }; - reader.onerror = () => { - return reject(this); - }; + reader.onerror = () => { + return reject(this); + }; - reader.readAsDataURL(file); - }); - }, - downloadFile(file) { - // Return the original as there is nothing to do. - return Promise.resolve(file); - } -}); + reader.readAsDataURL(file); + }); + }, + downloadFile(file) { + // Return the original as there is nothing to do. + return Promise.resolve(file); + } + }; +} base64.title = 'Base64'; export default base64; diff --git a/src/providers/storage/dropbox.js b/src/providers/storage/dropbox.js index 64122202bc..b1229fd674 100644 --- a/src/providers/storage/dropbox.js +++ b/src/providers/storage/dropbox.js @@ -1,65 +1,67 @@ import { setXhrHeaders } from './xhr'; -const dropbox = (formio) => ({ - uploadFile(file, fileName, dir, progressCallback, url, options, fileKey, groupPermissions, groupId, abortCallback) { - return new Promise(((resolve, reject) => { - // Send the file with data. - const xhr = new XMLHttpRequest(); +function dropbox(formio) { + return { + uploadFile(file, fileName, dir, progressCallback, url, options, fileKey, groupPermissions, groupId, abortCallback) { + return new Promise(((resolve, reject) => { + // Send the file with data. + const xhr = new XMLHttpRequest(); - if (typeof progressCallback === 'function') { - xhr.upload.onprogress = progressCallback; - } + if (typeof progressCallback === 'function') { + xhr.upload.onprogress = progressCallback; + } - if (typeof abortCallback === 'function') { - abortCallback(() => xhr.abort()); - } + if (typeof abortCallback === 'function') { + abortCallback(() => xhr.abort()); + } - const fd = new FormData(); - fd.append('name', fileName); - fd.append('dir', dir); - fd.append('file', file); + const fd = new FormData(); + fd.append('name', fileName); + fd.append('dir', dir); + fd.append('file', file); - // Fire on network error. - xhr.onerror = (err) => { - err.networkError = true; - reject(err); - }; + // Fire on network error. + xhr.onerror = (err) => { + err.networkError = true; + reject(err); + }; - xhr.onload = () => { - if (xhr.status >= 200 && xhr.status < 300) { - const response = JSON.parse(xhr.response); - response.storage = 'dropbox'; - response.size = file.size; - response.type = file.type; - response.groupId = groupId; - response.groupPermissions = groupPermissions; - response.url = response.path_lower; - resolve(response); - } - else { - reject(xhr.response || 'Unable to upload file'); - } - }; + xhr.onload = () => { + if (xhr.status >= 200 && xhr.status < 300) { + const response = JSON.parse(xhr.response); + response.storage = 'dropbox'; + response.size = file.size; + response.type = file.type; + response.groupId = groupId; + response.groupPermissions = groupPermissions; + response.url = response.path_lower; + resolve(response); + } + else { + reject(xhr.response || 'Unable to upload file'); + } + }; - xhr.onabort = reject; + xhr.onabort = reject; - xhr.open('POST', `${formio.formUrl}/storage/dropbox`); + xhr.open('POST', `${formio.formUrl}/storage/dropbox`); - setXhrHeaders(formio, xhr); + setXhrHeaders(formio, xhr); + const token = formio.getToken(); + if (token) { + xhr.setRequestHeader('x-jwt-token', token); + } + xhr.send(fd); + })); + }, + downloadFile(file) { const token = formio.getToken(); - if (token) { - xhr.setRequestHeader('x-jwt-token', token); - } - xhr.send(fd); - })); - }, - downloadFile(file) { - const token = formio.getToken(); - file.url = - `${formio.formUrl}/storage/dropbox?path_lower=${file.path_lower}${token ? `&x-jwt-token=${token}` : ''}`; - return Promise.resolve(file); - } -}); + file.url = + `${formio.formUrl}/storage/dropbox?path_lower=${file.path_lower}${token ? `&x-jwt-token=${token}` : ''}`; + return Promise.resolve(file); + } + }; +} dropbox.title = 'Dropbox'; export default dropbox; diff --git a/src/providers/storage/googleDrive.js b/src/providers/storage/googleDrive.js index 3540669ffe..639ba1093b 100644 --- a/src/providers/storage/googleDrive.js +++ b/src/providers/storage/googleDrive.js @@ -1,64 +1,66 @@ import { setXhrHeaders } from './xhr'; -const googledrive = (formio) => ({ - uploadFile(file, fileName, dir, progressCallback, url, options, fileKey, groupPermissions, groupId, abortCallback) { - return new Promise(((resolve, reject) => { - // Send the file with data. - const xhr = new XMLHttpRequest(); +function googledrive(formio) { + return { + uploadFile(file, fileName, dir, progressCallback, url, options, fileKey, groupPermissions, groupId, abortCallback) { + return new Promise(((resolve, reject) => { + // Send the file with data. + const xhr = new XMLHttpRequest(); - if (typeof progressCallback === 'function') { - xhr.upload.onprogress = progressCallback; - } + if (typeof progressCallback === 'function') { + xhr.upload.onprogress = progressCallback; + } - if (typeof abortCallback === 'function') { - abortCallback(() => xhr.abort()); - } + if (typeof abortCallback === 'function') { + abortCallback(() => xhr.abort()); + } - const fd = new FormData(); - fd.append('name', fileName); - fd.append('dir', dir); - fd.append('file', file); + const fd = new FormData(); + fd.append('name', fileName); + fd.append('dir', dir); + fd.append('file', file); - // Fire on network error. - xhr.onerror = (err) => { - err.networkError = true; - reject(err); - }; + // Fire on network error. + xhr.onerror = (err) => { + err.networkError = true; + reject(err); + }; - xhr.onload = () => { - if (xhr.status >= 200 && xhr.status < 300) { - const response = JSON.parse(xhr.response); - response.storage = 'googledrive'; - response.size = file.size; - response.type = file.type; - response.groupId = groupId; - response.groupPermissions = groupPermissions; - resolve(response); - } - else { - reject(xhr.response || 'Unable to upload file'); - } - }; + xhr.onload = () => { + if (xhr.status >= 200 && xhr.status < 300) { + const response = JSON.parse(xhr.response); + response.storage = 'googledrive'; + response.size = file.size; + response.type = file.type; + response.groupId = groupId; + response.groupPermissions = groupPermissions; + resolve(response); + } + else { + reject(xhr.response || 'Unable to upload file'); + } + }; - xhr.onabort = reject; + xhr.onabort = reject; - xhr.open('POST', `${formio.formUrl}/storage/gdrive`); + xhr.open('POST', `${formio.formUrl}/storage/gdrive`); - setXhrHeaders(formio, xhr); + setXhrHeaders(formio, xhr); + const token = formio.getToken(); + if (token) { + xhr.setRequestHeader('x-jwt-token', token); + } + xhr.send(fd); + })); + }, + downloadFile(file) { const token = formio.getToken(); - if (token) { - xhr.setRequestHeader('x-jwt-token', token); - } - xhr.send(fd); - })); - }, - downloadFile(file) { - const token = formio.getToken(); - file.url = - `${formio.formUrl}/storage/gdrive?fileId=${file.id}&fileName=${file.originalName}${token ? `&x-jwt-token=${token}` : ''}`; - return Promise.resolve(file); - } -}); + file.url = + `${formio.formUrl}/storage/gdrive?fileId=${file.id}&fileName=${file.originalName}${token ? `&x-jwt-token=${token}` : ''}`; + return Promise.resolve(file); + } + }; +} googledrive.title = 'Google Drive'; export default googledrive; diff --git a/src/providers/storage/indexeddb.js b/src/providers/storage/indexeddb.js index a24f7f4b8f..9e4d5d25ef 100644 --- a/src/providers/storage/indexeddb.js +++ b/src/providers/storage/indexeddb.js @@ -1,135 +1,137 @@ import { v4 as uuidv4 } from 'uuid'; -const indexeddb = () => ({ - title: 'indexedDB', - name: 'indexeddb', - uploadFile(file, fileName, dir, progressCallback, url, options) { - if (!('indexedDB' in window)) { - console.log('This browser doesn\'t support IndexedDB'); - return; - } - - return new Promise((resolve) => { - const request = indexedDB.open(options.indexeddb); - request.onsuccess = function(event) { - const db = event.target.result; - resolve(db); - }; - request.onupgradeneeded = function(e) { - const db = e.target.result; - db.createObjectStore(options.indexeddbTable); - }; - }).then((db) => { - const reader = new FileReader(); - - return new Promise((resolve, reject) => { - reader.onload = () => { - const blobObject = new Blob([file], { type: file.type }); - - const id = uuidv4(blobObject); - - const data = { - id, - data: blobObject, - name: file.name, - size: file.size, - type: file.type, - url, - }; +function indexeddb() { + return { + title: 'indexedDB', + name: 'indexeddb', + uploadFile(file, fileName, dir, progressCallback, url, options) { + if (!('indexedDB' in window)) { + console.log('This browser doesn\'t support IndexedDB'); + return; + } + + return new Promise((resolve) => { + const request = indexedDB.open(options.indexeddb); + request.onsuccess = function(event) { + const db = event.target.result; + resolve(db); + }; + request.onupgradeneeded = function(e) { + const db = e.target.result; + db.createObjectStore(options.indexeddbTable); + }; + }).then((db) => { + const reader = new FileReader(); - const trans = db.transaction([options.indexeddbTable], 'readwrite'); - const addReq = trans.objectStore(options.indexeddbTable).put(data, id); + return new Promise((resolve, reject) => { + reader.onload = () => { + const blobObject = new Blob([file], { type: file.type }); - addReq.onerror = function(e) { - console.log('error storing data'); - console.error(e); - }; + const id = uuidv4(blobObject); - trans.oncomplete = function() { - resolve({ - storage: 'indexeddb', + const data = { + id, + data: blobObject, name: file.name, size: file.size, type: file.type, - url: url, - id, - }); - }; - }; + url, + }; - reader.onerror = () => { - return reject(this); - }; + const trans = db.transaction([options.indexeddbTable], 'readwrite'); + const addReq = trans.objectStore(options.indexeddbTable).put(data, id); - reader.readAsDataURL(file); - }); - }); - }, - downloadFile(file, options) { - return new Promise((resolve) => { - const request = indexedDB.open(options.indexeddb); - - request.onsuccess = function(event) { - const db = event.target.result; - resolve(db); - }; - }).then((db) => { - return new Promise((resolve, reject) => { - const trans = db.transaction([options.indexeddbTable], 'readonly'); - const store = trans.objectStore(options.indexeddbTable).get(file.id); - store.onsuccess = () => { - trans.oncomplete = () => { - const result = store.result; - const dbFile = new File([store.result.data], file.name, { - type: store.result.type, - }); - - const reader = new FileReader(); - - reader.onload = (event) => { - result.url = event.target.result; - result.storage = file.storage; - resolve(result); + addReq.onerror = function(e) { + console.log('error storing data'); + console.error(e); }; - reader.onerror = () => { - return reject(this); + trans.oncomplete = function() { + resolve({ + storage: 'indexeddb', + name: file.name, + size: file.size, + type: file.type, + url: url, + id, + }); }; + }; - reader.readAsDataURL(dbFile); + reader.onerror = () => { + return reject(this); }; - }; - store.onerror = () => { - return reject(this); - }; + + reader.readAsDataURL(file); + }); }); - }); - }, - deleteFile(file, options) { - return new Promise((resolve) => { - const request = indexedDB.open(options.indexeddb); - - request.onsuccess = function(event) { - const db = event.target.result; - resolve(db); - }; - }).then((db) => { - return new Promise((resolve, reject) => { - const trans = db.transaction([options.indexeddbTable], 'readwrite'); - const store = trans.objectStore(options.indexeddbTable).delete(file.id); - store.onsuccess = () => { - trans.oncomplete = () => { - const result = store.result; - - resolve(result); - }; + }, + downloadFile(file, options) { + return new Promise((resolve) => { + const request = indexedDB.open(options.indexeddb); + + request.onsuccess = function(event) { + const db = event.target.result; + resolve(db); }; - store.onerror = () => { - return reject(this); + }).then((db) => { + return new Promise((resolve, reject) => { + const trans = db.transaction([options.indexeddbTable], 'readonly'); + const store = trans.objectStore(options.indexeddbTable).get(file.id); + store.onsuccess = () => { + trans.oncomplete = () => { + const result = store.result; + const dbFile = new File([store.result.data], file.name, { + type: store.result.type, + }); + + const reader = new FileReader(); + + reader.onload = (event) => { + result.url = event.target.result; + result.storage = file.storage; + resolve(result); + }; + + reader.onerror = () => { + return reject(this); + }; + + reader.readAsDataURL(dbFile); + }; + }; + store.onerror = () => { + return reject(this); + }; + }); + }); + }, + deleteFile(file, options) { + return new Promise((resolve) => { + const request = indexedDB.open(options.indexeddb); + + request.onsuccess = function(event) { + const db = event.target.result; + resolve(db); }; + }).then((db) => { + return new Promise((resolve, reject) => { + const trans = db.transaction([options.indexeddbTable], 'readwrite'); + const store = trans.objectStore(options.indexeddbTable).delete(file.id); + store.onsuccess = () => { + trans.oncomplete = () => { + const result = store.result; + + resolve(result); + }; + }; + store.onerror = () => { + return reject(this); + }; + }); }); - }); - } -}); + } + }; +} indexeddb.title = 'IndexedDB'; export default indexeddb; diff --git a/src/providers/storage/s3.js b/src/providers/storage/s3.js index 0963e3c57a..2ae8acd0cc 100644 --- a/src/providers/storage/s3.js +++ b/src/providers/storage/s3.js @@ -1,26 +1,71 @@ import XHR from './xhr'; -const s3 = (formio) => ({ - uploadFile(file, fileName, dir, progressCallback, url, options, fileKey, groupPermissions, groupId, abortCallback) { - return XHR.upload(formio, 's3', (xhr, response) => { - response.data.fileName = fileName; - response.data.key = XHR.path([response.data.key, dir, fileName]); - if (response.signed) { - xhr.openAndSetHeaders('PUT', response.signed); - Object.keys(response.data.headers).forEach(key => { - xhr.setRequestHeader(key, response.data.headers[key]); - }); - return file; - } - else { - const fd = new FormData(); - for (const key in response.data) { - fd.append(key, response.data[key]); +import { withRetries } from './util'; +function s3(formio) { + return { + async uploadFile(file, fileName, dir, progressCallback, url, options, fileKey, groupPermissions, groupId, abortCallback, multipartOptions) { + const xhrCallback = async(xhr, response, abortCallback) => { + response.data.fileName = fileName; + response.data.key = XHR.path([response.data.key, dir, fileName]); + if (response.signed) { + if (multipartOptions && Array.isArray(response.signed)) { + // patch abort callback + const abortController = new AbortController(); + const abortSignal = abortController.signal; + if (typeof abortCallback === 'function') { + abortCallback(() => abortController.abort()); + } + try { + const parts = await this.uploadParts( + file, + response.signed, + response.data.headers, + response.partSizeActual, + multipartOptions, + abortSignal + ); + await withRetries(this.completeMultipartUpload, [response, parts, multipartOptions], 3); + return; + } + catch (err) { + // abort in-progress fetch requests + abortController.abort(); + // attempt to cancel the multipart upload + this.abortMultipartUpload(response); + throw err; + } + } + else { + xhr.openAndSetHeaders('PUT', response.signed); + xhr.setRequestHeader('Content-Type', file.type); + Object.keys(response.data.headers).forEach((key) => { + xhr.setRequestHeader(key, response.data.headers[key]); + }); + return file; + } } - fd.append('file', file); - xhr.openAndSetHeaders('POST', response.url); - return fd; - } - }, file, fileName, dir, progressCallback, groupPermissions, groupId, abortCallback).then((response) => { + else { + const fd = new FormData(); + for (const key in response.data) { + fd.append(key, response.data[key]); + } + fd.append('file', file); + xhr.openAndSetHeaders('POST', response.url); + return fd; + } + }; + const response = await XHR.upload( + formio, + 's3', + xhrCallback, + file, + fileName, + dir, + progressCallback, + groupPermissions, + groupId, + abortCallback, + multipartOptions + ); return { storage: 's3', name: fileName, @@ -31,17 +76,77 @@ const s3 = (formio) => ({ size: file.size, type: file.type }; - }); - }, - downloadFile(file) { - if (file.acl !== 'public-read') { - return formio.makeRequest('file', `${formio.formUrl}/storage/s3?bucket=${XHR.trim(file.bucket)}&key=${XHR.trim(file.key)}`, 'GET'); - } - else { - return Promise.resolve(file); + }, + async completeMultipartUpload(serverResponse, parts, multipart) { + const { changeMessage } = multipart; + changeMessage('Completing AWS S3 multipart upload...'); + const response = await fetch(`${formio.formUrl}/storage/s3/multipart/complete`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ parts, uploadId: serverResponse.uploadId, key: serverResponse.key }) + }); + const message = await response.text(); + if (!response.ok) { + throw new Error(message); + } + // the AWS S3 SDK CompleteMultipartUpload command can return a HTTP 200 status header but still error; + // we need to parse, and according to AWS, to retry + if (message.match(/Error/)) { + throw new Error(message); + } + }, + abortMultipartUpload(serverResponse) { + const { uploadId, key } = serverResponse; + fetch(`${formio.formUrl}/storage/s3/multipart/abort`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ uploadId, key }) + }); + }, + uploadParts(file, urls, headers, partSize, multipart, abortSignal) { + const { changeMessage, progressCallback } = multipart; + changeMessage('Chunking and uploading parts to AWS S3...'); + const promises = []; + for (let i = 0; i < urls.length; i++) { + const start = i * partSize; + const end = (i + 1) * partSize; + const blob = i < urls.length ? file.slice(start, end) : file.slice(start); + const promise = fetch(urls[i], { + method: 'PUT', + headers, + body: blob, + signal: abortSignal, + }).then((res) => { + if (res.ok) { + progressCallback(urls.length); + const eTag = res.headers.get('etag'); + if (!eTag) { + throw new Error('ETag header not found; it must be exposed in S3 bucket CORS settings'); + } + return { ETag: eTag, PartNumber: i + 1 }; + } + else { + throw new Error(`Part no ${i} failed with status ${res.status}`); + } + }); + promises.push(promise); + } + return Promise.all(promises); + }, + downloadFile(file) { + if (file.acl !== 'public-read') { + return formio.makeRequest('file', `${formio.formUrl}/storage/s3?bucket=${XHR.trim(file.bucket)}&key=${XHR.trim(file.key)}`, 'GET'); + } + else { + return Promise.resolve(file); + } } - } - }); + }; +} s3.title = 'S3'; export default s3; diff --git a/src/providers/storage/s3.unit.js b/src/providers/storage/s3.unit.js new file mode 100644 index 0000000000..a4160170fa --- /dev/null +++ b/src/providers/storage/s3.unit.js @@ -0,0 +1,80 @@ +import assert from 'assert'; +import sinon from 'sinon'; +import fetchMock from 'fetch-mock'; + +import { Formio } from '../../Formio'; +import S3 from './s3'; +import { withRetries } from './util'; + +describe('S3 Provider', () => { + describe('Function Unit Tests', () => { + it('withRetries should retry a given function three times, then throw the provided error', (done) => { + function sleepAndReject(ms) { + return new Promise((_, reject) => setTimeout(reject, ms)); + } + + const spy = sinon.spy(sleepAndReject); + withRetries(spy, [200], 3, 'Custom error message').catch((err) => { + assert.equal(err.message, 'Custom error message'); + assert.equal(spy.callCount, 3); + done(); + }); + }); + }); + + describe('Provider Integration Tests', () => { + describe('AWS S3 Multipart Uploads', () => { + before('Mocks fetch', () => { + fetchMock + .post('https://fakeproject.form.io/fakeform/storage/s3', { + signed: new Array(5).fill('https://fakebucketurl.aws.com/signed'), + minio: false, + url: 'https://fakebucketurl.aws.com', + bucket: 'fakebucket', + uploadId: 'fakeuploadid', + key: 'test.jpg', + partSizeActual: 1, + data: {} + }) + .put('https://fakebucketurl.aws.com/signed', { status: 200, headers: { 'Etag': 'fakeetag' } }) + .post('https://fakeproject.form.io/fakeform/storage/s3/multipart/complete', 200) + .post('https://fakeproject.form.io/fakeform/storage/s3/multipart/abort', 200); + }); + it('Given an array of signed urls it should upload a file to S3 using multipart upload', (done) => { + const mockFormio = { + formUrl: 'https://fakeproject.form.io/fakeform', + getToken: () => {} + }; + const s3 = new S3(mockFormio); + const uploadSpy = sinon.spy(s3, 'uploadParts'); + const completeSpy = sinon.spy(s3, 'completeMultipartUpload'); + + const mockFile = new File(['test!'], 'test.jpg', { type: 'image/jpeg' }); + s3.uploadFile( + mockFile, + 'test.jpg', + '', + () => {}, + '', + {}, + 'test.jpg', + {}, + '', + () => {}, + { partSize: 1, changeMessage: () => {}, progressCallback: () => {} } + ).then((response) => { + assert.equal(response.storage, 's3'); + assert.equal(response.name, 'test.jpg'); + assert.equal(response.bucket, 'fakebucket'); + assert.equal(response.url, 'https://fakebucketurl.aws.com/test.jpg'); + assert.equal(response.acl, undefined); + assert.equal(response.size, 5); + assert.equal(response.type, 'image/jpeg'); + assert.equal(uploadSpy.callCount, 1); + assert.equal(completeSpy.callCount, 1); + done(); + }); + }); + }); + }); +}); diff --git a/src/providers/storage/url.js b/src/providers/storage/url.js index c58b383752..8acd63162f 100644 --- a/src/providers/storage/url.js +++ b/src/providers/storage/url.js @@ -1,4 +1,4 @@ -const url = (formio) => { +function url(formio) { const xhrRequest = (url, name, query, data, options, progressCallback, abortCallback) => { return new Promise((resolve, reject) => { const xhr = new XMLHttpRequest(); @@ -148,7 +148,7 @@ const url = (formio) => { return Promise.resolve(file); } }; -}; +} url.title = 'Url'; export default url; diff --git a/src/providers/storage/util.js b/src/providers/storage/util.js new file mode 100644 index 0000000000..f84101648f --- /dev/null +++ b/src/providers/storage/util.js @@ -0,0 +1,6 @@ +export async function withRetries(fn, args, retries = 3, err = null) { + if (!retries) { + throw new Error(err); + } + return fn(...args).catch(() => withRetries(fn, args, retries - 1, err)); +} diff --git a/src/providers/storage/xhr.js b/src/providers/storage/xhr.js index 928d6845b5..51f9c8f24c 100644 --- a/src/providers/storage/xhr.js +++ b/src/providers/storage/xhr.js @@ -21,87 +21,89 @@ const XHR = { path(items) { return items.filter(item => !!item).map(XHR.trim).join('/'); }, - upload(formio, type, xhrCb, file, fileName, dir, progressCallback, groupPermissions, groupId, abortCallback) { - return new Promise(((resolve, reject) => { - // Send the pre response to sign the upload. - const pre = new XMLHttpRequest(); - - // This only fires on a network error. - pre.onerror = (err) => { - err.networkError = true; - reject(err); + async upload(formio, type, xhrCallback, file, fileName, dir, progressCallback, groupPermissions, groupId, abortCallback, multipartOptions) { + // make request to Form.io server + const token = formio.getToken(); + let response; + try { + response = await fetch(`${formio.formUrl}/storage/${type}`, { + method: 'POST', + headers: { + 'Accept': 'application/json', + 'Content-Type': 'application/json; charset=UTF-8', + ...(token ? { 'x-jwt-token': token } : {}), + }, + body: JSON.stringify({ + name: XHR.path([dir, fileName]), + size: file.size, + type: file.type, + groupPermissions, + groupId, + multipart: multipartOptions + }) + }); + } + catch (err) { + // only throws on network errors + err.networkError = true; + throw err; + } + if (!response.ok) { + const message = await response.text(); + throw new Error(message || 'Unable to sign file.'); + } + const serverResponse = await response.json(); + return await XHR.makeXhrRequest(formio, xhrCallback, serverResponse, progressCallback, abortCallback); + }, + makeXhrRequest(formio, xhrCallback, serverResponse, progressCallback, abortCallback) { + return new Promise((resolve, reject) => { + // Send the file with data. + const xhr = new XMLHttpRequest(); + xhr.openAndSetHeaders = (...params) => { + xhr.open(...params); + setXhrHeaders(formio, xhr); }; + Promise.resolve(xhrCallback(xhr, serverResponse, abortCallback)).then((payload) => { + // if payload is nullish we can assume the provider took care of the entire upload process + if (!payload) { + return resolve(serverResponse); + } + // Fire on network error. + xhr.onerror = (err) => { + err.networkError = true; + reject(err); + }; - pre.onabort = reject; - pre.onload = () => { - if (pre.status >= 200 && pre.status < 300) { - const response = JSON.parse(pre.response); - - // Send the file with data. - const xhr = new XMLHttpRequest(); - - if (typeof progressCallback === 'function') { - xhr.upload.onprogress = progressCallback; - } - - if (typeof abortCallback === 'function') { - abortCallback(() => xhr.abort()); - } - - xhr.openAndSetHeaders = (...params) => { - xhr.open(...params); - setXhrHeaders(formio, xhr); - }; - - // Fire on network error. - xhr.onerror = (err) => { - err.networkError = true; - reject(err); - }; - - // Fire on network abort. - xhr.onabort = (err) => { - err.networkError = true; - reject(err); - }; - - // Fired when the response has made it back from the server. - xhr.onload = () => { - if (xhr.status >= 200 && xhr.status < 300) { - resolve(response); - } - else { - reject(xhr.response || 'Unable to upload file'); - } - }; + // Fire on network abort. + xhr.onabort = (err) => { + err.networkError = true; + reject(err); + }; - // Set the onabort error callback. - xhr.onabort = reject; + // Set the onabort error callback. + xhr.onabort = reject; - // Get the request and send it to the server. - xhr.send(xhrCb(xhr, response)); - } - else { - reject(pre.response || 'Unable to sign file'); + if (typeof progressCallback === 'function') { + xhr.upload.onprogress = progressCallback; } - }; - pre.open('POST', `${formio.formUrl}/storage/${type}`); - pre.setRequestHeader('Accept', 'application/json'); - pre.setRequestHeader('Content-Type', 'application/json; charset=UTF-8'); - const token = formio.getToken(); - if (token) { - pre.setRequestHeader('x-jwt-token', token); - } + if (typeof abortCallback === 'function') { + abortCallback(() => xhr.abort()); + } + // Fired when the response has made it back from the server. + xhr.onload = () => { + if (xhr.status >= 200 && xhr.status < 300) { + resolve(serverResponse); + } + else { + reject(xhr.response || 'Unable to upload file'); + } + }; - pre.send(JSON.stringify({ - name: XHR.path([dir, fileName]), - size: file.size, - type: file.type, - groupPermissions, - groupId, - })); - })); + // Get the request and send it to the server. + xhr.send(payload); + }).catch(reject); + }); } };