From c177aebdca36bb246c702886c6c99e30cf2df8be Mon Sep 17 00:00:00 2001 From: Jack Carlisle Date: Thu, 29 Sep 2016 16:53:05 +0100 Subject: [PATCH] adds complete sdkUpload demo with tests #32 --- examples/direct-upload/README.md | 20 ++ examples/sdk-upload/README.md | 306 ++++++++++++++++++++++++ examples/sdk-upload/lib/routes.js | 9 +- examples/sdk-upload/public/client.js | 28 ++- examples/sdk-upload/src/upload-to-s3.js | 2 - examples/sdk-upload/src/upload.js | 25 ++ examples/sdk-upload/test/index.test.js | 2 + examples/sdk-upload/test/server.test.js | 39 +++ examples/sdk-upload/test/upload.test.js | 0 package.json | 6 +- 10 files changed, 427 insertions(+), 10 deletions(-) delete mode 100644 examples/sdk-upload/src/upload-to-s3.js create mode 100644 examples/sdk-upload/src/upload.js create mode 100644 examples/sdk-upload/test/index.test.js create mode 100644 examples/sdk-upload/test/server.test.js create mode 100644 examples/sdk-upload/test/upload.test.js diff --git a/examples/direct-upload/README.md b/examples/direct-upload/README.md index c760bed..91d48b1 100644 --- a/examples/direct-upload/README.md +++ b/examples/direct-upload/README.md @@ -71,8 +71,28 @@ following into the CORS field (*it's basically saying that we are allowing GET, POST and PUT requests from any Allowed Origin with any Allowed Header*) ++ Finally we need to add a policy to the bucket to make it public readable. Click +on the 'Add bucket policy' button and then add the following to the input field: + +``` +{ + "Version": "2008-10-17", + "Statement": [ + { + "Sid": "AllowPublicRead", + "Effect": "Allow", + "Principal": { + "AWS": "*" + }, + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::[YOUR_BUCKET_NAME]/*" + } + ] +} +``` + Then click ***save*** + ![save CORS](https://cloud.githubusercontent.com/assets/12450298/18393882/359e3bf6-76af-11e6-90da-bcd993d035ff.png) #### Our bucket is now completely set up so that it will accept our POST request images! diff --git a/examples/sdk-upload/README.md b/examples/sdk-upload/README.md index e69de29..3a269f5 100644 --- a/examples/sdk-upload/README.md +++ b/examples/sdk-upload/README.md @@ -0,0 +1,306 @@ +# SDK Upload to S3 - A Complete Guide + +We are going to implement a simple solution for uploading images to an S3 bucket +using the AWS SDK. + +![upload example](https://cloud.githubusercontent.com/assets/12450298/18955390/2996f122-864f-11e6-92f5-f2b4c631b2d7.png) + +### Contents +- [Creating an S3 Bucket](#step-1---creating-the-bucket) +- [Creating IAM User with S3 Permissions](#step-2---creating-an-iam-user-with-s3-permissions) +- [AWS SDK Configuration](#step-3---configure-the-aws-adk) +- [Implement the SDK](#step-4---implement-the-sdk) +- [Create a Server](#step-5---create-a-server-to-handle-our-upload-file) +- [Client Side Code](#step-6---write-the-client-side-code-to-send-our-file-to-the-server) +- [Take it for a Spin](#take-it-for-a-spin) +- [Learning Resources](#learning-resources) + +### Step 1 - Creating the bucket + +In our direct-upload example we walk through the steps of how to create a new +bucket in S3. Check out [the tutorial](https://github.com/dwyl/image-uploads/blob/master/examples/direct-upload/README.md#step-1---creating-the-bucket) +and follow the steps. + +### Step 2 - Creating an IAM user with S3 permissions + +Just as in Step 1 we've already created a step-by-step process for creating a +new IAM user that has the correct permissions to access the bucket you created +[above](#step-1---creating-the-bucket). Take a look at [the tutorial](https://github.com/dwyl/image-uploads/blob/master/examples/direct-upload/README.md#step-2---creating-an-iam-user-with-s3-permissions) +and again follow the steps. + +### Step 3 - Configure the AWS SDK + +In order to upload to our S3 bucket, we have to use the AWS SDK. Install it using +the following command: + +`$ npm install aws-sdk --save` + +Next we'll need to configure our AWS SDK. You can do this in a number of ways. +We're going to create a credentials file at `~/.aws/credentials`. To do this we +take the following steps: + +* `$ cd` this takes us back to the root of our file system +* `$ mkdir .aws` creates a folder that will hold our credentials file +* `$ cd mkdir` navigate to our `.aws` directory +* `$ touch credentials` creates our credentials file +* Open your credentials file in your text editor and add the following: +``` +[default] +aws_access_key_id = your_access_key_id +aws_secret_access_key = your_secret_key +``` +* save the file + +The credentials should be the ones associated with the IAM user you created in +[Step 2]((#step-2---creating-an-iam-user-with-s3-permissions)). + +#### You will now be able to use the AWS SDK! + +### Step 4 - Implement the SDK + +First you'll need to set some environment variables. These are as follows: + +`export S3_REGION=` +`export S3_BUCKET=` + +Create a `src` directory in the root of your project. Create a file within this +directory called `upload.js`. This file will contain our SDK functionality. +Add the following to this file: + +```js +// load the AWS SDK +var AWS = require('aws-sdk') +// crypto is used for our unique filenames +var crypto = require('crypto') +// we use path to add the relative file extension +var path = require('path') + +// assign our region from our environment variables +AWS.config.region = process.env.S3_REGION + +/** +* Returns data specific to our upload that we send to the front end +* @param {Buffer} file - file that we are uploading +* @param {string} filename - name of the file to be uploaded +* @param {function} callback +**/ +function upload (file, filename, callback) { + // creating our new filename + var filenameHex = + filename.split('.')[0] + + crypto.randomBytes(8).toString('hex') + + path.extname(filename) + // loading our bucket name from our environment variables + var bucket = process.env.S3_BUCKET + // creating a new instance of our bucket + var s3Bucket = new AWS.S3({params: {Bucket: bucket}}) + // sets the params needed for the upload + var params = {Bucket: bucket, Key: filenameHex, Body: file} + // SDK upload to S3 + s3Bucket.upload(params, function (err, data) { + if (err) console.log(err, data) + // callback with the data that gets returned from S3 + else callback(null, data) + }) +} + +module.exports = { + upload +} +``` + +#### We've now set up our S3 upload function! + + +### Step 5 - Create a server to handle our upload file + +Create a directory called `lib`. This will hold our server + routes. + +Create a file called `index.js` and add the following: + +```js +'use strict' + +var Hapi = require('hapi') +var Inert = require('inert') +var assert = require('assert') + +// creating a new instance of our server +var server = new Hapi.Server() + +// connecting to port 8000 +server.connection({port: 8000}) +// registering the Inert module which allows us to render static files +server.register([Inert], function (err) { + assert(!err) // not much point continuing without plugins ... + + // adding our routes (we'll be creating this file next) + server.route(require('./routes.js')) + + // starting the server + server.start(function (err) { + assert(!err) // not much point continuing if the server does not start ... + console.log('The server is running on: ', server.info.uri) + }) +}) + +module.exports = server +``` + +Next create a file called `routes.js` and add the following: + +```js +// require the function we just created +var s3 = require('../src/upload.js') +// path is needed to resolve the filepath to our index.html +var path = require('path') + +module.exports = [ + // our index route + { + method: 'GET', + path: '/', + handler: function (request, reply) { + return reply.file(path.resolve(__dirname, '../public/index.html')) + } + }, + // our endpoint + { + method: 'POST', + path: '/file_submitted', + handler: function (request, reply) { + // we'll be sending through a file along with it's filename + var file = request.payload.file + var filename = request.payload.filename + // upload the file to S3 + s3.upload(file, filename, function (err, data) { + if (err) console.log(err) + // send the data back to the front end + reply(data) + }) + } + }, + // this is for our static files + { + method: 'GET', + path: '/{param*}', + handler: { + directory: { + path: path.resolve(__dirname, '../public'), + listing: true, + index: false + } + } + } +] +``` + +### Step 6 - Write the client side code to send our file to the server + +Create a new directory called `public`. Create an `index.html` file within this +directory. Add the following to the file: + +```html + + + + S3 Upload Demo + + + + + + + +

S3 Direct Upload Demo

+
+ +
+ +
+
+ + + +
+ +
+ + +``` + +Create the javascript file that we're including in our html file and call it +`client.js`. Add the following to the file: + +```js +// IIFE that will contain our global variables +var sdkDemo = (function () { + // file that we want to upload + var file + // filename of that file + var filename + + /** + * Saves our file and filename to the relevant global variables + * @param {Buffer} uploadFile - file that we are uploading + **/ + function saveFile (uploadFile) { + file = uploadFile[0] + filename = file.name + } + + /** + * Calls our sendFileToServer function + **/ + function submitFile () { + sendFileToServer(file, filename) + } + + /** + * Sends file to server and returns link to uploaded file + * @param {Buffer} file - file that we are uploading + * @param {string} filename - name of the file to be uploaded + **/ + function sendFileToServer (file, filename) { + // creates a new FormData instance so we can send our image file + var formData = new FormData() + // append the file to the formData + formData.append('file', file) + // append the filename to the formData + formData.append('filename', filename) + // create a new XHR request + var xhttp = new XMLHttpRequest() + // wait for a 200 status (OK) + xhttp.onreadystatechange = function () { + if (this.readyState === 4 && this.status === 200) { + // save the file location from the responseText + var fileLocation = JSON.parse(xhttp.responseText).Location + // add success message to index.html + var successMessage = document.createElement('h4') + successMessage.innerHTML = 'Image Successfully Uploaded at: ' + // create a link to the image + var imageATag = document.querySelector('a') + var link = fileLocation + // set the link to the image location + imageATag.setAttribute('href', link) + var imageLink = document.createElement('h4') + imageLink.innerHTML = link + var div = document.querySelector('div') + // add the success message and image link to the index.html + div.insertBefore(successMessage, div.firstChild) + imageATag.appendChild(imageLink) + } + } + // open the POST request + xhttp.open('POST', '/file_submitted', true) + // send the formData + xhttp.send(formData) + } + + return { + // make your functions available to your index.html + saveFile, + submitFile + } +}()) +``` diff --git a/examples/sdk-upload/lib/routes.js b/examples/sdk-upload/lib/routes.js index 809f46f..78237aa 100644 --- a/examples/sdk-upload/lib/routes.js +++ b/examples/sdk-upload/lib/routes.js @@ -1,4 +1,4 @@ -// require('env2')('./.env') +var s3 = require('../src/upload.js') var path = require('path') module.exports = [ @@ -13,7 +13,12 @@ module.exports = [ method: 'POST', path: '/file_submitted', handler: function (request, reply) { - console.log('-------->', request.payload) + var file = request.payload.file + var filename = request.payload.filename + s3.upload(file, filename, function (err, data) { + console.log(err) + reply(data) + }) } }, { diff --git a/examples/sdk-upload/public/client.js b/examples/sdk-upload/public/client.js index 88dbd57..936dedb 100644 --- a/examples/sdk-upload/public/client.js +++ b/examples/sdk-upload/public/client.js @@ -1,19 +1,39 @@ var sdkDemo = (function () { var file + var filename function saveFile (uploadFile) { file = uploadFile[0] + filename = file.name } function submitFile () { - sendFileToServer(file) + sendFileToServer(file, filename) } - function sendFileToServer (file) { + function sendFileToServer (file, filename) { + var formData = new FormData() + formData.append('file', file) + formData.append('filename', filename) var xhttp = new XMLHttpRequest() + xhttp.onreadystatechange = function () { + if (this.readyState === 4 && this.status === 200) { + var fileLocation = JSON.parse(xhttp.responseText).Location + console.log('FRONT END DATA', fileLocation) + var successMessage = document.createElement('h4') + successMessage.innerHTML = 'Image Successfully Uploaded at: ' + var link = fileLocation + var imageATag = document.querySelector('a') + imageATag.setAttribute('href', link) + var imageLink = document.createElement('h4') + imageLink.innerHTML = link + var div = document.querySelector('div') + div.insertBefore(successMessage, div.firstChild) + imageATag.appendChild(imageLink) + } + } xhttp.open('POST', '/file_submitted', true) - xhttp.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8') - xhttp.send(file) + xhttp.send(formData) } return { diff --git a/examples/sdk-upload/src/upload-to-s3.js b/examples/sdk-upload/src/upload-to-s3.js deleted file mode 100644 index 0769cf0..0000000 --- a/examples/sdk-upload/src/upload-to-s3.js +++ /dev/null @@ -1,2 +0,0 @@ -require('env2')('./.env') -var AWS = require('aws-sdk') diff --git a/examples/sdk-upload/src/upload.js b/examples/sdk-upload/src/upload.js new file mode 100644 index 0000000..de6a298 --- /dev/null +++ b/examples/sdk-upload/src/upload.js @@ -0,0 +1,25 @@ +require('env2')('./.env') +var AWS = require('aws-sdk') +var crypto = require('crypto') +var path = require('path') + +AWS.config.region = process.env.S3_REGION + +function upload (file, filename, callback) { + var filenameHex = + filename.split('.')[0] + + crypto.randomBytes(8).toString('hex') + + path.extname(filename) + var bucket = process.env.S3_BUCKET + console.log(bucket) + var s3Bucket = new AWS.S3({params: {Bucket: bucket}}) + var params = {Bucket: bucket, Key: filenameHex, Body: file} + s3Bucket.upload(params, function (err, data) { + console.log(err, data) + callback(null, data) + }) +} + +module.exports = { + upload +} diff --git a/examples/sdk-upload/test/index.test.js b/examples/sdk-upload/test/index.test.js new file mode 100644 index 0000000..e3bbc55 --- /dev/null +++ b/examples/sdk-upload/test/index.test.js @@ -0,0 +1,2 @@ +require('./upload.test.js') +require('./server.test.js') diff --git a/examples/sdk-upload/test/server.test.js b/examples/sdk-upload/test/server.test.js new file mode 100644 index 0000000..945a93a --- /dev/null +++ b/examples/sdk-upload/test/server.test.js @@ -0,0 +1,39 @@ +var test = require('tape') +var server = require('../lib/index.js') + +test('checks GET request for our index.html', function (t) { + var options = { + method: 'GET', + url: '/' + } + server.inject(options, function (response) { + t.equal(response.statusCode, 200, '✅ 200 status code returned') + t.end(server.stop(function () {})) + }) +}) + +test('checks GET request for our client.js', function (t) { + var options = { + method: 'GET', + url: '/client.js' + } + server.inject(options, function (response) { + t.equal(response.statusCode, 200, '✅ 200 status code returned') + t.end(server.stop(function () {})) + }) +}) + +test('checks POST request to our /file_submitted endpoint', function (t) { + var options = { + method: 'POST', + url: '/file_submitted', + payload: { + file: 'test', + filename: 'test' + } + } + server.inject(options, function (response) { + t.equal(response.statusCode, 200, '✅ 200 status code returned') + t.end(server.stop(function () {})) + }) +}) diff --git a/examples/sdk-upload/test/upload.test.js b/examples/sdk-upload/test/upload.test.js new file mode 100644 index 0000000..e69de29 diff --git a/package.json b/package.json index 451f058..a0f3974 100644 --- a/package.json +++ b/package.json @@ -21,8 +21,10 @@ "tape": "^4.6.0" }, "scripts": { - "test": "istanbul cover tape examples/direct-upload/test/**/*.js", - "coverage": "istanbul cover tape examples/direct-upload/test/**/*.js && codecov" + "direct-upload-test": "istanbul cover tape examples/direct-upload/test/**/*.js", + "direct-upload-coverage": "istanbul cover tape examples/direct-upload/test/**/*.js && codecov", + "sdk-upload-test": "istanbul cover tape examples/sdk-upload/test/**/*.js", + "sdk-upload-coverage": "istanbul cover tape examples/sdk-upload/test/**/*.js && codecov" }, "repository": { "type": "git",