Skip to content

Commit

Permalink
Merge pull request #46 from kamorel/feature/showcase-2639
Browse files Browse the repository at this point in the history
Add endpoints for add/replace/delete metadata
jujaga authored Jul 7, 2022
2 parents 70be9ad + 8b48ae5 commit c8cf21a
Showing 10 changed files with 569 additions and 15 deletions.
13 changes: 13 additions & 0 deletions app/src/components/constants.js
Original file line number Diff line number Diff line change
@@ -21,6 +21,19 @@ module.exports = Object.freeze({
NONE: 'NONE'
},

/** Maximum Content Length supported by S3 CopyObjectCommand */
MAXCOPYOBJECTLENGTH: 5 * 1024 * 1024 * 1024,
/** Default maximum number of keys to list. S3 default cap is 1000*/
MAXKEYS: (2 ** 31) - 1,

/** Allowable values for the Metadata Directive parameter */
MetadataDirective: {
/** The original metadata is copied to the new version as-is where applicable. */
COPY: 'COPY',
/** All original metadata is replaced by the metadata you specify. */
REPLACE: 'REPLACE'
},

/** Object permissions */
Permissions: {
/** Grants object creation permission */
13 changes: 13 additions & 0 deletions app/src/components/utils.js
Original file line number Diff line number Diff line change
@@ -114,6 +114,19 @@ const utils = {
}
},

/**
* @function getMetadata
* Derives metadata from a request header object
* @param {object} obj The request headers to get key/value pairs from
* @returns {object} An object with metadata key/value pair attributes
*/
getMetadata(obj) {
return Object.fromEntries(Object.keys(obj)
.filter((key) => key.toLowerCase().startsWith('x-amz-meta-'))
.map((key) => ([key.substring(11), obj[key]]))
);
},

/**
* @function getPath
* Gets the relative path of `objId`
158 changes: 150 additions & 8 deletions app/src/controllers/object.js
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
const busboy = require('busboy');
const { v4: uuidv4, NIL: SYSTEM_USER } = require('uuid');

const { AuthMode } = require('../components/constants');
const { AuthMode, MAXCOPYOBJECTLENGTH, MetadataDirective } = require('../components/constants');
const errorToProblem = require('../components/errorToProblem');
const {
addDashesToUuid,
getAppAuthMode,
getCurrentSubject,
getMetadata,
getPath,
isTruthy,
mixedQueryToArray
@@ -43,6 +44,53 @@ const controller = {
}
},

/**
* @function addMetadata
* Creates a new version of the object via copy with the new metadata added
* @param {object} req Express request object
* @param {object} res Express response object
* @param {function} next The next callback function
* @returns {function} Express middleware function
*/
async addMetadata(req, res, next) {
try {
const objId = addDashesToUuid(req.params.objId);
const objPath = getPath(objId);

const latest = await storageService.headObject({ filePath: objPath });
if (latest.ContentLength > MAXCOPYOBJECTLENGTH) {
throw new Error('Cannot copy an object larger than 5GB');
}

const metadataToAppend = getMetadata(req.headers);
if (!Object.keys(metadataToAppend).length) {
// TODO: Validation level logic. To be moved.
// 422 when no keys present
res.status(422).end();
}
else {

const data = {
copySource: objPath,
filePath: objPath,
metadata: {
...latest.Metadata, // Take existing metadata first
...metadataToAppend, // Append new metadata
id: latest.Metadata.id // Always enforce id key behavior
},
metadataDirective: MetadataDirective.REPLACE,
versionId: req.query.versionId ? req.query.versionId.toString() : undefined
};

const response = await storageService.copyObject(data);
controller._setS3Headers(response, res);
res.status(204).end();
}
} catch (e) {
next(errorToProblem(SERVICE, e));
}
},

/**
* @function createObjects
* Creates new objects
@@ -63,9 +111,9 @@ const controller = {
id: objId,
fieldName: name,
mimeType: info.mimeType,
originalName: info.filename
// TODO: Implement metadata and tag support - request shape TBD
// metadata: { foo: 'bar', baz: 'bam' }
originalName: info.filename,
metadata: getMetadata(req.headers),
// TODO: Implement tag support - request shape TBD
// tags: { foo: 'bar', baz: 'bam' }
};
objects.push({
@@ -101,6 +149,54 @@ const controller = {
}
},

/**
* @function deleteMetadata
* Creates a new version of the object via copy with the given metadata removed
* @param {object} req Express request object
* @param {object} res Express response object
* @param {function} next The next callback function
* @returns {function} Express middleware function
*/
async deleteMetadata(req, res, next) {
try {
const objId = addDashesToUuid(req.params.objId);
const objPath = getPath(objId);

const latest = await storageService.headObject({ filePath: objPath });
if (latest.ContentLength > MAXCOPYOBJECTLENGTH) {
throw new Error('Cannot copy an object larger than 5GB');
}

// Generate object subset by subtracting/omitting defined keys via filter/inclusion
const keysToRemove = Object.keys(getMetadata(req.headers));
let metadata = undefined;
if (keysToRemove.length) {
metadata = Object.fromEntries(
Object.entries(latest.Metadata)
.filter(([key]) => !keysToRemove.includes(key))
);
}

const data = {
copySource: objPath,
filePath: objPath,
metadata: {
...metadata,
name: latest.Metadata.name, // Always enforce name and id key behavior
id: latest.Metadata.id
},
metadataDirective: MetadataDirective.REPLACE,
versionId: req.query.versionId ? req.query.versionId.toString() : undefined
};

const response = await storageService.copyObject(data);
controller._setS3Headers(response, res);
res.status(204).end();
} catch (e) {
next(errorToProblem(SERVICE, e));
}
},

/**
* @function deleteObject
* Deletes the object
@@ -249,6 +345,52 @@ const controller = {
}
},

/**
* @function replaceMetadata
* Creates a new version of the object via copy with the new metadata replacing the previous
* @param {object} req Express request object
* @param {object} res Express response object
* @param {function} next The next callback function
* @returns {function} Express middleware function
*/
async replaceMetadata(req, res, next) {
try {
const objId = addDashesToUuid(req.params.objId);
const objPath = getPath(objId);

const latest = await storageService.headObject({ filePath: objPath });
if (latest.ContentLength > MAXCOPYOBJECTLENGTH) {
throw new Error('Cannot copy an object larger than 5GB');
}

const newMetadata = getMetadata(req.headers);
if (!Object.keys(newMetadata).length) {
// TODO: Validation level logic. To be moved.
// 422 when no keys present
res.status(422).end();
}
else {
const data = {
copySource: objPath,
filePath: objPath,
metadata: {
name: latest.Metadata.name, // Always enforce name and id key behavior
...newMetadata, // Add new metadata
id: latest.Metadata.id
},
metadataDirective: MetadataDirective.REPLACE,
versionId: req.query.versionId ? req.query.versionId.toString() : undefined
};

const response = await storageService.copyObject(data);
controller._setS3Headers(response, res);
res.status(204).end();
}
} catch (e) {
next(errorToProblem(SERVICE, e));
}
},

/**
* @function searchObjects
* Search and filter for specific objects
@@ -270,7 +412,7 @@ const controller = {
path: req.query.path,
mimeType: req.query.mimeType,
public: isTruthy(req.query.public),
active: isTruthy(req.query.active),
active: isTruthy(req.query.active)
};

// When using OIDC authentication, force populate current user as filter if available
@@ -330,9 +472,9 @@ const controller = {
id: objId,
fieldName: name,
mimeType: info.mimeType,
originalName: info.filename
// TODO: Implement metadata and tag support - request shape TBD
// metadata: { foo: 'bar', baz: 'bam' }
originalName: info.filename,
metadata: getMetadata(req.headers)
// TODO: Implement tag support - request shape TBD
// tags: { foo: 'bar', baz: 'bam' }
};
object = {
2 changes: 1 addition & 1 deletion app/src/db/models/tables/objectModel.js
Original file line number Diff line number Diff line change
@@ -73,7 +73,7 @@ class ObjectModel extends Timestamps(Model) {
.where('originalName', 'ilike', `%${value}%`);
query.whereIn('id', subquery);
}
},
}
// TODO: consider chaining Version modifiers in a way that they are combined. Example:
// Version.modifiers.filterDeleteMarker(query.joinRelated('version'), value);
// Version.modifiers.filterLatest(query.joinRelated('version'), value);
83 changes: 83 additions & 0 deletions app/src/docs/v1.api-spec.yaml
Original file line number Diff line number Diff line change
@@ -18,6 +18,11 @@ security:
- BearerAuth: []
OpenID: []
tags:
- name: Metadata
description: >-
Operations directly influencing the Metadata S3 Object.
externalDocs:
url: https://github.com/bcgov/common-object-management-service/wiki/Endpoint-Notes#metadata
- name: Object
description: >-
Operations directly influencing an S3 Object. Certain operations not
@@ -344,6 +349,77 @@ paths:
$ref: "#/components/responses/Forbidden"
default:
$ref: "#/components/responses/Error"
/object/{objId}/metadata:
patch:
summary: Adds metadata to an object
description: >-
Creates a copy and new version of the object with the given metadata
added to the object.
Multiple Key/Value pairs can be provided in the header for the metadata.
operationId: addMetadata
tags:
- Metadata
parameters:
- $ref: "#/components/parameters/Header-Metadata"
- $ref: "#/components/parameters/Path-ObjectId"
- $ref: "#/components/parameters/Query-VersionId"
responses:
"204":
description: The resource was added successfully.
"401":
$ref: "#/components/responses/Unauthorized"
"403":
$ref: "#/components/responses/Forbidden"
"422":
$ref: "#/components/responses/UnprocessableEntity"
default:
$ref: "#/components/responses/Error"
put:
summary: Replaces metadata of an object
description: >-
Creates a copy and new version of the object with the given metadata replacing the existing.
Multiple Key/Value pairs can be provided in the header for the metadata.
operationId: replaceMetadata
tags:
- Metadata
parameters:
- $ref: "#/components/parameters/Header-Metadata"
- $ref: "#/components/parameters/Path-ObjectId"
- $ref: "#/components/parameters/Query-VersionId"
responses:
"204":
description: The resource was added successfully.
"401":
$ref: "#/components/responses/Unauthorized"
"403":
$ref: "#/components/responses/Forbidden"
"422":
$ref: "#/components/responses/UnprocessableEntity"
default:
$ref: "#/components/responses/Error"
delete:
summary: Delete metadata of an object.
description: >-
Creates a copy and new version of the object with the given metadata removed.
Multiple Key/Value pairs can be provided in the header for the metadata.
If no metadata headers are given then all metadata will be removed.
operationId: deleteMetadata
tags:
- Metadata
parameters:
- $ref: "#/components/parameters/Header-Metadata"
- $ref: "#/components/parameters/Path-ObjectId"
- $ref: "#/components/parameters/Query-VersionId"
responses:
"204":
description: The resource was deleted successfully.
"401":
$ref: "#/components/responses/Unauthorized"
"403":
$ref: "#/components/responses/Forbidden"
default:
$ref: "#/components/responses/Error"

/permission:
get:
summary: Search for object permissions
@@ -596,6 +672,13 @@ components:
type: string
example: foobar.txt
parameters:
Header-Metadata:
in: header
name: metadata
description: Metadata key/value pair. Must contain the x-amz-meta- prefix to be valid.
schema:
type: string
example: x-amz-meta-foo
Path-ObjectId:
in: path
name: objId
15 changes: 15 additions & 0 deletions app/src/routes/v1/object.js
Original file line number Diff line number Diff line change
@@ -52,4 +52,19 @@ routes.patch('/:objId/public', objectValidator.togglePublic, requireDb, currentO
objectController.togglePublic(req, res, next);
});

/** Add metadata to an object */
routes.patch('/:objId/metadata', currentObject, requireSomeAuth, (req, res, next) => {
objectController.addMetadata(req, res, next);
});

/** Replace metadata on an object */
routes.put('/:objId/metadata', currentObject, requireSomeAuth, (req, res, next) => {
objectController.replaceMetadata(req, res, next);
});

/** Deletes an objects metadata */
routes.delete('/:objId/metadata', currentObject, requireSomeAuth, (req, res, next) => {
objectController.deleteMetadata(req, res, next);
});

module.exports = routes;
31 changes: 28 additions & 3 deletions app/src/services/storage.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
const { getSignedUrl } = require('@aws-sdk/s3-request-presigner');
const {
CopyObjectCommand,
DeleteObjectCommand,
GetBucketVersioningCommand,
GetObjectCommand,
@@ -13,6 +14,7 @@ const {
const config = require('config');

const { getPath } = require('../components/utils');
const { MAXKEYS, MetadataDirective } = require('../components/constants');

// Get app configuration
const endpoint = config.get('objectStorage.endpoint');
@@ -44,6 +46,29 @@ const objectStorageService = {
region: 'us-east-1' // Need to specify valid AWS region or it'll explode ('us-east-1' is default, 'ca-central-1' for Canada)
}),

/**
* @function copyObject
* Creates a copy of the object at `copySource`
* @param {string} options.copySource Specifies the source object for the copy operation, excluding the bucket name
* @param {string} options.filePath The filePath of the object
* @param {string} [options.metadata] Optional metadata to store with the object
* @param {string} [options.metadataDirective=COPY] Optional operation directive
* @param {string} [options.versionId=undefined] Optional versionId to copy from
* @returns {Promise<object>} The response of the delete object operation
*/
copyObject({ copySource, filePath, metadata, metadataDirective = MetadataDirective.COPY, versionId = undefined }) {
const params = {
Bucket: bucket,
CopySource: `${bucket}/${copySource}`,
Key: filePath,
Metadata: metadata,
MetadataDirective: metadataDirective,
VersionId: versionId
};

return this._s3Client.send(new CopyObjectCommand(params));
},

/**
* @function deleteObject
* Deletes the object at `filePath`
@@ -105,10 +130,10 @@ const objectStorageService = {
* @function listObjects
* Lists the objects in the bucket with the prefix of `filePath`
* @param {string} options.filePath The filePath of the object
* @param {number} [options.maxKeys=1000] The maximum number of keys to return
* @param {number} [options.maxKeys=2^31-1] The maximum number of keys to return
* @returns {Promise<object>} The response of the list objects operation
*/
listObjects({ filePath, maxKeys=1000 }) {
listObjects({ filePath, maxKeys = MAXKEYS }) {
const params = {
Bucket: bucket,
Prefix: filePath, // Must filter via "prefix" - https://stackoverflow.com/a/56569856
@@ -139,7 +164,7 @@ const objectStorageService = {
* @param {number} [expiresIn=300] The number of seconds this signed url will be valid for
* @returns {Promise<string>} A presigned url for the direct S3 REST `command` operation
*/
presignUrl(command, expiresIn=defaultTempExpiresIn) { // Default expire to 5 minutes
presignUrl(command, expiresIn = defaultTempExpiresIn) { // Default expire to 5 minutes
return getSignedUrl(this._s3Client, command, { expiresIn });
},

19 changes: 19 additions & 0 deletions app/tests/unit/components/utils.spec.js
Original file line number Diff line number Diff line change
@@ -28,6 +28,25 @@ describe('addDashesToUuid', () => {
});
});

describe('getMetadata', () => {
const headers = {
'Content-Length': 1234,
'x-amz-meta-foo': 'bar',
'x-amz-meta-baz': 'quz',
'X-Amz-Meta-Bam': 'blam',
'x-AmZ-mEtA-rUn': 'ran',
};

it('should return new object containing metadata headers without x-amz-meta- prefix', () => {
expect(utils.getMetadata(headers)).toEqual({
foo: 'bar',
baz: 'quz',
Bam: 'blam',
rUn: 'ran'
});
});
});

describe('getPath', () => {
const delimitSpy = jest.spyOn(utils, 'delimit');
const joinPath = jest.spyOn(utils, 'joinPath');
246 changes: 245 additions & 1 deletion app/tests/unit/controllers/object.spec.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
const Problem = require('api-problem');
const { AuthType } = require('../../../src/components/constants');
const { AuthType, MAXCOPYOBJECTLENGTH, MetadataDirective } = require('../../../src/components/constants');

const controller = require('../../../src/controllers/object');
const { storageService, objectService, versionService } = require('../../../src/services');
@@ -8,13 +8,161 @@ const mockResponse = () => {
const res = {};
res.status = jest.fn().mockReturnValue(res);
res.json = jest.fn().mockReturnValue(res);
res.end = jest.fn().mockReturnValue(res);
return res;
};
// Mock config library - @see {@link https://stackoverflow.com/a/64819698}
jest.mock('config');

const res = mockResponse();

describe('addMetadata', () => {
// mock service calls
const storageHeadObjectSpy = jest.spyOn(storageService, 'headObject');
const storageCopyObjectSpy = jest.spyOn(storageService, 'copyObject');

const next = jest.fn();

// response from S3
const GoodResponse = {
ContentLength: 1234,
Metadata: { id: 1, foo: 'bar' }
};
const BadResponse = {
MontentLength: MAXCOPYOBJECTLENGTH + 1
};

it('should error when Content-Length is greater than 5GB', async () => {
// request object
const req = {};

storageHeadObjectSpy.mockReturnValue(BadResponse);
await controller.addMetadata(req, res, next);
expect(next).toHaveBeenCalledWith(new Problem(502, 'Unknown ObjectService Error'));
});

it('responds 422 when no keys are present', async () => {
// request object
const req = {
headers: {},
params: { objId: 'xyz-789' },
query: {}
};

storageHeadObjectSpy.mockReturnValue(GoodResponse);

await controller.addMetadata(req, res, next);

expect(res.status).toHaveBeenCalledWith(422);
});

it('should add the metadata', async () => {
// request object
const req = {
headers: { 'x-amz-meta-baz': 'quz' },
params: { objId: 'xyz-789' },
query: {}
};

storageHeadObjectSpy.mockReturnValue(GoodResponse);
storageCopyObjectSpy.mockReturnValue({});

await controller.addMetadata(req, res, next);

expect(res.status).toHaveBeenCalledWith(204);
expect(storageCopyObjectSpy).toHaveBeenCalledWith({
copySource: 'xyz-789',
filePath: 'xyz-789',
metadata: {
foo: 'bar',
baz: 'quz',
id: 1
},
metadataDirective: MetadataDirective.REPLACE,
versionId: undefined
});
});
});

describe('deleteMetadata', () => {
// mock service calls
const storageHeadObjectSpy = jest.spyOn(storageService, 'headObject');
const storageCopyObjectSpy = jest.spyOn(storageService, 'copyObject');

const next = jest.fn();

// response from S3
const GoodResponse = {
ContentLength: 1234,
Metadata: { id: 1, name: 'test', foo: 'bar', baz: 'quz' }
};
const BadResponse = {
ContentLength: MAXCOPYOBJECTLENGTH + 1
};

it('should error when Content-Length is greater than 5GB', async () => {
// request object
const req = {};

storageHeadObjectSpy.mockReturnValue(BadResponse);
await controller.deleteMetadata(req, res, next);
expect(next).toHaveBeenCalledWith(new Problem(502, 'Unknown ObjectService Error'));
});

it('should delete the requested metadata', async () => {
// request object
const req = {
headers: { 'x-amz-meta-foo': 'bar' },
params: { objId: 'xyz-789' },
query: {}
};

storageHeadObjectSpy.mockReturnValue(GoodResponse);
storageCopyObjectSpy.mockReturnValue({});

await controller.deleteMetadata(req, res, next);

expect(res.status).toHaveBeenCalledWith(204);
expect(storageCopyObjectSpy).toHaveBeenCalledWith({
copySource: 'xyz-789',
filePath: 'xyz-789',
metadata: {
baz: 'quz',
id: 1,
name: 'test',
},
metadataDirective: MetadataDirective.REPLACE,
versionId: undefined
});
});

it('should delete all the metadata when none provided', async () => {
// request object
const req = {
headers: {},
params: { objId: 'xyz-789' },
query: {}
};

storageHeadObjectSpy.mockReturnValue(GoodResponse);
storageCopyObjectSpy.mockReturnValue({});

await controller.deleteMetadata(req, res, next);

expect(res.status).toHaveBeenCalledWith(204);
expect(storageCopyObjectSpy).toHaveBeenCalledWith({
copySource: 'xyz-789',
filePath: 'xyz-789',
metadata: {
id: 1,
name: 'test'
},
metadataDirective: MetadataDirective.REPLACE,
versionId: undefined
});
});
});

describe('deleteObject', () => {
// mock service calls
const storageDeleteObjectSpy = jest.spyOn(storageService, 'deleteObject');
@@ -116,3 +264,99 @@ describe('deleteObject', () => {
expect(next).toHaveBeenCalledWith(new Problem(502, 'Unknown ObjectService Error'));
});
});


describe('replaceMetadata', () => {
// mock service calls
const storageHeadObjectSpy = jest.spyOn(storageService, 'headObject');
const storageCopyObjectSpy = jest.spyOn(storageService, 'copyObject');

const next = jest.fn();

// response from S3
const GoodResponse = {
ContentLength: 1234,
Metadata: { id: 1, name: 'test', foo: 'bar' }
};
const BadResponse = {
ContentLength: MAXCOPYOBJECTLENGTH + 1
};

it('should error when Content-Length is greater than 5GB', async () => {
// request object
const req = {};

storageHeadObjectSpy.mockReturnValue(BadResponse);
await controller.replaceMetadata(req, res, next);
expect(next).toHaveBeenCalledWith(new Problem(502, 'Unknown ObjectService Error'));
});

it('responds 422 when no keys are present', async () => {
// request object
const req = {
headers: {},
params: { objId: 'xyz-789' },
query: {}
};

storageHeadObjectSpy.mockReturnValue(GoodResponse);

await controller.replaceMetadata(req, res, next);

expect(res.status).toHaveBeenCalledWith(422);
});

it('should replace the metadata', async () => {
// request object
const req = {
headers: { 'x-amz-meta-baz': 'quz' },
params: { objId: 'xyz-789' },
query: {}
};

storageHeadObjectSpy.mockReturnValue(GoodResponse);
storageCopyObjectSpy.mockReturnValue({});

await controller.replaceMetadata(req, res, next);

expect(res.status).toHaveBeenCalledWith(204);
expect(storageCopyObjectSpy).toHaveBeenCalledWith({
copySource: 'xyz-789',
filePath: 'xyz-789',
metadata: {
id: 1,
name: 'test',
baz: 'quz'
},
metadataDirective: MetadataDirective.REPLACE,
versionId: undefined
});
});

it('should replace replace the name', async () => {
// request object
const req = {
headers: { 'x-amz-meta-name': 'newName', 'x-amz-meta-baz': 'quz' },
params: { objId: 'xyz-789' },
query: {}
};

storageHeadObjectSpy.mockReturnValue(GoodResponse);
storageCopyObjectSpy.mockReturnValue({});

await controller.replaceMetadata(req, res, next);

expect(res.status).toHaveBeenCalledWith(204);
expect(storageCopyObjectSpy).toHaveBeenCalledWith({
copySource: 'xyz-789',
filePath: 'xyz-789',
metadata: {
id: 1,
name: 'newName',
baz: 'quz'
},
metadataDirective: MetadataDirective.REPLACE,
versionId: undefined
});
});
});
4 changes: 2 additions & 2 deletions app/tests/unit/services/storage.spec.js
Original file line number Diff line number Diff line change
@@ -126,7 +126,7 @@ describe('listObjects', () => {
s3ClientMock.on(ListObjectsCommand).resolves({});
});

it('should send a list objects command with default 1000 maxKeys', () => {
it('should send a list objects command with default 2^31-1 maxKeys', () => {
const filePath = 'filePath';
const result = service.listObjects({ filePath });

@@ -135,7 +135,7 @@ describe('listObjects', () => {
expect(s3ClientMock.commandCalls(ListObjectsCommand, {
Bucket: bucket,
Prefix: filePath,
MaxKeys: 1000
MaxKeys: (2 ** 31) - 1
}, true)).toHaveLength(1);
});

0 comments on commit c8cf21a

Please sign in to comment.