From b6830f08b5ee75c7bd3830f401b18d7d0dca63e6 Mon Sep 17 00:00:00 2001 From: Csaky Date: Thu, 7 Sep 2023 14:16:39 -0700 Subject: [PATCH 1/4] Validate default bucket configuration Check for conflict config > database on startup Let getBucket() throw if objectStorage not in config --- app/app.js | 7 ++++ app/config/custom-environment-variables.json | 2 +- app/config/default.json | 4 +-- app/src/components/utils.js | 36 ++++++++++---------- app/src/services/storage.js | 2 +- 5 files changed, 28 insertions(+), 23 deletions(-) diff --git a/app/app.js b/app/app.js index c11c98f9..005250ce 100644 --- a/app/app.js +++ b/app/app.js @@ -13,6 +13,7 @@ const QueueManager = require('./src/components/queueManager'); const { getAppAuthMode, getGitRevision } = require('./src/components/utils'); const DataConnection = require('./src/db/dataConnection'); const v1Router = require('./src/routes/v1'); +const { readUnique } = require('./src/services/bucket'); const dataConnection = new DataConnection(); const queueManager = new QueueManager(); @@ -215,6 +216,12 @@ function initializeConnections() { if (state.connections.data) { log.info('DataConnection Reachable', { function: 'initializeConnections' }); } + if (config.has('objectStorage')) { + readUnique(config.get('objectStorage')).then(() => { + log.error('Default bucket cannot also exist in database', { function: 'initializeConnections' }); + fatalErrorHandler(); + }).catch(() => { }); + } }) .catch(error => { log.error(`Initialization failed: Database OK = ${state.connections.data}`, { function: 'initializeConnections' }); diff --git a/app/config/custom-environment-variables.json b/app/config/custom-environment-variables.json index 44fa5786..4a0bc7b5 100644 --- a/app/config/custom-environment-variables.json +++ b/app/config/custom-environment-variables.json @@ -25,13 +25,13 @@ "objectStorage": { "accessKeyId": "OBJECTSTORAGE_ACCESSKEYID", "bucket": "OBJECTSTORAGE_BUCKET", - "defaultTempExpiresIn": "OBJECTSTORAGE_TEMP_EXPIRESIN", "endpoint": "OBJECTSTORAGE_ENDPOINT", "key": "OBJECTSTORAGE_KEY", "secretAccessKey": "OBJECTSTORAGE_SECRETACCESSKEY" }, "server": { "bodyLimit": "SERVER_BODYLIMIT", + "defaultTempExpiresIn": "SERVER_TEMP_EXPIRESIN", "hardReset": "SERVER_HARDRESET", "logFile": "SERVER_LOGFILE", "logLevel": "SERVER_LOGLEVEL", diff --git a/app/config/default.json b/app/config/default.json index 99d0c29b..ff1eaae4 100644 --- a/app/config/default.json +++ b/app/config/default.json @@ -7,11 +7,9 @@ "poolMax": "10", "username": "app" }, - "objectStorage": { - "defaultTempExpiresIn": "300" - }, "server": { "bodyLimit": "30mb", + "defaultTempExpiresIn": "300", "logLevel": "http", "maxRetries": "3", "port": "3000" diff --git a/app/src/components/utils.js b/app/src/components/utils.js index fd550221..921b1a5e 100644 --- a/app/src/components/utils.js +++ b/app/src/components/utils.js @@ -69,33 +69,33 @@ const utils = { * @throws If there are no records found with `bucketId` and `throwable` is true */ async getBucket(bucketId = undefined, throwable = false) { - const data = { - accessKeyId: config.get('objectStorage.accessKeyId'), - bucket: config.get('objectStorage.bucket'), - endpoint: config.get('objectStorage.endpoint'), - key: config.get('objectStorage.key'), - region: DEFAULTREGION, - secretAccessKey: config.get('objectStorage.secretAccessKey') - }; - - if (bucketId) { - // Function scoped import to avoid circular dependencies - const { bucketService } = require('../services'); - - try { + const data = { region: DEFAULTREGION }; + try { + if (bucketId) { + // Function scoped import to avoid circular dependencies + const { bucketService } = require('../services'); const bucketData = await bucketService.read(bucketId); + data.accessKeyId = bucketData.accessKeyId; data.bucket = bucketData.bucket; data.endpoint = bucketData.endpoint; data.key = bucketData.key; data.secretAccessKey = bucketData.secretAccessKey; if (bucketData.region) data.region = bucketData.region; - } catch (err) { - log.warn(err.message, { function: 'getBucket' }); - if (throwable) throw new Problem(404, { details: err.message }); + } else if (config.has('objectStorage')) { + data.accessKeyId = config.get('objectStorage.accessKeyId'); + data.bucket = config.get('objectStorage.bucket'); + data.endpoint = config.get('objectStorage.endpoint'); + data.key = config.get('objectStorage.key'); + data.secretAccessKey = config.get('objectStorage.secretAccessKey'); + if (config.has('objectStorage.region')) data.region = config.get('objectStorage.region'); + } else { + throw new Error('Unable to get bucket'); } + } catch (err) { + log.warn(err.message, { function: 'getBucket' }); + if (throwable) throw new Problem(404, { details: err.message }); } - return data; }, diff --git a/app/src/services/storage.js b/app/src/services/storage.js index bea3fb43..e8d64e13 100644 --- a/app/src/services/storage.js +++ b/app/src/services/storage.js @@ -27,7 +27,7 @@ const utils = require('../components/utils'); const DELIMITER = '/'; // Get app configuration -const defaultTempExpiresIn = parseInt(config.get('objectStorage.defaultTempExpiresIn'), 10); +const defaultTempExpiresIn = parseInt(config.get('server.defaultTempExpiresIn'), 10); /** * The Core S3 Object Storage Service From de7c4f42cfedcf2dca2eba88a479b68a403b5a5f Mon Sep 17 00:00:00 2001 From: Csaky Date: Fri, 8 Sep 2023 14:31:03 -0700 Subject: [PATCH 2/4] Remove `throwable` parameter from getBucket() Function always throws if no bucket found --- app/src/components/utils.js | 22 ++++++------ app/src/controllers/object.js | 8 ++--- app/src/services/storage.js | 2 +- app/tests/unit/components/utils.spec.js | 48 +++++++------------------ 4 files changed, 29 insertions(+), 51 deletions(-) diff --git a/app/src/components/utils.js b/app/src/components/utils.js index 921b1a5e..8b9da666 100644 --- a/app/src/components/utils.js +++ b/app/src/components/utils.js @@ -61,16 +61,16 @@ const utils = { /** * @function getBucket - * Acquire core S3 bucket credential information with graceful default fallback - * @param {string} [bucketId=undefined] An optional bucketId to lookup - * @param {boolean} [throwable=false] Throws an error if no `bucketId` is found + * Acquire core S3 bucket credential information from database or configuration + * @param {string} [bucketId=undefined] An optional bucket ID to query database for bucket * @returns {object} An object containing accessKeyId, bucket, endpoint, key, * region and secretAccessKey attributes - * @throws If there are no records found with `bucketId` and `throwable` is true + * @throws If there are no records found with `bucketId` or, if `bucketId` is undefined, + * no bucket details exist in the configuration */ - async getBucket(bucketId = undefined, throwable = false) { - const data = { region: DEFAULTREGION }; + async getBucket(bucketId = undefined) { try { + const data = { region: DEFAULTREGION }; if (bucketId) { // Function scoped import to avoid circular dependencies const { bucketService } = require('../services'); @@ -88,15 +88,17 @@ const utils = { data.endpoint = config.get('objectStorage.endpoint'); data.key = config.get('objectStorage.key'); data.secretAccessKey = config.get('objectStorage.secretAccessKey'); - if (config.has('objectStorage.region')) data.region = config.get('objectStorage.region'); + if (config.has('objectStorage.region')) { + data.region = config.get('objectStorage.region'); + } } else { throw new Error('Unable to get bucket'); } + return data; } catch (err) { - log.warn(err.message, { function: 'getBucket' }); - if (throwable) throw new Problem(404, { details: err.message }); + log.error(err.message, { function: 'getBucket' }); + throw new Problem(404, { details: err.message }); } - return data; }, /** diff --git a/app/src/controllers/object.js b/app/src/controllers/object.js index 8085bba5..ccfcfb02 100644 --- a/app/src/controllers/object.js +++ b/app/src/controllers/object.js @@ -239,7 +239,7 @@ const controller = { } // Preflight existence check for bucketId - const { key: bucketKey } = await getBucket(bucketId, true); + const { key: bucketKey } = await getBucket(bucketId); const objId = uuidv4(); const data = { @@ -373,7 +373,7 @@ const controller = { } // Preflight existence check for bucketId - const { key: bucketKey } = await getBucket(bucketId, true); + const { key: bucketKey } = await getBucket(bucketId); bb.on('file', async (name, stream, info) => { try { @@ -1059,7 +1059,7 @@ const controller = { // Preflight existence check for bucketId const bucketId = req.currentObject?.bucketId; - const { key: bucketKey } = await getBucket(bucketId, true); + const { key: bucketKey } = await getBucket(bucketId); const filename = req.currentObject?.path.match(/(?!.*\/)(.*)$/)[0]; const objId = addDashesToUuid(req.params.objectId); @@ -1196,7 +1196,7 @@ const controller = { // Preflight existence check for bucketId const bucketId = req.currentObject?.bucketId; - const { key: bucketKey } = await getBucket(bucketId, true); + const { key: bucketKey } = await getBucket(bucketId); bb.on('file', async (name, stream, info) => { try { diff --git a/app/src/services/storage.js b/app/src/services/storage.js index e8d64e13..2899bbce 100644 --- a/app/src/services/storage.js +++ b/app/src/services/storage.js @@ -207,7 +207,7 @@ const objectStorageService = { */ async headBucket(options = {}) { const data = options.bucketId - ? await utils.getBucket(options.bucketId, true) + ? await utils.getBucket(options.bucketId) : options; const params = { Bucket: data.bucket, diff --git a/app/tests/unit/components/utils.spec.js b/app/tests/unit/components/utils.spec.js index f6234083..7dfa9393 100644 --- a/app/tests/unit/components/utils.spec.js +++ b/app/tests/unit/components/utils.spec.js @@ -160,16 +160,15 @@ describe('getBucket', () => { }; const readBucketSpy = jest.spyOn(bucketService, 'read'); - beforeEach(() => { + it('should return config data when it exists, given no bucketId', async () => { + config.has.mockReturnValue(true); config.get .mockReturnValueOnce(cdata.accessKeyId) // objectStorage.accessKeyId .mockReturnValueOnce(cdata.bucket) // objectStorage.bucket .mockReturnValueOnce(cdata.endpoint) // objectStorage.endpoint .mockReturnValueOnce(cdata.key) // objectStorage.key - .mockReturnValueOnce(cdata.secretAccessKey); // objectStorage.secretAccessKey - }); - - it('should return config data given no bucketId and not throwable', async () => { + .mockReturnValueOnce(cdata.secretAccessKey) // objectStorage.secretAccessKey + .mockReturnValueOnce(cdata.region); // objectStorage.region const result = await utils.getBucket(); @@ -184,22 +183,16 @@ describe('getBucket', () => { expect(readBucketSpy).toHaveBeenCalledTimes(0); }); - it('should return config data given no bucketId and is throwable', async () => { + it('should throw when no config data exists, given no bucketId', async () => { + config.has.mockReturnValue(false); - const result = await utils.getBucket(undefined, true); + const result = (() => utils.getBucket(undefined))(); - expect(result).toBeTruthy(); - expect(result).toEqual(cdata); - expect(result).toHaveProperty('accessKeyId', cdata.accessKeyId); - expect(result).toHaveProperty('bucket', cdata.bucket); - expect(result).toHaveProperty('endpoint', cdata.endpoint); - expect(result).toHaveProperty('key', cdata.key); - expect(result).toHaveProperty('region', cdata.region); - expect(result).toHaveProperty('secretAccessKey', cdata.secretAccessKey); + expect(result).rejects.toThrow(); expect(readBucketSpy).toHaveBeenCalledTimes(0); }); - it('should return database data given a good bucketId and not throwable', async () => { + it('should return database data given a good bucketId', async () => { readBucketSpy.mockResolvedValue(ddata); const result = await utils.getBucket('bucketId'); @@ -215,31 +208,14 @@ describe('getBucket', () => { expect(readBucketSpy).toHaveBeenCalledWith('bucketId'); }); - it('should return config data given a bad bucketId and not throwable', async () => { - readBucketSpy.mockImplementation(() => { throw new Problem(422); }); - - const result = await (() => utils.getBucket('bucketId'))(); - - expect(result).toBeTruthy(); - expect(result).toEqual(cdata); - expect(result).toHaveProperty('accessKeyId', cdata.accessKeyId); - expect(result).toHaveProperty('bucket', cdata.bucket); - expect(result).toHaveProperty('endpoint', cdata.endpoint); - expect(result).toHaveProperty('key', cdata.key); - expect(result).toHaveProperty('region', cdata.region); - expect(result).toHaveProperty('secretAccessKey', cdata.secretAccessKey); - expect(readBucketSpy).toHaveBeenCalledTimes(1); - expect(readBucketSpy).toHaveBeenCalledWith('bucketId'); - }); - - it('should throw given a bucketId and is throwable', () => { + it('should throw given a bad bucketId', () => { readBucketSpy.mockImplementation(() => { throw new Problem(422); }); - const result = (() => utils.getBucket('bucketId', true))(); + const result = (() => utils.getBucket('bad bucketId'))(); expect(result).rejects.toThrow(); expect(readBucketSpy).toHaveBeenCalledTimes(1); - expect(readBucketSpy).toHaveBeenCalledWith('bucketId'); + expect(readBucketSpy).toHaveBeenCalledWith('bad bucketId'); }); }); From fdcf16c07a89cab022178beffad767b90dad85ef Mon Sep 17 00:00:00 2001 From: Csaky Date: Fri, 8 Sep 2023 16:08:47 -0700 Subject: [PATCH 3/4] Updated GitHub environments with variable name change OBJECTSTORAGE_TEMP_EXPIRESIN changed to SERVER_TEMP_EXPIRESIN --- .github/environments/values.dev.yaml | 2 +- .github/environments/values.prod.yaml | 2 +- .github/environments/values.test.yaml | 2 +- app/README.md | 2 +- app/tests/unit/services/storage.spec.js | 2 +- charts/coms/README.md | 2 +- charts/coms/values.yaml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/environments/values.dev.yaml b/.github/environments/values.dev.yaml index c6a1671e..b92f9b8b 100644 --- a/.github/environments/values.dev.yaml +++ b/.github/environments/values.dev.yaml @@ -19,7 +19,6 @@ config: KC_REALM: standard KC_SERVERURL: "https://dev.loginproxy.gov.bc.ca/auth" OBJECTSTORAGE_BUCKET: egejyy - OBJECTSTORAGE_TEMP_EXPIRESIN: "300" OBJECTSTORAGE_ENDPOINT: "https://nrs.objectstore.gov.bc.ca" # OBJECTSTORAGE_KEY: ~ SERVER_BODYLIMIT: 30mb @@ -27,6 +26,7 @@ config: SERVER_LOGLEVEL: http SERVER_PORT: "3000" SERVER_PRIVACY_MASK: "true" + SERVER_TEMP_EXPIRESIN: "300" patroni: enabled: true diff --git a/.github/environments/values.prod.yaml b/.github/environments/values.prod.yaml index 377f57de..3f569f14 100644 --- a/.github/environments/values.prod.yaml +++ b/.github/environments/values.prod.yaml @@ -19,7 +19,6 @@ config: KC_REALM: standard KC_SERVERURL: "https://loginproxy.gov.bc.ca/auth" OBJECTSTORAGE_BUCKET: egejyy - OBJECTSTORAGE_TEMP_EXPIRESIN: "300" OBJECTSTORAGE_ENDPOINT: "https://nrs.objectstore.gov.bc.ca" # OBJECTSTORAGE_KEY: ~ SERVER_BODYLIMIT: 30mb @@ -27,6 +26,7 @@ config: SERVER_LOGLEVEL: http SERVER_PORT: "3000" SERVER_PRIVACY_MASK: "true" + SERVER_TEMP_EXPIRESIN: "300" patroni: enabled: true diff --git a/.github/environments/values.test.yaml b/.github/environments/values.test.yaml index 62ce6f63..dbace25b 100644 --- a/.github/environments/values.test.yaml +++ b/.github/environments/values.test.yaml @@ -19,7 +19,6 @@ config: KC_REALM: standard KC_SERVERURL: "https://test.loginproxy.gov.bc.ca/auth" OBJECTSTORAGE_BUCKET: egejyy - OBJECTSTORAGE_TEMP_EXPIRESIN: "300" OBJECTSTORAGE_ENDPOINT: "https://nrs.objectstore.gov.bc.ca" # OBJECTSTORAGE_KEY: ~ SERVER_BODYLIMIT: 30mb @@ -27,6 +26,7 @@ config: SERVER_LOGLEVEL: http SERVER_PORT: "3000" SERVER_PRIVACY_MASK: "true" + SERVER_TEMP_EXPIRESIN: "300" patroni: enabled: true diff --git a/app/README.md b/app/README.md index 71a759d4..09820715 100644 --- a/app/README.md +++ b/app/README.md @@ -84,7 +84,6 @@ The following variables enable and enforce the use of OIDC Bearer Authentication | --- | --- | --- | --- | | `accessKeyId` | `OBJECTSTORAGE_ACCESSKEYID` | | The Access Key for your S3 compatible object storage account | | `bucket` | `OBJECTSTORAGE_BUCKET` | | The object storage bucket name | -| `defaultTempExpiresIn` | `OBJECTSTORAGE_TEMP_EXPIRESIN` | 300 | The expiry time for pre-signed URLs to objects in seconds | | `endpoint` | `OBJECTSTORAGE_ENDPOINT` | | Object store URL. eg: `https://nrs.objectstore.gov.bc.ca` | | `key` | `OBJECTSTORAGE_KEY` | | The base path for storage location | | `secretAccessKey` | `OBJECTSTORAGE_SECRETACCESSKEY` | | The Secret Access Key for your S3 compatible object storage account | @@ -96,6 +95,7 @@ The following variables alter the general Express application behavior. For most | Config Var | Env Var | Default | Notes | | --- | --- | --- | --- | | `bodyLimit` | `SERVER_BODYLIMIT` | 30mb | Maximum body size accepted for parsing to JSON body | +| `defaultTempExpiresIn` | `SERVER_TEMP_EXPIRESIN` | 300 | The expiry time for pre-signed S3 URLs to objects in seconds | | `logFile` | `SERVER_LOGFILE` | | Writes logs to the following file only if defined | | `logLevel` | `SERVER_LOGLEVEL` | http | The logging level of COMS | | `passphrase` | `SERVER_PASSPHRASE` | | A key to encrypt/decrypt bucket secretAccessKey's saved to the database | diff --git a/app/tests/unit/services/storage.spec.js b/app/tests/unit/services/storage.spec.js index dbf7c87c..24b2e9c7 100644 --- a/app/tests/unit/services/storage.spec.js +++ b/app/tests/unit/services/storage.spec.js @@ -29,7 +29,7 @@ const { MetadataDirective, TaggingDirective } = require('../../../src/components const DEFAULTREGION = 'us-east-1'; // Need to specify valid AWS region or it'll explode ('us-east-1' is default, 'ca-central-1' for Canada) const bucket = 'bucket'; const key = 'filePath'; -const defaultTempExpiresIn = parseInt(config.get('objectStorage.defaultTempExpiresIn'), 10); +const defaultTempExpiresIn = parseInt(config.get('server.defaultTempExpiresIn'), 10); const s3ClientMock = mockClient(S3Client); diff --git a/charts/coms/README.md b/charts/coms/README.md index 0967796a..28e14f20 100644 --- a/charts/coms/README.md +++ b/charts/coms/README.md @@ -35,7 +35,7 @@ Kubernetes: `>= 1.13.0` | autoscaling.targetCPUUtilizationPercentage | int | `80` | | | basicAuthSecretOverride.password | string | `nil` | | | basicAuthSecretOverride.username | string | `nil` | | -| config.configMap | object | `{"DB_PORT":"5432","KC_IDENTITYKEY":null,"KC_PUBLICKEY":null,"KC_REALM":null,"KC_SERVERURL":null,"OBJECTSTORAGE_BUCKET":null,"OBJECTSTORAGE_ENDPOINT":null,"OBJECTSTORAGE_KEY":null,"OBJECTSTORAGE_TEMP_EXPIRESIN":"300","SERVER_BODYLIMIT":"30mb","SERVER_LOGLEVEL":"http","SERVER_PORT":"3000"}` | These values will be wholesale added to the configmap as is; refer to the coms documentation for what each of these values mean and whether you need them defined. Ensure that all values are represented explicitly as strings, as non-string values will not translate over as expected into container environment variables. For configuration keys named `*_ENABLED`, either leave them commented/undefined, or set them to string value "true". | +| config.configMap | object | `{"DB_PORT":"5432","KC_IDENTITYKEY":null,"KC_PUBLICKEY":null,"KC_REALM":null,"KC_SERVERURL":null,"OBJECTSTORAGE_BUCKET":null,"OBJECTSTORAGE_ENDPOINT":null,"OBJECTSTORAGE_KEY":null,"SERVER_BODYLIMIT":"30mb","SERVER_LOGLEVEL":"http","SERVER_PORT":"3000","SERVER_TEMP_EXPIRESIN":"300"}` | These values will be wholesale added to the configmap as is; refer to the coms documentation for what each of these values mean and whether you need them defined. Ensure that all values are represented explicitly as strings, as non-string values will not translate over as expected into container environment variables. For configuration keys named `*_ENABLED`, either leave them commented/undefined, or set them to string value "true". | | config.enabled | bool | `false` | | | config.releaseScoped | bool | `false` | This should be set to true if and only if you require configmaps and secrets to be release scoped. In the event you want all instances in the same namespace to share a similar configuration, this should be set to false | | dbSecretOverride.password | string | `nil` | | diff --git a/charts/coms/values.yaml b/charts/coms/values.yaml index cf33a36a..fb68c77a 100644 --- a/charts/coms/values.yaml +++ b/charts/coms/values.yaml @@ -139,7 +139,6 @@ config: KC_SERVERURL: ~ OBJECTSTORAGE_BUCKET: ~ - OBJECTSTORAGE_TEMP_EXPIRESIN: "300" OBJECTSTORAGE_ENDPOINT: ~ OBJECTSTORAGE_KEY: ~ @@ -149,6 +148,7 @@ config: SERVER_LOGLEVEL: "http" SERVER_PORT: "3000" # SERVER_PRIVACY_MASK: "true" + SERVER_TEMP_EXPIRESIN: "300" # Modify the following variables if you need to acquire secret values from a custom-named resource basicAuthSecretOverride: From d5941aeeb711cf959644eed582dc951240439f15 Mon Sep 17 00:00:00 2001 From: Csaky Date: Mon, 11 Sep 2023 13:21:21 -0700 Subject: [PATCH 4/4] Update helm chart with optional objectStorage config --- .github/environments/values.dev.yaml | 4 +--- .github/environments/values.pr.yaml | 1 + .github/environments/values.prod.yaml | 4 +--- .github/environments/values.test.yaml | 4 +--- app/README.md | 6 ++++++ app/app.js | 2 +- app/config/custom-environment-variables.json | 1 + app/src/components/utils.js | 6 +++--- charts/coms/Chart.yaml | 2 +- charts/coms/README.md | 3 ++- charts/coms/templates/deploymentconfig.yaml | 2 ++ charts/coms/templates/secret.yaml | 2 +- charts/coms/values.yaml | 3 +++ 13 files changed, 24 insertions(+), 16 deletions(-) diff --git a/.github/environments/values.dev.yaml b/.github/environments/values.dev.yaml index b92f9b8b..615762c5 100644 --- a/.github/environments/values.dev.yaml +++ b/.github/environments/values.dev.yaml @@ -1,6 +1,7 @@ --- features: basicAuth: true + defaultBucket: false oidcAuth: true autoscaling: @@ -18,9 +19,6 @@ config: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuy7zfh2ZgpDV5mH/aXyLDTddZK81rGakJcTy4KvCNOkDDxt1KAhW02lmbCo8YhHCOzjNZBp1+Vi6QiMRgBqAe2GTPZYEiV70aXfROGZe3Nvwcjbtki6HoyRte3SpqLJEIPL2F+hjJkw1UPGnjPTWZkEx9p74b9i3BjuE8RnjJ0Sza2MWw83zoQUZEJRGiopSL0yuVej6t2LO2btVdVf7QuZfPt9ehkcQYlPKpVvJA+pfeqPAdnNt7OjEIeYxinjurZr8Z04hz8UhkRefcWlSbFzFQYmL7O7iArjW0bsSvq8yNUd5r0KCOQkFduwZy26yTzTxj8OLFT91fEmbBBl4rQIDAQAB KC_REALM: standard KC_SERVERURL: "https://dev.loginproxy.gov.bc.ca/auth" - OBJECTSTORAGE_BUCKET: egejyy - OBJECTSTORAGE_ENDPOINT: "https://nrs.objectstore.gov.bc.ca" - # OBJECTSTORAGE_KEY: ~ SERVER_BODYLIMIT: 30mb # SERVER_LOGFILE: ~ SERVER_LOGLEVEL: http diff --git a/.github/environments/values.pr.yaml b/.github/environments/values.pr.yaml index 002f9a80..42447e33 100644 --- a/.github/environments/values.pr.yaml +++ b/.github/environments/values.pr.yaml @@ -2,6 +2,7 @@ features: basicAuth: true oidcAuth: true + defaultBucket: false patroni: enabled: true diff --git a/.github/environments/values.prod.yaml b/.github/environments/values.prod.yaml index 3f569f14..9a4270e9 100644 --- a/.github/environments/values.prod.yaml +++ b/.github/environments/values.prod.yaml @@ -1,6 +1,7 @@ --- features: basicAuth: true + defaultBucket: true oidcAuth: true autoscaling: @@ -18,9 +19,6 @@ config: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAmHiuPKOkpkq4GXN1ktr23rJtDl6Vdu/Y37ZAd3PnQ8/IDfAODvy1Y81aAUZicKe9egolv+OTRANN3yOg+TAbRhkeXLE5p/473EK0aQ0NazTCuWo6Am3oDQ7Yt8x0pw56/qcLtkTuXNyo5EnVV2Z2BzCnnaL31JOhyitolku0DNT6GDoRBmT4o2ItqEVHk5nM25cf1t2zbwI2790W6if1B2qVRkxxivS8tbH7nYC61Is3XCPockKptkH22cm2ZQJmtYd5sZKuXaGsvtyzHmn8/l0Kd1xnHmUu4JNuQ67YiNZGu3hOkrF0Js3BzAk1Qm4kvYRaxbJFCs/qokLZ4Z0W9wIDAQAB KC_REALM: standard KC_SERVERURL: "https://loginproxy.gov.bc.ca/auth" - OBJECTSTORAGE_BUCKET: egejyy - OBJECTSTORAGE_ENDPOINT: "https://nrs.objectstore.gov.bc.ca" - # OBJECTSTORAGE_KEY: ~ SERVER_BODYLIMIT: 30mb # SERVER_LOGFILE: ~ SERVER_LOGLEVEL: http diff --git a/.github/environments/values.test.yaml b/.github/environments/values.test.yaml index dbace25b..71e1386d 100644 --- a/.github/environments/values.test.yaml +++ b/.github/environments/values.test.yaml @@ -1,6 +1,7 @@ --- features: basicAuth: true + defaultBucket: false oidcAuth: true autoscaling: @@ -18,9 +19,6 @@ config: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAiFdv9GA83uHuy8Eu9yiZHGGF9j6J8t7FkbcpaN81GDjwbjsIJ0OJO9dKRAx6BAtTC4ubJTBJMPvQER5ikOhIeBi4o25fg61jpgsU6oRZHkCXc9gX6mrjMjbsPaf3/bjjYxP5jicBDJQeD1oRa24+tiGggoQ7k6gDEN+cRYqqNpzC/GQbkUPk8YsgroncEgu8ChMh/3ERsLV2zorchMANUq76max16mHrhtWIQxrb/STpSt4JuSlUzzBV/dcXjJe5gywZHe0jAutFhNqjHzHdgyaC4RAd3eYQo+Kl/JOgy2AZrnx+CiPmvOJKe9tAW4k4H087ng8aVE40v4HW/FEbnwIDAQAB KC_REALM: standard KC_SERVERURL: "https://test.loginproxy.gov.bc.ca/auth" - OBJECTSTORAGE_BUCKET: egejyy - OBJECTSTORAGE_ENDPOINT: "https://nrs.objectstore.gov.bc.ca" - # OBJECTSTORAGE_KEY: ~ SERVER_BODYLIMIT: 30mb # SERVER_LOGFILE: ~ SERVER_LOGLEVEL: http diff --git a/app/README.md b/app/README.md index 09820715..906d1e16 100644 --- a/app/README.md +++ b/app/README.md @@ -82,6 +82,7 @@ The following variables enable and enforce the use of OIDC Bearer Authentication | Config Var | Env Var | Default | Notes | | --- | --- | --- | --- | +| `enabled` | `OBJECTSTORAGE_ENABLED` | | Whether to run COMS with a default bucket | | `accessKeyId` | `OBJECTSTORAGE_ACCESSKEYID` | | The Access Key for your S3 compatible object storage account | | `bucket` | `OBJECTSTORAGE_BUCKET` | | The object storage bucket name | | `endpoint` | `OBJECTSTORAGE_ENDPOINT` | | Object store URL. eg: `https://nrs.objectstore.gov.bc.ca` | @@ -122,6 +123,7 @@ Run COMS in **Unauthenticated mode** (replace environment values as necessary) ``` sh docker run -it --rm -p 3000:3000 \ + -e OBJECTSTORAGE_ENABLED=true \ -e OBJECTSTORAGE_ACCESSKEYID= \ -e OBJECTSTORAGE_BUCKET= \ -e OBJECTSTORAGE_ENDPOINT= \ @@ -134,6 +136,7 @@ Run COMS in **Basic Auth mode** (replace environment values as necessary) ``` sh docker run -it --rm -p 3000:3000 \ + -e OBJECTSTORAGE_ENABLED=true \ -e OBJECTSTORAGE_ACCESSKEYID= \ -e OBJECTSTORAGE_BUCKET= \ -e OBJECTSTORAGE_ENDPOINT= \ @@ -158,6 +161,7 @@ Run COMS in **OIDC Auth Mode** (replace environment values as necessary) ``` sh docker run -it --rm -p 3000:3000 \ + -e OBJECTSTORAGE_ENABLED=true \ -e OBJECTSTORAGE_ACCESSKEYID= \ -e OBJECTSTORAGE_BUCKET= \ -e OBJECTSTORAGE_ENDPOINT= \ @@ -178,6 +182,7 @@ Run COMS in **Full Auth Mode** (replace environment values as necessary) ``` sh docker run -it --rm -p 3000:3000 \ + -e OBJECTSTORAGE_ENABLED=true \ -e OBJECTSTORAGE_ACCESSKEYID= \ -e OBJECTSTORAGE_BUCKET= \ -e OBJECTSTORAGE_ENDPOINT= \ @@ -233,6 +238,7 @@ To run COMS in Full Auth mode you will want your `local.json` to have the follow "serverUrl": "" }, "objectStorage": { + "enabled": true, "secretAccessKey": "", "key": "", "accessKeyId": "", diff --git a/app/app.js b/app/app.js index 005250ce..3d11775c 100644 --- a/app/app.js +++ b/app/app.js @@ -216,7 +216,7 @@ function initializeConnections() { if (state.connections.data) { log.info('DataConnection Reachable', { function: 'initializeConnections' }); } - if (config.has('objectStorage')) { + if (config.has('objectStorage.enabled')) { readUnique(config.get('objectStorage')).then(() => { log.error('Default bucket cannot also exist in database', { function: 'initializeConnections' }); fatalErrorHandler(); diff --git a/app/config/custom-environment-variables.json b/app/config/custom-environment-variables.json index 4a0bc7b5..763b500e 100644 --- a/app/config/custom-environment-variables.json +++ b/app/config/custom-environment-variables.json @@ -25,6 +25,7 @@ "objectStorage": { "accessKeyId": "OBJECTSTORAGE_ACCESSKEYID", "bucket": "OBJECTSTORAGE_BUCKET", + "enabled": "OBJECTSTORAGE_ENABLED", "endpoint": "OBJECTSTORAGE_ENDPOINT", "key": "OBJECTSTORAGE_KEY", "secretAccessKey": "OBJECTSTORAGE_SECRETACCESSKEY" diff --git a/app/src/components/utils.js b/app/src/components/utils.js index 8b9da666..20f4f89d 100644 --- a/app/src/components/utils.js +++ b/app/src/components/utils.js @@ -73,8 +73,8 @@ const utils = { const data = { region: DEFAULTREGION }; if (bucketId) { // Function scoped import to avoid circular dependencies - const { bucketService } = require('../services'); - const bucketData = await bucketService.read(bucketId); + const { read } = require('../services/bucket'); + const bucketData = await read(bucketId); data.accessKeyId = bucketData.accessKeyId; data.bucket = bucketData.bucket; @@ -82,7 +82,7 @@ const utils = { data.key = bucketData.key; data.secretAccessKey = bucketData.secretAccessKey; if (bucketData.region) data.region = bucketData.region; - } else if (config.has('objectStorage')) { + } else if (config.has('objectStorage') && config.has('objectStorage.enabled')) { data.accessKeyId = config.get('objectStorage.accessKeyId'); data.bucket = config.get('objectStorage.bucket'); data.endpoint = config.get('objectStorage.endpoint'); diff --git a/charts/coms/Chart.yaml b/charts/coms/Chart.yaml index 9766b5bd..b839d59a 100644 --- a/charts/coms/Chart.yaml +++ b/charts/coms/Chart.yaml @@ -3,7 +3,7 @@ name: common-object-management-service # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.0.17 +version: 0.0.18 kubeVersion: ">= 1.13.0" description: A microservice for managing access control to S3 Objects # A chart can be either an 'application' or a 'library' chart. diff --git a/charts/coms/README.md b/charts/coms/README.md index 28e14f20..c8b5f5ce 100644 --- a/charts/coms/README.md +++ b/charts/coms/README.md @@ -1,6 +1,6 @@ # common-object-management-service -![Version: 0.0.17](https://img.shields.io/badge/Version-0.0.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.6.0](https://img.shields.io/badge/AppVersion-0.6.0-informational?style=flat-square) +![Version: 0.0.18](https://img.shields.io/badge/Version-0.0.18-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.6.0](https://img.shields.io/badge/AppVersion-0.6.0-informational?style=flat-square) A microservice for managing access control to S3 Objects @@ -42,6 +42,7 @@ Kubernetes: `>= 1.13.0` | dbSecretOverride.username | string | `nil` | | | failurePolicy | string | `"Retry"` | | | features.basicAuth | bool | `false` | Specifies whether basic auth is enabled | +| features.defaultBucket | bool | `false` | Specifies whether a default bucket is enabled | | features.oidcAuth | bool | `false` | Specifies whether oidc auth is enabled | | fullnameOverride | string | `nil` | String to fully override fullname | | image.pullPolicy | string | `"IfNotPresent"` | | diff --git a/charts/coms/templates/deploymentconfig.yaml b/charts/coms/templates/deploymentconfig.yaml index 209e3c43..09dd8a34 100644 --- a/charts/coms/templates/deploymentconfig.yaml +++ b/charts/coms/templates/deploymentconfig.yaml @@ -148,6 +148,7 @@ spec: key: password name: {{ include "coms.configname" . }}-keycloak {{- end }} + {{- if or .Values.features.defaultBucket .Values.config.configMap.OBJECTSTORAGE_ENABLED }} - name: OBJECTSTORAGE_ACCESSKEYID valueFrom: secretKeyRef: @@ -158,6 +159,7 @@ spec: secretKeyRef: key: password name: {{ include "coms.configname" . }}-objectstorage + {{- end }} - name: SERVER_PASSPHRASE valueFrom: secretKeyRef: diff --git a/charts/coms/templates/secret.yaml b/charts/coms/templates/secret.yaml index 39d37e7a..d12245f6 100644 --- a/charts/coms/templates/secret.yaml +++ b/charts/coms/templates/secret.yaml @@ -62,7 +62,7 @@ data: password: {{ .Values.keycloakSecretOverride.password | b64enc | quote }} username: {{ .Values.keycloakSecretOverride.username | b64enc | quote }} {{- end }} -{{- if and (not $osSecret) (and .Values.objectStorageSecretOverride.password .Values.objectStorageSecretOverride.username) }} +{{- if and .Values.features.defaultBucket (not $osSecret) (and .Values.objectStorageSecretOverride.password .Values.objectStorageSecretOverride.username) }} --- apiVersion: v1 kind: Secret diff --git a/charts/coms/values.yaml b/charts/coms/values.yaml index fb68c77a..b0b96382 100644 --- a/charts/coms/values.yaml +++ b/charts/coms/values.yaml @@ -106,6 +106,8 @@ resources: features: # -- Specifies whether basic auth is enabled basicAuth: false + # -- Specifies whether a default bucket is enabled + defaultBucket: false # -- Specifies whether oidc auth is enabled oidcAuth: false @@ -139,6 +141,7 @@ config: KC_SERVERURL: ~ OBJECTSTORAGE_BUCKET: ~ + # OBJECTSTORAGE_ENABLED: "true" OBJECTSTORAGE_ENDPOINT: ~ OBJECTSTORAGE_KEY: ~