From 702ef9d55ffcee529232b17723aa4dd6847b3b8b Mon Sep 17 00:00:00 2001
From: Sun <vsun@mitre.org>
Date: Fri, 6 Oct 2023 14:45:34 -0400
Subject: [PATCH 01/15] refactoring data components tests

---
 .../data-components/data-components.spec.js   | 425 +++++++-----------
 1 file changed, 154 insertions(+), 271 deletions(-)

diff --git a/app/tests/api/data-components/data-components.spec.js b/app/tests/api/data-components/data-components.spec.js
index 5075aed7..ef3486bc 100644
--- a/app/tests/api/data-components/data-components.spec.js
+++ b/app/tests/api/data-components/data-components.spec.js
@@ -56,199 +56,145 @@ describe('Data Components API', function () {
         passportCookie = await login.loginAnonymous(app);
     });
 
-    it('GET /api/data-components returns an empty array of data components', function (done) {
-        request(app)
+    it('GET /api/data-components returns an empty array of data components', async function () {
+        const res = await request(app)
             .get('/api/data-components')
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(200)
-            .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    // We expect to get an empty array
-                    const dataComponents = res.body;
-                    expect(dataComponents).toBeDefined();
-                    expect(Array.isArray(dataComponents)).toBe(true);
-                    expect(dataComponents.length).toBe(0);
-                    done();
-                }
-            });
+            .expect('Content-Type', /json/);
+
+        // We expect to get an empty array
+        const dataComponents = res.body;
+        expect(dataComponents).toBeDefined();
+        expect(Array.isArray(dataComponents)).toBe(true);
+        expect(dataComponents.length).toBe(0);
+
     });
 
-    it('POST /api/data-components does not create an empty data component', function (done) {
+    it('POST /api/data-components does not create an empty data component', async function () {
         const body = { };
-        request(app)
+        const res = await request(app)
             .post('/api/data-components')
             .send(body)
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
-            .expect(400)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    done();
-                }
-            });
+            .expect(400);
+
     });
 
     let dataComponent1;
-    it('POST /api/data-components creates a data component', function (done) {
+    it('POST /api/data-components creates a data component', async function () {
         const timestamp = new Date().toISOString();
         initialObjectData.stix.created = timestamp;
         initialObjectData.stix.modified = timestamp;
         const body = initialObjectData;
-        request(app)
+        const res = await request(app)
             .post('/api/data-components')
             .send(body)
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(201)
-            .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    // We expect to get the created data component
-                    dataComponent1 = res.body;
-                    expect(dataComponent1).toBeDefined();
-                    expect(dataComponent1.stix).toBeDefined();
-                    expect(dataComponent1.stix.id).toBeDefined();
-                    expect(dataComponent1.stix.created).toBeDefined();
-                    expect(dataComponent1.stix.modified).toBeDefined();
-                    expect(dataComponent1.stix.x_mitre_attack_spec_version).toBe(config.app.attackSpecVersion);
-
-                    done();
-                }
-            });
+            .expect('Content-Type', /json/);
+
+        // We expect to get the created data component
+        dataComponent1 = res.body;
+        expect(dataComponent1).toBeDefined();
+        expect(dataComponent1.stix).toBeDefined();
+        expect(dataComponent1.stix.id).toBeDefined();
+        expect(dataComponent1.stix.created).toBeDefined();
+        expect(dataComponent1.stix.modified).toBeDefined();
+        expect(dataComponent1.stix.x_mitre_attack_spec_version).toBe(config.app.attackSpecVersion);
+
     });
 
-    it('GET /api/data-components returns the added data component', function (done) {
-        request(app)
+    it('GET /api/data-components returns the added data component', async function () {
+        const res = await request(app)
             .get('/api/data-components')
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(200)
             .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    // We expect to get one data component in an array
-                    const dataComponent = res.body;
-                    expect(dataComponent).toBeDefined();
-                    expect(Array.isArray(dataComponent)).toBe(true);
-                    expect(dataComponent.length).toBe(1);
-                    done();
-                }
-            });
+
+            // We expect to get one data component in an array
+            const dataComponent = res.body;
+            expect(dataComponent).toBeDefined();
+            expect(Array.isArray(dataComponent)).toBe(true);
+            expect(dataComponent.length).toBe(1);
     });
 
-    it('GET /api/data-components/:id should not return a data component when the id cannot be found', function (done) {
-        request(app)
+    it('GET /api/data-components/:id should not return a data component when the id cannot be found', async function () {
+        const res = await request(app)
             .get('/api/data-components/not-an-id')
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
-            .expect(404)
-            .end(function (err, res) {
-                if (err) {
-                    done(err);
-                } else {
-                    done();
-                }
-            });
+            .expect(404);
     });
 
-    it('GET /api/data-components/:id returns the added data component', function (done) {
-        request(app)
+    it('GET /api/data-components/:id returns the added data component', async function () {
+        const res = await request(app)
             .get('/api/data-components/' + dataComponent1.stix.id)
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(200)
             .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    // We expect to get one data component in an array
-                    const dataComponents = res.body;
-                    expect(dataComponents).toBeDefined();
-                    expect(Array.isArray(dataComponents)).toBe(true);
-                    expect(dataComponents.length).toBe(1);
-
-                    const dataComponent = dataComponents[0];
-                    expect(dataComponent).toBeDefined();
-                    expect(dataComponent.stix).toBeDefined();
-                    expect(dataComponent.stix.id).toBe(dataComponent1.stix.id);
-                    expect(dataComponent.stix.type).toBe(dataComponent1.stix.type);
-                    expect(dataComponent.stix.name).toBe(dataComponent1.stix.name);
-                    expect(dataComponent.stix.description).toBe(dataComponent1.stix.description);
-                    expect(dataComponent.stix.spec_version).toBe(dataComponent1.stix.spec_version);
-                    expect(dataComponent.stix.object_marking_refs).toEqual(expect.arrayContaining(dataComponent1.stix.object_marking_refs));
-                    expect(dataComponent.stix.created_by_ref).toBe(dataComponent1.stix.created_by_ref);
-                    expect(dataComponent.stix.x_mitre_version).toBe(dataComponent1.stix.x_mitre_version);
-                    expect(dataComponent.stix.x_mitre_attack_spec_version).toBe(dataComponent1.stix.x_mitre_attack_spec_version);
-
-                    done();
-                }
-            });
+
+        // We expect to get one data component in an array
+        const dataComponents = res.body;
+        expect(dataComponents).toBeDefined();
+        expect(Array.isArray(dataComponents)).toBe(true);
+        expect(dataComponents.length).toBe(1);
+
+        const dataComponent = dataComponents[0];
+        expect(dataComponent).toBeDefined();
+        expect(dataComponent.stix).toBeDefined();
+        expect(dataComponent.stix.id).toBe(dataComponent1.stix.id);
+        expect(dataComponent.stix.type).toBe(dataComponent1.stix.type);
+        expect(dataComponent.stix.name).toBe(dataComponent1.stix.name);
+        expect(dataComponent.stix.description).toBe(dataComponent1.stix.description);
+        expect(dataComponent.stix.spec_version).toBe(dataComponent1.stix.spec_version);
+        expect(dataComponent.stix.object_marking_refs).toEqual(expect.arrayContaining(dataComponent1.stix.object_marking_refs));
+        expect(dataComponent.stix.created_by_ref).toBe(dataComponent1.stix.created_by_ref);
+        expect(dataComponent.stix.x_mitre_version).toBe(dataComponent1.stix.x_mitre_version);
+        expect(dataComponent.stix.x_mitre_attack_spec_version).toBe(dataComponent1.stix.x_mitre_attack_spec_version);
+
     });
 
-    it('PUT /api/data-components updates a data component', function (done) {
+    it('PUT /api/data-components updates a data component', async function () {
         const originalModified = dataComponent1.stix.modified;
         const timestamp = new Date().toISOString();
         dataComponent1.stix.modified = timestamp;
         dataComponent1.stix.description = 'This is an updated data component.'
         const body = dataComponent1;
-        request(app)
+        const res = await request(app)
             .put('/api/data-components/' + dataComponent1.stix.id + '/modified/' + originalModified)
             .send(body)
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(200)
             .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    // We expect to get the updated data component
-                    const dataComponent = res.body;
-                    expect(dataComponent).toBeDefined();
-                    expect(dataComponent.stix.id).toBe(dataComponent1.stix.id);
-                    expect(dataComponent.stix.modified).toBe(dataComponent1.stix.modified);
-                    done();
-                }
-            });
+
+        // We expect to get the updated data component
+        const dataComponent = res.body;
+        expect(dataComponent).toBeDefined();
+        expect(dataComponent.stix.id).toBe(dataComponent1.stix.id);
+        expect(dataComponent.stix.modified).toBe(dataComponent1.stix.modified);
+
     });
 
-    it('POST /api/data-components does not create a data component with the same id and modified date', function (done) {
+    it('POST /api/data-components does not create a data component with the same id and modified date', async function () {
         const body = dataComponent1;
-        request(app)
+        const res = await request(app)
             .post('/api/data-components')
             .send(body)
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
-            .expect(409)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    done();
-                }
-            });
+            .expect(409);
+
     });
 
     let dataComponent2;
-    it('POST /api/data-components should create a new version of a data component with a duplicate stix.id but different stix.modified date', function (done) {
+    it('POST /api/data-components should create a new version of a data component with a duplicate stix.id but different stix.modified date', async function () {
         dataComponent2 = _.cloneDeep(dataComponent1);
         dataComponent2._id = undefined;
         dataComponent2.__t = undefined;
@@ -256,121 +202,91 @@ describe('Data Components API', function () {
         const timestamp = new Date().toISOString();
         dataComponent2.stix.modified = timestamp;
         const body = dataComponent2;
-        request(app)
+        const res = await request(app)
             .post('/api/data-components')
             .send(body)
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(201)
-            .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    // We expect to get the created data component
-                    const dataComponent = res.body;
-                    expect(dataComponent).toBeDefined();
-                    done();
-                }
-            });
+            .expect('Content-Type', /json/);
+
+        // We expect to get the created data component
+        const dataComponent = res.body;
+        expect(dataComponent).toBeDefined();
+
     });
 
-    it('GET /api/data-components returns the latest added data component', function (done) {
-        request(app)
+    it('GET /api/data-components returns the latest added data component', async function () {
+        const res = await request(app)
             .get('/api/data-components/' + dataComponent2.stix.id)
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(200)
-            .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    // We expect to get one data component in an array
-                    const dataComponents = res.body;
-                    expect(dataComponents).toBeDefined();
-                    expect(Array.isArray(dataComponents)).toBe(true);
-                    expect(dataComponents.length).toBe(1);
-                    const dataComponent = dataComponents[0];
-                    expect(dataComponent.stix.id).toBe(dataComponent2.stix.id);
-                    expect(dataComponent.stix.modified).toBe(dataComponent2.stix.modified);
-                    done();
-                }
-            });
+            .expect('Content-Type', /json/);
+
+        // We expect to get one data component in an array
+        const dataComponents = res.body;
+        expect(dataComponents).toBeDefined();
+        expect(Array.isArray(dataComponents)).toBe(true);
+        expect(dataComponents.length).toBe(1);
+        const dataComponent = dataComponents[0];
+        expect(dataComponent.stix.id).toBe(dataComponent2.stix.id);
+        expect(dataComponent.stix.modified).toBe(dataComponent2.stix.modified);
+
     });
 
-    it('GET /api/data-components returns all added data component', function (done) {
-        request(app)
+    it('GET /api/data-components returns all added data component', async function () {
+        const res = await request(app)
             .get('/api/data-components/' + dataComponent1.stix.id + '?versions=all')
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(200)
             .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    // We expect to get two data components in an array
-                    const dataComponents = res.body;
-                    expect(dataComponents).toBeDefined();
-                    expect(Array.isArray(dataComponents)).toBe(true);
-                    expect(dataComponents.length).toBe(2);
-                    done();
-                }
-            });
+
+        // We expect to get two data components in an array
+        const dataComponents = res.body;
+        expect(dataComponents).toBeDefined();
+        expect(Array.isArray(dataComponents)).toBe(true);
+        expect(dataComponents.length).toBe(2);
+
     });
 
-    it('GET /api/data-components/:id/modified/:modified returns the first added data component', function (done) {
-        request(app)
+    it('GET /api/data-components/:id/modified/:modified returns the first added data component', async function () {
+        const res = await request(app)
             .get('/api/data-components/' + dataComponent1.stix.id + '/modified/' + dataComponent1.stix.modified)
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(200)
-            .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    // We expect to get one data component in an array
-                    const dataComponent = res.body;
-                    expect(dataComponent).toBeDefined();
-                    expect(dataComponent.stix).toBeDefined();
-                    expect(dataComponent.stix.id).toBe(dataComponent1.stix.id);
-                    expect(dataComponent.stix.modified).toBe(dataComponent1.stix.modified);
-                    done();
-                }
-            });
+            .expect('Content-Type', /json/);
+
+        // We expect to get one data component in an array
+        const dataComponent = res.body;
+        expect(dataComponent).toBeDefined();
+        expect(dataComponent.stix).toBeDefined();
+        expect(dataComponent.stix.id).toBe(dataComponent1.stix.id);
+        expect(dataComponent.stix.modified).toBe(dataComponent1.stix.modified);
+
     });
 
-    it('GET /api/data-components/:id/modified/:modified returns the second added data component', function (done) {
-        request(app)
+    it('GET /api/data-components/:id/modified/:modified returns the second added data component', async function () {
+        const res = await request(app)
             .get('/api/data-components/' + dataComponent2.stix.id + '/modified/' + dataComponent2.stix.modified)
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(200)
-            .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    // We expect to get one data component in an array
-                    const dataComponent = res.body;
-                    expect(dataComponent).toBeDefined();
-                    expect(dataComponent.stix).toBeDefined();
-                    expect(dataComponent.stix.id).toBe(dataComponent2.stix.id);
-                    expect(dataComponent.stix.modified).toBe(dataComponent2.stix.modified);
-                    done();
-                }
-            });
+            .expect('Content-Type', /json/);
+
+        // We expect to get one data component in an array
+        const dataComponent = res.body;
+        expect(dataComponent).toBeDefined();
+        expect(dataComponent.stix).toBeDefined();
+        expect(dataComponent.stix.id).toBe(dataComponent2.stix.id);
+        expect(dataComponent.stix.modified).toBe(dataComponent2.stix.modified);
+
     });
 
     let dataComponent3;
-    it('POST /api/data-components should create a new version of a data component with a duplicate stix.id but different stix.modified date', function (done) {
+    it('POST /api/data-components should create a new version of a data component with a duplicate stix.id but different stix.modified date', async function () {
         dataComponent3 = _.cloneDeep(dataComponent1);
         dataComponent3._id = undefined;
         dataComponent3.__t = undefined;
@@ -378,91 +294,58 @@ describe('Data Components API', function () {
         const timestamp = new Date().toISOString();
         dataComponent3.stix.modified = timestamp;
         const body = dataComponent3;
-        request(app)
+        const res = await request(app)
             .post('/api/data-components')
             .send(body)
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(201)
-            .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    // We expect to get the created data component
-                    const dataComponent = res.body;
-                    expect(dataComponent).toBeDefined();
-                    done();
-                }
-            });
+            .expect('Content-Type', /json/);
+
+        // We expect to get the created data component
+        const dataComponent = res.body;
+        expect(dataComponent).toBeDefined();
+
     });
 
-    it('DELETE /api/data-components/:id should not delete a data component when the id cannot be found', function (done) {
-        request(app)
+    it('DELETE /api/data-components/:id should not delete a data component when the id cannot be found', async function () {
+        const res = await request(app)
             .delete('/api/data-components/not-an-id')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
-            .expect(404)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    done();
-                }
-            });
+            .expect(404);
+
     });
 
-    it('DELETE /api/data-components/:id/modified/:modified deletes a data component', function (done) {
-        request(app)
+    it('DELETE /api/data-components/:id/modified/:modified deletes a data component', async function () {
+        const res = await request(app)
             .delete('/api/data-components/' + dataComponent1.stix.id + '/modified/' + dataComponent1.stix.modified)
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
-            .expect(204)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    done();
-                }
-            });
+            .expect(204);
+
     });
 
-    it('DELETE /api/data-components/:id should delete all the data components with the same stix id', function (done) {
-        request(app)
+    it('DELETE /api/data-components/:id should delete all the data components with the same stix id', async function () {
+        const res = await request(app)
             .delete('/api/data-components/' + dataComponent2.stix.id)
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
-            .expect(204)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    done();
-                }
-            });
+            .expect(204);
+
     }); 
 
-    it('GET /api/data-components returns an empty array of data components', function (done) {
-        request(app)
+    it('GET /api/data-components returns an empty array of data components', async function () {
+        const res = await request(app)
             .get('/api/data-components')
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(200)
             .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    // We expect to get an empty array
-                    const dataComponents = res.body;
-                    expect(dataComponents).toBeDefined();
-                    expect(Array.isArray(dataComponents)).toBe(true);
-                    expect(dataComponents.length).toBe(0);
-                    done();
-                }
-            });
+
+        // We expect to get an empty array
+        const dataComponents = res.body;
+        expect(dataComponents).toBeDefined();
+        expect(Array.isArray(dataComponents)).toBe(true);
+        expect(dataComponents.length).toBe(0);
+
     });
 
     after(async function() {

From f0482502c77dc2821dffc483b4297d22f2302d79 Mon Sep 17 00:00:00 2001
From: Sun <vsun@mitre.org>
Date: Fri, 6 Oct 2023 15:02:22 -0400
Subject: [PATCH 02/15] refactoring data components repo and service

---
 app/repository/data-components-repository.js |  13 +
 app/services/data-components-service.js      | 480 +++----------------
 2 files changed, 86 insertions(+), 407 deletions(-)
 create mode 100644 app/repository/data-components-repository.js

diff --git a/app/repository/data-components-repository.js b/app/repository/data-components-repository.js
new file mode 100644
index 00000000..35e5a80f
--- /dev/null
+++ b/app/repository/data-components-repository.js
@@ -0,0 +1,13 @@
+'use strict';
+
+const BaseRepository = require('./_base.repository');
+const DataComponent = require('../models/data-component-model');
+
+class DataComponentsRepository extends BaseRepository {
+
+    constructor() {
+        super(DataComponent);
+    }
+}
+
+module.exports = new DataComponentsRepository();
\ No newline at end of file
diff --git a/app/services/data-components-service.js b/app/services/data-components-service.js
index a2016989..3567676f 100644
--- a/app/services/data-components-service.js
+++ b/app/services/data-components-service.js
@@ -9,6 +9,10 @@ const config = require('../config/config');
 const regexValidator = require('../lib/regex');
 const {lastUpdatedByQueryHelper} = require('../lib/request-parameter-helper');
 
+const DataComponentsRepository = require('../repository/data-components-repository.js');
+
+const BaseService = require('./_base.service');
+
 const errors = {
     missingParameter: 'Missing required parameter',
     badlyFormattedParameter: 'Badly formatted parameter',
@@ -16,432 +20,94 @@ const errors = {
     notFound: 'Document not found',
     invalidQueryStringParameter: 'Invalid query string parameter'
 };
-exports.errors = errors;
 
-exports.retrieveAll = function(options, callback) {
-    // Build the query
-    const query = {};
-    if (!options.includeRevoked) {
-        query['stix.revoked'] = { $in: [null, false] };
-    }
-    if (!options.includeDeprecated) {
-        query['stix.x_mitre_deprecated'] = { $in: [null, false] };
-    }
-    if (typeof options.state !== 'undefined') {
-        if (Array.isArray(options.state)) {
-            query['workspace.workflow.state'] = { $in: options.state };
-        }
-        else {
-            query['workspace.workflow.state'] = options.state;
-        }
-    }
-    if (typeof options.lastUpdatedBy !== 'undefined') {
-      query['workspace.workflow.created_by_user_account'] = lastUpdatedByQueryHelper(options.lastUpdatedBy);
-    }
-
-    // Build the aggregation
-    // - Group the documents by stix.id, sorted by stix.modified
-    // - Use the first document in each group (according to the value of stix.modified)
-    // - Then apply query, skip and limit options
-    const aggregation = [
-        { $sort: { 'stix.id': 1, 'stix.modified': -1 } },
-        { $group: { _id: '$stix.id', document: { $first: '$$ROOT' }}},
-        { $replaceRoot: { newRoot: '$document' }},
-        { $sort: { 'stix.id': 1 }},
-        { $match: query }
-    ];
+class DataComponentsService extends BaseService {
 
-    if (typeof options.search !== 'undefined') {
-        options.search = regexValidator.sanitizeRegex(options.search);
-        const match = { $match: { $or: [
-                    { 'stix.name': { '$regex': options.search, '$options': 'i' }},
-                    { 'stix.description': { '$regex': options.search, '$options': 'i' }}
-                ]}};
-        aggregation.push(match);
-    }
-
-    const facet = {
-        $facet: {
-            totalCount: [ { $count: 'totalCount' }],
-            documents: [ ]
-        }
-    };
-    if (options.offset) {
-        facet.$facet.documents.push({ $skip: options.offset });
-    }
-    else {
-        facet.$facet.documents.push({ $skip: 0 });
+    constructor() {
+        super(DataComponentsRepository, DataComponent);
     }
-    if (options.limit) {
-        facet.$facet.documents.push({ $limit: options.limit });
-    }
-    aggregation.push(facet);
 
-    // Retrieve the documents
-    DataComponent.aggregate(aggregation, function(err, results) {
-        if (err) {
-            return callback(err);
+    async retrieveAllAsync(options) {
+        // Build the query
+        const query = {};
+        if (!options.includeRevoked) {
+            query['stix.revoked'] = { $in: [null, false] };
         }
-        else {
-            identitiesService.addCreatedByAndModifiedByIdentitiesToAll(results[0].documents)
-                .then(function() {
-                    if (options.includePagination) {
-                        let derivedTotalCount = 0;
-                        if (results[0].totalCount.length > 0) {
-                            derivedTotalCount = results[0].totalCount[0].totalCount;
-                        }
-                        const returnValue = {
-                            pagination: {
-                                total: derivedTotalCount,
-                                offset: options.offset,
-                                limit: options.limit
-                            },
-                            data: results[0].documents
-                        };
-                        return callback(null, returnValue);
-                    }
-                    else {
-                        return callback(null, results[0].documents);
-                    }
-                });
+        if (!options.includeDeprecated) {
+            query['stix.x_mitre_deprecated'] = { $in: [null, false] };
         }
-    });
-};
-
-exports.retrieveAllAsync = async function(options) {
-    // Build the query
-    const query = {};
-    if (!options.includeRevoked) {
-        query['stix.revoked'] = { $in: [null, false] };
-    }
-    if (!options.includeDeprecated) {
-        query['stix.x_mitre_deprecated'] = { $in: [null, false] };
-    }
-    if (typeof options.state !== 'undefined') {
-        if (Array.isArray(options.state)) {
-            query['workspace.workflow.state'] = { $in: options.state };
-        } else {
-            query['workspace.workflow.state'] = options.state;
-        }
-    }
-
-    // Build the aggregation
-    // - Group the documents by stix.id, sorted by stix.modified
-    // - Use the first document in each group (according to the value of stix.modified)
-    // - Then apply query, skip and limit options
-    const aggregation = [
-        { $sort: { 'stix.id': 1, 'stix.modified': -1 } },
-        { $group: { _id: '$stix.id', document: { $first: '$$ROOT' } } },
-        { $replaceRoot: { newRoot: '$document' } },
-        { $sort: { 'stix.id': 1 } },
-        { $match: query }
-    ];
-
-    if (typeof options.search !== 'undefined') {
-        const match = {
-            $match: {
-                $or: [
-                    { 'stix.name': { '$regex': options.search, '$options': 'i' } },
-                    { 'stix.description': { '$regex': options.search, '$options': 'i' } }
-                ]
+        if (typeof options.state !== 'undefined') {
+            if (Array.isArray(options.state)) {
+                query['workspace.workflow.state'] = { $in: options.state };
+            } else {
+                query['workspace.workflow.state'] = options.state;
             }
-        };
-        aggregation.push(match);
-    }
-
-    const facet = {
-        $facet: {
-            totalCount: [{ $count: 'totalCount' }],
-            documents: []
         }
-    };
-    if (options.offset) {
-        facet.$facet.documents.push({ $skip: options.offset });
-    } else {
-        facet.$facet.documents.push({ $skip: 0 });
-    }
-    if (options.limit) {
-        facet.$facet.documents.push({ $limit: options.limit });
-    }
-    aggregation.push(facet);
-
-    // Retrieve the documents
-    const results = await DataComponent.aggregate(aggregation);
-    await identitiesService.addCreatedByAndModifiedByIdentitiesToAll(results[0].documents);
-
-    if (options.includePagination) {
-        let derivedTotalCount = 0;
-        if (results[0].totalCount.length > 0) {
-            derivedTotalCount = results[0].totalCount[0].totalCount;
-        }
-        const returnValue = {
-            pagination: {
-                total: derivedTotalCount,
-                offset: options.offset,
-                limit: options.limit
-            },
-            data: results[0].documents
-        };
-        return returnValue;
-    }
-    else {
-        return results[0].documents;
-    }
-};
 
-exports.retrieveById = function(stixId, options, callback) {
-    // versions=all Retrieve all data components with the stixId
-    // versions=latest Retrieve the data components with the latest modified date for this stixId
-
-    if (!stixId) {
-        const error = new Error(errors.missingParameter);
-        error.parameterName = 'stixId';
-        return callback(error);
-    }
-
-    if (options.versions === 'all') {
-        DataComponent.find({'stix.id': stixId})
-            .lean()
-            .exec(function (err, dataComponents) {
-                if (err) {
-                    if (err.name === 'CastError') {
-                        const error = new Error(errors.badlyFormattedParameter);
-                        error.parameterName = 'stixId';
-                        return callback(error);
-                    } else {
-                        return callback(err);
-                    }
-                } else {
-                    identitiesService.addCreatedByAndModifiedByIdentitiesToAll(dataComponents)
-                        .then(() => callback(null, dataComponents));
+        // Build the aggregation
+        // - Group the documents by stix.id, sorted by stix.modified
+        // - Use the first document in each group (according to the value of stix.modified)
+        // - Then apply query, skip and limit options
+        const aggregation = [
+            { $sort: { 'stix.id': 1, 'stix.modified': -1 } },
+            { $group: { _id: '$stix.id', document: { $first: '$$ROOT' } } },
+            { $replaceRoot: { newRoot: '$document' } },
+            { $sort: { 'stix.id': 1 } },
+            { $match: query }
+        ];
+
+        if (typeof options.search !== 'undefined') {
+            const match = {
+                $match: {
+                    $or: [
+                        { 'stix.name': { '$regex': options.search, '$options': 'i' } },
+                        { 'stix.description': { '$regex': options.search, '$options': 'i' } }
+                    ]
                 }
-            });
-    }
-    else if (options.versions === 'latest') {
-        DataComponent.findOne({ 'stix.id': stixId })
-            .sort('-stix.modified')
-            .lean()
-            .exec(function(err, dataComponent) {
-                if (err) {
-                    if (err.name === 'CastError') {
-                        const error = new Error(errors.badlyFormattedParameter);
-                        error.parameterName = 'stixId';
-                        return callback(error);
-                    }
-                    else {
-                        return callback(err);
-                    }
-                }
-                else {
-                    // Note: document is null if not found
-                    if (dataComponent) {
-                        identitiesService.addCreatedByAndModifiedByIdentities(dataComponent)
-                            .then(() => callback(null, [ dataComponent ]));
-                    }
-                    else {
-                        return callback(null, []);
-                    }
-                }
-            });
-    }
-    else {
-        const error = new Error(errors.invalidQueryStringParameter);
-        error.parameterName = 'versions';
-        return callback(error);
-    }
-};
-
-exports.retrieveVersionById = function(stixId, modified, callback) {
-    // Retrieve the versions of the data component with the matching stixId and modified date
-
-    if (!stixId) {
-        const error = new Error(errors.missingParameter);
-        error.parameterName = 'stixId';
-        return callback(error);
-    }
-
-    if (!modified) {
-        const error = new Error(errors.missingParameter);
-        error.parameterName = 'modified';
-        return callback(error);
-    }
-
-    DataComponent.findOne({ 'stix.id': stixId, 'stix.modified': modified }, function(err, dataComponent) {
-        if (err) {
-            if (err.name === 'CastError') {
-                const error = new Error(errors.badlyFormattedParameter);
-                error.parameterName = 'stixId';
-                return callback(error);
-            }
-            else {
-                return callback(err);
-            }
-        }
-        else {
-            // Note: document is null if not found
-            if (dataComponent) {
-                identitiesService.addCreatedByAndModifiedByIdentities(dataComponent)
-                    .then(() => callback(null, dataComponent));
-            }
-            else {
-                return callback();
-            }
-        }
-    });
-};
-
-exports.createIsAsync = true;
-exports.create = async function(data, options) {
-    // This function handles two use cases:
-    //   1. This is a completely new object. Create a new object and generate the stix.id if not already
-    //      provided. Set both stix.created_by_ref and stix.x_mitre_modified_by_ref to the organization identity.
-    //   2. This is a new version of an existing object. Create a new object with the specified id.
-    //      Set stix.x_mitre_modified_by_ref to the organization identity.
-
-    // Create the document
-    const dataComponent = new DataComponent(data);
-
-    options = options || {};
-    if (!options.import) {
-        // Set the ATT&CK Spec Version
-        dataComponent.stix.x_mitre_attack_spec_version = dataComponent.stix.x_mitre_attack_spec_version ?? config.app.attackSpecVersion;
-
-        // Record the user account that created the object
-        if (options.userAccountId) {
-            dataComponent.workspace.workflow.created_by_user_account = options.userAccountId;
+            };
+            aggregation.push(match);
         }
 
-        // Set the default marking definitions
-        await attackObjectsService.setDefaultMarkingDefinitions(dataComponent);
-
-        // Get the organization identity
-        const organizationIdentityRef = await systemConfigurationService.retrieveOrganizationIdentityRef();
-
-        // Check for an existing object
-        let existingObject;
-        if (dataComponent.stix.id) {
-            existingObject = await DataComponent.findOne({ 'stix.id': dataComponent.stix.id });
-        }
-
-        if (existingObject) {
-            // New version of an existing object
-            // Only set the x_mitre_modified_by_ref property
-            dataComponent.stix.x_mitre_modified_by_ref = organizationIdentityRef;
-        }
-        else {
-            // New object
-            // Assign a new STIX id if not already provided
-            dataComponent.stix.id = dataComponent.stix.id || `x-mitre-data-component--${uuid.v4()}`;
-
-            // Set the created_by_ref and x_mitre_modified_by_ref properties
-            dataComponent.stix.created_by_ref = organizationIdentityRef;
-            dataComponent.stix.x_mitre_modified_by_ref = organizationIdentityRef;
-        }
-    }
-
-    // Save the document in the database
-    try {
-        const savedDataComponent = await dataComponent.save();
-        return savedDataComponent;
-    }
-    catch (err) {
-        if (err.name === 'MongoServerError' && err.code === 11000) {
-            // 11000 = Duplicate index
-            const error = new Error(errors.duplicateId);
-            throw error;
+        const facet = {
+            $facet: {
+                totalCount: [{ $count: 'totalCount' }],
+                documents: []
+            }
+        };
+        if (options.offset) {
+            facet.$facet.documents.push({ $skip: options.offset });
+        } else {
+            facet.$facet.documents.push({ $skip: 0 });
         }
-        else {
-            throw err;
+        if (options.limit) {
+            facet.$facet.documents.push({ $limit: options.limit });
         }
-    }
-};
+        aggregation.push(facet);
 
-exports.updateFull = function(stixId, stixModified, data, callback) {
-    if (!stixId) {
-        const error = new Error(errors.missingParameter);
-        error.parameterName = 'stixId';
-        return callback(error);
-    }
+        // Retrieve the documents
+        const results = await DataComponent.aggregate(aggregation);
+        await identitiesService.addCreatedByAndModifiedByIdentitiesToAll(results[0].documents);
 
-    if (!stixModified) {
-        const error = new Error(errors.missingParameter);
-        error.parameterName = 'modified';
-        return callback(error);
-    }
-
-    DataComponent.findOne({ 'stix.id': stixId, 'stix.modified': stixModified }, function(err, document) {
-        if (err) {
-            if (err.name === 'CastError') {
-                var error = new Error(errors.badlyFormattedParameter);
-                error.parameterName = 'stixId';
-                return callback(error);
-            }
-            else {
-                return callback(err);
+        if (options.includePagination) {
+            let derivedTotalCount = 0;
+            if (results[0].totalCount.length > 0) {
+                derivedTotalCount = results[0].totalCount[0].totalCount;
             }
-        }
-        else if (!document) {
-            // document not found
-            return callback(null);
+            const returnValue = {
+                pagination: {
+                    total: derivedTotalCount,
+                    offset: options.offset,
+                    limit: options.limit
+                },
+                data: results[0].documents
+            };
+            return returnValue;
         }
         else {
-            // Copy data to found document and save
-            Object.assign(document, data);
-            document.save(function(err, savedDocument) {
-                if (err) {
-                    if (err.name === 'MongoServerError' && err.code === 11000) {
-                        // 11000 = Duplicate index
-                        var error = new Error(errors.duplicateId);
-                        return callback(error);
-                    }
-                    else {
-                        return callback(err);
-                    }
-                }
-                else {
-                    return callback(null, savedDocument);
-                }
-            });
-        }
-    });
-};
-
-exports.deleteVersionById = function (stixId, stixModified, callback) {
-    if (!stixId) {
-        const error = new Error(errors.missingParameter);
-        error.parameterName = 'stixId';
-        return callback(error);
-    }
-
-    if (!stixModified) {
-        const error = new Error(errors.missingParameter);
-        error.parameterName = 'modified';
-        return callback(error);
-    }
-
-    DataComponent.findOneAndRemove({ 'stix.id': stixId, 'stix.modified': stixModified }, function (err, dataComponent) {
-        if (err) {
-            return callback(err);
-        } else {
-            // Note: data component is null if not found
-            return callback(null, dataComponent);
+            return results[0].documents;
         }
-    });
-};
+    };
 
-exports.deleteById = function (stixId, callback) {
-    if (!stixId) {
-        const error = new Error(errors.missingParameter);
-        error.parameterName = 'stixId';
-        return callback(error);
-    }
+}
 
-    DataComponent.deleteMany({ 'stix.id': stixId }, function (err, dataComponent) {
-        if (err) {
-            return callback(err);
-        } else {
-            //Note: dataComponent is null if not found
-            return callback(null, dataComponent);
-        }
-    });
-};
+module.exports = new DataComponentsService();

From c0e86f4fa44255833f056c63d473bc287c3b2fcc Mon Sep 17 00:00:00 2001
From: Sun <vsun@mitre.org>
Date: Fri, 6 Oct 2023 16:06:22 -0400
Subject: [PATCH 03/15] refactoring data component controller

---
 app/controllers/data-components-controller.js | 110 +++++++++---------
 1 file changed, 54 insertions(+), 56 deletions(-)

diff --git a/app/controllers/data-components-controller.js b/app/controllers/data-components-controller.js
index 3dc17f78..65daaece 100644
--- a/app/controllers/data-components-controller.js
+++ b/app/controllers/data-components-controller.js
@@ -2,8 +2,9 @@
 
 const dataComponentsService = require('../services/data-components-service');
 const logger = require('../lib/logger');
+const { BadlyFormattedParameterError, InvalidQueryStringParameterError, DuplicateIdError } = require('../exceptions');
 
-exports.retrieveAll = function(req, res) {
+exports.retrieveAll = async function(req, res) {
     const options = {
         offset: req.query.offset || 0,
         limit: req.query.limit || 0,
@@ -15,59 +16,65 @@ exports.retrieveAll = function(req, res) {
         includePagination: req.query.includePagination
     }
 
-    dataComponentsService.retrieveAll(options, function(err, results) {
-        if (err) {
-            logger.error('Failed with error: ' + err);
-            return res.status(500).send('Unable to get data components. Server error.');
+    try {
+        const results = await dataComponentsService.retrieveAll(options);
+        if (options.includePagination) {
+            logger.debug(`Success: Retrieved ${ results.data.length } of ${ results.pagination.total } total data component(s)`);
         }
         else {
-            if (options.includePagination) {
-                logger.debug(`Success: Retrieved ${ results.data.length } of ${ results.pagination.total } total data component(s)`);
-            }
-            else {
-                logger.debug(`Success: Retrieved ${ results.length } data component(s)`);
-            }
-            return res.status(200).send(results);
+            logger.debug(`Success: Retrieved ${ results.length } data component(s)`);
         }
-    });
+        return res.status(200).send(results);
+    } catch (err) {
+        logger.error('Failed with error: ' + err);
+        return res.status(500).send('Unable to get data components. Server error.');
+    }
+
 };
 
-exports.retrieveById = function(req, res) {
+exports.retrieveById = async function(req, res) {
     const options = {
         versions: req.query.versions || 'latest'
     }
 
-    dataComponentsService.retrieveById(req.params.stixId, options, function (err, dataComponents) {
-        if (err) {
-            if (err.message === dataComponentsService.errors.badlyFormattedParameter) {
-                logger.warn('Badly formatted stix id: ' + req.params.stixId);
-                return res.status(400).send('Stix id is badly formatted.');
-            }
-            else if (err.message === dataComponentsService.errors.invalidQueryStringParameter) {
-                logger.warn('Invalid query string: versions=' + req.query.versions);
-                return res.status(400).send('Query string parameter versions is invalid.');
-            }
-            else {
-                logger.error('Failed with error: ' + err);
-                return res.status(500).send('Unable to get data component. Server error.');
-            }
+    try {
+        const dataComponents = await dataComponentsService.retrieveById(req.params.stixId, options);
+        if (dataComponents.length === 0) {
+            return res.status(404).send('Data component not found.');
+        }
+        else {
+            logger.debug(`Success: Retrieved ${ dataComponents.length } data component(s) with id ${ req.params.stixId }`);
+            return res.status(200).send(dataComponents);
+        }
+    } catch (err) {
+        if (err instanceof BadlyFormattedParameterError) {
+            logger.warn('Badly formatted stix id: ' + req.params.stixId);
+            return res.status(400).send('Stix id is badly formatted.');
+        }
+        else if (err instanceof InvalidQueryStringParameterError) {
+            logger.warn('Invalid query string: versions=' + req.query.versions);
+            return res.status(400).send('Query string parameter versions is invalid.');
         }
         else {
-            if (dataComponents.length === 0) {
+            logger.error('Failed with error: ' + err);
+            return res.status(500).send('Unable to get data component. Server error.');
+        }
+    }
+
+};
+
+exports.retrieveVersionById = async function(req, res) {
+        try {
+            const dataComponent = dataComponentsService.retrieveVersionById(req.params.stixId, req.params.modified);
+            if (!dataComponent) {
                 return res.status(404).send('Data component not found.');
             }
             else {
-                logger.debug(`Success: Retrieved ${ dataComponents.length } data component(s) with id ${ req.params.stixId }`);
-                return res.status(200).send(dataComponents);
+                logger.debug(`Success: Retrieved data component with id ${dataComponent.id}`);
+                return res.status(200).send(dataComponent);
             }
-        }
-    });
-};
-
-exports.retrieveVersionById = function(req, res) {
-    dataComponentsService.retrieveVersionById(req.params.stixId, req.params.modified, function (err, dataComponent) {
-        if (err) {
-            if (err.message === dataComponentsService.errors.badlyFormattedParameter) {
+        }  catch (err) {
+            if (err instanceof BadlyFormattedParameterError) {
                 logger.warn('Badly formatted stix id: ' + req.params.stixId);
                 return res.status(400).send('Stix id is badly formatted.');
             }
@@ -75,16 +82,7 @@ exports.retrieveVersionById = function(req, res) {
                 logger.error('Failed with error: ' + err);
                 return res.status(500).send('Unable to get data component. Server error.');
             }
-        } else {
-            if (!dataComponent) {
-                return res.status(404).send('Data component not found.');
-            }
-            else {
-                logger.debug(`Success: Retrieved data component with id ${dataComponent.id}`);
-                return res.status(200).send(dataComponent);
-            }
-        }
-    });
+        } 
 };
 
 exports.create = async function(req, res) {
@@ -102,7 +100,7 @@ exports.create = async function(req, res) {
         return res.status(201).send(dataComponent);
     }
     catch(err) {
-        if (err.message === dataComponentsService.errors.duplicateId) {
+        if (err instanceof DuplicateIdError) {
             logger.warn("Duplicate stix.id and stix.modified");
             return res.status(409).send('Unable to create data component. Duplicate stix.id and stix.modified properties.');
         }
@@ -113,12 +111,12 @@ exports.create = async function(req, res) {
     }
 };
 
-exports.updateFull = function(req, res) {
+exports.updateFull = async function(req, res) {
     // Get the data from the request
     const dataComponentData = req.body;
 
     // Create the data component
-    dataComponentsService.updateFull(req.params.stixId, req.params.modified, dataComponentData, function(err, dataComponent) {
+    dataComponentsService.updateFull(req.params.stixId, req.params.modified, dataComponentData, async function(err, dataComponent) {
         if (err) {
             logger.error("Failed with error: " + err);
             return res.status(500).send("Unable to update data component. Server error.");
@@ -134,8 +132,8 @@ exports.updateFull = function(req, res) {
     });
 };
 
-exports.deleteVersionById = function(req, res) {
-    dataComponentsService.deleteVersionById(req.params.stixId, req.params.modified, function (err, dataComponent) {
+exports.deleteVersionById = async function(req, res) {
+    dataComponentsService.deleteVersionById(req.params.stixId, req.params.modified, async function (err, dataComponent) {
         if (err) {
             logger.error('Delete data component failed. ' + err);
             return res.status(500).send('Unable to delete data component. Server error.');
@@ -151,8 +149,8 @@ exports.deleteVersionById = function(req, res) {
     });
 };
 
-exports.deleteById = function(req, res) {
-    dataComponentsService.deleteById(req.params.stixId, function (err, dataComponents) {
+exports.deleteById = async function(req, res) {
+    dataComponentsService.deleteById(req.params.stixId, async function (err, dataComponents) {
         if (err) {
             logger.error('Delete data component failed. ' + err);
             return res.status(500).send('Unable to delete data component. Server error.');

From 531fbed3bc1f7c7d21ead9bd0818ff0ef8a250a7 Mon Sep 17 00:00:00 2001
From: Sun <vsun@mitre.org>
Date: Fri, 6 Oct 2023 16:09:42 -0400
Subject: [PATCH 04/15] refactoring data component controller part 2

---
 app/controllers/data-components-controller.js | 75 +++++++++----------
 1 file changed, 37 insertions(+), 38 deletions(-)

diff --git a/app/controllers/data-components-controller.js b/app/controllers/data-components-controller.js
index 65daaece..3c6d55b1 100644
--- a/app/controllers/data-components-controller.js
+++ b/app/controllers/data-components-controller.js
@@ -116,53 +116,52 @@ exports.updateFull = async function(req, res) {
     const dataComponentData = req.body;
 
     // Create the data component
-    dataComponentsService.updateFull(req.params.stixId, req.params.modified, dataComponentData, async function(err, dataComponent) {
-        if (err) {
-            logger.error("Failed with error: " + err);
-            return res.status(500).send("Unable to update data component. Server error.");
-        }
-        else {
-            if (!dataComponent) {
-                return res.status(404).send('Data component not found.');
-            } else {
-                logger.debug("Success: Updated data component with id " + dataComponent.stix.id);
-                return res.status(200).send(dataComponent);
-            }
+    
+    try {
+        const dataComponent = await dataComponentsService.updateFull(req.params.stixId, req.params.modified, dataComponentData);
+        if (!dataComponent) {
+            return res.status(404).send('Data component not found.');
+        } else {
+            logger.debug("Success: Updated data component with id " + dataComponent.stix.id);
+            return res.status(200).send(dataComponent);
         }
-    });
+    } catch (err) {
+        logger.error("Failed with error: " + err);
+        return res.status(500).send("Unable to update data component. Server error.");
+    }
+
 };
 
 exports.deleteVersionById = async function(req, res) {
-    dataComponentsService.deleteVersionById(req.params.stixId, req.params.modified, async function (err, dataComponent) {
-        if (err) {
-            logger.error('Delete data component failed. ' + err);
-            return res.status(500).send('Unable to delete data component. Server error.');
-        }
-        else {
-            if (!dataComponent) {
-                return res.status(404).send('Data component not found.');
-            } else {
-                logger.debug("Success: Deleted data component with id " + dataComponent.stix.id);
-                return res.status(204).end();
-            }
+
+    try {
+        const dataComponent = await dataComponentsService.deleteVersionById(req.params.stixId, req.params.modified);
+        if (!dataComponent) {
+            return res.status(404).send('Data component not found.');
+        } else {
+            logger.debug("Success: Deleted data component with id " + dataComponent.stix.id);
+            return res.status(204).end();
         }
-    });
+    } catch (err) {
+        logger.error('Delete data component failed. ' + err);
+        return res.status(500).send('Unable to delete data component. Server error.');
+    }
+
 };
 
 exports.deleteById = async function(req, res) {
-    dataComponentsService.deleteById(req.params.stixId, async function (err, dataComponents) {
-        if (err) {
-            logger.error('Delete data component failed. ' + err);
-            return res.status(500).send('Unable to delete data component. Server error.');
+    
+    try {
+        const dataComponents = await dataComponentsService.deleteById(req.params.stixId);
+        if (dataComponents.deletedCount === 0) {
+            return res.status(404).send('Data Component not found.');
         }
         else {
-            if (dataComponents.deletedCount === 0) {
-                return res.status(404).send('Data Component not found.');
-            }
-            else {
-                logger.debug(`Success: Deleted data component with id ${ req.params.stixId }`);
-                return res.status(204).end();
-            }
+            logger.debug(`Success: Deleted data component with id ${ req.params.stixId }`);
+            return res.status(204).end();
         }
-    });
+    } catch (err) {
+        logger.error('Delete data component failed. ' + err);
+        return res.status(500).send('Unable to delete data component. Server error.');
+    }
 };

From f58b15cc4dc4f84c6156dc18a928a740bd1a0546 Mon Sep 17 00:00:00 2001
From: Sun <vsun@mitre.org>
Date: Fri, 6 Oct 2023 16:32:28 -0400
Subject: [PATCH 05/15] fixing one issue with await in controller

---
 app/controllers/data-components-controller.js | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/app/controllers/data-components-controller.js b/app/controllers/data-components-controller.js
index 3c6d55b1..74c7acc4 100644
--- a/app/controllers/data-components-controller.js
+++ b/app/controllers/data-components-controller.js
@@ -65,7 +65,7 @@ exports.retrieveById = async function(req, res) {
 
 exports.retrieveVersionById = async function(req, res) {
         try {
-            const dataComponent = dataComponentsService.retrieveVersionById(req.params.stixId, req.params.modified);
+            const dataComponent = await dataComponentsService.retrieveVersionById(req.params.stixId, req.params.modified);
             if (!dataComponent) {
                 return res.status(404).send('Data component not found.');
             }
@@ -88,13 +88,13 @@ exports.retrieveVersionById = async function(req, res) {
 exports.create = async function(req, res) {
     // Get the data from the request
     const dataComponentData = req.body;
+    const options = {
+        import: false,
+        userAccountId: req.user?.userAccountId
+    };
 
     // Create the data component
     try {
-        const options = {
-            import: false,
-            userAccountId: req.user?.userAccountId
-        };
         const dataComponent = await dataComponentsService.create(dataComponentData, options);
         logger.debug("Success: Created data component with id " + dataComponent.stix.id);
         return res.status(201).send(dataComponent);

From 219d0483f20cccb7ff6efd37a8d593751b4bd55e Mon Sep 17 00:00:00 2001
From: Sun <vsun@mitre.org>
Date: Thu, 12 Oct 2023 15:29:11 -0400
Subject: [PATCH 06/15] resolving lint issues

---
 app/services/data-components-service.js          | 16 +---------------
 .../api/data-components/data-components.spec.js  | 12 ++++++------
 2 files changed, 7 insertions(+), 21 deletions(-)

diff --git a/app/services/data-components-service.js b/app/services/data-components-service.js
index 3567676f..46526257 100644
--- a/app/services/data-components-service.js
+++ b/app/services/data-components-service.js
@@ -1,26 +1,12 @@
 'use strict';
 
-const uuid = require('uuid');
 const DataComponent = require('../models/data-component-model');
-const systemConfigurationService = require('./system-configuration-service');
 const identitiesService = require('./identities-service');
-const attackObjectsService = require('./attack-objects-service');
-const config = require('../config/config');
-const regexValidator = require('../lib/regex');
-const {lastUpdatedByQueryHelper} = require('../lib/request-parameter-helper');
 
 const DataComponentsRepository = require('../repository/data-components-repository.js');
 
 const BaseService = require('./_base.service');
 
-const errors = {
-    missingParameter: 'Missing required parameter',
-    badlyFormattedParameter: 'Badly formatted parameter',
-    duplicateId: 'Duplicate id',
-    notFound: 'Document not found',
-    invalidQueryStringParameter: 'Invalid query string parameter'
-};
-
 class DataComponentsService extends BaseService {
 
     constructor() {
@@ -106,7 +92,7 @@ class DataComponentsService extends BaseService {
         else {
             return results[0].documents;
         }
-    };
+    }
 
 }
 
diff --git a/app/tests/api/data-components/data-components.spec.js b/app/tests/api/data-components/data-components.spec.js
index ef3486bc..52c139f5 100644
--- a/app/tests/api/data-components/data-components.spec.js
+++ b/app/tests/api/data-components/data-components.spec.js
@@ -74,7 +74,7 @@ describe('Data Components API', function () {
 
     it('POST /api/data-components does not create an empty data component', async function () {
         const body = { };
-        const res = await request(app)
+        await request(app)
             .post('/api/data-components')
             .send(body)
             .set('Accept', 'application/json')
@@ -124,7 +124,7 @@ describe('Data Components API', function () {
     });
 
     it('GET /api/data-components/:id should not return a data component when the id cannot be found', async function () {
-        const res = await request(app)
+        await request(app)
             .get('/api/data-components/not-an-id')
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
@@ -184,7 +184,7 @@ describe('Data Components API', function () {
 
     it('POST /api/data-components does not create a data component with the same id and modified date', async function () {
         const body = dataComponent1;
-        const res = await request(app)
+        await request(app)
             .post('/api/data-components')
             .send(body)
             .set('Accept', 'application/json')
@@ -309,7 +309,7 @@ describe('Data Components API', function () {
     });
 
     it('DELETE /api/data-components/:id should not delete a data component when the id cannot be found', async function () {
-        const res = await request(app)
+        await request(app)
             .delete('/api/data-components/not-an-id')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(404);
@@ -317,7 +317,7 @@ describe('Data Components API', function () {
     });
 
     it('DELETE /api/data-components/:id/modified/:modified deletes a data component', async function () {
-        const res = await request(app)
+        await request(app)
             .delete('/api/data-components/' + dataComponent1.stix.id + '/modified/' + dataComponent1.stix.modified)
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(204);
@@ -325,7 +325,7 @@ describe('Data Components API', function () {
     });
 
     it('DELETE /api/data-components/:id should delete all the data components with the same stix id', async function () {
-        const res = await request(app)
+        await request(app)
             .delete('/api/data-components/' + dataComponent2.stix.id)
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(204);

From b42452d6a2a888bfff65085de866998a68c3d83f Mon Sep 17 00:00:00 2001
From: Sun <vsun@mitre.org>
Date: Mon, 16 Oct 2023 13:22:01 -0400
Subject: [PATCH 07/15] changing function to static

---
 app/services/data-components-service.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/services/data-components-service.js b/app/services/data-components-service.js
index 46526257..132c0696 100644
--- a/app/services/data-components-service.js
+++ b/app/services/data-components-service.js
@@ -13,7 +13,7 @@ class DataComponentsService extends BaseService {
         super(DataComponentsRepository, DataComponent);
     }
 
-    async retrieveAllAsync(options) {
+    static async retrieveAllAsync(options) {
         // Build the query
         const query = {};
         if (!options.includeRevoked) {

From d40ff9fc4926de510bf6d1da6286092f94ecb845 Mon Sep 17 00:00:00 2001
From: Sun <vsun@mitre.org>
Date: Tue, 5 Dec 2023 14:39:44 -0500
Subject: [PATCH 08/15] fixing test

---
 app/services/data-components-service.js | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/app/services/data-components-service.js b/app/services/data-components-service.js
index 132c0696..99c86eaa 100644
--- a/app/services/data-components-service.js
+++ b/app/services/data-components-service.js
@@ -9,10 +9,6 @@ const BaseService = require('./_base.service');
 
 class DataComponentsService extends BaseService {
 
-    constructor() {
-        super(DataComponentsRepository, DataComponent);
-    }
-
     static async retrieveAllAsync(options) {
         // Build the query
         const query = {};
@@ -96,4 +92,4 @@ class DataComponentsService extends BaseService {
 
 }
 
-module.exports = new DataComponentsService();
+module.exports = new DataComponentsService('x-mitre-data-component', DataComponentsRepository);

From 4977b0ac7fd00d6c20e22421864d507a7560c6fe Mon Sep 17 00:00:00 2001
From: Sun <vsun@mitre.org>
Date: Wed, 6 Dec 2023 15:59:31 -0500
Subject: [PATCH 09/15] almost done with data components

---
 app/repository/data-components-repository.js | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/app/repository/data-components-repository.js b/app/repository/data-components-repository.js
index 35e5a80f..6b7983dc 100644
--- a/app/repository/data-components-repository.js
+++ b/app/repository/data-components-repository.js
@@ -5,9 +5,6 @@ const DataComponent = require('../models/data-component-model');
 
 class DataComponentsRepository extends BaseRepository {
 
-    constructor() {
-        super(DataComponent);
-    }
 }
 
-module.exports = new DataComponentsRepository();
\ No newline at end of file
+module.exports = new DataComponentsRepository(DataComponent);
\ No newline at end of file

From 34ccc2f5da60b4f28ac183520b60db240add8b6d Mon Sep 17 00:00:00 2001
From: Sun <vsun@mitre.org>
Date: Wed, 3 Jan 2024 14:03:45 -0500
Subject: [PATCH 10/15] made some progress on test

---
 app/services/data-components-service.js       |  2 +-
 .../api/data-sources/data-sources.spec.js     | 47 ++++++++++++-------
 2 files changed, 32 insertions(+), 17 deletions(-)

diff --git a/app/services/data-components-service.js b/app/services/data-components-service.js
index 99c86eaa..0f6471f9 100644
--- a/app/services/data-components-service.js
+++ b/app/services/data-components-service.js
@@ -9,7 +9,7 @@ const BaseService = require('./_base.service');
 
 class DataComponentsService extends BaseService {
 
-    static async retrieveAllAsync(options) {
+    async retrieveAllAsync(options) {
         // Build the query
         const query = {};
         if (!options.includeRevoked) {
diff --git a/app/tests/api/data-sources/data-sources.spec.js b/app/tests/api/data-sources/data-sources.spec.js
index d6f9022f..20d986dc 100644
--- a/app/tests/api/data-sources/data-sources.spec.js
+++ b/app/tests/api/data-sources/data-sources.spec.js
@@ -115,6 +115,11 @@ describe('Data Sources API', function () {
     let passportCookie;
 
     before(async function() {
+        // this.timeout(0);
+        // this.timeout(100000);
+
+
+
         // Establish the database connection
         // Use an in-memory database that we spin up for the test
         await database.initializeConnection();
@@ -274,28 +279,38 @@ describe('Data Sources API', function () {
             });
     });
 
-    it('GET /api/data-sources/:id returns the added data source with data components', async function () {
+    it('GET /api/data-sources/:id returns the added data source with data components', function (done) {
         initialDataComponentData.stix.x_mitre_data_source_ref = dataSource1.stix.id;
-        await loadDataComponents(initialDataComponentData);
+        loadDataComponents(initialDataComponentData);
 
-        const res = await request(app)
+        //this.timeout(0);
+
+        request(app)
             .get(`/api/data-sources/${dataSource1.stix.id}?retrieveDataComponents=true`)
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(200)
-            .expect('Content-Type', /json/);
-
-        // We expect to get one data source in an array
-        const dataSources = res.body;
-        expect(dataSources).toBeDefined();
-        expect(Array.isArray(dataSources)).toBe(true);
-        expect(dataSources.length).toBe(1);
-        const dataSource = dataSources[0];
-        expect(dataSource).toBeDefined();
-
-        // We expect to get 5 data components that reference this data source
-        expect(dataSource.dataComponents).toBeDefined();
-        expect(dataSource.dataComponents.length).toBe(5);
+            .expect('Content-Type', /json/)
+            .end(function(err, res) {
+                if (err) {
+                    done(err);
+                }
+                else {
+
+                    // We expect to get one data source in an array
+                    const dataSources = res.body;
+                    expect(dataSources).toBeDefined();
+                    expect(Array.isArray(dataSources)).toBe(true);
+                    expect(dataSources.length).toBe(1);
+                    const dataSource = dataSources[0];
+                    expect(dataSource).toBeDefined();
+
+                    // We expect to get 5 data components that reference this data source
+                    expect(dataSource.dataComponents).toBeDefined();
+                    expect(dataSource.dataComponents.length).toBe(5);
+                    done();
+                }
+            }); //.catch(done);
     });
 
     it('PUT /api/data-sources updates a data source', function (done) {

From 17e00dca0c67705166894b1450476e3f87bfcb8f Mon Sep 17 00:00:00 2001
From: Sun <vsun@mitre.org>
Date: Wed, 3 Jan 2024 14:07:46 -0500
Subject: [PATCH 11/15] all tests pass

---
 .../api/data-sources/data-sources.spec.js     | 42 +++++++------------
 1 file changed, 16 insertions(+), 26 deletions(-)

diff --git a/app/tests/api/data-sources/data-sources.spec.js b/app/tests/api/data-sources/data-sources.spec.js
index 20d986dc..6ebcd2e4 100644
--- a/app/tests/api/data-sources/data-sources.spec.js
+++ b/app/tests/api/data-sources/data-sources.spec.js
@@ -279,38 +279,28 @@ describe('Data Sources API', function () {
             });
     });
 
-    it('GET /api/data-sources/:id returns the added data source with data components', function (done) {
+    it('GET /api/data-sources/:id returns the added data source with data components', async function () {
         initialDataComponentData.stix.x_mitre_data_source_ref = dataSource1.stix.id;
-        loadDataComponents(initialDataComponentData);
+        await loadDataComponents(initialDataComponentData);
 
-        //this.timeout(0);
-
-        request(app)
+        const res = await request(app)
             .get(`/api/data-sources/${dataSource1.stix.id}?retrieveDataComponents=true`)
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(200)
-            .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-
-                    // We expect to get one data source in an array
-                    const dataSources = res.body;
-                    expect(dataSources).toBeDefined();
-                    expect(Array.isArray(dataSources)).toBe(true);
-                    expect(dataSources.length).toBe(1);
-                    const dataSource = dataSources[0];
-                    expect(dataSource).toBeDefined();
-
-                    // We expect to get 5 data components that reference this data source
-                    expect(dataSource.dataComponents).toBeDefined();
-                    expect(dataSource.dataComponents.length).toBe(5);
-                    done();
-                }
-            }); //.catch(done);
+            .expect('Content-Type', /json/);
+
+        // We expect to get one data source in an array
+        const dataSources = res.body;
+        expect(dataSources).toBeDefined();
+        expect(Array.isArray(dataSources)).toBe(true);
+        expect(dataSources.length).toBe(1);
+        const dataSource = dataSources[0];
+        expect(dataSource).toBeDefined();
+
+        // We expect to get 5 data components that reference this data source
+        expect(dataSource.dataComponents).toBeDefined();
+        expect(dataSource.dataComponents.length).toBe(5);
     });
 
     it('PUT /api/data-sources updates a data source', function (done) {

From c92e98ac87300a7b99b54ffa3c6a0e581ea2f359 Mon Sep 17 00:00:00 2001
From: Sun <vsun@mitre.org>
Date: Wed, 3 Jan 2024 14:10:20 -0500
Subject: [PATCH 12/15] merge ready

---
 app/services/data-components-service.js | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/app/services/data-components-service.js b/app/services/data-components-service.js
index 0f6471f9..db51c68c 100644
--- a/app/services/data-components-service.js
+++ b/app/services/data-components-service.js
@@ -1,6 +1,5 @@
 'use strict';
 
-const DataComponent = require('../models/data-component-model');
 const identitiesService = require('./identities-service');
 
 const DataComponentsRepository = require('../repository/data-components-repository.js');
@@ -67,7 +66,7 @@ class DataComponentsService extends BaseService {
         aggregation.push(facet);
 
         // Retrieve the documents
-        const results = await DataComponent.aggregate(aggregation);
+        const results = await this.repository.model.aggregate(aggregation);
         await identitiesService.addCreatedByAndModifiedByIdentitiesToAll(results[0].documents);
 
         if (options.includePagination) {

From ad0bff48c6026d7b4d15b24264c51918dfb42681 Mon Sep 17 00:00:00 2001
From: Sun <vsun@mitre.org>
Date: Wed, 3 Jan 2024 14:12:45 -0500
Subject: [PATCH 13/15] removed superfluous comments

---
 app/tests/api/data-sources/data-sources.spec.js | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/app/tests/api/data-sources/data-sources.spec.js b/app/tests/api/data-sources/data-sources.spec.js
index 6ebcd2e4..d6f9022f 100644
--- a/app/tests/api/data-sources/data-sources.spec.js
+++ b/app/tests/api/data-sources/data-sources.spec.js
@@ -115,11 +115,6 @@ describe('Data Sources API', function () {
     let passportCookie;
 
     before(async function() {
-        // this.timeout(0);
-        // this.timeout(100000);
-
-
-
         // Establish the database connection
         // Use an in-memory database that we spin up for the test
         await database.initializeConnection();

From 91f0996ca658fd6592c7ff58ea37fc3052006884 Mon Sep 17 00:00:00 2001
From: Sun <vsun@mitre.org>
Date: Thu, 4 Jan 2024 16:36:32 -0500
Subject: [PATCH 14/15] fixed review issues

---
 app/services/data-components-service.js | 83 -------------------------
 app/services/data-sources-service.js    |  2 +-
 2 files changed, 1 insertion(+), 84 deletions(-)

diff --git a/app/services/data-components-service.js b/app/services/data-components-service.js
index db51c68c..28d3f9fc 100644
--- a/app/services/data-components-service.js
+++ b/app/services/data-components-service.js
@@ -1,94 +1,11 @@
 'use strict';
 
-const identitiesService = require('./identities-service');
-
 const DataComponentsRepository = require('../repository/data-components-repository.js');
 
 const BaseService = require('./_base.service');
 
 class DataComponentsService extends BaseService {
 
-    async retrieveAllAsync(options) {
-        // Build the query
-        const query = {};
-        if (!options.includeRevoked) {
-            query['stix.revoked'] = { $in: [null, false] };
-        }
-        if (!options.includeDeprecated) {
-            query['stix.x_mitre_deprecated'] = { $in: [null, false] };
-        }
-        if (typeof options.state !== 'undefined') {
-            if (Array.isArray(options.state)) {
-                query['workspace.workflow.state'] = { $in: options.state };
-            } else {
-                query['workspace.workflow.state'] = options.state;
-            }
-        }
-
-        // Build the aggregation
-        // - Group the documents by stix.id, sorted by stix.modified
-        // - Use the first document in each group (according to the value of stix.modified)
-        // - Then apply query, skip and limit options
-        const aggregation = [
-            { $sort: { 'stix.id': 1, 'stix.modified': -1 } },
-            { $group: { _id: '$stix.id', document: { $first: '$$ROOT' } } },
-            { $replaceRoot: { newRoot: '$document' } },
-            { $sort: { 'stix.id': 1 } },
-            { $match: query }
-        ];
-
-        if (typeof options.search !== 'undefined') {
-            const match = {
-                $match: {
-                    $or: [
-                        { 'stix.name': { '$regex': options.search, '$options': 'i' } },
-                        { 'stix.description': { '$regex': options.search, '$options': 'i' } }
-                    ]
-                }
-            };
-            aggregation.push(match);
-        }
-
-        const facet = {
-            $facet: {
-                totalCount: [{ $count: 'totalCount' }],
-                documents: []
-            }
-        };
-        if (options.offset) {
-            facet.$facet.documents.push({ $skip: options.offset });
-        } else {
-            facet.$facet.documents.push({ $skip: 0 });
-        }
-        if (options.limit) {
-            facet.$facet.documents.push({ $limit: options.limit });
-        }
-        aggregation.push(facet);
-
-        // Retrieve the documents
-        const results = await this.repository.model.aggregate(aggregation);
-        await identitiesService.addCreatedByAndModifiedByIdentitiesToAll(results[0].documents);
-
-        if (options.includePagination) {
-            let derivedTotalCount = 0;
-            if (results[0].totalCount.length > 0) {
-                derivedTotalCount = results[0].totalCount[0].totalCount;
-            }
-            const returnValue = {
-                pagination: {
-                    total: derivedTotalCount,
-                    offset: options.offset,
-                    limit: options.limit
-                },
-                data: results[0].documents
-            };
-            return returnValue;
-        }
-        else {
-            return results[0].documents;
-        }
-    }
-
 }
 
 module.exports = new DataComponentsService('x-mitre-data-component', DataComponentsRepository);
diff --git a/app/services/data-sources-service.js b/app/services/data-sources-service.js
index 10eb4c47..bfbfbdb9 100644
--- a/app/services/data-sources-service.js
+++ b/app/services/data-sources-service.js
@@ -146,7 +146,7 @@ async function addDataComponents(dataSource) {
     // version doesn't.
 
     // Retrieve the latest version of all data components
-    const allDataComponents = await dataComponentsService.retrieveAllAsync({ includeDeprecated: true, includeRevoked: true });
+    const allDataComponents = await dataComponentsService.retrieveAll({ includeDeprecated: true, includeRevoked: true });
 
     // Add the data components that reference the data source
     dataSource.dataComponents = allDataComponents.filter(dataComponent => dataComponent.stix.x_mitre_data_source_ref === dataSource.stix.id);

From 434003f6aea11c454d4f3b6d3e5cce2017d05839 Mon Sep 17 00:00:00 2001
From: Sean Sica <23294618+seansica@users.noreply.github.com>
Date: Tue, 9 Jan 2024 10:04:51 -0500
Subject: [PATCH 15/15] DataComponents: Linting and change imported
 DataComponentsRepository to camelCase

---
 app/repository/data-components-repository.js | 4 +---
 app/services/data-components-service.js      | 8 +++-----
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/app/repository/data-components-repository.js b/app/repository/data-components-repository.js
index 6b7983dc..5729c336 100644
--- a/app/repository/data-components-repository.js
+++ b/app/repository/data-components-repository.js
@@ -3,8 +3,6 @@
 const BaseRepository = require('./_base.repository');
 const DataComponent = require('../models/data-component-model');
 
-class DataComponentsRepository extends BaseRepository {
-
-}
+class DataComponentsRepository extends BaseRepository {}
 
 module.exports = new DataComponentsRepository(DataComponent);
\ No newline at end of file
diff --git a/app/services/data-components-service.js b/app/services/data-components-service.js
index 28d3f9fc..83875bc4 100644
--- a/app/services/data-components-service.js
+++ b/app/services/data-components-service.js
@@ -1,11 +1,9 @@
 'use strict';
 
-const DataComponentsRepository = require('../repository/data-components-repository.js');
+const dataComponentsRepository = require('../repository/data-components-repository.js');
 
 const BaseService = require('./_base.service');
 
-class DataComponentsService extends BaseService {
+class DataComponentsService extends BaseService { }
 
-}
-
-module.exports = new DataComponentsService('x-mitre-data-component', DataComponentsRepository);
+module.exports = new DataComponentsService('x-mitre-data-component', dataComponentsRepository);