From 6f9b00760a0eb08d1ed180aa37a75d54baa3fdd2 Mon Sep 17 00:00:00 2001
From: Sun <vsun@mitre.org>
Date: Mon, 18 Dec 2023 14:46:06 -0500
Subject: [PATCH 1/9] refactoring service

---
 app/services/data-sources-service.js | 397 ++-------------------------
 1 file changed, 20 insertions(+), 377 deletions(-)

diff --git a/app/services/data-sources-service.js b/app/services/data-sources-service.js
index 10eb4c47..d27f3750 100644
--- a/app/services/data-sources-service.js
+++ b/app/services/data-sources-service.js
@@ -9,6 +9,7 @@ const attackObjectsService = require('./attack-objects-service');
 const config = require('../config/config');
 const regexValidator = require('../lib/regex');
 const {lastUpdatedByQueryHelper} = require('../lib/request-parameter-helper');
+const BaseService = require('./_base.service');
 
 const errors = {
     missingParameter: 'Missing required parameter',
@@ -19,392 +20,34 @@ const errors = {
 };
 exports.errors = errors;
 
-exports.retrieveAll = function(options, callback) {
-    // Build the query
-    const query = {};
-    if (!options.includeRevoked) {
-        query['stix.revoked'] = { $in: [null, false] };
-    }
-    if (!options.includeDeprecated) {
-        query['stix.x_mitre_deprecated'] = { $in: [null, false] };
-    }
-    if (typeof options.state !== 'undefined') {
-        if (Array.isArray(options.state)) {
-            query['workspace.workflow.state'] = { $in: options.state };
-        }
-        else {
-            query['workspace.workflow.state'] = options.state;
-        }
-    }
-    if (typeof options.domain !== 'undefined') {
-        if (Array.isArray(options.domain)) {
-            query['stix.x_mitre_domains'] = { $in: options.domain };
-        }
-        else {
-            query['stix.x_mitre_domains'] = options.domain;
-        }
-    }
-    if (typeof options.platform !== 'undefined') {
-        if (Array.isArray(options.platform)) {
-            query['stix.x_mitre_platforms'] = { $in: options.platform };
-        }
-        else {
-            query['stix.x_mitre_platforms'] = options.platform;
-        }
-    }
-    if (typeof options.lastUpdatedBy !== 'undefined') {
-      query['workspace.workflow.created_by_user_account'] = lastUpdatedByQueryHelper(options.lastUpdatedBy);
-    }
-
-    // Build the aggregation
-    // - Group the documents by stix.id, sorted by stix.modified
-    // - Use the first document in each group (according to the value of stix.modified)
-    // - Then apply query, skip and limit options
-    const aggregation = [
-        { $sort: { 'stix.id': 1, 'stix.modified': -1 } },
-        { $group: { _id: '$stix.id', document: { $first: '$$ROOT' }}},
-        { $replaceRoot: { newRoot: '$document' }},
-        { $sort: { 'stix.id': 1 }},
-        { $match: query }
-    ];
+class DataSourcesService extends BaseService {
 
-    if (typeof options.search !== 'undefined') {
-        options.search = regexValidator.sanitizeRegex(options.search);
-        const match = { $match: { $or: [
-                    { 'stix.name': { '$regex': options.search, '$options': 'i' }},
-                    { 'stix.description': { '$regex': options.search, '$options': 'i' }},
-                    { 'workspace.attack_id': { '$regex': options.search, '$options': 'i' }}
-                ]}};
-        aggregation.push(match);
-    }
-
-    const facet = {
-        $facet: {
-            totalCount: [ { $count: 'totalCount' }],
-            documents: [ ]
+    async addExtraData(dataSource, retrieveDataComponents) {
+        await identitiesService.addCreatedByAndModifiedByIdentities(dataSource);
+        if (retrieveDataComponents) {
+            await addDataComponents(dataSource);
         }
-    };
-    if (options.offset) {
-        facet.$facet.documents.push({ $skip: options.offset });
-    }
-    else {
-        facet.$facet.documents.push({ $skip: 0 });
     }
-    if (options.limit) {
-        facet.$facet.documents.push({ $limit: options.limit });
-    }
-    aggregation.push(facet);
 
-    // Retrieve the documents
-    DataSource.aggregate(aggregation, function(err, results) {
-        if (err) {
-            return callback(err);
-        }
-        else {
-            identitiesService.addCreatedByAndModifiedByIdentitiesToAll(results[0].documents)
-                .then(function() {
-                    if (options.includePagination) {
-                        let derivedTotalCount = 0;
-                        if (results[0].totalCount.length > 0) {
-                            derivedTotalCount = results[0].totalCount[0].totalCount;
-                        }
-                        const returnValue = {
-                            pagination: {
-                                total: derivedTotalCount,
-                                offset: options.offset,
-                                limit: options.limit
-                            },
-                            data: results[0].documents
-                        };
-                        return callback(null, returnValue);
-                    }
-                    else {
-                        return callback(null, results[0].documents);
-                    }
-                });
+    async addExtraDataToAll(dataSources, retrieveDataComponents) {
+        for (const dataSource of dataSources) {
+            // eslint-disable-next-line no-await-in-loop
+            await addExtraData(dataSource, retrieveDataComponents);
         }
-    });
-};
-
-async function addExtraData(dataSource, retrieveDataComponents) {
-    await identitiesService.addCreatedByAndModifiedByIdentities(dataSource);
-    if (retrieveDataComponents) {
-        await addDataComponents(dataSource);
     }
-}
 
-async function addExtraDataToAll(dataSources, retrieveDataComponents) {
-    for (const dataSource of dataSources) {
-        // eslint-disable-next-line no-await-in-loop
-        await addExtraData(dataSource, retrieveDataComponents);
-    }
-}
-
-async function addDataComponents(dataSource) {
-    // We have to work with the latest version of all data components to avoid mishandling a situation
-    // where an earlier version of a data component may reference a data source, but the latest
-    // version doesn't.
-
-    // Retrieve the latest version of all data components
-    const allDataComponents = await dataComponentsService.retrieveAllAsync({ includeDeprecated: true, includeRevoked: true });
-
-    // Add the data components that reference the data source
-    dataSource.dataComponents = allDataComponents.filter(dataComponent => dataComponent.stix.x_mitre_data_source_ref === dataSource.stix.id);
-}
-
-exports.retrieveById = function(stixId, options, callback) {
-    // versions=all    Retrieve all versions of the data source with the stixId
-    // versions=latest Retrieve the data source with the latest modified date for this stixId
-
-    if (!stixId) {
-        const error = new Error(errors.missingParameter);
-        error.parameterName = 'stixId';
-        return callback(error);
-    }
-
-    if (options.versions === 'all') {
-        DataSource.find({'stix.id': stixId})
-            .lean()
-            .exec(function (err, dataSources) {
-                if (err) {
-                    if (err.name === 'CastError') {
-                        const error = new Error(errors.badlyFormattedParameter);
-                        error.parameterName = 'stixId';
-                        return callback(error);
-                    } else {
-                        return callback(err);
-                    }
-                } else {
-                    addExtraDataToAll(dataSources, options.retrieveDataComponents)
-                        .then(() => callback(null, dataSources));
-                }
-            });
-    }
-    else if (options.versions === 'latest') {
-        DataSource.findOne({ 'stix.id': stixId })
-            .sort('-stix.modified')
-            .lean()
-            .exec(function(err, dataSource) {
-                if (err) {
-                    if (err.name === 'CastError') {
-                        const error = new Error(errors.badlyFormattedParameter);
-                        error.parameterName = 'stixId';
-                        return callback(error);
-                    }
-                    else {
-                        return callback(err);
-                    }
-                }
-                else {
-                    // Note: document is null if not found
-                    if (dataSource) {
-                        addExtraData(dataSource, options.retrieveDataComponents)
-                            .then(() => callback(null, [ dataSource ]));
-                    }
-                    else {
-                        return callback(null, []);
-                    }
-                }
-            });
-    }
-    else {
-        const error = new Error(errors.invalidQueryStringParameter);
-        error.parameterName = 'versions';
-        return callback(error);
-    }
-};
-
-exports.retrieveVersionById = function(stixId, modified, options, callback) {
-    // Retrieve the version of the data source with the matching stixId and modified date
+    async addDataComponents(dataSource) {
+        // We have to work with the latest version of all data components to avoid mishandling a situation
+        // where an earlier version of a data component may reference a data source, but the latest
+        // version doesn't.
 
-    if (!stixId) {
-        const error = new Error(errors.missingParameter);
-        error.parameterName = 'stixId';
-        return callback(error);
-    }
+        // Retrieve the latest version of all data components
+        const allDataComponents = await dataComponentsService.retrieveAllAsync({ includeDeprecated: true, includeRevoked: true });
 
-    if (!modified) {
-        const error = new Error(errors.missingParameter);
-        error.parameterName = 'modified';
-        return callback(error);
+        // Add the data components that reference the data source
+        dataSource.dataComponents = allDataComponents.filter(dataComponent => dataComponent.stix.x_mitre_data_source_ref === dataSource.stix.id);
     }
 
-    DataSource.findOne({ 'stix.id': stixId, 'stix.modified': modified }, function(err, dataSource) {
-        if (err) {
-            if (err.name === 'CastError') {
-                const error = new Error(errors.badlyFormattedParameter);
-                error.parameterName = 'stixId';
-                return callback(error);
-            }
-            else {
-                return callback(err);
-            }
-        }
-        else {
-            // Note: document is null if not found
-            if (dataSource) {
-                addExtraData(dataSource, options.retrieveDataComponents)
-                    .then(() => callback(null, dataSource));
-            }
-            else {
-                return callback();
-            }
-        }
-    });
-};
-
-exports.createIsAsync = true;
-exports.create = async function(data, options) {
-    // This function handles two use cases:
-    //   1. This is a completely new object. Create a new object and generate the stix.id if not already
-    //      provided. Set both stix.created_by_ref and stix.x_mitre_modified_by_ref to the organization identity.
-    //   2. This is a new version of an existing object. Create a new object with the specified id.
-    //      Set stix.x_mitre_modified_by_ref to the organization identity.
-
-    // Create the document
-    const dataSource = new DataSource(data);
-
-    options = options || {};
-    if (!options.import) {
-        // Set the ATT&CK Spec Version
-        dataSource.stix.x_mitre_attack_spec_version = dataSource.stix.x_mitre_attack_spec_version ?? config.app.attackSpecVersion;
-
-        // Record the user account that created the object
-        if (options.userAccountId) {
-            dataSource.workspace.workflow.created_by_user_account = options.userAccountId;
-        }
-
-        // Set the default marking definitions
-        await attackObjectsService.setDefaultMarkingDefinitions(dataSource);
-
-        // Get the organization identity
-        const organizationIdentityRef = await systemConfigurationService.retrieveOrganizationIdentityRef();
-
-        // Check for an existing object
-        let existingObject;
-        if (dataSource.stix.id) {
-            existingObject = await DataSource.findOne({ 'stix.id': dataSource.stix.id });
-        }
-
-        if (existingObject) {
-            // New version of an existing object
-            // Only set the x_mitre_modified_by_ref property
-            dataSource.stix.x_mitre_modified_by_ref = organizationIdentityRef;
-        }
-        else {
-            // New object
-            // Assign a new STIX id if not already provided
-            dataSource.stix.id = dataSource.stix.id || `x-mitre-data-source--${uuid.v4()}`;
-
-            // Set the created_by_ref and x_mitre_modified_by_ref properties
-            dataSource.stix.created_by_ref = organizationIdentityRef;
-            dataSource.stix.x_mitre_modified_by_ref = organizationIdentityRef;
-        }
-    }
-
-    // Save the document in the database
-    try {
-        const savedDataSource = await dataSource.save();
-        return savedDataSource;
-    }
-    catch (err) {
-        if (err.name === 'MongoServerError' && err.code === 11000) {
-            // 11000 = Duplicate index
-            const error = new Error(errors.duplicateId);
-            throw error;
-        }
-        else {
-            throw err;
-        }
-    }
-};
-
-exports.updateFull = function(stixId, stixModified, data, callback) {
-    if (!stixId) {
-        const error = new Error(errors.missingParameter);
-        error.parameterName = 'stixId';
-        return callback(error);
-    }
-
-    if (!stixModified) {
-        const error = new Error(errors.missingParameter);
-        error.parameterName = 'modified';
-        return callback(error);
-    }
-
-    DataSource.findOne({ 'stix.id': stixId, 'stix.modified': stixModified }, function(err, document) {
-        if (err) {
-            if (err.name === 'CastError') {
-                var error = new Error(errors.badlyFormattedParameter);
-                error.parameterName = 'stixId';
-                return callback(error);
-            }
-            else {
-                return callback(err);
-            }
-        }
-        else if (!document) {
-            // document not found
-            return callback(null);
-        }
-        else {
-            // Copy data to found document and save
-            Object.assign(document, data);
-            document.save(function(err, savedDocument) {
-                if (err) {
-                    if (err.name === 'MongoServerError' && err.code === 11000) {
-                        // 11000 = Duplicate index
-                        var error = new Error(errors.duplicateId);
-                        return callback(error);
-                    }
-                    else {
-                        return callback(err);
-                    }
-                }
-                else {
-                    return callback(null, savedDocument);
-                }
-            });
-        }
-    });
-};
-
-exports.deleteVersionById = function (stixId, stixModified, callback) {
-    if (!stixId) {
-        const error = new Error(errors.missingParameter);
-        error.parameterName = 'stixId';
-        return callback(error);
-    }
-
-    if (!stixModified) {
-        const error = new Error(errors.missingParameter);
-        error.parameterName = 'modified';
-        return callback(error);
-    }
-
-    DataSource.findOneAndRemove({ 'stix.id': stixId, 'stix.modified': stixModified }, function (err, dataSource) {
-        if (err) {
-            return callback(err);
-        } else {
-            // Note: data source is null if not found
-            return callback(null, dataSource);
-        }
-    });
-};
-
-exports.deleteById = function (stixId, callback) {
-    if (!stixId) {
-        const error = new Error(errors.missingParameter);
-        error.parameterName = 'stixId';
-        return callback(error);
-    }
+}
 
-    DataSource.deleteMany({ 'stix.id': stixId }, function (err, dataSource) {
-        if (err) {
-            return callback(err);
-        } else {
-            //Note: dataSource is null if not found
-            return callback(null, dataSource);
-        }
-    });
-};
+module.exports = new DataSourcesService('x-mitre-data-source', matrixRepository);
\ No newline at end of file

From b528aae8741cdf450e5fac8f1b2da3d65d38fea6 Mon Sep 17 00:00:00 2001
From: Sun <vsun@mitre.org>
Date: Mon, 18 Dec 2023 14:49:36 -0500
Subject: [PATCH 2/9] refactoring service

---
 app/repository/data-source-repository.js | 8 ++++++++
 app/services/data-sources-service.js     | 9 ++-------
 2 files changed, 10 insertions(+), 7 deletions(-)
 create mode 100644 app/repository/data-source-repository.js

diff --git a/app/repository/data-source-repository.js b/app/repository/data-source-repository.js
new file mode 100644
index 00000000..dc83baf6
--- /dev/null
+++ b/app/repository/data-source-repository.js
@@ -0,0 +1,8 @@
+'use strict';
+
+const BaseRepository = require('./_base.repository');
+const DataSource = require('../models/data-source-model');
+
+class DataSourceRepository extends BaseRepository { }
+
+module.exports = new DataSourceRepository(DataSource);
diff --git a/app/services/data-sources-service.js b/app/services/data-sources-service.js
index d27f3750..0a5465f4 100644
--- a/app/services/data-sources-service.js
+++ b/app/services/data-sources-service.js
@@ -1,13 +1,8 @@
 'use strict';
 
-const uuid = require('uuid');
-const DataSource = require('../models/data-source-model');
-const systemConfigurationService = require('./system-configuration-service');
+const DataSourceRepository = require('../repository/data-source-repository');
 const identitiesService = require('./identities-service');
 const dataComponentsService = require('./data-components-service');
-const attackObjectsService = require('./attack-objects-service');
-const config = require('../config/config');
-const regexValidator = require('../lib/regex');
 const {lastUpdatedByQueryHelper} = require('../lib/request-parameter-helper');
 const BaseService = require('./_base.service');
 
@@ -50,4 +45,4 @@ class DataSourcesService extends BaseService {
 
 }
 
-module.exports = new DataSourcesService('x-mitre-data-source', matrixRepository);
\ No newline at end of file
+module.exports = new DataSourcesService('x-mitre-data-source', DataSourceRepository);
\ No newline at end of file

From d7850768507236eacacdb68ae0d432686661031b Mon Sep 17 00:00:00 2001
From: Sun <vsun@mitre.org>
Date: Mon, 18 Dec 2023 14:52:58 -0500
Subject: [PATCH 3/9] refactoring tests

---
 .../api/data-sources/data-sources.spec.js     | 437 +++++++-----------
 1 file changed, 161 insertions(+), 276 deletions(-)

diff --git a/app/tests/api/data-sources/data-sources.spec.js b/app/tests/api/data-sources/data-sources.spec.js
index d6f9022f..db254881 100644
--- a/app/tests/api/data-sources/data-sources.spec.js
+++ b/app/tests/api/data-sources/data-sources.spec.js
@@ -129,149 +129,110 @@ describe('Data Sources API', function () {
         passportCookie = await login.loginAnonymous(app);
     });
 
-    it('GET /api/data-sources returns an empty array of data sources', function (done) {
-        request(app)
+    it('GET /api/data-sources returns an empty array of data sources', async function () {
+        const res = await request(app)
             .get('/api/data-sources')
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(200)
-            .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    // We expect to get an empty array
-                    const dataSources = res.body;
-                    expect(dataSources).toBeDefined();
-                    expect(Array.isArray(dataSources)).toBe(true);
-                    expect(dataSources.length).toBe(0);
-                    done();
-                }
-            });
+            .expect('Content-Type', /json/);
+        // We expect to get an empty array
+        const dataSources = res.body;
+        expect(dataSources).toBeDefined();
+        expect(Array.isArray(dataSources)).toBe(true);
+        expect(dataSources.length).toBe(0);
     });
 
-    it('POST /api/data-sources does not create an empty data source', function (done) {
+    it('POST /api/data-sources does not create an empty data source', async function () {
         const body = { };
-        request(app)
+        const res = await request(app)
             .post('/api/data-sources')
             .send(body)
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
-            .expect(400)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    done();
-                }
-            });
+            .expect(400);
+
     });
 
     let dataSource1;
-    it('POST /api/data-sources creates a data source', function (done) {
+    it('POST /api/data-sources creates a data source', async function () {
         const timestamp = new Date().toISOString();
         initialDataSourceData.stix.created = timestamp;
         initialDataSourceData.stix.modified = timestamp;
         const body = initialDataSourceData;
-        request(app)
+        const res = await request(app)
             .post('/api/data-sources')
             .send(body)
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(201)
-            .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    // We expect to get the created data source
-                    dataSource1 = res.body;
-                    expect(dataSource1).toBeDefined();
-                    expect(dataSource1.stix).toBeDefined();
-                    expect(dataSource1.stix.id).toBeDefined();
-                    expect(dataSource1.stix.created).toBeDefined();
-                    expect(dataSource1.stix.modified).toBeDefined();
-                    expect(dataSource1.stix.x_mitre_attack_spec_version).toBe(config.app.attackSpecVersion);
-
-                    done();
-                }
-            });
+            .expect('Content-Type', /json/);
+
+        // We expect to get the created data source
+        dataSource1 = res.body;
+        expect(dataSource1).toBeDefined();
+        expect(dataSource1.stix).toBeDefined();
+        expect(dataSource1.stix.id).toBeDefined();
+        expect(dataSource1.stix.created).toBeDefined();
+        expect(dataSource1.stix.modified).toBeDefined();
+        expect(dataSource1.stix.x_mitre_attack_spec_version).toBe(config.app.attackSpecVersion);
+
+
     });
 
-    it('GET /api/data-sources returns the added data source', function (done) {
-        request(app)
+    it('GET /api/data-sources returns the added data source', async function () {
+        const res = await request(app)
             .get('/api/data-sources')
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(200)
-            .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    // We expect to get one data source in an array
-                    const dataSource = res.body;
-                    expect(dataSource).toBeDefined();
-                    expect(Array.isArray(dataSource)).toBe(true);
-                    expect(dataSource.length).toBe(1);
-                    done();
-                }
-            });
+            .expect('Content-Type', /json/);
+
+        // We expect to get one data source in an array
+        const dataSource = res.body;
+        expect(dataSource).toBeDefined();
+        expect(Array.isArray(dataSource)).toBe(true);
+        expect(dataSource.length).toBe(1);
+
     });
 
-    it('GET /api/data-sources/:id should not return a data source when the id cannot be found', function (done) {
-        request(app)
+    it('GET /api/data-sources/:id should not return a data source when the id cannot be found', async function () {
+        const res = await request(app)
             .get('/api/data-sources/not-an-id')
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
-            .expect(404)
-            .end(function (err, res) {
-                if (err) {
-                    done(err);
-                } else {
-                    done();
-                }
-            });
+            .expect(404);
+
     });
 
-    it('GET /api/data-sources/:id returns the added data source', function (done) {
-        request(app)
+    it('GET /api/data-sources/:id returns the added data source', async function () {
+        const res = await request(app)
             .get('/api/data-sources/' + dataSource1.stix.id)
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(200)
-            .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    // We expect to get one data source in an array
-                    const dataSources = res.body;
-                    expect(dataSources).toBeDefined();
-                    expect(Array.isArray(dataSources)).toBe(true);
-                    expect(dataSources.length).toBe(1);
-
-                    const dataSource = dataSources[0];
-                    expect(dataSource).toBeDefined();
-                    expect(dataSource.stix).toBeDefined();
-                    expect(dataSource.stix.id).toBe(dataSource1.stix.id);
-                    expect(dataSource.stix.type).toBe(dataSource1.stix.type);
-                    expect(dataSource.stix.name).toBe(dataSource1.stix.name);
-                    expect(dataSource.stix.description).toBe(dataSource1.stix.description);
-                    expect(dataSource.stix.spec_version).toBe(dataSource1.stix.spec_version);
-                    expect(dataSource.stix.object_marking_refs).toEqual(expect.arrayContaining(dataSource1.stix.object_marking_refs));
-                    expect(dataSource.stix.created_by_ref).toBe(dataSource1.stix.created_by_ref);
-                    expect(dataSource.stix.x_mitre_version).toBe(dataSource1.stix.x_mitre_version);
-                    expect(dataSource.stix.x_mitre_attack_spec_version).toBe(dataSource1.stix.x_mitre_attack_spec_version);
-
-                    done();
-                }
-            });
+            .expect('Content-Type', /json/);
+
+        // We expect to get one data source in an array
+        const dataSources = res.body;
+        expect(dataSources).toBeDefined();
+        expect(Array.isArray(dataSources)).toBe(true);
+        expect(dataSources.length).toBe(1);
+
+        const dataSource = dataSources[0];
+        expect(dataSource).toBeDefined();
+        expect(dataSource.stix).toBeDefined();
+        expect(dataSource.stix.id).toBe(dataSource1.stix.id);
+        expect(dataSource.stix.type).toBe(dataSource1.stix.type);
+        expect(dataSource.stix.name).toBe(dataSource1.stix.name);
+        expect(dataSource.stix.description).toBe(dataSource1.stix.description);
+        expect(dataSource.stix.spec_version).toBe(dataSource1.stix.spec_version);
+        expect(dataSource.stix.object_marking_refs).toEqual(expect.arrayContaining(dataSource1.stix.object_marking_refs));
+        expect(dataSource.stix.created_by_ref).toBe(dataSource1.stix.created_by_ref);
+        expect(dataSource.stix.x_mitre_version).toBe(dataSource1.stix.x_mitre_version);
+        expect(dataSource.stix.x_mitre_attack_spec_version).toBe(dataSource1.stix.x_mitre_attack_spec_version);
+
+
     });
 
     it('GET /api/data-sources/:id returns the added data source with data components', async function () {
@@ -298,54 +259,41 @@ describe('Data Sources API', function () {
         expect(dataSource.dataComponents.length).toBe(5);
     });
 
-    it('PUT /api/data-sources updates a data source', function (done) {
+    it('PUT /api/data-sources updates a data source', async function () {
         const originalModified = dataSource1.stix.modified;
         const timestamp = new Date().toISOString();
         dataSource1.stix.modified = timestamp;
         dataSource1.stix.description = 'This is an updated data source.'
         const body = dataSource1;
-        request(app)
+        const res = await request(app)
             .put('/api/data-sources/' + dataSource1.stix.id + '/modified/' + originalModified)
             .send(body)
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(200)
-            .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    // We expect to get the updated data source
-                    const dataSource = res.body;
-                    expect(dataSource).toBeDefined();
-                    expect(dataSource.stix.id).toBe(dataSource1.stix.id);
-                    expect(dataSource.stix.modified).toBe(dataSource1.stix.modified);
-                    done();
-                }
-            });
+            .expect('Content-Type', /json/);
+
+        // We expect to get the updated data source
+        const dataSource = res.body;
+        expect(dataSource).toBeDefined();
+        expect(dataSource.stix.id).toBe(dataSource1.stix.id);
+        expect(dataSource.stix.modified).toBe(dataSource1.stix.modified);
+
     });
 
-    it('POST /api/data-sources does not create a data source with the same id and modified date', function (done) {
+    it('POST /api/data-sources does not create a data source with the same id and modified date', async function () {
         const body = dataSource1;
-        request(app)
+        const res = await request(app)
             .post('/api/data-sources')
             .send(body)
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
-            .expect(409)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    done();
-                }
-            });
+            .expect(409);
+
     });
 
     let dataSource2;
-    it('POST /api/data-sources should create a new version of a data source with a duplicate stix.id but different stix.modified date', function (done) {
+    it('POST /api/data-sources should create a new version of a data source with a duplicate stix.id but different stix.modified date', async function () {
         dataSource2 = _.cloneDeep(dataSource1);
         dataSource2._id = undefined;
         dataSource2.__t = undefined;
@@ -353,121 +301,91 @@ describe('Data Sources API', function () {
         const timestamp = new Date().toISOString();
         dataSource2.stix.modified = timestamp;
         const body = dataSource2;
-        request(app)
+        const res = await request(app)
             .post('/api/data-sources')
             .send(body)
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(201)
-            .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    // We expect to get the created data source
-                    const dataSource = res.body;
-                    expect(dataSource).toBeDefined();
-                    done();
-                }
-            });
+            .expect('Content-Type', /json/);
+
+        // We expect to get the created data source
+        const dataSource = res.body;
+        expect(dataSource).toBeDefined();
+
     });
 
-    it('GET /api/data-sources returns the latest added data source', function (done) {
-        request(app)
+    it('GET /api/data-sources returns the latest added data source', async function () {
+        const res = await request(app)
             .get('/api/data-sources/' + dataSource2.stix.id)
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(200)
-            .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    // We expect to get one data source in an array
-                    const dataSources = res.body;
-                    expect(dataSources).toBeDefined();
-                    expect(Array.isArray(dataSources)).toBe(true);
-                    expect(dataSources.length).toBe(1);
-                    const dataSource = dataSources[0];
-                    expect(dataSource.stix.id).toBe(dataSource2.stix.id);
-                    expect(dataSource.stix.modified).toBe(dataSource2.stix.modified);
-                    done();
-                }
-            });
+            .expect('Content-Type', /json/);
+
+        // We expect to get one data source in an array
+        const dataSources = res.body;
+        expect(dataSources).toBeDefined();
+        expect(Array.isArray(dataSources)).toBe(true);
+        expect(dataSources.length).toBe(1);
+        const dataSource = dataSources[0];
+        expect(dataSource.stix.id).toBe(dataSource2.stix.id);
+        expect(dataSource.stix.modified).toBe(dataSource2.stix.modified);
+
     });
 
-    it('GET /api/data-sources returns all added data source', function (done) {
-        request(app)
+    it('GET /api/data-sources returns all added data source', async function () {
+        const res = await request(app)
             .get('/api/data-sources/' + dataSource1.stix.id + '?versions=all')
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(200)
-            .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    // We expect to get two data sources in an array
-                    const dataSources = res.body;
-                    expect(dataSources).toBeDefined();
-                    expect(Array.isArray(dataSources)).toBe(true);
-                    expect(dataSources.length).toBe(2);
-                    done();
-                }
-            });
+            .expect('Content-Type', /json/);
+
+        // We expect to get two data sources in an array
+        const dataSources = res.body;
+        expect(dataSources).toBeDefined();
+        expect(Array.isArray(dataSources)).toBe(true);
+        expect(dataSources.length).toBe(2);
+
     });
 
-    it('GET /api/data-sources/:id/modified/:modified returns the first added data source', function (done) {
-        request(app)
+    it('GET /api/data-sources/:id/modified/:modified returns the first added data source', async function () {
+        const res = await request(app)
             .get('/api/data-sources/' + dataSource1.stix.id + '/modified/' + dataSource1.stix.modified)
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(200)
-            .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    // We expect to get one data source in an array
-                    const dataSource = res.body;
-                    expect(dataSource).toBeDefined();
-                    expect(dataSource.stix).toBeDefined();
-                    expect(dataSource.stix.id).toBe(dataSource1.stix.id);
-                    expect(dataSource.stix.modified).toBe(dataSource1.stix.modified);
-                    done();
-                }
-            });
+            .expect('Content-Type', /json/);
+
+        // We expect to get one data source in an array
+        const dataSource = res.body;
+        expect(dataSource).toBeDefined();
+        expect(dataSource.stix).toBeDefined();
+        expect(dataSource.stix.id).toBe(dataSource1.stix.id);
+        expect(dataSource.stix.modified).toBe(dataSource1.stix.modified);
+
     });
 
-    it('GET /api/data-sources/:id/modified/:modified returns the second added data source', function (done) {
-        request(app)
+    it('GET /api/data-sources/:id/modified/:modified returns the second added data source', async function () {
+        const res = await request(app)
             .get('/api/data-sources/' + dataSource2.stix.id + '/modified/' + dataSource2.stix.modified)
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(200)
-            .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    // We expect to get one data source in an array
-                    const dataSource = res.body;
-                    expect(dataSource).toBeDefined();
-                    expect(dataSource.stix).toBeDefined();
-                    expect(dataSource.stix.id).toBe(dataSource2.stix.id);
-                    expect(dataSource.stix.modified).toBe(dataSource2.stix.modified);
-                    done();
-                }
-            });
+            .expect('Content-Type', /json/);
+
+        // We expect to get one data source in an array
+        const dataSource = res.body;
+        expect(dataSource).toBeDefined();
+        expect(dataSource.stix).toBeDefined();
+        expect(dataSource.stix.id).toBe(dataSource2.stix.id);
+        expect(dataSource.stix.modified).toBe(dataSource2.stix.modified);
+
     });
 
     let dataSource3;
-    it('POST /api/data-sources should create a new version of a data source with a duplicate stix.id but different stix.modified date', function (done) {
+    it('POST /api/data-sources should create a new version of a data source with a duplicate stix.id but different stix.modified date', async function () {
         dataSource3 = _.cloneDeep(dataSource1);
         dataSource3._id = undefined;
         dataSource3.__t = undefined;
@@ -475,91 +393,58 @@ describe('Data Sources API', function () {
         const timestamp = new Date().toISOString();
         dataSource3.stix.modified = timestamp;
         const body = dataSource3;
-        request(app)
+        const res = await request(app)
             .post('/api/data-sources')
             .send(body)
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(201)
-            .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    // We expect to get the created data source
-                    const dataSource = res.body;
-                    expect(dataSource).toBeDefined();
-                    done();
-                }
-            });
+            .expect('Content-Type', /json/);
+
+        // We expect to get the created data source
+        const dataSource = res.body;
+        expect(dataSource).toBeDefined();
+
     });
 
-    it('DELETE /api/data-sources/:id should not delete a data source when the id cannot be found', function (done) {
-        request(app)
+    it('DELETE /api/data-sources/:id should not delete a data source when the id cannot be found', async function () {
+        const res = await request(app)
             .delete('/api/data-sources/not-an-id')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
-            .expect(404)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    done();
-                }
-            });
+            .expect(404);
+
     });
 
-    it('DELETE /api/data-sources/:id/modified/:modified deletes a data source', function (done) {
-        request(app)
+    it('DELETE /api/data-sources/:id/modified/:modified deletes a data source', async function () {
+        const res = await request(app)
             .delete('/api/data-sources/' + dataSource1.stix.id + '/modified/' + dataSource1.stix.modified)
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
-            .expect(204)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    done();
-                }
-            });
+            .expect(204);
+
     });
 
-    it('DELETE /api/data-sources/:id should delete all the data sources with the same stix id', function (done) {
-        request(app)
+    it('DELETE /api/data-sources/:id should delete all the data sources with the same stix id', async function () {
+        const res = await request(app)
             .delete('/api/data-sources/' + dataSource2.stix.id)
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
-            .expect(204)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    done();
-                }
-            });
+            .expect(204);
+
     });
 
-    it('GET /api/data-sources returns an empty array of data sources', function (done) {
-        request(app)
+    it('GET /api/data-sources returns an empty array of data sources', async function () {
+        const res = await request(app)
             .get('/api/data-sources')
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(200)
-            .expect('Content-Type', /json/)
-            .end(function(err, res) {
-                if (err) {
-                    done(err);
-                }
-                else {
-                    // We expect to get an empty array
-                    const dataSources = res.body;
-                    expect(dataSources).toBeDefined();
-                    expect(Array.isArray(dataSources)).toBe(true);
-                    expect(dataSources.length).toBe(0);
-                    done();
-                }
-            });
+            .expect('Content-Type', /json/);
+
+        // We expect to get an empty array
+        const dataSources = res.body;
+        expect(dataSources).toBeDefined();
+        expect(Array.isArray(dataSources)).toBe(true);
+        expect(dataSources.length).toBe(0);
+
     });
 
     after(async function() {

From 77d4f3a4d1b9e3c9347dba53ae22ce49ae4da441 Mon Sep 17 00:00:00 2001
From: Sun <vsun@mitre.org>
Date: Mon, 18 Dec 2023 15:04:44 -0500
Subject: [PATCH 4/9] refactored controller

---
 app/controllers/data-sources-controller.js | 173 ++++++++++-----------
 1 file changed, 83 insertions(+), 90 deletions(-)

diff --git a/app/controllers/data-sources-controller.js b/app/controllers/data-sources-controller.js
index 39741812..5b1055b1 100644
--- a/app/controllers/data-sources-controller.js
+++ b/app/controllers/data-sources-controller.js
@@ -2,8 +2,10 @@
 
 const dataSourcesService = require('../services/data-sources-service');
 const logger = require('../lib/logger');
+const { DuplicateIdError, BadlyFormattedParameterError, InvalidQueryStringParameterError } = require('../exceptions');
+const { InvalidPointerError } = require('@apidevtools/json-schema-ref-parser');
 
-exports.retrieveAll = function(req, res) {
+exports.retrieveAll = async function(req, res) {
     const options = {
         offset: req.query.offset || 0,
         limit: req.query.limit || 0,
@@ -16,65 +18,68 @@ exports.retrieveAll = function(req, res) {
         lastUpdatedBy: req.query.lastUpdatedBy,
         includePagination: req.query.includePagination
     }
-
-    dataSourcesService.retrieveAll(options, function(err, results) {
-        if (err) {
-            logger.error('Failed with error: ' + err);
-            return res.status(500).send('Unable to get data sources. Server error.');
+    try {
+        const results = await dataSourcesService.retrieveAll(options);
+        if (options.includePagination) {
+            logger.debug(`Success: Retrieved ${ results.data.length } of ${ results.pagination.total } total data source(s)`);
         }
         else {
-            if (options.includePagination) {
-                logger.debug(`Success: Retrieved ${ results.data.length } of ${ results.pagination.total } total data source(s)`);
-            }
-            else {
-                logger.debug(`Success: Retrieved ${ results.length } data source(s)`);
-            }
-            return res.status(200).send(results);
+            logger.debug(`Success: Retrieved ${ results.length } data source(s)`);
+        }
+        return res.status(200).send(results);
+    } catch (err) {
+            logger.error('Failed with error: ' + err);
+            return res.status(500).send('Unable to get data sources. Server error.');
         }
-    });
+
 };
 
-exports.retrieveById = function(req, res) {
+exports.retrieveById = async function(req, res) {
     const options = {
         versions: req.query.versions || 'latest',
         retrieveDataComponents: req.query.retrieveDataComponents
     }
-
-    dataSourcesService.retrieveById(req.params.stixId, options, function (err, dataSources) {
-        if (err) {
-            if (err.message === dataSourcesService.errors.badlyFormattedParameter) {
-                logger.warn('Badly formatted stix id: ' + req.params.stixId);
-                return res.status(400).send('Stix id is badly formatted.');
-            }
-            else if (err.message === dataSourcesService.errors.invalidQueryStringParameter) {
-                logger.warn('Invalid query string: versions=' + req.query.versions);
-                return res.status(400).send('Query string parameter versions is invalid.');
-            }
-            else {
-                logger.error('Failed with error: ' + err);
-                return res.status(500).send('Unable to get data sources. Server error.');
-            }
+    try {
+        const dataSources = await dataSourcesService.retrieveById(req.params.stixId, options);
+        if (dataSources.length === 0) {
+            return res.status(404).send('Data source not found.');
         }
         else {
-            if (dataSources.length === 0) {
-                return res.status(404).send('Data source not found.');
-            }
-            else {
-                logger.debug(`Success: Retrieved ${ dataSources.length } data source(s) with id ${ req.params.stixId }`);
-                return res.status(200).send(dataSources);
-            }
+            logger.debug(`Success: Retrieved ${ dataSources.length } data source(s) with id ${ req.params.stixId }`);
+            return res.status(200).send(dataSources);
+        }
+    } catch (err) {
+        if (err instanceof BadlyFormattedParameterError) {
+            logger.warn('Badly formatted stix id: ' + req.params.stixId);
+            return res.status(400).send('Stix id is badly formatted.');
         }
-    });
+        else if (err instanceof InvalidQueryStringParameterError) {
+            logger.warn('Invalid query string: versions=' + req.query.versions);
+            return res.status(400).send('Query string parameter versions is invalid.');
+        }
+        else {
+            logger.error('Failed with error: ' + err);
+            return res.status(500).send('Unable to get data sources. Server error.');
+        }
+    }
 };
 
-exports.retrieveVersionById = function(req, res) {
+exports.retrieveVersionById = async function(req, res) {
     const options = {
         retrieveDataComponents: req.query.retrieveDataComponents
     }
 
-    dataSourcesService.retrieveVersionById(req.params.stixId, req.params.modified, options, function (err, dataSource) {
-        if (err) {
-            if (err.message === dataSourcesService.errors.badlyFormattedParameter) {
+    try {
+        const dataSource = await dataSourcesService.retrieveVersionById(req.params.stixId, req.params.modified, options);
+        if (!dataSource) {
+            return res.status(404).send('Data source not found.');
+        }
+        else {
+            logger.debug(`Success: Retrieved data source with id ${dataSource.id}`);
+            return res.status(200).send(dataSource);
+        }
+    } catch (err) {
+            if (err instanceof BadlyFormattedParameterError) {
                 logger.warn('Badly formatted stix id: ' + req.params.stixId);
                 return res.status(400).send('Stix id is badly formatted.');
             }
@@ -82,16 +87,7 @@ exports.retrieveVersionById = function(req, res) {
                 logger.error('Failed with error: ' + err);
                 return res.status(500).send('Unable to get data source. Server error.');
             }
-        } else {
-            if (!dataSource) {
-                return res.status(404).send('Data source not found.');
-            }
-            else {
-                logger.debug(`Success: Retrieved data source with id ${dataSource.id}`);
-                return res.status(200).send(dataSource);
-            }
-        }
-    });
+        } 
 };
 
 exports.create = async function(req, res) {
@@ -109,7 +105,7 @@ exports.create = async function(req, res) {
         return res.status(201).send(dataSource);
     }
     catch(err) {
-        if (err.message === dataSourcesService.errors.duplicateId) {
+        if (err instanceof DuplicateIdError) {
             logger.warn("Duplicate stix.id and stix.modified");
             return res.status(409).send('Unable to create data source. Duplicate stix.id and stix.modified properties.');
         }
@@ -120,58 +116,55 @@ exports.create = async function(req, res) {
     }
 };
 
-exports.updateFull = function(req, res) {
+exports.updateFull = async function(req, res) {
     // Get the data from the request
     const dataSourceData = req.body;
 
     // Create the data source
-    dataSourcesService.updateFull(req.params.stixId, req.params.modified, dataSourceData, function(err, dataSource) {
-        if (err) {
+    try {
+        const dataSource = await dataSourcesService.updateFull(req.params.stixId, req.params.modified, dataSourceData);
+        if (!dataSource) {
+            return res.status(404).send('Data source not found.');
+        } else {
+            logger.debug("Success: Updated data source with id " + dataSource.stix.id);
+            return res.status(200).send(dataSource);
+        }
+    } catch (err) {
             logger.error("Failed with error: " + err);
             return res.status(500).send("Unable to update data source. Server error.");
         }
-        else {
-            if (!dataSource) {
-                return res.status(404).send('Data source not found.');
-            } else {
-                logger.debug("Success: Updated data source with id " + dataSource.stix.id);
-                return res.status(200).send(dataSource);
-            }
-        }
-    });
+
 };
 
-exports.deleteVersionById = function(req, res) {
-    dataSourcesService.deleteVersionById(req.params.stixId, req.params.modified, function (err, dataSource) {
-        if (err) {
+exports.deleteVersionById = async function(req, res) {
+
+    try {
+        const dataSource = await dataSourcesService.deleteVersionById(req.params.stixId, req.params.modified);
+        if (!dataSource) {
+            return res.status(404).send('Data source not found.');
+        } else {
+            logger.debug("Success: Deleted data source with id " + dataSource.stix.id);
+            return res.status(204).end();
+        }
+    } catch (err) {
             logger.error('Delete data source failed. ' + err);
             return res.status(500).send('Unable to delete data source. Server error.');
         }
-        else {
-            if (!dataSource) {
-                return res.status(404).send('Data source not found.');
-            } else {
-                logger.debug("Success: Deleted data source with id " + dataSource.stix.id);
-                return res.status(204).end();
-            }
-        }
-    });
+
 };
 
-exports.deleteById = function(req, res) {
-    dataSourcesService.deleteById(req.params.stixId, function (err, dataSources) {
-        if (err) {
-            logger.error('Delete data source failed. ' + err);
-            return res.status(500).send('Unable to delete data source. Server error.');
+exports.deleteById = async function(req, res) {
+    try {
+        const dataSources = await dataSourcesService.deleteById(req.params.stixId);
+        if (dataSources.deletedCount === 0) {
+            return res.status(404).send('Data Sources not found.');
         }
         else {
-            if (dataSources.deletedCount === 0) {
-                return res.status(404).send('Data Sources not found.');
-            }
-            else {
-                logger.debug(`Success: Deleted data source with id ${ req.params.stixId }`);
-                return res.status(204).end();
-            }
+            logger.debug(`Success: Deleted data source with id ${ req.params.stixId }`);
+            return res.status(204).end();
+        }
+    } catch (err) {
+            logger.error('Delete data source failed. ' + err);
+            return res.status(500).send('Unable to delete data source. Server error.');
         }
-    });
 };

From 800b45db35ddb68ba4d131a4e4a481a0ccaeb806 Mon Sep 17 00:00:00 2001
From: Sun <vsun@mitre.org>
Date: Thu, 21 Dec 2023 15:03:38 -0500
Subject: [PATCH 5/9] adding this keyword to internal function calls

---
 app/services/data-sources-service.js | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/app/services/data-sources-service.js b/app/services/data-sources-service.js
index 0a5465f4..27fa923a 100644
--- a/app/services/data-sources-service.js
+++ b/app/services/data-sources-service.js
@@ -1,6 +1,6 @@
 'use strict';
 
-const DataSourceRepository = require('../repository/data-source-repository');
+const dataSourceRepository = require('../repository/data-source-repository');
 const identitiesService = require('./identities-service');
 const dataComponentsService = require('./data-components-service');
 const {lastUpdatedByQueryHelper} = require('../lib/request-parameter-helper');
@@ -20,14 +20,14 @@ class DataSourcesService extends BaseService {
     async addExtraData(dataSource, retrieveDataComponents) {
         await identitiesService.addCreatedByAndModifiedByIdentities(dataSource);
         if (retrieveDataComponents) {
-            await addDataComponents(dataSource);
+            await this.addDataComponents(dataSource);
         }
     }
 
     async addExtraDataToAll(dataSources, retrieveDataComponents) {
         for (const dataSource of dataSources) {
             // eslint-disable-next-line no-await-in-loop
-            await addExtraData(dataSource, retrieveDataComponents);
+            await this.addExtraData(dataSource, retrieveDataComponents);
         }
     }
 
@@ -45,4 +45,4 @@ class DataSourcesService extends BaseService {
 
 }
 
-module.exports = new DataSourcesService('x-mitre-data-source', DataSourceRepository);
\ No newline at end of file
+module.exports = new DataSourcesService('x-mitre-data-source', dataSourceRepository);
\ No newline at end of file

From 703cca12bc9e95d85bbe40dd2eb57d278c30dca4 Mon Sep 17 00:00:00 2001
From: Sun <vsun@mitre.org>
Date: Fri, 5 Jan 2024 12:06:14 -0500
Subject: [PATCH 6/9] trying to readd functions

---
 app/services/data-sources-service.js | 123 +++++++++++++++++++++++++--
 1 file changed, 115 insertions(+), 8 deletions(-)

diff --git a/app/services/data-sources-service.js b/app/services/data-sources-service.js
index 27fa923a..8ae7c57a 100644
--- a/app/services/data-sources-service.js
+++ b/app/services/data-sources-service.js
@@ -5,18 +5,125 @@ const identitiesService = require('./identities-service');
 const dataComponentsService = require('./data-components-service');
 const {lastUpdatedByQueryHelper} = require('../lib/request-parameter-helper');
 const BaseService = require('./_base.service');
+const { MissingParameterError, BadlyFormattedParameterError, InvalidQueryStringParameterError } = require('../exceptions');
 
-const errors = {
-    missingParameter: 'Missing required parameter',
-    badlyFormattedParameter: 'Badly formatted parameter',
-    duplicateId: 'Duplicate id',
-    notFound: 'Document not found',
-    invalidQueryStringParameter: 'Invalid query string parameter'
-};
-exports.errors = errors;
 
 class DataSourcesService extends BaseService {
 
+    errors = {
+        missingParameter: 'Missing required parameter',
+        badlyFormattedParameter: 'Badly formatted parameter',
+        duplicateId: 'Duplicate id',
+        notFound: 'Document not found',
+        invalidQueryStringParameter: 'Invalid query string parameter'
+    }
+
+    async retrieveById(stixId, options, callback) {
+        try {
+            // versions=all    Retrieve all versions of the data source with the stixId
+            // versions=latest Retrieve the data source with the latest modified date for this stixId
+    
+            if (!stixId) {
+                if (callback) {
+                    return callback(new Error(this.errors.missingParameter));
+                }
+                throw new MissingParameterError;
+            }
+    
+            if (options.versions === 'all') {
+                const dataSources = await this.repository.model.find({ 'stix.id': stixId }).lean().exec();
+                await addExtraDataToAll(dataSources, options.retrieveDataComponents);
+                if (callback) {
+                    return callback(null, dataSources);
+                }
+                return dataSources;
+            } else if (options.versions === 'latest') {
+                const dataSource = await this.repository.model.findOne({ 'stix.id': stixId }).sort('-stix.modified').lean().exec();
+    
+                // Note: document is null if not found
+                if (dataSource) {
+                    await addExtraData(dataSource, options.retrieveDataComponents);
+                    if (callback) {
+                        return callback(null, [dataSource]);
+                    }
+                    return [dataSource];
+                } else {
+                    if (callback) {
+                        return callback(null, []);
+                    }
+                    return [];
+                }
+            } else {
+                if (callback) {
+                    return callback(new Error(this.errors.invalidQueryStringParameter));
+                }
+                throw new InvalidQueryStringParameterError;
+            }
+        } catch (err) {
+            if (err.name === 'CastError') {
+                if (callback) {
+                    return callback(new Error(this.errors.badlyFormattedParameter));
+                }
+                throw new BadlyFormattedParameterError;
+            } else {
+                if (callback) {
+                    return callback(err);
+                }
+                throw err;
+            }
+        }
+    };
+    
+    
+    async retrieveVersionById(stixId, modified, options, callback) {
+        try {
+            // Retrieve the version of the data source with the matching stixId and modified date
+    
+            if (!stixId) {
+                if (callback) {
+                    return callback(new Error(this.errors.missingParameter));
+                }
+                throw new MissingParameterError;
+            }
+    
+            if (!modified) {
+                if (callback) {
+                    return callback(new Error(this.errors.missingParameter));
+                }
+                throw new MissingParameterError;
+            }
+    
+            const dataSource = await this.repository.model.findOne({ 'stix.id': stixId, 'stix.modified': modified }).exec();
+    
+            // Note: document is null if not found
+            if (dataSource) {
+                await addExtraData(dataSource, options.retrieveDataComponents);
+                if (callback) {
+                    return callback(null, dataSource);
+                }
+                return dataSource;
+            } else {
+                if (callback) {
+                    return callback(null, null);
+                }
+                return null;
+            }
+        } catch (err) {
+            if (err.name === 'CastError') {
+                if (callback) {
+                    return callback(new Error(this.errors.badlyFormattedParameter));
+                }
+                throw new BadlyFormattedParameterError;
+            } else {
+                if (callback) {
+                    return callback(err);
+                }
+                throw err;
+            }
+        }
+    };
+    
+
     async addExtraData(dataSource, retrieveDataComponents) {
         await identitiesService.addCreatedByAndModifiedByIdentities(dataSource);
         if (retrieveDataComponents) {

From 4b428cee81e77451afc2d75916ff3ba2f72e60e6 Mon Sep 17 00:00:00 2001
From: Sun <vsun@mitre.org>
Date: Tue, 9 Jan 2024 18:44:38 -0500
Subject: [PATCH 7/9] all tests pass

---
 app/controllers/data-sources-controller.js    |  1 -
 app/services/data-sources-service.js          | 64 +++++++++----------
 .../api/data-sources/data-sources.spec.js     | 12 ++--
 3 files changed, 38 insertions(+), 39 deletions(-)

diff --git a/app/controllers/data-sources-controller.js b/app/controllers/data-sources-controller.js
index 5b1055b1..a0ccce2d 100644
--- a/app/controllers/data-sources-controller.js
+++ b/app/controllers/data-sources-controller.js
@@ -3,7 +3,6 @@
 const dataSourcesService = require('../services/data-sources-service');
 const logger = require('../lib/logger');
 const { DuplicateIdError, BadlyFormattedParameterError, InvalidQueryStringParameterError } = require('../exceptions');
-const { InvalidPointerError } = require('@apidevtools/json-schema-ref-parser');
 
 exports.retrieveAll = async function(req, res) {
     const options = {
diff --git a/app/services/data-sources-service.js b/app/services/data-sources-service.js
index 8ae7c57a..5cf8724d 100644
--- a/app/services/data-sources-service.js
+++ b/app/services/data-sources-service.js
@@ -3,7 +3,6 @@
 const dataSourceRepository = require('../repository/data-source-repository');
 const identitiesService = require('./identities-service');
 const dataComponentsService = require('./data-components-service');
-const {lastUpdatedByQueryHelper} = require('../lib/request-parameter-helper');
 const BaseService = require('./_base.service');
 const { MissingParameterError, BadlyFormattedParameterError, InvalidQueryStringParameterError } = require('../exceptions');
 
@@ -18,6 +17,33 @@ class DataSourcesService extends BaseService {
         invalidQueryStringParameter: 'Invalid query string parameter'
     }
 
+
+    async addExtraData(dataSource, retrieveDataComponents) {
+        await identitiesService.addCreatedByAndModifiedByIdentities(dataSource);
+        if (retrieveDataComponents) {
+            await this.addDataComponents(dataSource);
+        }
+    }
+
+    async addExtraDataToAll(dataSources, retrieveDataComponents) {
+        for (const dataSource of dataSources) {
+            // eslint-disable-next-line no-await-in-loop
+            await this.addExtraData(dataSource, retrieveDataComponents);
+        }
+    }
+
+    async addDataComponents(dataSource) {
+        // We have to work with the latest version of all data components to avoid mishandling a situation
+        // where an earlier version of a data component may reference a data source, but the latest
+        // version doesn't.
+
+        // Retrieve the latest version of all data components
+        const allDataComponents = await dataComponentsService.retrieveAllAsync({ includeDeprecated: true, includeRevoked: true });
+
+        // Add the data components that reference the data source
+        dataSource.dataComponents = allDataComponents.filter(dataComponent => dataComponent.stix.x_mitre_data_source_ref === dataSource.stix.id);
+    }
+
     async retrieveById(stixId, options, callback) {
         try {
             // versions=all    Retrieve all versions of the data source with the stixId
@@ -32,7 +58,7 @@ class DataSourcesService extends BaseService {
     
             if (options.versions === 'all') {
                 const dataSources = await this.repository.model.find({ 'stix.id': stixId }).lean().exec();
-                await addExtraDataToAll(dataSources, options.retrieveDataComponents);
+                await this.addExtraDataToAll(dataSources, options.retrieveDataComponents);
                 if (callback) {
                     return callback(null, dataSources);
                 }
@@ -42,7 +68,7 @@ class DataSourcesService extends BaseService {
     
                 // Note: document is null if not found
                 if (dataSource) {
-                    await addExtraData(dataSource, options.retrieveDataComponents);
+                    await this.addExtraData(dataSource, options.retrieveDataComponents);
                     if (callback) {
                         return callback(null, [dataSource]);
                     }
@@ -72,7 +98,7 @@ class DataSourcesService extends BaseService {
                 throw err;
             }
         }
-    };
+    }
     
     
     async retrieveVersionById(stixId, modified, options, callback) {
@@ -97,7 +123,7 @@ class DataSourcesService extends BaseService {
     
             // Note: document is null if not found
             if (dataSource) {
-                await addExtraData(dataSource, options.retrieveDataComponents);
+                await this.addExtraData(dataSource, options.retrieveDataComponents);
                 if (callback) {
                     return callback(null, dataSource);
                 }
@@ -121,34 +147,8 @@ class DataSourcesService extends BaseService {
                 throw err;
             }
         }
-    };
-    
-
-    async addExtraData(dataSource, retrieveDataComponents) {
-        await identitiesService.addCreatedByAndModifiedByIdentities(dataSource);
-        if (retrieveDataComponents) {
-            await this.addDataComponents(dataSource);
-        }
-    }
-
-    async addExtraDataToAll(dataSources, retrieveDataComponents) {
-        for (const dataSource of dataSources) {
-            // eslint-disable-next-line no-await-in-loop
-            await this.addExtraData(dataSource, retrieveDataComponents);
-        }
-    }
-
-    async addDataComponents(dataSource) {
-        // We have to work with the latest version of all data components to avoid mishandling a situation
-        // where an earlier version of a data component may reference a data source, but the latest
-        // version doesn't.
-
-        // Retrieve the latest version of all data components
-        const allDataComponents = await dataComponentsService.retrieveAllAsync({ includeDeprecated: true, includeRevoked: true });
-
-        // Add the data components that reference the data source
-        dataSource.dataComponents = allDataComponents.filter(dataComponent => dataComponent.stix.x_mitre_data_source_ref === dataSource.stix.id);
     }
+    
 
 }
 
diff --git a/app/tests/api/data-sources/data-sources.spec.js b/app/tests/api/data-sources/data-sources.spec.js
index db254881..c7d72033 100644
--- a/app/tests/api/data-sources/data-sources.spec.js
+++ b/app/tests/api/data-sources/data-sources.spec.js
@@ -145,7 +145,7 @@ describe('Data Sources API', function () {
 
     it('POST /api/data-sources does not create an empty data source', async function () {
         const body = { };
-        const res = await request(app)
+        await request(app)
             .post('/api/data-sources')
             .send(body)
             .set('Accept', 'application/json')
@@ -197,7 +197,7 @@ describe('Data Sources API', function () {
     });
 
     it('GET /api/data-sources/:id should not return a data source when the id cannot be found', async function () {
-        const res = await request(app)
+        await request(app)
             .get('/api/data-sources/not-an-id')
             .set('Accept', 'application/json')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
@@ -283,7 +283,7 @@ describe('Data Sources API', function () {
 
     it('POST /api/data-sources does not create a data source with the same id and modified date', async function () {
         const body = dataSource1;
-        const res = await request(app)
+        await request(app)
             .post('/api/data-sources')
             .send(body)
             .set('Accept', 'application/json')
@@ -408,7 +408,7 @@ describe('Data Sources API', function () {
     });
 
     it('DELETE /api/data-sources/:id should not delete a data source when the id cannot be found', async function () {
-        const res = await request(app)
+        await request(app)
             .delete('/api/data-sources/not-an-id')
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(404);
@@ -416,7 +416,7 @@ describe('Data Sources API', function () {
     });
 
     it('DELETE /api/data-sources/:id/modified/:modified deletes a data source', async function () {
-        const res = await request(app)
+        await request(app)
             .delete('/api/data-sources/' + dataSource1.stix.id + '/modified/' + dataSource1.stix.modified)
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(204);
@@ -424,7 +424,7 @@ describe('Data Sources API', function () {
     });
 
     it('DELETE /api/data-sources/:id should delete all the data sources with the same stix id', async function () {
-        const res = await request(app)
+        await request(app)
             .delete('/api/data-sources/' + dataSource2.stix.id)
             .set('Cookie', `${ login.passportCookieName }=${ passportCookie.value }`)
             .expect(204);

From 26dfb671e4ac19103e0ab2233536ed118de36452 Mon Sep 17 00:00:00 2001
From: Sun <vsun@mitre.org>
Date: Wed, 10 Jan 2024 15:12:13 -0500
Subject: [PATCH 8/9] fixing final lint issues

---
 app/services/data-sources-service.js | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/app/services/data-sources-service.js b/app/services/data-sources-service.js
index 5cf8724d..ba91dd9d 100644
--- a/app/services/data-sources-service.js
+++ b/app/services/data-sources-service.js
@@ -18,21 +18,21 @@ class DataSourcesService extends BaseService {
     }
 
 
-    async addExtraData(dataSource, retrieveDataComponents) {
+    static async addExtraData(dataSource, retrieveDataComponents) {
         await identitiesService.addCreatedByAndModifiedByIdentities(dataSource);
         if (retrieveDataComponents) {
-            await this.addDataComponents(dataSource);
+            await DataSourcesService.addDataComponents(dataSource);
         }
     }
 
-    async addExtraDataToAll(dataSources, retrieveDataComponents) {
+    static async addExtraDataToAll(dataSources, retrieveDataComponents) {
         for (const dataSource of dataSources) {
             // eslint-disable-next-line no-await-in-loop
-            await this.addExtraData(dataSource, retrieveDataComponents);
+            await DataSourcesService.addExtraData(dataSource, retrieveDataComponents);
         }
     }
 
-    async addDataComponents(dataSource) {
+    static async addDataComponents(dataSource) {
         // We have to work with the latest version of all data components to avoid mishandling a situation
         // where an earlier version of a data component may reference a data source, but the latest
         // version doesn't.
@@ -58,7 +58,7 @@ class DataSourcesService extends BaseService {
     
             if (options.versions === 'all') {
                 const dataSources = await this.repository.model.find({ 'stix.id': stixId }).lean().exec();
-                await this.addExtraDataToAll(dataSources, options.retrieveDataComponents);
+                await DataSourcesService.addExtraDataToAll(dataSources, options.retrieveDataComponents);
                 if (callback) {
                     return callback(null, dataSources);
                 }
@@ -68,7 +68,7 @@ class DataSourcesService extends BaseService {
     
                 // Note: document is null if not found
                 if (dataSource) {
-                    await this.addExtraData(dataSource, options.retrieveDataComponents);
+                    await DataSourcesService.addExtraData(dataSource, options.retrieveDataComponents);
                     if (callback) {
                         return callback(null, [dataSource]);
                     }
@@ -123,7 +123,7 @@ class DataSourcesService extends BaseService {
     
             // Note: document is null if not found
             if (dataSource) {
-                await this.addExtraData(dataSource, options.retrieveDataComponents);
+                await DataSourcesService.addExtraData(dataSource, options.retrieveDataComponents);
                 if (callback) {
                     return callback(null, dataSource);
                 }

From b4491ba8687c90742cf47cbe73dfb0fc0928b69b Mon Sep 17 00:00:00 2001
From: Jack Sheriff <>
Date: Fri, 12 Jan 2024 11:44:51 -0500
Subject: [PATCH 9/9] Make data sources repository plural to match other
 repositories.

---
 ...ource-repository.js => data-sources-repository.js} |  4 ++--
 app/services/data-sources-service.js                  | 11 +++--------
 2 files changed, 5 insertions(+), 10 deletions(-)
 rename app/repository/{data-source-repository.js => data-sources-repository.js} (53%)

diff --git a/app/repository/data-source-repository.js b/app/repository/data-sources-repository.js
similarity index 53%
rename from app/repository/data-source-repository.js
rename to app/repository/data-sources-repository.js
index dc83baf6..f44865bf 100644
--- a/app/repository/data-source-repository.js
+++ b/app/repository/data-sources-repository.js
@@ -3,6 +3,6 @@
 const BaseRepository = require('./_base.repository');
 const DataSource = require('../models/data-source-model');
 
-class DataSourceRepository extends BaseRepository { }
+class DataSourcesRepository extends BaseRepository { }
 
-module.exports = new DataSourceRepository(DataSource);
+module.exports = new DataSourcesRepository(DataSource);
diff --git a/app/services/data-sources-service.js b/app/services/data-sources-service.js
index ba91dd9d..b1b203be 100644
--- a/app/services/data-sources-service.js
+++ b/app/services/data-sources-service.js
@@ -1,12 +1,11 @@
 'use strict';
 
-const dataSourceRepository = require('../repository/data-source-repository');
+const dataSourcesRepository = require('../repository/data-sources-repository');
 const identitiesService = require('./identities-service');
 const dataComponentsService = require('./data-components-service');
 const BaseService = require('./_base.service');
 const { MissingParameterError, BadlyFormattedParameterError, InvalidQueryStringParameterError } = require('../exceptions');
 
-
 class DataSourcesService extends BaseService {
 
     errors = {
@@ -17,7 +16,6 @@ class DataSourcesService extends BaseService {
         invalidQueryStringParameter: 'Invalid query string parameter'
     }
 
-
     static async addExtraData(dataSource, retrieveDataComponents) {
         await identitiesService.addCreatedByAndModifiedByIdentities(dataSource);
         if (retrieveDataComponents) {
@@ -99,8 +97,7 @@ class DataSourcesService extends BaseService {
             }
         }
     }
-    
-    
+
     async retrieveVersionById(stixId, modified, options, callback) {
         try {
             // Retrieve the version of the data source with the matching stixId and modified date
@@ -148,8 +145,6 @@ class DataSourcesService extends BaseService {
             }
         }
     }
-    
-
 }
 
-module.exports = new DataSourcesService('x-mitre-data-source', dataSourceRepository);
\ No newline at end of file
+module.exports = new DataSourcesService('x-mitre-data-source', dataSourcesRepository);
\ No newline at end of file