create cli-pull-ot command to sync assessments from OneTrust to disk

README.md

@@ -172,6 +172,7 @@ yarn add -D @transcend-io/cli
 
 # cli commands available within package
 yarn tr-pull --auth=$TRANSCEND_API_KEY
+yarn tr-pull-ot --auth=$ONE_TRUST_OAUTH_TOKEN --hostname=$ONE_TRUST_HOSTNAME --file=$ONE_TRUST_OUTPUT_FILE
 yarn tr-push --auth=$TRANSCEND_API_KEY
 yarn tr-scan-packages --auth=$TRANSCEND_API_KEY
 yarn tr-discover-silos --auth=$TRANSCEND_API_KEY
@@ -212,6 +213,7 @@ npm i -D @transcend-io/cli
 
 # cli commands available within package
 tr-pull --auth=$TRANSCEND_API_KEY
+tr-pull-ot --auth=$ONE_TRUST_OAUTH_TOKEN --hostname=$ONE_TRUST_HOSTNAME --file=$ONE_TRUST_OUTPUT_FILE
 tr-push --auth=$TRANSCEND_API_KEY
 tr-scan-packages --auth=$TRANSCEND_API_KEY
 tr-discover-silos --auth=$TRANSCEND_API_KEY
@@ -571,6 +573,43 @@ tr-pull --auth=./transcend-api-keys.json --resources=consentManager --file=./tra
 
 Note: This command will overwrite the existing transcend.yml file that you have locally.
 
+### tr-pull-ot
+
+Pulls resources from a OneTrust instance. For now, it only supports retrieving OneTrust Assessments. It sends a request to the [Get List of Assessments](https://developer.onetrust.com/onetrust/reference/getallassessmentbasicdetailsusingget) endpoint to fetch a list of all Assessments in your account. Then, it queries the [Get Assessment](https://developer.onetrust.com/onetrust/reference/exportassessmentusingget) and [Get Risk](https://developer.onetrust.com/onetrust/reference/getriskusingget) endpoints to enrich these assessments with more details such as respondents, approvers, assessment questions and responses, and assessment risks. Finally, it syncs the enriched resources to disk in the specified file and format.
+
+This command can be helpful if you are looking to:
+
+- Pull resources from your OneTrust account.
+- Migrate your resources from your OneTrust account to Transcend.
+
+#### Authentication
+
+In order to use this command, you will need to generate a OneTrust OAuth Token with scope for accessing the following endpoints:
+
+- [GET /v2/assessments](https://developer.onetrust.com/onetrust/reference/getallassessmentbasicdetailsusingget)
+- [GET /v2/assessments/{assessmentId}/export](https://developer.onetrust.com/onetrust/reference/exportassessmentusingget)
+- [GET /risks/{riskId}](https://developer.onetrust.com/onetrust/reference/getriskusingget)
+
+To learn how to generate the token, see the [OAuth 2.0 Scopes](https://developer.onetrust.com/onetrust/reference/oauth-20-scopes) and [Generate Access Token](https://developer.onetrust.com/onetrust/reference/getoauthtoken) pages.
+
+#### Arguments
+
+| Argument   | Description                                                                                       | Type    | Default     | Required |
+| ---------- | ------------------------------------------------------------------------------------------------- | ------- | ----------- | -------- |
+| auth       | The OAuth access token with the scopes necessary to access the OneTrust Public APIs.              | string  | N/A         | true     |
+| hostname   | The domain of the OneTrust environment from which to pull the resource (e.g. trial.onetrust.com). | string  | N/A         | true     |
+| file       | Path to the file to pull the resource into. Its format must match the fileFormat argument.        | string  | N/A         | true     |
+| fileFormat | The format of the output file. For now, only json is supported.                                   | string  | json        | false    |
+| resource   | The resource to pull from OneTrust. For now, only assessments is supported.                       | string  | assessments | false    |
+| debug      | Whether to print detailed logs in case of error.                                                  | boolean | false       | false    |
+
+#### Usage
+
+```sh
+# Writes out file to ./oneTrustAssessments.json
+tr-pull-ot --auth=$ONE_TRUST_OAUTH_TOKEN --hostname=trial.onetrust.com --file=./oneTrustAssessments.json
+```
+
 ### tr-push
 
 Given a transcend.yml file, sync the contents up to your connected services view (https://app.transcend.io/privacy-requests/connected-services).

package.json

@@ -2,7 +2,7 @@
   "author": "Transcend Inc.",
   "name": "@transcend-io/cli",
   "description": "Small package containing useful typescript utilities.",
-  "version": "6.12.1",
+  "version": "6.13.0",
   "homepage": "https://github.com/transcend-io/cli",
   "repository": {
     "type": "git",
@@ -28,6 +28,7 @@
     "tr-pull-consent-metrics": "./build/cli-pull-consent-metrics.js",
     "tr-pull-consent-preferences": "./build/cli-pull-consent-preferences.js",
     "tr-pull-datapoints": "./build/cli-pull-datapoints.js",
+    "tr-pull-ot": "./build/cli-pull-ot.js",
     "tr-push": "./build/cli-push.js",
     "tr-request-approve": "./build/cli-request-approve.js",
     "tr-request-cancel": "./build/cli-request-cancel.js",

src/cli-pull-ot.ts

@@ -0,0 +1,76 @@
+#!/usr/bin/env node
+import { logger } from './logger';
+import colors from 'colors';
+import {
+  getListOfAssessments,
+  getAssessment,
+  writeOneTrustAssessment,
+  parseCliPullOtArguments,
+  createOneTrustGotInstance,
+} from './oneTrust';
+import { OneTrustPullResource } from './enums';
+import { mapSeries } from 'bluebird';
+
+/**
+ * Pull configuration from OneTrust down locally to disk
+ *
+ * Dev Usage:
+ * yarn ts-node ./src/cli-pull-ot.ts --hostname=customer.my.onetrust.com --auth=$ONE_TRUST_OAUTH_TOKEN --file=./oneTrustAssessment.json
+ *
+ * Standard usage
+ * yarn cli-pull-ot --hostname=customer.my.onetrust.com --auth=$ONE_TRUST_OAUTH_TOKEN --file=./oneTrustAssessment.json
+ */
+async function main(): Promise<void> {
+  const { file, fileFormat, hostname, auth, resource, debug } =
+    parseCliPullOtArguments();
+
+  try {
+    if (resource === OneTrustPullResource.Assessments) {
+      // use the hostname and auth token to instantiate a client to talk to OneTrust
+      const oneTrust = createOneTrustGotInstance({ hostname, auth });
+
+      // fetch the list of all assessments in the OneTrust organization
+      const assessments = await getListOfAssessments({ oneTrust });
+
+      // fetch details about one assessment at a time and sync to disk right away to avoid running out of memory
+      await mapSeries(assessments, async (assessment, index) => {
+        logger.info(
+          `Fetching details about assessment ${index + 1} of ${
+            assessments.length
+          }...`,
+        );
+        const assessmentDetails = await getAssessment({
+          oneTrust,
+          assessmentId: assessment.assessmentId,
+        });
+
+        writeOneTrustAssessment({
+          assessment,
+          assessmentDetails,
+          index,
+          total: assessments.length,
+          file,
+          fileFormat,
+        });
+      });
+    }
+  } catch (err) {
+    logger.error(
+      colors.red(
+        `An error occurred pulling the resource ${resource} from OneTrust: ${
+          debug ? err.stack : err.message
+        }`,
+      ),
+    );
+    process.exit(1);
+  }
+
+  // Indicate success
+  logger.info(
+    colors.green(
+      `Successfully synced OneTrust ${resource} to disk at "${file}"!`,
+    ),
+  );
+}
+
+main();

src/enums.ts

@@ -1,5 +1,18 @@
 import { makeEnum } from '@transcend-io/type-utils';
 
+/** Accepted file formats for exporting resources from OneTrust */
+export enum OneTrustFileFormat {
+  Json = 'json',
+  Csv = 'csv',
+}
+
+/**
+ * Resources that can be pulled in from OneTrust
+ */
+export enum OneTrustPullResource {
+  Assessments = 'assessments',
+}
+
 /**
  * Resources that can be pulled in
  */
@@ -50,7 +63,7 @@ export const PathfinderPolicyName = makeEnum({
  * Type override
  */
 export type PathfinderPolicyName =
-  typeof PathfinderPolicyName[keyof typeof PathfinderPolicyName];
+  (typeof PathfinderPolicyName)[keyof typeof PathfinderPolicyName];
 
 /**
  * The names of the OpenAI routes that we support setting policies for
@@ -76,4 +89,4 @@ export const OpenAIRouteName = makeEnum({
  * Type override
  */
 export type OpenAIRouteName =
-  typeof OpenAIRouteName[keyof typeof OpenAIRouteName];
+  (typeof OpenAIRouteName)[keyof typeof OpenAIRouteName];

src/oneTrust/createOneTrustGotInstance.ts

@@ -0,0 +1,25 @@
+import got, { Got } from 'got';
+
+/**
+ * Instantiate an instance of got that is capable of making requests to OneTrust
+ *
+ * @param param - information about the OneTrust URL
+ * @returns The instance of got that is capable of making requests to the customer ingress
+ */
+export const createOneTrustGotInstance = ({
+  hostname,
+  auth,
+}: {
+  /** Hostname of the OneTrust API */
+  hostname: string;
+  /** The OAuth access token */
+  auth: string;
+}): Got =>
+  got.extend({
+    prefixUrl: `https://${hostname}`,
+    headers: {
+      accept: 'application/json',
+      'content-type': 'application/json',
+      authorization: `Bearer ${auth}`,
+    },
+  });

src/oneTrust/getAssessment.ts

@@ -0,0 +1,24 @@
+import { Got } from 'got';
+import { OneTrustGetAssessmentResponse } from './types';
+
+/**
+ * Retrieve details about a particular assessment.
+ *
+ * @param param - the information about the OneTrust client and assessment to retrieve
+ * @returns details about the assessment
+ */
+export const getAssessment = async ({
+  oneTrust,
+  assessmentId,
+}: {
+  /** The OneTrust client instance */
+  oneTrust: Got;
+  /** The ID of the assessment to retrieve */
+  assessmentId: string;
+}): Promise<OneTrustGetAssessmentResponse> => {
+  const { body } = await oneTrust.get(
+    `api/assessment/v2/assessments/${assessmentId}/export?ExcludeSkippedQuestions=false`,
+  );
+
+  return JSON.parse(body) as OneTrustGetAssessmentResponse;
+};

src/oneTrust/getListOfAssessments.ts

@@ -0,0 +1,49 @@
+import { Got } from 'got';
+import { logger } from '../logger';
+import {
+  OneTrustAssessment,
+  OneTrustGetListOfAssessmentsResponse,
+} from './types';
+
+/**
+ * Fetch a list of all assessments from the OneTrust client.
+ *
+ * @param param - the information about the OneTrust client
+ * @returns a list of OneTrustAssessment
+ */
+export const getListOfAssessments = async ({
+  oneTrust,
+}: {
+  /** The OneTrust client instance */
+  oneTrust: Got;
+}): Promise<OneTrustAssessment[]> => {
+  let currentPage = 0;
+  let totalPages = 1;
+  let totalElements = 0;
+
+  const allAssessments: OneTrustAssessment[] = [];
+
+  logger.info('Getting list of all assessments from OneTrust...');
+  while (currentPage < totalPages) {
+    // eslint-disable-next-line no-await-in-loop
+    const { body } = await oneTrust.get(
+      `api/assessment/v2/assessments?page=${currentPage}&size=2000`,
+    );
+    const { page, content } = JSON.parse(
+      body,
+    ) as OneTrustGetListOfAssessmentsResponse;
+    allAssessments.push(...(content ?? []));
+    if (currentPage === 0) {
+      totalPages = page?.totalPages ?? 0;
+      totalElements = page?.totalElements ?? 0;
+    }
+    currentPage += 1;
+
+    // log progress
+    logger.info(
+      `Fetched ${allAssessments.length} of ${totalElements} assessments.`,
+    );
+  }
+
+  return allAssessments;
+};

src/oneTrust/index.ts

@@ -0,0 +1,5 @@
+export * from './getListOfAssessments';
+export * from './createOneTrustGotInstance';
+export * from './getAssessment';
+export * from './writeOneTrustAssessment';
+export * from './parseCliPullOtArguments';