Skip to content

Commit

Permalink
improve codecs and create flattenOneTrustQuestions helper
Browse files Browse the repository at this point in the history
  • Loading branch information
@abrantesarthur
abrantesarthur committed Jan 10, 2025
1 parent 5d9def9 commit 4822fba
Show file tree
Hide file tree
Showing 3 changed files with 168 additions and 129 deletions.
3 changes: 0 additions & 3 deletions src/cli-pull-ot.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,6 @@ async function main(): Promise<void> {
// fetch the list of all assessments in the OneTrust organization
const assessments = await getListOfOneTrustAssessments({ oneTrust });

// TODO: undo
// const theAssessments = assessments.slice(1902);

// fetch details about one assessment at a time and sync to disk right away to avoid running out of memory
await mapSeries(assessments, async (assessment, index) => {
logger.info(
Expand Down
155 changes: 90 additions & 65 deletions src/oneTrust/codecs.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ export type OneTrustAssessmentQuestionRiskCodec = t.TypeOf<
typeof OneTrustAssessmentQuestionRiskCodec
>;

export const OneTrustAssessmentQuestionResponsesCodec = t.type({
export const OneTrustAssessmentQuestionResponseCodec = t.type({
/** The responses */
responses: t.array(
t.type({
Expand Down Expand Up @@ -219,73 +219,80 @@ export const OneTrustAssessmentQuestionResponsesCodec = t.type({
});

/** Type override */
export type OneTrustAssessmentQuestionResponsesCodec = t.TypeOf<
typeof OneTrustAssessmentQuestionResponsesCodec
export type OneTrustAssessmentQuestionResponseCodec = t.TypeOf<
typeof OneTrustAssessmentQuestionResponseCodec
>;

export const OneTrustAssessmentNestedQuestionCodec = t.type({
/** ID of the question. */
id: t.string,
/** ID of the root version of the question. */
rootVersionId: t.string,
/** Order in which the question appears in the assessment. */
sequence: t.number,
/** Type of question in the assessment. */
questionType: t.union([
t.literal('TEXTBOX'),
t.literal('MULTICHOICE'),
t.literal('YESNO'),
t.literal('DATE'),
t.literal('STATEMENT'),
t.literal('INVENTORY'),
t.literal('ATTRIBUTE'),
t.literal('PERSONAL_DATA'),
t.literal('ENGAGEMENT'),
t.literal('ASSESS_CONTROL'),
t.null,
]),
/** Indicates whether a response to the question is required. */
required: t.boolean,
/** Data element attributes that are directly updated by the question. */
attributes: t.string,
/** Short, descriptive name for the question. */
friendlyName: t.union([t.string, t.null]),
/** Description of the question. */
description: t.union([t.string, t.null]),
/** Tooltip text within a hint for the question. */
hint: t.union([t.string, t.null]),
/** ID of the parent question. */
parentQuestionId: t.union([t.string, t.null]),
/** Indicates whether the response to the question is prepopulated. */
prePopulateResponse: t.boolean,
/** Indicates whether the assessment is linked to inventory records. */
linkAssessmentToInventory: t.boolean,
/** The question options */
options: t.union([t.array(OneTrustAssessmentQuestionOptionCodec), t.null]),
/** Indicates whether the question is valid. */
valid: t.boolean,
/** Type of question in the assessment. */
type: t.union([
t.literal('TEXTBOX'),
t.literal('MULTICHOICE'),
t.literal('YESNO'),
t.literal('DATE'),
t.literal('STATEMENT'),
t.literal('INVENTORY'),
t.literal('ATTRIBUTE'),
t.literal('PERSONAL_DATA'),
t.literal('ENGAGEMENT'),
t.literal('ASSESS_CONTROL'),
]),
/** Whether the response can be multi select */
allowMultiSelect: t.boolean,
/** The text of a question. */
content: t.string,
/** Indicates whether justification comments are required for the question. */
requireJustification: t.boolean,
});

/** Type override */
export type OneTrustAssessmentNestedQuestionCodec = t.TypeOf<
typeof OneTrustAssessmentNestedQuestionCodec
>;

export const OneTrustAssessmentQuestionCodec = t.type({
/** The question */
question: t.type({
/** ID of the question. */
id: t.string,
/** ID of the root version of the question. */
rootVersionId: t.string,
/** Order in which the question appears in the assessment. */
sequence: t.number,
/** Type of question in the assessment. */
questionType: t.union([
t.literal('TEXTBOX'),
t.literal('MULTICHOICE'),
t.literal('YESNO'),
t.literal('DATE'),
t.literal('STATEMENT'),
t.literal('INVENTORY'),
t.literal('ATTRIBUTE'),
t.literal('PERSONAL_DATA'),
t.literal('ENGAGEMENT'),
t.literal('ASSESS_CONTROL'),
t.null,
]),
/** Indicates whether a response to the question is required. */
required: t.boolean,
/** Data element attributes that are directly updated by the question. */
attributes: t.string,
/** Short, descriptive name for the question. */
friendlyName: t.union([t.string, t.null]),
/** Description of the question. */
description: t.union([t.string, t.null]),
/** Tooltip text within a hint for the question. */
hint: t.union([t.string, t.null]),
/** ID of the parent question. */
parentQuestionId: t.union([t.string, t.null]),
/** Indicates whether the response to the question is prepopulated. */
prePopulateResponse: t.boolean,
/** Indicates whether the assessment is linked to inventory records. */
linkAssessmentToInventory: t.boolean,
/** The question options */
options: t.union([t.array(OneTrustAssessmentQuestionOptionCodec), t.null]),
/** Indicates whether the question is valid. */
valid: t.boolean,
/** Type of question in the assessment. */
type: t.union([
t.literal('TEXTBOX'),
t.literal('MULTICHOICE'),
t.literal('YESNO'),
t.literal('DATE'),
t.literal('STATEMENT'),
t.literal('INVENTORY'),
t.literal('ATTRIBUTE'),
t.literal('PERSONAL_DATA'),
t.literal('ENGAGEMENT'),
t.literal('ASSESS_CONTROL'),
]),
/** Whether the response can be multi select */
allowMultiSelect: t.boolean,
/** The text of a question. */
content: t.string,
/** Indicates whether justification comments are required for the question. */
requireJustification: t.boolean,
}),
question: OneTrustAssessmentNestedQuestionCodec,
/** Indicates whether the question is hidden on the assessment. */
hidden: t.boolean,
/** Reason for locking the question in the assessment. */
Expand All @@ -299,7 +306,7 @@ export const OneTrustAssessmentQuestionCodec = t.type({
/** Indicates whether navigation rules are enabled for the question. */
hasNavigationRules: t.boolean,
/** The responses to this question */
questionResponses: t.array(OneTrustAssessmentQuestionResponsesCodec),
questionResponses: t.array(OneTrustAssessmentQuestionResponseCodec),
/** The risks associated with this question */
risks: t.union([t.array(OneTrustAssessmentQuestionRiskCodec), t.null]),
/** List of IDs associated with the question root requests. */
Expand All @@ -315,6 +322,24 @@ export type OneTrustAssessmentQuestionCodec = t.TypeOf<
typeof OneTrustAssessmentQuestionCodec
>;

// TODO: do not add to privacy types
// The OneTrustAssessmentQuestionCodec without nested properties
export const OneTrustAssessmentQuestionFlatCodec = t.type({
hidden: OneTrustAssessmentQuestionCodec.props.hidden,
lockReason: OneTrustAssessmentQuestionCodec.props.lockReason,
copyErrors: OneTrustAssessmentQuestionCodec.props.copyErrors,
hasNavigationRules: OneTrustAssessmentQuestionCodec.props.hasNavigationRules,
rootRequestInformationIds:
OneTrustAssessmentQuestionCodec.props.rootRequestInformationIds,
totalAttachments: OneTrustAssessmentQuestionCodec.props.totalAttachments,
attachmentIds: OneTrustAssessmentQuestionCodec.props.attachmentIds,
});

/** Type override */
export type OneTrustAssessmentQuestionFlatCodec = t.TypeOf<
typeof OneTrustAssessmentQuestionFlatCodec
>;

export const OneTrustAssessmentSectionHeaderRiskStatisticsCodec = t.union([
t.type({
/** Maximum level of risk in the section. */
Expand Down
139 changes: 78 additions & 61 deletions src/oneTrust/flattenOneTrustAssessment.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,11 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import {
OneTrustAssessmentCodec,
OneTrustAssessmentNestedQuestionCodec,
OneTrustAssessmentQuestionCodec,
// OneTrustAssessmentQuestionResponsesCodec,
OneTrustAssessmentQuestionFlatCodec,
OneTrustAssessmentQuestionResponseCodec,
OneTrustAssessmentQuestionRiskCodec,
OneTrustAssessmentSectionCodec,
OneTrustAssessmentSectionFlatHeaderCodec,
OneTrustAssessmentSectionHeaderCodec,
Expand Down Expand Up @@ -45,6 +48,7 @@ const flattenList = (list: any[], prefix: string): any => {
const flattenedList = list.map((obj) => flattenObject(obj, prefix));

// get all possible keys from the flattenedList
// TODO: make helper
const allKeys = Array.from(
new Set(flattenedList.flatMap((a) => Object.keys(a))),
);
Expand All @@ -58,7 +62,7 @@ const flattenList = (list: any[], prefix: string): any => {
};

// const flattenOneTrustQuestionResponses = (
// questionResponses: OneTrustAssessmentQuestionResponsesCodec[],
// questionResponses: OneTrustAssessmentQuestionResponseCodec[],
// prefix: string,
// ): any => {
// if (questionResponses.length === 0) {
Expand All @@ -73,60 +77,74 @@ const flattenList = (list: any[], prefix: string): any => {
// };
// };

// const flattenOneTrustQuestion = (
// oneTrustQuestion: OneTrustAssessmentQuestionCodec,
// prefix: string,
// ): any => {
// const {
// question: { options: questionOptions, ...restQuestion },
// questionResponses,
// // risks,
// ...rest
// } = oneTrustQuestion;
// const newPrefix = `${prefix}_${restQuestion.sequence}`;

// return {
// ...flattenObject({ ...restQuestion, ...rest }, newPrefix),
// ...flattenList(questionOptions ?? [], `${newPrefix}_options`),
// ...flattenOneTrustQuestionResponses(
// questionResponses ?? [],
// `${newPrefix}_responses`,
// ),
// };
// };
const flattenOneTrustQuestions = (
allSectionQuestions: OneTrustAssessmentQuestionCodec[][],
prefix: string,
): any => {
// each entry of sectionQuestions is the list of questions of one section
const allSectionQuestionsFlat = allSectionQuestions.map(
(sectionQuestions) => {
// extract nested properties (TODO: try to make a helper for this!!!)
const {
// TODO: flatten the questions, allQuestionResponses, and risks too!
// questions,
// allQuestionResponses,
// allRisks,
unnestedSectionQuestions,
} = sectionQuestions.reduce<{
/** The nested questions */
questions: OneTrustAssessmentNestedQuestionCodec[];
/** The responses of all questions in the section */
allQuestionResponses: OneTrustAssessmentQuestionResponseCodec[][];
/** The risks of all questions in the section */
allRisks: OneTrustAssessmentQuestionRiskCodec[][];
/** The parent questions without nested questions */
unnestedSectionQuestions: OneTrustAssessmentQuestionFlatCodec[];
}>(
(acc, sectionQuestion) => {
const { question, questionResponses, risks, ...rest } =
sectionQuestion;
return {
questions: [...acc.questions, question],
allQuestionResponses: [
...acc.allQuestionResponses,
questionResponses,
],
allRisks: [...acc.allRisks, risks ?? []],
unnestedSectionQuestions: [...acc.unnestedSectionQuestions, rest],
};
},
{
questions: [],
allQuestionResponses: [],
allRisks: [],
unnestedSectionQuestions: [],
},
);

// const flattenOneTrustQuestions = (
// questions: OneTrustAssessmentQuestionCodec[],
// prefix: string,
// ): any =>
// questions.reduce(
// (acc, question) => ({
// ...acc,
// ...flattenOneTrustQuestion(question, prefix),
// }),
// {},
// );

// const flattenOneTrustSection = (
// section: OneTrustAssessmentSectionCodec,
// ): any => {
// const { questions, header, ...rest } = section;
return flattenList(unnestedSectionQuestions, prefix);
},
);

// // the flattened section key has format like sections_${sequence}_sectionId
// const prefix = `sections_${section.sequence}`;
// return {
// ...flattenObject({ ...header, ...rest }, prefix),
// ...flattenOneTrustQuestions(questions, `${prefix}_questions`),
// };
// };
// extract all keys across allSectionQuestionsFlat
const allKeys = Array.from(
new Set(allSectionQuestionsFlat.flatMap((a) => Object.keys(a))),
);

// const flattenOneTrustSections = (
// sections: OneTrustAssessmentSectionCodec[],
// ): any =>
// sections.reduce(
// (acc, section) => ({ ...acc, ...flattenOneTrustSection(section) }),
// {},
// );
// TODO: comment
return allSectionQuestionsFlat.reduce(
(acc, flatSectionQuestions) =>
Object.fromEntries(
allKeys.map((key) => [
key,
`${acc[key] === undefined ? '' : `${acc[key]},`}[${
flatSectionQuestions[key] ?? ''
}]`,
]),
),
{},
);
};

const flattenOneTrustSectionHeaders = (
headers: OneTrustAssessmentSectionHeaderCodec[],
Expand Down Expand Up @@ -172,11 +190,7 @@ const flattenOneTrustSections = (
sections: OneTrustAssessmentSectionCodec[],
prefix: string,
): any => {
const {
// allQuestions,
headers,
unnestedSections,
} = sections.reduce<{
const { allQuestions, headers, unnestedSections } = sections.reduce<{
/** The sections questions */
allQuestions: OneTrustAssessmentQuestionCodec[][];
/** The sections headers */
Expand All @@ -200,8 +214,12 @@ const flattenOneTrustSections = (
);
const flattenedSections = flattenList(unnestedSections, prefix);
const flattenedHeaders = flattenOneTrustSectionHeaders(headers, prefix);
const flattenedQuestions = flattenOneTrustQuestions(
allQuestions,
`${prefix}_questions`,
);

return { ...flattenedSections, ...flattenedHeaders };
return { ...flattenedSections, ...flattenedHeaders, ...flattenedQuestions };
};

export const flattenOneTrustAssessment = ({
Expand All @@ -227,7 +245,7 @@ export const flattenOneTrustAssessment = ({
// approvers,
// primaryEntityDetails,
// respondents,
// eslint-disable-next-line @typescript-eslint/no-unused-vars
// eslintdisablenextline @typescripteslint/nounusedvars
respondent,
sections,
...rest
Expand All @@ -243,7 +261,6 @@ export const flattenOneTrustAssessment = ({
...flattenOneTrustSections(sections, 'sections'),
};
};

/**
*
*
Expand Down

0 comments on commit 4822fba

Please sign in to comment.