diff --git a/package-lock.json b/package-lock.json index c886d8a2..07f31ab2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@bcgsc-pori/graphkb-loader", - "version": "8.0.0", + "version": "8.0.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@bcgsc-pori/graphkb-loader", - "version": "8.0.0", + "version": "8.0.1", "license": "GPL-3", "dependencies": { "@bcgsc-pori/graphkb-parser": "^1.1.1", diff --git a/package.json b/package.json index be57fea4..42db1765 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@bcgsc-pori/graphkb-loader", "main": "src/index.js", - "version": "8.0.0", + "version": "8.0.1", "repository": { "type": "git", "url": "https://github.com/bcgsc/pori_graphkb_loader.git" diff --git a/src/civic/evidenceItems.graphql b/src/civic/evidenceItems.graphql index 1c6158cf..2d98ddbd 100644 --- a/src/civic/evidenceItems.graphql +++ b/src/civic/evidenceItems.graphql @@ -72,15 +72,21 @@ query evidenceItems( id name parsedName { - ... on Gene { entrezId } + __typename ... on MolecularProfileTextSegment { text } ... on Variant { id } } rawName variants { - gene { - entrezId - name + feature { + featureInstance { + __typename + ... on Factor { id } + ... on Gene { + entrezId + name + } + } } id name diff --git a/src/civic/index.js b/src/civic/index.js index bf78bef1..26151043 100644 --- a/src/civic/index.js +++ b/src/civic/index.js @@ -26,8 +26,6 @@ const { EvidenceItem: evidenceSpec } = require('./specs.json'); class NotImplementedError extends ErrorMixin { } -const ajv = new Ajv(); - const BASE_URL = 'https://civicdb.org/api/graphql'; /** @@ -50,6 +48,8 @@ const VOCAB = { const EVIDENCE_LEVEL_CACHE = {}; // avoid unecessary requests by caching the evidence levels +// Spec compiler +const ajv = new Ajv(); const validateEvidenceSpec = ajv.compile(evidenceSpec); @@ -213,11 +213,28 @@ const processEvidenceRecord = async (opt) => { conn, rawRecord, sources, variantsCache, oneToOne = false, } = opt; - const [level, relevance, [feature]] = await Promise.all([ + // Relevance & EvidenceLevel + const [level, relevance] = await Promise.all([ getEvidenceLevel(opt), getRelevance(opt), - _entrezGene.fetchAndLoadByIds(conn, [rawRecord.variant.gene.entrezId]), ]); + + // Variant's Feature + let feature; + const civicFeature = rawRecord.variant.feature.featureInstance; + + if (civicFeature.__typename === 'Gene') { + [feature] = await _entrezGene.fetchAndLoadByIds(conn, [civicFeature.entrezId]); + } else if (civicFeature.__typename === 'Factor') { + // TODO: Deal with __typename === 'Factor' + // No actual case as April 22nd, 2024 + throw new NotImplementedError( + 'unable to process variant\'s feature of type Factor', + ); + } + + + // Variant let variants; if (variantsCache.records[rawRecord.variant.id]) { @@ -236,7 +253,6 @@ const processEvidenceRecord = async (opt) => { } } - // get the disease by doid let disease; @@ -382,6 +398,7 @@ const processEvidenceRecord = async (opt) => { // update the existing record return conn.updateRecord('Statement', rid(original), content); } + // create a new record return conn.addRecord({ content, @@ -486,6 +503,7 @@ const downloadEvidenceRecords = async (url, trustedCurators) => { } records.push(record); } + logger.info(`${records.length}/${evidenceItems.length} evidenceItem records successfully validated with the specs`); return { counts, errorList, records }; }; @@ -604,7 +622,7 @@ const upload = async ({ record.conditions = Mp.process().conditions; } catch (err) { logger.error(`evidence (${record.id}) ${err}`); - counts.skip += 1; + counts.skip++; continue; } diff --git a/src/civic/profile.js b/src/civic/profile.js index 00222b1d..2530a99c 100644 --- a/src/civic/profile.js +++ b/src/civic/profile.js @@ -226,8 +226,8 @@ const MolecularProfile = (molecularProfile) => ({ `unable to process molecular profile with NOT operator (${this.profile.id || ''})`, ); } - // Filters out unwanted gene's info from expression - const filteredParsedName = parsedName.filter(el => !el.entrezId); + // Filters out unwanted Feature info from expression + const filteredParsedName = parsedName.filter(el => el.__typename !== 'Feature'); // Parse expression into conditions this.conditions = this._parse(filteredParsedName); diff --git a/src/civic/specs.json b/src/civic/specs.json index 2a087ac1..338a1072 100644 --- a/src/civic/specs.json +++ b/src/civic/specs.json @@ -84,32 +84,21 @@ }, "parsedName": { "items": { - "anyOf": [ - { - "properties": { - "entrezId": { - "type": "number" - } - }, - "type": "object" + "properties": { + "__typename": { + "type": "string" }, - { - "properties": { - "id": { - "type": "number" - } - }, - "type": "object" + "id": { + "type": "number" }, - { - "properties": { - "text": { - "type": "string" - } - }, - "type": "object" + "text": { + "type": "string" } - ] + }, + "required":[ + "__typename" + ], + "type": "object" }, "type": "array" }, @@ -122,39 +111,45 @@ "variants": { "items": { "properties": { - "gene": { + "feature": { "properties": { - "entrezId": { - "type": [ - "null", - "number" - ] - }, - "name": { - "type": [ - "null", - "string" - ] + "featureInstance": { + "properties": { + "__typename": { + "type": "string" + }, + "entrezId": { + "type": "number" + }, + "name": { + "type": "string" + } + }, + "required":[ + "__typename", + "entrezId", + "name" + ], + "type": "object" } }, - "type": [ - "null", - "object" - ] + "required": [ + "featureInstance" + ], + "type": "object" }, "id": { - "type": [ - "null", - "number" - ] + "type": "number" }, "name": { - "type": [ - "null", - "string" - ] + "type": "string" } }, + "required": [ + "feature", + "id", + "name" + ], "type": [ "null", "object" @@ -166,10 +161,14 @@ ] } }, - "type": [ - "null", - "object" - ] + "required": [ + "id", + "name", + "parsedName", + "rawName", + "variants" + ], + "type": "object" }, "phenotypes": { "items": { @@ -317,9 +316,22 @@ ] } }, - "type": [ - "null", - "object" - ] + "required":[ + "description", + "disease", + "evidenceDirection", + "evidenceLevel", + "evidenceRating", + "evidenceType", + "id", + "molecularProfile", + "phenotypes", + "significance", + "source", + "status", + "therapies", + "therapyInteractionType" + ], + "type": "object" } } diff --git a/src/civic/variant.js b/src/civic/variant.js index 9c92b398..5550069c 100644 --- a/src/civic/variant.js +++ b/src/civic/variant.js @@ -1,15 +1,11 @@ const kbParser = require('@bcgsc-pori/graphkb-parser'); - -const { error: { ParsingError } } = kbParser; -const { - rid, -} = require('../graphkb'); +const { rid } = require('../graphkb'); const _entrezGene = require('../entrez/gene'); const _snp = require('../entrez/snp'); +const { civic: SOURCE_DEFN } = require('../sources'); -const { - civic: SOURCE_DEFN, -} = require('../sources'); +const { error: { ErrorMixin, ParsingError } } = kbParser; +class NotImplementedError extends ErrorMixin { } // based on discussion with cam here: https://www.bcgsc.ca/jira/browse/KBDEV-844 @@ -229,8 +225,8 @@ const normalizeVariantRecord = ({ }; /** - * Given some normalized variant record from CIViC load into graphkb, create links and - * return the record + * Given some normalized variant record from CIViC, + * load into graphkb, create links and return the record * * @param {ApiConnection} conn the connection to GraphKB * @param {Object} normalizedVariant the normalized variant record @@ -328,11 +324,30 @@ const uploadNormalizedVariant = async (conn, normalizedVariant, feature) => { /** * Given some variant record and a feature, process the variant and return a GraphKB equivalent + * + * @param {ApiConnection} conn the connection to GraphKB + * @param {Object} civicVariantRecord the raw variant record from CIViC + * @param {Object} feature the gene feature already grabbed from GraphKB */ const processVariantRecord = async (conn, civicVariantRecord, feature) => { + const featureInstance = civicVariantRecord.feature.featureInstance; + let entrezId, + entrezName; + + if (featureInstance.__typename === 'Gene') { + entrezId = featureInstance.entrezId; + entrezName = featureInstance.name; + } else if (featureInstance.__typename === 'Factor') { + // TODO: Deal with __typename === 'Factor' + // No actual case as April 22nd, 2024 + throw new NotImplementedError( + 'unable to process variant\'s feature of type Factor', + ); + } + const variants = normalizeVariantRecord({ - entrezId: civicVariantRecord.gene.entrezId, - entrezName: civicVariantRecord.gene.name, + entrezId, + entrezName, name: civicVariantRecord.name, }); diff --git a/src/clinicaltrialsgov/index.js b/src/clinicaltrialsgov/index.js index 381a2e64..099271de 100644 --- a/src/clinicaltrialsgov/index.js +++ b/src/clinicaltrialsgov/index.js @@ -301,7 +301,8 @@ const formatDate = (date) => `${date.getFullYear()}-${date.getMonth() + 1}-${dat const upload = async ({ conn, maxRecords, days }) => { const source = await conn.addSource(SOURCE_DEFN); - let options = {}; + let options, + optionsWithToken; if (days) { const startDate = new Date(Date.now() - days * 24 * 60 * 60 * 1000); @@ -320,7 +321,9 @@ const upload = async ({ conn, maxRecords, days }) => { while (next) { if (nextToken) { - options = { pageToken: nextToken, ...options }; + optionsWithToken = { pageToken: nextToken, ...options }; + } else { + optionsWithToken = options; } const trials = await requestWithRetry({ json: true, @@ -331,7 +334,7 @@ const upload = async ({ conn, maxRecords, days }) => { pageSize: 1000, 'query.cond': 'cancer', sort: 'LastUpdatePostDate', - ...options, + ...optionsWithToken, }, uri: BASE_URL, }); diff --git a/test/civic.profile.test.js b/test/civic.profile.test.js index 9702e98b..5f70ac31 100644 --- a/test/civic.profile.test.js +++ b/test/civic.profile.test.js @@ -70,12 +70,12 @@ describe('MolecularProfile._end()', () => { describe('MolecularProfile._not()', () => { test('check for presence of NOT operator in expression', () => { expect(MolecularProfile()._not([ - { entrezId: 9 }, { id: 1 }, { text: 'AND' }, { text: 'NOT' }, { text: '(' }, - { entrezId: 9 }, { id: 2 }, { text: 'OR' }, { entrezId: 9 }, { id: 3 }, { text: ')' }, + { __typename: 'Feature' }, { id: 1 }, { text: 'AND' }, { text: 'NOT' }, { text: '(' }, + { __typename: 'Feature' }, { id: 2 }, { text: 'OR' }, { __typename: 'Feature' }, { id: 3 }, { text: ')' }, ])).toBe(true); expect(MolecularProfile()._not([ - { entrezId: 9 }, { id: 1 }, { text: 'AND' }, { text: '(' }, { entrezId: 9 }, - { id: 2 }, { text: 'OR' }, { entrezId: 9 }, { id: 3 }, { text: ')' }, + { __typename: 'Feature' }, { id: 1 }, { text: 'AND' }, { text: '(' }, { __typename: 'Feature' }, + { id: 2 }, { text: 'OR' }, { __typename: 'Feature' }, { id: 3 }, { text: ')' }, ])).toBe(false); }); }); @@ -187,7 +187,7 @@ describe('MolecularProfile._variants()', () => { describe('MolecularProfile.process()', () => { test('gene infos not interfering', () => { expect(MolecularProfile({ - parsedName: [{ entrezId: 9 }, { id: 1 }], + parsedName: [{ __typename: 'Feature' }, { id: 1 }], variants: [{ id: 1, name: 'a1' }], }).process().conditions).toEqual([[{ id: 1, name: 'a1' }]]); }); @@ -215,8 +215,8 @@ describe('MolecularProfile.process()', () => { const molecularProfile = { id: 1, parsedName: [ - { entrezId: 9 }, { id: 1 }, { text: 'AND' }, { text: 'NOT' }, { text: '(' }, - { entrezId: 9 }, { id: 2 }, { text: 'OR' }, { entrezId: 9 }, { id: 3 }, { text: ')' }, + { __typename: 'Feature' }, { id: 1 }, { text: 'AND' }, { text: 'NOT' }, { text: '(' }, + { __typename: 'Feature' }, { id: 2 }, { text: 'OR' }, { __typename: 'Feature' }, { id: 3 }, { text: ')' }, ], }; expect(() => MolecularProfile(molecularProfile).process()).toThrow(