From 0cf8d6ffe6277874fa58bc6b03fab25c0c16ad9f Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Wed, 27 Jul 2022 11:09:17 +0000 Subject: [PATCH 01/13] fix scaling --- taxonium_web_client/src/utils/processNewick.js | 2 +- .../src/utils/processNextstrain.js | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/taxonium_web_client/src/utils/processNewick.js b/taxonium_web_client/src/utils/processNewick.js index b1598ece..8dac78c2 100644 --- a/taxonium_web_client/src/utils/processNewick.js +++ b/taxonium_web_client/src/utils/processNewick.js @@ -186,7 +186,7 @@ export async function processNewick(data, sendStatusMessage) { rootId: 0, overwrite_config: { num_tips: total_tips }, }; - console.log(JSON.stringify(output)); + return output; } diff --git a/taxonium_web_client/src/utils/processNextstrain.js b/taxonium_web_client/src/utils/processNextstrain.js index 265677c8..c8d48bd1 100644 --- a/taxonium_web_client/src/utils/processNextstrain.js +++ b/taxonium_web_client/src/utils/processNextstrain.js @@ -169,6 +169,7 @@ async function processJsTree(tree, data, sendStatusMessage) { tree.node.forEach((node) => { node.d = node.pre_x_dist; }); + kn_calxy(tree, true); // kn_calxy sets x -> move x to x_dist tree.node.forEach((node) => { @@ -220,10 +221,12 @@ async function processJsTree(tree, data, sendStatusMessage) { rootId: 0, overwrite_config: { num_tips: total_tips }, }; + return output; } function json_preorder(root) { + let n_tips = 0; const parents = {}; parents[root.name] = null; const path = []; @@ -273,15 +276,17 @@ function json_preorder(root) { parents[childJson.name] = parsedNode; stack.push(childJson); } + } else { + n_tips += 1; } } - return [path, parents]; + return { path, parents, n_tips }; } async function json_to_tree(json) { const rootJson = json.tree; - const [preorder, parents] = json_preorder(rootJson); - let n_tips = 0; + const { path: preorder, parents, n_tips } = json_preorder(rootJson); + const nodes = []; let root; for (const node of preorder) { @@ -303,13 +308,15 @@ async function json_to_tree(json) { nodes.push(node); } - return { + const jstree_format = { // tree in jstree.js format node: nodes, error: 0, n_tips: n_tips, root: root, }; + + return jstree_format; } export async function processNextstrain(data, sendStatusMessage) { From 53bd1c335cbbfeb4b54737cdbb2a92e8aced6473 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Wed, 27 Jul 2022 11:42:44 +0000 Subject: [PATCH 02/13] Capture NS metadata --- .../src/utils/processNextstrain.js | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/taxonium_web_client/src/utils/processNextstrain.js b/taxonium_web_client/src/utils/processNextstrain.js index c8d48bd1..6628f2c3 100644 --- a/taxonium_web_client/src/utils/processNextstrain.js +++ b/taxonium_web_client/src/utils/processNextstrain.js @@ -129,7 +129,7 @@ async function cleanup(tree) { }); } -async function processJsTree(tree, data, sendStatusMessage) { +async function processJsTree(tree, data, config, sendStatusMessage) { function assignNumTips(node) { if (node.child.length === 0) { node.num_tips = 1; @@ -219,7 +219,7 @@ async function processJsTree(tree, data, sendStatusMessage) { node_to_mut: {}, rootMutations: [], rootId: 0, - overwrite_config: { num_tips: total_tips }, + overwrite_config: { ...config, num_tips: total_tips }, }; return output; @@ -308,7 +308,7 @@ async function json_to_tree(json) { nodes.push(node); } - const jstree_format = { + const jsTree = { // tree in jstree.js format node: nodes, error: 0, @@ -316,7 +316,22 @@ async function json_to_tree(json) { root: root, }; - return jstree_format; + const config = {}; + console.log("META", json.meta); + config.title = json.meta.title; + console.log("META PROV", json.meta.data_provenance); + config.source = + json.meta.data_provenance.map((source) => source.name).join(" & ") + + " on " + + json.meta.updated + + " in a build maintained by " + + json.meta.maintainers.map((source) => source.name).join(" & "); + config.overlay = `

This is a tree generated from a NextStrain JSON file, being visualised in Taxonium.

.`; + if (json.meta.build_url) { + config.overlay += `

The NextStrain build is available here.

`; + } + + return { jsTree, config }; } export async function processNextstrain(data, sendStatusMessage) { @@ -333,9 +348,9 @@ export async function processNextstrain(data, sendStatusMessage) { const input_string = the_data; - const jsTree = await json_to_tree(JSON.parse(input_string)); + const { jsTree, config } = await json_to_tree(JSON.parse(input_string)); - const output = await processJsTree(jsTree, data, sendStatusMessage); + const output = await processJsTree(jsTree, data, config, sendStatusMessage); return output; } From dba540a7294ad272cd95a43c4d9006e9a79c2af4 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Wed, 27 Jul 2022 12:39:27 +0000 Subject: [PATCH 03/13] Process nuc mutations (not displaying in panel for some reason) --- .../src/utils/processNextstrain.js | 78 +++++++++++++++++-- 1 file changed, 71 insertions(+), 7 deletions(-) diff --git a/taxonium_web_client/src/utils/processNextstrain.js b/taxonium_web_client/src/utils/processNextstrain.js index 6628f2c3..aa161182 100644 --- a/taxonium_web_client/src/utils/processNextstrain.js +++ b/taxonium_web_client/src/utils/processNextstrain.js @@ -5,6 +5,50 @@ import { kn_expand_node, kn_calxy } from "./jstree"; const emptyList = []; +const nodeMutationsFromNextStrainToTaxonium = ( + mutations, + unique_mutations, + mutation_lookup +) => { + //console.log("mutations", mutations); + const keys = Object.keys(mutations); + const nuc_muts = mutations["nuc"] ? mutations["nuc"] : []; + + const genes = keys.filter((key) => key !== "nuc"); + const taxonium_muts = []; + nuc_muts.forEach((nuc_mut) => { + // input format is like "C123T", we want to break this into old_residue, position and new_residue + // use regex to match the position + const position = nuc_mut.match(/\d+/g); + const index_of_position = nuc_mut.indexOf(position[0]); + const previous_residue = nuc_mut.substring(0, index_of_position); + const new_residue = nuc_mut.substring( + index_of_position + position[0].length + ); + const tax_format = { + type: "nt", + gene: "nt", + previous_residue, + new_residue, + residue_pos: parseInt(position[0]), + }; + const jsonned = JSON.stringify(tax_format); + //console.log("jsonned", jsonned); + if (mutation_lookup[jsonned]) { + taxonium_muts.push(mutation_lookup[jsonned]); + } else { + unique_mutations.push({ + ...tax_format, + mutation_id: unique_mutations.length, + }); + const this_index = unique_mutations.length - 1; + mutation_lookup[jsonned] = this_index; + taxonium_muts.push(this_index); + } + }); + return taxonium_muts; +}; + async function do_fetch(url, sendStatusMessage, whatIsBeingDownloaded) { if (!sendStatusMessage) { sendStatusMessage = () => {}; @@ -81,7 +125,7 @@ async function cleanup(tree) { const cleaned = { name: node.name.replace(/'/g, ""), parent_id: node.parent ? node.parent.node_id : node.node_id, - mutations: emptyList, + mutations: node.mutations ? node.mutations : emptyList, y: node.y, num_tips: node.num_tips, is_tip: node.child.length === 0, @@ -231,6 +275,10 @@ function json_preorder(root) { parents[root.name] = null; const path = []; const stack = [root]; + + const unique_mutations = []; + + const mutation_lookup = {}; while (stack.length > 0) { const nodeJson = stack.pop(); let div = null; @@ -241,7 +289,7 @@ function json_preorder(root) { if (nodeJson.node_attrs.num_date) { time = nodeJson.node_attrs.num_date.value; } - + //console.log(nodeJson); // this is the node format for downstream processing const parsedNode = { name: nodeJson.name, @@ -249,6 +297,14 @@ function json_preorder(root) { meta: "", hl: false, hidden: false, + mutations: + nodeJson.branch_attrs && nodeJson.branch_attrs.mutations + ? nodeMutationsFromNextStrainToTaxonium( + nodeJson.branch_attrs.mutations, + unique_mutations, + mutation_lookup + ) + : [], }; // assign distance @@ -280,12 +336,17 @@ function json_preorder(root) { n_tips += 1; } } - return { path, parents, n_tips }; + return { path, parents, n_tips, unique_mutations }; } async function json_to_tree(json) { const rootJson = json.tree; - const { path: preorder, parents, n_tips } = json_preorder(rootJson); + const { + path: preorder, + parents, + n_tips, + unique_mutations, + } = json_preorder(rootJson); const nodes = []; let root; @@ -317,6 +378,7 @@ async function json_to_tree(json) { }; const config = {}; + console.log("META", json.meta); config.title = json.meta.title; console.log("META PROV", json.meta.data_provenance); @@ -331,7 +393,7 @@ async function json_to_tree(json) { config.overlay += `

The NextStrain build is available here.

`; } - return { jsTree, config }; + return { jsTree, config, unique_mutations }; } export async function processNextstrain(data, sendStatusMessage) { @@ -348,9 +410,11 @@ export async function processNextstrain(data, sendStatusMessage) { const input_string = the_data; - const { jsTree, config } = await json_to_tree(JSON.parse(input_string)); + const { jsTree, config, unique_mutations } = await json_to_tree( + JSON.parse(input_string) + ); const output = await processJsTree(jsTree, data, config, sendStatusMessage); - return output; + return { ...output, mutations: unique_mutations }; } From 302d3385d26e64027fae68326e64372fe85fceb0 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Wed, 27 Jul 2022 12:48:46 +0000 Subject: [PATCH 04/13] Fix missing panel muts --- taxonium_web_client/src/utils/processNextstrain.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/taxonium_web_client/src/utils/processNextstrain.js b/taxonium_web_client/src/utils/processNextstrain.js index aa161182..77609cff 100644 --- a/taxonium_web_client/src/utils/processNextstrain.js +++ b/taxonium_web_client/src/utils/processNextstrain.js @@ -415,6 +415,7 @@ export async function processNextstrain(data, sendStatusMessage) { ); const output = await processJsTree(jsTree, data, config, sendStatusMessage); + const node_to_mut = output.nodes.map((x) => x.mutations); - return { ...output, mutations: unique_mutations }; + return { ...output, mutations: unique_mutations, node_to_mut: node_to_mut }; } From e4dd6473c1cf3be8c7b01e49316df23b171b0689 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Wed, 27 Jul 2022 12:52:43 +0000 Subject: [PATCH 05/13] Process AA muts too --- .../src/utils/processNextstrain.js | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/taxonium_web_client/src/utils/processNextstrain.js b/taxonium_web_client/src/utils/processNextstrain.js index 77609cff..7dc57d36 100644 --- a/taxonium_web_client/src/utils/processNextstrain.js +++ b/taxonium_web_client/src/utils/processNextstrain.js @@ -46,6 +46,41 @@ const nodeMutationsFromNextStrainToTaxonium = ( taxonium_muts.push(this_index); } }); + + genes.forEach((gene) => { + const gene_muts = mutations[gene]; + gene_muts.forEach((gene_mut) => { + // input format is like "Q123F", we want to break this into old_residue, position and new_residue + // use regex to match the position + const position = gene_mut.match(/\d+/g); + const index_of_position = gene_mut.indexOf(position[0]); + const previous_residue = gene_mut.substring(0, index_of_position); + const new_residue = gene_mut.substring( + index_of_position + position[0].length + ); + const tax_format = { + type: "aa", + gene, + previous_residue, + new_residue, + residue_pos: parseInt(position[0]), + }; + const jsonned = JSON.stringify(tax_format); + //console.log("jsonned", jsonned); + if (mutation_lookup[jsonned]) { + taxonium_muts.push(mutation_lookup[jsonned]); + } else { + unique_mutations.push({ + ...tax_format, + mutation_id: unique_mutations.length, + }); + const this_index = unique_mutations.length - 1; + mutation_lookup[jsonned] = this_index; + taxonium_muts.push(this_index); + } + }); + }); + return taxonium_muts; }; From 6eccfa8637941aa760364d1c3d84f1f5dbddd40d Mon Sep 17 00:00:00 2001 From: Alex Kramer Date: Wed, 27 Jul 2022 14:03:22 -0700 Subject: [PATCH 06/13] fix small bug in setting xtype --- taxonium_web_client/src/Taxonium.jsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taxonium_web_client/src/Taxonium.jsx b/taxonium_web_client/src/Taxonium.jsx index b0d86f62..f5e2f5c3 100644 --- a/taxonium_web_client/src/Taxonium.jsx +++ b/taxonium_web_client/src/Taxonium.jsx @@ -71,9 +71,9 @@ function Taxonium({ // This can happen with e.g. nextstrain json if (data.base_data && data.base_data.nodes) { const n = data.base_data.nodes[0]; - if (!n.x_dist) { + if (!n.hasOwnProperty("x_dist")) { setxType("x_time"); - } else if (!n.x_time) { + } else if (!n.hasOwnProperty("x_time")) { setxType("x_dist"); } } From 1f29fc3e3007e1b8a95a0639400530cce5d4bbc0 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Thu, 28 Jul 2022 15:20:21 +0100 Subject: [PATCH 07/13] fix ladderize --- taxonium_web_client/src/hooks/useInputHelper.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taxonium_web_client/src/hooks/useInputHelper.js b/taxonium_web_client/src/hooks/useInputHelper.js index 796208d9..de9c7c4b 100644 --- a/taxonium_web_client/src/hooks/useInputHelper.js +++ b/taxonium_web_client/src/hooks/useInputHelper.js @@ -149,7 +149,7 @@ export const useInputHelper = ({ const tree_file = inputs.find((input) => input.filetype === "nwk"); const newQuery = { treeUrl: tree_file.name, - ladderizeTree: tree_file.ladderize === "true", + ladderizeTree: tree_file.ladderize === true, }; if (meta_file) { newQuery.metaUrl = meta_file.name; From 6c3863211e836de30d4fb1877a5e75ce5b5a6430 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Thu, 28 Jul 2022 16:49:29 +0100 Subject: [PATCH 08/13] Remove debug logging --- taxonium_web_client/src/utils/processNewick.js | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/taxonium_web_client/src/utils/processNewick.js b/taxonium_web_client/src/utils/processNewick.js index 8dac78c2..0a30e75f 100644 --- a/taxonium_web_client/src/utils/processNewick.js +++ b/taxonium_web_client/src/utils/processNewick.js @@ -98,7 +98,7 @@ async function cleanup(tree) { const scale_x = 450 / ref_x; - console.log(scale_y, "scale_y"); + tree.node.forEach((node) => { node.x_dist = node.x_dist * scale_x; node.y = node.y * scale_y; @@ -106,7 +106,7 @@ async function cleanup(tree) { } export async function processNewick(data, sendStatusMessage) { - console.log("got data", data); + let the_data; the_data = await fetch_or_extract(data, sendStatusMessage, "tree"); @@ -139,10 +139,10 @@ export async function processNewick(data, sendStatusMessage) { } assignNumTips(tree.root); const total_tips = tree.root.num_tips; - console.log("tree.root.num_tips", tree.root.num_tips); + if (data.ladderize) { - console.log("ladderizing"); + sortWithNumTips(tree.root); tree.node = kn_expand_node(tree.root); @@ -198,8 +198,6 @@ export async function processMetadataFile(data, sendStatusMessage) { the_data = await fetch_or_extract(data, logStatusToConsole, "metadata"); - console.log("Got metadata file"); - const lines = the_data.split("\n"); const output = {}; let separator; @@ -275,6 +273,6 @@ export async function processNewickAndMetadata(data, sendStatusMessage) { Object.assign(node, blanks); } }); - console.log(tree); + return tree; } From a2a0efb61bfdad2154b54ddee23df570beab36dc Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Thu, 28 Jul 2022 16:51:00 +0100 Subject: [PATCH 09/13] Update localBackendWorker.js --- taxonium_web_client/src/webworkers/localBackendWorker.js | 1 - 1 file changed, 1 deletion(-) diff --git a/taxonium_web_client/src/webworkers/localBackendWorker.js b/taxonium_web_client/src/webworkers/localBackendWorker.js index 3880f0a5..14c5393c 100644 --- a/taxonium_web_client/src/webworkers/localBackendWorker.js +++ b/taxonium_web_client/src/webworkers/localBackendWorker.js @@ -332,7 +332,6 @@ onmessage = async (event) => { //Process uploaded data: console.log("Worker onmessage"); const { data } = event; - console.log(data, "data"); if ( data.type === "upload" && data.data && From 0582496accb55c4c1a7ed8e9ad04870725e37943 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Thu, 28 Jul 2022 16:52:29 +0100 Subject: [PATCH 10/13] Update processNextstrain.js --- taxonium_web_client/src/utils/processNextstrain.js | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/taxonium_web_client/src/utils/processNextstrain.js b/taxonium_web_client/src/utils/processNextstrain.js index 7dc57d36..299a8ac6 100644 --- a/taxonium_web_client/src/utils/processNextstrain.js +++ b/taxonium_web_client/src/utils/processNextstrain.js @@ -433,20 +433,15 @@ async function json_to_tree(json) { export async function processNextstrain(data, sendStatusMessage) { console.log("got data", data); - let the_data; - - the_data = await fetch_or_extract(data, sendStatusMessage, "tree"); - - console.log("the_data", the_data); + const the_data = await fetch_or_extract(data, sendStatusMessage, "tree"); sendStatusMessage({ message: "Parsing NS file", }); - const input_string = the_data; const { jsTree, config, unique_mutations } = await json_to_tree( - JSON.parse(input_string) + JSON.parse(the_data) ); const output = await processJsTree(jsTree, data, config, sendStatusMessage); From ac10547853864a4f2fe2d935e8bce828ec91da55 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Thu, 28 Jul 2022 16:53:12 +0100 Subject: [PATCH 11/13] Update localBackendWorker.js --- taxonium_web_client/src/webworkers/localBackendWorker.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taxonium_web_client/src/webworkers/localBackendWorker.js b/taxonium_web_client/src/webworkers/localBackendWorker.js index 14c5393c..17eb2779 100644 --- a/taxonium_web_client/src/webworkers/localBackendWorker.js +++ b/taxonium_web_client/src/webworkers/localBackendWorker.js @@ -358,7 +358,7 @@ onmessage = async (event) => { data.data.filename && data.data.filetype === "nextstrain" ) { - console.log("got nextstrain file", data.data); + processedUploadedData = await processNextstrain( data.data, From 504c35ae3a9cffff7ccc9af804eea5a47ad068a5 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Thu, 28 Jul 2022 16:54:05 +0100 Subject: [PATCH 12/13] Update processNextstrain.js --- taxonium_web_client/src/utils/processNextstrain.js | 1 - 1 file changed, 1 deletion(-) diff --git a/taxonium_web_client/src/utils/processNextstrain.js b/taxonium_web_client/src/utils/processNextstrain.js index 299a8ac6..3e7ae160 100644 --- a/taxonium_web_client/src/utils/processNextstrain.js +++ b/taxonium_web_client/src/utils/processNextstrain.js @@ -432,7 +432,6 @@ async function json_to_tree(json) { } export async function processNextstrain(data, sendStatusMessage) { - console.log("got data", data); const the_data = await fetch_or_extract(data, sendStatusMessage, "tree"); sendStatusMessage({ From 50e8778c3d41d9aa750c737243c23e31a6af0942 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 28 Jul 2022 16:28:42 +0000 Subject: [PATCH 13/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .all-contributorsrc | 2 +- taxonium_web_client/src/utils/processNewick.js | 5 ----- taxonium_web_client/src/utils/processNextstrain.js | 1 - taxonium_web_client/src/webworkers/localBackendWorker.js | 2 -- 4 files changed, 1 insertion(+), 9 deletions(-) diff --git a/.all-contributorsrc b/.all-contributorsrc index 7a3ba708..f600380d 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -77,7 +77,7 @@ "infra" ] } - + ], "contributorsPerLine": 7, "projectName": "taxonium", diff --git a/taxonium_web_client/src/utils/processNewick.js b/taxonium_web_client/src/utils/processNewick.js index 0a30e75f..227291b9 100644 --- a/taxonium_web_client/src/utils/processNewick.js +++ b/taxonium_web_client/src/utils/processNewick.js @@ -98,7 +98,6 @@ async function cleanup(tree) { const scale_x = 450 / ref_x; - tree.node.forEach((node) => { node.x_dist = node.x_dist * scale_x; node.y = node.y * scale_y; @@ -106,7 +105,6 @@ async function cleanup(tree) { } export async function processNewick(data, sendStatusMessage) { - let the_data; the_data = await fetch_or_extract(data, sendStatusMessage, "tree"); @@ -140,10 +138,7 @@ export async function processNewick(data, sendStatusMessage) { assignNumTips(tree.root); const total_tips = tree.root.num_tips; - if (data.ladderize) { - - sortWithNumTips(tree.root); tree.node = kn_expand_node(tree.root); } diff --git a/taxonium_web_client/src/utils/processNextstrain.js b/taxonium_web_client/src/utils/processNextstrain.js index 3e7ae160..6434cd3c 100644 --- a/taxonium_web_client/src/utils/processNextstrain.js +++ b/taxonium_web_client/src/utils/processNextstrain.js @@ -438,7 +438,6 @@ export async function processNextstrain(data, sendStatusMessage) { message: "Parsing NS file", }); - const { jsTree, config, unique_mutations } = await json_to_tree( JSON.parse(the_data) ); diff --git a/taxonium_web_client/src/webworkers/localBackendWorker.js b/taxonium_web_client/src/webworkers/localBackendWorker.js index 17eb2779..dfa734db 100644 --- a/taxonium_web_client/src/webworkers/localBackendWorker.js +++ b/taxonium_web_client/src/webworkers/localBackendWorker.js @@ -358,8 +358,6 @@ onmessage = async (event) => { data.data.filename && data.data.filetype === "nextstrain" ) { - - processedUploadedData = await processNextstrain( data.data, sendStatusMessage