From cd26bc1f96bbf291350e70ad9d8951544de2c2ec Mon Sep 17 00:00:00 2001 From: rashidakanchwala Date: Thu, 7 Mar 2024 16:53:33 +0000 Subject: [PATCH 1/8] first draft --- src/selectors/linked-nodes.js | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/selectors/linked-nodes.js b/src/selectors/linked-nodes.js index 7c0660f7e8..ca03e13c87 100644 --- a/src/selectors/linked-nodes.js +++ b/src/selectors/linked-nodes.js @@ -72,3 +72,29 @@ export const getLinkedNodes = createSelector( return linkedNodes; } ); + +export const getLinkedNodesinBetween = createSelector( + [getVisibleEdgesByNode, getSelectedStartNode, getSelectedEndNode], + ({ sourceEdges, targetEdges }, startID, endID) => { + if (!startID || !endID) { + return {}; + } + + const linkedNodesBeforeEnd = {}; + findLinkedNodes(endID, sourceEdges, linkedNodes); + + const linkedNodesAfterStart = {}; + findLinkedNodes(startID, targetEdges, linkedNodes); + + const linkedNodesBetween = {}; + for (const nodeID in linkedNodesBeforeEnd) { + if (linkedNodesAfterStart[nodeID]) { + linkedNodesBetween[nodeID] = true; + } + } + return linkedNodesBetween; + } +); + + + From 02019399a22c926de86d5bf3d0c072c950bdd5b4 Mon Sep 17 00:00:00 2001 From: rashidakanchwala Date: Thu, 14 Mar 2024 10:39:15 +0000 Subject: [PATCH 2/8] progress --- .eslintrc.json | 2 +- src/actions/nodes.js | 12 ++ src/components/flowchart/flowchart.js | 2 +- src/components/node-list/index.js | 35 ++++++ src/components/node-list/node-list-groups.js | 112 ++++++++++++++---- src/components/node-list/node-list.js | 14 +++ src/components/node-list/styles/_filters.scss | 0 src/config.js | 7 +- src/reducers/index.js | 2 + src/selectors/linked-nodes.js | 51 ++++---- src/selectors/node-types.js | 22 ++++ src/store/normalize-data.js | 4 + 12 files changed, 207 insertions(+), 56 deletions(-) create mode 100644 src/components/node-list/styles/_filters.scss diff --git a/.eslintrc.json b/.eslintrc.json index 8c3135079c..71021adf2b 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -4,7 +4,7 @@ "curly": ["error"], "valid-typeof": ["error"], "camelcase": "error", - "id-length": ["error", { "min": 3, "exceptions": ["_","a","b","d","e","i","j","k","x","y","id","el","pi","PI","up"] }], + "id-length": ["error", { "min": 3, "exceptions": ["_","a","b","d","e","i","j","k","x","y","id","el","pi","PI","up","to"] }], "no-var": ["error"], "lines-between-class-members": ["error", "always"] } diff --git a/src/actions/nodes.js b/src/actions/nodes.js index 8a543b8830..589e6f1c37 100644 --- a/src/actions/nodes.js +++ b/src/actions/nodes.js @@ -55,6 +55,15 @@ export function toggleNodeDataLoading(loading) { }; } +export const ADD_NODE_FILTERS = 'ADD_NODE_FILTERS'; + +export function filterNodes(filterNodes) { + return { + type: ADD_NODE_FILTERS, + filterNodes + }; +} + export const ADD_NODE_METADATA = 'ADD_NODE_METADATA'; /** @@ -87,3 +96,6 @@ export function loadNodeData(nodeID) { } }; } + + + diff --git a/src/components/flowchart/flowchart.js b/src/components/flowchart/flowchart.js index 1590a7823e..9db87574cf 100644 --- a/src/components/flowchart/flowchart.js +++ b/src/components/flowchart/flowchart.js @@ -135,7 +135,7 @@ export class FlowChart extends Component { drawNodes.call(this, changed); } - if (changed('edges', 'nodes', 'layers', 'chartSize', 'clickedNode')) { + if (changed('edges', 'nodes', 'layers', 'chartSize', 'clickedNode','linkedNodes')) { // Don't zoom out when the metadata or code panels are opened or closed if (prevProps.visibleMetaSidebar !== this.props.visibleMetaSidebar) { drawNodes.call(this, changed); diff --git a/src/components/node-list/index.js b/src/components/node-list/index.js index 3327ec1f29..cadaea7d5c 100644 --- a/src/components/node-list/index.js +++ b/src/components/node-list/index.js @@ -12,6 +12,8 @@ import { import { getNodeTypes, isModularPipelineType, + getTaskNodes, + getDatasets, } from '../../selectors/node-types'; import { getTagData, getTagNodeCounts } from '../../selectors/tags'; import { @@ -36,6 +38,8 @@ import { loadNodeData, toggleNodeHovered, toggleNodesDisabled, + filterNodes, + toggleNodeClicked } from '../../actions/nodes'; import { useGeneratePathname } from '../../utils/hooks/use-generate-pathname'; import './styles/node-list.scss'; @@ -45,12 +49,15 @@ import './styles/node-list.scss'; * Also handles user interaction and dispatches updates back to the store. */ const NodeListProvider = ({ + flags, faded, nodes, nodeSelected, tags, tagNodeCounts, nodeTypes, + taskNodes, + datasets, onToggleNodesDisabled, onToggleNodeSelected, onToggleNodeActive, @@ -61,7 +68,9 @@ const NodeListProvider = ({ onToggleModularPipelineDisabled, onToggleModularPipelineExpanded, onToggleTypeDisabled, + onToggleNodeClicked, onToggleFocusMode, + onFilterNodes, modularPipelinesTree, focusMode, disabledModularPipeline, @@ -69,9 +78,19 @@ const NodeListProvider = ({ }) => { const [searchValue, updateSearchValue] = useState(''); + const [toNodes, selectedToNodes] = useState(null); + const [fromNodes, selectedFromNodes] = useState(null); + const { toSelectedPipeline, toSelectedNode, toFocusedModularPipeline } = useGeneratePathname(); + + useEffect(() => { + onFilterNodes(fromNodes,toNodes) + onToggleNodeClicked(null) + }, [fromNodes,toNodes, onFilterNodes, onToggleNodeClicked]); + + const items = getFilteredItems({ nodes, tags, @@ -214,13 +233,20 @@ const NodeListProvider = ({ return ( ({ + flags: state.flags, tags: getTagData(state), tagNodeCounts: getTagNodeCounts(state), nodes: getGroupedNodes(state), nodeSelected: getNodeSelected(state), nodeTypes: getNodeTypes(state), + taskNodes: getTaskNodes(state), + datasets: getDatasets(state), focusMode: getFocusedModularPipeline(state), disabledModularPipeline: state.modularPipeline.disabled, inputOutputDataNodes: getInputOutputNodesForFocusedModularPipeline(state), @@ -280,6 +309,12 @@ export const mapDispatchToProps = (dispatch) => ({ onToggleFocusMode: (modularPipeline) => { dispatch(toggleFocusMode(modularPipeline)); }, + onToggleNodeClicked: (nodeID) =>{ + dispatch(toggleNodeClicked(nodeID)) + }, + onFilterNodes: (from,to) => { + dispatch(filterNodes({from,to})); + } }); export default connect(mapStateToProps, mapDispatchToProps)(NodeListProvider); diff --git a/src/components/node-list/node-list-groups.js b/src/components/node-list/node-list-groups.js index a91dd31e94..b3b61e6564 100644 --- a/src/components/node-list/node-list-groups.js +++ b/src/components/node-list/node-list-groups.js @@ -2,11 +2,15 @@ import React, { useState } from 'react'; import { loadLocalStorage, saveLocalStorage } from '../../store/helpers'; import NodeListGroup from './node-list-group'; import { localStorageName } from '../../config'; - +import Dropdown from '../ui/dropdown'; +import MenuOption from '../ui/menu-option'; const storedState = loadLocalStorage(localStorageName); const NodeListGroups = ({ + flags, groups, + datasets, + taskNodes, items, onGroupToggleChanged, onItemChange, @@ -14,8 +18,15 @@ const NodeListGroups = ({ onItemMouseEnter, onItemMouseLeave, searchValue, + toNodes, + fromNodes, + selectedFromNodes, + selectedToNodes, }) => { const [collapsed, setCollapsed] = useState(storedState.groupsCollapsed || {}); + const [toNodeName, selectedToNodeName] = useState(''); + const [fromNodeName, selectedFromNodeName] = useState(''); + const isSlicingEnabled = flags.slicePipeline; // Collapse/expand node group const onToggleGroupCollapsed = (groupID) => { @@ -29,31 +40,80 @@ const NodeListGroups = ({ }; return ( - + <> + {!isSlicingEnabled ? ( + + ) : ( + + )} + ); }; diff --git a/src/components/node-list/node-list.js b/src/components/node-list/node-list.js index 67c0edc794..2edbf8d970 100644 --- a/src/components/node-list/node-list.js +++ b/src/components/node-list/node-list.js @@ -12,7 +12,10 @@ import './styles/node-list.scss'; * Scrollable list of toggleable items, with search & filter functionality */ const NodeList = ({ + flags, faded, + datasets, + taskNodes, items, modularPipelinesTree, modularPipelinesSearchResult, @@ -20,6 +23,10 @@ const NodeList = ({ searchValue, getGroupState, onUpdateSearchValue, + toNodes, + fromNodes, + onSelectFromNodes, + onSelectToNodes, onGroupToggleChanged, onItemClick, onItemMouseEnter, @@ -83,9 +90,16 @@ const NodeList = ({ Filters state.node.clicked; +const getFromNodes = (state) => state.filterNodes.from; +const getToNodes = (state) => state.filterNodes.to; /** * Gets a map of visible nodeIDs to successors nodeIDs in both directions * @param {Array} edges @@ -57,44 +59,39 @@ const findLinkedNodes = (nodeID, edgesByNode, visited) => { * @param {String} nodeID */ export const getLinkedNodes = createSelector( - [getVisibleEdgesByNode, getClickedNode], - ({ sourceEdges, targetEdges }, nodeID) => { + [getVisibleEdgesByNode, getClickedNode, getFromNodes, getToNodes], + ({ sourceEdges, targetEdges }, nodeID, startID, endID) => { if (!nodeID) { - return {}; + if (!startID && !endID) { + return {}; + } + + const linkedNodesBeforeEnd = {}; + findLinkedNodes(endID, sourceEdges, linkedNodesBeforeEnd); + + const linkedNodesAfterStart = {}; + findLinkedNodes(startID, targetEdges, linkedNodesAfterStart); + + const linkedNodesBetween = {}; + for (const nodeID in linkedNodesBeforeEnd) { + if (linkedNodesAfterStart[nodeID]) { + linkedNodesBetween[nodeID] = true; + } + } + return linkedNodesBetween; } const linkedNodes = {}; findLinkedNodes(nodeID, sourceEdges, linkedNodes); - + console.log(linkedNodes); + linkedNodes[nodeID] = false; findLinkedNodes(nodeID, targetEdges, linkedNodes); - + return linkedNodes; } ); -export const getLinkedNodesinBetween = createSelector( - [getVisibleEdgesByNode, getSelectedStartNode, getSelectedEndNode], - ({ sourceEdges, targetEdges }, startID, endID) => { - if (!startID || !endID) { - return {}; - } - - const linkedNodesBeforeEnd = {}; - findLinkedNodes(endID, sourceEdges, linkedNodes); - - const linkedNodesAfterStart = {}; - findLinkedNodes(startID, targetEdges, linkedNodes); - - const linkedNodesBetween = {}; - for (const nodeID in linkedNodesBeforeEnd) { - if (linkedNodesAfterStart[nodeID]) { - linkedNodesBetween[nodeID] = true; - } - } - return linkedNodesBetween; - } -); diff --git a/src/selectors/node-types.js b/src/selectors/node-types.js index e9437129a8..9a29060317 100644 --- a/src/selectors/node-types.js +++ b/src/selectors/node-types.js @@ -2,13 +2,35 @@ import { createSelector } from 'reselect'; import { getNodeDisabled } from './disabled'; import { arrayToObject } from '../utils'; + + const getNodeIDs = (state) => state.node.ids; const getNodeType = (state) => state.node.type; +const getNodeName = (state) => state.node.name; export const getNodeTypeIDs = (state) => state.nodeType.ids; const getNodeTypeName = (state) => state.nodeType.name; const getNodeTypeDisabled = (state) => state.nodeType.disabled; export const isModularPipelineType = (type) => type === 'modularPipeline'; + + +export const getTaskNodes = createSelector( + [getNodeType, getNodeName], + (nodeType, nodeName) => { + const taskNodeIds = Object.keys(nodeName).filter(id => nodeType[id] === 'task'); + return arrayToObject(taskNodeIds, id => nodeName[id]); + } +); + +export const getDatasets = createSelector( + [getNodeType, getNodeName], + (nodeType, nodeName) => { + const dataNodeIds = Object.keys(nodeName).filter(id => nodeType[id] === 'data'); + return arrayToObject(dataNodeIds, id => nodeName[id]); + } +); + + /** * Calculate the total number of nodes (and the number of visible nodes) * for each node-type diff --git a/src/store/normalize-data.js b/src/store/normalize-data.js index 1546526930..e63be38674 100644 --- a/src/store/normalize-data.js +++ b/src/store/normalize-data.js @@ -78,6 +78,10 @@ export const createInitialPipelineState = () => ({ active: {}, enabled: {}, }, + filterNodes: { + from: [], + to:[], + }, hoveredParameters: false, hoveredFocusMode: false, }); From a42102f08c4f99e46ce41c48fd3a7c8c7bb2bbe5 Mon Sep 17 00:00:00 2001 From: rashidakanchwala Date: Tue, 19 Mar 2024 11:07:27 +0000 Subject: [PATCH 3/8] WIP --- demo-project/src/demo_project/settings.py | 1 + .../integrations/kedro/data_loader.py | 46 ++++++++++--------- package/kedro_viz/models/flowchart.py | 8 +++- src/components/flowchart/draw.js | 8 +++- src/components/flowchart/flowchart.js | 20 +++++++- src/selectors/linked-nodes.js | 18 +++----- 6 files changed, 63 insertions(+), 38 deletions(-) diff --git a/demo-project/src/demo_project/settings.py b/demo-project/src/demo_project/settings.py index 304f7d8c1a..fefed89d7e 100644 --- a/demo-project/src/demo_project/settings.py +++ b/demo-project/src/demo_project/settings.py @@ -11,6 +11,7 @@ SESSION_STORE_CLASS = SQLiteStore SESSION_STORE_ARGS = {"path": str(Path(__file__).parents[2] / "data")} + # Setup for collaborative experiment tracking. # SESSION_STORE_ARGS = {"path": str(Path(__file__).parents[2] / "data"), # "remote_path": "s3://{path-to-session_store}" } diff --git a/package/kedro_viz/integrations/kedro/data_loader.py b/package/kedro_viz/integrations/kedro/data_loader.py index dccf95d8af..267b37e828 100644 --- a/package/kedro_viz/integrations/kedro/data_loader.py +++ b/package/kedro_viz/integrations/kedro/data_loader.py @@ -89,25 +89,27 @@ def load_data( from kedro.framework.startup import bootstrap_project bootstrap_project(project_path) - - with KedroSession.create( - project_path=project_path, - env=env, - save_on_close=False, - extra_params=extra_params, - ) as session: - # check for --ignore-plugins option - if ignore_plugins: - session._hook_manager = _VizNullPluginManager() # type: ignore - - context = session.load_context() - session_store = session._store - catalog = context.catalog - - # Pipelines is a lazy dict-like object, so we force it to populate here - # in case user doesn't have an active session down the line when it's first accessed. - # Useful for users who have `get_current_session` in their `register_pipelines()`. - pipelines_dict = dict(pipelines) - stats_dict = _get_dataset_stats(project_path) - - return catalog, pipelines_dict, session_store, stats_dict + + pipelines_dict = dict(pipelines) + c + + # with KedroSession.create( + # project_path=project_path, + # env=env, + # save_on_close=False, + # extra_params=extra_params, + # ) as session: + # # check for --ignore-plugins option + # if ignore_plugins: + # session._hook_manager = _VizNullPluginManager() # type: ignore + + # context = session.load_context() + # session_store = session._store + # catalog = context.catalog + + # # Pipelines is a lazy dict-like object, so we force it to populate here + # # in case user doesn't have an active session down the line when it's first accessed. + # # Useful for users who have `get_current_session` in their `register_pipelines()`. + # stats_dict = _get_dataset_stats(project_path) + + # return catalog, pipelines_dict, session_store, stats_dict diff --git a/package/kedro_viz/models/flowchart.py b/package/kedro_viz/models/flowchart.py index c7e0b4aed7..56e8382115 100644 --- a/package/kedro_viz/models/flowchart.py +++ b/package/kedro_viz/models/flowchart.py @@ -425,14 +425,18 @@ def inputs(self) -> Set[str]: Intuitively, the set of inputs for this modular pipeline is the set of all external and internal inputs, excluding the ones also serving as outputs. """ - return (self.external_inputs | self.internal_inputs) - self.internal_outputs + return (self.external_inputs | self.internal_inputs) - ( + self.external_outputs | self.internal_outputs + ) @property def outputs(self) -> Set[str]: """Return a set of inputs for this modular pipeline. Follow the same logic as the inputs calculation. """ - return self.external_outputs | (self.internal_outputs - self.internal_inputs) + return (self.external_outputs | self.internal_outputs) - ( + self.external_inputs | self.internal_inputs + ) class TaskNodeMetadata(GraphNodeMetadata): diff --git a/src/components/flowchart/draw.js b/src/components/flowchart/draw.js index d221bf4d29..b27b12a840 100644 --- a/src/components/flowchart/draw.js +++ b/src/components/flowchart/draw.js @@ -235,7 +235,9 @@ export const drawNodes = function (changed) { 'clickedNode', 'linkedNodes', 'focusMode', - 'inputOutputDataNodes' + 'inputOutputDataNodes', + 'fromNodes', + 'toNodes' ) ) { allNodes @@ -409,7 +411,9 @@ export const drawEdges = function (changed) { 'clickedNode', 'linkedNodes', 'focusMode', - 'inputOutputDataEdges' + 'inputOutputDataEdges', + 'fromNodes', + 'toNodes' ) ) { allEdges diff --git a/src/components/flowchart/flowchart.js b/src/components/flowchart/flowchart.js index 9db87574cf..1f3526817c 100644 --- a/src/components/flowchart/flowchart.js +++ b/src/components/flowchart/flowchart.js @@ -110,6 +110,8 @@ export class FlowChart extends Component { 'edges', 'clickedNode', 'linkedNodes', + 'fromNodes', + 'toNodes', 'focusMode', 'inputOutputDataEdges' ) @@ -122,6 +124,8 @@ export class FlowChart extends Component { 'nodes', 'clickedNode', 'linkedNodes', + 'fromNodes', + 'toNodes', 'nodeTypeDisabled', 'nodeActive', 'nodeSelected', @@ -132,10 +136,22 @@ export class FlowChart extends Component { 'hoveredFocusMode' ) ) { + console.log(this.props.linkedNodes); drawNodes.call(this, changed); } - if (changed('edges', 'nodes', 'layers', 'chartSize', 'clickedNode','linkedNodes')) { + if ( + changed( + 'edges', + 'nodes', + 'layers', + 'chartSize', + 'clickedNode', + 'linkedNodes', + 'fromNodes', + 'toNodes' + ) + ) { // Don't zoom out when the metadata or code panels are opened or closed if (prevProps.visibleMetaSidebar !== this.props.visibleMetaSidebar) { drawNodes.call(this, changed); @@ -687,6 +703,8 @@ const emptyGraphSize = {}; export const mapStateToProps = (state, ownProps) => ({ clickedNode: state.node.clicked, + fromNodes: state.filterNodes.from, + toNodes: state.filterNodes.to, chartSize: getChartSize(state), chartZoom: getChartZoom(state), displayGlobalToolbar: state.display.globalToolbar, diff --git a/src/selectors/linked-nodes.js b/src/selectors/linked-nodes.js index c550c83661..f045ffde22 100644 --- a/src/selectors/linked-nodes.js +++ b/src/selectors/linked-nodes.js @@ -3,7 +3,7 @@ import { getVisibleEdges } from './edges'; const getClickedNode = (state) => state.node.clicked; const getFromNodes = (state) => state.filterNodes.from; -const getToNodes = (state) => state.filterNodes.to; +const getToNodes = (state) => state.filterNodes.to; /** * Gets a map of visible nodeIDs to successors nodeIDs in both directions * @param {Array} edges @@ -65,13 +65,13 @@ export const getLinkedNodes = createSelector( if (!startID && !endID) { return {}; } - + const linkedNodesBeforeEnd = {}; findLinkedNodes(endID, sourceEdges, linkedNodesBeforeEnd); - + const linkedNodesAfterStart = {}; findLinkedNodes(startID, targetEdges, linkedNodesAfterStart); - + const linkedNodesBetween = {}; for (const nodeID in linkedNodesBeforeEnd) { if (linkedNodesAfterStart[nodeID]) { @@ -83,15 +83,11 @@ export const getLinkedNodes = createSelector( const linkedNodes = {}; findLinkedNodes(nodeID, sourceEdges, linkedNodes); - console.log(linkedNodes); - + linkedNodes[nodeID] = false; findLinkedNodes(nodeID, targetEdges, linkedNodes); - + + console.log(linkedNodes); return linkedNodes; } ); - - - - From 898cf2485c5768cc8bdfc1a1741759e0650bbe62 Mon Sep 17 00:00:00 2001 From: rashidakanchwala Date: Mon, 25 Mar 2024 10:40:44 +0000 Subject: [PATCH 4/8] still WIP --- demo-project/conf/base/catalog_01_raw.yml | 1 + .../integrations/kedro/data_loader.py | 46 +++++++------- src/actions/filters.js | 12 ++++ src/actions/nodes.js | 12 ---- src/components/flowchart/draw.js | 18 +++--- src/components/flowchart/flowchart.js | 20 +++--- src/components/node-list/index.js | 31 +++------- src/components/node-list/node-list-groups.js | 56 ++++++++++------- src/components/node-list/node-list.js | 12 ++-- src/reducers/filters.js | 20 ++++++ src/reducers/index.js | 4 +- src/selectors/disabled.js | 5 +- src/selectors/linked-nodes.js | 61 ++++++++++++------- src/store/normalize-data.js | 6 +- 14 files changed, 168 insertions(+), 136 deletions(-) create mode 100644 src/actions/filters.js create mode 100644 src/reducers/filters.js diff --git a/demo-project/conf/base/catalog_01_raw.yml b/demo-project/conf/base/catalog_01_raw.yml index b2d71bf82c..f594c64d0d 100644 --- a/demo-project/conf/base/catalog_01_raw.yml +++ b/demo-project/conf/base/catalog_01_raw.yml @@ -6,6 +6,7 @@ companies: layer: raw preview_args: nrows: 5 + reviews: type: pandas.CSVDataset diff --git a/package/kedro_viz/integrations/kedro/data_loader.py b/package/kedro_viz/integrations/kedro/data_loader.py index 267b37e828..f83d395bba 100644 --- a/package/kedro_viz/integrations/kedro/data_loader.py +++ b/package/kedro_viz/integrations/kedro/data_loader.py @@ -89,27 +89,25 @@ def load_data( from kedro.framework.startup import bootstrap_project bootstrap_project(project_path) - - pipelines_dict = dict(pipelines) - c - - # with KedroSession.create( - # project_path=project_path, - # env=env, - # save_on_close=False, - # extra_params=extra_params, - # ) as session: - # # check for --ignore-plugins option - # if ignore_plugins: - # session._hook_manager = _VizNullPluginManager() # type: ignore - - # context = session.load_context() - # session_store = session._store - # catalog = context.catalog - - # # Pipelines is a lazy dict-like object, so we force it to populate here - # # in case user doesn't have an active session down the line when it's first accessed. - # # Useful for users who have `get_current_session` in their `register_pipelines()`. - # stats_dict = _get_dataset_stats(project_path) - - # return catalog, pipelines_dict, session_store, stats_dict + + with KedroSession.create( + project_path=project_path, + env=env, + save_on_close=False, + extra_params=extra_params, + ) as session: + # check for --ignore-plugins option + if ignore_plugins: + session._hook_manager = _VizNullPluginManager() # type: ignore + + context = session.load_context() + session_store = session._store + catalog = context.catalog + + # Pipelines is a lazy dict-like object, so we force it to populate here + # in case user doesn't have an active session down the line when it's first accessed. + # Useful for users who have `get_current_session` in their `register_pipelines()`. + stats_dict = _get_dataset_stats(project_path) + pipelines_dict = dict(pipelines) + + return catalog, pipelines_dict, session_store, stats_dict diff --git a/src/actions/filters.js b/src/actions/filters.js new file mode 100644 index 0000000000..3143ae19d3 --- /dev/null +++ b/src/actions/filters.js @@ -0,0 +1,12 @@ +export const FILTER_NODES = 'FILTER_NODES'; + +export const filterNodes = (from, to) => ({ + type: FILTER_NODES, + filters: { from, to }, +}); + +export const RESET_NODES_FILTER = 'RESET_NODES_FILTER'; + +export const resetNodesFilter = () => ({ + type: RESET_NODES_FILTER, +}); diff --git a/src/actions/nodes.js b/src/actions/nodes.js index 589e6f1c37..8a543b8830 100644 --- a/src/actions/nodes.js +++ b/src/actions/nodes.js @@ -55,15 +55,6 @@ export function toggleNodeDataLoading(loading) { }; } -export const ADD_NODE_FILTERS = 'ADD_NODE_FILTERS'; - -export function filterNodes(filterNodes) { - return { - type: ADD_NODE_FILTERS, - filterNodes - }; -} - export const ADD_NODE_METADATA = 'ADD_NODE_METADATA'; /** @@ -96,6 +87,3 @@ export function loadNodeData(nodeID) { } }; } - - - diff --git a/src/components/flowchart/draw.js b/src/components/flowchart/draw.js index b27b12a840..ed84c05705 100644 --- a/src/components/flowchart/draw.js +++ b/src/components/flowchart/draw.js @@ -132,13 +132,18 @@ export const drawNodes = function (changed) { hoveredParameters, nodesWithInputParams, inputOutputDataNodes, + slicedPipeline, nodes, focusMode, hoveredFocusMode, } = this.props; - const isInputOutputNode = (nodeID) => - focusMode !== null && inputOutputDataNodes[nodeID]; + const isInputOutputNode = (nodeID) => { + const result = + (focusMode !== null && inputOutputDataNodes[nodeID]) || + slicedPipeline[nodeID]; + return result; + }; if (changed('nodes')) { this.el.nodes = this.el.nodeGroup @@ -236,8 +241,7 @@ export const drawNodes = function (changed) { 'linkedNodes', 'focusMode', 'inputOutputDataNodes', - 'fromNodes', - 'toNodes' + 'slicedPipeline' ) ) { allNodes @@ -358,7 +362,7 @@ export const drawEdges = function (changed) { .merge(exitEdges) .filter((edge) => edge); - if (changed('edges', 'focusMode', 'inputOutputDataNodes')) { + if (changed('edges', 'focusMode', 'inputOutputDataNodes', 'slicedPipeline')) { enterEdges.append('path'); allEdges .select('path') @@ -411,9 +415,7 @@ export const drawEdges = function (changed) { 'clickedNode', 'linkedNodes', 'focusMode', - 'inputOutputDataEdges', - 'fromNodes', - 'toNodes' + 'inputOutputDataEdges' ) ) { allEdges diff --git a/src/components/flowchart/flowchart.js b/src/components/flowchart/flowchart.js index 1f3526817c..ccced105e8 100644 --- a/src/components/flowchart/flowchart.js +++ b/src/components/flowchart/flowchart.js @@ -14,7 +14,10 @@ import { import { getInputOutputDataEdges } from '../../selectors/edges'; import { getChartSize, getChartZoom } from '../../selectors/layout'; import { getLayers } from '../../selectors/layers'; -import { getLinkedNodes } from '../../selectors/linked-nodes'; +import { + getLinkedNodes, + getSlicedGraphNodes, +} from '../../selectors/linked-nodes'; import { getVisibleMetaSidebar } from '../../selectors/metadata'; import { drawNodes, drawEdges, drawLayers, drawLayerNames } from './draw'; import { @@ -110,8 +113,6 @@ export class FlowChart extends Component { 'edges', 'clickedNode', 'linkedNodes', - 'fromNodes', - 'toNodes', 'focusMode', 'inputOutputDataEdges' ) @@ -124,8 +125,6 @@ export class FlowChart extends Component { 'nodes', 'clickedNode', 'linkedNodes', - 'fromNodes', - 'toNodes', 'nodeTypeDisabled', 'nodeActive', 'nodeSelected', @@ -133,10 +132,10 @@ export class FlowChart extends Component { 'nodesWithInputParams', 'focusMode', 'inputOutputDataNodes', - 'hoveredFocusMode' + 'hoveredFocusMode', + 'slicedPipeline' ) ) { - console.log(this.props.linkedNodes); drawNodes.call(this, changed); } @@ -147,9 +146,7 @@ export class FlowChart extends Component { 'layers', 'chartSize', 'clickedNode', - 'linkedNodes', - 'fromNodes', - 'toNodes' + 'linkedNodes' ) ) { // Don't zoom out when the metadata or code panels are opened or closed @@ -703,8 +700,6 @@ const emptyGraphSize = {}; export const mapStateToProps = (state, ownProps) => ({ clickedNode: state.node.clicked, - fromNodes: state.filterNodes.from, - toNodes: state.filterNodes.to, chartSize: getChartSize(state), chartZoom: getChartZoom(state), displayGlobalToolbar: state.display.globalToolbar, @@ -722,6 +717,7 @@ export const mapStateToProps = (state, ownProps) => ({ nodesWithInputParams: getNodesWithInputParams(state), inputOutputDataNodes: getInputOutputNodesForFocusedModularPipeline(state), inputOutputDataEdges: getInputOutputDataEdges(state), + slicedPipeline: getSlicedGraphNodes(state), visibleGraph: state.visible.graph, visibleSidebar: state.visible.sidebar, visibleCode: state.visible.code, diff --git a/src/components/node-list/index.js b/src/components/node-list/index.js index cadaea7d5c..a8897319ec 100644 --- a/src/components/node-list/index.js +++ b/src/components/node-list/index.js @@ -38,9 +38,8 @@ import { loadNodeData, toggleNodeHovered, toggleNodesDisabled, - filterNodes, - toggleNodeClicked } from '../../actions/nodes'; +import { filterNodes, resetNodesFilter } from '../../actions/filters'; import { useGeneratePathname } from '../../utils/hooks/use-generate-pathname'; import './styles/node-list.scss'; @@ -68,9 +67,9 @@ const NodeListProvider = ({ onToggleModularPipelineDisabled, onToggleModularPipelineExpanded, onToggleTypeDisabled, - onToggleNodeClicked, onToggleFocusMode, onFilterNodes, + onResetNodesFilter, modularPipelinesTree, focusMode, disabledModularPipeline, @@ -78,19 +77,9 @@ const NodeListProvider = ({ }) => { const [searchValue, updateSearchValue] = useState(''); - const [toNodes, selectedToNodes] = useState(null); - const [fromNodes, selectedFromNodes] = useState(null); - const { toSelectedPipeline, toSelectedNode, toFocusedModularPipeline } = useGeneratePathname(); - - useEffect(() => { - onFilterNodes(fromNodes,toNodes) - onToggleNodeClicked(null) - }, [fromNodes,toNodes, onFilterNodes, onToggleNodeClicked]); - - const items = getFilteredItems({ nodes, tags, @@ -242,11 +231,7 @@ const NodeListProvider = ({ taskNodes={taskNodes} datasets={datasets} searchValue={searchValue} - toNodes={toNodes} - fromNodes={fromNodes} onUpdateSearchValue={debounce(updateSearchValue, 250)} - onSelectFromNodes = {selectedFromNodes} - onSelectToNodes = {selectedToNodes} onModularPipelineToggleExpanded={handleToggleModularPipelineExpanded} onGroupToggleChanged={onGroupToggleChanged} onToggleFocusMode={onToggleFocusMode} @@ -254,6 +239,8 @@ const NodeListProvider = ({ onItemMouseEnter={onItemMouseEnter} onItemMouseLeave={onItemMouseLeave} onItemChange={onItemChange} + onFilterNodes={onFilterNodes} + onResetNodesFilter={onResetNodesFilter} focusMode={focusMode} disabledModularPipeline={disabledModularPipeline} /> @@ -309,12 +296,12 @@ export const mapDispatchToProps = (dispatch) => ({ onToggleFocusMode: (modularPipeline) => { dispatch(toggleFocusMode(modularPipeline)); }, - onToggleNodeClicked: (nodeID) =>{ - dispatch(toggleNodeClicked(nodeID)) + onFilterNodes: (from, to) => { + dispatch(filterNodes(from, to)); + }, + onResetNodesFilter: () => { + dispatch(resetNodesFilter()); }, - onFilterNodes: (from,to) => { - dispatch(filterNodes({from,to})); - } }); export default connect(mapStateToProps, mapDispatchToProps)(NodeListProvider); diff --git a/src/components/node-list/node-list-groups.js b/src/components/node-list/node-list-groups.js index b3b61e6564..f4bf0681b3 100644 --- a/src/components/node-list/node-list-groups.js +++ b/src/components/node-list/node-list-groups.js @@ -4,6 +4,7 @@ import NodeListGroup from './node-list-group'; import { localStorageName } from '../../config'; import Dropdown from '../ui/dropdown'; import MenuOption from '../ui/menu-option'; +import Button from '../ui/button'; const storedState = loadLocalStorage(localStorageName); const NodeListGroups = ({ @@ -18,14 +19,12 @@ const NodeListGroups = ({ onItemMouseEnter, onItemMouseLeave, searchValue, - toNodes, - fromNodes, - selectedFromNodes, - selectedToNodes, + onFilterNodes, + onResetNodesFilters, }) => { const [collapsed, setCollapsed] = useState(storedState.groupsCollapsed || {}); - const [toNodeName, selectedToNodeName] = useState(''); - const [fromNodeName, selectedFromNodeName] = useState(''); + const [toNode, selectedToNode] = useState({}); + const [fromNode, selectedFromNode] = useState({}); const isSlicingEnabled = flags.slicePipeline; // Collapse/expand node group @@ -75,13 +74,11 @@ const NodeListGroups = ({ From node { - selectedFromNodes(selectedNode.value); - selectedFromNodeName(selectedNode.label); - } - } + defaultText={fromNode.label} + placeholderText={!fromNode.label ? 'Select a node' : null} + onChanged={(selectedNode) => { + selectedFromNode(selectedNode); + }} width={null} > {Object.entries(taskNodes).map(([value, label]) => ( @@ -94,23 +91,36 @@ const NodeListGroups = ({ To node { - selectedToNodes(selectedNode.value); - selectedToNodeName(selectedNode.label); - } - } + defaultText={toNode.label} + placeholderText={!toNode.label ? 'Select a node' : null} + onChanged={(selectedNode) => { + selectedToNode(selectedNode); + }} width={null} > {Object.entries(taskNodes).map(([value, label]) => ( ))} +
+ + +
- - - + )} diff --git a/src/components/node-list/node-list.js b/src/components/node-list/node-list.js index 2edbf8d970..ebb25fa49b 100644 --- a/src/components/node-list/node-list.js +++ b/src/components/node-list/node-list.js @@ -23,10 +23,8 @@ const NodeList = ({ searchValue, getGroupState, onUpdateSearchValue, - toNodes, - fromNodes, - onSelectFromNodes, - onSelectToNodes, + onFilterNodes, + onResetNodesFilter, onGroupToggleChanged, onItemClick, onItemMouseEnter, @@ -96,10 +94,8 @@ const NodeList = ({ items={items} groups={groups} searchValue={searchValue} - toNodes={toNodes} - fromNodes={fromNodes} - selectedFromNodes={onSelectFromNodes} - selectedToNodes={onSelectToNodes} + onFilterNodes={onFilterNodes} + onResetNodesFilter={onResetNodesFilter} getGroupState={getGroupState} onItemClick={onItemClick} onItemMouseEnter={onItemMouseEnter} diff --git a/src/reducers/filters.js b/src/reducers/filters.js new file mode 100644 index 0000000000..efdb40d885 --- /dev/null +++ b/src/reducers/filters.js @@ -0,0 +1,20 @@ +import { FILTER_NODES, RESET_NODES_FILTER } from '../actions/filters'; + +// Reducer for filtering nodes +const filterNodesReducer = (filterState = {}, action) => { + const updateState = (newState) => Object.assign({}, filterState, newState); + + switch (action.type) { + case FILTER_NODES: + return updateState({ + from: action.filters.from, + to: action.filters.to, + }); + case RESET_NODES_FILTER: + return {}; + default: + return filterState; + } +}; + +export default filterNodesReducer; diff --git a/src/reducers/index.js b/src/reducers/index.js index 83c90cb845..1c572cb69f 100644 --- a/src/reducers/index.js +++ b/src/reducers/index.js @@ -7,6 +7,7 @@ import loading from './loading'; import node from './nodes'; import nodeType from './node-type'; import pipeline from './pipeline'; +import filters from './filters'; import tag from './tags'; import modularPipeline from './modular-pipelines'; import visible from './visible'; @@ -22,7 +23,6 @@ import { UPDATE_ZOOM, } from '../actions'; import { TOGGLE_PARAMETERS_HOVERED } from '../actions'; -import { ADD_NODE_FILTERS } from '../actions/nodes'; /** * Create a generic reducer @@ -62,6 +62,7 @@ const combinedReducer = combineReducers({ node, nodeType, pipeline, + filters, tag, modularPipeline, visible, @@ -76,7 +77,6 @@ const combinedReducer = combineReducers({ textLabels: createReducer(true, TOGGLE_TEXT_LABELS, 'textLabels'), theme: createReducer('dark', TOGGLE_THEME, 'theme'), isPrettyName: createReducer(true, TOGGLE_IS_PRETTY_NAME, 'isPrettyName'), - filterNodes: createReducer({}, ADD_NODE_FILTERS,'filterNodes'), showFeatureHints: createReducer( true, TOGGLE_SHOW_FEATURE_HINTS, diff --git a/src/selectors/disabled.js b/src/selectors/disabled.js index aa0e8bba79..a296f4f41b 100644 --- a/src/selectors/disabled.js +++ b/src/selectors/disabled.js @@ -6,6 +6,7 @@ import { getModularPipelinesTree, } from './modular-pipelines'; import { getTagCount } from './tags'; +import { getSlicedGraphNodes } from './linked-nodes'; const getNodeIDs = (state) => state.node.ids; const getNodeDisabledNode = (state) => state.node.disabled; @@ -78,6 +79,7 @@ export const getNodeDisabled = createSelector( getVisibleSidebarNodes, getVisibleModularPipelineInputsOutputs, getDisabledModularPipeline, + getSlicedGraphNodes, ], ( nodeIDs, @@ -91,7 +93,8 @@ export const getNodeDisabled = createSelector( focusedModularPipeline, visibleSidebarNodes, visibleModularPipelineInputsOutputs, - disabledModularPipeline + disabledModularPipeline, + slicedGraphNodes ) => arrayToObject(nodeIDs, (id) => { let isDisabledViaModularPipeline = diff --git a/src/selectors/linked-nodes.js b/src/selectors/linked-nodes.js index f045ffde22..96b2abfe5c 100644 --- a/src/selectors/linked-nodes.js +++ b/src/selectors/linked-nodes.js @@ -1,9 +1,10 @@ import { createSelector } from 'reselect'; import { getVisibleEdges } from './edges'; +import { getGraphNodes } from './nodes'; const getClickedNode = (state) => state.node.clicked; -const getFromNodes = (state) => state.filterNodes.from; -const getToNodes = (state) => state.filterNodes.to; +const getFromNodes = (state) => state.filters.from; +const getToNodes = (state) => state.filters.to; /** * Gets a map of visible nodeIDs to successors nodeIDs in both directions * @param {Array} edges @@ -59,26 +60,10 @@ const findLinkedNodes = (nodeID, edgesByNode, visited) => { * @param {String} nodeID */ export const getLinkedNodes = createSelector( - [getVisibleEdgesByNode, getClickedNode, getFromNodes, getToNodes], - ({ sourceEdges, targetEdges }, nodeID, startID, endID) => { + [getVisibleEdgesByNode, getClickedNode], + ({ sourceEdges, targetEdges }, nodeID) => { if (!nodeID) { - if (!startID && !endID) { - return {}; - } - - const linkedNodesBeforeEnd = {}; - findLinkedNodes(endID, sourceEdges, linkedNodesBeforeEnd); - - const linkedNodesAfterStart = {}; - findLinkedNodes(startID, targetEdges, linkedNodesAfterStart); - - const linkedNodesBetween = {}; - for (const nodeID in linkedNodesBeforeEnd) { - if (linkedNodesAfterStart[nodeID]) { - linkedNodesBetween[nodeID] = true; - } - } - return linkedNodesBetween; + return {}; } const linkedNodes = {}; @@ -87,7 +72,39 @@ export const getLinkedNodes = createSelector( linkedNodes[nodeID] = false; findLinkedNodes(nodeID, targetEdges, linkedNodes); - console.log(linkedNodes); return linkedNodes; } ); + +export const getSlicedGraphNodes = createSelector( + [getVisibleEdgesByNode, getFromNodes, getToNodes, getGraphNodes], + ({ sourceEdges, targetEdges }, startID, endID, graphNodes) => { + if (!startID && !endID && !graphNodes) { + return {}; + } + + const linkedNodesBeforeEnd = {}; + findLinkedNodes(endID, sourceEdges, linkedNodesBeforeEnd); + + const linkedNodesAfterStart = {}; + findLinkedNodes(startID, targetEdges, linkedNodesAfterStart); + + const linkedNodesBetween = []; + for (const nodeID in linkedNodesBeforeEnd) { + if (linkedNodesAfterStart[nodeID]) { + linkedNodesBetween.push(nodeID); + } + } + + // Traverse graphNodes and add those whose ID is in linkedNodesBetween + const filteredNodes = {}; + for (const nodeID of linkedNodesBetween) { + // Changed this loop + if (graphNodes[nodeID]) { + // Check if the graphNode's ID is present + filteredNodes[nodeID] = graphNodes[nodeID]; + } + } + return filteredNodes; + } +); diff --git a/src/store/normalize-data.js b/src/store/normalize-data.js index e63be38674..3961cec45e 100644 --- a/src/store/normalize-data.js +++ b/src/store/normalize-data.js @@ -78,9 +78,11 @@ export const createInitialPipelineState = () => ({ active: {}, enabled: {}, }, - filterNodes: { + filters: { from: [], - to:[], + to: [], + active: {}, + disabled: {}, }, hoveredParameters: false, hoveredFocusMode: false, From fdc5f93d23497b91999f58722725e117f3f83063 Mon Sep 17 00:00:00 2001 From: rashidakanchwala Date: Fri, 29 Mar 2024 18:43:18 +0000 Subject: [PATCH 5/8] wip --- src/components/flowchart/draw.js | 5 +-- src/components/flowchart/flowchart.js | 12 +++---- src/selectors/disabled.js | 5 +-- src/selectors/linked-nodes.js | 52 ++++++++++++++------------- 4 files changed, 33 insertions(+), 41 deletions(-) diff --git a/src/components/flowchart/draw.js b/src/components/flowchart/draw.js index ed84c05705..dfa2e24f99 100644 --- a/src/components/flowchart/draw.js +++ b/src/components/flowchart/draw.js @@ -132,16 +132,13 @@ export const drawNodes = function (changed) { hoveredParameters, nodesWithInputParams, inputOutputDataNodes, - slicedPipeline, nodes, focusMode, hoveredFocusMode, } = this.props; const isInputOutputNode = (nodeID) => { - const result = - (focusMode !== null && inputOutputDataNodes[nodeID]) || - slicedPipeline[nodeID]; + const result = focusMode !== null && inputOutputDataNodes[nodeID]; return result; }; diff --git a/src/components/flowchart/flowchart.js b/src/components/flowchart/flowchart.js index ccced105e8..08f49b0574 100644 --- a/src/components/flowchart/flowchart.js +++ b/src/components/flowchart/flowchart.js @@ -14,10 +14,7 @@ import { import { getInputOutputDataEdges } from '../../selectors/edges'; import { getChartSize, getChartZoom } from '../../selectors/layout'; import { getLayers } from '../../selectors/layers'; -import { - getLinkedNodes, - getSlicedGraphNodes, -} from '../../selectors/linked-nodes'; +import { getLinkedNodes, getFilteredNodes } from '../../selectors/linked-nodes'; import { getVisibleMetaSidebar } from '../../selectors/metadata'; import { drawNodes, drawEdges, drawLayers, drawLayerNames } from './draw'; import { @@ -132,8 +129,7 @@ export class FlowChart extends Component { 'nodesWithInputParams', 'focusMode', 'inputOutputDataNodes', - 'hoveredFocusMode', - 'slicedPipeline' + 'hoveredFocusMode' ) ) { drawNodes.call(this, changed); @@ -710,14 +706,14 @@ export const mapStateToProps = (state, ownProps) => ({ hoveredFocusMode: state.hoveredFocusMode, layers: getLayers(state), linkedNodes: getLinkedNodes(state), - nodes: state.graph.nodes || emptyNodes, + nodes: getFilteredNodes(state) || emptyNodes, + // nodes: state.graph.nodes || emptyNodes, nodeTypeDisabled: state.nodeType.disabled, nodeActive: getNodeActive(state), nodeSelected: getNodeSelected(state), nodesWithInputParams: getNodesWithInputParams(state), inputOutputDataNodes: getInputOutputNodesForFocusedModularPipeline(state), inputOutputDataEdges: getInputOutputDataEdges(state), - slicedPipeline: getSlicedGraphNodes(state), visibleGraph: state.visible.graph, visibleSidebar: state.visible.sidebar, visibleCode: state.visible.code, diff --git a/src/selectors/disabled.js b/src/selectors/disabled.js index a296f4f41b..aa0e8bba79 100644 --- a/src/selectors/disabled.js +++ b/src/selectors/disabled.js @@ -6,7 +6,6 @@ import { getModularPipelinesTree, } from './modular-pipelines'; import { getTagCount } from './tags'; -import { getSlicedGraphNodes } from './linked-nodes'; const getNodeIDs = (state) => state.node.ids; const getNodeDisabledNode = (state) => state.node.disabled; @@ -79,7 +78,6 @@ export const getNodeDisabled = createSelector( getVisibleSidebarNodes, getVisibleModularPipelineInputsOutputs, getDisabledModularPipeline, - getSlicedGraphNodes, ], ( nodeIDs, @@ -93,8 +91,7 @@ export const getNodeDisabled = createSelector( focusedModularPipeline, visibleSidebarNodes, visibleModularPipelineInputsOutputs, - disabledModularPipeline, - slicedGraphNodes + disabledModularPipeline ) => arrayToObject(nodeIDs, (id) => { let isDisabledViaModularPipeline = diff --git a/src/selectors/linked-nodes.js b/src/selectors/linked-nodes.js index 96b2abfe5c..633c036e81 100644 --- a/src/selectors/linked-nodes.js +++ b/src/selectors/linked-nodes.js @@ -72,39 +72,41 @@ export const getLinkedNodes = createSelector( linkedNodes[nodeID] = false; findLinkedNodes(nodeID, targetEdges, linkedNodes); + console.log(linkedNodes); + return linkedNodes; } ); -export const getSlicedGraphNodes = createSelector( +export const getFilteredNodes = createSelector( [getVisibleEdgesByNode, getFromNodes, getToNodes, getGraphNodes], ({ sourceEdges, targetEdges }, startID, endID, graphNodes) => { - if (!startID && !endID && !graphNodes) { - return {}; - } - - const linkedNodesBeforeEnd = {}; - findLinkedNodes(endID, sourceEdges, linkedNodesBeforeEnd); - - const linkedNodesAfterStart = {}; - findLinkedNodes(startID, targetEdges, linkedNodesAfterStart); - - const linkedNodesBetween = []; - for (const nodeID in linkedNodesBeforeEnd) { - if (linkedNodesAfterStart[nodeID]) { - linkedNodesBetween.push(nodeID); + // Initialize as an array this time + let filteredNodes = []; + + if ((!startID || !startID.length) && (!endID || !endID.length)) { + // If no startID or endID, return all graphNodes as an array + filteredNodes = Object.values(graphNodes); // Convert all graphNodes values to an array + } else { + const linkedNodesBeforeEnd = {}; + findLinkedNodes(endID, sourceEdges, linkedNodesBeforeEnd); + + const linkedNodesAfterStart = {}; + findLinkedNodes(startID, targetEdges, linkedNodesAfterStart); + + const linkedNodesBetween = []; + for (const nodeID in linkedNodesBeforeEnd) { + if (linkedNodesAfterStart[nodeID]) { + linkedNodesBetween.push(nodeID); + } } - } - // Traverse graphNodes and add those whose ID is in linkedNodesBetween - const filteredNodes = {}; - for (const nodeID of linkedNodesBetween) { - // Changed this loop - if (graphNodes[nodeID]) { - // Check if the graphNode's ID is present - filteredNodes[nodeID] = graphNodes[nodeID]; - } + // Populate filteredNodes array with nodes that are linked between start and end + filteredNodes = linkedNodesBetween + .map((nodeID) => graphNodes[nodeID]) + .filter((node) => node !== undefined); } - return filteredNodes; + + return filteredNodes; // This is now an array } ); From 82541010809f06d060c316319b9c2d0b15eb9271 Mon Sep 17 00:00:00 2001 From: rashidakanchwala Date: Tue, 23 Apr 2024 12:34:09 +0100 Subject: [PATCH 6/8] first working draft --- src/components/flowchart/flowchart.js | 5 +- src/selectors/disabled.js | 11 ++- src/selectors/filtered-pipeline.js | 107 ++++++++++++++++++++++++++ src/selectors/linked-nodes.js | 38 --------- 4 files changed, 119 insertions(+), 42 deletions(-) create mode 100644 src/selectors/filtered-pipeline.js diff --git a/src/components/flowchart/flowchart.js b/src/components/flowchart/flowchart.js index 08f49b0574..a6a8d6aa18 100644 --- a/src/components/flowchart/flowchart.js +++ b/src/components/flowchart/flowchart.js @@ -14,7 +14,7 @@ import { import { getInputOutputDataEdges } from '../../selectors/edges'; import { getChartSize, getChartZoom } from '../../selectors/layout'; import { getLayers } from '../../selectors/layers'; -import { getLinkedNodes, getFilteredNodes } from '../../selectors/linked-nodes'; +import { getLinkedNodes } from '../../selectors/linked-nodes'; import { getVisibleMetaSidebar } from '../../selectors/metadata'; import { drawNodes, drawEdges, drawLayers, drawLayerNames } from './draw'; import { @@ -706,8 +706,7 @@ export const mapStateToProps = (state, ownProps) => ({ hoveredFocusMode: state.hoveredFocusMode, layers: getLayers(state), linkedNodes: getLinkedNodes(state), - nodes: getFilteredNodes(state) || emptyNodes, - // nodes: state.graph.nodes || emptyNodes, + nodes: state.graph.nodes || emptyNodes, nodeTypeDisabled: state.nodeType.disabled, nodeActive: getNodeActive(state), nodeSelected: getNodeSelected(state), diff --git a/src/selectors/disabled.js b/src/selectors/disabled.js index aa0e8bba79..44cae1b15a 100644 --- a/src/selectors/disabled.js +++ b/src/selectors/disabled.js @@ -6,6 +6,7 @@ import { getModularPipelinesTree, } from './modular-pipelines'; import { getTagCount } from './tags'; +import { getFilteredPipeline } from './filtered-pipeline'; const getNodeIDs = (state) => state.node.ids; const getNodeDisabledNode = (state) => state.node.disabled; @@ -78,6 +79,7 @@ export const getNodeDisabled = createSelector( getVisibleSidebarNodes, getVisibleModularPipelineInputsOutputs, getDisabledModularPipeline, + getFilteredPipeline, ], ( nodeIDs, @@ -91,12 +93,18 @@ export const getNodeDisabled = createSelector( focusedModularPipeline, visibleSidebarNodes, visibleModularPipelineInputsOutputs, - disabledModularPipeline + disabledModularPipeline, + filteredPipeline ) => arrayToObject(nodeIDs, (id) => { let isDisabledViaModularPipeline = disabledModularPipeline[nodeModularPipelines[id]]; + let isDisabledViaFilters = false; + if (filteredPipeline.length > 0) { + isDisabledViaFilters = !filteredPipeline.includes(id); + } + const isDisabledViaSidebar = !visibleSidebarNodes[id] && !visibleModularPipelineInputsOutputs.has(id); @@ -126,6 +134,7 @@ export const getNodeDisabled = createSelector( isDisabledViaSidebar, isDisabledViaModularPipeline, isDisabledViaFocusedModularPipeline, + isDisabledViaFilters, ].some(Boolean); }) ); diff --git a/src/selectors/filtered-pipeline.js b/src/selectors/filtered-pipeline.js new file mode 100644 index 0000000000..600a89e7ca --- /dev/null +++ b/src/selectors/filtered-pipeline.js @@ -0,0 +1,107 @@ +import { createSelector } from 'reselect'; + +const getEdgeIDs = (state) => state.edge.ids; +const getEdgeSources = (state) => state.edge.sources; +const getEdgeTargets = (state) => state.edge.targets; +const getFromNodes = (state) => state.filters.from; +const getToNodes = (state) => state.filters.to; + +/** + * Selector to get all edges formatted as an array of objects with id, source, and target properties. + * @param {Object} state - The global state object. + * @returns {Array} An array of edge objects. + */ +const getEdges = createSelector( + [getEdgeIDs, getEdgeSources, getEdgeTargets], + (edgeIDs, edgeSources, edgeTargets) => + edgeIDs.map((id) => ({ + id, + source: edgeSources[id], + target: edgeTargets[id], + })) +); + +/** + * Selector to organize edges by their source and target nodes. + * @param {Array} edges - Array of edge objects. + * @returns {Object} An object containing edges mapped by source and target nodes. + */ + +export const getEdgesByNode = createSelector([getEdges], (edges) => { + const sourceEdges = {}; + const targetEdges = {}; + + for (const edge of edges) { + if (!sourceEdges[edge.target]) { + sourceEdges[edge.target] = []; + } + + sourceEdges[edge.target].push(edge.source); + + if (!targetEdges[edge.source]) { + targetEdges[edge.source] = []; + } + + targetEdges[edge.source].push(edge.target); + } + + return { sourceEdges, targetEdges }; +}); + +/** + * Recursive function to find all linked nodes starting from a given node ID. + * @param {string} nodeID - The starting node ID. + * @param {Object} edgesByNode - A map of node IDs to their connected node IDs. + * @param {Object} visited - A map to keep track of visited nodes. + * @returns {Object} A map of visited nodes. + */ + +const findLinkedNodes = (nodeID, edgesByNode, visited) => { + if (!visited[nodeID]) { + visited[nodeID] = true; + + if (edgesByNode[nodeID]) { + edgesByNode[nodeID].forEach((nodeID) => + findLinkedNodes(nodeID, edgesByNode, visited) + ); + } + } + + return visited; +}; + +/** + * Selector to filter nodes that are connected between two specified node IDs. + * @param {Object} edgesByNode - Edges organized by node IDs. + * @param {string} startID - Starting node ID. + * @param {string} endID - Ending node ID. + * @returns {Array} Array of node IDs that are connected from startID to endID. + */ + +export const getFilteredPipeline = createSelector( + [getEdgesByNode, getFromNodes, getToNodes], + ({ sourceEdges, targetEdges }, startID, endID) => { + let filteredNodeIDs = []; + + if ((!startID || !startID.length) && (!endID || !endID.length)) { + return filteredNodeIDs; + } else { + const linkedNodesBeforeEnd = {}; + findLinkedNodes(endID, sourceEdges, linkedNodesBeforeEnd); + + const linkedNodesAfterStart = {}; + findLinkedNodes(startID, targetEdges, linkedNodesAfterStart); + + const linkedNodesBetween = []; + for (const nodeID in linkedNodesBeforeEnd) { + if (linkedNodesAfterStart[nodeID]) { + linkedNodesBetween.push(nodeID); + } + } + + filteredNodeIDs = linkedNodesBetween; + + return filteredNodeIDs; + } + } +); diff --git a/src/selectors/linked-nodes.js b/src/selectors/linked-nodes.js index 633c036e81..7c0660f7e8 100644 --- a/src/selectors/linked-nodes.js +++ b/src/selectors/linked-nodes.js @@ -1,10 +1,7 @@ import { createSelector } from 'reselect'; import { getVisibleEdges } from './edges'; -import { getGraphNodes } from './nodes'; const getClickedNode = (state) => state.node.clicked; -const getFromNodes = (state) => state.filters.from; -const getToNodes = (state) => state.filters.to; /** * Gets a map of visible nodeIDs to successors nodeIDs in both directions * @param {Array} edges @@ -72,41 +69,6 @@ export const getLinkedNodes = createSelector( linkedNodes[nodeID] = false; findLinkedNodes(nodeID, targetEdges, linkedNodes); - console.log(linkedNodes); - return linkedNodes; } ); - -export const getFilteredNodes = createSelector( - [getVisibleEdgesByNode, getFromNodes, getToNodes, getGraphNodes], - ({ sourceEdges, targetEdges }, startID, endID, graphNodes) => { - // Initialize as an array this time - let filteredNodes = []; - - if ((!startID || !startID.length) && (!endID || !endID.length)) { - // If no startID or endID, return all graphNodes as an array - filteredNodes = Object.values(graphNodes); // Convert all graphNodes values to an array - } else { - const linkedNodesBeforeEnd = {}; - findLinkedNodes(endID, sourceEdges, linkedNodesBeforeEnd); - - const linkedNodesAfterStart = {}; - findLinkedNodes(startID, targetEdges, linkedNodesAfterStart); - - const linkedNodesBetween = []; - for (const nodeID in linkedNodesBeforeEnd) { - if (linkedNodesAfterStart[nodeID]) { - linkedNodesBetween.push(nodeID); - } - } - - // Populate filteredNodes array with nodes that are linked between start and end - filteredNodes = linkedNodesBetween - .map((nodeID) => graphNodes[nodeID]) - .filter((node) => node !== undefined); - } - - return filteredNodes; // This is now an array - } -); From ff20b165f35327eb7be67356370f1a3bf9753d66 Mon Sep 17 00:00:00 2001 From: rashidakanchwala Date: Tue, 23 Apr 2024 12:44:56 +0100 Subject: [PATCH 7/8] remove unwanted code --- demo-project/conf/base/catalog_01_raw.yml | 3 +-- demo-project/src/demo_project/settings.py | 1 - package/kedro_viz/integrations/kedro/data_loader.py | 2 +- package/kedro_viz/models/flowchart.py | 8 ++------ src/components/flowchart/draw.js | 11 ++++------- src/components/flowchart/flowchart.js | 11 +---------- 6 files changed, 9 insertions(+), 27 deletions(-) diff --git a/demo-project/conf/base/catalog_01_raw.yml b/demo-project/conf/base/catalog_01_raw.yml index f594c64d0d..e2f3f2b324 100644 --- a/demo-project/conf/base/catalog_01_raw.yml +++ b/demo-project/conf/base/catalog_01_raw.yml @@ -5,8 +5,7 @@ companies: kedro-viz: layer: raw preview_args: - nrows: 5 - + nrows: 5 reviews: type: pandas.CSVDataset diff --git a/demo-project/src/demo_project/settings.py b/demo-project/src/demo_project/settings.py index fefed89d7e..304f7d8c1a 100644 --- a/demo-project/src/demo_project/settings.py +++ b/demo-project/src/demo_project/settings.py @@ -11,7 +11,6 @@ SESSION_STORE_CLASS = SQLiteStore SESSION_STORE_ARGS = {"path": str(Path(__file__).parents[2] / "data")} - # Setup for collaborative experiment tracking. # SESSION_STORE_ARGS = {"path": str(Path(__file__).parents[2] / "data"), # "remote_path": "s3://{path-to-session_store}" } diff --git a/package/kedro_viz/integrations/kedro/data_loader.py b/package/kedro_viz/integrations/kedro/data_loader.py index f83d395bba..dccf95d8af 100644 --- a/package/kedro_viz/integrations/kedro/data_loader.py +++ b/package/kedro_viz/integrations/kedro/data_loader.py @@ -107,7 +107,7 @@ def load_data( # Pipelines is a lazy dict-like object, so we force it to populate here # in case user doesn't have an active session down the line when it's first accessed. # Useful for users who have `get_current_session` in their `register_pipelines()`. - stats_dict = _get_dataset_stats(project_path) pipelines_dict = dict(pipelines) + stats_dict = _get_dataset_stats(project_path) return catalog, pipelines_dict, session_store, stats_dict diff --git a/package/kedro_viz/models/flowchart.py b/package/kedro_viz/models/flowchart.py index 56e8382115..c7e0b4aed7 100644 --- a/package/kedro_viz/models/flowchart.py +++ b/package/kedro_viz/models/flowchart.py @@ -425,18 +425,14 @@ def inputs(self) -> Set[str]: Intuitively, the set of inputs for this modular pipeline is the set of all external and internal inputs, excluding the ones also serving as outputs. """ - return (self.external_inputs | self.internal_inputs) - ( - self.external_outputs | self.internal_outputs - ) + return (self.external_inputs | self.internal_inputs) - self.internal_outputs @property def outputs(self) -> Set[str]: """Return a set of inputs for this modular pipeline. Follow the same logic as the inputs calculation. """ - return (self.external_outputs | self.internal_outputs) - ( - self.external_inputs | self.internal_inputs - ) + return self.external_outputs | (self.internal_outputs - self.internal_inputs) class TaskNodeMetadata(GraphNodeMetadata): diff --git a/src/components/flowchart/draw.js b/src/components/flowchart/draw.js index dfa2e24f99..d221bf4d29 100644 --- a/src/components/flowchart/draw.js +++ b/src/components/flowchart/draw.js @@ -137,10 +137,8 @@ export const drawNodes = function (changed) { hoveredFocusMode, } = this.props; - const isInputOutputNode = (nodeID) => { - const result = focusMode !== null && inputOutputDataNodes[nodeID]; - return result; - }; + const isInputOutputNode = (nodeID) => + focusMode !== null && inputOutputDataNodes[nodeID]; if (changed('nodes')) { this.el.nodes = this.el.nodeGroup @@ -237,8 +235,7 @@ export const drawNodes = function (changed) { 'clickedNode', 'linkedNodes', 'focusMode', - 'inputOutputDataNodes', - 'slicedPipeline' + 'inputOutputDataNodes' ) ) { allNodes @@ -359,7 +356,7 @@ export const drawEdges = function (changed) { .merge(exitEdges) .filter((edge) => edge); - if (changed('edges', 'focusMode', 'inputOutputDataNodes', 'slicedPipeline')) { + if (changed('edges', 'focusMode', 'inputOutputDataNodes')) { enterEdges.append('path'); allEdges .select('path') diff --git a/src/components/flowchart/flowchart.js b/src/components/flowchart/flowchart.js index a6a8d6aa18..1590a7823e 100644 --- a/src/components/flowchart/flowchart.js +++ b/src/components/flowchart/flowchart.js @@ -135,16 +135,7 @@ export class FlowChart extends Component { drawNodes.call(this, changed); } - if ( - changed( - 'edges', - 'nodes', - 'layers', - 'chartSize', - 'clickedNode', - 'linkedNodes' - ) - ) { + if (changed('edges', 'nodes', 'layers', 'chartSize', 'clickedNode')) { // Don't zoom out when the metadata or code panels are opened or closed if (prevProps.visibleMetaSidebar !== this.props.visibleMetaSidebar) { drawNodes.call(this, changed); From 5b824bbe0a4525a1d88cdf66c902d135a807163c Mon Sep 17 00:00:00 2001 From: huongg Date: Thu, 2 May 2024 11:30:25 +0100 Subject: [PATCH 8/8] to slice nodes from the startID and any nodes before endID Signed-off-by: huongg --- src/selectors/filtered-pipeline.js | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/selectors/filtered-pipeline.js b/src/selectors/filtered-pipeline.js index 600a89e7ca..78c7ceea58 100644 --- a/src/selectors/filtered-pipeline.js +++ b/src/selectors/filtered-pipeline.js @@ -86,20 +86,24 @@ export const getFilteredPipeline = createSelector( if ((!startID || !startID.length) && (!endID || !endID.length)) { return filteredNodeIDs; } else { + const linkedNodesBetween = []; const linkedNodesBeforeEnd = {}; findLinkedNodes(endID, sourceEdges, linkedNodesBeforeEnd); + const linkedNodeBeforeStart = {}; + findLinkedNodes(startID, sourceEdges, linkedNodeBeforeStart); - const linkedNodesAfterStart = {}; - findLinkedNodes(startID, targetEdges, linkedNodesAfterStart); + // keep any nodes before the endID + filteredNodeIDs = linkedNodesBetween.concat( + Object.keys(linkedNodesBeforeEnd) + ); - const linkedNodesBetween = []; - for (const nodeID in linkedNodesBeforeEnd) { - if (linkedNodesAfterStart[nodeID]) { - linkedNodesBetween.push(nodeID); + // remove any nodes before startID + Object.keys(linkedNodeBeforeStart).map((node) => { + if (node !== startID) { + const index = filteredNodeIDs.indexOf(node); + filteredNodeIDs.splice(index, 1); } - } - - filteredNodeIDs = linkedNodesBetween; + }); return filteredNodeIDs; }