From 89c8f7a9d0c5190690e8251006cd715c6eb70f79 Mon Sep 17 00:00:00 2001 From: JacobiClark Date: Thu, 16 Jan 2025 10:04:57 -0800 Subject: [PATCH] wip: Initiated work on entity extraction process --- .../native-ui/dialogs/open-file.js | 22 +++- .../DatasetTreeViewRenderer/ContextMenu.jsx | 17 ++- .../shared/DatasetTreeViewRenderer/index.jsx | 124 ++++++++++++------ src/renderer/src/scripts/others/renderer.js | 32 +++-- .../src/stores/slices/datasetTreeViewSlice.js | 106 ++++++++------- .../src/stores/utils/folderAndFileActions.js | 24 ++++ 6 files changed, 215 insertions(+), 110 deletions(-) diff --git a/src/main/main-process/native-ui/dialogs/open-file.js b/src/main/main-process/native-ui/dialogs/open-file.js index 369312069..3ab655af6 100644 --- a/src/main/main-process/native-ui/dialogs/open-file.js +++ b/src/main/main-process/native-ui/dialogs/open-file.js @@ -593,17 +593,31 @@ ipcMain.on("open-files-organize-datasets-dialog", async (event) => { } }); -ipcMain.on("open-folders-organize-datasets-dialog", async (event) => { +ipcMain.on("open-folders-organize-datasets-dialog", async (event, args) => { + if (!args?.importRelativePath) { + throw new Error("The 'importRelativePath' property is required but was not provided."); + } + + console.log("args.importRelativePath:", args.importRelativePath); + let mainWindow = BrowserWindow.getFocusedWindow(); + const importRelativePath = args.importRelativePath; + let folders = await dialog.showOpenDialog(mainWindow, { properties: ["openDirectory", "multiSelections"], - title: "Import a folder", + title: `Import a folder to ${importRelativePath}`, }); - if (folders) { - mainWindow.webContents.send("selected-folders-organize-datasets", folders.filePaths); + if (folders.canceled) { + return; // Exit if the dialog is canceled } + + // Send the selected folders and the relative path back to the renderer + mainWindow.webContents.send("selected-folders-organize-datasets", { + filePaths: folders.filePaths, + importRelativePath, + }); }); // Generate manifest file locally diff --git a/src/renderer/src/components/shared/DatasetTreeViewRenderer/ContextMenu.jsx b/src/renderer/src/components/shared/DatasetTreeViewRenderer/ContextMenu.jsx index f178c4d48..57422cb41 100644 --- a/src/renderer/src/components/shared/DatasetTreeViewRenderer/ContextMenu.jsx +++ b/src/renderer/src/components/shared/DatasetTreeViewRenderer/ContextMenu.jsx @@ -68,13 +68,13 @@ const ContextMenu = () => {
- + {contextMenuItemType === "folder" ? ( ) : ( )} - + {contextMenuItemName} @@ -85,7 +85,7 @@ const ContextMenu = () => { closeContextMenu(); }} > - Move + Move {contextMenuItemType} { @@ -98,10 +98,17 @@ const ContextMenu = () => { closeContextMenu(); }} > - Delete + Delete {contextMenuItemType} {contextMenuItemType === "folder" && ( - console.log("Import data")}> + { + e.preventDefault(); + window.electron.ipcRenderer.send("open-folders-organize-datasets-dialog", { + importRelativePath: contextMenuItemData.relativePath, + }); + }} + > Import data into {contextMenuItemName} )} diff --git a/src/renderer/src/components/shared/DatasetTreeViewRenderer/index.jsx b/src/renderer/src/components/shared/DatasetTreeViewRenderer/index.jsx index f32066ee1..2fc9043fd 100644 --- a/src/renderer/src/components/shared/DatasetTreeViewRenderer/index.jsx +++ b/src/renderer/src/components/shared/DatasetTreeViewRenderer/index.jsx @@ -11,6 +11,7 @@ import { Space, Center, Button, + Loader, } from "@mantine/core"; import { useHover } from "@mantine/hooks"; import { IconFolder, IconFolderOpen, IconFile, IconSearch } from "@tabler/icons-react"; @@ -22,7 +23,10 @@ import { setFolderMoveMode, moveFolderToNewLocation, } from "../../../stores/slices/datasetTreeViewSlice"; -import { moveFoldersToTargetLocation } from "../../../stores/utils/folderAndFileActions"; +import { + moveFoldersToTargetLocation, + moveFilesToTargetLocation, +} from "../../../stores/utils/folderAndFileActions"; import { naturalSort } from "../utils/util-functions"; const ICON_SETTINGS = { @@ -111,8 +115,6 @@ const FolderItem = ({ !content || (Object.keys(content.folders).length === 0 && Object.keys(content.files).length === 0); - console.log("content", content); - const folderIsPassThrough = content.folderIsPassThrough === true; if (folderIsPassThrough) { console.log("Folder is pass through", name); @@ -150,10 +152,16 @@ const FolderItem = ({ readOnly disabled={content.relativePath.includes(contextMenuItemData.relativePath)} onClick={() => { - moveFoldersToTargetLocation( - [contextMenuItemData.relativePath], - content.relativePath - ); + contextMenuItemType === "folder" + ? moveFoldersToTargetLocation( + [contextMenuItemData.relativePath], + content.relativePath + ) + : moveFilesToTargetLocation( + [contextMenuItemData.relativePath], + content.relativePath + ); + setFolderMoveMode(false); }} /> @@ -211,15 +219,41 @@ const DatasetTreeViewRenderer = ({ folderActions, fileActions, allowStructureEdi const { renderDatasetStructureJSONObj, datasetStructureSearchFilter, folderMoveModeIsActive } = useGlobalStore(); + const searcDebounceTime = 300; // Set the debounce time for the search filter (in milliseconds) + + const [inputSearchFilter, setInputSearchFilter] = useState(datasetStructureSearchFilter); // Local state for input + const searchTimeoutRef = useRef(null); + + // Debounce the search filter change + const handleSearchChange = (event) => { + const value = event.target.value; + setInputSearchFilter(value); // Update the input immediately + + // Clear the previous timeout if there is one + if (searchTimeoutRef.current) { + clearTimeout(searchTimeoutRef.current); + } + + // Set a new timeout to update the global state + searchTimeoutRef.current = setTimeout(() => { + setDatasetStructureSearchFilter(value); // Update the global state after the debounce delay + }, searcDebounceTime); + }; + + useEffect(() => { + return () => { + // Cleanup the timeout on component unmount + if (searchTimeoutRef.current) { + clearTimeout(searchTimeoutRef.current); + } + }; + }, []); + const handleMenuClose = () => { console.log("Closing context menu"); setMenuOpened(false); }; - const handleSearchChange = (event) => { - setDatasetStructureSearchFilter(event.target.value); - }; - const handleAllFilesSelectClick = () => { Object.keys(renderDatasetStructureJSONObj.files).forEach((fileName) => fileActions["on-file-click"](fileName, renderDatasetStructureJSONObj.files[fileName]) @@ -236,9 +270,7 @@ const DatasetTreeViewRenderer = ({ folderActions, fileActions, allowStructureEdi }; const handleDeleteAllItemsClick = () => { - console.log( - 'Deleting all items containing "' + datasetStructureSearchFilter + '" in their name' - ); + console.log('Deleting all items containing "' + inputSearchFilter + '" in their name'); }; const renderObjIsEmpty = @@ -246,12 +278,14 @@ const DatasetTreeViewRenderer = ({ folderActions, fileActions, allowStructureEdi (Object.keys(renderDatasetStructureJSONObj?.folders).length === 0 && Object.keys(renderDatasetStructureJSONObj?.files).length === 0); + const searchDebounceIsActive = datasetStructureSearchFilter !== inputSearchFilter; + return ( } mb="sm" @@ -306,32 +340,40 @@ const DatasetTreeViewRenderer = ({ folderActions, fileActions, allowStructureEdi ) : ( <> - {naturalSort(Object.keys(renderDatasetStructureJSONObj?.folders || {})).map( - (folderName) => ( - - ) - )} - {naturalSort(Object.keys(renderDatasetStructureJSONObj?.files || {})).map( - (fileName) => ( - - ) + {searchDebounceIsActive ? ( +
+ +
+ ) : ( + <> + {naturalSort(Object.keys(renderDatasetStructureJSONObj?.folders || {})).map( + (folderName) => ( + + ) + )} + {naturalSort(Object.keys(renderDatasetStructureJSONObj?.files || {})).map( + (fileName) => ( + + ) + )} + )} )} diff --git a/src/renderer/src/scripts/others/renderer.js b/src/renderer/src/scripts/others/renderer.js index ee2b36cb1..821b9ad3b 100644 --- a/src/renderer/src/scripts/others/renderer.js +++ b/src/renderer/src/scripts/others/renderer.js @@ -85,6 +85,7 @@ import { blockedMessage, hostFirewallMessage, } from "../check-firewall/checkFirewall"; +import { setDatasetEntityObj } from "../../stores/slices/datasetEntitySelectorSlice"; // add jquery to the window object window.$ = jQuery; @@ -4163,41 +4164,52 @@ organizeDSaddFolders.addEventListener("click", function () { // Event listener for when folder(s) are imported into the file explorer window.electron.ipcRenderer.on( "selected-folders-organize-datasets", - async (event, importedFolders) => { + async (event, { filePaths: importedFolders, importRelativePath }) => { try { - const currentFileExplorerPath = window.organizeDSglobalPath.value.trim(); - console.log("Importing folders at path", currentFileExplorerPath); + if (!importRelativePath) { + throw new Error("The 'importRelativePath' property is missing in the response."); + } + + // Use the current file explorer path or the provided relative path + const currentFileExplorerPath = `dataset_root/${importRelativePath}`; + + console.log("currentFileExplorerPath we are merging into at", currentFileExplorerPath); + const builtDatasetStructureFromImportedFolders = await window.buildDatasetStructureJsonFromImportedData( importedFolders, currentFileExplorerPath ); + console.log( "builtDatasetStructureFromImportedFolders after importing data", builtDatasetStructureFromImportedFolders ); + // Add the imported folders to the dataset structure await mergeLocalAndRemoteDatasetStructure( builtDatasetStructureFromImportedFolders, currentFileExplorerPath ); - // Step 4: Update successful, show success message + // Show success message window.notyf.open({ type: "success", message: `Data successfully imported`, duration: 3000, }); - /*await mergeNewDatasetStructureToExistingDatasetStructureAtPath( - builtDatasetStructureFromImportedFolders, - currentFileExplorerPath - );*/ } catch (error) { console.error("Error importing folders", error); + + // Optionally show an error notification + window.notyf.open({ + type: "error", + message: `Error importing data: ${error.message}`, + duration: 3000, + }); } } ); - /* ################################################################################## */ /* ################################################################################## */ /* ################################################################################## */ @@ -4801,7 +4813,7 @@ const mergeLocalAndRemoteDatasetStructure = async ( ); console.log("currentPathArray", currentPathArray.slice(1)); - setTreeViewDatasetStructure(window.datasetStructureJSONObj, currentPathArray.slice(1)); + setTreeViewDatasetStructure(window.datasetStructureJSONObj, ["primary"]); }; const mergeNewDatasetStructureToExistingDatasetStructureAtPath = async ( diff --git a/src/renderer/src/stores/slices/datasetTreeViewSlice.js b/src/renderer/src/stores/slices/datasetTreeViewSlice.js index 9774b04c8..a5f96bcb1 100644 --- a/src/renderer/src/stores/slices/datasetTreeViewSlice.js +++ b/src/renderer/src/stores/slices/datasetTreeViewSlice.js @@ -31,64 +31,68 @@ const traverseStructureByPath = (structure, pathToRender) => { }; // Determines if a folder or its subfolders/files match the search filter -const folderObjMatchesSearch = (folderObj, searchFilter) => { - if (!searchFilter) { - return { matchesDirectly: true, matchesFilesDirectly: true, passThrough: false }; - } - - const folderRelativePath = folderObj.relativePath.toLowerCase(); +const folderMatchesSearch = (folder, searchFilter) => { + const folderRelativePath = folder.relativePath.toLowerCase(); const matchesDirectly = folderRelativePath.includes(searchFilter); - const matchesFilesDirectly = Object.values(folderObj.files || {}).some((file) => + const matchesFilesDirectly = Object.values(folder.files || {}).some((file) => file.relativePath.toLowerCase().includes(searchFilter) ); - const subfolderMatches = Object.values(folderObj.folders || {}).some((subFolder) => { - const result = folderObjMatchesSearch(subFolder, searchFilter); - return result.matchesDirectly || result.matchesFilesDirectly || result.passThrough; - }); - const passThrough = !matchesDirectly && !matchesFilesDirectly && subfolderMatches; + const subfolderMatches = Object.values(folder.folders || {}).some((subFolder) => + folderMatchesSearch(subFolder, searchFilter) + ); - return { matchesDirectly, matchesFilesDirectly, passThrough }; + return { matchesDirectly, matchesFilesDirectly, subfolderMatches }; }; -// Filters the dataset structure based on the current search filter -const filterStructure = (structure, searchFilter) => { - if (!searchFilter) return structure; +// Determines if a folder should be kept or pruned based on search filter +const pruneFolder = (folder, searchFilter) => { + // Check if the folder itself or its contents match the search filter + const { matchesDirectly, matchesFilesDirectly, subfolderMatches } = folderMatchesSearch( + folder, + searchFilter + ); + + // If nothing matches and the folder has no subfolders or files to pass through, return null + if (!matchesDirectly && !matchesFilesDirectly && !subfolderMatches) return null; + + // Mark if the folder should pass through (i.e., it has matching subfolders or files but itself does not match) + const passThrough = !matchesDirectly && !matchesFilesDirectly && subfolderMatches; - const pruneStructure = (folderObj, searchFilter) => { - const { matchesDirectly, matchesFilesDirectly, passThrough } = folderObjMatchesSearch( - folderObj, - searchFilter - ); - - // If the folder does not match and is not a pass-through, return null (exclude it) - if (!matchesDirectly && !matchesFilesDirectly && !passThrough) return null; - - // If the folder is pass-through, it should still be included in the structure - return produce(folderObj, (draft) => { - // If it's a pass-through, we include it even if it doesn't match directly - if (passThrough) { - console.log("Pass-through folder:", draft.relativePath); // Debug log - draft.folderIsPassThrough = true; // Mark it as pass-through in the JSON - } - - // Recursively prune subfolders that don't match - for (const subFolder in draft.folders || {}) { - if (!pruneStructure(draft.folders[subFolder], searchFilter)) { - delete draft.folders[subFolder]; - } - } - - // Remove files that don't match the filter - for (const fileName in draft.files || {}) { - if (!draft.files[fileName].relativePath.toLowerCase().includes(searchFilter)) { - delete draft.files[fileName]; - } - } - }); + // Keep the folder and its properties + const prunedFolder = { + ...folder, + matchesDirectly, + matchesFilesDirectly, + passThrough, }; - // Start the filtering process - return pruneStructure(structure, searchFilter.toLowerCase()); + // Prune subfolders recursively + if (prunedFolder.folders) { + prunedFolder.folders = Object.entries(prunedFolder.folders) + .map(([key, subFolder]) => { + const prunedSubFolder = pruneFolder(subFolder, searchFilter); + if (!prunedSubFolder) return null; + return { [key]: prunedSubFolder }; + }) + .filter(Boolean) + .reduce((acc, curr) => ({ ...acc, ...curr }), {}); + } + + // Prune files based on the search filter + if (prunedFolder.files) { + prunedFolder.files = Object.entries(prunedFolder.files) + .filter(([_, file]) => file.relativePath.toLowerCase().includes(searchFilter)) + .reduce((acc, [key, file]) => ({ ...acc, [key]: file }), {}); + } + + return prunedFolder; +}; + +// Filter structure based on search filter +const filterStructure = (structure, searchFilter) => { + if (!searchFilter) return structure; + const lowerCaseSearchFilter = searchFilter.toLowerCase(); + return pruneFolder(structure, lowerCaseSearchFilter); }; // Updates the dataset search filter and modifies the rendered structure @@ -96,7 +100,9 @@ export const setDatasetStructureSearchFilter = (searchFilter) => { const globalStore = useGlobalStore.getState(); const originalStructure = globalStore.datasetStructureJSONObj; - const structureToFilter = traverseStructureByPath(originalStructure, globalStore.pathToRender); + let structureToFilter = traverseStructureByPath(originalStructure, globalStore.pathToRender); + structureToFilter = JSON.parse(JSON.stringify(structureToFilter)); // Avoid proxy-related issues + const filteredStructure = filterStructure(structureToFilter, searchFilter); useGlobalStore.setState({ diff --git a/src/renderer/src/stores/utils/folderAndFileActions.js b/src/renderer/src/stores/utils/folderAndFileActions.js index 9335d8d4b..e6bbaaf13 100644 --- a/src/renderer/src/stores/utils/folderAndFileActions.js +++ b/src/renderer/src/stores/utils/folderAndFileActions.js @@ -95,3 +95,27 @@ export const moveFoldersToTargetLocation = ( // Update the tree view structure to reflect the changes. setTreeViewDatasetStructure(window.datasetStructureJSONObj); }; + +export const moveFilesToTargetLocation = (arrayOfRelativePathsToMove, destionationRelativePath) => { + const { + parentFolder: destinationParentFolder, + itemName: destinationItemName, + itemObject: destinationItemObject, + } = getItemAtPath(destionationRelativePath, "folder"); + + for (const relativePathToMove of arrayOfRelativePathsToMove) { + const { parentFolder, itemName, itemObject } = getItemAtPath(relativePathToMove, "file"); + console.log("parentFolder for file", parentFolder); + console.log("itemName for file", itemName); + console.log("itemObject for file", itemObject); + + // Move the file to the destination folder. + destinationItemObject["files"][itemName] = itemObject; + + // Remove the file from its original location. + delete parentFolder["files"][itemName]; + } + + // Update the tree view structure to reflect the changes. + setTreeViewDatasetStructure(window.datasetStructureJSONObj); +};