DYN-6901 Pm - retainfolderstructure rework

src/DynamoCoreWpf/ViewModels/PackageManager/PublishPackageViewModel.cs

@@ -895,8 +895,9 @@ public bool RetainFolderStructureOverride
         /// <summary>
         /// The root directory of the package
         /// </summary>
-        private string CurrentPackageDirectory { get; set; }
+        private IEnumerable<string> CurrentPackageRootDirectories { get; set; }
         private static MetadataLoadContext sharedMetaDataLoadContext = null;
+
         /// <summary>
         /// A shared MetaDataLoadContext that is used for assembly inspection during package publishing.
         /// This member is shared so the behavior is similar to the ReflectionOnlyLoadContext this is replacing.
@@ -1073,34 +1074,30 @@ internal List<PackageItemRootViewModel> BindParentToChild(Dictionary<string, Pac
             }
 
             // Only add the folder items, they contain the files
-            var updatedItems = GetRootItems(items);
+            var updatedItems = OrganizePackageRootItems(items);
             return updatedItems;
         }
 
         /// <summary>
-        /// Gets the list PackageItemRootViewModel items which will be at the root directory of the package with all the child items.
+        /// Organizes package items into root items based on common paths and hierarchical structure.
+        /// This includes determining root items, establishing parent-child relationships, and collecting all child items.
         /// </summary>
-        /// <param name="items"></param>
-        /// <returns></returns>
-        private List<PackageItemRootViewModel> GetRootItems(Dictionary<string, PackageItemRootViewModel> items)
+        /// <param name="items">A dictionary of package item keys and their corresponding PackageItemRootViewModel objects.</param>
+        /// <returns>A list of PackageItemRootViewModel items representing the organized root items and their child items.</returns>
+        private List<PackageItemRootViewModel> OrganizePackageRootItems(Dictionary<string, PackageItemRootViewModel> items)
         {
             var rootItems = items.Values.Where(x => !x.isChild).ToList();
             if (!rootItems.Any()) return rootItems;
 
             var roots = new List<PackageItemRootViewModel>();
+
+            var commonPaths = GetCommonPaths(items.Keys.ToArray());
+            if (commonPaths == null) return null;
 
-            if (CurrentPackageDirectory != null)
-            {
-                roots.Add(new PackageItemRootViewModel(CurrentPackageDirectory));
-            }
-            else
-            {
-                var commonPaths = GetCommonPaths(items.Keys.ToArray());
-                if (commonPaths == null) return null;
+            CurrentPackageRootDirectories = commonPaths;
 
-                // Add a new root item for each common path found
-                commonPaths.ForEach(p => roots.Add(new PackageItemRootViewModel(p)));
-            }
+            // Add a new root item for each common path found
+            commonPaths.ForEach(p => roots.Add(new PackageItemRootViewModel(p)));
 
             // Check each root item and create any missing connections
             foreach (var item in rootItems)
@@ -1532,7 +1529,7 @@ internal static PublishPackageViewModel FromLocalPackage(DynamoViewModel dynamoV
                 CopyrightHolder = pkg.CopyrightHolder,
                 CopyrightYear = pkg.CopyrightYear,
                 IsPublishFromLocalPackage = true,
-                CurrentPackageDirectory = pkg.RootDirectory,
+                CurrentPackageRootDirectories = new List<string> { pkg.RootDirectory },
                 //default retain folder structure to true when publishing a new version from local.
                 RetainFolderStructureOverride = retainFolderStructure
             };
@@ -2270,7 +2267,7 @@ private void Submit()
             {
                 // begin submission
                 var pmExtension = dynamoViewModel.Model.GetPackageManagerExtension();
-                var handle = pmExtension.PackageManagerClient.PublishAsync(Package, RetainFolderStructureOverride ? updatedFiles : contentFiles, MarkdownFiles, IsNewVersion, RetainFolderStructureOverride);
+                var handle = pmExtension.PackageManagerClient.PublishAsync(Package, RetainFolderStructureOverride ? updatedFiles : contentFiles, MarkdownFiles, IsNewVersion, CurrentPackageRootDirectories, RetainFolderStructureOverride);
 
                 // start upload
                 Uploading = true;
@@ -2365,11 +2362,13 @@ private void PublishLocally()
                     var remapper = new CustomNodePathRemapper(DynamoViewModel.Model.CustomNodeManager,
                         DynamoModel.IsTestMode);
                     var builder = new PackageDirectoryBuilder(new MutatingFileSystem(), remapper);
-                    if (string.IsNullOrEmpty(Package.RootDirectory))
-                    {
-                        Package.RootDirectory = CurrentPackageDirectory;
-                    }
-                    builder.BuildRetainDirectory(Package, publishPath, updatedFiles, MarkdownFiles);
+
+                    //if (string.IsNullOrEmpty(Package.RootDirectory))
+                    //{
+                    //    Package.RootDirectory = CurrentPackageRootDirectories;
+                    //}
+
+                    builder.BuildRetainDirectory(Package, publishPath, CurrentPackageRootDirectories, updatedFiles, MarkdownFiles);
                     UploadState = PackageUploadHandle.State.Uploaded;
                 }
                 else
@@ -2770,11 +2769,11 @@ internal PackageItemRootViewModel GetExistingRootItemViewModel(string publishPat
         {
             if (!PackageContents.Any()) return null;
             if (PackageContents.Count(x => x.DependencyType.Equals(DependencyType.Folder)) == 1) {
-                // If there is only one root item, nest it under the new root package folder
+                // If there is only one root item, this root item becomes the new folder
                 var item = PackageContents.First(x => x.DependencyType.Equals(DependencyType.Folder));
 
                 item = new PackageItemRootViewModel(Path.Combine(publishPath, packageName));
-                item.AddChildren(new List<PackageItemRootViewModel> { PackageContents.First() } );
+                item.AddChildren( PackageContents.First().ChildItems.ToList() );
 
                 return item;
             }
@@ -2783,6 +2782,9 @@ internal PackageItemRootViewModel GetExistingRootItemViewModel(string publishPat
             var rootItem = new PackageItemRootViewModel(Path.Combine(publishPath, packageName));
             foreach(var item in PackageContents)
             {
+                // Skip 'bare' custom nodes, they will be represented by their CustomNodePreview counterparts
+                if(item.DependencyType.Equals(DependencyType.CustomNode)) { continue; }
+
                 item.isChild = true;
                 rootItem.AddChildren(item);
             }

src/DynamoPackages/PackageDirectoryBuilder.cs

@@ -11,7 +11,7 @@ namespace Dynamo.PackageManager
     public interface IPackageDirectoryBuilder
     {
         IDirectoryInfo BuildDirectory(Package packages, string packagesDirectory, IEnumerable<string> files, IEnumerable<string> markdownfiles);
-        IDirectoryInfo BuildRetainDirectory(Package package, string packagesDirectory, IEnumerable<IEnumerable<string>> contentFiles, IEnumerable<string> markdownFiles);
+        IDirectoryInfo BuildRetainDirectory(Package package, string packagesDirectory, IEnumerable<string> roots, IEnumerable<IEnumerable<string>> contentFiles, IEnumerable<string> markdownFiles);
     }
 
     /// <summary>
@@ -69,24 +69,24 @@ public IDirectoryInfo BuildDirectory(Package package, string packagesDirectory,
         /// </summary>
         /// <param name="package">The package to be formed</param>
         /// <param name="packagesDirectory">The parent directory (the published folder or the default packages directory)</param>
+        /// <param name="roots">All possible root folders for this collection of contentFiles</param>
         /// <param name="contentFiles">The collection of files to be moved</param>
         /// <param name="markdownFiles">Separately provided markdown files</param>
         /// <returns></returns>
-        public IDirectoryInfo BuildRetainDirectory(Package package, string packagesDirectory, IEnumerable<IEnumerable<string>> contentFiles, IEnumerable<string> markdownFiles)
+        public IDirectoryInfo BuildRetainDirectory(Package package, string packagesDirectory, IEnumerable<string> roots, IEnumerable<IEnumerable<string>> contentFiles, IEnumerable<string> markdownFiles)
         {
 
             var rootPath = Path.Combine(packagesDirectory, package.Name);
             var rootDir = fileSystem.TryCreateDirectory(rootPath);
-            var sourcePackageDir = package.RootDirectory;
             package.RootDirectory = rootDir.FullName;
 
             var dyfFiles = new List<string>();
 
             RemoveUnselectedFiles(contentFiles.SelectMany(files => files).ToList(), rootDir);
-            CopyFilesIntoRetainedPackageDirectory(contentFiles, markdownFiles, sourcePackageDir, rootDir, out dyfFiles);
+            CopyFilesIntoRetainedPackageDirectory(contentFiles, markdownFiles, roots, rootDir, out dyfFiles);
             RemoveRetainDyfFiles(contentFiles.SelectMany(files => files).ToList(), dyfFiles);  
-
             RemapRetainCustomNodeFilePaths(contentFiles.SelectMany(files => files).ToList(), dyfFiles);
+
             WritePackageHeader(package, rootDir);
 
             return rootDir;
@@ -114,8 +114,7 @@ private void RemapRetainCustomNodeFilePaths(IEnumerable<string> filePaths, List<
         {
             foreach (var func in filePaths.Where(x => x.EndsWith(".dyf")))
             {
-                //var remapLocation = dyfFiles.First(x => Path.GetDirectoryName(x).Equals(Path.GetDirectoryName(func)));
-                var remapLocation = dyfFiles.First(x =>
+                var remapLocation = dyfFiles.FirstOrDefault(x =>
                 {
                     var p1 = Path.GetFileName(Path.GetDirectoryName(x));
                     var f1 = Path.GetFileName(x);
@@ -127,7 +126,15 @@ private void RemapRetainCustomNodeFilePaths(IEnumerable<string> filePaths, List<
 
                     return r1.Equals(r2);
                 });
-                pathRemapper.SetPath(func, remapLocation);
+
+                // If no full path match is found, try to match based on filename only
+                if (remapLocation == null)
+                {
+                    remapLocation = dyfFiles.FirstOrDefault(x =>
+                        Path.GetFileName(x).Equals(Path.GetFileName(func), StringComparison.OrdinalIgnoreCase));
+                }
+
+                pathRemapper.SetPath(func, remapLocation);                
             }
         }
 
@@ -221,10 +228,25 @@ private void WritePackageHeader(Package package, IDirectoryInfo rootDir)
             fileSystem.WriteAllText(headerPath, pkgHeaderStr);
         }
 
-        internal void CopyFilesIntoRetainedPackageDirectory(IEnumerable<IEnumerable<string>> contentFiles, IEnumerable<string> markdownFiles, string sourcePackageDir, IDirectoryInfo rootDir, out List<string> dyfFiles)
+        internal void CopyFilesIntoRetainedPackageDirectory(IEnumerable<IEnumerable<string>> contentFiles,
+                                                            IEnumerable<string> markdownFiles,
+                                                            IEnumerable<string> roots,
+                                                            IDirectoryInfo rootDir,
+                                                            out List<string> dyfFiles)
         {
             dyfFiles = new List<string>();
 
+            // Normalize roots to ensure consistent comparison
+            var normalizedRoots = roots.Select(r => Path.GetFullPath(r)).ToList();
+
+            // Determine if all files are under a single folder
+            var distinctFolders = contentFiles.SelectMany(f => f)
+                                              .Where(f => f != null)
+                                              .Select(f => f.Substring(normalizedRoots.First().Length).TrimStart(new char[] { '/', '\\' }))
+                                              .Select(rp => rp.Split(new char[] { '/', '\\' })[0])
+                                              .Distinct()
+                                              .Count();
+
             foreach (var files in contentFiles)
             {
                 foreach (var file in files.Where(x => x != null))
@@ -235,24 +257,34 @@ internal void CopyFilesIntoRetainedPackageDirectory(IEnumerable<IEnumerable<stri
                         continue;
                     }
 
-                    // TODO: This will be properly fixed in the next PR
-                    var relativePath = sourcePackageDir != null ? file.Substring(sourcePackageDir.Length) : Path.GetFileName(file);
+                    string relativePath = "";
+
+                    foreach (var root in normalizedRoots)
+                    {
+                        var normalizedFile = Path.GetFullPath(file);
+                        if (normalizedFile.StartsWith(root, StringComparison.OrdinalIgnoreCase))
+                        {
+                            relativePath = normalizedFile.Substring(root.Length);
+                            // Trim leading directory separators
+                            relativePath = relativePath.TrimStart(new char[] { '/', '\\' });
+                        }
+                    }
+
+                    // If we have more than 1 root, then we need to nest into a new root folder
+                    // If we don't, and in order to preserve 1-to-1 folder structure, we remove the original root and replace with the package name
+                    if (normalizedRoots.Count() == 1 && distinctFolders == 1)
+                    {
+                        relativePath = RemoveFirstFolder(relativePath);
+                    }
 
                     // Ensure the relative path starts with a directory separator.
                     if (!string.IsNullOrEmpty(relativePath) && relativePath[0] != Path.DirectorySeparatorChar)
                     {
-                        relativePath = relativePath.TrimStart(new char[] { '/', '\\' });
                         relativePath = Path.DirectorySeparatorChar + relativePath;
                     }
 
                     var destPath = Path.Combine(rootDir.FullName, relativePath.TrimStart('\\'));
 
-                    // We are already creating the pkg.json file ourselves, so skip it, also skip if we are copying the file to itself.
-                    if (destPath.Equals(Path.Combine(rootDir.FullName, "pkg.json")) || destPath.Equals(file))
-                    {
-                        continue;
-                    }
-
                     if (fileSystem.FileExists(destPath))
                     {
                         fileSystem.DeleteFile(destPath);
@@ -271,6 +303,9 @@ internal void CopyFilesIntoRetainedPackageDirectory(IEnumerable<IEnumerable<stri
                     }
                 }
             }
+
+
+
             // All files under Markdown directory do not apply to the rule above,
             // because they may fall into extra folder instead of docs folder,
             // currently there is on obvious way to filter them properly only based on path string.
@@ -288,6 +323,15 @@ internal void CopyFilesIntoRetainedPackageDirectory(IEnumerable<IEnumerable<stri
             }
         }
 
+        private static string RemoveFirstFolder(string path)
+        {
+            var parts = path.Split(new string[] { "\\" }, StringSplitOptions.RemoveEmptyEntries);
+
+            if (parts.Length > 1) return "\\" + String.Join("\\", parts, 1, parts.Length - 1);
+
+            return "\\" + parts[0];
+        }
+
         internal void CopyFilesIntoPackageDirectory(IEnumerable<string> files, IEnumerable<string> markdownFiles,
                                                     IDirectoryInfo dyfDir, IDirectoryInfo binDir,
                                                     IDirectoryInfo extraDir, IDirectoryInfo docDir)

src/DynamoPackages/PackageManagerClient.cs

@@ -226,27 +226,27 @@ private bool ExecuteTermsOfUseCall(bool queryAcceptanceStatus)
             }, false);
         }
 
-        internal PackageUploadHandle PublishAsync(Package package, object files, IEnumerable<string> markdownFiles, bool isNewVersion, bool retainFolderStructure)
+        internal PackageUploadHandle PublishAsync(Package package, object files, IEnumerable<string> markdownFiles, bool isNewVersion, IEnumerable<string> roots, bool retainFolderStructure)
         {
             var packageUploadHandle = new PackageUploadHandle(PackageUploadBuilder.NewRequestBody(package));
 
             Task.Factory.StartNew(() =>
             {
-                Publish(package, files, markdownFiles, isNewVersion, packageUploadHandle, retainFolderStructure);
+                Publish(package, files, markdownFiles, isNewVersion, packageUploadHandle, roots, retainFolderStructure);
             });
 
             return packageUploadHandle;
         }
 
-        internal void Publish(Package package, object files, IEnumerable<string> markdownFiles, bool isNewVersion, PackageUploadHandle packageUploadHandle, bool retainFolderStructure = false)
+        internal void Publish(Package package, object files, IEnumerable<string> markdownFiles, bool isNewVersion, PackageUploadHandle packageUploadHandle, IEnumerable<string> roots, bool retainFolderStructure = false)
         {
             try
             {
                 ResponseBody ret = null;
                 if (isNewVersion)
                 {
                     var pkg = retainFolderStructure ?
-                        uploadBuilder.NewPackageVersionRetainUpload(package, packageUploadDirectory, (IEnumerable<IEnumerable<string>>)files, markdownFiles,
+                        uploadBuilder.NewPackageVersionRetainUpload(package, packageUploadDirectory, roots, (IEnumerable<IEnumerable<string>>)files, markdownFiles,
                         packageUploadHandle)
                         : uploadBuilder.NewPackageVersionUpload(package, packageUploadDirectory, (IEnumerable<string>)files, markdownFiles,
                         packageUploadHandle);
@@ -256,7 +256,7 @@ internal void Publish(Package package, object files, IEnumerable<string> markdow
                 else
                 {
                     var pkg = retainFolderStructure ?
-                        uploadBuilder.NewPackageRetainUpload(package, packageUploadDirectory, (IEnumerable<IEnumerable<string>>)files, markdownFiles,
+                        uploadBuilder.NewPackageRetainUpload(package, packageUploadDirectory, roots, (IEnumerable<IEnumerable<string>>)files, markdownFiles,
                         packageUploadHandle)
                         : uploadBuilder.NewPackageUpload(package, packageUploadDirectory, (IEnumerable<string>)files, markdownFiles,
                         packageUploadHandle);