diff --git a/src/PollinationSDK/Helper/Helper.cs b/src/PollinationSDK/Helper/Helper.cs index 48e9f807..92e76c1f 100644 --- a/src/PollinationSDK/Helper/Helper.cs +++ b/src/PollinationSDK/Helper/Helper.cs @@ -279,6 +279,17 @@ public static bool GetRecipeFromRecipeSourceURL(string recipeSource, out string } + /// + /// Download all cloud reference assets (file or folder) from a project folder + /// + /// + /// + /// + /// + /// + /// + /// + /// public static async Task> DownloadAssetsAsync(string projSlug, IEnumerable assets, string saveAsDir, Action reportProgressAction, bool useCached = false, System.Threading.CancellationToken cancelToken = default) { var tasks = new List(); @@ -354,6 +365,84 @@ public static async Task> DownloadAssetsAsync(string p } + public static async Task> DownloadAssetsAsync(string projSlug, string jobId, IEnumerable assets, string saveAsDir, Action reportProgressAction, bool useCached = false, System.Threading.CancellationToken cancelToken = default) + { + var tasks = new List(); + if (assets == null || !assets.Any()) return tasks; + + + var dir = Path.GetFullPath(Path.Combine(saveAsDir, projSlug)); + System.IO.Directory.CreateDirectory(dir); + + + var proj = projSlug.Split('/'); + var projOwner = proj[0]; + var projName = proj[1]; + + // check folder assets + var gp = assets.GroupBy(_ => Path.HasExtension(_.RelativePath)); + var fileAssets = gp.FirstOrDefault(_ => _.Key == true)?.Select(_ => _)?.ToList() ?? new List(); + var folderAssets = gp.FirstOrDefault(_ => _.Key == false)?.Select(_ => _.RelativePath)?.ToList(); + + var api = new PollinationSDK.Api.JobsApi(); + if (folderAssets != null && folderAssets.Any()) + { + var allFiles = await GetAllFilesAsync(api, projOwner, projName, jobId, folderAssets, saveAsDir); + var cloudRefs = allFiles.Select(_ => new CloudReferenceAsset(projOwner, projName, jobId, _.Key)); + fileAssets.AddRange(cloudRefs); + } + + // check if cached + if (useCached) + { + fileAssets = CheckCached(fileAssets, dir).ToList(); + } + + var total = fileAssets.Count(); + var completed = 0; + foreach (var asset in fileAssets) + { + try + { + if (asset.IsPathAsset() && !asset.IsSaved()) + { + + Action individualProgress = (percent) => { + reportProgressAction?.Invoke($"[{completed}]: {percent}%"); + }; + + var savedFolderOrFilePath = await DownloadArtifactAsync(api, projOwner, projName, jobId, asset.RelativePath, dir, individualProgress, cancelToken); + + // check folder with single file + savedFolderOrFilePath = CheckPathForDir(savedFolderOrFilePath); + + // update saved path + var dup = asset.Duplicate() as CloudReferenceAsset; + dup.LocalPath = savedFolderOrFilePath; + tasks.Add(dup); + } + else + { + tasks.Add(asset); + } + + completed++; + } + catch (Exception e) + { + //canceled by user + if (e is OperationCanceledException) + return null; + + throw new ArgumentException($"Failed to download asset {asset.Name}.\n -{e.Message}"); + } + } + return tasks; + + } + + + /// /// List all FileMeta from all sub-folders if exists /// @@ -380,9 +469,24 @@ public static async Task> GetAllFilesAsync(ArtifactsApi api, stri } + public static async Task> GetAllFilesAsync(JobsApi api, string projOwner, string projName, string jobId, List relativeFolderPaths, string saveAsDir) + { + var fs = api.SearchJobFolder(projOwner, projName, jobId, relativeFolderPaths, null, 500).Resources; + var gp = fs.GroupBy(_ => _.FileType == "folder"); + var files = gp.FirstOrDefault(_ => _.Key == false)?.Select(_ => _)?.ToList() ?? new List(); + var folders = gp.FirstOrDefault(_ => _.Key == true)?.Select(_ => _.Key)?.ToList() ?? new List(); + if (folders.Any()) + { + var subItems = await GetAllFilesAsync(api, projOwner, projName, jobId, folders, saveAsDir); + files.AddRange(subItems); + } + return files; + + } + /// - /// + /// Download file artifacts from a project folder /// /// "model\grid\room.pts" public static async Task DownloadArtifactAsync(ArtifactsApi api, string projOwner, string projName, string relativePath, string saveAsDir, Action reportProgressAction = default, System.Threading.CancellationToken cancelToken = default) @@ -409,7 +513,31 @@ public static async Task DownloadArtifactAsync(ArtifactsApi api, string return path; } - + public static async Task DownloadArtifactAsync(JobsApi api, string projOwner, string projName, string jobId, string relativePath, string saveAsDir, Action reportProgressAction = default, System.Threading.CancellationToken cancelToken = default) + { + var fileRelativePath = relativePath.Replace('\\', '/'); + if (fileRelativePath.StartsWith("/")) + fileRelativePath = fileRelativePath.Substring(1); + + if (!Path.HasExtension(fileRelativePath)) // dir + { + var msg = $"Cannot download the following folder directly: {fileRelativePath}"; + throw new ArgumentException(msg); + } + + var url = (await api.DownloadJobArtifactAsync(projOwner, projName, jobId, fileRelativePath, cancelToken))?.ToString(); + + Helper.Logger.Information($"DownloadJobArtifactAsync: downloading {fileRelativePath} from \n -{url}\n"); + // get relative path correct + saveAsDir = Path.GetDirectoryName(Path.Combine(saveAsDir, relativePath)); + saveAsDir = Path.GetFullPath(saveAsDir); + var path = await Helper.DownloadUrlAsync(url.ToString(), saveAsDir, reportProgressAction, null, cancelToken); + + Helper.Logger.Information($"DownloadJobArtifactAsync: saved {fileRelativePath} to {path}"); + return path; + } + + public static async Task DownloadArtifactAsync(Project proj, FileMeta file, string saveAsFolder, Action reportProgressAction = default) { var api = new ArtifactsApi(); diff --git a/src/PollinationSDK/Wrapper/CloudReferenceAsset.cs b/src/PollinationSDK/Wrapper/CloudReferenceAsset.cs index fb237e48..66575c9a 100644 --- a/src/PollinationSDK/Wrapper/CloudReferenceAsset.cs +++ b/src/PollinationSDK/Wrapper/CloudReferenceAsset.cs @@ -7,7 +7,10 @@ namespace PollinationSDK.Wrapper { public class CloudReferenceAsset : PollinationSDK.Wrapper.AssetBase { + private static readonly string _cloudReferenceAssetKey = "CloudReferenceAsset"; public string ProjectSlug { get; set; } + public bool IsCloudJobReferenceAsset => !this.RunSource.EndsWith(_cloudReferenceAssetKey); + public string JobId => IsCloudJobReferenceAsset ? this.RunSource.Split('/').LastOrDefault() : string.Empty; [JsonConstructorAttribute] public CloudReferenceAsset() @@ -28,14 +31,28 @@ public CloudReferenceAsset(string projOwner, string projName, string assetPath) this.Description = $"CLOUD:{ProjectSlug}/{RelativePath}"; - - this.RunSource = $"CLOUD:{ProjectSlug}/CloudReferenceAsset"; + this.RunSource = $"CLOUD:{ProjectSlug}/{_cloudReferenceAssetKey}"; } + public CloudReferenceAsset(string projOwner, string projName, string jobID, string assetPath) + { + // get name + this.Name = System.IO.Path.GetFileNameWithoutExtension(assetPath); + + + // check path type + this.RelativePath = assetPath; + this.ProjectSlug = $"{projOwner}/{projName}"; + + this.Description = $"CLOUD:{ProjectSlug}/{jobID}/{RelativePath}"; + + this.RunSource = $"CLOUD:{ProjectSlug}/{jobID}"; + } public override string ToString() { - return this.ToJobPathArgument().ToUserFriendlyString(true); + return this.Description; + //return this.ToJobPathArgument().ToUserFriendlyString(true); }