Skip to content

Commit

Permalink
Merge branch 'feature/folder-saver'
Browse files Browse the repository at this point in the history
  • Loading branch information
benbierens committed Dec 16, 2024
2 parents c24861e + 1d9f22c commit 95aa6fd
Show file tree
Hide file tree
Showing 21 changed files with 1,222 additions and 298 deletions.
15 changes: 14 additions & 1 deletion ProjectPlugins/CodexPlugin/CodexAccess.cs
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,20 @@ public LocalDataset DownloadManifestOnly(ContentId cid)

public LocalDatasetList LocalFiles()
{
return mapper.Map(OnCodex(api => api.ListDataAsync()));
// API for listData mismatches.
//return mapper.Map(OnCodex(api => api.ListDataAsync()));

return mapper.Map(CrashCheck(() =>
{
var endpoint = GetEndpoint();
return Time.Retry(() =>
{
var str = endpoint.HttpGetString("data");
if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
return JsonConvert.DeserializeObject<LocalDatasetListJson>(str)!;
}, nameof(LocalFiles));
}));

}

public StorageAvailability SalesAvailability(StorageAvailability request)
Expand Down
2 changes: 1 addition & 1 deletion ProjectPlugins/CodexPlugin/CodexNode.cs
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ public ContentId UploadFile(TrackedFile file)

public ContentId UploadFile(TrackedFile file, Action<Failure> onFailure)
{
return UploadFile(file, "application/x-binary", $"attachment; filename=\"{file.Filename}\"", onFailure);
return UploadFile(file, "application/octet-stream", $"attachment; filename=\"{Path.GetFileName(file.Filename)}\"", onFailure);
}

public ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure)
Expand Down
67 changes: 67 additions & 0 deletions ProjectPlugins/CodexPlugin/Mapper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,14 @@ public DebugInfo Map(CodexOpenApi.DebugInfo debugInfo)
};
}

public LocalDatasetList Map(LocalDatasetListJson json)
{
return new LocalDatasetList
{
Content = json.Content.Select(Map).ToArray()
};
}

public LocalDatasetList Map(CodexOpenApi.DataList dataList)
{
return new LocalDatasetList
Expand All @@ -38,6 +46,15 @@ public LocalDataset Map(CodexOpenApi.DataItem dataItem)
};
}

public LocalDataset Map(LocalDatasetListJsonItem item)
{
return new LocalDataset
{
Cid = new ContentId(item.Cid),
Manifest = MapManifest(item.Manifest)
};
}

public CodexOpenApi.SalesAvailabilityCREATE Map(StorageAvailability availability)
{
return new CodexOpenApi.SalesAvailabilityCREATE
Expand Down Expand Up @@ -188,6 +205,18 @@ private Manifest MapManifest(CodexOpenApi.ManifestItem manifest)
};
}

public Manifest MapManifest(LocalDatasetListJsonItemManifest manifest)
{
return new Manifest
{
// needs update
BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)),
OriginalBytes = new ByteSize(Convert.ToInt64(manifest.DatasetSize)),
RootHash = manifest.TreeCid,
Protected = manifest.Protected
};
}

private JArray JArray(IDictionary<string, object> map, string name)
{
return (JArray)map[name];
Expand Down Expand Up @@ -243,4 +272,42 @@ private ByteSize ToByteSize(string size)
return new ByteSize(Convert.ToInt64(size));
}
}


//"content": [
// {
// "cid": "zDvZRwzkxLxVaGces3kpkHjo8EcTPXudvYMfNxdoH21Ask1Js5fJ",
// "manifest": {
// "treeCid": "zDzSvJTf8GBRyEDNuAzXS9VnRfh8cNuYuRPwTLW6RUQReSgKnhCt",
// "datasetSize": 5242880,
// "blockSize": 65536,
// "filename": null,
// "mimetype": "application/octet-stream",
// "uploadedAt": 1731426230,
// "protected": false
// }
// }
// ]

public class LocalDatasetListJson
{
public LocalDatasetListJsonItem[] Content { get; set; } = Array.Empty<LocalDatasetListJsonItem>();
}

public class LocalDatasetListJsonItem
{
public string Cid { get; set; } = string.Empty;
public LocalDatasetListJsonItemManifest Manifest { get; set; } = new();
}

public class LocalDatasetListJsonItemManifest
{
public string TreeCid { get; set; } = string.Empty;
public int DatasetSize { get; set; }
public int BlockSize { get; set; }
public string? Filename { get; set; } = string.Empty;
public string? MimeType { get; set; } = string.Empty;
public int? UploadedAt { get; set; }
public bool Protected { get; set; }
}
}
6 changes: 6 additions & 0 deletions ProjectPlugins/CodexPlugin/MarketplaceTypes.cs
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,12 @@ public class StoragePurchase
public string State { get; set; } = string.Empty;
public string Error { get; set; } = string.Empty;
public StorageRequest Request { get; set; } = null!;

public bool IsCancelled => State.ToLowerInvariant().Contains("cancel");
public bool IsError => State.ToLowerInvariant().Contains("error");
public bool IsFinished => State.ToLowerInvariant().Contains("finished");
public bool IsStarted => State.ToLowerInvariant().Contains("started");
public bool IsSubmitted => State.ToLowerInvariant().Contains("submitted");
}

public class StorageRequest
Expand Down
13 changes: 12 additions & 1 deletion Tools/AutoClient/App.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using Logging;
using AutoClient.Modes.FolderStore;
using Logging;

namespace AutoClient
{
Expand All @@ -19,6 +20,15 @@ public App(Configuration config)
new FileLog(Path.Combine(config.LogPath, "performance")),
new ConsoleLog()
));

if (!string.IsNullOrEmpty(config.FolderToStore))
{
FolderWorkDispatcher = new FolderWorkDispatcher(Log, config.FolderToStore);
}
else
{
FolderWorkDispatcher = null!;
}
}

public Configuration Config { get; }
Expand All @@ -27,6 +37,7 @@ public App(Configuration config)
public CancellationTokenSource Cts { get; } = new CancellationTokenSource();
public CidRepo CidRepo { get; }
public Performance Performance { get; }
public FolderWorkDispatcher FolderWorkDispatcher { get; }

private IFileGenerator CreateGenerator()
{
Expand Down
165 changes: 165 additions & 0 deletions Tools/AutoClient/AutomaticPurchaser.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
using CodexOpenApi;
using CodexPlugin;
using Logging;
using Newtonsoft.Json;
using Utils;

namespace AutoClient
{
public class AutomaticPurchaser
{
private readonly ILog log;
private readonly ICodexInstance instance;
private readonly CodexNode codex;
private Task workerTask = Task.CompletedTask;
private App app => instance.App;

public AutomaticPurchaser(ILog log, ICodexInstance instance, CodexNode codex)
{
this.log = log;
this.instance = instance;
this.codex = codex;
}

public void Start()
{
workerTask = Task.Run(Worker);
}

public void Stop()
{
workerTask.Wait();
}

private async Task Worker()
{
log.Log("Worker started.");
while (!app.Cts.Token.IsCancellationRequested)
{
try
{
var pid = await StartNewPurchase();
await WaitTillFinished(pid);
await DownloadForeignCid();
}
catch (Exception ex)
{
log.Error("Worker failed with: " + ex);
await Task.Delay(TimeSpan.FromHours(6));
}
}
}

private async Task DownloadForeignCid()
{
var cid = app.CidRepo.GetForeignCid(instance.NodeId);
if (cid == null) return;

var size = app.CidRepo.GetSizeForCid(cid);
if (size == null) return;

var filename = Guid.NewGuid().ToString().ToLowerInvariant();
await codex.DownloadCid(filename, cid, size);

DeleteFile(filename);
}

private async Task<string> StartNewPurchase()
{
var file = await CreateFile();
try
{
var cid = await codex.UploadFile(file);
var response = await codex.RequestStorage(cid);
return response.PurchaseId;
}
finally
{
DeleteFile(file);
}
}

private async Task<string> CreateFile()
{
return await app.Generator.Generate();
}

private void DeleteFile(string file)
{
try
{
File.Delete(file);
}
catch (Exception exc)
{
app.Log.Error($"Failed to delete file '{file}': {exc}");
}
}

private async Task WaitTillFinished(string pid)
{
try
{
var emptyResponseTolerance = 10;
while (!app.Cts.Token.IsCancellationRequested)
{
var purchase = await codex.GetStoragePurchase(pid);
if (purchase == null)
{
await FixedShortDelay();
emptyResponseTolerance--;
if (emptyResponseTolerance == 0)
{
log.Log("Received 10 empty responses. Stop tracking this purchase.");
await ExpiryTimeDelay();
return;
}
continue;
}
if (purchase.IsCancelled)
{
app.Performance.StorageContractCancelled();
return;
}
if (purchase.IsError)
{
app.Performance.StorageContractErrored(purchase.Error);
return;
}
if (purchase.IsFinished)
{
app.Performance.StorageContractFinished();
return;
}
if (purchase.IsStarted)
{
app.Performance.StorageContractStarted();
await FixedDurationDelay();
}

await FixedShortDelay();
}
}
catch (Exception ex)
{
log.Log($"Wait failed with exception: {ex}. Assume contract will expire: Wait expiry time.");
await ExpiryTimeDelay();
}
}

private async Task FixedDurationDelay()
{
await Task.Delay(app.Config.ContractDurationMinutes * 60 * 1000, app.Cts.Token);
}

private async Task ExpiryTimeDelay()
{
await Task.Delay(app.Config.ContractExpiryMinutes * 60 * 1000, app.Cts.Token);
}

private async Task FixedShortDelay()
{
await Task.Delay(15 * 1000, app.Cts.Token);
}
}
}
1 change: 1 addition & 0 deletions Tools/AutoClient/CidRepo.cs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ public void Add(string nodeId, string cid, long knownSize)
lock (_lock)
{
entries.Add(new CidEntry(nodeId, cid, knownSize));
if (entries.Count > 1000) entries.Clear();
}
}

Expand Down
Loading

0 comments on commit 95aa6fd

Please sign in to comment.