Skip to content

Commit

Permalink
Merge pull request #1936 from julien-louis/fix-download-root-folder
Browse files Browse the repository at this point in the history 
unzip option
  • Loading branch information
@julien-louis
julien-louis authored Oct 31, 2023
2 parents 982e440 + d5f21fc commit 60d9be5
Show file tree
Hide file tree
Showing 3 changed files with 45 additions and 26 deletions.
6 changes: 6 additions & 0 deletions shanoir-ng-front/src/app/shared/mass-download/download-setup/download-setup.component.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,12 @@ <h2 class="header">Download Datasets</h2>
<select-box formControlName="format" [options]="formatOptions"></select-box>
</span>
</li>
<li>
<label>Unzip datasets</label>
<span class="right-col">
<checkbox formControlName="unzip"></checkbox>
</span>
</li>
</ol>
</fieldset>
<div class="footer">
Expand Down
6 changes: 4 additions & 2 deletions shanoir-ng-front/src/app/shared/mass-download/download-setup/download-setup.component.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import { Option } from '../../select/select.component';

export class DownloadSetupComponent implements OnInit {

@Output() go: EventEmitter<{format: Format, nbQueues: number}> = new EventEmitter();
@Output() go: EventEmitter<{format: Format, nbQueues: number, unzip: boolean}> = new EventEmitter();
@Output() close: EventEmitter<void> = new EventEmitter();
form: UntypedFormGroup;
@Input() format: Format;
Expand All @@ -53,14 +53,16 @@ export class DownloadSetupComponent implements OnInit {
let formGroup = this.formBuilder.group({
'format': [{value: this.format || 'dcm', disabled: this.format}, [Validators.required]],
'nbQueues': [4, [Validators.required, Validators.min(1), Validators.max(1024)]],
'unzip': [false, []],
});
return formGroup;
}

downloadNow() {
this.go.emit({
format: this.form.get('format').value,
nbQueues: this.form.get('nbQueues').value
nbQueues: this.form.get('nbQueues').value,
unzip: this.form.get('unzip').value
});
}

Expand Down
59 changes: 35 additions & 24 deletions shanoir-ng-front/src/app/shared/mass-download/mass-download.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ export type Report = {
duration?: number;
format: Format;
nbQueues: number;
unzip: boolean;
};

@Injectable()
Expand All @@ -67,7 +68,7 @@ export class MassDownloadService {
downloadByIds(datasetIds: number[], format?: Format): Promise<void> {
return this.openModal(format).then(ret => {
if (ret != 'cancel') {
return this._downloadByIds(datasetIds, ret.format, ret.nbQueues)
return this._downloadByIds(datasetIds, ret.format, ret.nbQueues, ret.unzip);
} else return Promise.resolve();
}).catch(error => {
if (error == this.BROWSER_COMPAT_ERROR_MSG) {
Expand All @@ -84,7 +85,7 @@ export class MassDownloadService {
return this.openModal().then(ret => {
if (ret != 'cancel') {
return this.datasetService.getByStudyId(studyId).then(datasets => {
this._downloadDatasets(datasets, ret.format, ret.nbQueues);
this._downloadDatasets(datasets, ret.format, ret.nbQueues, ret.unzip);
})
} else return Promise.resolve();
}).catch(error => {
Expand All @@ -103,7 +104,7 @@ export class MassDownloadService {
return this.openModal(format, options).then(ret => {
if (ret != 'cancel') {
return datasetsPromise.then(datasets => {
this._downloadDatasets(datasets, ret.format, ret.nbQueues, downloadState);
this._downloadDatasets(datasets, ret.format, ret.nbQueues, ret.unzip, downloadState);
})
} else return Promise.resolve();
}).catch(error => {
Expand Down Expand Up @@ -141,15 +142,15 @@ export class MassDownloadService {
return this.openModal().then(ret => {
if (ret != 'cancel') {
return this.datasetService.getByStudyIdAndSubjectId(studyId, subjectId).then(datasets => {
return this._downloadDatasets(datasets, ret.format, ret.nbQueues).then(r => {
return this._downloadDatasets(datasets, ret.format, ret.nbQueues, ret.unzip).then(r => {
return r;
});
})
} else return Promise.resolve();
});
}

private _downloadByIds(datasetIds: number[], format: Format, nbQueues: number = 4, task?: Task, report?: Report, parentHandle?: FileSystemDirectoryHandle): Promise<void> {
private _downloadByIds(datasetIds: number[], format: Format, nbQueues: number = 4, unzip: boolean= false, task?: Task, report?: Report, parentHandle?: FileSystemDirectoryHandle): Promise<void> {
if (datasetIds.length == 0) return;
let directoryHandlePromise: Promise<FileSystemDirectoryHandle>;
if (parentHandle) {
Expand All @@ -168,11 +169,11 @@ export class MassDownloadService {
this.notificationService.pushLocalTask(task);
const start: number = Date.now();
let ids = [...datasetIds]; // copy array
if (!report) report = this.initReport(datasetIds, task.id, parentFolderHandle.name, format, nbQueues);
if (!report) report = this.initReport(datasetIds, task.id, parentFolderHandle.name, format, nbQueues, unzip);
let promises: Promise<void>[] = [];
for (let queueIndex = 0; queueIndex < nbQueues; queueIndex++) { // build the dl queues
promises.push(
this.recursiveSave(ids.shift(), format, parentFolderHandle, ids, report, task)
this.recursiveSave(ids.shift(), format, parentFolderHandle, ids, report, task, unzip)
);
}
return Promise.all(promises).then(() => {
Expand Down Expand Up @@ -270,7 +271,7 @@ export class MassDownloadService {
});
}

private _downloadDatasets(datasets: Dataset[], format: Format, nbQueues: number = 4, downloadState?: TaskState): Promise<void> {
private _downloadDatasets(datasets: Dataset[], format: Format, nbQueues: number = 4, unzip: boolean = false, downloadState?: TaskState): Promise<void> {
if (datasets.length == 0) return;
return this.getFolderHandle()
// add a subdirectory
Expand All @@ -285,12 +286,12 @@ export class MassDownloadService {
task.lastUpdate = new Date();
const start: number = Date.now();
let ids = [...datasets.map(ds => ds.id)];
let report: Report = this.initReport(datasets.map(ds => ds.id), task.id, parentFolderHandle.name, format, nbQueues);
let report: Report = this.initReport(datasets.map(ds => ds.id), task.id, parentFolderHandle.name, format, nbQueues, unzip);
let promises: Promise<void>[] = [];
let j = 0;
for (let queueIndex = 0; queueIndex < nbQueues; queueIndex++) { // build the dl queues
promises.push(
this.recursiveSave(ids.shift(), format, parentFolderHandle, ids, report, task, datasets)
this.recursiveSave(ids.shift(), format, parentFolderHandle, ids, report, task, unzip, datasets)
);
}
return Promise.all(promises).then(() => {
Expand All @@ -313,7 +314,7 @@ export class MassDownloadService {
private _downloadFromReport(report: Report, task: Task, parentHandle: FileSystemDirectoryHandle) {
if (!report) throw new Error('report can\'t be null !');
const noSuccessIds: number[] = Object.keys(report.list).filter(key => report.list[key].status != 'SUCCESS').map(key => parseInt(key));
this._downloadByIds(noSuccessIds, report.format, report.nbQueues, task, report, parentHandle);
this._downloadByIds(noSuccessIds, report.format, report.nbQueues, report.unzip, task, report, parentHandle);
}

private handleEnd(task: Task, report: Report, start: number) {
Expand All @@ -335,28 +336,27 @@ export class MassDownloadService {
this.notificationService.pushLocalTask(task);
}

private recursiveSave(id: number, format: Format, userFolderHandle: FileSystemDirectoryHandle, remainingIds: number[], report: Report, task: Task, datasets?: Dataset[]): Promise<void> {
private recursiveSave(id: number, format: Format, userFolderHandle: FileSystemDirectoryHandle, remainingIds: number[], report: Report, task: Task, unzip: boolean = false, datasets?: Dataset[]): Promise<void> {
if (!id) return Promise.resolve();
return this.saveDataset(id, format, userFolderHandle, report, task, datasets?.find(ds => ds.id == id)).then(() => {
return this.saveDataset(id, format, userFolderHandle, report, task, unzip, datasets?.find(ds => ds.id == id)).then(() => {
if (remainingIds.length > 0) {
return this.recursiveSave(remainingIds.shift(), format, userFolderHandle, remainingIds, report, task, datasets);
return this.recursiveSave(remainingIds.shift(), format, userFolderHandle, remainingIds, report, task, unzip, datasets);
} else {
return Promise.resolve();
}
});
}

private saveDataset(id: number, format: Format, userFolderHandle: FileSystemDirectoryHandle, report: Report, task: Task, dataset?: Dataset): Promise<void> {
private saveDataset(id: number, format: Format, userFolderHandle: FileSystemDirectoryHandle, report: Report, task: Task, unzip: boolean = false, dataset?: Dataset): Promise<void> {
const metadataPromise: Promise<Dataset> = (dataset?.id == id && dataset.datasetAcquisition?.examination?.subject) ? Promise.resolve(dataset) : this.datasetService.get(id, 'lazy');
const downloadPromise: Promise<HttpResponse<Blob>> = this.datasetService.downloadToBlob(id, format);
return Promise.all([metadataPromise, downloadPromise]).then(([dataset, httpResponse]) => {
const blob: Blob = httpResponse.body;
const filename: string = this.getFilename(httpResponse) || 'dataset_' + id;
const path: string = this.buildDatasetPath(dataset) + filename;

// Check ERRORS file in zip
var zip = new JSZip();
const errorsCheckPromise: Promise<void> = zip.loadAsync(httpResponse.body).then(dataFiles => {
const unzipPromise: Promise<any> = zip.loadAsync(httpResponse.body).then(dataFiles => {
if (dataFiles.files['ERRORS.json']) {
return dataFiles.files['ERRORS.json'].async('string').then(content => {
const errorsJson: any = JSON.parse(content);
Expand All @@ -374,10 +374,20 @@ export class MassDownloadService {
task.lastUpdate = new Date();
task.message = '(' + report.nbSuccess + '/' + report.requestedDatasetIds.length + ') dataset n°' + id + ' successfully saved';
}
return dataFiles;
});

return Promise.all([errorsCheckPromise, this.writeMyFile(path, blob, userFolderHandle)]).then(() => null);

if (unzip) {
return unzipPromise.then(data => {
for(let [name, file] of Object.entries(data.files)) {
const path: string = this.buildAcquisitionPath(dataset) + filename.replace('.zip', '') + '/' + name;
this.writeMyFile(path, file, userFolderHandle);
}
});
} else {
const path: string = this.buildAcquisitionPath(dataset) + filename;
return Promise.all([unzipPromise, this.writeMyFile(path, blob, userFolderHandle)]).then(() => null);
}
}).catch(reason => {
report.list[id].status = 'ERROR';
report.list[id].error = reason;
Expand All @@ -399,7 +409,7 @@ export class MassDownloadService {
});
}

private buildDatasetPath(dataset: Dataset): string {
private buildAcquisitionPath(dataset: Dataset): string {
return dataset.datasetAcquisition?.examination?.subject?.name
+ '_' + dataset.datasetAcquisition?.examination?.subject?.id
+ '/'
Expand Down Expand Up @@ -445,7 +455,7 @@ export class MassDownloadService {
// Create a FileSystemWritableFileStream to write to.
const writable: FileSystemWritableFileStream = await fileHandle.createWritable();
// Write the contents of the file to the stream.
await writable.write(contents);
await writable.write({type: 'write', data: contents});
// Close the file and write the contents to disk.
await writable.close();
}
Expand All @@ -467,7 +477,7 @@ export class MassDownloadService {
return contentDispHeader?.slice(contentDispHeader.indexOf(prefix) + prefix.length, contentDispHeader.length).replace('/', '_');
}

private initReport(datasetIds: number[], taskId: number, folderName: string, format: Format, nbQueues: number): Report {
private initReport(datasetIds: number[], taskId: number, folderName: string, format: Format, nbQueues: number, unzip: boolean): Report {
let report: Report = {
taskId: taskId,
folderName: folderName,
Expand All @@ -477,7 +487,8 @@ export class MassDownloadService {
nbError: 0,
nbSuccess: 0,
format : format,
nbQueues: nbQueues
nbQueues: nbQueues,
unzip: unzip
};
datasetIds.forEach(id => report.list[id] = { status: 'QUEUED' });
return report;
Expand All @@ -504,7 +515,7 @@ export class MassDownloadService {
return task;
}

private openModal(format?: Format, options?: DownloadSetupOptions): Promise<{format: Format, nbQueues: number} | 'cancel'> {
private openModal(format?: Format, options?: DownloadSetupOptions): Promise<{format: Format, nbQueues: number, unzip: boolean} | 'cancel'> {
// @ts-ignore
if (window.showDirectoryPicker) { // test compatibility
let modalRef: ComponentRef<DownloadSetupComponent> = ServiceLocator.rootViewContainerRef.createComponent(DownloadSetupComponent);
Expand Down

0 comments on commit 60d9be5

Please sign in to comment.