From b9ce27c7b20512cbdfc63ab2ad19251371177fa4 Mon Sep 17 00:00:00 2001 From: carlosribas Date: Mon, 8 Jul 2024 11:23:10 +0100 Subject: [PATCH 01/29] Explore the new RNAcentral microservice to export data --- .../static/js/components/routes.service.js | 1 + .../text-search/export-results.search.js | 96 ++++++++++++++++--- .../text-search-results.component.js | 18 +++- .../text-search-results.html | 2 +- .../portal/search/export-job-results.html | 31 ++---- 5 files changed, 106 insertions(+), 42 deletions(-) diff --git a/rnacentral/portal/static/js/components/routes.service.js b/rnacentral/portal/static/js/components/routes.service.js index f38d3181c..c8c744576 100644 --- a/rnacentral/portal/static/js/components/routes.service.js +++ b/rnacentral/portal/static/js/components/routes.service.js @@ -62,6 +62,7 @@ angular.module("routes", []).service('routes', ['$interpolate', function($interp quickGoSummaryPage: 'https://www.ebi.ac.uk/QuickGO/term/{{ term_id }}', quickGoChart: 'https://www.ebi.ac.uk/QuickGO/services/ontology/{{ ontology }}/terms/{{ term_ids }}/chart?base64=true', qcStatusApi: '/api/v1/rna/{{ upi }}/qc-status/{{ taxid }}', + exportApp: 'http://hx-rke-wp-webadmin-12-worker-3.caas.ebi.ac.uk:31867', }; diff --git a/rnacentral/portal/static/js/components/text-search/export-results.search.js b/rnacentral/portal/static/js/components/text-search/export-results.search.js index 68cb55fec..c3b17f0bf 100644 --- a/rnacentral/portal/static/js/components/text-search/export-results.search.js +++ b/rnacentral/portal/static/js/components/text-search/export-results.search.js @@ -15,18 +15,21 @@ limitations under the License. * Angular.js code for exporting metadata search results. */ -;angular.module('textSearch').controller('ExportResultsCtrl', ['$scope', '$location', '$http', '$interval', '$window', function($scope, $location, $http, $interval, $window) { +;angular.module('textSearch').controller('ExportResultsCtrl', ['$scope', '$location', '$http', '$interval', '$window', 'routes', function($scope, $location, $http, $interval, $window, routes) { $scope.export = { query: null, job_id: null, + data_type: null, status: null, - hits: 0, + hits: null, progress: -1, + downloadUrl: null, ended_at: null, enqueued_at: null, error_message: '', }; + $scope.routes = routes; var interval; @@ -35,19 +38,40 @@ limitations under the License. */ function get_job_status() { return $http({ - url: '/export/job-status?job=' + $scope.export.job_id, + url: routes.exportApp() + '/download/' + $scope.export.job_id + '/' + $scope.export.data_type, method: 'GET' }).then( function(response) { - $scope.export = _.extend($scope.export, response.data); - $scope.export.expiration = new Date($scope.export.expiration); - if (response.data.status === 'finished' || response.data.status === 'failed') { + if (response.headers('content-type').includes('application/json')) { + $scope.export.hits = response.data.hit_count; + + // Check progress + if ($scope.export.data_type === 'fasta') { + $scope.export.progress = (response.data.progress_ids + response.data.progress_fasta) / 2 || 0; + } else if ($scope.export.data_type === 'json') { + $scope.export.progress = (response.data.progress_ids + response.data.progress_db_data) / 2 || 0; + } else { + $scope.export.progress = response.data.progress_ids || 0; + } + + // Check status + $scope.export.status = response.data.state === 'RUNNING' ? 'running' : response.data.state ? response.data.state : 'pending'; + if ($scope.export.status === 'FAILURE') { + $interval.cancel(interval); + } + } else { $interval.cancel(interval); + $scope.export.status = 'finished'; + $scope.export.progress = 100; + + // Store the download URL + var blob = new Blob([response.data], { type: response.headers('content-type') }); + $scope.export.downloadUrl = window.URL.createObjectURL(blob); } update_page_title(); }, function(response) { - if ( response.status === 404 ) { + if (response.status === 404) { $scope.export.error_message = 'Job not found'; } else { $scope.export.error_message = 'Unknown error'; @@ -56,7 +80,28 @@ limitations under the License. update_page_title(); } ); - } + } + + /** + * Download the file. + */ + function fetch_file() { + return $http({ + url: routes.exportApp() + '/download/' + $scope.export.job_id + '/' + $scope.export.data_type, + method: 'GET', + responseType: 'arraybuffer' // Ensure the response is handled as binary for file download + }).then( + function(response) { + var blob = new Blob([response.data], { type: response.headers('content-type') }); + $scope.export.downloadUrl = window.URL.createObjectURL(blob); // Create the download URL + $scope.downloadFile(); // Trigger the download + }, + function(response) { + $scope.export.error_message = 'Error downloading file'; + update_page_title(); + } + ); + } /** * Format progress for use with ng-style and CSS. @@ -70,20 +115,18 @@ limitations under the License. * Set polling interval dynamically based on the number of hits. */ function poll_job_status() { - var min_interval = 1, // 1 second - max_interval = 3, // 3 seconds - polling_interval = Math.max(min_interval, Math.min($scope.export.hits / 1000, max_interval)) * 1000; + var max_interval = 3000; // 3 seconds interval = $interval(function(){ get_job_status(); - }, polling_interval); + }, max_interval); } /** * Show progress in page title. */ function update_page_title() { - if ($scope.export.status === 'failed') { + if ($scope.export.status === 'FAILURE') { $window.document.title = 'Export failed'; } else if ($scope.export.error_message !== '') { $window.document.title = 'Results expired'; @@ -92,16 +135,39 @@ limitations under the License. } } + /** + * Function to handle file download via button. + */ + $scope.downloadFile = function() { + if ($scope.export.downloadUrl) { + var a = document.createElement('a'); + a.style.display = 'none'; + a.href = $scope.export.downloadUrl; + a.download = $scope.export.job_id + '.' + ($scope.export.data_type === 'json' ? 'json.gz' : $scope.export.data_type === 'fasta' ? 'fasta.gz' : 'txt.gz'); + document.body.appendChild(a); + a.click(); + window.URL.revokeObjectURL($scope.export.downloadUrl); + } + }; + + /** + * Fetch the file for download. + */ + $scope.triggerDownload = function() { + fetch_file(); + }; + /** * Get job id from the url and begin retrieving data. */ function initialize() { - if ($location.url().indexOf("/export/results?job=") > -1) { + if ($location.url().indexOf("/export/results?job=") > -1) { $scope.export.job_id = $location.search().job; + $scope.export.data_type = $location.search().data_type; get_job_status().then(function(){ poll_job_status(); }); - } + } } // get job id and start updating the status diff --git a/rnacentral/portal/static/js/components/text-search/text-search-results/text-search-results.component.js b/rnacentral/portal/static/js/components/text-search/text-search-results/text-search-results.component.js index 089f0b8a6..7a5205a47 100644 --- a/rnacentral/portal/static/js/components/text-search/text-search-results/text-search-results.component.js +++ b/rnacentral/portal/static/js/components/text-search/text-search-results/text-search-results.component.js @@ -355,10 +355,24 @@ var textSearchResults = { * - open the results page in a new window. */ ctrl.exportResults = function(format) { - $http.get(ctrl.routes.submitQuery() + '?q=' + search.preprocessQuery(ctrl.search.query) + '&format=' + format).then( + var queryParam = encodeURIComponent(`(${search.preprocessQuery(ctrl.search.query)})`); + var apiUrl = `https://www.ebi.ac.uk/ebisearch/ws/rest/rnacentral?query=${queryParam}&size=1000&sort=id&format=json`; + + var payload = { + api_url: apiUrl, + data_type: format + }; + + $http.post(routes.exportApp() + '/fetch-data/', JSON.stringify(payload), { + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json' + } + }).then( function(response) { ctrl.showExportError = false; - window.location.href = ctrl.routes.resultsPage() + '?job=' + response.data.job_id; + var task_id = response.data.task_id; + window.location.href = ctrl.routes.resultsPage() + '?job=' + task_id + '&data_type=' + format; }, function(response) { ctrl.showExportError = true; diff --git a/rnacentral/portal/static/js/components/text-search/text-search-results/text-search-results.html b/rnacentral/portal/static/js/components/text-search/text-search-results/text-search-results.html index 163e82b9f..94991b473 100644 --- a/rnacentral/portal/static/js/components/text-search/text-search-results/text-search-results.html +++ b/rnacentral/portal/static/js/components/text-search/text-search-results/text-search-results.html @@ -22,7 +22,7 @@

diff --git a/rnacentral/portal/templates/portal/search/export-job-results.html b/rnacentral/portal/templates/portal/search/export-job-results.html index da08dc1f3..12ca6ca96 100644 --- a/rnacentral/portal/templates/portal/search/export-job-results.html +++ b/rnacentral/portal/templates/portal/search/export-job-results.html @@ -37,18 +37,17 @@

-