diff --git a/README.md b/README.md index 6a2b22f..0092d8d 100644 --- a/README.md +++ b/README.md @@ -1,48 +1,17 @@ # edly Electron diffraction simulator web dashboard. -## Installation -### python dependencies -pip install flask plotly wheel tarikDrevonUtils - -### Development environment -``` -python3 -m venv .env -cd .env -source bin/activate -git clone git@github.com:ccp4/electron-diffraction.git -cd electron-diffraction -pip install -e . -``` -Then a simple git pull on these dependencies will keep them up to date - -### Installing the js dependencies -``` -bower install jquery angular angular-aria angular-touch angular-bootstrap bootstrap-css bootstrap angular-chart jquery-ui plotly MathJax - -cd static -ln -s ../bower_components . -mkdir css -wget -O css/all.css https://css.gg/css -``` +![](static/images/edly.png) - - - -### installing the test dataset -``` -cd static/data/; ln -s ../../test test -``` +## Installation +Run the `install.sh` file to complete installation. ## Miscellaneous +### datasets +Experimental dataset (raw frames) are located in the `database` folder. Locally available datasets can manually be put in this directory. + ### Clear a session info -They should be cleared every day but can be cleared manually with : +Automatically cleared every day but can also be cleared manually with : ``` rm -rf static/data/tmp/ ``` diff --git a/app.py b/app.py index 11a766a..e35d65f 100644 --- a/app.py +++ b/app.py @@ -2,7 +2,7 @@ from subprocess import check_output,Popen,PIPE import json,os,sys,glob,time,datetime,crystals,re #,base64,hashlib import tifffile,mrcfile,cbf -from flask import Flask,Blueprint,request,url_for,redirect,jsonify,session,render_template +from flask import Flask,Blueprint,request,url_for,redirect,jsonify,session,render_template,send_file from functools import wraps import numpy as np,pandas as pd from EDutils import utilities as ut #;imp.reload(ut) @@ -27,7 +27,7 @@ def wrap(*args, **kwargs): return redirect(url_for('login')) # records=ut.load_pkl('static/spg/records.pkl') days=7 - +dl_jobs={} @app.route('/set_mode', methods=['POST']) def set_mode(): @@ -55,6 +55,11 @@ def upload_file(): ################ #### import frames +@app.route('/frames_test.zip', methods=['GET']) +def get_test_frames(): + time.sleep(2) + return send_file('static/spg/frames_test.zip') + @app.route('/update_zenodo', methods=['POST']) def update_zenodo(): fetch = json.loads(request.data.decode())['fetch'] @@ -65,7 +70,7 @@ def update_zenodo(): records = json.load(f) return json.dumps(records) -@app.route('/get_dl_state', methods=['GET']) +@app.route('/get_dl_state', methods=['POST']) def get_dl_state(): log_file="%s/dl.log" %(session['path']) cmd=r"grep '%%' %s" %log_file @@ -78,8 +83,30 @@ def get_dl_state(): cmd = 'tail -n1 %s' %os.path.realpath("%s/uncompress.log" %session['path']) file_extract=check_output(cmd,shell=True).decode().strip() msg='Extracting : \n%s\n' %file_extract + + link = request.data.decode() + job_id=short_hash(link) + print(dl_jobs) + p = dl_jobs[job_id] + poll=p.poll() + if isinstance(poll,int): + msg = 'done:%d' %p.poll() + # print(msg) return msg +@app.route('/cancel_download', methods=['POST']) +def cancel_download(): + link = request.data.decode() + job_id=short_hash(link) + p = dl_jobs[job_id] + # print(p.communicate()) + p.kill() + print('cancelling job %s : %s. Status : ' %(job_id,p.args),p.poll()) + cmd="if [ -d {filepath} ];then rm -rf {filepath};fi".format( + filepath = os.path.realpath(data_path(link))) + out=check_output(cmd,shell=True).decode().strip() + return 'cancelled' + @app.route('/download_frames', methods=['POST']) def download_frames(): link = request.data.decode() @@ -102,12 +129,19 @@ def download_frames(): cmd_log = os.path.realpath("%s/uncompress.log" %session['path']), )#;print(job) - job_file="%s/dl.sh" %(session['path']) #;print(job_file) + job_id=short_hash(link) + + job_file="%s/dl_%s.sh" %(session['path'],job_id) #;print(job_file) with open(job_file,'w') as f : f.write(job) cmd = 'bash %s' %job_file - out=check_output(cmd,shell=True).decode().strip()#;print(out) + p=p_open(cmd) + dl_jobs[job_id]=p + # p.wait() + # out=str(p.poll()) + # print('job done with status : %s'%out) + out='ok' return out @app.route('/check_dl_format', methods=['POST']) @@ -123,7 +157,7 @@ def check_dl_format(): @app.route('/check_dl_frames', methods=['POST']) def check_dl_frames(): link = request.data.decode() #;print(colors.red,link,colors.black) - filepath = data_path(link) #;print(filepath) + filepath = data_path(link) #;print(filepath) dl = os.path.exists(filepath) #;print(dl) folders = [] if dl: diff --git a/changelog.md b/changelog.md index 999bdd9..44799a0 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,16 @@ # Version log ## 0.0.7dev +### Wed 11 Oct 19:06:36 BST 2023 +- issue#63 : cancel download + - added ids to relevant html DOM + - link is updated only when clicking back on the set_link button + - added `download.downloading` in *mainCtrl.js* + - download jobs have an ID +- test : + - added download/cancel download and manipulate zenodo entries +- update README + ### Mon 9 Oct 20:45:54 BST 2023 - added test option to run only specific files as `run_tests.sh -m 'frames'` - login and close are run no matter what diff --git a/in_out.py b/in_out.py index be0264c..6703fd2 100644 --- a/in_out.py +++ b/in_out.py @@ -1,6 +1,6 @@ from string import ascii_letters,digits -import os,glob,crystals,numpy as np -from subprocess import check_output +import os,glob,crystals,hashlib,numpy as np +from subprocess import check_output,Popen,PIPE from crystals import Crystal as Crys from utils import glob_colors as colors from EDutils import felix as fe #;imp.reload(fe) @@ -24,6 +24,8 @@ 'dials': ['*.expt', '*.refl','reflections.txt'], } +p_open=lambda cmd:Popen(cmd,shell=True,stdout=PIPE,stderr=PIPE) +short_hash=lambda s:hashlib.shake_256(s.encode('utf-8')).hexdigest(5) def get_frames_fmt(path): '''return the type of frames in the folder if any ''' diff --git a/static/images/edly.png b/static/images/edly.png index ca1ced2..bad011c 100644 Binary files a/static/images/edly.png and b/static/images/edly.png differ diff --git a/static/js/mainCtrl.js b/static/js/mainCtrl.js index f856791..2a8f785 100644 --- a/static/js/mainCtrl.js +++ b/static/js/mainCtrl.js @@ -239,29 +239,42 @@ angular.module('app'). $scope.download.info='done'; } + var interval_dl; $scope.download_frames=function(){ $http.post('check_dl_format',$scope.download.link) .then(function(response){ $scope.download.info=response.data.msg; - - // $log.log($scope.download.info,$scope.download.info=='ready to download' ) if ($scope.download.info=='ready to download'){ - var interval = $interval(function () { - $http.get('get_dl_state') + interval_dl = $interval(function () { + $http.post('get_dl_state',$scope.download.link) .then(function(response) { $scope.download.info=response.data; + if(response.data.includes('done')){ + $scope.download.info='done'; + $scope.download.downloading=false; + $interval.cancel(interval_dl); + if (response.data.split(':')[1]=='0'){ + $log.log('download success') + $scope.check_dl_frames(); + } + } }) - },500); + },200); + + $scope.download.downloading=true; $http.post('download_frames',$scope.download.link) .then(function(response){ - $interval.cancel(interval); - $scope.download.info='done'; - $scope.check_dl_frames(); + $log.log('download_frames request complete') }) } }) } + $scope.cancel_download=function(){ + $log.log('cancelling download') + $http.post('cancel_download',$scope.download.link) + } + $scope.import_frames=function(folder){ $http.post('load_frames',folder) .then(function(response){ @@ -490,6 +503,16 @@ angular.module('app'). update_formula($scope.crys.chemical_formula); } + + ////////////////////////////////////////////////////////////////////////////// + // misc + ////////////////////////////////////////////////////////////////////////////// + $scope.change_show_input_link = function(){ + $scope.show_input_link=!$scope.show_input_link + if (!$scope.show_input_link){ + $scope.check_dl_frames(); + } + } ////////////////////////////////////////////////////////////////////////////// // init stuffs ////////////////////////////////////////////////////////////////////////////// @@ -546,12 +569,12 @@ angular.module('app'). $scope.update(); }) $scope.changed=true; - $scope.download={'zenodo':true,'link':'','info':'done','downloaded':false}; + $scope.download={'zenodo':true,'link':'','info':'done','downloaded':false, 'downloading':false}; $scope.zenodo={'record':'','records':''} $scope.local_frames={'name':'','filtered':[],'folders':[]} $scope.show_input_link=false; $scope.import_style = {open_struct:'',frames:'',dat:'',cif:''}; - $scope.import_mode ='open_struct'; + $scope.import_mode ='frames'; $scope.import_style[$scope.import_mode]=sel_style; $scope.dat_type_files = { 'xds' : 'A single XDS_ASCII.HKL file ' , @@ -575,7 +598,7 @@ angular.module('app'). $scope.frames = {offset:0,active_frame:0,reload:true,manual:true,jump_frames:10} $scope.expand_str={false:'expand',true:'minimize'}; $scope.expand={ - 'importer':false,'struct':false, + 'importer':true,'struct':false, //'rock_settings':true,'load_rock':true, }; diff --git a/static/views/upload_panel.html b/static/views/upload_panel.html index ca16f4e..9ce8855 100644 --- a/static/views/upload_panel.html +++ b/static/views/upload_panel.html @@ -124,7 +124,7 @@ @@ -134,7 +134,7 @@ -
+
@@ -158,15 +158,26 @@
- Link : + + Link : + Enter a link manually (as it would be fetched with wget).
Click again to actually update the link +
- - - - - Download frames from cloud to server - - + + + + + + + Download frames from cloud to server + + + + + cancel download + + +
@@ -184,7 +195,7 @@ datasets import -
+
diff --git a/tests/conftest.py b/tests/conftest.py index 1df8504..1b55947 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,7 @@ def pytest_addoption(parser): parser.addoption('--port' ,action="store" ,default=8020) parser.addoption('--ip' ,action="store" ,default="localhost") parser.addoption('--address' ,action="store" ,default='') - parser.addoption('--sleep' ,action="store" ,default=1) + parser.addoption('--sleep' ,action="store" ,default=0.3) parser.addoption('--headless' ,action='store_true',default=False) parser.addoption("--slow", action="store_true", default=False, help="run slow tests") @@ -22,7 +22,7 @@ def pytest_configure(config): config.addinivalue_line("markers", "slow: mark test as slow to run") config.addinivalue_line("markers", "opt: mark test as optional") config.addinivalue_line("markers", "old: mark test as old") - for i in range(1,3): + for i in range(1,5): config.addinivalue_line("markers", "lvl%d: test level %d" %(i,i)) config.addinivalue_line("markers", "new: mark test as new to run") @@ -64,6 +64,14 @@ def pytest_collection_modifyitems(config, items): +def get_address(pytestconfig): + if pytestconfig.getoption('address'): + address=pytestconfig.getoption('address') + else: + port = pytestconfig.getoption('port') + ip = pytestconfig.getoption('ip') + address = 'http://%s:%s' %(ip,port) + return address @pytest.fixture(scope="package") def chrome_driver(pytestconfig): @@ -72,13 +80,8 @@ def chrome_driver(pytestconfig): # options.add_argument("--window-size=1920,1080") # options.add_argument("--start-maximized") options.add_argument('--headless') + address=get_address(pytestconfig) - if pytestconfig.getoption('address'): - address=pytestconfig.getoption('address') - else: - port = pytestconfig.getoption('port') - ip = pytestconfig.getoption('ip') - address = 'http://%s:%s' %(ip,port) chrome_driver = webdriver.Chrome(options=options) # chrome_driver.set_network_conditions(offline=True,latency=5,throughput=500*1024) # print(address) @@ -94,3 +97,12 @@ def sec(pytestconfig): sleep=0 print(sleep) return sleep + + +# @pytest.fixture(scope="package") +# def port(pytestconfig): +# return pytestconfig.getoption('port') + +@pytest.fixture(scope="package") +def address(pytestconfig): + return get_address(pytestconfig) diff --git a/tests/mail.txt b/tests/mail.txt index 3c358d4..9163b64 100644 --- a/tests/mail.txt +++ b/tests/mail.txt @@ -1,5 +1,5 @@ # If using gmail an app password must be set by the sender # This is done in the browser directly from the gmail account -sender_email = -password = +sender_email = ronandrevon@gmail.com +password = tlrd hrci iwuo ffuc receiver_email = tarik.drevon@stfc.ac.uk diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 93eef78..b7e5319 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -129,8 +129,8 @@ if [ $do_report -eq 1 ];then ##check the port is correct log_port=0 i=0 - # If the server takes more than 15 seconds to lauch,increase this - wait_for_server=15 + # If the server takes more than 60 seconds to lauch,increase this + wait_for_server=60 while [[ ! "$log_port" -eq "$port_report" && $i -lt $wait_for_server ]] ;do running_line=$( grep "Running on http" $server_log | sed 's/[^0-9.:]//g') ip_address=$(echo $running_line | cut -d":" -f2) diff --git a/tests/test_bloch.py b/tests/test_bloch.py index 0091c69..20308a0 100644 --- a/tests/test_bloch.py +++ b/tests/test_bloch.py @@ -57,7 +57,7 @@ def test_bloch_mode(chrome_driver,sec): click(chrome_driver,'bloch_tab',sec) check_text(chrome_driver,'mode_title_panel','Bloch solver') -@pytest.mark.lvl2 +@pytest.mark.lvl1 def test_bloch_single(chrome_driver,sec): print(colors.blue+", bloch single "+colors.black,end="") click(chrome_driver,'u_single',sec) diff --git a/tests/test_close.py b/tests/test_close.py index bd4031b..4661eb7 100644 --- a/tests/test_close.py +++ b/tests/test_close.py @@ -4,6 +4,7 @@ from selenium.webdriver.common.keys import Keys from selenium_utils import* +# @pytest.mark.new @pytest.mark.lvl1 def test_delete_struct(chrome_driver,sec): print(colors.green+"\nDeleting structure %s" %struct_test+colors.black,end="") @@ -26,6 +27,7 @@ def test_delete_struct(chrome_driver,sec): click(chrome_driver,'expand_import_menu',sec) sleep(2) +@pytest.mark.new def test_close(chrome_driver): print(colors.green+"\nclosing browser. Good bye"+colors.black) chrome_driver.close() diff --git a/tests/test_frames.py b/tests/test_frames.py index ccc1cf5..0b0809e 100644 --- a/tests/test_frames.py +++ b/tests/test_frames.py @@ -2,9 +2,9 @@ from time import sleep from utils import glob_colors as colors from selenium.webdriver.common.keys import Keys +from subprocess import check_output from selenium_utils import* - @pytest.mark.lvl1 def test_import_frames(chrome_driver,sec): if not chrome_driver.find_element("id", "import_menu_open_btn").is_displayed(): @@ -61,6 +61,7 @@ def test_frame_key_nav(chrome_driver,sec): check_text(chrome_driver,'frame_nb_span',str(frame)) click(chrome_driver,'frame_mode_select_btn',sec) + @pytest.mark.lvl2 def test_frame_input_nav(chrome_driver,sec): print(colors.green+", input nav"+colors.black,end="") @@ -81,3 +82,68 @@ def test_frame_heatmap(chrome_driver,sec): print(colors.green+", heatmap"+colors.black,end="") select_by_text(chrome_driver,'heatmap_select','Greys_r',sec) select_by_text(chrome_driver,'heatmap_select','hot',sec) + + + +################ +#### zenodo bit +################ +# @pytest.mark.new +@pytest.mark.lvl3 +def test_zenodo_entry(chrome_driver,sec): + search='lysozyme' + record='Electron diffraction datasets of hen egg-white lysozyme' + name="Lys_ED_Dataset_1.tar.gz" + data_folder="1250447_Lys_ED_Dataset_1/Lys_ED_Dataset_1" + if not chrome_driver.find_element("id", "import_menu_open_btn").is_displayed(): + click(chrome_driver,'expand_import_menu',sec) + + print(colors.green+",Select from zenodo"+colors.black,end="") + if not chrome_driver.find_element("id", "search_frames").is_displayed(): + click(chrome_driver,'import_frames_toggle_btn',sec) + print(colors.green+",pick record : %s" %record+colors.black,end="") + write(chrome_driver,'search_frames',search,sec,clear=True) + click(chrome_driver,'li_zenodo_%s' %record,sec,exec=False) + + print(colors.green+",pick first dataset"+colors.black,end="") + click(chrome_driver,'td_record_%s' %name,sec,exec=False) + check_text(chrome_driver,'td_frames_folder_%s' %data_folder,data_folder) + print(colors.green+",done"+colors.black) + +def remove_folder(data_folder): + #### delete folder and reload the page + folder_path='../static/database/%s' %data_folder + if os.path.exists(folder_path): + cmd="rm -rf %s" %folder_path + check_output(cmd,shell=True) + +# @pytest.mark.new +@pytest.mark.lvl3 +def test_zenodo_download(chrome_driver,sec,address): + #http://localhost:8020/frames_test.zip + link="%s/frames_test.zip" %address + data_folder="localhost_8020_frames_test" + remove_folder(data_folder) + + print(colors.green+"\nDownload_test "+colors.yellow+data_folder+colors.black,end="") + chrome_driver.get(address) + + print(colors.green+", Setting link"+colors.black,end="") + if not chrome_driver.find_element("id", "input_download_link").is_displayed(): + click(chrome_driver,'download_link_btn',sec,exec=False) + write(chrome_driver,'input_download_link',link,sec,clear=True) + click(chrome_driver,'download_link_btn',sec,exec=False) + + print(colors.green+", Downloading"+colors.black,end="") + click(chrome_driver,'btn_download',sec,exec=False) + # sleep(0.1) + + print(colors.green+", Cancelling download"+colors.black,end="") + click(chrome_driver,'btn_cancel_download',sec,exec=False) + + print(colors.green+", Re Downloading"+colors.black,end="") + click(chrome_driver,'btn_download',sec,exec=False) + check_text(chrome_driver,'td_frames_folder_%s' %data_folder,data_folder) + print(colors.green+",download complete"+colors.black,end="") + + # remove_folder(data_folder) diff --git a/tests/test_login.py b/tests/test_login.py index d3ddfb6..8ca9542 100644 --- a/tests/test_login.py +++ b/tests/test_login.py @@ -4,6 +4,7 @@ from selenium.webdriver.common.keys import Keys from selenium_utils import* +@pytest.mark.new def test_login(chrome_driver,sec): print(colors.green+"\nLogin : "+colors.black,end="") sleep(sec) @@ -13,7 +14,7 @@ def test_login(chrome_driver,sec): check_text(chrome_driver,'span_expand_import_menu','Import menu') print(colors.green+",done"+colors.black) - +# @pytest.mark.new @pytest.mark.lvl1 def test_new_structure(chrome_driver,sec): print(colors.green+"\nCreating structure %s" %struct_test+colors.black,end="")