Skip to content

Commit

Permalink
issue#63 complete
Browse files Browse the repository at this point in the history
  • Loading branch information
@ronandrevon
ronandrevon committed Oct 11, 2023
1 parent 45e45f7 commit 45bf864
Show file tree
Hide file tree
Showing 14 changed files with 213 additions and 83 deletions.
45 changes: 7 additions & 38 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -1,48 +1,17 @@
# edly
Electron diffraction simulator web dashboard.

## Installation
### python dependencies
pip install flask plotly wheel tarikDrevonUtils

### Development environment
```
python3 -m venv .env
cd .env
source bin/activate
git clone [email protected]:ccp4/electron-diffraction.git
cd electron-diffraction
pip install -e .
```
Then a simple git pull on these dependencies will keep them up to date

### Installing the js dependencies
```
bower install jquery angular angular-aria angular-touch angular-bootstrap bootstrap-css bootstrap angular-chart jquery-ui plotly MathJax
cd static
ln -s ../bower_components .
mkdir css
wget -O css/all.css https://css.gg/css
```
![](static/images/edly.png)

<!-- ## Installing jsmol
```
cd static
wget https://sourceforge.net/projects/jmol/files/latest/download
unzip download
mv jmol<version> jmol
cd jmol;unzip jsmol -->


### installing the test dataset
```
cd static/data/; ln -s ../../test test
```
## Installation
Run the `install.sh` file to complete installation.

## Miscellaneous
### datasets
Experimental dataset (raw frames) are located in the `database` folder. Locally available datasets can manually be put in this directory.

### Clear a session info
They should be cleared every day but can be cleared manually with :
Automatically cleared every day but can also be cleared manually with :
```
rm -rf static/data/tmp/<session_id>
```
46 changes: 40 additions & 6 deletions app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
from subprocess import check_output,Popen,PIPE
import json,os,sys,glob,time,datetime,crystals,re #,base64,hashlib
import tifffile,mrcfile,cbf
from flask import Flask,Blueprint,request,url_for,redirect,jsonify,session,render_template
from flask import Flask,Blueprint,request,url_for,redirect,jsonify,session,render_template,send_file
from functools import wraps
import numpy as np,pandas as pd
from EDutils import utilities as ut #;imp.reload(ut)
Expand All @@ -27,7 +27,7 @@ def wrap(*args, **kwargs):
return redirect(url_for('login'))
# records=ut.load_pkl('static/spg/records.pkl')
days=7

dl_jobs={}

@app.route('/set_mode', methods=['POST'])
def set_mode():
Expand Down Expand Up @@ -55,6 +55,11 @@ def upload_file():

################
#### import frames
@app.route('/frames_test.zip', methods=['GET'])
def get_test_frames():
time.sleep(2)
return send_file('static/spg/frames_test.zip')

@app.route('/update_zenodo', methods=['POST'])
def update_zenodo():
fetch = json.loads(request.data.decode())['fetch']
Expand All @@ -65,7 +70,7 @@ def update_zenodo():
records = json.load(f)
return json.dumps(records)

@app.route('/get_dl_state', methods=['GET'])
@app.route('/get_dl_state', methods=['POST'])
def get_dl_state():
log_file="%s/dl.log" %(session['path'])
cmd=r"grep '%%' %s" %log_file
Expand All @@ -78,8 +83,30 @@ def get_dl_state():
cmd = 'tail -n1 %s' %os.path.realpath("%s/uncompress.log" %session['path'])
file_extract=check_output(cmd,shell=True).decode().strip()
msg='Extracting : \n%s\n' %file_extract

link = request.data.decode()
job_id=short_hash(link)
print(dl_jobs)
p = dl_jobs[job_id]
poll=p.poll()
if isinstance(poll,int):
msg = 'done:%d' %p.poll()
# print(msg)
return msg

@app.route('/cancel_download', methods=['POST'])
def cancel_download():
link = request.data.decode()
job_id=short_hash(link)
p = dl_jobs[job_id]
# print(p.communicate())
p.kill()
print('cancelling job %s : %s. Status : ' %(job_id,p.args),p.poll())
cmd="if [ -d {filepath} ];then rm -rf {filepath};fi".format(
filepath = os.path.realpath(data_path(link)))
out=check_output(cmd,shell=True).decode().strip()
return 'cancelled'

@app.route('/download_frames', methods=['POST'])
def download_frames():
link = request.data.decode()
Expand All @@ -102,12 +129,19 @@ def download_frames():
cmd_log = os.path.realpath("%s/uncompress.log" %session['path']),
)#;print(job)

job_file="%s/dl.sh" %(session['path']) #;print(job_file)
job_id=short_hash(link)

job_file="%s/dl_%s.sh" %(session['path'],job_id) #;print(job_file)
with open(job_file,'w') as f :
f.write(job)

cmd = 'bash %s' %job_file
out=check_output(cmd,shell=True).decode().strip()#;print(out)
p=p_open(cmd)
dl_jobs[job_id]=p
# p.wait()
# out=str(p.poll())
# print('job done with status : %s'%out)
out='ok'
return out

@app.route('/check_dl_format', methods=['POST'])
Expand All @@ -123,7 +157,7 @@ def check_dl_format():
@app.route('/check_dl_frames', methods=['POST'])
def check_dl_frames():
link = request.data.decode() #;print(colors.red,link,colors.black)
filepath = data_path(link) #;print(filepath)
filepath = data_path(link) #;print(filepath)
dl = os.path.exists(filepath) #;print(dl)
folders = []
if dl:
Expand Down
10 changes: 10 additions & 0 deletions changelog.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,16 @@
# Version log
## 0.0.7dev

### Wed 11 Oct 19:06:36 BST 2023
- issue#63 : cancel download
- added ids to relevant html DOM
- link is updated only when clicking back on the set_link button
- added `download.downloading` in *mainCtrl.js*
- download jobs have an ID
- test :
- added download/cancel download and manipulate zenodo entries
- update README

### Mon 9 Oct 20:45:54 BST 2023
- added test option to run only specific files as `run_tests.sh -m 'frames'`
- login and close are run no matter what
Expand Down
6 changes: 4 additions & 2 deletions in_out.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from string import ascii_letters,digits
import os,glob,crystals,numpy as np
from subprocess import check_output
import os,glob,crystals,hashlib,numpy as np
from subprocess import check_output,Popen,PIPE
from crystals import Crystal as Crys
from utils import glob_colors as colors
from EDutils import felix as fe #;imp.reload(fe)
Expand All @@ -24,6 +24,8 @@
'dials': ['*.expt', '*.refl','reflections.txt'],
}

p_open=lambda cmd:Popen(cmd,shell=True,stdout=PIPE,stderr=PIPE)
short_hash=lambda s:hashlib.shake_256(s.encode('utf-8')).hexdigest(5)

def get_frames_fmt(path):
'''return the type of frames in the folder if any '''
Expand Down
Binary file modified static/images/edly.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
45 changes: 34 additions & 11 deletions static/js/mainCtrl.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,29 +239,42 @@ angular.module('app').
$scope.download.info='done';
}

var interval_dl;
$scope.download_frames=function(){
$http.post('check_dl_format',$scope.download.link)
.then(function(response){
$scope.download.info=response.data.msg;

// $log.log($scope.download.info,$scope.download.info=='ready to download' )
if ($scope.download.info=='ready to download'){
var interval = $interval(function () {
$http.get('get_dl_state')
interval_dl = $interval(function () {
$http.post('get_dl_state',$scope.download.link)
.then(function(response) {
$scope.download.info=response.data;
if(response.data.includes('done')){
$scope.download.info='done';
$scope.download.downloading=false;
$interval.cancel(interval_dl);
if (response.data.split(':')[1]=='0'){
$log.log('download success')
$scope.check_dl_frames();
}
}
})
},500);
},200);

$scope.download.downloading=true;
$http.post('download_frames',$scope.download.link)
.then(function(response){
$interval.cancel(interval);
$scope.download.info='done';
$scope.check_dl_frames();
$log.log('download_frames request complete')
})
}
})
}

$scope.cancel_download=function(){
$log.log('cancelling download')
$http.post('cancel_download',$scope.download.link)
}

$scope.import_frames=function(folder){
$http.post('load_frames',folder)
.then(function(response){
Expand Down Expand Up @@ -490,6 +503,16 @@ angular.module('app').
update_formula($scope.crys.chemical_formula);
}


//////////////////////////////////////////////////////////////////////////////
// misc
//////////////////////////////////////////////////////////////////////////////
$scope.change_show_input_link = function(){
$scope.show_input_link=!$scope.show_input_link
if (!$scope.show_input_link){
$scope.check_dl_frames();
}
}
//////////////////////////////////////////////////////////////////////////////
// init stuffs
//////////////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -546,12 +569,12 @@ angular.module('app').
$scope.update();
})
$scope.changed=true;
$scope.download={'zenodo':true,'link':'','info':'done','downloaded':false};
$scope.download={'zenodo':true,'link':'','info':'done','downloaded':false, 'downloading':false};
$scope.zenodo={'record':'','records':''}
$scope.local_frames={'name':'','filtered':[],'folders':[]}
$scope.show_input_link=false;
$scope.import_style = {open_struct:'',frames:'',dat:'',cif:''};
$scope.import_mode ='open_struct';
$scope.import_mode ='frames';
$scope.import_style[$scope.import_mode]=sel_style;
$scope.dat_type_files = {
'xds' : 'A single XDS_ASCII.HKL file ' ,
Expand All @@ -575,7 +598,7 @@ angular.module('app').
$scope.frames = {offset:0,active_frame:0,reload:true,manual:true,jump_frames:10}
$scope.expand_str={false:'expand',true:'minimize'};
$scope.expand={
'importer':false,'struct':false,
'importer':true,'struct':false,
//'rock_settings':true,'load_rock':true,
};

Expand Down
33 changes: 22 additions & 11 deletions static/views/upload_panel.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@
</a></span>
</div>
<ul id="search_ul" style="overflow-y:scroll;max-height:200px;margin-left:20px;width:280px;border-top:none;border-bottom:none;">
<li class="li_search" ng-mousedown="select_record(x)" ng-repeat=" x in zenodo.frames_filtered"><span ng-bind="x"/></li>
<li id="li_zenodo_{a x a}" class="li_search" ng-mousedown="select_record(x)" ng-repeat=" x in zenodo.frames_filtered"><span ng-bind="x"/></li>
</ul>
</div>
<!-- record file table -->
Expand All @@ -134,7 +134,7 @@
<tbody>
<tr ng-repeat="(name,x) in zenodo.record.files">
<td style="word-wrap:break-word;">
<div style="cursor: pointer" ng-click="update_link(x)" ng-bind="name"> </div>
<div id="td_record{a name a}" style="cursor: pointer" ng-click="update_link(x)" ng-bind="name"> </div>
</td>
<td style=""><span ng-bind="x.size"/></td>
</tr>
Expand All @@ -158,15 +158,26 @@

<!-- link input and download button -->
<div>
<span id="download_link_btn" style="cursor:pointer;" ng-click="show_input_link=!show_input_link"> Link </span> :
<span id="download_link_btn" class="popup" style="cursor:pointer;" ng-click="change_show_input_link()" ng-mouseenter="showIt('edit_link')" ng-mouseleave="hideIt('edit_link')">
Link :
<span class="popuptext" ng-show="popup['edit_link']"> Enter a link manually (as it would be fetched with wget).<br> Click again to actually update the link</span>
</span>
<div style="margin-left:5px;display: flex;">
<span id="span_dl_link" ng-hide="show_input_link" style="width: 230px;word-wrap:break-word;font-style:italic;" ng-bind="download.link"/>
<input ng-show="show_input_link" style="width: 230px;margin-right: 5px;" type="text" ng-model="download.link" ng-key-enter="check_dl_frames()" ng-change="check_dl_frames()">
<a ng-hide="download.downloaded" style="margin:0px;height:30px;" class="btn-sm btn-primary popup" ng-click="download_frames()" ng-mouseenter="showIt('download_frames')" ng-mouseleave="hideIt('download_frames')">
<span class="glyphicon glyphicon-cloud-download" />
<span class="popuptext" ng-show="popup['download_frames']"> Download frames from cloud to server </span>
</a>
<span ng-show="download.downloaded" style="color:green;margin-left:5px;">
<span id="span_dl_link" ng-hide="show_input_link" style="width: 230px;word-wrap:break-word;font-style:italic;" ng-bind="download.link" />
<input id="input_download_link" ng-show="show_input_link" style="width: 230px;margin-right: 5px;" type="text" ng-model="download.link" ng-key-enter="check_dl_frames()">
<span ng-hide="download.downloaded || show_input_link" >
<!-- Download btn -->
<a id="btn_download" ng-hide="download.downloading" style="margin:0px;height:30px;" class="btn-sm btn-primary popup" ng-click="download_frames()" ng-mouseenter="showIt('download_frames')" ng-mouseleave="hideIt('download_frames')">
<span class="glyphicon glyphicon-cloud-download" />
<span class="popuptext" ng-show="popup['download_frames']"> Download frames from cloud to server </span>
</a>
<!-- cancel btn -->
<a id="btn_cancel_download" ng-show="download.downloading" style="margin:0px;height:30px;" class="btn-sm btn-primary popup" ng-click="cancel_download()" ng-mouseenter="showIt('cancel_download')" ng-mouseleave="hideIt('cancel_download')">
<span class="glyphicon glyphicon-remove" />
<span class="popuptext" ng-show="popup['cancel_download']"> cancel download </span>
</a>
</span>
<span ng-show="download.downloaded && !show_input_link" style="color:green;margin-left:5px;">
<span class="glyphicon glyphicon-saved" />
</span>
</div>
Expand All @@ -184,7 +195,7 @@
<thead><tr style="width:80%"><th> datasets </th><th style="width:20%">import</th></tr>
<tbody>
<tr ng-repeat="x in data_folders">
<td style="word-wrap:break-word;"><div ng-bind="x"> </div></td>
<td id="td_frames_folder_{a x a}" style="word-wrap:break-word;"><div ng-bind="x"> </div></td>
<td style="">
<a id="import_frames_btn_{a x a}" class="btn-sm btn-primary" style="cursor: pointer" ng-click="import_frames(x)" >
<span class="glyphicon glyphicon-import" />
Expand Down
Loading

0 comments on commit 45bf864

Please sign in to comment.