From d38ff24a5b51801bcb8d261e4a8d4f9d10a528a8 Mon Sep 17 00:00:00 2001
From: James McLaughlin <james@mclgh.net>
Date: Thu, 26 Dec 2024 23:54:16 +0000
Subject: [PATCH] restructure ftp

---
 dataload/nextflow/load_subgraph.nf | 23 ++++++++++++-----------
 dataload/scripts/dataload_codon.sh |  2 +-
 2 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/dataload/nextflow/load_subgraph.nf b/dataload/nextflow/load_subgraph.nf
index 5736f23..39b3f74 100644
--- a/dataload/nextflow/load_subgraph.nf
+++ b/dataload/nextflow/load_subgraph.nf
@@ -451,7 +451,7 @@ process run_materialised_queries {
     path(neo_db)
 
     output:
-    path("materialised_queries")
+    path("materialised_queries/*")
 
     script:
     """
@@ -636,8 +636,8 @@ process copy_neo_to_ftp {
     """
     #!/usr/bin/env bash
     set -Eeuo pipefail
-    mkdir -p /nfs/ftp/public/databases/spot/kg/${params.config}/${params.timestamp.trim()}
-    cp -f neo4j.tgz /nfs/ftp/public/databases/spot/kg/${params.config}/${params.timestamp.trim()}/${params.subgraph}_neo4j.tgz
+    mkdir -p /nfs/ftp/public/databases/spot/kg/${params.subgraph}/${params.timestamp.trim()}
+    cp -f neo4j.tgz /nfs/ftp/public/databases/spot/kg/${params.subgraph}/${params.timestamp.trim()}/${params.subgraph}_neo4j.tgz
     """
 }
 
@@ -655,8 +655,8 @@ process copy_summary_to_ftp {
     """
     #!/usr/bin/env bash
     set -Eeuo pipefail
-    mkdir -p /nfs/ftp/public/databases/spot/kg/${params.config}/${params.timestamp.trim()}
-    cp -f ${summary_json} /nfs/ftp/public/databases/spot/kg/${params.config}/${params.timestamp.trim()}/
+    mkdir -p /nfs/ftp/public/databases/spot/kg/${params.subgraph}/${params.timestamp.trim()}
+    cp -f ${summary_json} /nfs/ftp/public/databases/spot/kg/${params.subgraph}/${params.timestamp.trim()}/
     """
 }
 
@@ -674,8 +674,8 @@ process copy_solr_to_ftp {
     """
     #!/usr/bin/env bash
     set -Eeuo pipefail
-    mkdir -p /nfs/ftp/public/databases/spot/kg/${params.config}/${params.timestamp.trim()}
-    cp -f solr.tgz /nfs/ftp/public/databases/spot/kg/${params.config}/${params.timestamp.trim()}/${params.subgraph}_solr.tgz
+    mkdir -p /nfs/ftp/public/databases/spot/kg/${params.subgraph}/${params.timestamp.trim()}
+    cp -f solr.tgz /nfs/ftp/public/databases/spot/kg/${params.subgraph}/${params.timestamp.trim()}/${params.subgraph}_solr.tgz
     """
 }
 
@@ -693,8 +693,8 @@ process copy_sqlite_to_ftp {
     """
     #!/usr/bin/env bash
     set -Eeuo pipefail
-    mkdir -p /nfs/ftp/public/databases/spot/kg/${params.config}/${params.timestamp.trim()}
-    cp -f ${params.subgraph}.sqlite3 /nfs/ftp/public/databases/spot/kg/${params.config}/${params.timestamp.trim()}/${params.subgraph}.sqlite3
+    mkdir -p /nfs/ftp/public/databases/spot/kg/${params.subgraph}/${params.timestamp.trim()}
+    cp -f ${params.subgraph}.sqlite3 /nfs/ftp/public/databases/spot/kg/${params.subgraph}/${params.timestamp.trim()}/${params.subgraph}.sqlite3
     """
 }
 
@@ -713,8 +713,9 @@ process copy_mat_queries_to_ftp {
     """
     #!/usr/bin/env bash
     set -Eeuo pipefail
-    mkdir -p /nfs/ftp/public/databases/spot/kg/${params.config}/${params.timestamp.trim()}
-    cp -f ${csvs} ${sqlite} /nfs/ftp/public/databases/spot/kg/${params.config}/${params.timestamp.trim()}/
+    mkdir -p /nfs/ftp/public/databases/spot/kg/${params.subgraph}/${params.timestamp.trim()}/query_results
+    cp -f ${csvs} /nfs/ftp/public/databases/spot/kg/${params.subgraph}/${params.timestamp.trim()}/query_results/
+    cp -f ${sqlite} /nfs/ftp/public/databases/spot/kg/${params.subgraph}/${params.timestamp.trim()}/query_results/all_query_results.sqlite3.gz
     """
 }
 
diff --git a/dataload/scripts/dataload_codon.sh b/dataload/scripts/dataload_codon.sh
index ac2426f..202c687 100755
--- a/dataload/scripts/dataload_codon.sh
+++ b/dataload/scripts/dataload_codon.sh
@@ -4,7 +4,7 @@ export GREBI_QUERY_YAMLS_PATH=/nfs/production/parkinso/spot/grebi/materialised_q
 export GREBI_TMP=/hps/nobackup/parkinso/spot/grebi/tmp
 export GREBI_CONFIG=ebi
 export GREBI_IS_EBI=true
-export GREBI_TIMESTAMP=$(date +%Y_%m_%d__%H_%M)
+export GREBI_TIMESTAMP=$(date +"%Y-%b-%d")
 export GREBI_MAX_ENTITIES=1000000000
 export GREBI_NEXTFLOW_CONFIG=$GREBI_DATALOAD_HOME/nextflow/codon_nextflow.config
 module load nextflow-22.10.1-gcc-11.2.0-ju5saqw