allow default override for local development

Zenysis · Jul 16, 2024 · 730b150 · 730b150
1 parent 6ec4d6e
commit 730b150
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 1 deletion.
diff --git a/pipeline/harmony_demo/index/run/00_druid/00_index b/pipeline/harmony_demo/index/run/00_druid/00_index
@@ -5,5 +5,6 @@ set -o pipefail
   --data_files="${DRUID_SHARED_FOLDER:-/home/share}/data/harmony_demo/*/current/processed_rows.*" \
   --task_id_file="${PIPELINE_TMP_DIR}/task_id" \
   --task_hash_dir "${DRUID_SHARED_FOLDER:-/home/share}/data/logs/druid_indexing/hash" \
+  --druid_server_shared_folder="${DRUID_SHARED_FOLDER:-/home/share}" \
   --local_server_shared_folder "${DRUID_SHARED_FOLDER:-/home/share}" \
   --min_data_date='1970-01-01'
diff --git a/scripts/clean_published_pipeline.py b/scripts/clean_published_pipeline.py
@@ -9,8 +9,10 @@
 
 from util.file.directory_util import compute_dir_hash, equal_dir_content
 
+DRUID_SHARED_FOLDER = os.environ.get('DRUID_SHARED_FOLDER', '/home/share')
+
 # Base directory of where to begin search.
-MAIN_DIR = '/home/share/data'
+MAIN_DIR = f'{DRUID_SHARED_FOLDER}/data'
 
 # Directories to whitelist
 WHITELIST_DIRS = set(['current'])