another approach for daily updates

kauevestena · Sep 23, 2024 · a06b162 · a06b162
1 parent e260fa5
commit a06b162
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 15 deletions.
diff --git a/functions.py b/functions.py
@@ -529,6 +529,7 @@ def get_datetime_last_update(
     return_parsed=True,
     return_special_tuple=True,
 ):
+    # TODO: use osmapi!
 
     h_url = get_feature_history_url(featureid, featuretype)
 

diff --git a/runners/daily.sh b/runners/daily.sh
@@ -0,0 +1,15 @@
+
+python oswm_codebase/getting_data.py || echo "getting_data.py failed"; \
+# this structure assures independence on successful completion of the previous step
+
+python oswm_codebase/filtering_adapting_data.py || echo "filtering_adapting_data.py failed"; \
+
+python oswm_codebase/generation/vec_tiles_gen.py || echo "vec_tiles_gen.py failed"; \
+
+python oswm_codebase/webmap/create_webmap_new.py || echo "create_webmap_new.py failed"; \
+
+python oswm_codebase/data_quality/tag_values_checking.py || echo "tag_values_checking.py failed"; \
+
+python oswm_codebase/data_quality/quality_check_compiling.py || echo "quality_check_compiling.py failed"; \
+
+python oswm_codebase/statistics_generation.py || echo "statistics_generation.py failed"
diff --git a/special_updates.py b/special_updates.py
@@ -4,6 +4,8 @@
 
 """
 script reserved for eventual modifications that shall happen on all nodes
+
+generally not works if running from a gh action workflow
 """
 
 # Replacing the workflows from the node to the newest ones:

diff --git a/workflows/data_daily_updating.yml b/workflows/data_daily_updating.yml
@@ -53,21 +53,7 @@ jobs:
 
       #### main ________________________________________________________________
 
-      - run: python oswm_codebase/getting_data.py
-
-      - run: python oswm_codebase/filtering_adapting_data.py
-
-      - run: python oswm_codebase/generation/vec_tiles_gen.py
-
-      # - run: python oswm_codebase/create_webmap.py
-
-      - run: python oswm_codebase/webmap/create_webmap_new.py
-
-      - run: python oswm_codebase/data_quality/tag_values_checking.py
-
-      - run: python oswm_codebase/data_quality/quality_check_compiling.py
-
-      - run: python oswm_codebase/statistics_generation.py
+      - run: sh oswm_codebase/runners/daily.sh
 
       #### upload ______________________________________________________________
       # @see https://github.com/actions/checkout#push-a-commit-using-the-built-in-token
-Original file line number
+Diff line change
@@ Expand Up / @@ -4,6 +4,8 @@ @@
     """
     script reserved for eventual modifications that shall happen on all nodes
+    generally not works if running from a gh action workflow
     """
     # Replacing the workflows from the node to the newest ones:
@@ Expand Down @@