sync scheduler overhaul

docker-compose-dev.yaml

@@ -44,6 +44,8 @@ services:
   sync-scheduler:
     build: ./mirror-sync-scheduler
     container_name: mirror-sync-scheduler
+    expose:
+      - 9281
     volumes:
       - "./mirror-sync-scheduler/storage:/storage"
       - "./mirror-sync-scheduler/configs:/mirror/configs:ro"

docker-compose-prod.yaml

@@ -62,6 +62,8 @@ services:
   sync-scheduler:
     build: ./mirror-sync-scheduler
     container_name: mirror-sync-scheduler
+    expose:
+      - 9281
     restart: unless-stopped
     volumes:
       - "/storage:/storage"

mirror-sync-scheduler/.clang-tidy

@@ -0,0 +1,3 @@
+InheritParentConfig: true
+Checks:
+    - -clang-analyzer-optin.cplusplus.UninitializedObject # spdlog causing issues

mirror-sync-scheduler/CMakeLists.txt

@@ -1,23 +1,16 @@
-cmake_minimum_required(VERSION 3.22)
-project(syncScheduler)
+cmake_minimum_required(VERSION 3.20...3.30)
+
+find_program(CLANG_TIDY_EXE NAMES "clang-tidy-20" "clang-tidy" REQUIRED)
+set(CLANG_TIDY_COMMAND ${CLANG_TIDY_EXE} --header-filter=${CMAKE_SOURCE_DIR}/include)
 
-add_executable(${PROJECT_NAME}
-    src/main.cpp
-    src/schedule.cpp
-    src/queue.cpp
-)
 
-include(FetchContent)
+set(CPPZMQ_BUILD_TESTS OFF CACHE INTERNAL "")
 
-FetchContent_Declare(json
-    GIT_REPOSITORY https://github.com/nlohmann/json
-    GIT_TAG v3.11.3
+project(syncScheduler
+    LANGUAGES CXX
 )
-FetchContent_MakeAvailable(json)
 
-add_subdirectory(lib/mirror-logging)
+include(cmake/fetch_deps.cmake)
+
+add_subdirectory(src)
 
-target_link_libraries(${PROJECT_NAME} 
-    PUBLIC mirror-logging
-    PRIVATE nlohmann_json::nlohmann_json
-)

mirror-sync-scheduler/Dockerfile

@@ -5,18 +5,17 @@
 FROM ubuntu:24.04 as builder
 RUN apt update && apt install -y g++ cmake libzmq3-dev ninja-build git
 WORKDIR /mirror
-COPY ./src /mirror/src
-COPY ./lib /mirror/lib
-COPY ./CMakeLists.txt /mirror/CMakeLists.txt
-RUN cmake -S/mirror -B/mirror/build -G Ninja
-RUN cmake --build /mirror/build --target clean
-RUN cmake --build /mirror/build --target all
+COPY src src
+COPY include include
+COPY cmake cmake
+COPY CMakeLists.txt CMakeLists.txt
+RUN cmake -B build -G Ninja
+RUN cmake --build build --target all
 
 # Run
 FROM ubuntu:24.04
-RUN apt update && apt install -y python3 curl wget
-RUN apt install -y libzmq3-dev rsync
+RUN apt update && apt install -y python3 curl wget libzmq3-dev rsync
 WORKDIR /mirror
-COPY --from=builder /mirror/build/syncScheduler .
-RUN chmod 744 syncScheduler
-ENTRYPOINT ["./syncScheduler"]
+COPY --from=builder /mirror/build/src/sync_scheduler .
+RUN chmod 744 sync_scheduler
+ENTRYPOINT ["./sync_scheduler"]

mirror-sync-scheduler/cmake/fetch_deps.cmake

@@ -0,0 +1,19 @@
+include(FetchContent)
+
+FetchContent_Declare(json
+    GIT_REPOSITORY https://github.com/nlohmann/json
+    GIT_TAG v3.11.3
+)
+FetchContent_MakeAvailable(json)
+
+FetchContent_Declare(spdlog
+    GIT_REPOSITORY https://github.com/gabime/spdlog
+    GIT_TAG v1.15.1
+)
+FetchContent_MakeAvailable(spdlog)
+
+FetchContent_Declare(cppzmq
+    GIT_REPOSITORY https://github.com/zeromq/cppzmq
+    GIT_TAG v4.10.0
+)
+FetchContent_MakeAvailable(cppzmq)

mirror-sync-scheduler/configs/sync-scheduler.json

@@ -1,6 +1,3 @@
 {
-    "dryrun" : true,
-    "queueThreads" : 8,
-    "logServerPort": 4001,
-    "logServerHost": "localhost"
-}
+    "dry_run": true
+}

mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp

@@ -0,0 +1,60 @@
+/**
+ * @file JobManager.hpp
+ * @author Cary Keesler
+ * @brief
+ */
+
+#pragma once
+
+// System Includes
+#include <sys/types.h>
+
+// Standard Library Includes
+#include <chrono>
+#include <condition_variable>
+#include <filesystem>
+#include <map>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <vector>
+
+// Project Includes
+#include <mirror/sync_scheduler/SyncJob.hpp>
+
+namespace mirror::sync_scheduler
+{
+class JobManager
+{
+  public: // Constructors
+    JobManager();
+    ~JobManager();
+
+  public: // Methods
+    auto job_is_running(const std::string& jobName) -> bool
+    {
+        return m_ActiveJobs.contains(jobName);
+    }
+
+    auto start_job(
+        const std::string&           jobName,
+        std::string                  command,
+        const std::filesystem::path& passwordFile
+    ) -> bool;
+
+  private:
+    auto register_job(const std::string& jobName, const ::pid_t processID)
+        -> void;
+    auto kill_job(const std::string& jobName) -> void;
+    auto kill_all_jobs() -> void;
+    auto reap_processes() -> std::vector<std::string>;
+    auto deregister_jobs(const std::vector<std::string>& completedJobs) -> void;
+
+  private: // Members
+    std::map<std::string, SyncJob> m_ActiveJobs;
+    std::jthread                   m_ProcessReaper;
+    std::condition_variable        m_SleepVariable;
+    std::mutex                     m_JobMutex;
+    std::mutex                     m_ReaperMutex;
+};
+} // namespace mirror::sync_scheduler

mirror-sync-scheduler/include/mirror/sync_scheduler/ProjectCatalogue.hpp

@@ -0,0 +1,19 @@
+/**
+ * @file ProjectCatalogue.hpp
+ * @author Cary Keesler
+ * @brief
+ */
+
+#pragma once
+
+// Standard Library Includes
+#include <map>
+#include <string>
+
+// Project Includes
+#include <mirror/sync_scheduler/SyncDetails.hpp>
+
+namespace mirror::sync_scheduler
+{
+using ProjectCatalogue = std::map<std::string, SyncDetails>;
+} // namespace mirror::sync_scheduler

mirror-sync-scheduler/include/mirror/sync_scheduler/Schedule.hpp

@@ -0,0 +1,47 @@
+/**
+ * @file Schedule.hpp
+ * @author Cary Keesler
+ * @brief
+ */
+
+#pragma once
+
+// Standard Library Includes
+#include <chrono>
+#include <cstddef>
+#include <map>
+#include <set>
+#include <string>
+#include <utility>
+#include <vector>
+
+// Third Party Library Includes
+#include <nlohmann/json.hpp>
+
+// Project Includes
+#include <mirror/sync_scheduler/ProjectCatalogue.hpp>
+#include <mirror/sync_scheduler/SyncDetails.hpp>
+
+namespace mirror::sync_scheduler
+{
+class Schedule
+{
+  public: // Constructors
+    explicit Schedule(const ProjectCatalogue& projects);
+
+  public: // Methods
+    [[nodiscard]]
+    auto get_next_sync_batch() -> std::pair<
+        std::chrono::time_point<std::chrono::system_clock>,
+        std::set<std::string>>;
+
+  private: // Methods
+    auto build(const ProjectCatalogue& projects) -> void;
+    auto verify(const ProjectCatalogue& projects) -> void;
+    auto determine_sync_lcm(const ProjectCatalogue& projects) -> void;
+
+  private: // Members
+    std::vector<std::set<std::string>> m_SyncIntervals;
+    std::size_t                        m_SyncLCM;
+};
+} // namespace mirror::sync_scheduler

mirror-sync-scheduler/include/mirror/sync_scheduler/SyncDetails.hpp

@@ -0,0 +1,87 @@
+/**
+ * @file SyncDetails.hpp
+ * @author Cary Keesler
+ * @brief
+ */
+
+#pragma once
+
+// Standard Library Includes
+#include <cstddef>
+#include <cstdint>
+#include <stdexcept>
+#include <string>
+#include <utility>
+#include <vector>
+
+// Third Party Library Includes
+#include <nlohmann/json.hpp>
+
+namespace mirror::sync_scheduler
+{
+enum class SyncMethod : std::uint8_t
+{
+    SCRIPT,
+    RSYNC,
+    UNSET,
+};
+
+class SyncDetails
+{
+  public: // Constructors
+    explicit SyncDetails(const nlohmann::json& project);
+
+  public: // Methods
+    [[nodiscard]]
+    auto get_syncs_per_day() const -> std::size_t
+    {
+        return m_SyncConfig.at("syncs_per_day").get<std::size_t>();
+    }
+
+    [[nodiscard]]
+    auto get_sync_method() const -> SyncMethod
+    {
+        return m_SyncMethod;
+    }
+
+    [[nodiscard]]
+    auto get_password_file() const -> nlohmann::json
+    {
+        return m_SyncConfig.value("password_file", "");
+    }
+
+    [[nodiscard]]
+    auto get_sync_config() const -> nlohmann::json
+    {
+        return m_SyncConfig;
+    }
+
+  private: // Methods
+    [[nodiscard]]
+    static auto compose_rsync_command(
+        const nlohmann::json& rsyncConfig,
+        const std::string&    optionsKey
+    ) -> std::string;
+
+    [[nodiscard]]
+    static auto generate_sync_config(const nlohmann::json& project)
+        -> std::pair<SyncMethod, nlohmann::json>;
+
+    [[nodiscard]]
+    static auto generate_rsync_config(const nlohmann::json& project)
+        -> std::pair<SyncMethod, nlohmann::json>;
+
+    [[nodiscard]]
+    static auto generate_script_config(const nlohmann::json& project)
+        -> std::pair<SyncMethod, nlohmann::json>;
+
+  private: // Members
+    SyncMethod     m_SyncMethod;
+    nlohmann::json m_SyncConfig;
+};
+
+struct static_project_exception : std::runtime_error
+{
+    using std::runtime_error::runtime_error;
+};
+} // namespace mirror::sync_scheduler

mirror-sync-scheduler/include/mirror/sync_scheduler/SyncJob.hpp

@@ -0,0 +1,22 @@
+/**
+ * @file SyncJob.hpp
+ * @author Cary Keesler
+ * @brief
+ */
+
+#pragma once
+
+// System Includes
+#include <unistd.h>
+
+// Standard Library Includes
+#include <chrono>
+
+namespace mirror::sync_scheduler
+{
+struct SyncJob
+{
+    ::pid_t                                            processID = 0;
+    std::chrono::time_point<std::chrono::system_clock> startTime;
+};
+} // namespace mirror::sync_scheduler