Merge pull request #42 from openlawlibrary/ndusan/history-rdf-fixes

fix: insert history rdf fixes and add filesystem logging

Cargo.toml

@@ -25,9 +25,10 @@ lazy_static = "1.4.0"
 regex = "1"
 serde = "1.0"
 serde_json = "1.0"
-tracing = "0.1.37"
-tracing-subscriber = "0.3.16"
-tracing-actix-web = "0.6.2"
+tracing = "0.1"
+tracing-appender = "0.2"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+tracing-actix-web = "0.7"
 derive_more = "0.99.17"
 toml = "0.8.8"
 toml_edit = "0.22"

src/history/changes.rs

@@ -19,6 +19,7 @@ use crate::db::statements::queries::{
 };
 use crate::history::rdf::graph::StelaeGraph;
 use crate::history::rdf::namespaces::{dcterms, oll};
+use crate::stelae::stele::Stele;
 use crate::utils::archive::get_name_parts;
 use crate::utils::git::Repo;
 use crate::{
@@ -61,6 +62,7 @@ pub async fn insert(
             process::exit(1);
         }
     };
+    tracing::info!("Inserting history into archive");
     if let Some(_stele) = stele {
         insert_changes_single_stele()?;
     } else {
@@ -91,22 +93,52 @@ async fn insert_changes_archive(
         false,
     )?;
 
+    let mut errors = Vec::new();
     for (name, mut stele) in archive.get_stelae() {
-        if let Some(repositories) = stele.get_repositories()? {
-            let Some(rdf_data) = repositories.get_rdf_repository() else {
-                continue;
-            };
-            let rdf_repo_path = archive_path.to_path_buf().join(&rdf_data.name);
-            if !rdf_repo_path.exists() {
-                anyhow::bail!(
-                    "RDF repository should exist on disk but not found: {}",
-                    rdf_repo_path.display()
-                );
-            }
-            let (rdf_org, rdf_name) = get_name_parts(&rdf_data.name)?;
-            let rdf_repo = Repo::new(archive_path, &rdf_org, &rdf_name)?;
-            insert_changes_from_rdf_repository(conn, rdf_repo, &name).await?;
+        match process_stele(conn, &name, &mut stele, archive_path).await {
+            Ok(()) => (),
+            Err(err) => errors.push(err),
+        }
+    }
+    if !errors.is_empty() {
+        let error_msg = errors
+            .into_iter()
+            .map(|err| err.to_string())
+            .collect::<Vec<_>>()
+            .join("\n");
+        return Err(anyhow::anyhow!(
+            "Errors occurred while inserting changes:\n{error_msg}"
+        ));
+    }
+    Ok(())
+}
+
+/// Process the stele and insert changes into the database
+async fn process_stele(
+    conn: &DatabaseConnection,
+    name: &str,
+    stele: &mut Stele,
+    archive_path: &Path,
+) -> anyhow::Result<()> {
+    let Ok(found_repositories) = stele.get_repositories() else {
+        tracing::warn!("No repositories found for stele: {name}");
+        return Ok(());
+    };
+    if let Some(repositories) = found_repositories {
+        let Some(rdf_repo) = repositories.get_rdf_repository() else {
+            tracing::warn!("No RDF repository found for stele: {name}");
+            return Ok(());
+        };
+        let rdf_repo_path = archive_path.to_path_buf().join(&rdf_repo.name);
+        if !rdf_repo_path.exists() {
+            return Err(anyhow::anyhow!(
+                "RDF repository should exist on disk but not found: {}",
+                rdf_repo_path.display()
+            ));
         }
+        let (rdf_org, rdf_name) = get_name_parts(&rdf_repo.name)?;
+        let rdf = Repo::new(archive_path, &rdf_org, &rdf_name)?;
+        insert_changes_from_rdf_repository(conn, rdf, name).await?;
     }
     Ok(())
 }

src/server/api/versions/mod.rs

@@ -85,7 +85,7 @@ pub async fn versions(
     });
 
     if active_version == current_date {
-        active_version = CURRENT_VERSION_DATE.to_owned();
+        CURRENT_VERSION_DATE.clone_into(&mut active_version);
     }
 
     let messages = messages::historical(
@@ -97,7 +97,7 @@ pub async fn versions(
     );
 
     if active_publication_name == current_publication.name.clone() {
-        active_publication_name = CURRENT_PUBLICATION_NAME.to_owned();
+        CURRENT_PUBLICATION_NAME.clone_into(&mut active_publication_name);
     }
 
     response::Version::insert_if_not_present(&mut versions, params.date.clone());

src/server/api/versions/response/mod.rs

@@ -24,7 +24,7 @@ pub struct Versions {
     /// Resolves to "current" if the latest version is selected.
     pub active_version: String,
     /// Currently selected version to compare against.
-    /// If compare_date is specified, this will be the date to compare against.
+    /// If `compare_date` is specified, this will be the date to compare against.
     pub active_compare_to: Option<String>,
     /// Features for the versions endpoint.
     pub features: Features,

src/server/tracing.rs

@@ -3,6 +3,7 @@
 use std::time::Instant;
 
 use actix_web::{
+    body::MessageBody,
     dev::{ServiceRequest, ServiceResponse},
     HttpMessage,
 };
@@ -45,7 +46,7 @@ impl RootSpanBuilder for StelaeRootSpanBuilder {
         )
     }
 
-    fn on_request_end<B>(
+    fn on_request_end<B: MessageBody>(
         span: tracing::Span,
         outcome: &Result<ServiceResponse<B>, actix_web::Error>,
     ) {

src/stelae/types/repositories.rs

@@ -77,7 +77,7 @@ pub struct Custom {
     /// Routing to use when locating current blobs from the data repository.
     /// Example:
     ///
-    /// Given a `["_underscore/.*"] glob pattern, the following urls are expected to be routed to the current data repository:
+    /// Given a `["_underscore/.*"]` glob pattern, the following urls are expected to be routed to the current data repository:
     ///
     /// - `/_underscore/`
     /// - `/_underscore/any/path`

src/utils/cli.rs

@@ -11,9 +11,15 @@ use clap::Parser;
 use std::env;
 use std::io;
 use std::path::Path;
+use std::path::PathBuf;
 use std::process;
 use tracing;
+use tracing::Level;
+use tracing_appender::rolling;
 use tracing_subscriber::fmt;
+use tracing_subscriber::fmt::writer::MakeWriterExt;
+use tracing_subscriber::Layer;
+use tracing_subscriber::{filter::EnvFilter, layer::SubscriberExt, util::SubscriberInitExt};
 
 /// Stelae is currently just a simple git server.
 /// run from the library directory or pass
@@ -56,9 +62,41 @@ enum Subcommands {
     },
 }
 
+/// Place to initialize tracing
 ///
-fn init_tracing() {
-    fmt::init();
+/// We create `debug` and `error` log files in `.stelae` dir.
+/// `debug` log file contains all logs, `error` log file contains only `warn` and `error`
+/// NOTE: once `https://github.com/tokio-rs/tracing/pull/2497` is merged,
+/// update `init_tracing` to rotate log files based on size.
+#[allow(clippy::expect_used)]
+fn init_tracing(archive_path: &Path) {
+    let stelae_dir = archive_path.join(PathBuf::from("./.stelae"));
+
+    let debug_file_appender =
+        rolling::never(&stelae_dir, "stelae-debug.log").with_max_level(Level::DEBUG);
+    let error_file_appender =
+        rolling::never(&stelae_dir, "stelae-error.log").with_max_level(Level::WARN);
+
+    let mut debug_layer = fmt::layer().with_writer(debug_file_appender);
+    let mut error_layer = fmt::layer().with_writer(error_file_appender);
+
+    // disable ANSI colors.
+    // this is to avoid color coding in log files and to make it easier to read.
+    debug_layer = debug_layer.with_ansi(false);
+    error_layer = error_layer.with_ansi(false);
+    // also log to console
+    let console_layer = fmt::layer().with_target(true).with_filter(
+        EnvFilter::try_from_default_env()
+            .or_else(|_| EnvFilter::try_new("info"))
+            .expect("Failed to initialize console logging"),
+    );
+
+    tracing_subscriber::registry()
+        .with(debug_layer)
+        .with(error_layer)
+        .with(console_layer)
+        .init();
+
     if env::var("RUST_LOG").is_err() {
         env::set_var("RUST_LOG", "info");
     }
@@ -69,7 +107,6 @@ fn init_tracing() {
 /// # Errors
 /// TODO: This function should not return errors
 pub fn run() -> io::Result<()> {
-    init_tracing();
     tracing::debug!("Starting application");
     let cli = Cli::parse();
     let archive_path_wd = Path::new(&cli.archive_path);
@@ -81,13 +118,14 @@ pub fn run() -> io::Result<()> {
         process::exit(1);
     };
 
+    init_tracing(&archive_path);
+
     match cli.subcommands {
         Subcommands::Git { port } => serve_git(&cli.archive_path, archive_path, port),
         Subcommands::Serve { port, individual } => {
             serve_archive(&cli.archive_path, archive_path, port, individual)
         }
         Subcommands::InsertHistory { stele } => {
-            tracing::info!("Inserting history into archive");
             changes::insert(&cli.archive_path, archive_path, stele)
         }
     }