From 304a042f1df3986332489c6de32c301681c70af8 Mon Sep 17 00:00:00 2001 From: Lan Lou Date: Wed, 6 Nov 2024 20:26:13 -0500 Subject: [PATCH 01/28] Add implementation for create_new_epoch --- optd-persistent/src/cost_model/orm.rs | 87 ++++++++++++++++++++++++++- 1 file changed, 85 insertions(+), 2 deletions(-) diff --git a/optd-persistent/src/cost_model/orm.rs b/optd-persistent/src/cost_model/orm.rs index b6bd8ad..195e1b6 100644 --- a/optd-persistent/src/cost_model/orm.rs +++ b/optd-persistent/src/cost_model/orm.rs @@ -1,6 +1,11 @@ #![allow(dead_code, unused_imports, unused_variables)] -use crate::{BackendManager, CostModelStorageLayer, StorageResult}; +use sea_orm::{sqlx::types::chrono::Utc, EntityTrait}; + +use crate::{ + entities::event::{self, Entity as Event}, + BackendManager, CostModelStorageLayer, StorageResult, +}; use super::interface::CatalogSource; @@ -17,7 +22,20 @@ impl CostModelStorageLayer for BackendManager { source: String, data: String, ) -> StorageResult { - todo!() + let new_event = event::ActiveModel { + source_variant: sea_orm::ActiveValue::Set(source), + timestamp: sea_orm::ActiveValue::Set(Utc::now()), + data: sea_orm::ActiveValue::Set(sea_orm::JsonValue::String(data)), + ..Default::default() + }; + let res = Event::insert(new_event).exec(&self.db).await; + Ok(res.and_then(|insert_res| { + self.latest_epoch_id.store( + insert_res.last_insert_id as usize, + std::sync::atomic::Ordering::Relaxed, + ); + Ok(insert_res.last_insert_id) + })?) } async fn update_stats_from_catalog( @@ -88,3 +106,68 @@ impl CostModelStorageLayer for BackendManager { todo!() } } + +#[cfg(test)] +mod tests { + use crate::{migrate, CostModelStorageLayer}; + use sea_orm::{ConnectionTrait, Database, EntityTrait, ModelTrait}; + use serde_json::de; + + use crate::entities::event::Entity as Event; + + async fn run_migration(db_file: &str) -> String { + let database_url = format!("sqlite:./{}?mode=rwc", db_file); + remove_db_file(db_file); + + let db = Database::connect(database_url.clone()) + .await + .expect("Unable to connect to the database"); + + migrate(&db) + .await + .expect("Something went wrong during migration"); + + db.execute(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Sqlite, + "PRAGMA foreign_keys = ON;".to_owned(), + )) + .await + .expect("Unable to enable foreign keys"); + database_url.clone() + } + + fn remove_db_file(db_file: &str) { + let database_file = format!("./{}", db_file); + let _ = std::fs::remove_file(database_file); + } + + #[tokio::test] + async fn test_create_new_epoch() { + const DATABASE_FILE: &str = "test_create_new_epoch.db"; + let database_url = run_migration(DATABASE_FILE).await; + let mut binding = super::BackendManager::new(Some(&database_url)).await; + let backend_manager = binding.as_mut().unwrap(); + let res = backend_manager + .create_new_epoch("source".to_string(), "data".to_string()) + .await; + println!("{:?}", res); + assert!(res.is_ok()); + assert_eq!( + super::Event::find() + .all(&backend_manager.db) + .await + .unwrap() + .len(), + 1 + ); + println!( + "{:?}", + super::Event::find().all(&backend_manager.db).await.unwrap()[0] + ); + assert_eq!( + super::Event::find().all(&backend_manager.db).await.unwrap()[0].epoch_id, + res.unwrap() + ); + remove_db_file(DATABASE_FILE); + } +} From 99f36cf02ef5a293d390309ef543afefd8d5e3e2 Mon Sep 17 00:00:00 2001 From: Yuanxin Cao Date: Wed, 6 Nov 2024 22:33:13 -0500 Subject: [PATCH 02/28] implement get_stats methods --- optd-persistent/src/cost_model/interface.rs | 20 ++--- optd-persistent/src/cost_model/orm.rs | 87 +++++++++++++++------ 2 files changed, 70 insertions(+), 37 deletions(-) diff --git a/optd-persistent/src/cost_model/interface.rs b/optd-persistent/src/cost_model/interface.rs index 647a83a..d713560 100644 --- a/optd-persistent/src/cost_model/interface.rs +++ b/optd-persistent/src/cost_model/interface.rs @@ -5,6 +5,7 @@ use crate::entities::event::Model as event_model; use crate::entities::logical_expression; use crate::entities::physical_expression; use crate::StorageResult; +use sea_orm::prelude::Json; use sea_orm::*; use sea_orm_migration::prelude::*; use serde_json::json; @@ -43,8 +44,7 @@ pub trait CostModelStorageLayer { epoch_id: Self::EpochId, ) -> StorageResult<()>; - // i32 in `stats:i32` is a placeholder for the stats type - async fn update_stats(&self, stats: i32, epoch_id: Self::EpochId) -> StorageResult<()>; + async fn update_stats(&self, stats: Json, epoch_id: Self::EpochId) -> StorageResult<()>; async fn store_cost( &self, @@ -68,27 +68,17 @@ pub trait CostModelStorageLayer { // TODO: Add enum for stat_type stat_type: i32, epoch_id: Option, - ) -> StorageResult>; + ) -> StorageResult>; - /// Get the statistics for a given attribute. + /// Get the (joint) statistics for one or more attributes. /// /// If `epoch_id` is None, it will return the latest statistics. async fn get_stats_for_attr( - &self, - attr_id: Self::AttrId, - stat_type: i32, - epoch_id: Option, - ) -> StorageResult>; - - /// Get the joint statistics for a list of attributes. - /// - /// If `epoch_id` is None, it will return the latest statistics. - async fn get_stats_for_attrs( &self, attr_ids: Vec, stat_type: i32, epoch_id: Option, - ) -> StorageResult>; + ) -> StorageResult>; async fn get_cost_analysis( &self, diff --git a/optd-persistent/src/cost_model/orm.rs b/optd-persistent/src/cost_model/orm.rs index 195e1b6..ed8e1bc 100644 --- a/optd-persistent/src/cost_model/orm.rs +++ b/optd-persistent/src/cost_model/orm.rs @@ -1,11 +1,10 @@ #![allow(dead_code, unused_imports, unused_variables)] +use crate::entities::{prelude::*, *}; +use crate::{BackendManager, CostModelStorageLayer, StorageResult}; +use sea_orm::prelude::Json; use sea_orm::{sqlx::types::chrono::Utc, EntityTrait}; - -use crate::{ - entities::event::{self, Entity as Event}, - BackendManager, CostModelStorageLayer, StorageResult, -}; +use sea_orm::{ColumnTrait, QueryFilter, QueryOrder, QuerySelect}; use super::interface::CatalogSource; @@ -46,7 +45,7 @@ impl CostModelStorageLayer for BackendManager { todo!() } - async fn update_stats(&self, stats: i32, epoch_id: Self::EpochId) -> StorageResult<()> { + async fn update_stats(&self, stats: Json, epoch_id: Self::EpochId) -> StorageResult<()> { todo!() } @@ -72,26 +71,67 @@ impl CostModelStorageLayer for BackendManager { table_id: i32, stat_type: i32, epoch_id: Option, - ) -> StorageResult> { - todo!() + ) -> StorageResult> { + match epoch_id { + Some(epoch_id) => Ok(VersionedStatistic::find() + .filter(versioned_statistic::Column::EpochId.eq(epoch_id)) + .inner_join(statistic::Entity) + .filter(statistic::Column::TableId.eq(table_id)) + .filter(statistic::Column::StatisticType.eq(stat_type)) + .one(&self.db) + .await? + .map(|stat| stat.statistic_value)), + + None => Ok(VersionedStatistic::find() + .inner_join(statistic::Entity) + .filter(statistic::Column::TableId.eq(table_id)) + .filter(statistic::Column::StatisticType.eq(stat_type)) + .order_by_desc(versioned_statistic::Column::EpochId) + .one(&self.db) + .await? + .map(|stat| stat.statistic_value)), + } } async fn get_stats_for_attr( &self, - attr_id: i32, + mut attr_ids: Vec, stat_type: i32, epoch_id: Option, - ) -> StorageResult> { - todo!() - } - - async fn get_stats_for_attrs( - &self, - attr_ids: Vec, - stat_type: i32, - epoch_id: Option, - ) -> StorageResult> { - todo!() + ) -> StorageResult> { + let attr_num = attr_ids.len() as i32; + // The description is to concat `attr_ids` using commas + // Note that `attr_ids` should be sorted before concatenation + // e.g. [1, 2, 3] -> "1,2,3" + attr_ids.sort(); + let description = attr_ids + .iter() + .map(|id| id.to_string()) + .collect::>() + .join(","); + + // We don't join with junction table here for faster lookup. + match epoch_id { + Some(epoch_id) => Ok(VersionedStatistic::find() + .filter(versioned_statistic::Column::EpochId.eq(epoch_id)) + .inner_join(statistic::Entity) + .filter(statistic::Column::NumberOfAttributes.eq(attr_num)) + .filter(statistic::Column::Description.eq(description)) + .filter(statistic::Column::StatisticType.eq(stat_type)) + .one(&self.db) + .await? + .map(|stat| stat.statistic_value)), + + None => Ok(VersionedStatistic::find() + .inner_join(statistic::Entity) + .filter(statistic::Column::NumberOfAttributes.eq(attr_num)) + .filter(statistic::Column::Description.eq(description)) + .filter(statistic::Column::StatisticType.eq(stat_type)) + .order_by_desc(versioned_statistic::Column::EpochId) + .one(&self.db) + .await? + .map(|stat| stat.statistic_value)), + } } async fn get_cost_analysis( @@ -110,10 +150,13 @@ impl CostModelStorageLayer for BackendManager { #[cfg(test)] mod tests { use crate::{migrate, CostModelStorageLayer}; - use sea_orm::{ConnectionTrait, Database, EntityTrait, ModelTrait}; + use sea_orm::{ + ColumnTrait, ConnectionTrait, Database, DbBackend, EntityTrait, ModelTrait, QueryFilter, + QuerySelect, QueryTrait, + }; use serde_json::de; - use crate::entities::event::Entity as Event; + use crate::entities::{prelude::*, *}; async fn run_migration(db_file: &str) -> String { let database_url = format!("sqlite:./{}?mode=rwc", db_file); From 9f71c05aa46172d3ec7a8f89379dedc509620140 Mon Sep 17 00:00:00 2001 From: Lan Lou Date: Thu, 7 Nov 2024 08:18:10 -0500 Subject: [PATCH 03/28] Initial draft of update_stat --- optd-persistent/src/cost_model/interface.rs | 12 +- optd-persistent/src/cost_model/orm.rs | 164 ++++++++++++++++++-- optd-persistent/src/lib.rs | 1 + 3 files changed, 166 insertions(+), 11 deletions(-) diff --git a/optd-persistent/src/cost_model/interface.rs b/optd-persistent/src/cost_model/interface.rs index d713560..2062a59 100644 --- a/optd-persistent/src/cost_model/interface.rs +++ b/optd-persistent/src/cost_model/interface.rs @@ -22,6 +22,15 @@ pub enum StatisticType { Max, } +pub struct Stat { + pub stat_type: i32, + // TODO(lanlou): what should I use for the value type? + pub stat_value: String, + pub attr_ids: Vec, + pub table_id: Option, + pub name: String, +} + #[trait_variant::make(Send)] pub trait CostModelStorageLayer { type GroupId; @@ -44,7 +53,8 @@ pub trait CostModelStorageLayer { epoch_id: Self::EpochId, ) -> StorageResult<()>; - async fn update_stats(&self, stats: Json, epoch_id: Self::EpochId) -> StorageResult<()>; + // i32 in `stats:i32` is a placeholder for the stats type + async fn update_stats(&self, stat: Stat, epoch_id: Self::EpochId) -> StorageResult<()>; async fn store_cost( &self, diff --git a/optd-persistent/src/cost_model/orm.rs b/optd-persistent/src/cost_model/orm.rs index ed8e1bc..7a909e4 100644 --- a/optd-persistent/src/cost_model/orm.rs +++ b/optd-persistent/src/cost_model/orm.rs @@ -1,12 +1,33 @@ #![allow(dead_code, unused_imports, unused_variables)] +use std::i32; +use std::ptr::null; + use crate::entities::{prelude::*, *}; -use crate::{BackendManager, CostModelStorageLayer, StorageResult}; +use crate::{BackendError, BackendManager, CostModelError, CostModelStorageLayer, StorageResult}; use sea_orm::prelude::Json; use sea_orm::{sqlx::types::chrono::Utc, EntityTrait}; -use sea_orm::{ColumnTrait, QueryFilter, QueryOrder, QuerySelect}; +use sea_orm::{ + ActiveModelTrait, ColumnTrait, DbErr, ModelTrait, QueryFilter, QueryOrder, QuerySelect, + RuntimeErr, +}; + +use super::interface::{CatalogSource, Stat}; -use super::interface::CatalogSource; +impl BackendManager { + fn get_description_from_attr_ids( + &self, + attr_ids: Vec<::AttrId>, + ) -> String { + let mut attr_ids = attr_ids; + attr_ids.sort(); + attr_ids + .iter() + .map(|id| id.to_string()) + .collect::>() + .join(",") + } +} impl CostModelStorageLayer for BackendManager { type GroupId = i32; @@ -45,8 +66,135 @@ impl CostModelStorageLayer for BackendManager { todo!() } - async fn update_stats(&self, stats: Json, epoch_id: Self::EpochId) -> StorageResult<()> { - todo!() + async fn update_stats(&self, stat: Stat, epoch_id: Self::EpochId) -> StorageResult<()> { + // 0. Check if the stat already exists. If exists, get stat_id, else insert into statistic table. + let stat_id = match stat.table_id { + Some(table_id) => { + let res = Statistic::find() + .filter(statistic::Column::TableId.eq(table_id)) + .filter(statistic::Column::StatisticType.eq(stat.stat_type)) + .one(&self.db) + .await?; + match res { + Some(stat_data) => stat_data.id, + None => { + let new_stat = statistic::ActiveModel { + name: sea_orm::ActiveValue::Set(stat.name.clone()), + table_id: sea_orm::ActiveValue::Set(table_id), + number_of_attributes: sea_orm::ActiveValue::Set( + stat.attr_ids.len() as i32 + ), + created_time: sea_orm::ActiveValue::Set(Utc::now()), + statistic_type: sea_orm::ActiveValue::Set(stat.stat_type), + description: sea_orm::ActiveValue::Set("".to_string()), + ..Default::default() + }; + let res = Statistic::insert(new_stat).exec(&self.db).await; + match res { + Ok(insert_res) => insert_res.last_insert_id, + Err(_) => { + return Err(BackendError::Database(DbErr::Exec( + RuntimeErr::Internal( + "Failed to insert into statistic table".to_string(), + ), + ))) + } + } + } + } + } + None => { + let description = self.get_description_from_attr_ids(stat.attr_ids.clone()); + let res = Statistic::find() + .filter(statistic::Column::NumberOfAttributes.eq(stat.attr_ids.len() as i32)) + .filter(statistic::Column::Description.eq(description.clone())) + .filter(statistic::Column::StatisticType.eq(stat.stat_type)) + .one(&self.db) + .await?; + match res { + Some(stat_data) => stat_data.id, + None => { + let new_stat = statistic::ActiveModel { + name: sea_orm::ActiveValue::Set(stat.name.clone()), + number_of_attributes: sea_orm::ActiveValue::Set( + stat.attr_ids.len() as i32 + ), + created_time: sea_orm::ActiveValue::Set(Utc::now()), + statistic_type: sea_orm::ActiveValue::Set(stat.stat_type), + description: sea_orm::ActiveValue::Set(description), + ..Default::default() + }; + // TODO(lanlou): we should not clone here maybe... + let insert_res = Statistic::insert(new_stat.clone()).exec(&self.db).await?; + for attr_id in stat.attr_ids { + let new_junction = statistic_to_attribute_junction::ActiveModel { + statistic_id: sea_orm::ActiveValue::Set(insert_res.last_insert_id), + attribute_id: sea_orm::ActiveValue::Set(attr_id), + }; + let res = StatisticToAttributeJunction::insert(new_junction) + .exec(&self.db) + .await; + if res.is_err() { + let _ = new_stat.delete(&self.db).await; + return Err(BackendError::Database(DbErr::Exec( + RuntimeErr::Internal( + "Failed to insert into statistic_to_attribute_junction table".to_string(), + ), + ))); + } + } + insert_res.last_insert_id + } + } + } + }; + // TODO(lanlou): use join previously to filter, so we don't need another query here. + let res = VersionedStatistic::find() + .filter(versioned_statistic::Column::StatisticId.eq(stat_id)) + .order_by_desc(versioned_statistic::Column::EpochId) + .one(&self.db) + .await?; + if res.is_some() { + if res.unwrap().statistic_value == sea_orm::JsonValue::String(stat.stat_value.clone()) { + return Ok(()); + } + } + // 1. Insert into attr_stats and related junction tables. + let new_stats = versioned_statistic::ActiveModel { + epoch_id: sea_orm::ActiveValue::Set(epoch_id), + statistic_id: sea_orm::ActiveValue::Set(stat_id), + statistic_value: sea_orm::ActiveValue::Set(sea_orm::JsonValue::String(stat.stat_value)), + ..Default::default() + }; + let _ = VersionedStatistic::insert(new_stats).exec(&self.db).await?; + + // 2. Invalidate all the related cost. + // TODO(lanlou): better handle error, let everything atomic :( + let related_exprs = physical_expression_to_statistic_junction::Entity::find() + .filter(physical_expression_to_statistic_junction::Column::StatisticId.eq(stat_id)) + .all(&self.db) + .await?; + for expr in related_exprs { + let res = PlanCost::find() + .filter(plan_cost::Column::PhysicalExpressionId.eq(expr.physical_expression_id)) + .filter(plan_cost::Column::IsValid.eq(true)) + .all(&self.db) + .await?; + assert!(res.len() <= 1); + for cost in res { + // TODO(lanlou): better way to update the cost? + let new_cost = plan_cost::ActiveModel { + id: sea_orm::ActiveValue::Set(cost.id), + physical_expression_id: sea_orm::ActiveValue::Set(cost.physical_expression_id), + epoch_id: sea_orm::ActiveValue::Set(cost.epoch_id), + cost: sea_orm::ActiveValue::Set(cost.cost), + is_valid: sea_orm::ActiveValue::Set(false), + }; + let _ = new_cost.update(&self.db).await?; + } + } + + Ok(()) } async fn store_cost( @@ -104,11 +252,7 @@ impl CostModelStorageLayer for BackendManager { // Note that `attr_ids` should be sorted before concatenation // e.g. [1, 2, 3] -> "1,2,3" attr_ids.sort(); - let description = attr_ids - .iter() - .map(|id| id.to_string()) - .collect::>() - .join(","); + let description = self.get_description_from_attr_ids(attr_ids); // We don't join with junction table here for faster lookup. match epoch_id { diff --git a/optd-persistent/src/lib.rs b/optd-persistent/src/lib.rs index 70d7016..adb3687 100644 --- a/optd-persistent/src/lib.rs +++ b/optd-persistent/src/lib.rs @@ -19,6 +19,7 @@ pub type StorageResult = Result; pub enum CostModelError { // TODO: Add more error types UnknownStatisticType, + VersionedStatisticNotFound, } #[derive(Debug)] From 5dce3dda4acf0c33084ea10cdc060daa81c294bd Mon Sep 17 00:00:00 2001 From: Lan Lou Date: Thu, 7 Nov 2024 08:57:16 -0500 Subject: [PATCH 04/28] Small optimization --- optd-persistent/src/cost_model/orm.rs | 75 +++++++++++++++------------ 1 file changed, 41 insertions(+), 34 deletions(-) diff --git a/optd-persistent/src/cost_model/orm.rs b/optd-persistent/src/cost_model/orm.rs index 7a909e4..d9430f0 100644 --- a/optd-persistent/src/cost_model/orm.rs +++ b/optd-persistent/src/cost_model/orm.rs @@ -5,7 +5,7 @@ use std::ptr::null; use crate::entities::{prelude::*, *}; use crate::{BackendError, BackendManager, CostModelError, CostModelStorageLayer, StorageResult}; -use sea_orm::prelude::Json; +use sea_orm::prelude::{Expr, Json}; use sea_orm::{sqlx::types::chrono::Utc, EntityTrait}; use sea_orm::{ ActiveModelTrait, ColumnTrait, DbErr, ModelTrait, QueryFilter, QueryOrder, QuerySelect, @@ -70,13 +70,23 @@ impl CostModelStorageLayer for BackendManager { // 0. Check if the stat already exists. If exists, get stat_id, else insert into statistic table. let stat_id = match stat.table_id { Some(table_id) => { + // TODO(lanlou): only select needed fields let res = Statistic::find() .filter(statistic::Column::TableId.eq(table_id)) .filter(statistic::Column::StatisticType.eq(stat.stat_type)) + // FIX_ME: Do we need the following filter? + .inner_join(versioned_statistic::Entity) + .select_also(versioned_statistic::Entity) + .order_by_desc(versioned_statistic::Column::EpochId) .one(&self.db) .await?; match res { - Some(stat_data) => stat_data.id, + Some(stat_data) => { + if stat_data.1.unwrap().statistic_value == stat.stat_value { + return Ok(()); + } + stat_data.0.id + } None => { let new_stat = statistic::ActiveModel { name: sea_orm::ActiveValue::Set(stat.name.clone()), @@ -109,10 +119,19 @@ impl CostModelStorageLayer for BackendManager { .filter(statistic::Column::NumberOfAttributes.eq(stat.attr_ids.len() as i32)) .filter(statistic::Column::Description.eq(description.clone())) .filter(statistic::Column::StatisticType.eq(stat.stat_type)) + // FIX_ME: Do we need the following filter? + .inner_join(versioned_statistic::Entity) + .select_also(versioned_statistic::Entity) + .order_by_desc(versioned_statistic::Column::EpochId) .one(&self.db) .await?; match res { - Some(stat_data) => stat_data.id, + Some(stat_data) => { + if stat_data.1.unwrap().statistic_value == stat.stat_value { + return Ok(()); + } + stat_data.0.id + } None => { let new_stat = statistic::ActiveModel { name: sea_orm::ActiveValue::Set(stat.name.clone()), @@ -148,17 +167,6 @@ impl CostModelStorageLayer for BackendManager { } } }; - // TODO(lanlou): use join previously to filter, so we don't need another query here. - let res = VersionedStatistic::find() - .filter(versioned_statistic::Column::StatisticId.eq(stat_id)) - .order_by_desc(versioned_statistic::Column::EpochId) - .one(&self.db) - .await?; - if res.is_some() { - if res.unwrap().statistic_value == sea_orm::JsonValue::String(stat.stat_value.clone()) { - return Ok(()); - } - } // 1. Insert into attr_stats and related junction tables. let new_stats = versioned_statistic::ActiveModel { epoch_id: sea_orm::ActiveValue::Set(epoch_id), @@ -170,29 +178,28 @@ impl CostModelStorageLayer for BackendManager { // 2. Invalidate all the related cost. // TODO(lanlou): better handle error, let everything atomic :( - let related_exprs = physical_expression_to_statistic_junction::Entity::find() + let related_costs: Vec = plan_cost::Entity::find() + .filter(plan_cost::Column::IsValid.eq(true)) + .join_rev( + sea_orm::JoinType::InnerJoin, + physical_expression_to_statistic_junction::Entity::belongs_to(plan_cost::Entity) + .from(physical_expression_to_statistic_junction::Column::PhysicalExpressionId) + .to(plan_cost::Column::PhysicalExpressionId) + .into(), + ) .filter(physical_expression_to_statistic_junction::Column::StatisticId.eq(stat_id)) + .select_only() + .column(plan_cost::Column::Id) .all(&self.db) + .await? + .iter() + .map(|cost| cost.id) + .collect(); + let _ = plan_cost::Entity::update_many() + .col_expr(plan_cost::Column::IsValid, Expr::value(false)) + .filter(plan_cost::Column::Id.is_in(related_costs)) + .exec(&self.db) .await?; - for expr in related_exprs { - let res = PlanCost::find() - .filter(plan_cost::Column::PhysicalExpressionId.eq(expr.physical_expression_id)) - .filter(plan_cost::Column::IsValid.eq(true)) - .all(&self.db) - .await?; - assert!(res.len() <= 1); - for cost in res { - // TODO(lanlou): better way to update the cost? - let new_cost = plan_cost::ActiveModel { - id: sea_orm::ActiveValue::Set(cost.id), - physical_expression_id: sea_orm::ActiveValue::Set(cost.physical_expression_id), - epoch_id: sea_orm::ActiveValue::Set(cost.epoch_id), - cost: sea_orm::ActiveValue::Set(cost.cost), - is_valid: sea_orm::ActiveValue::Set(false), - }; - let _ = new_cost.update(&self.db).await?; - } - } Ok(()) } From c05d6642a4c50525a08516d69274f18e75dc7882 Mon Sep 17 00:00:00 2001 From: Lan Lou Date: Thu, 7 Nov 2024 09:22:02 -0500 Subject: [PATCH 05/28] Basic error handling --- optd-persistent/src/cost_model/orm.rs | 73 +++++++++++++++++++-------- 1 file changed, 51 insertions(+), 22 deletions(-) diff --git a/optd-persistent/src/cost_model/orm.rs b/optd-persistent/src/cost_model/orm.rs index d9430f0..60bba51 100644 --- a/optd-persistent/src/cost_model/orm.rs +++ b/optd-persistent/src/cost_model/orm.rs @@ -6,10 +6,11 @@ use std::ptr::null; use crate::entities::{prelude::*, *}; use crate::{BackendError, BackendManager, CostModelError, CostModelStorageLayer, StorageResult}; use sea_orm::prelude::{Expr, Json}; +use sea_orm::sea_query::Query; use sea_orm::{sqlx::types::chrono::Utc, EntityTrait}; use sea_orm::{ - ActiveModelTrait, ColumnTrait, DbErr, ModelTrait, QueryFilter, QueryOrder, QuerySelect, - RuntimeErr, + ActiveModelTrait, ColumnTrait, DbErr, DeleteResult, EntityOrSelect, ModelTrait, QueryFilter, + QueryOrder, QuerySelect, RuntimeErr, }; use super::interface::{CatalogSource, Stat}; @@ -68,6 +69,7 @@ impl CostModelStorageLayer for BackendManager { async fn update_stats(&self, stat: Stat, epoch_id: Self::EpochId) -> StorageResult<()> { // 0. Check if the stat already exists. If exists, get stat_id, else insert into statistic table. + let mut stat_inserted = false; let stat_id = match stat.table_id { Some(table_id) => { // TODO(lanlou): only select needed fields @@ -88,6 +90,7 @@ impl CostModelStorageLayer for BackendManager { stat_data.0.id } None => { + stat_inserted = true; let new_stat = statistic::ActiveModel { name: sea_orm::ActiveValue::Set(stat.name.clone()), table_id: sea_orm::ActiveValue::Set(table_id), @@ -133,6 +136,7 @@ impl CostModelStorageLayer for BackendManager { stat_data.0.id } None => { + stat_inserted = true; let new_stat = statistic::ActiveModel { name: sea_orm::ActiveValue::Set(stat.name.clone()), number_of_attributes: sea_orm::ActiveValue::Set( @@ -174,32 +178,57 @@ impl CostModelStorageLayer for BackendManager { statistic_value: sea_orm::ActiveValue::Set(sea_orm::JsonValue::String(stat.stat_value)), ..Default::default() }; - let _ = VersionedStatistic::insert(new_stats).exec(&self.db).await?; + let insert_res = VersionedStatistic::insert(new_stats).exec(&self.db).await; + if insert_res.is_err() && stat_inserted { + // TODO(lanlou): update it with txn. + let delete_res = Statistic::delete_by_id(stat_id).exec(&self.db).await; + return Err(BackendError::Database(DbErr::Exec(RuntimeErr::Internal( + format!("Failed to insert into versioned_statistic table. And statistic with id {} is{} deleted.", stat_id, if delete_res.is_err() { "not" } else { "" }), + )))); + } // 2. Invalidate all the related cost. // TODO(lanlou): better handle error, let everything atomic :( - let related_costs: Vec = plan_cost::Entity::find() + let update_res = plan_cost::Entity::update_many() + .col_expr(plan_cost::Column::IsValid, Expr::value(false)) .filter(plan_cost::Column::IsValid.eq(true)) - .join_rev( - sea_orm::JoinType::InnerJoin, - physical_expression_to_statistic_junction::Entity::belongs_to(plan_cost::Entity) - .from(physical_expression_to_statistic_junction::Column::PhysicalExpressionId) - .to(plan_cost::Column::PhysicalExpressionId) - .into(), + .filter( + plan_cost::Column::PhysicalExpressionId.in_subquery( + (*Query::select() + .column( + physical_expression_to_statistic_junction::Column::PhysicalExpressionId, + ) + .from(physical_expression_to_statistic_junction::Entity) + .and_where( + physical_expression_to_statistic_junction::Column::StatisticId + .eq(stat_id), + )) + .to_owned(), + ), ) - .filter(physical_expression_to_statistic_junction::Column::StatisticId.eq(stat_id)) - .select_only() - .column(plan_cost::Column::Id) - .all(&self.db) - .await? - .iter() - .map(|cost| cost.id) - .collect(); - let _ = plan_cost::Entity::update_many() - .col_expr(plan_cost::Column::IsValid, Expr::value(false)) - .filter(plan_cost::Column::Id.is_in(related_costs)) .exec(&self.db) - .await?; + .await; + if update_res.is_err() { + let delete_versioned_res = + VersionedStatistic::delete_by_id(insert_res.unwrap().last_insert_id) + .exec(&self.db) + .await; + let delete_res = if stat_inserted { + Statistic::delete_by_id(stat_id).exec(&self.db).await + } else { + Ok(DeleteResult { rows_affected: (0) }) + }; + return Err(BackendError::Database(DbErr::Exec(RuntimeErr::Internal( + format!( + "Failed to update plan_cost table. And related deletion is{} done.", + if !delete_res.is_err() && !delete_versioned_res.is_err() { + "" + } else { + " not" + } + ), + )))); + } Ok(()) } From 38406732f424613b3e8bf3a09f8264c8418a1f00 Mon Sep 17 00:00:00 2001 From: Lan Lou Date: Thu, 7 Nov 2024 09:27:45 -0500 Subject: [PATCH 06/28] Finish update_stats and store_expr_stats_mappings --- optd-persistent/src/cost_model/orm.rs | 54 +++++++++------------------ 1 file changed, 18 insertions(+), 36 deletions(-) diff --git a/optd-persistent/src/cost_model/orm.rs b/optd-persistent/src/cost_model/orm.rs index 60bba51..dfe7440 100644 --- a/optd-persistent/src/cost_model/orm.rs +++ b/optd-persistent/src/cost_model/orm.rs @@ -10,7 +10,7 @@ use sea_orm::sea_query::Query; use sea_orm::{sqlx::types::chrono::Utc, EntityTrait}; use sea_orm::{ ActiveModelTrait, ColumnTrait, DbErr, DeleteResult, EntityOrSelect, ModelTrait, QueryFilter, - QueryOrder, QuerySelect, RuntimeErr, + QueryOrder, QuerySelect, RuntimeErr, TransactionTrait, }; use super::interface::{CatalogSource, Stat}; @@ -68,8 +68,8 @@ impl CostModelStorageLayer for BackendManager { } async fn update_stats(&self, stat: Stat, epoch_id: Self::EpochId) -> StorageResult<()> { + let transaction = self.db.begin().await?; // 0. Check if the stat already exists. If exists, get stat_id, else insert into statistic table. - let mut stat_inserted = false; let stat_id = match stat.table_id { Some(table_id) => { // TODO(lanlou): only select needed fields @@ -90,7 +90,6 @@ impl CostModelStorageLayer for BackendManager { stat_data.0.id } None => { - stat_inserted = true; let new_stat = statistic::ActiveModel { name: sea_orm::ActiveValue::Set(stat.name.clone()), table_id: sea_orm::ActiveValue::Set(table_id), @@ -136,7 +135,6 @@ impl CostModelStorageLayer for BackendManager { stat_data.0.id } None => { - stat_inserted = true; let new_stat = statistic::ActiveModel { name: sea_orm::ActiveValue::Set(stat.name.clone()), number_of_attributes: sea_orm::ActiveValue::Set( @@ -178,18 +176,10 @@ impl CostModelStorageLayer for BackendManager { statistic_value: sea_orm::ActiveValue::Set(sea_orm::JsonValue::String(stat.stat_value)), ..Default::default() }; - let insert_res = VersionedStatistic::insert(new_stats).exec(&self.db).await; - if insert_res.is_err() && stat_inserted { - // TODO(lanlou): update it with txn. - let delete_res = Statistic::delete_by_id(stat_id).exec(&self.db).await; - return Err(BackendError::Database(DbErr::Exec(RuntimeErr::Internal( - format!("Failed to insert into versioned_statistic table. And statistic with id {} is{} deleted.", stat_id, if delete_res.is_err() { "not" } else { "" }), - )))); - } + let _ = VersionedStatistic::insert(new_stats).exec(&self.db).await; // 2. Invalidate all the related cost. - // TODO(lanlou): better handle error, let everything atomic :( - let update_res = plan_cost::Entity::update_many() + let _ = plan_cost::Entity::update_many() .col_expr(plan_cost::Column::IsValid, Expr::value(false)) .filter(plan_cost::Column::IsValid.eq(true)) .filter( @@ -208,28 +198,8 @@ impl CostModelStorageLayer for BackendManager { ) .exec(&self.db) .await; - if update_res.is_err() { - let delete_versioned_res = - VersionedStatistic::delete_by_id(insert_res.unwrap().last_insert_id) - .exec(&self.db) - .await; - let delete_res = if stat_inserted { - Statistic::delete_by_id(stat_id).exec(&self.db).await - } else { - Ok(DeleteResult { rows_affected: (0) }) - }; - return Err(BackendError::Database(DbErr::Exec(RuntimeErr::Internal( - format!( - "Failed to update plan_cost table. And related deletion is{} done.", - if !delete_res.is_err() && !delete_versioned_res.is_err() { - "" - } else { - " not" - } - ), - )))); - } + transaction.commit().await?; Ok(()) } @@ -247,7 +217,19 @@ impl CostModelStorageLayer for BackendManager { expr_id: Self::ExprId, stat_ids: Vec, ) -> StorageResult<()> { - todo!() + let to_insert_mappings = stat_ids + .iter() + .map( + |stat_id| physical_expression_to_statistic_junction::ActiveModel { + physical_expression_id: sea_orm::ActiveValue::Set(expr_id), + statistic_id: sea_orm::ActiveValue::Set(*stat_id), + }, + ) + .collect::>(); + let _ = PhysicalExpressionToStatisticJunction::insert_many(to_insert_mappings) + .exec(&self.db) + .await?; + Ok(()) } async fn get_stats_for_table( From d05c4938731bee519049a6bb597c493cbed58128 Mon Sep 17 00:00:00 2001 From: Lan Lou Date: Thu, 7 Nov 2024 09:36:08 -0500 Subject: [PATCH 07/28] Fix clippy --- optd-persistent/src/cost_model/orm.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/optd-persistent/src/cost_model/orm.rs b/optd-persistent/src/cost_model/orm.rs index dfe7440..173a6e9 100644 --- a/optd-persistent/src/cost_model/orm.rs +++ b/optd-persistent/src/cost_model/orm.rs @@ -49,14 +49,12 @@ impl CostModelStorageLayer for BackendManager { data: sea_orm::ActiveValue::Set(sea_orm::JsonValue::String(data)), ..Default::default() }; - let res = Event::insert(new_event).exec(&self.db).await; - Ok(res.and_then(|insert_res| { - self.latest_epoch_id.store( - insert_res.last_insert_id as usize, - std::sync::atomic::Ordering::Relaxed, - ); - Ok(insert_res.last_insert_id) - })?) + let insert_res = Event::insert(new_event).exec(&self.db).await?; + self.latest_epoch_id.store( + insert_res.last_insert_id as usize, + std::sync::atomic::Ordering::Relaxed, + ); + Ok(insert_res.last_insert_id) } async fn update_stats_from_catalog( From 6d1f82edc39ab2560185be6c989f482c7138f9b5 Mon Sep 17 00:00:00 2001 From: Yuanxin Cao Date: Thu, 7 Nov 2024 09:44:09 -0500 Subject: [PATCH 08/28] integrate w nullable columns --- optd-persistent/src/cost_model/orm.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/optd-persistent/src/cost_model/orm.rs b/optd-persistent/src/cost_model/orm.rs index 173a6e9..d4e8250 100644 --- a/optd-persistent/src/cost_model/orm.rs +++ b/optd-persistent/src/cost_model/orm.rs @@ -73,7 +73,7 @@ impl CostModelStorageLayer for BackendManager { // TODO(lanlou): only select needed fields let res = Statistic::find() .filter(statistic::Column::TableId.eq(table_id)) - .filter(statistic::Column::StatisticType.eq(stat.stat_type)) + .filter(statistic::Column::VariantTag.eq(stat.stat_type)) // FIX_ME: Do we need the following filter? .inner_join(versioned_statistic::Entity) .select_also(versioned_statistic::Entity) @@ -90,12 +90,12 @@ impl CostModelStorageLayer for BackendManager { None => { let new_stat = statistic::ActiveModel { name: sea_orm::ActiveValue::Set(stat.name.clone()), - table_id: sea_orm::ActiveValue::Set(table_id), + table_id: sea_orm::ActiveValue::Set(Some(table_id)), number_of_attributes: sea_orm::ActiveValue::Set( stat.attr_ids.len() as i32 ), created_time: sea_orm::ActiveValue::Set(Utc::now()), - statistic_type: sea_orm::ActiveValue::Set(stat.stat_type), + variant_tag: sea_orm::ActiveValue::Set(stat.stat_type), description: sea_orm::ActiveValue::Set("".to_string()), ..Default::default() }; @@ -118,7 +118,7 @@ impl CostModelStorageLayer for BackendManager { let res = Statistic::find() .filter(statistic::Column::NumberOfAttributes.eq(stat.attr_ids.len() as i32)) .filter(statistic::Column::Description.eq(description.clone())) - .filter(statistic::Column::StatisticType.eq(stat.stat_type)) + .filter(statistic::Column::VariantTag.eq(stat.stat_type)) // FIX_ME: Do we need the following filter? .inner_join(versioned_statistic::Entity) .select_also(versioned_statistic::Entity) @@ -139,7 +139,7 @@ impl CostModelStorageLayer for BackendManager { stat.attr_ids.len() as i32 ), created_time: sea_orm::ActiveValue::Set(Utc::now()), - statistic_type: sea_orm::ActiveValue::Set(stat.stat_type), + variant_tag: sea_orm::ActiveValue::Set(stat.stat_type), description: sea_orm::ActiveValue::Set(description), ..Default::default() }; @@ -241,7 +241,7 @@ impl CostModelStorageLayer for BackendManager { .filter(versioned_statistic::Column::EpochId.eq(epoch_id)) .inner_join(statistic::Entity) .filter(statistic::Column::TableId.eq(table_id)) - .filter(statistic::Column::StatisticType.eq(stat_type)) + .filter(statistic::Column::VariantTag.eq(stat_type)) .one(&self.db) .await? .map(|stat| stat.statistic_value)), @@ -249,7 +249,7 @@ impl CostModelStorageLayer for BackendManager { None => Ok(VersionedStatistic::find() .inner_join(statistic::Entity) .filter(statistic::Column::TableId.eq(table_id)) - .filter(statistic::Column::StatisticType.eq(stat_type)) + .filter(statistic::Column::VariantTag.eq(stat_type)) .order_by_desc(versioned_statistic::Column::EpochId) .one(&self.db) .await? @@ -277,7 +277,7 @@ impl CostModelStorageLayer for BackendManager { .inner_join(statistic::Entity) .filter(statistic::Column::NumberOfAttributes.eq(attr_num)) .filter(statistic::Column::Description.eq(description)) - .filter(statistic::Column::StatisticType.eq(stat_type)) + .filter(statistic::Column::VariantTag.eq(stat_type)) .one(&self.db) .await? .map(|stat| stat.statistic_value)), @@ -286,7 +286,7 @@ impl CostModelStorageLayer for BackendManager { .inner_join(statistic::Entity) .filter(statistic::Column::NumberOfAttributes.eq(attr_num)) .filter(statistic::Column::Description.eq(description)) - .filter(statistic::Column::StatisticType.eq(stat_type)) + .filter(statistic::Column::VariantTag.eq(stat_type)) .order_by_desc(versioned_statistic::Column::EpochId) .one(&self.db) .await? From 2d12016e75e7f939eade4aefae6d2cd76afb7428 Mon Sep 17 00:00:00 2001 From: Lan Lou Date: Thu, 7 Nov 2024 09:59:36 -0500 Subject: [PATCH 09/28] Update test_create_new_epoch --- optd-persistent/src/cost_model/orm.rs | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/optd-persistent/src/cost_model/orm.rs b/optd-persistent/src/cost_model/orm.rs index d4e8250..234d59f 100644 --- a/optd-persistent/src/cost_model/orm.rs +++ b/optd-persistent/src/cost_model/orm.rs @@ -355,22 +355,20 @@ mod tests { .await; println!("{:?}", res); assert!(res.is_ok()); + let inserted_id = res.unwrap(); + let lookup_res = Event::find_by_id(inserted_id) + .all(&backend_manager.db) + .await + .unwrap(); + println!("{:?}", lookup_res); + assert_eq!(lookup_res.len(), 1); + assert_eq!(lookup_res[0].source_variant, "source"); assert_eq!( - super::Event::find() - .all(&backend_manager.db) - .await - .unwrap() - .len(), - 1 - ); - println!( - "{:?}", - super::Event::find().all(&backend_manager.db).await.unwrap()[0] - ); - assert_eq!( - super::Event::find().all(&backend_manager.db).await.unwrap()[0].epoch_id, - res.unwrap() + lookup_res[0].data, + serde_json::Value::String("data".to_string()) ); + assert_eq!(lookup_res[0].epoch_id, inserted_id); + remove_db_file(DATABASE_FILE); } } From 72a07d1ee212ad212b696d863fd2ff5c17b05551 Mon Sep 17 00:00:00 2001 From: unw9527 <1041593558@qq.com> Date: Thu, 7 Nov 2024 14:03:49 -0500 Subject: [PATCH 10/28] add: cost methods --- optd-persistent/src/cost_model/orm.rs | 66 ++++++++++++++++++++++----- 1 file changed, 54 insertions(+), 12 deletions(-) diff --git a/optd-persistent/src/cost_model/orm.rs b/optd-persistent/src/cost_model/orm.rs index 234d59f..fa59499 100644 --- a/optd-persistent/src/cost_model/orm.rs +++ b/optd-persistent/src/cost_model/orm.rs @@ -1,6 +1,5 @@ #![allow(dead_code, unused_imports, unused_variables)] -use std::i32; use std::ptr::null; use crate::entities::{prelude::*, *}; @@ -201,15 +200,6 @@ impl CostModelStorageLayer for BackendManager { Ok(()) } - async fn store_cost( - &self, - expr_id: Self::ExprId, - cost: i32, - epoch_id: Self::EpochId, - ) -> StorageResult<()> { - todo!() - } - async fn store_expr_stats_mappings( &self, expr_id: Self::ExprId, @@ -299,11 +289,63 @@ impl CostModelStorageLayer for BackendManager { expr_id: Self::ExprId, epoch_id: Self::EpochId, ) -> StorageResult> { - todo!() + let cost = PlanCost::find() + .filter(plan_cost::Column::PhysicalExpressionId.eq(expr_id)) + .filter(plan_cost::Column::EpochId.eq(epoch_id)) + .one(&self.db) + .await?; + assert!(cost.is_some(), "Cost not found in Cost table"); + assert!(cost.clone().unwrap().is_valid, "Cost is not valid"); + Ok(cost.map(|c| c.cost)) } async fn get_cost(&self, expr_id: Self::ExprId) -> StorageResult> { - todo!() + let cost = PlanCost::find() + .filter(plan_cost::Column::PhysicalExpressionId.eq(expr_id)) + .order_by_desc(plan_cost::Column::EpochId) + .one(&self.db) + .await?; + assert!(cost.is_some(), "Cost not found in Cost table"); + assert!(cost.clone().unwrap().is_valid, "Cost is not valid"); + Ok(cost.map(|c| c.cost)) + } + + async fn store_cost( + &self, + expr_id: Self::ExprId, + cost: i32, + epoch_id: Self::EpochId, + ) -> StorageResult<()> { + let expr_exists = PhysicalExpression::find_by_id(expr_id) + .one(&self.db) + .await?; + if expr_exists.is_none() { + return Err(BackendError::Database(DbErr::RecordNotFound( + "ExprId not found in PhysicalExpression table".to_string(), + ))); + } + + // Check if epoch_id exists in Event table + let epoch_exists = Event::find() + .filter(event::Column::EpochId.eq(epoch_id)) + .one(&self.db) + .await + .unwrap(); + if epoch_exists.is_none() { + return Err(BackendError::Database(DbErr::RecordNotFound( + "EpochId not found in Event table".to_string(), + ))); + } + + let new_cost = plan_cost::ActiveModel { + physical_expression_id: sea_orm::ActiveValue::Set(expr_id), + epoch_id: sea_orm::ActiveValue::Set(epoch_id), + cost: sea_orm::ActiveValue::Set(cost), + is_valid: sea_orm::ActiveValue::Set(true), + ..Default::default() + }; + let _ = PlanCost::insert(new_cost).exec(&self.db).await?; + Ok(()) } } From 586c51fe987a15f1b13322df6cc78bc367eefef4 Mon Sep 17 00:00:00 2001 From: Lan Lou Date: Thu, 7 Nov 2024 14:19:47 -0500 Subject: [PATCH 11/28] add update_stats_from_catalog --- .../src/cost_model/catalog/mock_catalog.rs | 11 ++- optd-persistent/src/cost_model/catalog/mod.rs | 7 ++ optd-persistent/src/cost_model/orm.rs | 85 ++++++++++++++++++- optd-persistent/src/entities/attribute.rs | 2 +- .../src/entities/database_metadata.rs | 1 + optd-persistent/src/entities/index.rs | 2 +- .../src/entities/table_metadata.rs | 4 +- .../catalog/m20241029_000001_attribute.rs | 4 +- .../m20241029_000001_database_metadata.rs | 2 + .../catalog/m20241029_000001_index.rs | 2 +- .../m20241029_000001_table_metadata.rs | 6 +- 11 files changed, 114 insertions(+), 12 deletions(-) diff --git a/optd-persistent/src/cost_model/catalog/mock_catalog.rs b/optd-persistent/src/cost_model/catalog/mock_catalog.rs index b3aeb6c..b514916 100644 --- a/optd-persistent/src/cost_model/catalog/mock_catalog.rs +++ b/optd-persistent/src/cost_model/catalog/mock_catalog.rs @@ -1,6 +1,6 @@ use crate::cost_model::interface::StatisticType; -use super::IndexType; +use super::{AttrType, IndexType}; pub struct MockDatabaseMetadata { pub id: i32, @@ -24,6 +24,9 @@ pub struct MockAttribute { pub name: String, pub attr_index: i32, pub table_id: i32, + pub compression_method: char, + pub attr_type: i32, + pub is_not_null: bool, } pub struct MockStatistic { @@ -90,12 +93,18 @@ impl MockCatalog { name: "attr1".to_string(), attr_index: 1, table_id: 1, + compression_method: 'n', + attr_type: AttrType::Integer as i32, + is_not_null: true, }, MockAttribute { id: 2, name: "attr2".to_string(), attr_index: 2, table_id: 1, + compression_method: 'n', + attr_type: AttrType::Integer as i32, + is_not_null: false, }, ]; let statistics: Vec = vec![ diff --git a/optd-persistent/src/cost_model/catalog/mod.rs b/optd-persistent/src/cost_model/catalog/mod.rs index bb8ab4a..60535d8 100644 --- a/optd-persistent/src/cost_model/catalog/mod.rs +++ b/optd-persistent/src/cost_model/catalog/mod.rs @@ -4,3 +4,10 @@ pub enum IndexType { BTree, Hash, } + +pub enum AttrType { + Integer, + Float, + Varchar, + Boolean, +} diff --git a/optd-persistent/src/cost_model/orm.rs b/optd-persistent/src/cost_model/orm.rs index fa59499..b0f644d 100644 --- a/optd-persistent/src/cost_model/orm.rs +++ b/optd-persistent/src/cost_model/orm.rs @@ -12,6 +12,7 @@ use sea_orm::{ QueryOrder, QuerySelect, RuntimeErr, TransactionTrait, }; +use super::catalog::mock_catalog::{self, MockCatalog}; use super::interface::{CatalogSource, Stat}; impl BackendManager { @@ -61,7 +62,89 @@ impl CostModelStorageLayer for BackendManager { c: CatalogSource, epoch_id: Self::EpochId, ) -> StorageResult<()> { - todo!() + match c { + CatalogSource::Mock => { + let mock_catalog = MockCatalog::new(); + DatabaseMetadata::insert_many(mock_catalog.databases.iter().map(|database| { + database_metadata::ActiveModel { + name: sea_orm::ActiveValue::Set(database.name.clone()), + creation_time: sea_orm::ActiveValue::Set(Utc::now()), + ..Default::default() + } + })) + .exec(&self.db) + .await?; + NamespaceMetadata::insert_many(mock_catalog.namespaces.iter().map(|namespace| { + namespace_metadata::ActiveModel { + name: sea_orm::ActiveValue::Set(namespace.name.clone()), + database_id: sea_orm::ActiveValue::Set(namespace.database_id), + creation_time: sea_orm::ActiveValue::Set(Utc::now()), + ..Default::default() + } + })) + .exec(&self.db) + .await?; + TableMetadata::insert_many(mock_catalog.tables.iter().map(|table| { + table_metadata::ActiveModel { + name: sea_orm::ActiveValue::Set(table.name.clone()), + namespace_id: sea_orm::ActiveValue::Set(table.namespace_id), + creation_time: sea_orm::ActiveValue::Set(Utc::now()), + ..Default::default() + } + })) + .exec(&self.db) + .await?; + Attribute::insert_many(mock_catalog.attributes.iter().map(|attr| { + attribute::ActiveModel { + table_id: sea_orm::ActiveValue::Set(attr.table_id), + name: sea_orm::ActiveValue::Set(attr.name.clone()), + compression_method: sea_orm::ActiveValue::Set( + attr.compression_method.to_string(), + ), + variant_tag: sea_orm::ActiveValue::Set(attr.attr_type), + base_attribute_number: sea_orm::ActiveValue::Set(attr.attr_index), + is_not_null: sea_orm::ActiveValue::Set(attr.is_not_null), + ..Default::default() + } + })) + .exec(&self.db) + .await?; + Statistic::insert_many(mock_catalog.statistics.iter().map(|stat| { + statistic::ActiveModel { + name: sea_orm::ActiveValue::Set(stat.name.clone()), + table_id: sea_orm::ActiveValue::Set(stat.table_id), + number_of_attributes: sea_orm::ActiveValue::Set(stat.attr_ids.len() as i32), + created_time: sea_orm::ActiveValue::Set(Utc::now()), + variant_tag: sea_orm::ActiveValue::Set(stat.stat_type), + description: sea_orm::ActiveValue::Set( + self.get_description_from_attr_ids(stat.attr_ids.clone()), + ), + ..Default::default() + } + })) + .exec(&self.db) + .await?; + Index::insert_many(mock_catalog.indexes.iter().map(|index| index::ActiveModel { + name: sea_orm::ActiveValue::Set(index.name.clone()), + table_id: sea_orm::ActiveValue::Set(index.table_id), + number_of_attributes: sea_orm::ActiveValue::Set(index.attr_ids.len() as i32), + variant_tag: sea_orm::ActiveValue::Set(index.index_type), + is_unique: sea_orm::ActiveValue::Set(index.is_unique), + nulls_not_distinct: sea_orm::ActiveValue::Set(index.nulls_not_distinct), + is_primary: sea_orm::ActiveValue::Set(index.is_primary), + is_clustered: sea_orm::ActiveValue::Set(index.is_clustered), + is_exclusion: sea_orm::ActiveValue::Set(index.is_exclusion), + description: sea_orm::ActiveValue::Set( + self.get_description_from_attr_ids(index.attr_ids.clone()), + ), + ..Default::default() + })) + .exec(&self.db) + .await?; + Ok(()) + } + CatalogSource::Iceberg() => todo!(), + } } async fn update_stats(&self, stat: Stat, epoch_id: Self::EpochId) -> StorageResult<()> { diff --git a/optd-persistent/src/entities/attribute.rs b/optd-persistent/src/entities/attribute.rs index 2fa2ba2..aa312c0 100644 --- a/optd-persistent/src/entities/attribute.rs +++ b/optd-persistent/src/entities/attribute.rs @@ -10,7 +10,7 @@ pub struct Model { pub table_id: i32, pub name: String, pub compression_method: String, - pub r#type: i32, + pub variant_tag: i32, pub base_attribute_number: i32, pub is_not_null: bool, } diff --git a/optd-persistent/src/entities/database_metadata.rs b/optd-persistent/src/entities/database_metadata.rs index daff586..98106d6 100644 --- a/optd-persistent/src/entities/database_metadata.rs +++ b/optd-persistent/src/entities/database_metadata.rs @@ -7,6 +7,7 @@ use sea_orm::entity::prelude::*; pub struct Model { #[sea_orm(primary_key)] pub id: i32, + pub name: String, pub creation_time: DateTimeUtc, } diff --git a/optd-persistent/src/entities/index.rs b/optd-persistent/src/entities/index.rs index d77baa0..fb72452 100644 --- a/optd-persistent/src/entities/index.rs +++ b/optd-persistent/src/entities/index.rs @@ -16,7 +16,7 @@ pub struct Model { pub is_primary: bool, pub is_clustered: bool, pub is_exclusion: bool, - pub description: Json, + pub description: String, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/optd-persistent/src/entities/table_metadata.rs b/optd-persistent/src/entities/table_metadata.rs index 619ed29..f4d5fcf 100644 --- a/optd-persistent/src/entities/table_metadata.rs +++ b/optd-persistent/src/entities/table_metadata.rs @@ -8,7 +8,7 @@ pub struct Model { #[sea_orm(primary_key)] pub id: i32, pub name: String, - pub schema_id: i32, + pub namespace_id: i32, pub creation_time: DateTimeUtc, } @@ -20,7 +20,7 @@ pub enum Relation { Index, #[sea_orm( belongs_to = "super::namespace_metadata::Entity", - from = "Column::SchemaId", + from = "Column::NamespaceId", to = "super::namespace_metadata::Column::Id", on_update = "Cascade", on_delete = "Cascade" diff --git a/optd-persistent/src/migrator/catalog/m20241029_000001_attribute.rs b/optd-persistent/src/migrator/catalog/m20241029_000001_attribute.rs index a0d3858..543c745 100644 --- a/optd-persistent/src/migrator/catalog/m20241029_000001_attribute.rs +++ b/optd-persistent/src/migrator/catalog/m20241029_000001_attribute.rs @@ -8,7 +8,7 @@ pub enum Attribute { TableId, Name, CompressionMethod, - Type, + VariantTag, BaseAttributeNumber, IsNotNull, } @@ -35,7 +35,7 @@ impl MigrationTrait for Migration { ) .col(string(Attribute::Name)) .col(char(Attribute::CompressionMethod)) - .col(integer(Attribute::Type)) + .col(integer(Attribute::VariantTag)) .col(integer(Attribute::BaseAttributeNumber)) .col(boolean(Attribute::IsNotNull)) .to_owned(), diff --git a/optd-persistent/src/migrator/catalog/m20241029_000001_database_metadata.rs b/optd-persistent/src/migrator/catalog/m20241029_000001_database_metadata.rs index 7a0a482..4423557 100644 --- a/optd-persistent/src/migrator/catalog/m20241029_000001_database_metadata.rs +++ b/optd-persistent/src/migrator/catalog/m20241029_000001_database_metadata.rs @@ -4,6 +4,7 @@ use sea_orm_migration::{prelude::*, schema::*}; pub enum DatabaseMetadata { Table, Id, + Name, CreationTime, } @@ -19,6 +20,7 @@ impl MigrationTrait for Migration { .table(DatabaseMetadata::Table) .if_not_exists() .col(pk_auto(DatabaseMetadata::Id)) + .col(string(DatabaseMetadata::Name)) .col(timestamp(DatabaseMetadata::CreationTime)) .to_owned(), ) diff --git a/optd-persistent/src/migrator/catalog/m20241029_000001_index.rs b/optd-persistent/src/migrator/catalog/m20241029_000001_index.rs index 184249f..1c8ede9 100644 --- a/optd-persistent/src/migrator/catalog/m20241029_000001_index.rs +++ b/optd-persistent/src/migrator/catalog/m20241029_000001_index.rs @@ -45,7 +45,7 @@ impl MigrationTrait for Migration { .col(boolean(Index::IsPrimary)) .col(boolean(Index::IsClustered)) .col(boolean(Index::IsExclusion)) - .col(json(Index::Description)) + .col(string(Index::Description)) .to_owned(), ) .await diff --git a/optd-persistent/src/migrator/catalog/m20241029_000001_table_metadata.rs b/optd-persistent/src/migrator/catalog/m20241029_000001_table_metadata.rs index 3c43354..66a0ec3 100644 --- a/optd-persistent/src/migrator/catalog/m20241029_000001_table_metadata.rs +++ b/optd-persistent/src/migrator/catalog/m20241029_000001_table_metadata.rs @@ -6,7 +6,7 @@ pub enum TableMetadata { Table, Id, Name, - SchemaId, + NamespaceId, CreationTime, } @@ -23,10 +23,10 @@ impl MigrationTrait for Migration { .if_not_exists() .col(pk_auto(TableMetadata::Id)) .col(string(TableMetadata::Name)) - .col(integer(TableMetadata::SchemaId)) + .col(integer(TableMetadata::NamespaceId)) .foreign_key( ForeignKey::create() - .from(TableMetadata::Table, TableMetadata::SchemaId) + .from(TableMetadata::Table, TableMetadata::NamespaceId) .to(NamespaceMetadata::Table, NamespaceMetadata::Id) .on_delete(ForeignKeyAction::Cascade) .on_update(ForeignKeyAction::Cascade), From 959610b0fa175c432dd0774b16bf14d55c3c3ae3 Mon Sep 17 00:00:00 2001 From: Lan Lou Date: Thu, 7 Nov 2024 14:22:22 -0500 Subject: [PATCH 12/28] Add test_update_stats_from_catalog --- optd-persistent/src/cost_model/orm.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/optd-persistent/src/cost_model/orm.rs b/optd-persistent/src/cost_model/orm.rs index b0f644d..9bc41fc 100644 --- a/optd-persistent/src/cost_model/orm.rs +++ b/optd-persistent/src/cost_model/orm.rs @@ -434,7 +434,7 @@ impl CostModelStorageLayer for BackendManager { #[cfg(test)] mod tests { - use crate::{migrate, CostModelStorageLayer}; + use crate::{cost_model::interface::Stat, migrate, CostModelStorageLayer}; use sea_orm::{ ColumnTrait, ConnectionTrait, Database, DbBackend, EntityTrait, ModelTrait, QueryFilter, QuerySelect, QueryTrait, @@ -496,4 +496,23 @@ mod tests { remove_db_file(DATABASE_FILE); } + + #[tokio::test] + async fn test_update_stats_from_catalog() { + const DATABASE_FILE: &str = "test_update_stats_from_catalog.db"; + let database_url = run_migration(DATABASE_FILE).await; + let mut binding = super::BackendManager::new(Some(&database_url)).await; + let backend_manager = binding.as_mut().unwrap(); + let res = backend_manager + .update_stats_from_catalog(super::CatalogSource::Mock, 1) + .await; + println!("{:?}", res); + assert!(res.is_ok()); + + let lookup_res = Statistic::find().all(&backend_manager.db).await.unwrap(); + println!("{:?}", lookup_res); + assert_eq!(lookup_res.len(), 3); + + remove_db_file(DATABASE_FILE); + } } From 3cd2897a3d7e6886545043d420b9f8dae0b43885 Mon Sep 17 00:00:00 2001 From: unw9527 <1041593558@qq.com> Date: Thu, 7 Nov 2024 17:50:00 -0500 Subject: [PATCH 13/28] feat: enable unit test with every table intialized --- .gitignore | 2 + optd-persistent/src/bin/init.rs | 341 ++++++++++++++++++ optd-persistent/src/cost_model/mod.rs | 1 + optd-persistent/src/cost_model/orm.rs | 91 +++-- optd-persistent/src/entities/attribute.rs | 2 +- .../entities/attribute_constraint_junction.rs | 12 +- .../attribute_foreign_constraint_junction.rs | 12 +- .../src/entities/cascades_group.rs | 2 +- optd-persistent/src/entities/constraint.rs | 2 +- .../src/entities/constraint_metadata.rs | 68 ++++ .../src/entities/database_metadata.rs | 2 +- optd-persistent/src/entities/event.rs | 2 +- optd-persistent/src/entities/group_winner.rs | 2 +- optd-persistent/src/entities/index.rs | 2 +- .../src/entities/index_metadata.rs | 48 +++ .../src/entities/logical_children.rs | 2 +- .../src/entities/logical_expression.rs | 2 +- .../src/entities/logical_property.rs | 2 +- optd-persistent/src/entities/mod.rs | 6 +- .../src/entities/namespace_metadata.rs | 2 +- .../src/entities/physical_children.rs | 2 +- .../src/entities/physical_expression.rs | 2 +- ...ysical_expression_to_statistic_junction.rs | 2 +- .../src/entities/physical_property.rs | 2 +- optd-persistent/src/entities/plan_cost.rs | 2 +- optd-persistent/src/entities/prelude.rs | 7 +- optd-persistent/src/entities/statistic.rs | 4 +- .../statistic_to_attribute_junction.rs | 2 +- .../src/entities/table_metadata.rs | 10 +- optd-persistent/src/entities/trigger.rs | 2 +- .../src/entities/versioned_statistic.rs | 2 +- optd-persistent/src/lib.rs | 4 +- ...29_000001_attribute_constraint_junction.rs | 4 +- ...1_attribute_foreign_constraint_junction.rs | 4 +- ...> m20241029_000001_constraint_metadata.rs} | 30 +- ....rs => m20241029_000001_index_metadata.rs} | 30 +- optd-persistent/src/migrator/catalog/mod.rs | 8 +- .../cost_model/m20241029_000001_statistic.rs | 4 +- optd-persistent/src/migrator/mod.rs | 4 +- schema/all_tables.dbml | 15 +- 40 files changed, 626 insertions(+), 117 deletions(-) create mode 100644 optd-persistent/src/bin/init.rs create mode 100644 optd-persistent/src/entities/constraint_metadata.rs create mode 100644 optd-persistent/src/entities/index_metadata.rs rename optd-persistent/src/migrator/catalog/{m20241029_000001_constraint.rs => m20241029_000001_constraint_metadata.rs} (58%) rename optd-persistent/src/migrator/catalog/{m20241029_000001_index.rs => m20241029_000001_index_metadata.rs} (55%) diff --git a/.gitignore b/.gitignore index ff2dba2..f777bce 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ target/ **/*.db .DS_Store + +optd-persistent/sql/ \ No newline at end of file diff --git a/optd-persistent/src/bin/init.rs b/optd-persistent/src/bin/init.rs new file mode 100644 index 0000000..333452f --- /dev/null +++ b/optd-persistent/src/bin/init.rs @@ -0,0 +1,341 @@ +use optd_persistent::entities::*; +use optd_persistent::migrate; +use sea_orm::sqlx::types::chrono::Utc; +use sea_orm::*; +use serde_json::json; + +async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { + let database_url = format!("sqlite:./{}?mode=rwc", db_file); + let database_file = format!("./{}", db_file); + let _ = std::fs::remove_file(database_file); + + let db = Database::connect(database_url.clone()) + .await + .expect("Unable to connect to the database"); + + migrate(&db) + .await + .expect("Something went wrong during migration"); + + // Inserting into database_metadata + let database_metadata = database_metadata::ActiveModel { + id: Set(1), + name: Set("database1".to_owned()), + creation_time: Set(Utc::now()), + ..Default::default() + }; + database_metadata::Entity::insert(database_metadata) + .exec(&db) + .await + .expect("Unable to insert database metadata"); + + // Inserting into namespace_metadata + let namespace_metadata = namespace_metadata::ActiveModel { + id: Set(1), + database_id: Set(1), + name: Set("default".to_owned()), + creation_time: Set(Utc::now()), + ..Default::default() + }; + namespace_metadata::Entity::insert(namespace_metadata) + .exec(&db) + .await + .expect("Unable to insert namespace metadata"); + + // Inserting into table_metadata + let table_metadata = table_metadata::ActiveModel { + id: Set(1), + namespace_id: Set(1), + name: Set("users".to_owned()), + creation_time: Set(Utc::now()), + ..Default::default() + }; + table_metadata::Entity::insert(table_metadata) + .exec(&db) + .await + .expect("Unable to insert table metadata"); + + // Inserting into attribute + let attribute = attribute::ActiveModel { + id: Set(1), + table_id: Set(1), + name: Set("user_id".to_owned()), + compression_method: Set("N".to_owned()), + variant_tag: Set(1), + base_attribute_number: Set(1), + is_not_null: Set(true), + ..Default::default() + }; + attribute::Entity::insert(attribute) + .exec(&db) + .await + .expect("Unable to insert attribute"); + + // Inserting into statistic + let statistic = statistic::ActiveModel { + id: Set(1), + name: Set("row_count".to_owned()), + table_id: Set(Some(1)), + creation_time: Set(Utc::now()), + number_of_attributes: Set(0), + variant_tag: Set(1), + description: Set("Total rows".to_owned()), + ..Default::default() + }; + statistic::Entity::insert(statistic) + .exec(&db) + .await + .expect("Unable to insert statistic"); + // Inserting into event + let event = event::ActiveModel { + epoch_id: Set(1), + source_variant: Set("insert".to_owned()), + timestamp: Set(Utc::now()), + data: Set(json!(r#"{"user_id": 1}"#)), + ..Default::default() + }; + + event::Entity::insert(event) + .exec(&db) + .await + .expect("Unable to insert event"); + + // Inserting into versioned_statistic + let versioned_statistic = versioned_statistic::ActiveModel { + id: Set(1), + epoch_id: Set(1), + statistic_id: Set(1), + statistic_value: Set(json!(r#"{"row_count": 0}"#)), + ..Default::default() + }; + versioned_statistic::Entity::insert(versioned_statistic) + .exec(&db) + .await + .expect("Unable to insert versioned statistic"); + + // Inserting into index_metadata + let index_metadata = index_metadata::ActiveModel { + id: Set(1), + name: Set("user_id_index".to_owned()), + table_id: Set(1), + is_unique: Set(true), + nulls_not_distinct: Set(false), + is_primary: Set(true), + is_clustered: Set(false), + is_exclusion: Set(false), + variant_tag: Set(1), + number_of_attributes: Set(1), + description: Set("random".to_owned()), + ..Default::default() + }; + index_metadata::Entity::insert(index_metadata) + .exec(&db) + .await + .expect("Unable to insert index metadata"); + + // Inserting into trigger + let trigger = trigger::ActiveModel { + id: Set(1), + name: Set("after_insert_user".to_owned()), + table_id: Set(1), + parent_trigger_id: Set(1), + function: Set(json!(r#"{"function": "insert"}"#)), + ..Default::default() + }; + trigger::Entity::insert(trigger) + .exec(&db) + .await + .expect("Unable to insert trigger"); + + // Inserting into constraint_metadata + let constraint_metadata = constraint_metadata::ActiveModel { + id: Set(1), + name: Set("pk_user_id".to_owned()), + variant_tag: Set(1), + table_id: Set(Some(1)), + index_id: Set(Some(1)), + foreign_ref_id: Set(None), + check_src: Set("hello".to_owned()), + ..Default::default() + }; + constraint_metadata::Entity::insert(constraint_metadata) + .exec(&db) + .await + .expect("Unable to insert constraint metadata"); + + // Inserting into attribute_constraint_junction + let attribute_constraint_junction = attribute_constraint_junction::ActiveModel { + attribute_id: Set(1), + constraint_id: Set(1), + ..Default::default() + }; + attribute_constraint_junction::Entity::insert(attribute_constraint_junction) + .exec(&db) + .await + .expect("Unable to insert attribute_constraint_junction"); + + // Inserting into attribute_foreign_constraint_junction + let attribute_foreign_constraint_junction = + attribute_foreign_constraint_junction::ActiveModel { + attribute_id: Set(1), + constraint_id: Set(1), + ..Default::default() + }; + attribute_foreign_constraint_junction::Entity::insert(attribute_foreign_constraint_junction) + .exec(&db) + .await + .expect("Unable to insert attribute_foreign_constraint_junction"); + + // Inserting into statistic_to_attribute_junction + let statistic_to_attribute_junction = statistic_to_attribute_junction::ActiveModel { + statistic_id: Set(1), + attribute_id: Set(1), + ..Default::default() + }; + statistic_to_attribute_junction::Entity::insert(statistic_to_attribute_junction) + .exec(&db) + .await + .expect("Unable to insert statistic_to_attribute_junction"); + + // Inserting into cascades_group + let cascades_group = cascades_group::ActiveModel { + id: Set(1), + latest_winner: Set(None), + in_progress: Set(true), + is_optimized: Set(false), + ..Default::default() + }; + cascades_group::Entity::insert(cascades_group) + .exec(&db) + .await + .expect("Unable to insert cascades group"); + + // Inserting into logical_expression + let logical_expression = logical_expression::ActiveModel { + id: Set(1), + group_id: Set(1), + fingerprint: Set(12345), + variant_tag: Set(1), + data: Set(json!(r#"{"expr": "index_scan"}"#)), + ..Default::default() + }; + logical_expression::Entity::insert(logical_expression) + .exec(&db) + .await + .expect("Unable to insert logical expression"); + + // Inserting into physical_expression + let physical_expression = physical_expression::ActiveModel { + id: Set(1), + group_id: Set(1), + fingerprint: Set(12345), + variant_tag: Set(1), + data: Set(json!(r#"{"expr": "index_scan"}"#)), + ..Default::default() + }; + physical_expression::Entity::insert(physical_expression) + .exec(&db) + .await + .expect("Unable to insert physical expression"); + + // Inserting into physical_property + let physical_property = physical_property::ActiveModel { + id: Set(1), + physical_expression_id: Set(1), + variant_tag: Set(1), + data: Set(json!(r#"{"property": "indexed"}"#)), + ..Default::default() + }; + physical_property::Entity::insert(physical_property) + .exec(&db) + .await + .expect("Unable to insert physical property"); + + // Inserting into logical_property + let logical_property = logical_property::ActiveModel { + id: Set(1), + group_id: Set(1), + variant_tag: Set(1), + data: Set(json!(r#"{"property": "indexed"}"#)), + ..Default::default() + }; + logical_property::Entity::insert(logical_property) + .exec(&db) + .await + .expect("Unable to insert logical property"); + + let logical_children = logical_children::ActiveModel { + logical_expression_id: Set(1), + group_id: Set(1), + ..Default::default() + }; + logical_children::Entity::insert(logical_children) + .exec(&db) + .await + .expect("Unable to insert logical children"); + + let physical_children = physical_children::ActiveModel { + physical_expression_id: Set(1), + group_id: Set(1), + ..Default::default() + }; + physical_children::Entity::insert(physical_children) + .exec(&db) + .await + .expect("Unable to insert physical children"); + + // Inserting into plan_cost + let plan_cost = plan_cost::ActiveModel { + id: Set(1), + physical_expression_id: Set(1), + epoch_id: Set(1), + cost: Set(10), + is_valid: Set(true), + ..Default::default() + }; + plan_cost::Entity::insert(plan_cost) + .exec(&db) + .await + .expect("Unable to insert plan cost"); + + // Inserting into physical_expression_to_statistic_junction + let physical_expression_to_statistic_junction = + physical_expression_to_statistic_junction::ActiveModel { + physical_expression_id: Set(1), + statistic_id: Set(1), + ..Default::default() + }; + physical_expression_to_statistic_junction::Entity::insert( + physical_expression_to_statistic_junction, + ) + .exec(&db) + .await + .expect("Unable to insert physical_expression_to_statistic_junction"); + + // Inserting into group_winner + let group_winner = group_winner::ActiveModel { + id: Set(1), + group_id: Set(1), + physical_expression_id: Set(1), + cost_id: Set(1), + epoch_id: Set(1), + ..Default::default() + }; + group_winner::Entity::insert(group_winner) + .exec(&db) + .await + .expect("Unable to insert group winner"); + + Ok(()) +} + +#[tokio::main] +async fn main() { + let db_file = "init.db"; + if let Err(e) = init_all_tables(db_file).await { + eprintln!("Error initializing database: {}", e); + std::process::exit(1); + } + + println!("Database initialized successfully"); +} diff --git a/optd-persistent/src/cost_model/mod.rs b/optd-persistent/src/cost_model/mod.rs index 566435a..46514a1 100644 --- a/optd-persistent/src/cost_model/mod.rs +++ b/optd-persistent/src/cost_model/mod.rs @@ -1,3 +1,4 @@ pub mod catalog; + pub mod interface; pub mod orm; diff --git a/optd-persistent/src/cost_model/orm.rs b/optd-persistent/src/cost_model/orm.rs index 9bc41fc..f383a2f 100644 --- a/optd-persistent/src/cost_model/orm.rs +++ b/optd-persistent/src/cost_model/orm.rs @@ -114,7 +114,7 @@ impl CostModelStorageLayer for BackendManager { name: sea_orm::ActiveValue::Set(stat.name.clone()), table_id: sea_orm::ActiveValue::Set(stat.table_id), number_of_attributes: sea_orm::ActiveValue::Set(stat.attr_ids.len() as i32), - created_time: sea_orm::ActiveValue::Set(Utc::now()), + creation_time: sea_orm::ActiveValue::Set(Utc::now()), variant_tag: sea_orm::ActiveValue::Set(stat.stat_type), description: sea_orm::ActiveValue::Set( self.get_description_from_attr_ids(stat.attr_ids.clone()), @@ -124,21 +124,28 @@ impl CostModelStorageLayer for BackendManager { })) .exec(&self.db) .await?; - Index::insert_many(mock_catalog.indexes.iter().map(|index| index::ActiveModel { - name: sea_orm::ActiveValue::Set(index.name.clone()), - table_id: sea_orm::ActiveValue::Set(index.table_id), - number_of_attributes: sea_orm::ActiveValue::Set(index.attr_ids.len() as i32), - variant_tag: sea_orm::ActiveValue::Set(index.index_type), - is_unique: sea_orm::ActiveValue::Set(index.is_unique), - nulls_not_distinct: sea_orm::ActiveValue::Set(index.nulls_not_distinct), - is_primary: sea_orm::ActiveValue::Set(index.is_primary), - is_clustered: sea_orm::ActiveValue::Set(index.is_clustered), - is_exclusion: sea_orm::ActiveValue::Set(index.is_exclusion), - description: sea_orm::ActiveValue::Set( - self.get_description_from_attr_ids(index.attr_ids.clone()), - ), - ..Default::default() - })) + IndexMetadata::insert_many( + mock_catalog + .indexes + .iter() + .map(|index| index_metadata::ActiveModel { + name: sea_orm::ActiveValue::Set(index.name.clone()), + table_id: sea_orm::ActiveValue::Set(index.table_id), + number_of_attributes: sea_orm::ActiveValue::Set( + index.attr_ids.len() as i32 + ), + variant_tag: sea_orm::ActiveValue::Set(index.index_type), + is_unique: sea_orm::ActiveValue::Set(index.is_unique), + nulls_not_distinct: sea_orm::ActiveValue::Set(index.nulls_not_distinct), + is_primary: sea_orm::ActiveValue::Set(index.is_primary), + is_clustered: sea_orm::ActiveValue::Set(index.is_clustered), + is_exclusion: sea_orm::ActiveValue::Set(index.is_exclusion), + description: sea_orm::ActiveValue::Set( + self.get_description_from_attr_ids(index.attr_ids.clone()), + ), + ..Default::default() + }), + ) .exec(&self.db) .await?; Ok(()) @@ -176,7 +183,7 @@ impl CostModelStorageLayer for BackendManager { number_of_attributes: sea_orm::ActiveValue::Set( stat.attr_ids.len() as i32 ), - created_time: sea_orm::ActiveValue::Set(Utc::now()), + creation_time: sea_orm::ActiveValue::Set(Utc::now()), variant_tag: sea_orm::ActiveValue::Set(stat.stat_type), description: sea_orm::ActiveValue::Set("".to_string()), ..Default::default() @@ -220,7 +227,7 @@ impl CostModelStorageLayer for BackendManager { number_of_attributes: sea_orm::ActiveValue::Set( stat.attr_ids.len() as i32 ), - created_time: sea_orm::ActiveValue::Set(Utc::now()), + creation_time: sea_orm::ActiveValue::Set(Utc::now()), variant_tag: sea_orm::ActiveValue::Set(stat.stat_type), description: sea_orm::ActiveValue::Set(description), ..Default::default() @@ -395,11 +402,11 @@ impl CostModelStorageLayer for BackendManager { async fn store_cost( &self, - expr_id: Self::ExprId, + physical_expression_id: Self::ExprId, cost: i32, epoch_id: Self::EpochId, ) -> StorageResult<()> { - let expr_exists = PhysicalExpression::find_by_id(expr_id) + let expr_exists = PhysicalExpression::find_by_id(physical_expression_id) .one(&self.db) .await?; if expr_exists.is_none() { @@ -421,7 +428,7 @@ impl CostModelStorageLayer for BackendManager { } let new_cost = plan_cost::ActiveModel { - physical_expression_id: sea_orm::ActiveValue::Set(expr_id), + physical_expression_id: sea_orm::ActiveValue::Set(physical_expression_id), epoch_id: sea_orm::ActiveValue::Set(epoch_id), cost: sea_orm::ActiveValue::Set(cost), is_valid: sea_orm::ActiveValue::Set(true), @@ -435,6 +442,8 @@ impl CostModelStorageLayer for BackendManager { #[cfg(test)] mod tests { use crate::{cost_model::interface::Stat, migrate, CostModelStorageLayer}; + use sea_orm::sqlx::database; + use sea_orm::Statement; use sea_orm::{ ColumnTrait, ConnectionTrait, Database, DbBackend, EntityTrait, ModelTrait, QueryFilter, QuerySelect, QueryTrait, @@ -469,6 +478,13 @@ mod tests { let _ = std::fs::remove_file(database_file); } + async fn copy_init_db(db_file: &str) -> String { + let original_db = "init.db"; + let _ = std::fs::copy(original_db, format!("./{}", db_file)); + let database_url = format!("sqlite:./{}?mode=rwc", format!("./{}", db_file)); + database_url + } + #[tokio::test] async fn test_create_new_epoch() { const DATABASE_FILE: &str = "test_create_new_epoch.db"; @@ -515,4 +531,37 @@ mod tests { remove_db_file(DATABASE_FILE); } + + #[tokio::test] + #[ignore] // Need to update all tables + async fn test_store_cost() { + const DATABASE_FILE: &str = "test_store_cost.db"; + let database_url = copy_init_db(DATABASE_FILE).await; + let mut binding = super::BackendManager::new(Some(&database_url)).await; + let backend_manager = binding.as_mut().unwrap(); + let epoch_id = backend_manager + .create_new_epoch("source".to_string(), "data".to_string()) + .await + .unwrap(); + let physical_expression_id = 1; + let cost = 42; + let res = backend_manager + .store_cost(physical_expression_id, cost, epoch_id) + .await; + match res { + Ok(_) => assert!(true), + Err(e) => { + println!("Error: {:?}", e); + assert!(false); + } + } + let costs = super::PlanCost::find() + .all(&backend_manager.db) + .await + .unwrap(); + assert_eq!(costs.len(), 2); // The first row one is the initialized data + assert_eq!(costs[1].epoch_id, epoch_id); + assert_eq!(costs[1].physical_expression_id, physical_expression_id); + assert_eq!(costs[1].cost, cost); + } } diff --git a/optd-persistent/src/entities/attribute.rs b/optd-persistent/src/entities/attribute.rs index aa312c0..f45d04a 100644 --- a/optd-persistent/src/entities/attribute.rs +++ b/optd-persistent/src/entities/attribute.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/attribute_constraint_junction.rs b/optd-persistent/src/entities/attribute_constraint_junction.rs index 731afea..e1d8052 100644 --- a/optd-persistent/src/entities/attribute_constraint_junction.rs +++ b/optd-persistent/src/entities/attribute_constraint_junction.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 use sea_orm::entity::prelude::*; @@ -22,13 +22,13 @@ pub enum Relation { )] Attribute, #[sea_orm( - belongs_to = "super::constraint::Entity", + belongs_to = "super::constraint_metadata::Entity", from = "Column::ConstraintId", - to = "super::constraint::Column::Id", + to = "super::constraint_metadata::Column::Id", on_update = "Cascade", on_delete = "Cascade" )] - Constraint, + ConstraintMetadata, } impl Related for Entity { @@ -37,9 +37,9 @@ impl Related for Entity { } } -impl Related for Entity { +impl Related for Entity { fn to() -> RelationDef { - Relation::Constraint.def() + Relation::ConstraintMetadata.def() } } diff --git a/optd-persistent/src/entities/attribute_foreign_constraint_junction.rs b/optd-persistent/src/entities/attribute_foreign_constraint_junction.rs index 354a61f..c2f0b61 100644 --- a/optd-persistent/src/entities/attribute_foreign_constraint_junction.rs +++ b/optd-persistent/src/entities/attribute_foreign_constraint_junction.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 use sea_orm::entity::prelude::*; @@ -22,13 +22,13 @@ pub enum Relation { )] Attribute, #[sea_orm( - belongs_to = "super::constraint::Entity", + belongs_to = "super::constraint_metadata::Entity", from = "Column::ConstraintId", - to = "super::constraint::Column::Id", + to = "super::constraint_metadata::Column::Id", on_update = "Cascade", on_delete = "Cascade" )] - Constraint, + ConstraintMetadata, } impl Related for Entity { @@ -37,9 +37,9 @@ impl Related for Entity { } } -impl Related for Entity { +impl Related for Entity { fn to() -> RelationDef { - Relation::Constraint.def() + Relation::ConstraintMetadata.def() } } diff --git a/optd-persistent/src/entities/cascades_group.rs b/optd-persistent/src/entities/cascades_group.rs index 9ea79d1..d582a1c 100644 --- a/optd-persistent/src/entities/cascades_group.rs +++ b/optd-persistent/src/entities/cascades_group.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/constraint.rs b/optd-persistent/src/entities/constraint.rs index de9042d..0dd2334 100644 --- a/optd-persistent/src/entities/constraint.rs +++ b/optd-persistent/src/entities/constraint.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/constraint_metadata.rs b/optd-persistent/src/entities/constraint_metadata.rs new file mode 100644 index 0000000..5630e3e --- /dev/null +++ b/optd-persistent/src/entities/constraint_metadata.rs @@ -0,0 +1,68 @@ +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "constraint_metadata")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: i32, + pub name: String, + pub variant_tag: i32, + pub table_id: Option, + pub index_id: Option, + pub foreign_ref_id: Option, + pub check_src: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm(has_many = "super::attribute_constraint_junction::Entity")] + AttributeConstraintJunction, + #[sea_orm(has_many = "super::attribute_foreign_constraint_junction::Entity")] + AttributeForeignConstraintJunction, + #[sea_orm( + belongs_to = "super::index_metadata::Entity", + from = "Column::IndexId", + to = "super::index_metadata::Column::Id", + on_update = "Cascade", + on_delete = "Cascade" + )] + IndexMetadata, + #[sea_orm( + belongs_to = "super::table_metadata::Entity", + from = "Column::ForeignRefId", + to = "super::table_metadata::Column::Id", + on_update = "Cascade", + on_delete = "Cascade" + )] + TableMetadata2, + #[sea_orm( + belongs_to = "super::table_metadata::Entity", + from = "Column::TableId", + to = "super::table_metadata::Column::Id", + on_update = "Cascade", + on_delete = "Cascade" + )] + TableMetadata1, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::AttributeConstraintJunction.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::AttributeForeignConstraintJunction.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::IndexMetadata.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/optd-persistent/src/entities/database_metadata.rs b/optd-persistent/src/entities/database_metadata.rs index 98106d6..190bc73 100644 --- a/optd-persistent/src/entities/database_metadata.rs +++ b/optd-persistent/src/entities/database_metadata.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/event.rs b/optd-persistent/src/entities/event.rs index ad9157d..c26c7b1 100644 --- a/optd-persistent/src/entities/event.rs +++ b/optd-persistent/src/entities/event.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/group_winner.rs b/optd-persistent/src/entities/group_winner.rs index 32cbcb4..efd8661 100644 --- a/optd-persistent/src/entities/group_winner.rs +++ b/optd-persistent/src/entities/group_winner.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/index.rs b/optd-persistent/src/entities/index.rs index fb72452..824c7b4 100644 --- a/optd-persistent/src/entities/index.rs +++ b/optd-persistent/src/entities/index.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/index_metadata.rs b/optd-persistent/src/entities/index_metadata.rs new file mode 100644 index 0000000..49a1ea2 --- /dev/null +++ b/optd-persistent/src/entities/index_metadata.rs @@ -0,0 +1,48 @@ +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "index_metadata")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: i32, + pub table_id: i32, + pub name: String, + pub number_of_attributes: i32, + pub variant_tag: i32, + pub is_unique: bool, + pub nulls_not_distinct: bool, + pub is_primary: bool, + pub is_clustered: bool, + pub is_exclusion: bool, + pub description: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm(has_many = "super::constraint_metadata::Entity")] + ConstraintMetadata, + #[sea_orm( + belongs_to = "super::table_metadata::Entity", + from = "Column::TableId", + to = "super::table_metadata::Column::Id", + on_update = "Cascade", + on_delete = "Cascade" + )] + TableMetadata, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::ConstraintMetadata.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::TableMetadata.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/optd-persistent/src/entities/logical_children.rs b/optd-persistent/src/entities/logical_children.rs index 120641f..92d53bd 100644 --- a/optd-persistent/src/entities/logical_children.rs +++ b/optd-persistent/src/entities/logical_children.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/logical_expression.rs b/optd-persistent/src/entities/logical_expression.rs index 6beff04..aa3b314 100644 --- a/optd-persistent/src/entities/logical_expression.rs +++ b/optd-persistent/src/entities/logical_expression.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/logical_property.rs b/optd-persistent/src/entities/logical_property.rs index 755575c..f42ffdd 100644 --- a/optd-persistent/src/entities/logical_property.rs +++ b/optd-persistent/src/entities/logical_property.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/mod.rs b/optd-persistent/src/entities/mod.rs index f5b2787..1d4a0fa 100644 --- a/optd-persistent/src/entities/mod.rs +++ b/optd-persistent/src/entities/mod.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 pub mod prelude; @@ -6,11 +6,11 @@ pub mod attribute; pub mod attribute_constraint_junction; pub mod attribute_foreign_constraint_junction; pub mod cascades_group; -pub mod constraint; +pub mod constraint_metadata; pub mod database_metadata; pub mod event; pub mod group_winner; -pub mod index; +pub mod index_metadata; pub mod logical_children; pub mod logical_expression; pub mod logical_property; diff --git a/optd-persistent/src/entities/namespace_metadata.rs b/optd-persistent/src/entities/namespace_metadata.rs index 8f63146..f3f3f37 100644 --- a/optd-persistent/src/entities/namespace_metadata.rs +++ b/optd-persistent/src/entities/namespace_metadata.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/physical_children.rs b/optd-persistent/src/entities/physical_children.rs index d8f9db0..18c1794 100644 --- a/optd-persistent/src/entities/physical_children.rs +++ b/optd-persistent/src/entities/physical_children.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/physical_expression.rs b/optd-persistent/src/entities/physical_expression.rs index 4876d35..02795d2 100644 --- a/optd-persistent/src/entities/physical_expression.rs +++ b/optd-persistent/src/entities/physical_expression.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/physical_expression_to_statistic_junction.rs b/optd-persistent/src/entities/physical_expression_to_statistic_junction.rs index f8020bd..0332f02 100644 --- a/optd-persistent/src/entities/physical_expression_to_statistic_junction.rs +++ b/optd-persistent/src/entities/physical_expression_to_statistic_junction.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/physical_property.rs b/optd-persistent/src/entities/physical_property.rs index 51d98dc..944a94d 100644 --- a/optd-persistent/src/entities/physical_property.rs +++ b/optd-persistent/src/entities/physical_property.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/plan_cost.rs b/optd-persistent/src/entities/plan_cost.rs index d284762..1344713 100644 --- a/optd-persistent/src/entities/plan_cost.rs +++ b/optd-persistent/src/entities/plan_cost.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/prelude.rs b/optd-persistent/src/entities/prelude.rs index 5c62836..d904dab 100644 --- a/optd-persistent/src/entities/prelude.rs +++ b/optd-persistent/src/entities/prelude.rs @@ -1,15 +1,14 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 -#![allow(dead_code, unused_imports, unused_variables)] +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 pub use super::attribute::Entity as Attribute; pub use super::attribute_constraint_junction::Entity as AttributeConstraintJunction; pub use super::attribute_foreign_constraint_junction::Entity as AttributeForeignConstraintJunction; pub use super::cascades_group::Entity as CascadesGroup; -pub use super::constraint::Entity as Constraint; +pub use super::constraint_metadata::Entity as ConstraintMetadata; pub use super::database_metadata::Entity as DatabaseMetadata; pub use super::event::Entity as Event; pub use super::group_winner::Entity as GroupWinner; -pub use super::index::Entity as Index; +pub use super::index_metadata::Entity as IndexMetadata; pub use super::logical_children::Entity as LogicalChildren; pub use super::logical_expression::Entity as LogicalExpression; pub use super::logical_property::Entity as LogicalProperty; diff --git a/optd-persistent/src/entities/statistic.rs b/optd-persistent/src/entities/statistic.rs index a368d35..303dc0c 100644 --- a/optd-persistent/src/entities/statistic.rs +++ b/optd-persistent/src/entities/statistic.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 use sea_orm::entity::prelude::*; @@ -9,7 +9,7 @@ pub struct Model { pub id: i32, pub name: String, pub table_id: Option, - pub created_time: DateTimeUtc, + pub creation_time: DateTimeUtc, pub number_of_attributes: i32, pub variant_tag: i32, pub description: String, diff --git a/optd-persistent/src/entities/statistic_to_attribute_junction.rs b/optd-persistent/src/entities/statistic_to_attribute_junction.rs index 63d23fb..4581dd0 100644 --- a/optd-persistent/src/entities/statistic_to_attribute_junction.rs +++ b/optd-persistent/src/entities/statistic_to_attribute_junction.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/table_metadata.rs b/optd-persistent/src/entities/table_metadata.rs index f4d5fcf..c31140c 100644 --- a/optd-persistent/src/entities/table_metadata.rs +++ b/optd-persistent/src/entities/table_metadata.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 use sea_orm::entity::prelude::*; @@ -16,8 +16,8 @@ pub struct Model { pub enum Relation { #[sea_orm(has_many = "super::attribute::Entity")] Attribute, - #[sea_orm(has_many = "super::index::Entity")] - Index, + #[sea_orm(has_many = "super::index_metadata::Entity")] + IndexMetadata, #[sea_orm( belongs_to = "super::namespace_metadata::Entity", from = "Column::NamespaceId", @@ -38,9 +38,9 @@ impl Related for Entity { } } -impl Related for Entity { +impl Related for Entity { fn to() -> RelationDef { - Relation::Index.def() + Relation::IndexMetadata.def() } } diff --git a/optd-persistent/src/entities/trigger.rs b/optd-persistent/src/entities/trigger.rs index a5f9de8..1e3c763 100644 --- a/optd-persistent/src/entities/trigger.rs +++ b/optd-persistent/src/entities/trigger.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/versioned_statistic.rs b/optd-persistent/src/entities/versioned_statistic.rs index c4533bf..112cc59 100644 --- a/optd-persistent/src/entities/versioned_statistic.rs +++ b/optd-persistent/src/entities/versioned_statistic.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/lib.rs b/optd-persistent/src/lib.rs index adb3687..7cbf50f 100644 --- a/optd-persistent/src/lib.rs +++ b/optd-persistent/src/lib.rs @@ -7,10 +7,10 @@ use sea_orm_migration::prelude::*; use migrator::Migrator; -mod entities; +pub mod entities; mod migrator; -mod cost_model; +pub mod cost_model; pub use cost_model::interface::CostModelStorageLayer; pub type StorageResult = Result; diff --git a/optd-persistent/src/migrator/catalog/m20241029_000001_attribute_constraint_junction.rs b/optd-persistent/src/migrator/catalog/m20241029_000001_attribute_constraint_junction.rs index d09fe5b..c97ff99 100644 --- a/optd-persistent/src/migrator/catalog/m20241029_000001_attribute_constraint_junction.rs +++ b/optd-persistent/src/migrator/catalog/m20241029_000001_attribute_constraint_junction.rs @@ -5,7 +5,7 @@ //! //! One constraint might be associated with multiple attributes, for example, a composite primary key. -use crate::migrator::catalog::{attribute::Attribute, constraint::Constraint}; +use crate::migrator::catalog::{attribute::Attribute, constraint_metadata::ConstraintMetadata}; use sea_orm_migration::{prelude::*, schema::*}; #[derive(Iden)] @@ -49,7 +49,7 @@ impl MigrationTrait for Migration { AttributeConstraintJunction::Table, AttributeConstraintJunction::ConstraintId, ) - .to(Constraint::Table, Constraint::Id) + .to(ConstraintMetadata::Table, ConstraintMetadata::Id) .on_delete(ForeignKeyAction::Cascade) .on_update(ForeignKeyAction::Cascade), ) diff --git a/optd-persistent/src/migrator/catalog/m20241029_000001_attribute_foreign_constraint_junction.rs b/optd-persistent/src/migrator/catalog/m20241029_000001_attribute_foreign_constraint_junction.rs index 5b6cc7c..7b95600 100644 --- a/optd-persistent/src/migrator/catalog/m20241029_000001_attribute_foreign_constraint_junction.rs +++ b/optd-persistent/src/migrator/catalog/m20241029_000001_attribute_foreign_constraint_junction.rs @@ -9,7 +9,7 @@ //! One foreign key constraint might be associated with multiple attributes, for example, a composite //! foreign key. -use crate::migrator::catalog::{attribute::Attribute, constraint::Constraint}; +use crate::migrator::catalog::{attribute::Attribute, constraint_metadata::ConstraintMetadata}; use sea_orm_migration::{prelude::*, schema::*}; #[derive(Iden)] @@ -53,7 +53,7 @@ impl MigrationTrait for Migration { AttributeForeignConstraintJunction::Table, AttributeForeignConstraintJunction::ConstraintId, ) - .to(Constraint::Table, Constraint::Id) + .to(ConstraintMetadata::Table, ConstraintMetadata::Id) .on_delete(ForeignKeyAction::Cascade) .on_update(ForeignKeyAction::Cascade), ) diff --git a/optd-persistent/src/migrator/catalog/m20241029_000001_constraint.rs b/optd-persistent/src/migrator/catalog/m20241029_000001_constraint_metadata.rs similarity index 58% rename from optd-persistent/src/migrator/catalog/m20241029_000001_constraint.rs rename to optd-persistent/src/migrator/catalog/m20241029_000001_constraint_metadata.rs index 4160602..f6435eb 100644 --- a/optd-persistent/src/migrator/catalog/m20241029_000001_constraint.rs +++ b/optd-persistent/src/migrator/catalog/m20241029_000001_constraint_metadata.rs @@ -1,8 +1,8 @@ -use crate::migrator::catalog::{index::Index, table_metadata::TableMetadata}; +use crate::migrator::catalog::{index_metadata::IndexMetadata, table_metadata::TableMetadata}; use sea_orm_migration::{prelude::*, schema::*}; #[derive(Iden)] -pub enum Constraint { +pub enum ConstraintMetadata { Table, Id, Name, @@ -22,36 +22,36 @@ impl MigrationTrait for Migration { manager .create_table( Table::create() - .table(Constraint::Table) + .table(ConstraintMetadata::Table) .if_not_exists() - .col(pk_auto(Constraint::Id)) - .col(string(Constraint::Name)) - .col(integer(Constraint::VariantTag)) - .col(integer_null(Constraint::TableId)) + .col(pk_auto(ConstraintMetadata::Id)) + .col(string(ConstraintMetadata::Name)) + .col(integer(ConstraintMetadata::VariantTag)) + .col(integer_null(ConstraintMetadata::TableId)) .foreign_key( ForeignKey::create() - .from(Constraint::Table, Constraint::TableId) + .from(ConstraintMetadata::Table, ConstraintMetadata::TableId) .to(TableMetadata::Table, TableMetadata::Id) .on_delete(ForeignKeyAction::Cascade) .on_update(ForeignKeyAction::Cascade), ) - .col(integer_null(Constraint::IndexId)) + .col(integer_null(ConstraintMetadata::IndexId)) .foreign_key( ForeignKey::create() - .from(Constraint::Table, Constraint::IndexId) - .to(Index::Table, Index::Id) + .from(ConstraintMetadata::Table, ConstraintMetadata::IndexId) + .to(IndexMetadata::Table, IndexMetadata::Id) .on_delete(ForeignKeyAction::Cascade) .on_update(ForeignKeyAction::Cascade), ) - .col(integer_null(Constraint::ForeignRefId)) + .col(integer_null(ConstraintMetadata::ForeignRefId)) .foreign_key( ForeignKey::create() - .from(Constraint::Table, Constraint::ForeignRefId) + .from(ConstraintMetadata::Table, ConstraintMetadata::ForeignRefId) .to(TableMetadata::Table, TableMetadata::Id) .on_delete(ForeignKeyAction::Cascade) .on_update(ForeignKeyAction::Cascade), ) - .col(string(Constraint::CheckSrc)) + .col(string(ConstraintMetadata::CheckSrc)) .to_owned(), ) .await @@ -59,7 +59,7 @@ impl MigrationTrait for Migration { async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { manager - .drop_table(Table::drop().table(Constraint::Table).to_owned()) + .drop_table(Table::drop().table(ConstraintMetadata::Table).to_owned()) .await } } diff --git a/optd-persistent/src/migrator/catalog/m20241029_000001_index.rs b/optd-persistent/src/migrator/catalog/m20241029_000001_index_metadata.rs similarity index 55% rename from optd-persistent/src/migrator/catalog/m20241029_000001_index.rs rename to optd-persistent/src/migrator/catalog/m20241029_000001_index_metadata.rs index 1c8ede9..1d51a6f 100644 --- a/optd-persistent/src/migrator/catalog/m20241029_000001_index.rs +++ b/optd-persistent/src/migrator/catalog/m20241029_000001_index_metadata.rs @@ -2,7 +2,7 @@ use crate::migrator::catalog::table_metadata::TableMetadata; use sea_orm_migration::{prelude::*, schema::*}; #[derive(Iden)] -pub enum Index { +pub enum IndexMetadata { Table, Id, TableId, @@ -26,26 +26,26 @@ impl MigrationTrait for Migration { manager .create_table( Table::create() - .table(Index::Table) + .table(IndexMetadata::Table) .if_not_exists() - .col(pk_auto(Index::Id)) - .col(integer(Index::TableId)) + .col(pk_auto(IndexMetadata::Id)) + .col(integer(IndexMetadata::TableId)) .foreign_key( ForeignKey::create() - .from(Index::Table, Index::TableId) + .from(IndexMetadata::Table, IndexMetadata::TableId) .to(TableMetadata::Table, TableMetadata::Id) .on_delete(ForeignKeyAction::Cascade) .on_update(ForeignKeyAction::Cascade), ) - .col(string(Index::Name)) - .col(integer(Index::NumberOfAttributes)) - .col(integer(Index::VariantTag)) - .col(boolean(Index::IsUnique)) - .col(boolean(Index::NullsNotDistinct)) - .col(boolean(Index::IsPrimary)) - .col(boolean(Index::IsClustered)) - .col(boolean(Index::IsExclusion)) - .col(string(Index::Description)) + .col(string(IndexMetadata::Name)) + .col(integer(IndexMetadata::NumberOfAttributes)) + .col(integer(IndexMetadata::VariantTag)) + .col(boolean(IndexMetadata::IsUnique)) + .col(boolean(IndexMetadata::NullsNotDistinct)) + .col(boolean(IndexMetadata::IsPrimary)) + .col(boolean(IndexMetadata::IsClustered)) + .col(boolean(IndexMetadata::IsExclusion)) + .col(string(IndexMetadata::Description)) .to_owned(), ) .await @@ -53,7 +53,7 @@ impl MigrationTrait for Migration { async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { manager - .drop_table(Table::drop().table(Index::Table).to_owned()) + .drop_table(Table::drop().table(IndexMetadata::Table).to_owned()) .await } } diff --git a/optd-persistent/src/migrator/catalog/mod.rs b/optd-persistent/src/migrator/catalog/mod.rs index 1fa3880..bef234a 100644 --- a/optd-persistent/src/migrator/catalog/mod.rs +++ b/optd-persistent/src/migrator/catalog/mod.rs @@ -1,9 +1,9 @@ pub(crate) mod m20241029_000001_attribute; pub(crate) mod m20241029_000001_attribute_constraint_junction; pub(crate) mod m20241029_000001_attribute_foreign_constraint_junction; -pub(crate) mod m20241029_000001_constraint; +pub(crate) mod m20241029_000001_constraint_metadata; pub(crate) mod m20241029_000001_database_metadata; -pub(crate) mod m20241029_000001_index; +pub(crate) mod m20241029_000001_index_metadata; pub(crate) mod m20241029_000001_namespace_metadata; pub(crate) mod m20241029_000001_table_metadata; pub(crate) mod m20241029_000001_trigger; @@ -11,9 +11,9 @@ pub(crate) mod m20241029_000001_trigger; pub(crate) use m20241029_000001_attribute as attribute; pub(crate) use m20241029_000001_attribute_constraint_junction as attribute_constraint_junction; pub(crate) use m20241029_000001_attribute_foreign_constraint_junction as attribute_foreign_constraint_junction; -pub(crate) use m20241029_000001_constraint as constraint; +pub(crate) use m20241029_000001_constraint_metadata as constraint_metadata; pub(crate) use m20241029_000001_database_metadata as database_metadata; -pub(crate) use m20241029_000001_index as index; +pub(crate) use m20241029_000001_index_metadata as index_metadata; pub(crate) use m20241029_000001_namespace_metadata as namespace_metadata; pub(crate) use m20241029_000001_table_metadata as table_metadata; pub(crate) use m20241029_000001_trigger as trigger; diff --git a/optd-persistent/src/migrator/cost_model/m20241029_000001_statistic.rs b/optd-persistent/src/migrator/cost_model/m20241029_000001_statistic.rs index 7284422..3b2bb3c 100644 --- a/optd-persistent/src/migrator/cost_model/m20241029_000001_statistic.rs +++ b/optd-persistent/src/migrator/cost_model/m20241029_000001_statistic.rs @@ -15,7 +15,7 @@ pub enum Statistic { Name, // null if not a table statistic. TableId, - CreatedTime, + CreationTime, // 0 if a table statistic. NumberOfAttributes, VariantTag, @@ -47,7 +47,7 @@ impl MigrationTrait for Migration { .on_delete(ForeignKeyAction::Cascade) .on_update(ForeignKeyAction::Cascade), ) - .col(timestamp(Statistic::CreatedTime)) + .col(timestamp(Statistic::CreationTime)) .col(integer(Statistic::NumberOfAttributes)) .col(integer(Statistic::VariantTag)) .col(string(Statistic::Description)) diff --git a/optd-persistent/src/migrator/mod.rs b/optd-persistent/src/migrator/mod.rs index 377f89b..2571143 100644 --- a/optd-persistent/src/migrator/mod.rs +++ b/optd-persistent/src/migrator/mod.rs @@ -16,9 +16,9 @@ impl MigratorTrait for Migrator { Box::new(catalog::attribute::Migration), Box::new(catalog::attribute_constraint_junction::Migration), Box::new(catalog::attribute_foreign_constraint_junction::Migration), - Box::new(catalog::index::Migration), + Box::new(catalog::index_metadata::Migration), Box::new(catalog::trigger::Migration), - Box::new(catalog::constraint::Migration), + Box::new(catalog::constraint_metadata::Migration), Box::new(cost_model::statistic::Migration), Box::new(cost_model::versioned_statistic::Migration), Box::new(cost_model::statistic_to_attribute_junction::Migration), diff --git a/schema/all_tables.dbml b/schema/all_tables.dbml index c4350ed..91185b2 100644 --- a/schema/all_tables.dbml +++ b/schema/all_tables.dbml @@ -3,21 +3,22 @@ Table database_metadata { id integer PK - created_time timestamp + name varchar + creation_time timestamp } Table namespace_metadata { id integer PK database_id integer [ref: > database_metadata.id] name varchar - created_time timestamp + creation_time timestamp } Table table_metadata { id integer PK - schema_id integer [ref: > namespace_metadata.id] + namespace_id integer [ref: > namespace_metadata.id] name varchar - created_time timestamp + creation_time timestamp } Table attribute { @@ -25,8 +26,8 @@ Table attribute { table_id integer [ref: > table_metadata.id] name varchar compression_method char - type integer // Data type of this attribute. Should we make another table to explain the type mapping? - base_col_number integer // local index within the table + variant_tag integer // Data type of this attribute. Should we make another table to explain the type mapping? + base_attribute_number integer // local index within the table is_not_null boolean // physical property } @@ -34,7 +35,7 @@ Table statistic { id integer PK name varchar table_id integer [ref: > table_metadata.id, null] // null if not a table statistic - created_time timestamp + creation_time timestamp number_of_attributes integer // 0 if a table statistic variant_tag integer // Should we make another table to explain the type mapping? description varchar // Store the sorted attribute ids of this statistic, to support quick lookup (OR we can use junction table to look up) From ea154e62c022e884e3b9b46452cf44a51efec1ac Mon Sep 17 00:00:00 2001 From: Lan Lou Date: Thu, 7 Nov 2024 18:01:48 -0500 Subject: [PATCH 14/28] Fix clippy --- optd-persistent/src/bin/init.rs | 23 ------------------- .../src/cost_model/catalog/mock_catalog.rs | 1 + optd-persistent/src/cost_model/orm.rs | 6 ++--- optd-persistent/src/entities/attribute.rs | 2 +- .../entities/attribute_constraint_junction.rs | 2 +- .../attribute_foreign_constraint_junction.rs | 2 +- .../src/entities/cascades_group.rs | 2 +- .../src/entities/constraint_metadata.rs | 2 +- .../src/entities/database_metadata.rs | 2 +- optd-persistent/src/entities/event.rs | 2 +- optd-persistent/src/entities/group_winner.rs | 2 +- .../src/entities/index_metadata.rs | 2 +- .../src/entities/logical_children.rs | 2 +- .../src/entities/logical_expression.rs | 2 +- .../src/entities/logical_property.rs | 2 +- optd-persistent/src/entities/mod.rs | 2 +- .../src/entities/namespace_metadata.rs | 2 +- .../src/entities/physical_children.rs | 2 +- .../src/entities/physical_expression.rs | 2 +- ...ysical_expression_to_statistic_junction.rs | 2 +- .../src/entities/physical_property.rs | 2 +- optd-persistent/src/entities/plan_cost.rs | 2 +- optd-persistent/src/entities/prelude.rs | 2 +- optd-persistent/src/entities/statistic.rs | 2 +- .../statistic_to_attribute_junction.rs | 2 +- .../src/entities/table_metadata.rs | 2 +- optd-persistent/src/entities/trigger.rs | 2 +- .../src/entities/versioned_statistic.rs | 2 +- 28 files changed, 29 insertions(+), 51 deletions(-) diff --git a/optd-persistent/src/bin/init.rs b/optd-persistent/src/bin/init.rs index 333452f..37140c3 100644 --- a/optd-persistent/src/bin/init.rs +++ b/optd-persistent/src/bin/init.rs @@ -22,7 +22,6 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { id: Set(1), name: Set("database1".to_owned()), creation_time: Set(Utc::now()), - ..Default::default() }; database_metadata::Entity::insert(database_metadata) .exec(&db) @@ -35,7 +34,6 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { database_id: Set(1), name: Set("default".to_owned()), creation_time: Set(Utc::now()), - ..Default::default() }; namespace_metadata::Entity::insert(namespace_metadata) .exec(&db) @@ -48,7 +46,6 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { namespace_id: Set(1), name: Set("users".to_owned()), creation_time: Set(Utc::now()), - ..Default::default() }; table_metadata::Entity::insert(table_metadata) .exec(&db) @@ -64,7 +61,6 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { variant_tag: Set(1), base_attribute_number: Set(1), is_not_null: Set(true), - ..Default::default() }; attribute::Entity::insert(attribute) .exec(&db) @@ -80,7 +76,6 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { number_of_attributes: Set(0), variant_tag: Set(1), description: Set("Total rows".to_owned()), - ..Default::default() }; statistic::Entity::insert(statistic) .exec(&db) @@ -92,7 +87,6 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { source_variant: Set("insert".to_owned()), timestamp: Set(Utc::now()), data: Set(json!(r#"{"user_id": 1}"#)), - ..Default::default() }; event::Entity::insert(event) @@ -106,7 +100,6 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { epoch_id: Set(1), statistic_id: Set(1), statistic_value: Set(json!(r#"{"row_count": 0}"#)), - ..Default::default() }; versioned_statistic::Entity::insert(versioned_statistic) .exec(&db) @@ -126,7 +119,6 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { variant_tag: Set(1), number_of_attributes: Set(1), description: Set("random".to_owned()), - ..Default::default() }; index_metadata::Entity::insert(index_metadata) .exec(&db) @@ -140,7 +132,6 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { table_id: Set(1), parent_trigger_id: Set(1), function: Set(json!(r#"{"function": "insert"}"#)), - ..Default::default() }; trigger::Entity::insert(trigger) .exec(&db) @@ -156,7 +147,6 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { index_id: Set(Some(1)), foreign_ref_id: Set(None), check_src: Set("hello".to_owned()), - ..Default::default() }; constraint_metadata::Entity::insert(constraint_metadata) .exec(&db) @@ -167,7 +157,6 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { let attribute_constraint_junction = attribute_constraint_junction::ActiveModel { attribute_id: Set(1), constraint_id: Set(1), - ..Default::default() }; attribute_constraint_junction::Entity::insert(attribute_constraint_junction) .exec(&db) @@ -179,7 +168,6 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { attribute_foreign_constraint_junction::ActiveModel { attribute_id: Set(1), constraint_id: Set(1), - ..Default::default() }; attribute_foreign_constraint_junction::Entity::insert(attribute_foreign_constraint_junction) .exec(&db) @@ -190,7 +178,6 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { let statistic_to_attribute_junction = statistic_to_attribute_junction::ActiveModel { statistic_id: Set(1), attribute_id: Set(1), - ..Default::default() }; statistic_to_attribute_junction::Entity::insert(statistic_to_attribute_junction) .exec(&db) @@ -203,7 +190,6 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { latest_winner: Set(None), in_progress: Set(true), is_optimized: Set(false), - ..Default::default() }; cascades_group::Entity::insert(cascades_group) .exec(&db) @@ -217,7 +203,6 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { fingerprint: Set(12345), variant_tag: Set(1), data: Set(json!(r#"{"expr": "index_scan"}"#)), - ..Default::default() }; logical_expression::Entity::insert(logical_expression) .exec(&db) @@ -231,7 +216,6 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { fingerprint: Set(12345), variant_tag: Set(1), data: Set(json!(r#"{"expr": "index_scan"}"#)), - ..Default::default() }; physical_expression::Entity::insert(physical_expression) .exec(&db) @@ -244,7 +228,6 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { physical_expression_id: Set(1), variant_tag: Set(1), data: Set(json!(r#"{"property": "indexed"}"#)), - ..Default::default() }; physical_property::Entity::insert(physical_property) .exec(&db) @@ -257,7 +240,6 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { group_id: Set(1), variant_tag: Set(1), data: Set(json!(r#"{"property": "indexed"}"#)), - ..Default::default() }; logical_property::Entity::insert(logical_property) .exec(&db) @@ -267,7 +249,6 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { let logical_children = logical_children::ActiveModel { logical_expression_id: Set(1), group_id: Set(1), - ..Default::default() }; logical_children::Entity::insert(logical_children) .exec(&db) @@ -277,7 +258,6 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { let physical_children = physical_children::ActiveModel { physical_expression_id: Set(1), group_id: Set(1), - ..Default::default() }; physical_children::Entity::insert(physical_children) .exec(&db) @@ -291,7 +271,6 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { epoch_id: Set(1), cost: Set(10), is_valid: Set(true), - ..Default::default() }; plan_cost::Entity::insert(plan_cost) .exec(&db) @@ -303,7 +282,6 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { physical_expression_to_statistic_junction::ActiveModel { physical_expression_id: Set(1), statistic_id: Set(1), - ..Default::default() }; physical_expression_to_statistic_junction::Entity::insert( physical_expression_to_statistic_junction, @@ -319,7 +297,6 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { physical_expression_id: Set(1), cost_id: Set(1), epoch_id: Set(1), - ..Default::default() }; group_winner::Entity::insert(group_winner) .exec(&db) diff --git a/optd-persistent/src/cost_model/catalog/mock_catalog.rs b/optd-persistent/src/cost_model/catalog/mock_catalog.rs index b514916..1ca4f96 100644 --- a/optd-persistent/src/cost_model/catalog/mock_catalog.rs +++ b/optd-persistent/src/cost_model/catalog/mock_catalog.rs @@ -61,6 +61,7 @@ pub struct MockTrigger { pub function: String, } +#[derive(Default)] pub struct MockCatalog { pub databases: Vec, pub namespaces: Vec, diff --git a/optd-persistent/src/cost_model/orm.rs b/optd-persistent/src/cost_model/orm.rs index f383a2f..7e6c14c 100644 --- a/optd-persistent/src/cost_model/orm.rs +++ b/optd-persistent/src/cost_model/orm.rs @@ -481,7 +481,7 @@ mod tests { async fn copy_init_db(db_file: &str) -> String { let original_db = "init.db"; let _ = std::fs::copy(original_db, format!("./{}", db_file)); - let database_url = format!("sqlite:./{}?mode=rwc", format!("./{}", db_file)); + let database_url = format!("sqlite:./{}?mode=rwc", db_file); database_url } @@ -549,10 +549,10 @@ mod tests { .store_cost(physical_expression_id, cost, epoch_id) .await; match res { - Ok(_) => assert!(true), + Ok(_) => {} Err(e) => { println!("Error: {:?}", e); - assert!(false); + panic!(); } } let costs = super::PlanCost::find() diff --git a/optd-persistent/src/entities/attribute.rs b/optd-persistent/src/entities/attribute.rs index f45d04a..aa312c0 100644 --- a/optd-persistent/src/entities/attribute.rs +++ b/optd-persistent/src/entities/attribute.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/attribute_constraint_junction.rs b/optd-persistent/src/entities/attribute_constraint_junction.rs index e1d8052..7c61c72 100644 --- a/optd-persistent/src/entities/attribute_constraint_junction.rs +++ b/optd-persistent/src/entities/attribute_constraint_junction.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/attribute_foreign_constraint_junction.rs b/optd-persistent/src/entities/attribute_foreign_constraint_junction.rs index c2f0b61..f219a39 100644 --- a/optd-persistent/src/entities/attribute_foreign_constraint_junction.rs +++ b/optd-persistent/src/entities/attribute_foreign_constraint_junction.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/cascades_group.rs b/optd-persistent/src/entities/cascades_group.rs index d582a1c..9ea79d1 100644 --- a/optd-persistent/src/entities/cascades_group.rs +++ b/optd-persistent/src/entities/cascades_group.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/constraint_metadata.rs b/optd-persistent/src/entities/constraint_metadata.rs index 5630e3e..8adf1d7 100644 --- a/optd-persistent/src/entities/constraint_metadata.rs +++ b/optd-persistent/src/entities/constraint_metadata.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/database_metadata.rs b/optd-persistent/src/entities/database_metadata.rs index 190bc73..98106d6 100644 --- a/optd-persistent/src/entities/database_metadata.rs +++ b/optd-persistent/src/entities/database_metadata.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/event.rs b/optd-persistent/src/entities/event.rs index c26c7b1..ad9157d 100644 --- a/optd-persistent/src/entities/event.rs +++ b/optd-persistent/src/entities/event.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/group_winner.rs b/optd-persistent/src/entities/group_winner.rs index efd8661..32cbcb4 100644 --- a/optd-persistent/src/entities/group_winner.rs +++ b/optd-persistent/src/entities/group_winner.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/index_metadata.rs b/optd-persistent/src/entities/index_metadata.rs index 49a1ea2..47263b7 100644 --- a/optd-persistent/src/entities/index_metadata.rs +++ b/optd-persistent/src/entities/index_metadata.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/logical_children.rs b/optd-persistent/src/entities/logical_children.rs index 92d53bd..120641f 100644 --- a/optd-persistent/src/entities/logical_children.rs +++ b/optd-persistent/src/entities/logical_children.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/logical_expression.rs b/optd-persistent/src/entities/logical_expression.rs index aa3b314..6beff04 100644 --- a/optd-persistent/src/entities/logical_expression.rs +++ b/optd-persistent/src/entities/logical_expression.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/logical_property.rs b/optd-persistent/src/entities/logical_property.rs index f42ffdd..755575c 100644 --- a/optd-persistent/src/entities/logical_property.rs +++ b/optd-persistent/src/entities/logical_property.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/mod.rs b/optd-persistent/src/entities/mod.rs index 1d4a0fa..9434c2a 100644 --- a/optd-persistent/src/entities/mod.rs +++ b/optd-persistent/src/entities/mod.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 pub mod prelude; diff --git a/optd-persistent/src/entities/namespace_metadata.rs b/optd-persistent/src/entities/namespace_metadata.rs index f3f3f37..8f63146 100644 --- a/optd-persistent/src/entities/namespace_metadata.rs +++ b/optd-persistent/src/entities/namespace_metadata.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/physical_children.rs b/optd-persistent/src/entities/physical_children.rs index 18c1794..d8f9db0 100644 --- a/optd-persistent/src/entities/physical_children.rs +++ b/optd-persistent/src/entities/physical_children.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/physical_expression.rs b/optd-persistent/src/entities/physical_expression.rs index 02795d2..4876d35 100644 --- a/optd-persistent/src/entities/physical_expression.rs +++ b/optd-persistent/src/entities/physical_expression.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/physical_expression_to_statistic_junction.rs b/optd-persistent/src/entities/physical_expression_to_statistic_junction.rs index 0332f02..f8020bd 100644 --- a/optd-persistent/src/entities/physical_expression_to_statistic_junction.rs +++ b/optd-persistent/src/entities/physical_expression_to_statistic_junction.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/physical_property.rs b/optd-persistent/src/entities/physical_property.rs index 944a94d..51d98dc 100644 --- a/optd-persistent/src/entities/physical_property.rs +++ b/optd-persistent/src/entities/physical_property.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/plan_cost.rs b/optd-persistent/src/entities/plan_cost.rs index 1344713..d284762 100644 --- a/optd-persistent/src/entities/plan_cost.rs +++ b/optd-persistent/src/entities/plan_cost.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/prelude.rs b/optd-persistent/src/entities/prelude.rs index d904dab..753e58b 100644 --- a/optd-persistent/src/entities/prelude.rs +++ b/optd-persistent/src/entities/prelude.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 pub use super::attribute::Entity as Attribute; pub use super::attribute_constraint_junction::Entity as AttributeConstraintJunction; diff --git a/optd-persistent/src/entities/statistic.rs b/optd-persistent/src/entities/statistic.rs index 303dc0c..1eaf96b 100644 --- a/optd-persistent/src/entities/statistic.rs +++ b/optd-persistent/src/entities/statistic.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/statistic_to_attribute_junction.rs b/optd-persistent/src/entities/statistic_to_attribute_junction.rs index 4581dd0..63d23fb 100644 --- a/optd-persistent/src/entities/statistic_to_attribute_junction.rs +++ b/optd-persistent/src/entities/statistic_to_attribute_junction.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/table_metadata.rs b/optd-persistent/src/entities/table_metadata.rs index c31140c..35ea923 100644 --- a/optd-persistent/src/entities/table_metadata.rs +++ b/optd-persistent/src/entities/table_metadata.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/trigger.rs b/optd-persistent/src/entities/trigger.rs index 1e3c763..a5f9de8 100644 --- a/optd-persistent/src/entities/trigger.rs +++ b/optd-persistent/src/entities/trigger.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 use sea_orm::entity::prelude::*; diff --git a/optd-persistent/src/entities/versioned_statistic.rs b/optd-persistent/src/entities/versioned_statistic.rs index 112cc59..c4533bf 100644 --- a/optd-persistent/src/entities/versioned_statistic.rs +++ b/optd-persistent/src/entities/versioned_statistic.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.1 +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 use sea_orm::entity::prelude::*; From f56946c42035c46e48155e4f8a239a5e2c002bfd Mon Sep 17 00:00:00 2001 From: Yuanxin Cao Date: Thu, 7 Nov 2024 18:39:30 -0500 Subject: [PATCH 15/28] fix description --- optd-persistent/src/bin/init.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/optd-persistent/src/bin/init.rs b/optd-persistent/src/bin/init.rs index 37140c3..0364baf 100644 --- a/optd-persistent/src/bin/init.rs +++ b/optd-persistent/src/bin/init.rs @@ -75,7 +75,7 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { creation_time: Set(Utc::now()), number_of_attributes: Set(0), variant_tag: Set(1), - description: Set("Total rows".to_owned()), + description: Set("".to_owned()), }; statistic::Entity::insert(statistic) .exec(&db) @@ -118,7 +118,7 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { is_exclusion: Set(false), variant_tag: Set(1), number_of_attributes: Set(1), - description: Set("random".to_owned()), + description: Set("1".to_owned()), }; index_metadata::Entity::insert(index_metadata) .exec(&db) From 8cc4f0e07c5e1ddfc4ae5a66716ba133a6e68a98 Mon Sep 17 00:00:00 2001 From: Lan Lou Date: Thu, 7 Nov 2024 20:42:22 -0500 Subject: [PATCH 16/28] Modify update_stat and add one related test --- optd-persistent/src/cost_model/interface.rs | 1 + optd-persistent/src/cost_model/orm.rs | 211 +++++++++++++++++--- 2 files changed, 188 insertions(+), 24 deletions(-) diff --git a/optd-persistent/src/cost_model/interface.rs b/optd-persistent/src/cost_model/interface.rs index 2062a59..aec2ea7 100644 --- a/optd-persistent/src/cost_model/interface.rs +++ b/optd-persistent/src/cost_model/interface.rs @@ -18,6 +18,7 @@ pub enum CatalogSource { pub enum StatisticType { Count, + Cardinality, Min, Max, } diff --git a/optd-persistent/src/cost_model/orm.rs b/optd-persistent/src/cost_model/orm.rs index 7e6c14c..1dd41c4 100644 --- a/optd-persistent/src/cost_model/orm.rs +++ b/optd-persistent/src/cost_model/orm.rs @@ -8,8 +8,8 @@ use sea_orm::prelude::{Expr, Json}; use sea_orm::sea_query::Query; use sea_orm::{sqlx::types::chrono::Utc, EntityTrait}; use sea_orm::{ - ActiveModelTrait, ColumnTrait, DbErr, DeleteResult, EntityOrSelect, ModelTrait, QueryFilter, - QueryOrder, QuerySelect, RuntimeErr, TransactionTrait, + ActiveModelTrait, ColumnTrait, DbBackend, DbErr, DeleteResult, EntityOrSelect, ModelTrait, + QueryFilter, QueryOrder, QuerySelect, QueryTrait, RuntimeErr, TransactionTrait, }; use super::catalog::mock_catalog::{self, MockCatalog}; @@ -154,8 +154,10 @@ impl CostModelStorageLayer for BackendManager { } } + // **IMPORTANT**: It is the caller's responsibility to ensure that the updated stat is not the same with the last stored stat if + // if it is already exists. async fn update_stats(&self, stat: Stat, epoch_id: Self::EpochId) -> StorageResult<()> { - let transaction = self.db.begin().await?; + // let transaction = self.db.begin().await?; // 0. Check if the stat already exists. If exists, get stat_id, else insert into statistic table. let stat_id = match stat.table_id { Some(table_id) => { @@ -163,18 +165,29 @@ impl CostModelStorageLayer for BackendManager { let res = Statistic::find() .filter(statistic::Column::TableId.eq(table_id)) .filter(statistic::Column::VariantTag.eq(stat.stat_type)) - // FIX_ME: Do we need the following filter? - .inner_join(versioned_statistic::Entity) - .select_also(versioned_statistic::Entity) - .order_by_desc(versioned_statistic::Column::EpochId) + /* + TODO(FIX_ME, lanlou): Do we need the following filter? + I am really not sure although I add the top comment... + Since we already increase the epoch, so we should update the stat anyway. + (In theory, we can increase the epoch without updating the stat, but it is not + a straightforward design, and the epoch table will be very large.) + But it will increase the overhead, since the caller will need to make another + query to check if the stat is the same with the last one. We cannot put everything + in one query. + Let us assume we should update the stat anyway for now. + */ + // .inner_join(versioned_statistic::Entity) + // .select_also(versioned_statistic::Entity) + // .order_by_desc(versioned_statistic::Column::EpochId) .one(&self.db) .await?; match res { Some(stat_data) => { - if stat_data.1.unwrap().statistic_value == stat.stat_value { - return Ok(()); - } - stat_data.0.id + // if stat_data.1.unwrap().statistic_value == stat.stat_value { + // return Ok(()); + // } + // stat_data.0.id + stat_data.id } None => { let new_stat = statistic::ActiveModel { @@ -208,18 +221,18 @@ impl CostModelStorageLayer for BackendManager { .filter(statistic::Column::NumberOfAttributes.eq(stat.attr_ids.len() as i32)) .filter(statistic::Column::Description.eq(description.clone())) .filter(statistic::Column::VariantTag.eq(stat.stat_type)) - // FIX_ME: Do we need the following filter? - .inner_join(versioned_statistic::Entity) - .select_also(versioned_statistic::Entity) - .order_by_desc(versioned_statistic::Column::EpochId) + // .inner_join(versioned_statistic::Entity) + // .select_also(versioned_statistic::Entity) + // .order_by_desc(versioned_statistic::Column::EpochId) .one(&self.db) .await?; match res { Some(stat_data) => { - if stat_data.1.unwrap().statistic_value == stat.stat_value { - return Ok(()); - } - stat_data.0.id + // if stat_data.1.unwrap().statistic_value == stat.stat_value { + // return Ok(()); + // } + // stat_data.0.id + stat_data.id } None => { let new_stat = statistic::ActiveModel { @@ -269,24 +282,25 @@ impl CostModelStorageLayer for BackendManager { let _ = plan_cost::Entity::update_many() .col_expr(plan_cost::Column::IsValid, Expr::value(false)) .filter(plan_cost::Column::IsValid.eq(true)) + .filter(plan_cost::Column::EpochId.lt(epoch_id)) .filter( plan_cost::Column::PhysicalExpressionId.in_subquery( - (*Query::select() + Query::select() .column( physical_expression_to_statistic_junction::Column::PhysicalExpressionId, ) .from(physical_expression_to_statistic_junction::Entity) - .and_where( + .cond_where( physical_expression_to_statistic_junction::Column::StatisticId .eq(stat_id), - )) - .to_owned(), + ) + .to_owned(), ), ) .exec(&self.db) .await; - transaction.commit().await?; + // transaction.commit().await?; Ok(()) } @@ -441,6 +455,7 @@ impl CostModelStorageLayer for BackendManager { #[cfg(test)] mod tests { + use crate::cost_model::interface::StatisticType; use crate::{cost_model::interface::Stat, migrate, CostModelStorageLayer}; use sea_orm::sqlx::database; use sea_orm::Statement; @@ -532,6 +547,154 @@ mod tests { remove_db_file(DATABASE_FILE); } + #[tokio::test] + async fn test_update_attr_stats() { + const DATABASE_FILE: &str = "test_update_attr_stats.db"; + let database_url = copy_init_db(DATABASE_FILE).await; + let mut binding = super::BackendManager::new(Some(&database_url)).await; + let backend_manager = binding.as_mut().unwrap(); + // 1. Update non-existed stat + let epoch_id1 = backend_manager + .create_new_epoch("test".to_string(), "InsertTest".to_string()) + .await + .unwrap(); + let stat = Stat { + stat_type: StatisticType::Count as i32, + stat_value: "100".to_string(), + attr_ids: vec![1], + table_id: None, + name: "CountAttr1".to_string(), + }; + let res = backend_manager.update_stats(stat, epoch_id1).await; + assert!(res.is_ok()); + let stat_res = Statistic::find() + .filter(statistic::Column::Name.eq("CountAttr1")) + .all(&backend_manager.db) + .await + .unwrap(); + assert_eq!(stat_res.len(), 1); + println!("{:?}", stat_res); + assert_eq!(stat_res[0].number_of_attributes, 1); + assert_eq!(stat_res[0].description, "1".to_string()); + assert_eq!(stat_res[0].variant_tag, StatisticType::Count as i32); + let stat_attr_res = StatisticToAttributeJunction::find() + .filter(statistic_to_attribute_junction::Column::StatisticId.eq(stat_res[0].id)) + .all(&backend_manager.db) + .await + .unwrap(); + assert_eq!(stat_attr_res.len(), 1); + assert_eq!(stat_attr_res[0].attribute_id, 1); + let versioned_stat_res = VersionedStatistic::find() + .filter(versioned_statistic::Column::StatisticId.eq(stat_res[0].id)) + .all(&backend_manager.db) + .await + .unwrap(); + assert_eq!(versioned_stat_res.len(), 1); + assert_eq!( + versioned_stat_res[0].statistic_value, + serde_json::Value::String("100".to_string()) + ); + assert_eq!(versioned_stat_res[0].epoch_id, epoch_id1); + + // 2. Normal update + // 2.1 Insert some costs + let res = PhysicalExpression::insert(physical_expression::ActiveModel { + group_id: sea_orm::ActiveValue::Set(1), + fingerprint: sea_orm::ActiveValue::Set(12346), + variant_tag: sea_orm::ActiveValue::Set(1), + data: sea_orm::ActiveValue::Set(serde_json::Value::String("data".to_string())), + ..Default::default() + }); + let expr_id = res.exec(&backend_manager.db).await.unwrap().last_insert_id; + let res = PhysicalExpressionToStatisticJunction::insert( + physical_expression_to_statistic_junction::ActiveModel { + physical_expression_id: sea_orm::ActiveValue::Set(expr_id), + statistic_id: sea_orm::ActiveValue::Set(stat_res[0].id), + }, + ) + .exec(&backend_manager.db) + .await + .unwrap(); + backend_manager + .store_cost(expr_id, 42, versioned_stat_res[0].epoch_id) + .await + .unwrap(); + let cost_res = PlanCost::find() + .filter(plan_cost::Column::PhysicalExpressionId.eq(expr_id)) + .all(&backend_manager.db) + .await + .unwrap(); + assert_eq!(cost_res.len(), 1); + assert!(cost_res[0].is_valid); + // 2.2 Normal update + let epoch_id2 = backend_manager + .create_new_epoch("test".to_string(), "InsertTest".to_string()) + .await + .unwrap(); + let stat2 = Stat { + stat_type: StatisticType::Count as i32, + stat_value: "200".to_string(), + attr_ids: vec![1], + table_id: None, + name: "CountAttr1".to_string(), + }; + let res = backend_manager.update_stats(stat2, epoch_id2).await; + assert!(res.is_ok()); + // 2.3 Check statistic table + let stat_res = Statistic::find() + .filter(statistic::Column::Name.eq("CountAttr1")) + .all(&backend_manager.db) + .await + .unwrap(); + assert_eq!(stat_res.len(), 1); + assert_eq!(stat_res[0].number_of_attributes, 1); + assert_eq!(stat_res[0].description, "1".to_string()); + assert_eq!(stat_res[0].variant_tag, StatisticType::Count as i32); + // 2.4 Check statistic_to_attribute_junction table + let stat_attr_res = StatisticToAttributeJunction::find() + .filter(statistic_to_attribute_junction::Column::StatisticId.eq(stat_res[0].id)) + .all(&backend_manager.db) + .await + .unwrap(); + assert_eq!(stat_attr_res.len(), 1); + assert_eq!(stat_attr_res[0].attribute_id, 1); + // 2.5 Check versioned_statistic table + let versioned_stat_res = VersionedStatistic::find() + .filter(versioned_statistic::Column::StatisticId.eq(stat_res[0].id)) + .all(&backend_manager.db) + .await + .unwrap(); + assert_eq!(versioned_stat_res.len(), 2); + assert_eq!( + versioned_stat_res[0].statistic_value, + serde_json::Value::String("100".to_string()) + ); + assert_eq!(versioned_stat_res[0].epoch_id, epoch_id1); + assert_eq!(versioned_stat_res[0].statistic_id, stat_res[0].id); + assert_eq!( + versioned_stat_res[1].statistic_value, + serde_json::Value::String("200".to_string()) + ); + assert_eq!(versioned_stat_res[1].epoch_id, epoch_id2); + assert_eq!(versioned_stat_res[1].statistic_id, stat_res[0].id); + assert!(epoch_id1 < epoch_id2); + // 2.6 Check plan_cost table (cost invalidation) + let cost_res = PlanCost::find() + .filter(plan_cost::Column::PhysicalExpressionId.eq(expr_id)) + .all(&backend_manager.db) + .await + .unwrap(); + assert_eq!(cost_res.len(), 1); + assert_eq!(cost_res[0].cost, 42); + assert_eq!(cost_res[0].epoch_id, epoch_id1); + assert!(!cost_res[0].is_valid); + + remove_db_file(DATABASE_FILE); + } + + #[tokio::test] + async fn test_update_table_stats() {} + #[tokio::test] #[ignore] // Need to update all tables async fn test_store_cost() { From 6371d610c65e9f129b2cdc2a6e76e69655177a56 Mon Sep 17 00:00:00 2001 From: Yuanxin Cao Date: Thu, 7 Nov 2024 21:59:38 -0500 Subject: [PATCH 17/28] refine test infra and track init.db --- .gitignore | 1 + optd-persistent/Cargo.toml | 1 + optd-persistent/src/bin/init.rs | 13 ++++++------- optd-persistent/src/bin/migrate.rs | 4 ++-- optd-persistent/src/cost_model/orm.rs | 8 ++++---- optd-persistent/src/db/init.db | Bin 0 -> 147456 bytes optd-persistent/src/lib.rs | 25 ++++++++++++++++++++++--- 7 files changed, 36 insertions(+), 16 deletions(-) create mode 100644 optd-persistent/src/db/init.db diff --git a/.gitignore b/.gitignore index f777bce..df275d4 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ target/ **/*.rs.bk **/*.db +!init.db .DS_Store diff --git a/optd-persistent/Cargo.toml b/optd-persistent/Cargo.toml index e9b9905..55de4b7 100644 --- a/optd-persistent/Cargo.toml +++ b/optd-persistent/Cargo.toml @@ -21,3 +21,4 @@ trait-variant = "0.1.2" async-trait = "0.1.43" async-stream = "0.3.1" strum = "0.26.1" +lazy_static = "1" diff --git a/optd-persistent/src/bin/init.rs b/optd-persistent/src/bin/init.rs index 0364baf..ec396c7 100644 --- a/optd-persistent/src/bin/init.rs +++ b/optd-persistent/src/bin/init.rs @@ -1,15 +1,15 @@ use optd_persistent::entities::*; use optd_persistent::migrate; +use optd_persistent::TEST_DATABASE_FILENAME; +use optd_persistent::TEST_DATABASE_URL; use sea_orm::sqlx::types::chrono::Utc; use sea_orm::*; use serde_json::json; -async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { - let database_url = format!("sqlite:./{}?mode=rwc", db_file); - let database_file = format!("./{}", db_file); - let _ = std::fs::remove_file(database_file); +async fn init_all_tables() -> Result<(), sea_orm::error::DbErr> { + let _ = std::fs::remove_file(TEST_DATABASE_FILENAME); - let db = Database::connect(database_url.clone()) + let db = Database::connect(TEST_DATABASE_URL.clone()) .await .expect("Unable to connect to the database"); @@ -308,8 +308,7 @@ async fn init_all_tables(db_file: &str) -> Result<(), sea_orm::error::DbErr> { #[tokio::main] async fn main() { - let db_file = "init.db"; - if let Err(e) = init_all_tables(db_file).await { + if let Err(e) = init_all_tables().await { eprintln!("Error initializing database: {}", e); std::process::exit(1); } diff --git a/optd-persistent/src/bin/migrate.rs b/optd-persistent/src/bin/migrate.rs index 9f6d040..d6a36a6 100644 --- a/optd-persistent/src/bin/migrate.rs +++ b/optd-persistent/src/bin/migrate.rs @@ -1,10 +1,10 @@ -use optd_persistent::{migrate, DATABASE_FILE, DATABASE_URL}; +use optd_persistent::{migrate, DATABASE_FILENAME, DATABASE_URL}; use sea_orm::*; use sea_orm_migration::prelude::*; #[tokio::main] async fn main() { - let _ = std::fs::remove_file(DATABASE_FILE); + let _ = std::fs::remove_file(DATABASE_FILENAME); let db = Database::connect(DATABASE_URL) .await diff --git a/optd-persistent/src/cost_model/orm.rs b/optd-persistent/src/cost_model/orm.rs index 1dd41c4..b7bfb11 100644 --- a/optd-persistent/src/cost_model/orm.rs +++ b/optd-persistent/src/cost_model/orm.rs @@ -457,6 +457,7 @@ impl CostModelStorageLayer for BackendManager { mod tests { use crate::cost_model::interface::StatisticType; use crate::{cost_model::interface::Stat, migrate, CostModelStorageLayer}; + use crate::{get_sqlite_url, TEST_DATABASE_FILE}; use sea_orm::sqlx::database; use sea_orm::Statement; use sea_orm::{ @@ -468,7 +469,7 @@ mod tests { use crate::entities::{prelude::*, *}; async fn run_migration(db_file: &str) -> String { - let database_url = format!("sqlite:./{}?mode=rwc", db_file); + let database_url = get_sqlite_url(db_file); remove_db_file(db_file); let db = Database::connect(database_url.clone()) @@ -494,9 +495,8 @@ mod tests { } async fn copy_init_db(db_file: &str) -> String { - let original_db = "init.db"; - let _ = std::fs::copy(original_db, format!("./{}", db_file)); - let database_url = format!("sqlite:./{}?mode=rwc", db_file); + let _ = std::fs::copy(TEST_DATABASE_FILE.clone(), db_file); + let database_url = get_sqlite_url(db_file); database_url } diff --git a/optd-persistent/src/db/init.db b/optd-persistent/src/db/init.db new file mode 100644 index 0000000000000000000000000000000000000000..303510371fe8830ad446329575f08f45b25698b3 GIT binary patch literal 147456 zcmeI5Pi)*)e#gn7G$YD0LpzSm#7-ibu@g-!#WR+T*iM5~k!e-1G;;K(hBwX<)QFbo z%#dS} zSJ_X6dz~^s_yY;79nNLMg9q8W1Fp7E)m__ITGH+Hm;Bg_GCHS7bE9ufDw4b&DocZs zY%I!>VY)gImdev}Qh9!IQo1lRF*Q1KNjk4wl1As}rYFiIgDIsvHzdiXR?}tax@K3F zwN?@m;g_o>te1j*XsGypZ_|XFV$GA?*dk ztK3#nfOsoKNQK){HX{}a?6*GjaT(^KzM?j3OMlAIN~0sY)7^$D>Zn+=QCrY$)w-f; zE>Yk@!_}R*(GVpQqcAj2%i2<6yy2(~)40|k_OoDFRb4Y9+Lg^lwd$y*<*JK@;~K;g z62(X+b=#B3!n} zu-(42c+kYMpgE*hH76%En6NrcLEa4G%{580!?q9ZtLM`G{C|7grxhS;8V>y_Lo0`l z>`p_Sr+BicbLv_}@3b$f(pu1@!6 z#8Qd9J>i3iXBQ7i(?H__4;iYr0QD_1sV8~UURO+`xzb3(nC^&P=t+x5kFuA1p6fG} zqm$K|S<%}*_EIF)n8ju3f9DHDk&4yM*JkeNNG>Xpi=$%0o z<3pXaYOlJMO1v|!cZ^aA&j`FcUdwW|cor>LlCb4QYx;`4ghzKJX>NTNOZKN>7M}2- z{-m~2B1ie8{31@wrVqo*Z=jERBVOzEvqVqerA(3%=o2$f?u|5eBc4&K#T%Tkc*#%p za>ho}76?#c3G3v5!qjQLbN(S9;!iCK#g zEaCSAHuq8f&x9{>KhAw4|F5}h?s)bq;jgpb&Hhd9i>#IV^ZZKwfbbpS9hiP120AnX z1V8`;KmY_l00ck)1VCVW3B1I;%|6(-AZ{$y+}7_Ob zJbvcn$eEFoua2C4?Zn8)(PHsTvFPpp_vAleg?|@*Cj3PBE8#B$TlkKklJ5eH3nzr< zh2H$H^IzouHvh>^0f2gf00@8p2!H?xfB*=900@8p2s~i~ggx95K6;@dyPF%}!@CiC z1#Xy+-SNnC$N2CLMDx2W`*Yk#E3|coV_%jlg;}(M_w3>dJbh1Ne?Q3~a^T;*pRlKo zh|)I>a%paW4R1Pd{XBP&U%#-Riy3Z&O}H+>_4RTh@89dl^^oM@je*o2jub%O5ZIR@ zwT@p1c$y_mZ~6touNDO7|377fpFUyMf#!e!2!H?xfB*=900@8p2!H?xfWQ+?V2JHy z4)Rlr`W3BFbvvB@zxLYc*7-jx`~yS&;Q<0500JNY0w4eaAOHd&00JNY0wA!12)sf( z03tutaL8X8NZ$XSDivRA?f+B4KQY38k|#Vs00ck)1V8`;KmY_l00ck)1V8`;b{2uX zsXm5(USL_?^8!3m00ck)1V8`; zKmY_l00ck)1U5y0V>zDp{{A0JHvicFZ;Ba&KmY_l00ck)1V8`;KmY_l00cl_rxEDs zA@=}q{=d`IhWdg42!H?xfB*=900@8p2!H?xfWVdr;QW6}xF8GyAOHd&00JNY0w4ea zAOHd&00KLbK;-=YOGfx|N45c#1_2NN0T2KI5C8!X009sH0T2KI5LhR0fXy=e%jB|u zAK%v}cs~b7e)&%|%td{r6x{#+oDn_;E)W0#5C8!X009sH0T2KI5C8!X0D&DqV1!LG z$9P`RuDIR~-z58dSM_%L@`sn@D-E;a8kTukJ|kV0gUHMBO*z>A|AG;Iu>)F&DuMtA zfB*=900@8p2!H?xfB*=900?YEKqPg8DT*9 zIDawUoBh$QtNhP1pYb2{emnCHca{B=2;lKJ6IeT(%ZLXLvUdkuZK0~Wwz0HCTtC4} zer!e=ol~T_(YGcQNnQ_?r9nwH7G=pWU7ZL^<>@)8JU=-pU6`4e8lAZ$omVbNqw{mq z6J?UYlv17>l4S3XNXpW6&8{qKtt2GEFPB(UJU!3L7RK57^V(fw>*%^|(2IXS7pgw<&Z@@5!su1OmFw|#J5J(u?9 z|J&m}tpHimaOh7NS~+xNcN*$E#gj#yQ`ahbrSif8Mj>YPI^tYc#LC zFk5=$Zbda)zqN67b-FhrmP+jH2_H;6yLd>N1{xQ5$WXlnsBf7`J;|H)x?&p5l|~xI zbVu|;Pg*>Bl)dEhT%V~NovhZ(ir)6Imm=8~Ip~2MA}I^E6fPqkJI1b__VbL@FBZ^4 z8B4a`B^;;ibBb<7d$iw(>$)8*K-)J` z(Gix|h*qyx4SkUeZe+MJmdKhfTrx|AH&bbGcDU2j(O#}vY7gPahPKb!)aKGhn11$$ z3~y%sIX&DX_I}3xko&%HGWUM=So#Qw-1(1Nf3PnjiXyuj%!T@OvK{rGo}snnhxsbf z>sDnsaXgIbD7HTihsF&?N!hU)HW_T!=e4A5hHv~y`(Uy`u=i#}Rb}sv_~uw&zTp@Z ztxC2v;Y8|MDjA+#Zy!|&Hwl{-+)pFfbUGhfl5~dwuvNum0eaWaA!E`>sC;7EhYsoJ zu`@x4w5Ds{0bL_%M-fJCC4Z@4Kb;Y8^{SmZdqVxyL?cj3n-&$fN{N^I)=AJGwg~j*PH( zZu(ZSK9i9-D$F1uXfLh(1PNc`NqVNV*zS$&JV>{X3kl{Cv54zh)pJWd*zFh=NSj1+ z+K?mhUHBYBgxs6%_!D(3(-Abm`nKPDOxjOcz62n)_#f8nQtrfY)U-ML`j=l zgb~4q=7#*gcJST)M&9p0fAo$IC9-pGF5DA>1X@iA5)WBn5888gRA}JYqIz8AhE z)~_TQ!m?g@Pj&1{lKpJ+#=L!S4%S zDq`ToPyz$ioQg&|mAYhGje3RpbfC>_smSguwyKfy2UmUHFimo+xizc@MrN4gc)(in zJW5Fp9+|lRE#lj+86PB{57352mqlwSVI7rV5wptrnOf|;i7ez_-|vsjcLR>~s^&TO z9XEXQEKM&A?+#o-h3S5{e*8zq+He+Y;cZ2FRg7?NMG&@R9`$;D{M?1++Mz1l0N(U@ zUf3qOlrCp(;uUyuc9h=ze@?i}2!AJhm%QKs0w4eaAOHd&00JNY0w4eaAOHd&utfr= zxIX4Zeu`!HRy2FjFo}c2y>YyF{KRmnG+aD2S3G{^o;DHkmQ0T2KI z5C8!X009sH0T2KI5CDO70zExE&j;uKA2Y(o-~s|500JNY0w4eaAOHd&00JNY0wD01 z5f~>w-#0Z}>hSCS#nU6fulwsOdZpnSmZ|FIl40r}%8Lt{d`7NocD+$`bopkm|Nku` z{PrZfu2=^Yl8lplV00JNY0w4eaAOHd& z00JNY0w4eaUz0%EzW_kK|DXAs;n^C)pUnKM_g>~tdOqR)kgaih^WO^!Akla{ECOpU zzK{`*9b@k_Ty3GM*OzZNMn$Wtm1U#4XzOMyKpvY>M&}f1ZuG55MUvz3(qJT1U#Z)= z<9J{9FcxLWFkO8~x25v*oK&8loE(zmCEIG$J49ZXnV1@#xg?!eE=hx}qUcPcc@&1E zbJH`*#CUn*GzDo!Ij77hZPk5srIO_!y!%A`UXJT^KzHhNa^Lgp`=^$@Iw zw3ivSh%Tj~ITdYDcT_)qD+Ne}(Sfu$e~e`eb5UP$u2l_JSG9&~`JdI84%AXC=*88A z=QHBS2z&cITG1_-VVYLZp$^J!Nsx9(9;ujWrKs{o=jWy;$|SQXr92l^SS%TdykeNd zM(Q>(7Fk*_$d`oNsKH#MPN-v~6sIo>tF1 zml21D+1oGskgL{`-1Jbj2d-p6zX7U@tn3YDm;Q==gos@adx=Vu+yx5lek)H4;k3? z3Df7zRIBQ<8L?DiZyS_qTWV>hY=u0ZtlpNuHuR1iJo4P(p2>(uj(_N}?j4jP@tar(Nd<;UeJK_S>}|!jLVC$Ve>y;0A|2VCEmk#GCsq_# zePl%vEHn*MCG+r-w;UzQ3d^eMni*YJ7>;VyU881vpp)fHT!6MTx-1&s=JY?S1Z&{< zf>J81y}myq4h*n&1CI5kuERIa()7ac?!YBfxI=eqSwzyxEilJ~MLuDBU8{P_#0PQh zP?c@~Z+gKVvGVA)phGWM{ifvanA#W8%|%|=f@Z+sHeEHhSk3rt*V#i`^o31p&BC6Y zJXGaKh2Z@E3r6_jVQq4e4+ww&2!H?xfB*=900@8p2!H?xfWV_8aM(NHPw`6ewEW>^ z*>fUXmd{9+<=_~T9FLlp<(qP_|G&=&_a9x&P!b4$00@8p2!H?xfB*=900@8p2!OyN zBfzm7&-at!ShD%Y{{NA2M?oL}0w4eaAOHd&00JNY0w4eaAn+&&^z`uL`~RdwR`?f& z{KEqTKmY_l00ck)1V8`;KmY_l00cl_2NICUPyF-zxOe@(`SS;nuK?(at^5D@{{IeC zFRBazAOHd&00JNY0w4eaAOHd&00M*nzW)z1009sH0T2KI5C8!X009sH0T2KI5Mc2A ze;5G>fB*=900@8p2!H?xfB*=900=z!1n~X;Ctt^CAqao~2!H?xfB*=900@8p2!O!< L0|J}*{{R012`_Y& literal 0 HcmV?d00001 diff --git a/optd-persistent/src/lib.rs b/optd-persistent/src/lib.rs index 7cbf50f..8f8a1d6 100644 --- a/optd-persistent/src/lib.rs +++ b/optd-persistent/src/lib.rs @@ -13,6 +13,28 @@ mod migrator; pub mod cost_model; pub use cost_model::interface::CostModelStorageLayer; +pub const DATABASE_FILENAME: &str = "sqlite.db"; +pub const DATABASE_URL: &str = "sqlite:./sqlite.db?mode=rwc"; + +pub const TEST_DATABASE_FILENAME: &str = "init.db"; +lazy_static::lazy_static! { + pub static ref TEST_DATABASE_FILE: String = { + std::env::current_dir().unwrap() + .join("src") + .join("db") + .join(TEST_DATABASE_FILENAME) + .to_str() + .unwrap() + .to_owned() + }; + pub static ref TEST_DATABASE_URL: String = + get_sqlite_url(TEST_DATABASE_FILE.as_str()); +} + +fn get_sqlite_url(file: &str) -> String { + format!("sqlite:{}?mode=rwc", file) +} + pub type StorageResult = Result; #[derive(Debug)] @@ -56,9 +78,6 @@ impl BackendManager { } } -pub const DATABASE_URL: &str = "sqlite:./sqlite.db?mode=rwc"; -pub const DATABASE_FILE: &str = "./sqlite.db"; - pub async fn migrate(db: &DatabaseConnection) -> Result<(), DbErr> { Migrator::refresh(db).await } From fe891b0e8be19063d74e237eb4f14500a314ca7d Mon Sep 17 00:00:00 2001 From: Yuanxin Cao Date: Thu, 7 Nov 2024 22:00:36 -0500 Subject: [PATCH 18/28] add more initial data into stat-related tables --- optd-persistent/src/bin/init.rs | 140 +++++++++++++++++++++++++------- 1 file changed, 111 insertions(+), 29 deletions(-) diff --git a/optd-persistent/src/bin/init.rs b/optd-persistent/src/bin/init.rs index ec396c7..0abdbff 100644 --- a/optd-persistent/src/bin/init.rs +++ b/optd-persistent/src/bin/init.rs @@ -53,59 +53,151 @@ async fn init_all_tables() -> Result<(), sea_orm::error::DbErr> { .expect("Unable to insert table metadata"); // Inserting into attribute - let attribute = attribute::ActiveModel { + let attribute1 = attribute::ActiveModel { id: Set(1), table_id: Set(1), name: Set("user_id".to_owned()), compression_method: Set("N".to_owned()), - variant_tag: Set(1), + variant_tag: Set(1), // integer base_attribute_number: Set(1), is_not_null: Set(true), }; - attribute::Entity::insert(attribute) + let attribute2 = attribute::ActiveModel { + id: Set(2), + table_id: Set(1), + name: Set("username".to_owned()), + compression_method: Set("N".to_owned()), + variant_tag: Set(2), // varchar + base_attribute_number: Set(2), + is_not_null: Set(true), + }; + attribute::Entity::insert(attribute1) .exec(&db) .await .expect("Unable to insert attribute"); + attribute::Entity::insert(attribute2) + .exec(&db) + .await + .expect("Unable to insert attribute"); + + // Inserting into event + let event = event::ActiveModel { + epoch_id: Set(1), + source_variant: Set("execution_engine".to_owned()), + timestamp: Set(Utc::now()), + data: Set(json!({"dba": "parpulse"})), + }; + event::Entity::insert(event) + .exec(&db) + .await + .expect("Unable to insert event"); // Inserting into statistic - let statistic = statistic::ActiveModel { + + // Table statistic + let table_statistic = statistic::ActiveModel { id: Set(1), name: Set("row_count".to_owned()), table_id: Set(Some(1)), creation_time: Set(Utc::now()), number_of_attributes: Set(0), - variant_tag: Set(1), + variant_tag: Set(1), // row count description: Set("".to_owned()), }; - statistic::Entity::insert(statistic) + let table_versioned_statistic = versioned_statistic::ActiveModel { + id: Set(1), + epoch_id: Set(1), + statistic_id: Set(1), + statistic_value: Set(json!(0)), + }; + statistic::Entity::insert(table_statistic) .exec(&db) .await .expect("Unable to insert statistic"); - // Inserting into event - let event = event::ActiveModel { + versioned_statistic::Entity::insert(table_versioned_statistic) + .exec(&db) + .await + .expect("Unable to insert versioned statistic"); + + // Single-column attribute statistic + let single_column_attribute_statistic = statistic::ActiveModel { + id: Set(2), + name: Set("cardinality".to_owned()), + table_id: Set(Some(1)), + creation_time: Set(Utc::now()), + number_of_attributes: Set(1), + variant_tag: Set(2), // cardinality + description: Set("1".to_owned()), + }; + let single_column_attribute_versioned_statistic = versioned_statistic::ActiveModel { + id: Set(2), epoch_id: Set(1), - source_variant: Set("insert".to_owned()), - timestamp: Set(Utc::now()), - data: Set(json!(r#"{"user_id": 1}"#)), + statistic_id: Set(2), + statistic_value: Set(json!(0)), }; + statistic::Entity::insert(single_column_attribute_statistic) + .exec(&db) + .await + .expect("Unable to insert statistic"); + versioned_statistic::Entity::insert(single_column_attribute_versioned_statistic) + .exec(&db) + .await + .expect("Unable to insert versioned statistic"); - event::Entity::insert(event) + let single_column_statistic_to_attribute_junction = + statistic_to_attribute_junction::ActiveModel { + statistic_id: Set(2), // cardinality + attribute_id: Set(1), // user_id + }; + statistic_to_attribute_junction::Entity::insert(single_column_statistic_to_attribute_junction) .exec(&db) .await - .expect("Unable to insert event"); + .expect("Unable to insert statistic_to_attribute_junction"); - // Inserting into versioned_statistic - let versioned_statistic = versioned_statistic::ActiveModel { - id: Set(1), + // Multi-column attribute statistic + let multi_column_attribute_statistic = statistic::ActiveModel { + id: Set(3), + name: Set("cardinality".to_owned()), + table_id: Set(Some(1)), + creation_time: Set(Utc::now()), + number_of_attributes: Set(2), + variant_tag: Set(2), // cardinality + description: Set("1,2".to_owned()), + }; + let multi_column_attribute_versioned_statistic = versioned_statistic::ActiveModel { + id: Set(3), epoch_id: Set(1), - statistic_id: Set(1), - statistic_value: Set(json!(r#"{"row_count": 0}"#)), + statistic_id: Set(3), + statistic_value: Set(json!(0)), }; - versioned_statistic::Entity::insert(versioned_statistic) + statistic::Entity::insert(multi_column_attribute_statistic) + .exec(&db) + .await + .expect("Unable to insert statistic"); + versioned_statistic::Entity::insert(multi_column_attribute_versioned_statistic) .exec(&db) .await .expect("Unable to insert versioned statistic"); + let multi_column_statistic_to_attribute_junction1 = + statistic_to_attribute_junction::ActiveModel { + statistic_id: Set(3), // joint cardinality + attribute_id: Set(1), // user_id + }; + let multi_column_statistic_to_attribute_junction2 = + statistic_to_attribute_junction::ActiveModel { + statistic_id: Set(3), // joint cardinality + attribute_id: Set(2), // username + }; + statistic_to_attribute_junction::Entity::insert(multi_column_statistic_to_attribute_junction1) + .exec(&db) + .await + .expect("Unable to insert statistic_to_attribute_junction"); + statistic_to_attribute_junction::Entity::insert(multi_column_statistic_to_attribute_junction2) + .exec(&db) + .await + .expect("Unable to insert statistic_to_attribute_junction"); + // Inserting into index_metadata let index_metadata = index_metadata::ActiveModel { id: Set(1), @@ -174,16 +266,6 @@ async fn init_all_tables() -> Result<(), sea_orm::error::DbErr> { .await .expect("Unable to insert attribute_foreign_constraint_junction"); - // Inserting into statistic_to_attribute_junction - let statistic_to_attribute_junction = statistic_to_attribute_junction::ActiveModel { - statistic_id: Set(1), - attribute_id: Set(1), - }; - statistic_to_attribute_junction::Entity::insert(statistic_to_attribute_junction) - .exec(&db) - .await - .expect("Unable to insert statistic_to_attribute_junction"); - // Inserting into cascades_group let cascades_group = cascades_group::ActiveModel { id: Set(1), From 295697141bd0e3d05fd2041e0c1a24041d5659c6 Mon Sep 17 00:00:00 2001 From: Yuanxin Cao Date: Thu, 7 Nov 2024 22:07:07 -0500 Subject: [PATCH 19/28] add new line at eof --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index df275d4..db6bb3a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,4 @@ target/ .DS_Store -optd-persistent/sql/ \ No newline at end of file +optd-persistent/sql/ From 6d70ad04be80c6963db7dc0a236cee06bedd75e8 Mon Sep 17 00:00:00 2001 From: Yuanxin Cao Date: Thu, 7 Nov 2024 22:29:23 -0500 Subject: [PATCH 20/28] refine variant tag in init --- optd-persistent/src/bin/init.rs | 26 ++++++++++-------- .../src/cost_model/catalog/mock_catalog.rs | 10 +++---- optd-persistent/src/cost_model/catalog/mod.rs | 12 -------- optd-persistent/src/cost_model/interface.rs | 21 +++++++++++++- optd-persistent/src/cost_model/orm.rs | 12 ++++---- optd-persistent/src/db/init.db | Bin 147456 -> 147456 bytes 6 files changed, 45 insertions(+), 36 deletions(-) diff --git a/optd-persistent/src/bin/init.rs b/optd-persistent/src/bin/init.rs index 0abdbff..7b0ceca 100644 --- a/optd-persistent/src/bin/init.rs +++ b/optd-persistent/src/bin/init.rs @@ -1,3 +1,7 @@ +use optd_persistent::cost_model::interface::AttrType; +use optd_persistent::cost_model::interface::ConstraintType; +use optd_persistent::cost_model::interface::IndexType; +use optd_persistent::cost_model::interface::StatType; use optd_persistent::entities::*; use optd_persistent::migrate; use optd_persistent::TEST_DATABASE_FILENAME; @@ -58,7 +62,7 @@ async fn init_all_tables() -> Result<(), sea_orm::error::DbErr> { table_id: Set(1), name: Set("user_id".to_owned()), compression_method: Set("N".to_owned()), - variant_tag: Set(1), // integer + variant_tag: Set(AttrType::Integer as i32), base_attribute_number: Set(1), is_not_null: Set(true), }; @@ -67,7 +71,7 @@ async fn init_all_tables() -> Result<(), sea_orm::error::DbErr> { table_id: Set(1), name: Set("username".to_owned()), compression_method: Set("N".to_owned()), - variant_tag: Set(2), // varchar + variant_tag: Set(AttrType::Varchar as i32), base_attribute_number: Set(2), is_not_null: Set(true), }; @@ -101,7 +105,7 @@ async fn init_all_tables() -> Result<(), sea_orm::error::DbErr> { table_id: Set(Some(1)), creation_time: Set(Utc::now()), number_of_attributes: Set(0), - variant_tag: Set(1), // row count + variant_tag: Set(StatType::Count as i32), description: Set("".to_owned()), }; let table_versioned_statistic = versioned_statistic::ActiveModel { @@ -126,7 +130,7 @@ async fn init_all_tables() -> Result<(), sea_orm::error::DbErr> { table_id: Set(Some(1)), creation_time: Set(Utc::now()), number_of_attributes: Set(1), - variant_tag: Set(2), // cardinality + variant_tag: Set(StatType::Cardinality as i32), description: Set("1".to_owned()), }; let single_column_attribute_versioned_statistic = versioned_statistic::ActiveModel { @@ -161,7 +165,7 @@ async fn init_all_tables() -> Result<(), sea_orm::error::DbErr> { table_id: Set(Some(1)), creation_time: Set(Utc::now()), number_of_attributes: Set(2), - variant_tag: Set(2), // cardinality + variant_tag: Set(StatType::Cardinality as i32), description: Set("1,2".to_owned()), }; let multi_column_attribute_versioned_statistic = versioned_statistic::ActiveModel { @@ -208,7 +212,7 @@ async fn init_all_tables() -> Result<(), sea_orm::error::DbErr> { is_primary: Set(true), is_clustered: Set(false), is_exclusion: Set(false), - variant_tag: Set(1), + variant_tag: Set(IndexType::Hash as i32), number_of_attributes: Set(1), description: Set("1".to_owned()), }; @@ -234,7 +238,7 @@ async fn init_all_tables() -> Result<(), sea_orm::error::DbErr> { let constraint_metadata = constraint_metadata::ActiveModel { id: Set(1), name: Set("pk_user_id".to_owned()), - variant_tag: Set(1), + variant_tag: Set(ConstraintType::PrimaryKey as i32), table_id: Set(Some(1)), index_id: Set(Some(1)), foreign_ref_id: Set(None), @@ -283,7 +287,7 @@ async fn init_all_tables() -> Result<(), sea_orm::error::DbErr> { id: Set(1), group_id: Set(1), fingerprint: Set(12345), - variant_tag: Set(1), + variant_tag: Set(0), data: Set(json!(r#"{"expr": "index_scan"}"#)), }; logical_expression::Entity::insert(logical_expression) @@ -296,7 +300,7 @@ async fn init_all_tables() -> Result<(), sea_orm::error::DbErr> { id: Set(1), group_id: Set(1), fingerprint: Set(12345), - variant_tag: Set(1), + variant_tag: Set(0), data: Set(json!(r#"{"expr": "index_scan"}"#)), }; physical_expression::Entity::insert(physical_expression) @@ -308,7 +312,7 @@ async fn init_all_tables() -> Result<(), sea_orm::error::DbErr> { let physical_property = physical_property::ActiveModel { id: Set(1), physical_expression_id: Set(1), - variant_tag: Set(1), + variant_tag: Set(0), data: Set(json!(r#"{"property": "indexed"}"#)), }; physical_property::Entity::insert(physical_property) @@ -320,7 +324,7 @@ async fn init_all_tables() -> Result<(), sea_orm::error::DbErr> { let logical_property = logical_property::ActiveModel { id: Set(1), group_id: Set(1), - variant_tag: Set(1), + variant_tag: Set(0), data: Set(json!(r#"{"property": "indexed"}"#)), }; logical_property::Entity::insert(logical_property) diff --git a/optd-persistent/src/cost_model/catalog/mock_catalog.rs b/optd-persistent/src/cost_model/catalog/mock_catalog.rs index 1ca4f96..7d9b8aa 100644 --- a/optd-persistent/src/cost_model/catalog/mock_catalog.rs +++ b/optd-persistent/src/cost_model/catalog/mock_catalog.rs @@ -1,6 +1,4 @@ -use crate::cost_model::interface::StatisticType; - -use super::{AttrType, IndexType}; +use crate::cost_model::interface::{AttrType, IndexType, StatType}; pub struct MockDatabaseMetadata { pub id: i32, @@ -111,7 +109,7 @@ impl MockCatalog { let statistics: Vec = vec![ MockStatistic { id: 1, - stat_type: StatisticType::Count as i32, + stat_type: StatType::Count as i32, stat_value: "100".to_string(), attr_ids: vec![1], table_id: None, @@ -119,7 +117,7 @@ impl MockCatalog { }, MockStatistic { id: 2, - stat_type: StatisticType::Count as i32, + stat_type: StatType::Count as i32, stat_value: "200".to_string(), attr_ids: vec![2], table_id: None, @@ -127,7 +125,7 @@ impl MockCatalog { }, MockStatistic { id: 3, - stat_type: StatisticType::Count as i32, + stat_type: StatType::Count as i32, stat_value: "300".to_string(), attr_ids: vec![], table_id: Some(1), diff --git a/optd-persistent/src/cost_model/catalog/mod.rs b/optd-persistent/src/cost_model/catalog/mod.rs index 60535d8..4df69d8 100644 --- a/optd-persistent/src/cost_model/catalog/mod.rs +++ b/optd-persistent/src/cost_model/catalog/mod.rs @@ -1,13 +1 @@ pub mod mock_catalog; - -pub enum IndexType { - BTree, - Hash, -} - -pub enum AttrType { - Integer, - Float, - Varchar, - Boolean, -} diff --git a/optd-persistent/src/cost_model/interface.rs b/optd-persistent/src/cost_model/interface.rs index aec2ea7..e5cff09 100644 --- a/optd-persistent/src/cost_model/interface.rs +++ b/optd-persistent/src/cost_model/interface.rs @@ -16,7 +16,26 @@ pub enum CatalogSource { Mock, } -pub enum StatisticType { +pub enum AttrType { + Integer, + Float, + Varchar, + Boolean, +} + +pub enum IndexType { + BTree, + Hash, +} + +pub enum ConstraintType { + PrimaryKey, + ForeignKey, + Unique, + Check, +} + +pub enum StatType { Count, Cardinality, Min, diff --git a/optd-persistent/src/cost_model/orm.rs b/optd-persistent/src/cost_model/orm.rs index b7bfb11..f3ebe5c 100644 --- a/optd-persistent/src/cost_model/orm.rs +++ b/optd-persistent/src/cost_model/orm.rs @@ -455,7 +455,7 @@ impl CostModelStorageLayer for BackendManager { #[cfg(test)] mod tests { - use crate::cost_model::interface::StatisticType; + use crate::cost_model::interface::StatType; use crate::{cost_model::interface::Stat, migrate, CostModelStorageLayer}; use crate::{get_sqlite_url, TEST_DATABASE_FILE}; use sea_orm::sqlx::database; @@ -464,7 +464,7 @@ mod tests { ColumnTrait, ConnectionTrait, Database, DbBackend, EntityTrait, ModelTrait, QueryFilter, QuerySelect, QueryTrait, }; - use serde_json::de; + use serde_json::{de, json}; use crate::entities::{prelude::*, *}; @@ -559,7 +559,7 @@ mod tests { .await .unwrap(); let stat = Stat { - stat_type: StatisticType::Count as i32, + stat_type: StatType::Count as i32, stat_value: "100".to_string(), attr_ids: vec![1], table_id: None, @@ -576,7 +576,7 @@ mod tests { println!("{:?}", stat_res); assert_eq!(stat_res[0].number_of_attributes, 1); assert_eq!(stat_res[0].description, "1".to_string()); - assert_eq!(stat_res[0].variant_tag, StatisticType::Count as i32); + assert_eq!(stat_res[0].variant_tag, StatType::Count as i32); let stat_attr_res = StatisticToAttributeJunction::find() .filter(statistic_to_attribute_junction::Column::StatisticId.eq(stat_res[0].id)) .all(&backend_manager.db) @@ -632,7 +632,7 @@ mod tests { .await .unwrap(); let stat2 = Stat { - stat_type: StatisticType::Count as i32, + stat_type: StatType::Count as i32, stat_value: "200".to_string(), attr_ids: vec![1], table_id: None, @@ -649,7 +649,7 @@ mod tests { assert_eq!(stat_res.len(), 1); assert_eq!(stat_res[0].number_of_attributes, 1); assert_eq!(stat_res[0].description, "1".to_string()); - assert_eq!(stat_res[0].variant_tag, StatisticType::Count as i32); + assert_eq!(stat_res[0].variant_tag, StatType::Count as i32); // 2.4 Check statistic_to_attribute_junction table let stat_attr_res = StatisticToAttributeJunction::find() .filter(statistic_to_attribute_junction::Column::StatisticId.eq(stat_res[0].id)) diff --git a/optd-persistent/src/db/init.db b/optd-persistent/src/db/init.db index 303510371fe8830ad446329575f08f45b25698b3..5340a787a981c89f635c4caba5487ff153c81881 100644 GIT binary patch delta 1275 zcmchXU2GIp6vyZ6eBS+-+3g40c0Xo!x@n>9wllkJcefQ062yd%0&8Q?+J)GSRA{%X zMXg``%06m}mt-Y8)DRQ(!3P-hh7bWI8jT{pAbunmjXs$8gb5EN0nhAO#KdQB@|&AE z=YQ_Gcjnx)us{|T$VyuSEO8u{K-R~pV?8MsEFIh5+<4woCO^rak(b5SQxWyr{Akld%LR>z-LeKh6_CQiue_Pmp&;U0%Ct5p9OH!SIIsPuCF*b zmi>zT19<~Lwx>Lg07&+S-Vxvzw~O>~^;0UUd!;QdDVR^v19A(!cDJ4C+Usr%47A|G zub%M_dtH@0-5lN|Rr1}<{3zWT4$&Wb+|tv@N`7M#{Wu)<&Xh|t#q#W&W1<(s8|6yA ztAWo8)SqmnBS|-L9EkwGL!jfyh)|OaM>>Q`KCaO{(Iz?<@pM%35tYvgQztR^+1#MLR=BC%r^}|YIeV>iTLZr}6)KJR}Q$LjxI9B&?n-(W*EtHI~|a=D3} z2KW=CW;wVh7MvQYe!|0J(7@URcpt<@DR^XO&HSqfJ0ZYK3C-YVH)Oz~)YuUMmt3wE znO!^&4+8|*i$3@W>IuG9#yRbZR@KJsfVo?9XLyBcC%b?&R^QSg7{B-J`>o8N*=?Fb zrs=YJQvI%)j*#lA4lUdMk(vGJTwP{apl~|br6|5~>G{G_rK8icPTeIfD`{pQHSfiw zbAzb?wx+{3h5PLW8X430MN>^=imxoKmQ5fm&W7l)CG7j0LnW*Uj-8s_RC$kgEhnO zwJSE?j@Z#$GuAu6m@yO^GbX1eiZ2u=#*9~tzW`}~HeUb$ delta 1131 zcmcgrTWDNG7(U;*?(FUC=HjNw?w-wQNH59eoU<2pH?g)=Vucbzt)UdxZi`uLdda41 zqDEbtm}en%#I5*H6+wOQp*@vB#I#xzMQeTW0z$!}4}wn>N=c#hoU<-g@Y#WxVZQ(S z=Kp8ro24bPv_w{WJGcps;|84GZat@lGdwqO>R@-L9XLXMRK6fDNG~go2}f{Ay=m`^ zj6qx(2r11g0UV?|LP>j^-vk8hu=!;~(uo6-1YvdI` zwif~q0g}BIoCN(o{D`nTF3?E$b9(sBX?Z&@$M&Y_A*F|otNnhbs%Xp^BB67Uu^`_t z=(hM4xnT@-3sZDwypR4o?3bTRH;m0)^wW4eI9IRDRqOMME<`WHH!BTeYlpB`q>=On zI+^wpH!u(tc8T;rIw5+rvBaR*F#1)xC)q_86M?~okq8SV@p#6}%J5ayq)pI*ZTHBH4y$7&(X2h6c6;!z5_TiPOxb~>;vGnu9Gfz{vsyMbA;I?BmG!0QlRI*?*-ui?HZ$h*H>feFH{-N9|gHr`SryY%PFdi&>< T_0d#WpP8MmK3knG>ka)csD~@} From a82d6c43710bacc5190ab9c048abdeffecc21c7e Mon Sep 17 00:00:00 2001 From: Yuanxin Cao Date: Thu, 7 Nov 2024 22:40:11 -0500 Subject: [PATCH 21/28] use json for stat type --- optd-persistent/src/bin/init.rs | 2 +- optd-persistent/src/cost_model/interface.rs | 3 +-- optd-persistent/src/cost_model/orm.rs | 6 +++--- optd-persistent/src/db/init.db | Bin 147456 -> 147456 bytes 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/optd-persistent/src/bin/init.rs b/optd-persistent/src/bin/init.rs index 7b0ceca..e282f98 100644 --- a/optd-persistent/src/bin/init.rs +++ b/optd-persistent/src/bin/init.rs @@ -161,7 +161,7 @@ async fn init_all_tables() -> Result<(), sea_orm::error::DbErr> { // Multi-column attribute statistic let multi_column_attribute_statistic = statistic::ActiveModel { id: Set(3), - name: Set("cardinality".to_owned()), + name: Set("joint_cardinality".to_owned()), table_id: Set(Some(1)), creation_time: Set(Utc::now()), number_of_attributes: Set(2), diff --git a/optd-persistent/src/cost_model/interface.rs b/optd-persistent/src/cost_model/interface.rs index e5cff09..1815962 100644 --- a/optd-persistent/src/cost_model/interface.rs +++ b/optd-persistent/src/cost_model/interface.rs @@ -44,8 +44,7 @@ pub enum StatType { pub struct Stat { pub stat_type: i32, - // TODO(lanlou): what should I use for the value type? - pub stat_value: String, + pub stat_value: Json, pub attr_ids: Vec, pub table_id: Option, pub name: String, diff --git a/optd-persistent/src/cost_model/orm.rs b/optd-persistent/src/cost_model/orm.rs index f3ebe5c..df74600 100644 --- a/optd-persistent/src/cost_model/orm.rs +++ b/optd-persistent/src/cost_model/orm.rs @@ -273,7 +273,7 @@ impl CostModelStorageLayer for BackendManager { let new_stats = versioned_statistic::ActiveModel { epoch_id: sea_orm::ActiveValue::Set(epoch_id), statistic_id: sea_orm::ActiveValue::Set(stat_id), - statistic_value: sea_orm::ActiveValue::Set(sea_orm::JsonValue::String(stat.stat_value)), + statistic_value: sea_orm::ActiveValue::Set(stat.stat_value), ..Default::default() }; let _ = VersionedStatistic::insert(new_stats).exec(&self.db).await; @@ -560,7 +560,7 @@ mod tests { .unwrap(); let stat = Stat { stat_type: StatType::Count as i32, - stat_value: "100".to_string(), + stat_value: json!(100), attr_ids: vec![1], table_id: None, name: "CountAttr1".to_string(), @@ -633,7 +633,7 @@ mod tests { .unwrap(); let stat2 = Stat { stat_type: StatType::Count as i32, - stat_value: "200".to_string(), + stat_value: json!(200), attr_ids: vec![1], table_id: None, name: "CountAttr1".to_string(), diff --git a/optd-persistent/src/db/init.db b/optd-persistent/src/db/init.db index 5340a787a981c89f635c4caba5487ff153c81881..1fc7eb9a38c4ba11b8be7fac114e62aecf285964 100644 GIT binary patch delta 484 zcmZo@;B08%oFFB{RK~!-paR4&@Nc4y5ff9{#)R9_OzpoX+sbINwg1)@We%O}C?h@j zwhk9tx^DY#4Ux$QWn_2?it-Cmi%Kd%LX-E)Xt1~c))Hb4W|=IYD>>O;mx~i5p(4oa z&N4YpSC$2=%vx5BrTw=e|Kvb9k;#>^+-mK=W%-ybS#pgGj7$s-j4a~~K)^6QwJbHS z1f<1@m!**rWc~Eh%!~>UQ(nrNgPf-%rwyX~rvgzjUrrt*cR)@J zC}+#YT*)%oL0@{Zg1$7^S*EPanII88u!j^`!0!1c#|hEgD6a<6yi#5Zs9BSdIgn-Y zJ$)sx+$M#0@;t_tR>qckCdS4_hQ^zJ*?&P2F*7pU{44$oKU|H8fu;H8U->)>I3gJM z&+<>_kJv0|;KeUs%gn)`&*{s^DV&v`nO71&eW5&~BwUxdsfp$EUU^1cxQLmliShJf f@{AgA5mR$>lkLCc885{ngp7?Xw%@2{)HnbDTb`08 delta 478 zcmZo@;B08%oFFB{*u=oVpaR4&uxO%=5ffw6#)R9_Os&C_ZDlmsT7$JknL{T#%1BSX zt;5BZuG<={Au{=(j0{gfQGP*cQAs68X!3p;4ffVxEg|M$mdOITl9T;)xi~=*DuT@J zER*AOWm&+=tYy_$T7wn&CkM)jOs0J!qRN>ulO(ga5WZ27M7cT Date: Thu, 7 Nov 2024 22:42:42 -0500 Subject: [PATCH 22/28] add test_get_stats_for_table --- optd-persistent/src/cost_model/orm.rs | 52 +++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/optd-persistent/src/cost_model/orm.rs b/optd-persistent/src/cost_model/orm.rs index df74600..f146a20 100644 --- a/optd-persistent/src/cost_model/orm.rs +++ b/optd-persistent/src/cost_model/orm.rs @@ -727,4 +727,56 @@ mod tests { assert_eq!(costs[1].physical_expression_id, physical_expression_id); assert_eq!(costs[1].cost, cost); } + + #[tokio::test] + async fn test_get_stats_for_table() { + const DATABASE_FILE: &str = "test_get_stats_for_table.db"; + let database_url = copy_init_db(DATABASE_FILE).await; + let mut binding = super::BackendManager::new(Some(&database_url)).await; + let backend_manager = binding.as_mut().unwrap(); + let epoch_id = 1; + let table_id = 1; + let stat_type = StatType::Count as i32; + + // Get initial stats + let res = backend_manager + .get_stats_for_table(table_id, stat_type, None) + .await + .unwrap() + .unwrap(); + let row_count = res.as_i64().unwrap(); + assert_eq!(row_count, 0); + + // Update stats + let epoch_id2 = backend_manager + .create_new_epoch("test".to_string(), "test_get_stats_for_table".to_string()) + .await + .unwrap(); + let stat = Stat { + stat_type: StatType::Count as i32, + stat_value: json!(100), + attr_ids: vec![], + table_id: Some(table_id), + name: "row_count".to_string(), + }; + backend_manager.update_stats(stat, epoch_id2).await.unwrap(); + + // Get updated stats + let res = backend_manager + .get_stats_for_table(table_id, stat_type, None) + .await + .unwrap() + .unwrap(); + let row_count = res.as_i64().unwrap(); + assert_eq!(row_count, 100); + + // Get stats for a specific epoch + let res = backend_manager + .get_stats_for_table(table_id, stat_type, Some(epoch_id)) + .await + .unwrap() + .unwrap(); + let row_count = res.as_i64().unwrap(); + assert_eq!(row_count, 0); + } } From 7310fd2fb0f1b953de37c210a05b50f9b72c3bb2 Mon Sep 17 00:00:00 2001 From: Yuanxin Cao Date: Thu, 7 Nov 2024 22:48:36 -0500 Subject: [PATCH 23/28] minor fixes --- optd-persistent/src/cost_model/orm.rs | 40 ++++++++++----------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/optd-persistent/src/cost_model/orm.rs b/optd-persistent/src/cost_model/orm.rs index f146a20..24e15d2 100644 --- a/optd-persistent/src/cost_model/orm.rs +++ b/optd-persistent/src/cost_model/orm.rs @@ -464,6 +464,7 @@ mod tests { ColumnTrait, ConnectionTrait, Database, DbBackend, EntityTrait, ModelTrait, QueryFilter, QuerySelect, QueryTrait, }; + use sea_orm_migration::schema::json; use serde_json::{de, json}; use crate::entities::{prelude::*, *}; @@ -555,7 +556,7 @@ mod tests { let backend_manager = binding.as_mut().unwrap(); // 1. Update non-existed stat let epoch_id1 = backend_manager - .create_new_epoch("test".to_string(), "InsertTest".to_string()) + .create_new_epoch("test".to_string(), "test_update_attr_stats".to_string()) .await .unwrap(); let stat = Stat { @@ -563,12 +564,12 @@ mod tests { stat_value: json!(100), attr_ids: vec![1], table_id: None, - name: "CountAttr1".to_string(), + name: "countattr1".to_string(), }; let res = backend_manager.update_stats(stat, epoch_id1).await; assert!(res.is_ok()); let stat_res = Statistic::find() - .filter(statistic::Column::Name.eq("CountAttr1")) + .filter(statistic::Column::Name.eq("countattr1")) .all(&backend_manager.db) .await .unwrap(); @@ -590,10 +591,7 @@ mod tests { .await .unwrap(); assert_eq!(versioned_stat_res.len(), 1); - assert_eq!( - versioned_stat_res[0].statistic_value, - serde_json::Value::String("100".to_string()) - ); + assert_eq!(versioned_stat_res[0].statistic_value, json!(100)); assert_eq!(versioned_stat_res[0].epoch_id, epoch_id1); // 2. Normal update @@ -628,7 +626,7 @@ mod tests { assert!(cost_res[0].is_valid); // 2.2 Normal update let epoch_id2 = backend_manager - .create_new_epoch("test".to_string(), "InsertTest".to_string()) + .create_new_epoch("test".to_string(), "test_update_attr_stats".to_string()) .await .unwrap(); let stat2 = Stat { @@ -636,13 +634,13 @@ mod tests { stat_value: json!(200), attr_ids: vec![1], table_id: None, - name: "CountAttr1".to_string(), + name: "countattr1".to_string(), }; let res = backend_manager.update_stats(stat2, epoch_id2).await; assert!(res.is_ok()); // 2.3 Check statistic table let stat_res = Statistic::find() - .filter(statistic::Column::Name.eq("CountAttr1")) + .filter(statistic::Column::Name.eq("countattr1")) .all(&backend_manager.db) .await .unwrap(); @@ -665,16 +663,10 @@ mod tests { .await .unwrap(); assert_eq!(versioned_stat_res.len(), 2); - assert_eq!( - versioned_stat_res[0].statistic_value, - serde_json::Value::String("100".to_string()) - ); + assert_eq!(versioned_stat_res[0].statistic_value, json!(100)); assert_eq!(versioned_stat_res[0].epoch_id, epoch_id1); assert_eq!(versioned_stat_res[0].statistic_id, stat_res[0].id); - assert_eq!( - versioned_stat_res[1].statistic_value, - serde_json::Value::String("200".to_string()) - ); + assert_eq!(versioned_stat_res[1].statistic_value, json!(200)); assert_eq!(versioned_stat_res[1].epoch_id, epoch_id2); assert_eq!(versioned_stat_res[1].statistic_id, stat_res[0].id); assert!(epoch_id1 < epoch_id2); @@ -710,14 +702,8 @@ mod tests { let cost = 42; let res = backend_manager .store_cost(physical_expression_id, cost, epoch_id) - .await; - match res { - Ok(_) => {} - Err(e) => { - println!("Error: {:?}", e); - panic!(); - } - } + .await + .unwrap(); let costs = super::PlanCost::find() .all(&backend_manager.db) .await @@ -778,5 +764,7 @@ mod tests { .unwrap(); let row_count = res.as_i64().unwrap(); assert_eq!(row_count, 0); + + remove_db_file(DATABASE_FILE); } } From 3c222824053e4bf788deeafe0e69a1d8b255bc33 Mon Sep 17 00:00:00 2001 From: Yuanxin Cao Date: Thu, 7 Nov 2024 22:53:17 -0500 Subject: [PATCH 24/28] add test_get_stats_for_single_attr and test_get_stats_for_multiple_attrs --- optd-persistent/src/cost_model/orm.rs | 114 ++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/optd-persistent/src/cost_model/orm.rs b/optd-persistent/src/cost_model/orm.rs index 24e15d2..a51340d 100644 --- a/optd-persistent/src/cost_model/orm.rs +++ b/optd-persistent/src/cost_model/orm.rs @@ -767,4 +767,118 @@ mod tests { remove_db_file(DATABASE_FILE); } + + #[tokio::test] + async fn test_get_stats_for_single_attr() { + const DATABASE_FILE: &str = "test_get_stats_for_single_attr.db"; + let database_url = copy_init_db(DATABASE_FILE).await; + let mut binding = super::BackendManager::new(Some(&database_url)).await; + let backend_manager = binding.as_mut().unwrap(); + let epoch_id = 1; + let attr_ids = vec![1]; + let stat_type = StatType::Cardinality as i32; + + // Get initial stats + let res = backend_manager + .get_stats_for_attr(attr_ids.clone(), stat_type, None) + .await + .unwrap() + .unwrap(); + let cardinality = res.as_i64().unwrap(); + assert_eq!(cardinality, 0); + + // Update stats + let epoch_id2 = backend_manager + .create_new_epoch( + "test".to_string(), + "test_get_stats_for_single_attr".to_string(), + ) + .await + .unwrap(); + let stat = Stat { + stat_type: StatType::Cardinality as i32, + stat_value: json!(100), + attr_ids: attr_ids.clone(), + table_id: None, + name: "cardinality".to_string(), + }; + backend_manager.update_stats(stat, epoch_id2).await.unwrap(); + + // Get updated stats + let res = backend_manager + .get_stats_for_attr(attr_ids.clone(), stat_type, None) + .await + .unwrap() + .unwrap(); + let cardinality = res.as_i64().unwrap(); + assert_eq!(cardinality, 100); + + // Get stats for a specific epoch + let res = backend_manager + .get_stats_for_attr(attr_ids.clone(), stat_type, Some(epoch_id)) + .await + .unwrap() + .unwrap(); + let cardinality = res.as_i64().unwrap(); + assert_eq!(cardinality, 0); + + remove_db_file(DATABASE_FILE); + } + + #[tokio::test] + async fn test_get_stats_for_multiple_attrs() { + const DATABASE_FILE: &str = "test_get_stats_for_multiple_attrs.db"; + let database_url = copy_init_db(DATABASE_FILE).await; + let mut binding = super::BackendManager::new(Some(&database_url)).await; + let backend_manager = binding.as_mut().unwrap(); + let epoch_id = 1; + let attr_ids = vec![2, 1]; + let stat_type = StatType::Cardinality as i32; + + // Get initial stats + let res = backend_manager + .get_stats_for_attr(attr_ids.clone(), stat_type, None) + .await + .unwrap() + .unwrap(); + let cardinality = res.as_i64().unwrap(); + assert_eq!(cardinality, 0); + + // Update stats + let epoch_id2 = backend_manager + .create_new_epoch( + "test".to_string(), + "test_get_stats_for_multiple_attrs".to_string(), + ) + .await + .unwrap(); + let stat = Stat { + stat_type: StatType::Cardinality as i32, + stat_value: json!(111), + attr_ids: attr_ids.clone(), + table_id: None, + name: "cardinality".to_string(), + }; + backend_manager.update_stats(stat, epoch_id2).await.unwrap(); + + // Get updated stats + let res = backend_manager + .get_stats_for_attr(attr_ids.clone(), stat_type, None) + .await + .unwrap() + .unwrap(); + let cardinality = res.as_i64().unwrap(); + assert_eq!(cardinality, 111); + + // Get stats for a specific epoch + let res = backend_manager + .get_stats_for_attr(attr_ids.clone(), stat_type, Some(epoch_id)) + .await + .unwrap() + .unwrap(); + let cardinality = res.as_i64().unwrap(); + assert_eq!(cardinality, 0); + + remove_db_file(DATABASE_FILE); + } } From 5ad0c7a330f708fbf05566faafb91db7c5916572 Mon Sep 17 00:00:00 2001 From: Yuanxin Cao Date: Thu, 7 Nov 2024 22:55:43 -0500 Subject: [PATCH 25/28] remove unused comments --- optd-persistent/src/cost_model/interface.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/optd-persistent/src/cost_model/interface.rs b/optd-persistent/src/cost_model/interface.rs index 1815962..b4e27c7 100644 --- a/optd-persistent/src/cost_model/interface.rs +++ b/optd-persistent/src/cost_model/interface.rs @@ -72,7 +72,6 @@ pub trait CostModelStorageLayer { epoch_id: Self::EpochId, ) -> StorageResult<()>; - // i32 in `stats:i32` is a placeholder for the stats type async fn update_stats(&self, stat: Stat, epoch_id: Self::EpochId) -> StorageResult<()>; async fn store_cost( From 1d12502247fb25e47e8c12a0b7d36ee30c2d0ab9 Mon Sep 17 00:00:00 2001 From: unw9527 <1041593558@qq.com> Date: Fri, 8 Nov 2024 00:00:03 -0500 Subject: [PATCH 26/28] add: cost related tests --- optd-persistent/init.db | Bin 0 -> 147456 bytes optd-persistent/src/cost_model/orm.rs | 73 +++++++++++++++++++++++++- 2 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 optd-persistent/init.db diff --git a/optd-persistent/init.db b/optd-persistent/init.db new file mode 100644 index 0000000000000000000000000000000000000000..fe10513887383e8e2132b98465aa2dc75a4b69bb GIT binary patch literal 147456 zcmeI5PjKAUea8WT+$E^pg=ATRW!WaQmTayiT3k}9ExE2sQxMDHKbe23Q7i`oE}oy@p@a%q~Thg>?7IHOA+ z00N7*0G8xNluhZoBayrB{ocRN@Av-w-dk|tt!drSq^qV?R~>1T*~hRf^T(3JFw6z= z{}}lnyq+R&dV&w+Hye4M_Vxnv^81Zm61zwEjVtqva9{XM?gx9X@juIc%757Vt?b*} zHTDzXi?j?7zA1tAWBIIj>4cR!DP5SGmnSD?y;=>n ziYrO;@&$Qbo*kDLf?CuyM_o}Jl{C?9VM&^smCno4GGQ=2wlF?+UUoy4E}nO(Y=pEI z5U+AuNj~E36d{%FNV%+7DzV@Az{8~*E80yZs4e{|Pb-a%>`r$bRg|h&qgh|pEXBO4 zs18x!a?{c5xX}miyG>7BA#;Tc&zfE}l!|HCj-`@bt6baYFk2GZOB>Xd(s*{(l1%B{ks2jSL{#4=v@wXfI6JgcZp z$Mm~%SpQDSRmxiu%dL<0WyOgJ_AVLx$P~~@6lp}< z#O4N)z&>gi3DN!Q`?6VaV1T`A(gqP)Mqi4eJ*6h~oHrJaK z%VqY?qz5LRT|6X11C0wjV5sf_)U(W_p5#t@T``U3N+S(px+8kACnKIX!Cv-wZp>7+ zMpkP^MQi)mOOb4g9CX2+Bq>X`WiBh8I>oM?_416>FBZ^4h+pZ*Q=J-C2YIx^B%et?a^K%ZfKUj0Bv7jMn_m;BWj~j)3p^cxRK#X zuaY%ixMY?}Z|up43qxsBM|-(Sxh;h3Dff%y$G&`?{1kIfFv7nIe=pn?zAn62_@Ba$ z3g0bQg}Fj8|Ev7pLbG=r^~}5XO&_1RaP3Sw`{$l)`$n*Y>?BXTh*aI(n^OqidI(j+6p=RaYauY zgh+t|gD{b`tmD}1y+ zsjZaA2|g*mh!ZpLVYvAX^l`7pYrS!n=nA}$O;Q4VV&=-dp5boAGitTC!wHL<{7f&W zcS;_qew5wOo+KP$sa^xPqy~82hVXRIm3wJVlKsX#S*~n}P3o61pNcEmPsS`UXEK5* z{Gq_+KP>#E@Ol2@{MQQqki_sY7n9UGc%=++19+-JkGeN2#FD z121w5>3OxveBvlK5tobl5cWUMO~oZ>|G~cJxUslQ=sOUe=FY~WJ-{5e`~!W7fN9@A z_8fOBoe zcF&{0o#MlL5W)9Z4&}MgR%q)U$H5#|4zp+l@88Rnc>1o!p?;D>p)5(3a7Cgo!->3il_S?n+7#Z6$n}s<;gx~?9LbQrA#iXHsd4;5z>_R# zcHkEXzgpm*|Nn#$e)46h5aogZ2!H?xfB*=900@8p2!H?xfB*<=pTJ4BmpQ`EtY}x& zX3a@F|35uEeEMwb{GS#6g(3g&0s#;J0T2KI5C8!X009sH0T2KI5ZFZoULqa)-#s&j|NBBtsMkfB*=900@8p2!H?xfB*=900@AO+ylV*e@8}$1OX5L0T2KI5C8!X z009sH0T2Lz-A(}K|GQm-s6PmR00@8p2!H?xfB*=900@8p2y9Lua{m8wM)>*W2_OLi zAOHd&00JNY0w4eaAOHd&00JQJl_W60<{17(a?ihy@9R6f@z6g-H&(QpmTIh+b^reV zXN>UKSF--76bOI-2!H?xfB*=900@8p2!H?xfB;Qkl+7@wcwSboI_|Um4f2G4M{&3O z#Sg9&uQrW}qnpN+;yLL`(T}`R{HW;f|9{B{zoaw6ClCMu5C8!X009sH0T2KI5C8!X z0D)acKqPL#=B? z>4s`mR@GJ#65*9gEGnKKiEOBrW;n`5$!Su7f2t^5vrR*Bw3|*?sgu%$xp{eVV%DqG zV5_*2G%sI}=jGXPc_FApU31hG)lo?k-4>RlxmoGFJS`Ij<6{fsW9MZzWa;90m&!&+ zdjat(x0U3x-A)lw>5i1kilq|!jSoCr-W?nzs4e{|Pb-a%>`r$bRg|h&qgh`jzC`n? zqB=x@%S}hK6O~MiLRVcatJTDK-By~0e!WTTXW2AsnrcL}TQr)rnynb7qpaw*qZ3bM zq8Q1fVd-_%x|J4PsWoll{M3jgCKN-`Yd2jH;s;F>Skdf?r8itZX_B4quoX(B^{M`> zc}pX1!r)wmTfV6XB{! zhVAyH#by)BvTBoF6--WwKVhX!LGBFW&NWGc|BestYZo%!{C{V{qvazD42S-drIkZR zcBi5ADV{9q?1oy=(pF{_i;O3(iX7ct{<%MGN3AzHuhHD{!fff0yA>6*etYBU+FWl| zESK3klOC9OcJYu54KyzBfT6kzP|q@xdXhWsb;UHAD~&Xa>5k~do{V_n1bf-zxiM4O z8dJ+I7Noatx3$M8n>-!elzV(+Ky_qe|i z&g9?Aoyr_1k-PtO`%eyLMNwqe{JBuOLAImbkEdvDd10Q4w1!z(O&kxSI*RR&!=bUm zC|R`4rbPzZjd?9;o8cLM(mt4M5Uc}PQBl}?qn&dKmyH`o;>XA5Y0s2eGRgKqZzN-Gtdmqf9i+{~9b3Z%BgsByHdNuXv4m0EQDOV8lP;YykuYjI z`AemDp2&#Tr_wAWtaP$^$n4Nry$APx=1&&uC-!B<(NXs9N1jz|%w%Ma3NuIuIzVeb zLBi8`lAb9owtFMf2kG{4A;CN%7I8zZxo)Y=-Hu^_v`GZhMlllKh0ifW=*~NI31QUs zjADJC5ZwcYxrA??pfwYQcc%|`?Uy*6dA35&ro_`sl(e-)7!iD6Zpbg&!FT6t1+N2r z^KB1GWal0%+!KNXT1^QO4_INF?YUDG>U*}Rp6qKn@LyWm)kH&B)hh2QwpB^8pB>(q zw-3%?8|Xgh2M1c(PDgAnpp6AxjE}aPVx$sTsjuwxInbIc`c4elH(=GSsH9UVRm*HP zD%7W^+su}V?9O6qDmi~}l=pPQAh()Z!@6%|xQck zL*??&@GFZWBj-lWoI729dGz$i$ne<{!^7u>hZo6FcugYWw!i=X1ta|85hjGDfdB}A z00@8p2!H?xfB*=900@A<<3Qjr+sp9$8TUDWp^LzM5a4j|{J;B*aQ|@tfi{8w2!H?x zfB*=900@8p2!H?xfWTu%fMYqH?$>tyX|Hn@MXfOzX00@8p2!H?xfB*=900@A< z<3XUOhx|bR|NcMr|Br{t(Mk{i0T2KI5C8!X009sH0T2Lz$B+Qd{~tr8qsbru0w4ea zAOHd&00JNY0w4eaj~#)?`Tvg@;m40%`_W(!009sH0T2KI5C8!X009sH0T9?d1YRf4 z_M16ePI==0+0irp6aRI?)-1<;0$`K8A%FbQeg0oLc>dpS8R56Pr^%=(2!H?xfB*=9 z00@8p2!H?xfB*=9z?KMP*$nxM04(3{fB*k?jPScHrGpR%fB*=900@8p2!H?xfB*=9 z00@A!3m00ck)1V8`;KmY_l00ck)1hzsT<6QtC z&;QSU#_(*N;m>4$+WSTJ&wGBz{V7}L4ivuY=R%_KdQb$`pMNeZo;t3;) zvh|8uQ!1-^ZN<`zSU_=nULISNrNyx~r)8-akCz4`q1w%crP;P_8sz&OMM*att*Tkl z?A)R>yEHw0QYuz0v)M?Ayf{BOGd6!&nvyR|gRP?IOrv>}PD&T%=HEd}8!A3}XnPH3QQYxxlQCBou z@#42rfK(bA$cRg)SVlKiw43(zn(k# z(+WD8qU@FgX@}&Jim6tLDsOCQac**!WHuwuE=Cm=OGYBE>ISiqhDD5}C@t$%@;hoU zH&jbk4M%a*YEiQ5YOU5Lu%bF@QMzUmn{%|A(X_N}?0}xu&OMVAhlbcYFM5z`X4UIV z4a;nhuR7go|18mX+oN}Y?N=Gzy706I@9tX^ykH~>=mejiONJLjd+>rZEqhHFX|^#8 zCFdD6>ewihVrlW5wlgX`gkk56BN=gFC~epYsvi(n%IzTodwFl~XOkXb-{RiW4MVe{?~l+_qftA+Ghs$a z>wgu~cGBXshFMuniQM5Ky}gsAw1}SR6NG-Bjvb(BE2O3xiW~L&WF3;F^VV)??%X?A zjKptYAtaR;X3u|#r4oBb_Nm->+fr1ITIJ4EsZXV#^+}j zBhYpD<}gh!4DSwHLWNVhTgxJnR&KsICM@y^+Z$@lT_$eEHKi)u0N(O~J!0k2Z9z&e zSo^xpZ(!nAxY(WrktW8(J7AuI~ah*M|MPJyo)-3GW$pcl6RPxXNKWBu`AJirn z`G5cjfB*=900@8p2!H?xfB*=900=xh0>|7F{yn@reAa#HpX)?$ANhBs=pSQ}<5A;E zF?jypeMY$d@M?yVKmY_l00ck)1V8`;KmY_l00ck)1Rfdzj^%j1pA^TE%|G`44~;tt z0s#;J0T2KI5C8!X009sH0T2Lzhe@EPhbQ0vCnd7Nzcb_?ULXJhAOHd&00JNY0w4ea zAOHd&00O&^K#@G+pXVps>;J*y4 Date: Fri, 8 Nov 2024 12:02:12 -0500 Subject: [PATCH 27/28] Fix update_stats and add all tests --- optd-persistent/init.db | Bin 147456 -> 0 bytes optd-persistent/src/cost_model/interface.rs | 14 +- optd-persistent/src/cost_model/orm.rs | 272 +++++++++++++++----- optd-persistent/src/db/init.db | Bin 147456 -> 147456 bytes 4 files changed, 221 insertions(+), 65 deletions(-) delete mode 100644 optd-persistent/init.db diff --git a/optd-persistent/init.db b/optd-persistent/init.db deleted file mode 100644 index fe10513887383e8e2132b98465aa2dc75a4b69bb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 147456 zcmeI5PjKAUea8WT+$E^pg=ATRW!WaQmTayiT3k}9ExE2sQxMDHKbe23Q7i`oE}oy@p@a%q~Thg>?7IHOA+ z00N7*0G8xNluhZoBayrB{ocRN@Av-w-dk|tt!drSq^qV?R~>1T*~hRf^T(3JFw6z= z{}}lnyq+R&dV&w+Hye4M_Vxnv^81Zm61zwEjVtqva9{XM?gx9X@juIc%757Vt?b*} zHTDzXi?j?7zA1tAWBIIj>4cR!DP5SGmnSD?y;=>n ziYrO;@&$Qbo*kDLf?CuyM_o}Jl{C?9VM&^smCno4GGQ=2wlF?+UUoy4E}nO(Y=pEI z5U+AuNj~E36d{%FNV%+7DzV@Az{8~*E80yZs4e{|Pb-a%>`r$bRg|h&qgh|pEXBO4 zs18x!a?{c5xX}miyG>7BA#;Tc&zfE}l!|HCj-`@bt6baYFk2GZOB>Xd(s*{(l1%B{ks2jSL{#4=v@wXfI6JgcZp z$Mm~%SpQDSRmxiu%dL<0WyOgJ_AVLx$P~~@6lp}< z#O4N)z&>gi3DN!Q`?6VaV1T`A(gqP)Mqi4eJ*6h~oHrJaK z%VqY?qz5LRT|6X11C0wjV5sf_)U(W_p5#t@T``U3N+S(px+8kACnKIX!Cv-wZp>7+ zMpkP^MQi)mOOb4g9CX2+Bq>X`WiBh8I>oM?_416>FBZ^4h+pZ*Q=J-C2YIx^B%et?a^K%ZfKUj0Bv7jMn_m;BWj~j)3p^cxRK#X zuaY%ixMY?}Z|up43qxsBM|-(Sxh;h3Dff%y$G&`?{1kIfFv7nIe=pn?zAn62_@Ba$ z3g0bQg}Fj8|Ev7pLbG=r^~}5XO&_1RaP3Sw`{$l)`$n*Y>?BXTh*aI(n^OqidI(j+6p=RaYauY zgh+t|gD{b`tmD}1y+ zsjZaA2|g*mh!ZpLVYvAX^l`7pYrS!n=nA}$O;Q4VV&=-dp5boAGitTC!wHL<{7f&W zcS;_qew5wOo+KP$sa^xPqy~82hVXRIm3wJVlKsX#S*~n}P3o61pNcEmPsS`UXEK5* z{Gq_+KP>#E@Ol2@{MQQqki_sY7n9UGc%=++19+-JkGeN2#FD z121w5>3OxveBvlK5tobl5cWUMO~oZ>|G~cJxUslQ=sOUe=FY~WJ-{5e`~!W7fN9@A z_8fOBoe zcF&{0o#MlL5W)9Z4&}MgR%q)U$H5#|4zp+l@88Rnc>1o!p?;D>p)5(3a7Cgo!->3il_S?n+7#Z6$n}s<;gx~?9LbQrA#iXHsd4;5z>_R# zcHkEXzgpm*|Nn#$e)46h5aogZ2!H?xfB*=900@8p2!H?xfB*<=pTJ4BmpQ`EtY}x& zX3a@F|35uEeEMwb{GS#6g(3g&0s#;J0T2KI5C8!X009sH0T2KI5ZFZoULqa)-#s&j|NBBtsMkfB*=900@8p2!H?xfB*=900@AO+ylV*e@8}$1OX5L0T2KI5C8!X z009sH0T2Lz-A(}K|GQm-s6PmR00@8p2!H?xfB*=900@8p2y9Lua{m8wM)>*W2_OLi zAOHd&00JNY0w4eaAOHd&00JQJl_W60<{17(a?ihy@9R6f@z6g-H&(QpmTIh+b^reV zXN>UKSF--76bOI-2!H?xfB*=900@8p2!H?xfB;Qkl+7@wcwSboI_|Um4f2G4M{&3O z#Sg9&uQrW}qnpN+;yLL`(T}`R{HW;f|9{B{zoaw6ClCMu5C8!X009sH0T2KI5C8!X z0D)acKqPL#=B? z>4s`mR@GJ#65*9gEGnKKiEOBrW;n`5$!Su7f2t^5vrR*Bw3|*?sgu%$xp{eVV%DqG zV5_*2G%sI}=jGXPc_FApU31hG)lo?k-4>RlxmoGFJS`Ij<6{fsW9MZzWa;90m&!&+ zdjat(x0U3x-A)lw>5i1kilq|!jSoCr-W?nzs4e{|Pb-a%>`r$bRg|h&qgh`jzC`n? zqB=x@%S}hK6O~MiLRVcatJTDK-By~0e!WTTXW2AsnrcL}TQr)rnynb7qpaw*qZ3bM zq8Q1fVd-_%x|J4PsWoll{M3jgCKN-`Yd2jH;s;F>Skdf?r8itZX_B4quoX(B^{M`> zc}pX1!r)wmTfV6XB{! zhVAyH#by)BvTBoF6--WwKVhX!LGBFW&NWGc|BestYZo%!{C{V{qvazD42S-drIkZR zcBi5ADV{9q?1oy=(pF{_i;O3(iX7ct{<%MGN3AzHuhHD{!fff0yA>6*etYBU+FWl| zESK3klOC9OcJYu54KyzBfT6kzP|q@xdXhWsb;UHAD~&Xa>5k~do{V_n1bf-zxiM4O z8dJ+I7Noatx3$M8n>-!elzV(+Ky_qe|i z&g9?Aoyr_1k-PtO`%eyLMNwqe{JBuOLAImbkEdvDd10Q4w1!z(O&kxSI*RR&!=bUm zC|R`4rbPzZjd?9;o8cLM(mt4M5Uc}PQBl}?qn&dKmyH`o;>XA5Y0s2eGRgKqZzN-Gtdmqf9i+{~9b3Z%BgsByHdNuXv4m0EQDOV8lP;YykuYjI z`AemDp2&#Tr_wAWtaP$^$n4Nry$APx=1&&uC-!B<(NXs9N1jz|%w%Ma3NuIuIzVeb zLBi8`lAb9owtFMf2kG{4A;CN%7I8zZxo)Y=-Hu^_v`GZhMlllKh0ifW=*~NI31QUs zjADJC5ZwcYxrA??pfwYQcc%|`?Uy*6dA35&ro_`sl(e-)7!iD6Zpbg&!FT6t1+N2r z^KB1GWal0%+!KNXT1^QO4_INF?YUDG>U*}Rp6qKn@LyWm)kH&B)hh2QwpB^8pB>(q zw-3%?8|Xgh2M1c(PDgAnpp6AxjE}aPVx$sTsjuwxInbIc`c4elH(=GSsH9UVRm*HP zD%7W^+su}V?9O6qDmi~}l=pPQAh()Z!@6%|xQck zL*??&@GFZWBj-lWoI729dGz$i$ne<{!^7u>hZo6FcugYWw!i=X1ta|85hjGDfdB}A z00@8p2!H?xfB*=900@A<<3Qjr+sp9$8TUDWp^LzM5a4j|{J;B*aQ|@tfi{8w2!H?x zfB*=900@8p2!H?xfWTu%fMYqH?$>tyX|Hn@MXfOzX00@8p2!H?xfB*=900@A< z<3XUOhx|bR|NcMr|Br{t(Mk{i0T2KI5C8!X009sH0T2Lz$B+Qd{~tr8qsbru0w4ea zAOHd&00JNY0w4eaj~#)?`Tvg@;m40%`_W(!009sH0T2KI5C8!X009sH0T9?d1YRf4 z_M16ePI==0+0irp6aRI?)-1<;0$`K8A%FbQeg0oLc>dpS8R56Pr^%=(2!H?xfB*=9 z00@8p2!H?xfB*=9z?KMP*$nxM04(3{fB*k?jPScHrGpR%fB*=900@8p2!H?xfB*=9 z00@A!3m00ck)1V8`;KmY_l00ck)1hzsT<6QtC z&;QSU#_(*N;m>4$+WSTJ&wGBz{V7}L4ivuY=R%_KdQb$`pMNeZo;t3;) zvh|8uQ!1-^ZN<`zSU_=nULISNrNyx~r)8-akCz4`q1w%crP;P_8sz&OMM*att*Tkl z?A)R>yEHw0QYuz0v)M?Ayf{BOGd6!&nvyR|gRP?IOrv>}PD&T%=HEd}8!A3}XnPH3QQYxxlQCBou z@#42rfK(bA$cRg)SVlKiw43(zn(k# z(+WD8qU@FgX@}&Jim6tLDsOCQac**!WHuwuE=Cm=OGYBE>ISiqhDD5}C@t$%@;hoU zH&jbk4M%a*YEiQ5YOU5Lu%bF@QMzUmn{%|A(X_N}?0}xu&OMVAhlbcYFM5z`X4UIV z4a;nhuR7go|18mX+oN}Y?N=Gzy706I@9tX^ykH~>=mejiONJLjd+>rZEqhHFX|^#8 zCFdD6>ewihVrlW5wlgX`gkk56BN=gFC~epYsvi(n%IzTodwFl~XOkXb-{RiW4MVe{?~l+_qftA+Ghs$a z>wgu~cGBXshFMuniQM5Ky}gsAw1}SR6NG-Bjvb(BE2O3xiW~L&WF3;F^VV)??%X?A zjKptYAtaR;X3u|#r4oBb_Nm->+fr1ITIJ4EsZXV#^+}j zBhYpD<}gh!4DSwHLWNVhTgxJnR&KsICM@y^+Z$@lT_$eEHKi)u0N(O~J!0k2Z9z&e zSo^xpZ(!nAxY(WrktW8(J7AuI~ah*M|MPJyo)-3GW$pcl6RPxXNKWBu`AJirn z`G5cjfB*=900@8p2!H?xfB*=900=xh0>|7F{yn@reAa#HpX)?$ANhBs=pSQ}<5A;E zF?jypeMY$d@M?yVKmY_l00ck)1V8`;KmY_l00ck)1Rfdzj^%j1pA^TE%|G`44~;tt z0s#;J0T2KI5C8!X009sH0T2Lzhe@EPhbQ0vCnd7Nzcb_?ULXJhAOHd&00JNY0w4ea zAOHd&00O&^K#@G+pXVps>;J*y4 EpochId? + Existed(i32), + New(String, String), +} + +#[derive(Clone)] pub struct Stat { pub stat_type: i32, pub stat_value: Json, @@ -72,7 +80,11 @@ pub trait CostModelStorageLayer { epoch_id: Self::EpochId, ) -> StorageResult<()>; - async fn update_stats(&self, stat: Stat, epoch_id: Self::EpochId) -> StorageResult<()>; + async fn update_stats( + &mut self, + stat: Stat, + epoch_option: EpochOption, + ) -> StorageResult>; async fn store_cost( &self, diff --git a/optd-persistent/src/cost_model/orm.rs b/optd-persistent/src/cost_model/orm.rs index c43c809..0c47117 100644 --- a/optd-persistent/src/cost_model/orm.rs +++ b/optd-persistent/src/cost_model/orm.rs @@ -13,7 +13,7 @@ use sea_orm::{ }; use super::catalog::mock_catalog::{self, MockCatalog}; -use super::interface::{CatalogSource, Stat}; +use super::interface::{CatalogSource, EpochOption, Stat}; impl BackendManager { fn get_description_from_attr_ids( @@ -154,40 +154,47 @@ impl CostModelStorageLayer for BackendManager { } } - // **IMPORTANT**: It is the caller's responsibility to ensure that the updated stat is not the same with the last stored stat if - // if it is already exists. - async fn update_stats(&self, stat: Stat, epoch_id: Self::EpochId) -> StorageResult<()> { - // let transaction = self.db.begin().await?; + /* Update the statistics in the database. + * The statistic can be newly inserted or updated. If the statistic value + * is the same as the latest existing one, the update will be ignored, and + * the return value will be None. + * If `epoch_option` is `EpochOption::Existed(epoch_id)`, the new statistic + * will be associated with the given epoch_id. If `epoch_option` is + * `EpochOption::New(source, data)`, a new epoch will be created with the + * given source and data, and the new statistic will be associated with the + * new epoch. And return the new epoch_id. + * If the statistic value is the same as the latest existing one, this function + * won't create a new epoch. + * + * For batch updates, if the caller can directly call this function with + * New epoch option at the first time, and if the epoch_id is returned, the + * caller can use the returned epoch_id for the rest of the updates. + * But if the epoch_id is not returned, the caller should continue using + * the New epoch option for the next statistic update. + */ + async fn update_stats( + &mut self, + stat: Stat, + epoch_option: EpochOption, + ) -> StorageResult> { + let transaction = self.db.begin().await?; // 0. Check if the stat already exists. If exists, get stat_id, else insert into statistic table. let stat_id = match stat.table_id { Some(table_id) => { // TODO(lanlou): only select needed fields let res = Statistic::find() .filter(statistic::Column::TableId.eq(table_id)) - .filter(statistic::Column::VariantTag.eq(stat.stat_type)) - /* - TODO(FIX_ME, lanlou): Do we need the following filter? - I am really not sure although I add the top comment... - Since we already increase the epoch, so we should update the stat anyway. - (In theory, we can increase the epoch without updating the stat, but it is not - a straightforward design, and the epoch table will be very large.) - But it will increase the overhead, since the caller will need to make another - query to check if the stat is the same with the last one. We cannot put everything - in one query. - Let us assume we should update the stat anyway for now. - */ - // .inner_join(versioned_statistic::Entity) - // .select_also(versioned_statistic::Entity) - // .order_by_desc(versioned_statistic::Column::EpochId) - .one(&self.db) + .inner_join(versioned_statistic::Entity) + .select_also(versioned_statistic::Entity) + .order_by_desc(versioned_statistic::Column::EpochId) + .one(&transaction) .await?; match res { Some(stat_data) => { - // if stat_data.1.unwrap().statistic_value == stat.stat_value { - // return Ok(()); - // } - // stat_data.0.id - stat_data.id + if stat_data.1.unwrap().statistic_value == stat.stat_value { + return Ok(None); + } + stat_data.0.id } None => { let new_stat = statistic::ActiveModel { @@ -201,7 +208,7 @@ impl CostModelStorageLayer for BackendManager { description: sea_orm::ActiveValue::Set("".to_string()), ..Default::default() }; - let res = Statistic::insert(new_stat).exec(&self.db).await; + let res = Statistic::insert(new_stat).exec(&transaction).await; match res { Ok(insert_res) => insert_res.last_insert_id, Err(_) => { @@ -221,18 +228,17 @@ impl CostModelStorageLayer for BackendManager { .filter(statistic::Column::NumberOfAttributes.eq(stat.attr_ids.len() as i32)) .filter(statistic::Column::Description.eq(description.clone())) .filter(statistic::Column::VariantTag.eq(stat.stat_type)) - // .inner_join(versioned_statistic::Entity) - // .select_also(versioned_statistic::Entity) - // .order_by_desc(versioned_statistic::Column::EpochId) - .one(&self.db) + .inner_join(versioned_statistic::Entity) + .select_also(versioned_statistic::Entity) + .order_by_desc(versioned_statistic::Column::EpochId) + .one(&transaction) .await?; match res { Some(stat_data) => { - // if stat_data.1.unwrap().statistic_value == stat.stat_value { - // return Ok(()); - // } - // stat_data.0.id - stat_data.id + if stat_data.1.unwrap().statistic_value == stat.stat_value { + return Ok(None); + } + stat_data.0.id } None => { let new_stat = statistic::ActiveModel { @@ -246,23 +252,17 @@ impl CostModelStorageLayer for BackendManager { ..Default::default() }; // TODO(lanlou): we should not clone here maybe... - let insert_res = Statistic::insert(new_stat.clone()).exec(&self.db).await?; + let insert_res = Statistic::insert(new_stat.clone()) + .exec(&transaction) + .await?; for attr_id in stat.attr_ids { let new_junction = statistic_to_attribute_junction::ActiveModel { statistic_id: sea_orm::ActiveValue::Set(insert_res.last_insert_id), attribute_id: sea_orm::ActiveValue::Set(attr_id), }; let res = StatisticToAttributeJunction::insert(new_junction) - .exec(&self.db) - .await; - if res.is_err() { - let _ = new_stat.delete(&self.db).await; - return Err(BackendError::Database(DbErr::Exec( - RuntimeErr::Internal( - "Failed to insert into statistic_to_attribute_junction table".to_string(), - ), - ))); - } + .exec(&transaction) + .await?; } insert_res.last_insert_id } @@ -270,13 +270,30 @@ impl CostModelStorageLayer for BackendManager { } }; // 1. Insert into attr_stats and related junction tables. + let mut insert_new_epoch = false; + let epoch_id = match epoch_option { + EpochOption::Existed(e) => e, + EpochOption::New(source, data) => { + insert_new_epoch = true; + let new_event = event::ActiveModel { + source_variant: sea_orm::ActiveValue::Set(source), + timestamp: sea_orm::ActiveValue::Set(Utc::now()), + data: sea_orm::ActiveValue::Set(sea_orm::JsonValue::String(data)), + ..Default::default() + }; + let insert_res = Event::insert(new_event).exec(&transaction).await?; + insert_res.last_insert_id + } + }; let new_stats = versioned_statistic::ActiveModel { epoch_id: sea_orm::ActiveValue::Set(epoch_id), statistic_id: sea_orm::ActiveValue::Set(stat_id), statistic_value: sea_orm::ActiveValue::Set(stat.stat_value), ..Default::default() }; - let _ = VersionedStatistic::insert(new_stats).exec(&self.db).await; + let _ = VersionedStatistic::insert(new_stats) + .exec(&transaction) + .await?; // 2. Invalidate all the related cost. let _ = plan_cost::Entity::update_many() @@ -297,11 +314,17 @@ impl CostModelStorageLayer for BackendManager { .to_owned(), ), ) - .exec(&self.db) - .await; + .exec(&transaction) + .await?; - // transaction.commit().await?; - Ok(()) + // TODO(lanlou): consider the update conflict for latest_epoch_id in multiple threads + if insert_new_epoch { + self.latest_epoch_id + .store(epoch_id as usize, std::sync::atomic::Ordering::Relaxed); + } + + transaction.commit().await?; + Ok(Some(epoch_id)) } async fn store_expr_stats_mappings( @@ -455,10 +478,11 @@ impl CostModelStorageLayer for BackendManager { #[cfg(test)] mod tests { - use crate::cost_model::interface::StatType; + use crate::cost_model::interface::{EpochOption, StatType}; use crate::{cost_model::interface::Stat, migrate, CostModelStorageLayer}; use crate::{get_sqlite_url, TEST_DATABASE_FILE}; use sea_orm::sqlx::database; + use sea_orm::sqlx::types::chrono::Utc; use sea_orm::Statement; use sea_orm::{ ColumnTrait, ConnectionTrait, Database, DbBackend, EntityTrait, ModelTrait, QueryFilter, @@ -497,8 +521,7 @@ mod tests { async fn copy_init_db(db_file: &str) -> String { let _ = std::fs::copy(TEST_DATABASE_FILE.clone(), db_file); - let database_url = get_sqlite_url(db_file); - database_url + get_sqlite_url(db_file) } #[tokio::test] @@ -566,7 +589,9 @@ mod tests { table_id: None, name: "countattr1".to_string(), }; - let res = backend_manager.update_stats(stat, epoch_id1).await; + let res = backend_manager + .update_stats(stat, EpochOption::Existed(epoch_id1)) + .await; assert!(res.is_ok()); let stat_res = Statistic::find() .filter(statistic::Column::Name.eq("countattr1")) @@ -636,7 +661,9 @@ mod tests { table_id: None, name: "countattr1".to_string(), }; - let res = backend_manager.update_stats(stat2, epoch_id2).await; + let res = backend_manager + .update_stats(stat2, EpochOption::Existed(epoch_id2)) + .await; assert!(res.is_ok()); // 2.3 Check statistic table let stat_res = Statistic::find() @@ -645,9 +672,7 @@ mod tests { .await .unwrap(); assert_eq!(stat_res.len(), 1); - assert_eq!(stat_res[0].number_of_attributes, 1); assert_eq!(stat_res[0].description, "1".to_string()); - assert_eq!(stat_res[0].variant_tag, StatType::Count as i32); // 2.4 Check statistic_to_attribute_junction table let stat_attr_res = StatisticToAttributeJunction::find() .filter(statistic_to_attribute_junction::Column::StatisticId.eq(stat_res[0].id)) @@ -681,11 +706,121 @@ mod tests { assert_eq!(cost_res[0].epoch_id, epoch_id1); assert!(!cost_res[0].is_valid); + // 3. Update existed stat with the same value + let epoch_num = Event::find().all(&backend_manager.db).await.unwrap().len(); + let stat3 = Stat { + stat_type: StatType::Count as i32, + stat_value: json!(200), + attr_ids: vec![1], + table_id: None, + name: "CountAttr1".to_string(), + }; + let res = backend_manager + .update_stats( + stat3, + EpochOption::New("source".to_string(), "data".to_string()), + ) + .await; + assert!(res.is_ok()); + assert!(res.unwrap().is_none()); + let epoch_num2 = Event::find().all(&backend_manager.db).await.unwrap().len(); + assert_eq!(epoch_num, epoch_num2); + remove_db_file(DATABASE_FILE); } #[tokio::test] - async fn test_update_table_stats() {} + async fn test_update_table_stats() { + // Simulate batch updates, first insert an existed same stat with none epoch_id, + // then insert some non-existed or different stats with New epoch_option. + const DATABASE_FILE: &str = "test_update_table_stats.db"; + let database_url = copy_init_db(DATABASE_FILE).await; + let mut binding = super::BackendManager::new(Some(&database_url)).await; + let backend_manager = binding.as_mut().unwrap(); + + let table_inserted_res = TableMetadata::insert(table_metadata::ActiveModel { + name: sea_orm::ActiveValue::Set("Table2".to_string()), + namespace_id: sea_orm::ActiveValue::Set(1), + creation_time: sea_orm::ActiveValue::Set(Utc::now()), + ..Default::default() + }) + .exec(&backend_manager.db) + .await + .unwrap(); + + let statistics: Vec = vec![ + Stat { + stat_type: StatType::Count as i32, + stat_value: json!(0), + attr_ids: vec![], + table_id: Some(1), + name: "row_count".to_string(), + }, + Stat { + stat_type: StatType::Count as i32, + stat_value: json!(20), + attr_ids: vec![], + table_id: Some(1), + name: "row_count".to_string(), + }, + Stat { + stat_type: StatType::Count as i32, + stat_value: json!(100), + attr_ids: vec![], + table_id: Some(table_inserted_res.last_insert_id), + name: "Table2Count1".to_string(), + }, + ]; + + let mut epoch_id: Option = None; + for stat in statistics { + match epoch_id { + Some(e) => { + let res = backend_manager + .update_stats(stat.clone(), EpochOption::Existed(e)) + .await; + assert!(res.is_ok()); + assert!(stat.name == "Table2Count1"); + let res = res.unwrap(); + assert!(res.is_some()); + assert!(res.unwrap() == e); + let stat_res = Statistic::find() + .filter(statistic::Column::Name.eq(stat.name.clone())) + .all(&backend_manager.db) + .await + .unwrap(); + assert_eq!(stat_res.len(), 1); + let versioned_stat_res = VersionedStatistic::find() + .filter(versioned_statistic::Column::StatisticId.eq(stat_res[0].id)) + .all(&backend_manager.db) + .await + .unwrap(); + assert_eq!(versioned_stat_res.len(), 1); + assert_eq!(versioned_stat_res[0].statistic_value, stat.stat_value); + assert_eq!(versioned_stat_res[0].epoch_id, e); + } + None => { + let res = backend_manager + .update_stats( + stat.clone(), + EpochOption::New("source".to_string(), "data".to_string()), + ) + .await; + assert!(res.is_ok()); + if stat.stat_value == json!(0) { + assert!(res.unwrap().is_none()); + } else { + assert!(stat.stat_value == json!(20)); + let res = res.unwrap(); + assert!(res.is_some()); + epoch_id = Some(res.unwrap()); + } + } + } + } + + remove_db_file(DATABASE_FILE); + } #[tokio::test] async fn test_store_cost() { @@ -699,7 +834,7 @@ mod tests { .unwrap(); let physical_expression_id = 1; let cost = 42; - let res = backend_manager + backend_manager .store_cost(physical_expression_id, cost, epoch_id) .await .unwrap(); @@ -816,7 +951,10 @@ mod tests { table_id: Some(table_id), name: "row_count".to_string(), }; - backend_manager.update_stats(stat, epoch_id2).await.unwrap(); + backend_manager + .update_stats(stat, EpochOption::Existed(epoch_id2)) + .await + .unwrap(); // Get updated stats let res = backend_manager @@ -873,7 +1011,10 @@ mod tests { table_id: None, name: "cardinality".to_string(), }; - backend_manager.update_stats(stat, epoch_id2).await.unwrap(); + backend_manager + .update_stats(stat, EpochOption::Existed(epoch_id2)) + .await + .unwrap(); // Get updated stats let res = backend_manager @@ -930,7 +1071,10 @@ mod tests { table_id: None, name: "cardinality".to_string(), }; - backend_manager.update_stats(stat, epoch_id2).await.unwrap(); + backend_manager + .update_stats(stat, EpochOption::Existed(epoch_id2)) + .await + .unwrap(); // Get updated stats let res = backend_manager diff --git a/optd-persistent/src/db/init.db b/optd-persistent/src/db/init.db index 1fc7eb9a38c4ba11b8be7fac114e62aecf285964..d88b92d67fbd7019ce2728841a6ce9ee457f8357 100644 GIT binary patch delta 485 zcmZo@;B08%oFFB{oW;PvpaR5Dz!)`A$B2nJYh%JKX=Xj=1(R)MG}wW(wkUJxWJek4 z$+va5*wTRl8X}Vq%E<5(6y+DB7L`UK_Yr!?(sumfprQRdLejxy4d zZ|iWer2_>tL?$1Uk>M#Q$}dPQDyaksP2Mk~&H<#ggqVX_CJX3FPWIR3;snX42r|2~ zOpeo)WdW8F_)6(FX)lr;mpR7p+?O#90zfLx)?$sEcudA6PkMAdvbIk4;j zIaQFXEgN$s%VYB$QE(qK25vNC6aMD)PkQ)B_TZSp@kPKfqKc~!9XmGYV(?V60t zfh?2n=_`R{Hz~Z9=Ql96GPVRFJriSNBSYiOzwAGv3Yr-iZvGYjksn#5iGij0=3n_O z>yZV`O-(GP_sTPBAq$$Bnix+%CeNtKhh&kdx%u{=@{AYaQ3Q>Rw_mSk)I0zHw&#;3 From 7a8d4dfdb40765b5dcebe2c30867e678fffb68fa Mon Sep 17 00:00:00 2001 From: Lan Lou Date: Sat, 9 Nov 2024 13:30:36 -0500 Subject: [PATCH 28/28] Rebase on main --- optd-persistent/Cargo.lock | 1 + optd-persistent/src/cost_model/orm.rs | 3 +++ 2 files changed, 4 insertions(+) diff --git a/optd-persistent/Cargo.lock b/optd-persistent/Cargo.lock index 3a8e5be..adf1584 100644 --- a/optd-persistent/Cargo.lock +++ b/optd-persistent/Cargo.lock @@ -1146,6 +1146,7 @@ version = "0.1.0" dependencies = [ "async-stream", "async-trait", + "lazy_static", "sea-orm", "sea-orm-migration", "serde_json", diff --git a/optd-persistent/src/cost_model/orm.rs b/optd-persistent/src/cost_model/orm.rs index 0c47117..07ddbdb 100644 --- a/optd-persistent/src/cost_model/orm.rs +++ b/optd-persistent/src/cost_model/orm.rs @@ -318,6 +318,9 @@ impl CostModelStorageLayer for BackendManager { .await?; // TODO(lanlou): consider the update conflict for latest_epoch_id in multiple threads + // Assume the txn fails to commit, and the epoch_id is updated. But the epoch_id + // is always increasing and won't be overwritten even if the record is deleted, it + // might be fine. if insert_new_epoch { self.latest_epoch_id .store(epoch_id as usize, std::sync::atomic::Ordering::Relaxed);