Skip to content

Commit

Permalink
Registry from fs (#16)
Browse files Browse the repository at this point in the history
* add: derive macros

* feat: read packages from root folder

* feat: add settings in model layer

* fix: warning
  • Loading branch information
DarthB authored Feb 3, 2025
1 parent 182034a commit 2128ce6
Show file tree
Hide file tree
Showing 17 changed files with 406 additions and 42 deletions.
3 changes: 3 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,6 @@ serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
strum = "0.26"
strum_macros = "0.26"

uuid = { version = "1.12", features = ["v4"] }
config = "0.15"
4 changes: 3 additions & 1 deletion nebula_common/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,9 @@ strum_macros.workspace = true
serde.workspace = true
serde_json.workspace = true

config = "0.15"
uuid.workspace = true
config.workspace = true


[build-dependencies]
tonic-build = "0.12.3"
Expand Down
4 changes: 3 additions & 1 deletion nebula_common/proto/nebula.proto
Original file line number Diff line number Diff line change
Expand Up @@ -54,11 +54,13 @@ enum SortOption {
CREATION_DATE = 0; // Sort by creation date
DOWNLOADS = 1; // Sort by download count descending
NAME = 2; // Sort Alphabetical by name
AUTHOR = 3; // Sort Alphabetical by auhtor
}

message SortParameter {
SortOption sort_by = 1; // Sort option
optional bool descending = 2; // Sort direction, default: ascending
repeated bytes params = 3; // interpretation based on SortOption, e.g. DateRange for DOWNLOADs
}

message FieldOptions {
Expand All @@ -70,7 +72,7 @@ message FieldOptions {
// message returns complete package info
message PackageRequest {
string search_query = 1; // searches for EXACT package-name
PackageType package_type = 2; // filters by dataset, model or both
optional PackageType package_type = 2; // filters by dataset, model or both
}

message ListPackagesRequest {
Expand Down
2 changes: 1 addition & 1 deletion nebula_common/src/client/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ pub async fn get_package_info(
name: String,
) -> Result<PackageInfo, Box<dyn std::error::Error>> {
let request =
Request::new(PackageRequest { search_query: name, package_type: PackageType::Both as i32 });
Request::new(PackageRequest { search_query: name, package_type: None });
let response = client.get_package_info(request).await?;

Ok(response.into_inner())
Expand Down
12 changes: 10 additions & 2 deletions nebula_common/src/configuration/registry.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
use std::str::FromStr;

use serde::Deserialize;

#[derive(Debug, Copy, Clone, PartialEq, strum_macros::EnumString, strum_macros::Display)]
pub enum Environment {
#[strum(ascii_case_insensitive)]
Expand All @@ -9,19 +11,25 @@ pub enum Environment {
Production,
}

#[derive(Debug, Clone, serde::Deserialize)]
#[derive(Debug, Clone, Deserialize)]
pub struct Settings {
pub application: ApplicationSettings,
pub root_folder: Option<RootFolder>,
}

#[derive(Debug, Clone, serde::Deserialize)]
#[derive(Debug, Clone, Deserialize)]
pub struct ApplicationSettings {
//#[serde(deserialize_with = "deserialize_number_from_string")]
pub port: u16,
pub host: String,
pub base_url: String,
}

#[derive(Debug, Clone, Deserialize)]
pub struct RootFolder {
pub path: String,
}

pub fn get_configuration() -> Result<Settings, config::ConfigError> {
let base_path = std::env::current_dir().expect("Failed to determine the current directory");
let configuration_directory = base_path.join("configuration");
Expand Down
4 changes: 2 additions & 2 deletions nebula_common/src/datapackage/delta.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
use serde::{Deserialize, Serialize};

#[derive(Serialize, Deserialize)]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct DeltaDataPackageNotValidated {
pub category: String,
pub classes: Option<u32>,
Expand All @@ -15,7 +15,7 @@ pub struct DeltaDataPackageNotValidated {
pub mirror: Option<String>,
}

#[derive(Serialize, Deserialize)]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct DeltaDataResourceNotValidated {
pub origin: String,
pub format: Option<String>,
Expand Down
19 changes: 9 additions & 10 deletions nebula_common/src/datapackage/pod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ pub fn datapackage_meta_from_file_not_validated(
}

/// A mapping for the Data Package json format that is not validated in respect to the schema.
#[derive(Serialize, Deserialize)]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct DataPackageNotValidated {
pub resources: Vec<DataResourceNotValidated>,

Expand All @@ -41,7 +41,7 @@ pub struct DataPackageNotValidated {

pub id: Option<String>,

pub licenses: Option<Vec<DataPackageLicense>>,
pub licenses: Vec<DataPackageLicense>,

pub title: Option<String>,

Expand All @@ -63,9 +63,8 @@ pub struct DataPackageNotValidated {

pub delta: Option<DeltaDataPackageNotValidated>,
}

/// A mapping for the Data Resource json format that is not validated in respect to the schema.
#[derive(Serialize, Deserialize)]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct DataResourceNotValidated {
name: String,

Expand Down Expand Up @@ -95,14 +94,14 @@ pub struct DataResourceNotValidated {
delta: Option<DeltaDataResourceNotValidated>,
}

#[derive(Serialize, Deserialize)]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum PathSingleOrVec {
Single(String),
Vec(Vec<String>),
}

#[derive(Serialize, Deserialize)]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum DataStringOrObj {
String(String),
Expand All @@ -111,15 +110,15 @@ pub enum DataStringOrObj {
Object(),
}

#[derive(Serialize, Deserialize)]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum DataResourcesData {
Array(Vec<HashMap<String, String>>),

String(String),
}

#[derive(Serialize, Deserialize)]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct DataPackageContributor {
pub title: Option<String>,

Expand All @@ -136,7 +135,7 @@ pub struct DataPackageContributor {
pub organziation: Option<String>,
}

#[derive(Serialize, Deserialize)]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct DataPackageSource {
pub title: Option<String>,

Expand All @@ -147,7 +146,7 @@ pub struct DataPackageSource {
pub version: Option<String>,
}

#[derive(Serialize, Deserialize)]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct DataPackageLicense {
pub name: String,

Expand Down
7 changes: 4 additions & 3 deletions nebula_common/src/datapackage/validated.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,9 @@ pub trait ValidateData {
}

/// A wrapper typ that marks input data as validated
pub struct Validated<T: Sized>(T);
impl<T: Sized> Validated<T> {
#[derive(Debug, Clone, PartialEq)]
pub struct Validated<T: Sized + Sync + Send>(T);
impl<T: Sized + Sync + Send> Validated<T> {
pub fn into_inner(self) -> T {
self.0
}
Expand All @@ -47,7 +48,7 @@ impl<T: Sized> Validated<T> {
}
}

impl<T> Deref for Validated<T> {
impl<T: Sized + Sync + Send> Deref for Validated<T> {
type Target = T;

fn deref(&self) -> &T {
Expand Down
4 changes: 4 additions & 0 deletions nebula_common/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
//! Nebula common library crate with functionality for both registry and cli
pub mod client;
pub mod configuration;
pub mod datapackage;
pub mod model;
pub mod server;
pub mod storage;

pub mod nebula_proto {
tonic::include_proto!("nebula.v1");
Expand Down
62 changes: 62 additions & 0 deletions nebula_common/src/model/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
//! Contains the model internally used by nebula
//!
//!
pub mod pb_mapper;

pub type PackageType = super::server::PackageType;

/// Optional MetaData Fields
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MetaDataField {
PreviewImages,

DataPackage,

// todo more?
}


/// Settings to select additional (heavy) fields
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct FieldSettings {
optional_fields: Vec<MetaDataField>, // todo: on stack? set semantic...
}

/// Pagation Settings
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct PagationSettings {
pub limit: u32,

pub offset: u32,
}

impl Default for PagationSettings {
fn default() -> Self {
Self { limit: 30, offset: Default::default() }
}
}

/// multi level sort settinggs, not implemented
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SortSettings {

}

impl Default for SortSettings {
fn default() -> Self {
Self { }
}
}

/// Filter Settings, not implemnted
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FilterSettings {
package_type: PackageType,
}

impl Default for FilterSettings {
fn default() -> Self {
Self { package_type: crate::server::PackageType::Both }
}
}
98 changes: 98 additions & 0 deletions nebula_common/src/model/pb_mapper.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
//! Contains functionality to map protobuf related types to nebulas internal model
use crate::{datapackage::DataPackage, server::PackageInfo};

use super::{FilterSettings, PackageType, PagationSettings, SortSettings};

/// Maps self to Pagation Settings
pub trait PagationMapper {
fn as_pagation(&self) -> Result<PagationSettings, Box<dyn std::error::Error>>;
}

/// Maps self to Filter Settings
pub trait FilterMapper {
fn as_filter(&self) -> Result<FilterSettings, Box<dyn std::error::Error>>;
fn into_filter(self) -> Result<FilterSettings, Box<dyn std::error::Error>>;
}

pub trait SortMapper {
fn as_sort(&self) -> Result<SortSettings, Box<dyn std::error::Error>>;
}

impl PagationMapper for super::super::server::ListPackagesRequest {
fn as_pagation(&self) -> Result<PagationSettings, Box<dyn std::error::Error>> {
let mut reval = PagationSettings::default();
if let Some(limit) = self.limit {reval.limit = limit as u32;}
if let Some(offset) = self.offset {reval.offset = offset as u32;}
Ok(reval)
}
}

impl PagationMapper for super::super::server::SearchPackagesRequest {
fn as_pagation(&self) -> Result<PagationSettings, Box<dyn std::error::Error>> {
let mut reval = PagationSettings::default();
if let Some(limit) = self.limit {reval.limit = limit as u32;}
if let Some(offset) = self.offset {reval.offset = offset as u32;}
Ok(reval)
}
}

impl FilterMapper for super::super::server::PackageRequest {
fn as_filter(&self) -> Result<FilterSettings, Box<dyn std::error::Error>> {
let mut reval = FilterSettings::default();
if let Some(pt) = self.package_type {reval.package_type = PackageType::try_from(pt).unwrap();}
Ok(reval)
}

fn into_filter(self) -> Result<FilterSettings, Box<dyn std::error::Error>> {
self.as_filter()
}
}

impl FilterMapper for super::super::server::SearchPackagesRequest {
fn as_filter(&self) -> Result<FilterSettings, Box<dyn std::error::Error>> {
Ok(FilterSettings::default())
}

fn into_filter(self) -> Result<FilterSettings, Box<dyn std::error::Error>> {
Ok(FilterSettings::default())
}

}

impl SortMapper for super::super::server::SearchPackagesRequest {
fn as_sort(&self) -> Result<SortSettings, Box<dyn std::error::Error>> {
Ok(SortSettings::default())
}
}

impl Into<PackageInfo> for DataPackage {
fn into(self) -> PackageInfo {
let mut inner = self.into_inner();
PackageInfo {
name: match inner.name.take() {
Some(v) => v,
None => "No name".to_string(),
},
version: match inner.version.take() {
Some(v) => v,
None => "0.1.0".to_string(),
},
description: match inner.description.take() {
Some(v) => v,
None => "No Information".to_string(),
},
license: {
let mut reval = String::new();
for lic in &inner.licenses {
reval += lic.name.as_str();
}
if reval.is_empty() {
reval = "UKNOWN".to_string()
}
reval
},
..Default::default()
}
}
}
Loading

0 comments on commit 2128ce6

Please sign in to comment.