From 72af7b510587140a8f65b67681aa5bf9ac39c756 Mon Sep 17 00:00:00 2001
From: Tim Janus <tim@janus.rs>
Date: Fri, 24 Jan 2025 08:20:25 +0100
Subject: [PATCH 1/4] add: derive macros

---
 nebula_common/src/datapackage/delta.rs     |  4 ++--
 nebula_common/src/datapackage/pod.rs       | 18 +++++++++---------
 nebula_common/src/datapackage/validated.rs |  1 +
 3 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/nebula_common/src/datapackage/delta.rs b/nebula_common/src/datapackage/delta.rs
index 8250d71..f138cf2 100644
--- a/nebula_common/src/datapackage/delta.rs
+++ b/nebula_common/src/datapackage/delta.rs
@@ -4,7 +4,7 @@
 
 use serde::{Deserialize, Serialize};
 
-#[derive(Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct DeltaDataPackageNotValidated {
     pub category: String,
     pub classes: Option<u32>,
@@ -15,7 +15,7 @@ pub struct DeltaDataPackageNotValidated {
     pub mirror: Option<String>,
 }
 
-#[derive(Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct DeltaDataResourceNotValidated {
     pub origin: String,
     pub format: Option<String>,
diff --git a/nebula_common/src/datapackage/pod.rs b/nebula_common/src/datapackage/pod.rs
index a0f5f8a..8b33160 100644
--- a/nebula_common/src/datapackage/pod.rs
+++ b/nebula_common/src/datapackage/pod.rs
@@ -30,7 +30,7 @@ pub fn datapackage_meta_from_file_not_validated(
 }
 
 /// A mapping for the Data Package json format that is not validated in respect to the schema.
-#[derive(Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct DataPackageNotValidated {
     pub resources: Vec<DataResourceNotValidated>,
 
@@ -41,7 +41,7 @@ pub struct DataPackageNotValidated {
 
     pub id: Option<String>,
 
-    pub licenses: Option<Vec<DataPackageLicense>>,
+    pub licenses: Vec<DataPackageLicense>,
 
     pub title: Option<String>,
 
@@ -65,7 +65,7 @@ pub struct DataPackageNotValidated {
 }
 
 /// A mapping for the Data Resource json format that is not validated in respect to the schema.
-#[derive(Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct DataResourceNotValidated {
     name: String,
 
@@ -95,14 +95,14 @@ pub struct DataResourceNotValidated {
     delta: Option<DeltaDataResourceNotValidated>,
 }
 
-#[derive(Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(untagged)]
 pub enum PathSingleOrVec {
     Single(String),
     Vec(Vec<String>),
 }
 
-#[derive(Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(untagged)]
 pub enum DataStringOrObj {
     String(String),
@@ -111,7 +111,7 @@ pub enum DataStringOrObj {
     Object(),
 }
 
-#[derive(Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(untagged)]
 pub enum DataResourcesData {
     Array(Vec<HashMap<String, String>>),
@@ -119,7 +119,7 @@ pub enum DataResourcesData {
     String(String),
 }
 
-#[derive(Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct DataPackageContributor {
     pub title: Option<String>,
 
@@ -136,7 +136,7 @@ pub struct DataPackageContributor {
     pub organziation: Option<String>,
 }
 
-#[derive(Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct DataPackageSource {
     pub title: Option<String>,
 
@@ -147,7 +147,7 @@ pub struct DataPackageSource {
     pub version: Option<String>,
 }
 
-#[derive(Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct DataPackageLicense {
     pub name: String,
 
diff --git a/nebula_common/src/datapackage/validated.rs b/nebula_common/src/datapackage/validated.rs
index 942f8d6..f4735cb 100644
--- a/nebula_common/src/datapackage/validated.rs
+++ b/nebula_common/src/datapackage/validated.rs
@@ -35,6 +35,7 @@ pub trait ValidateData {
 }
 
 /// A wrapper typ that marks input data as validated
+#[derive(Debug, Clone, PartialEq)]
 pub struct Validated<T: Sized>(T);
 impl<T: Sized> Validated<T> {
     pub fn into_inner(self) -> T {

From 921f2b494539af14e1e15fc67b27d222e876e0b4 Mon Sep 17 00:00:00 2001
From: Tim Janus <tim@janus.rs>
Date: Fri, 24 Jan 2025 08:23:47 +0100
Subject: [PATCH 2/4] feat: read packages from root folder

---
 Cargo.toml                                    |  3 +
 nebula_common/Cargo.toml                      |  4 +-
 nebula_common/src/configuration/registry.rs   | 12 ++-
 nebula_common/src/lib.rs                      |  1 +
 nebula_common/src/server/endpoints.rs         | 62 ++++++++------
 nebula_common/src/server/metadata_fetchers.rs | 69 ++++++++++++++++
 nebula_common/src/server/mod.rs               |  1 +
 nebula_common/src/storage/data_source.rs      | 12 +++
 nebula_common/src/storage/mod.rs              |  2 +
 nebula_common/src/storage/root_folder.rs      | 80 +++++++++++++++++++
 nebula_registry/configuration/local.yaml      |  4 +-
 nebula_registry/src/main.rs                   |  8 +-
 12 files changed, 230 insertions(+), 28 deletions(-)
 create mode 100644 nebula_common/src/server/metadata_fetchers.rs
 create mode 100644 nebula_common/src/storage/data_source.rs
 create mode 100644 nebula_common/src/storage/mod.rs
 create mode 100644 nebula_common/src/storage/root_folder.rs

diff --git a/Cargo.toml b/Cargo.toml
index 8dd193e..a707748 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -15,3 +15,6 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 strum = "0.26"
 strum_macros = "0.26"
+
+uuid = { version = "1.12", features = ["v4"] }
+config = "0.15"
diff --git a/nebula_common/Cargo.toml b/nebula_common/Cargo.toml
index a47519c..bfe8926 100644
--- a/nebula_common/Cargo.toml
+++ b/nebula_common/Cargo.toml
@@ -28,7 +28,9 @@ strum_macros.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 
-config = "0.15"
+uuid.workspace = true
+config.workspace = true
+
 
 [build-dependencies]
 tonic-build = "0.12.3"
diff --git a/nebula_common/src/configuration/registry.rs b/nebula_common/src/configuration/registry.rs
index 02baf76..4fd659d 100644
--- a/nebula_common/src/configuration/registry.rs
+++ b/nebula_common/src/configuration/registry.rs
@@ -1,5 +1,7 @@
 use std::str::FromStr;
 
+use serde::Deserialize;
+
 #[derive(Debug, Copy, Clone, PartialEq, strum_macros::EnumString, strum_macros::Display)]
 pub enum Environment {
     #[strum(ascii_case_insensitive)]
@@ -9,12 +11,13 @@ pub enum Environment {
     Production,
 }
 
-#[derive(Debug, Clone, serde::Deserialize)]
+#[derive(Debug, Clone, Deserialize)]
 pub struct Settings {
     pub application: ApplicationSettings,
+    pub root_folder: Option<RootFolder>,
 }
 
-#[derive(Debug, Clone, serde::Deserialize)]
+#[derive(Debug, Clone, Deserialize)]
 pub struct ApplicationSettings {
     //#[serde(deserialize_with = "deserialize_number_from_string")]
     pub port: u16,
@@ -22,6 +25,11 @@ pub struct ApplicationSettings {
     pub base_url: String,
 }
 
+#[derive(Debug, Clone, Deserialize)]
+pub struct RootFolder {
+    pub path: String,
+}
+
 pub fn get_configuration() -> Result<Settings, config::ConfigError> {
     let base_path = std::env::current_dir().expect("Failed to determine the current directory");
     let configuration_directory = base_path.join("configuration");
diff --git a/nebula_common/src/lib.rs b/nebula_common/src/lib.rs
index 195d3c5..5ba5f3a 100644
--- a/nebula_common/src/lib.rs
+++ b/nebula_common/src/lib.rs
@@ -2,6 +2,7 @@ pub mod client;
 pub mod configuration;
 pub mod datapackage;
 pub mod server;
+pub mod storage;
 
 pub mod nebula_proto {
     tonic::include_proto!("nebula.v1");
diff --git a/nebula_common/src/server/endpoints.rs b/nebula_common/src/server/endpoints.rs
index 932f569..d3b8a25 100644
--- a/nebula_common/src/server/endpoints.rs
+++ b/nebula_common/src/server/endpoints.rs
@@ -1,33 +1,51 @@
+//! Contains the entry points for the grpc endpoints
+//!
+//! An endpoint has to implement an autogenerated trait of grpc.
+
+use crate::server::metadata_fetchers::{fetch_list_packages, fetch_packages};
+use crate::storage::data_source::DataSource;
+
 use super::nebula_package_query_server::NebulaPackageQuery;
 use super::{ListPackagesRequest, PackageInfo, PackageList, PackageRequest, SearchPackagesRequest};
 
 use tonic::{Request, Response, Status};
 
-#[derive(Debug, Default)]
-pub struct NebulaPackageQueryMockImpl {}
-
-fn dummy_package() -> PackageInfo {
-    PackageInfo {
-        description: "Dataset image classification".to_string(),
-        name: "cifar10".to_string(),
-        version: "2".to_string(),
-        download_size: 0,
-        installed_size: 0,
-        license: "todo".into(),
-        datapackage_json: None,
-        preview_images: vec![],
+#[derive(Debug)]
+pub struct NebulaPackageQueryMockImpl<T>
+where
+    T: DataSource + Send + Sync,
+{
+    inner_ds: T,
+}
+
+impl<T> NebulaPackageQueryMockImpl<T>
+where
+    T: DataSource + Send + Sync,
+{
+    pub fn new(ds: T) -> Self {
+        Self { inner_ds: ds }
     }
 }
 
 #[tonic::async_trait]
-impl NebulaPackageQuery for NebulaPackageQueryMockImpl {
+impl<T> NebulaPackageQuery for NebulaPackageQueryMockImpl<T>
+where
+    T: DataSource + Send + Sync + 'static,
+{
     async fn get_package_info(
         &self,
         request: Request<PackageRequest>,
     ) -> Result<Response<PackageInfo>, Status> {
         println!("Got a request: {:?}", request);
 
-        Ok(Response::new(dummy_package()))
+        match fetch_packages(
+            &self.inner_ds,
+            request.get_ref().search_query.as_str(),
+            request.get_ref().package_type,
+        ) {
+            Ok(body) => Ok(Response::new(body)),
+            Err(_) => Err(Status::internal("Error handling not implemented")),
+        }
     }
 
     async fn list_packages(
@@ -36,20 +54,18 @@ impl NebulaPackageQuery for NebulaPackageQueryMockImpl {
     ) -> Result<Response<PackageList>, Status> {
         println!("Got a request: {:?}", request);
 
-        let response_body = PackageList {
-            packages: vec![dummy_package()],
-            total_count: 1,
-            limit: None,
-            offset: None,
-        };
+        // todo: params from request
+        let body = fetch_list_packages(&self.inner_ds, 0, 30, 0);
 
-        Ok(Response::new(response_body))
+        Ok(Response::new(body))
     }
 
     async fn search_packages(
         &self,
-        _request: Request<SearchPackagesRequest>,
+        request: Request<SearchPackagesRequest>,
     ) -> Result<Response<PackageList>, Status> {
+        println!("Got a request: {:?}", request);
+
         Err(Status::internal("not implemented"))
     }
 }
diff --git a/nebula_common/src/server/metadata_fetchers.rs b/nebula_common/src/server/metadata_fetchers.rs
new file mode 100644
index 0000000..c73a7b5
--- /dev/null
+++ b/nebula_common/src/server/metadata_fetchers.rs
@@ -0,0 +1,69 @@
+//! Contains functions that use [crate::storage::DataSource] to fetch data from a storage backend
+
+use super::{PackageInfo, PackageList};
+use crate::{datapackage::DataPackage, storage::data_source::DataSource};
+
+#[derive(Debug, strum_macros::Display)]
+pub enum FetchError {
+    NotFound,
+    Internal,
+}
+
+impl std::error::Error for FetchError {}
+
+pub(super) fn fetch_packages<T>(
+    _ds: &T,
+    _search_query: &str,
+    _package_type: i32,
+) -> Result<PackageInfo, FetchError> {
+    todo!();
+}
+
+fn convert_to_package_info(data: DataPackage) -> Result<PackageInfo, FetchError> {
+    let info = PackageInfo {
+        name: data.name.clone().unwrap(),
+        version: match &data.version {
+            Some(v) => v.clone(),
+            None => "0.1.0".to_string(),
+        },
+        description: match &data.description {
+            Some(v) => v.clone(),
+            None => "No Information".to_string(),
+        },
+        license: {
+            let mut reval = String::new();
+            for lic in &data.licenses {
+                reval += lic.name.as_str();
+            }
+            if reval.is_empty() {
+                reval = "UKNOWN".to_string()
+            }
+            reval
+        },
+        ..Default::default()
+    };
+
+    Ok(info)
+}
+
+pub(super) fn fetch_list_packages<T>(
+    ds: &T,
+    _package_type: i32,
+    _limit: i32,
+    _offset: i32,
+) -> PackageList
+where
+    T: DataSource,
+{
+    let ids = ds.list_packages();
+    let mut packages = Vec::with_capacity(ids.len());
+    for id in ids {
+        if let Some(package) = ds.get_package(id) {
+            packages
+                .push(convert_to_package_info(package).expect("valid packages in data sources"));
+        }
+    }
+
+    let len = packages.len();
+    PackageList { packages, total_count: len as i32, limit: None, offset: None }
+}
diff --git a/nebula_common/src/server/mod.rs b/nebula_common/src/server/mod.rs
index 12731c4..eb198a9 100644
--- a/nebula_common/src/server/mod.rs
+++ b/nebula_common/src/server/mod.rs
@@ -6,4 +6,5 @@ pub use super::nebula_proto::nebula_package_query_server::NebulaPackageQueryServ
 pub(crate) use super::nebula_proto::*;
 
 pub mod endpoints;
+pub mod metadata_fetchers;
 pub use endpoints::NebulaPackageQueryMockImpl;
diff --git a/nebula_common/src/storage/data_source.rs b/nebula_common/src/storage/data_source.rs
new file mode 100644
index 0000000..852fed9
--- /dev/null
+++ b/nebula_common/src/storage/data_source.rs
@@ -0,0 +1,12 @@
+use uuid::Uuid;
+
+use crate::datapackage::DataPackage;
+
+pub(crate) type PackageId = Uuid;
+
+/// Trait to receive package information from a data source like the filesystem or a database
+pub trait DataSource {
+    fn list_packages(&self) -> Vec<PackageId>;
+
+    fn get_package(&self, id: PackageId) -> Option<DataPackage>;
+}
diff --git a/nebula_common/src/storage/mod.rs b/nebula_common/src/storage/mod.rs
new file mode 100644
index 0000000..dfecb9d
--- /dev/null
+++ b/nebula_common/src/storage/mod.rs
@@ -0,0 +1,2 @@
+pub mod data_source;
+pub mod root_folder;
diff --git a/nebula_common/src/storage/root_folder.rs b/nebula_common/src/storage/root_folder.rs
new file mode 100644
index 0000000..e1dbd2d
--- /dev/null
+++ b/nebula_common/src/storage/root_folder.rs
@@ -0,0 +1,80 @@
+use std::{collections::HashMap, path::PathBuf};
+
+use uuid::Uuid;
+
+use crate::datapackage::{DataPackage, datapackage_meta_from_file};
+
+use super::data_source::{DataSource, PackageId};
+
+/// Reads datapackage.json files from the filesystem
+pub struct RootFolderSource {
+    path: PathBuf,
+
+    buf: HashMap<PathBuf, (Uuid, DataPackage)>,
+}
+
+fn get_datapackage_file_candidates_from_folder(
+    path: &PathBuf,
+    candidates: &mut Vec<PathBuf>,
+) -> Result<(), Box<dyn std::error::Error>> {
+    // check folders for datapackage.json:
+    if path.is_dir() {
+        for entry in std::fs::read_dir(path)? {
+            let entry = entry?;
+            let mut path = entry.path();
+            if path.is_dir() {
+                // 1. check for datapackage.json
+                path.push("datapackage.json");
+                if path.is_file() {
+                    candidates.push(path);
+                }
+            }
+        }
+    }
+
+    Ok(())
+}
+
+impl RootFolderSource {
+    pub fn new_from_folder(path: PathBuf) -> Self {
+        let mut reval = RootFolderSource { path, buf: HashMap::new() };
+        reval.sync_all().unwrap();
+        reval
+    }
+
+    fn sync_all(&mut self) -> Result<(), Box<dyn std::error::Error>> {
+        let mut candidates = vec![];
+        get_datapackage_file_candidates_from_folder(&self.path, &mut candidates)?;
+
+        for candidate in candidates {
+            match self.sync_file(candidate) {
+                Ok(_) => {}
+                Err(_) => {
+                    // todo error reporting
+                }
+            }
+        }
+        Ok(())
+    }
+
+    fn sync_file(&mut self, file_path: PathBuf) -> Result<(), Box<dyn std::error::Error>> {
+        let dp = datapackage_meta_from_file(&file_path)?;
+        let id = if let Some((id, _)) = self.buf.get(&file_path) { *id } else { Uuid::new_v4() };
+        self.buf.insert(file_path, (id, dp));
+        Ok(())
+    }
+}
+
+impl DataSource for RootFolderSource {
+    fn list_packages(&self) -> Vec<Uuid> {
+        self.buf.values().map(|(id, _)| *id).collect()
+    }
+
+    fn get_package(&self, id: PackageId) -> Option<crate::datapackage::DataPackage> {
+        self.buf
+            .values()
+            .filter_map(|(inner_id, v)| if inner_id == &id { Some(v) } else { None })
+            .nth(0)
+            .cloned()
+    }
+}
diff --git a/nebula_registry/configuration/local.yaml b/nebula_registry/configuration/local.yaml
index c8061ac..bbe7396 100644
--- a/nebula_registry/configuration/local.yaml
+++ b/nebula_registry/configuration/local.yaml
@@ -1,3 +1,5 @@
 application:
   host: 0.0.0.0 # this ensures connection are accepted by all network devices
-  base_url: http://127.0.0.1
\ No newline at end of file
+  base_url: http://127.0.0.1
+root_folder:
+  path: ./data
diff --git a/nebula_registry/src/main.rs b/nebula_registry/src/main.rs
index b05ca39..8ef1d7c 100644
--- a/nebula_registry/src/main.rs
+++ b/nebula_registry/src/main.rs
@@ -1,8 +1,11 @@
+use std::{path::PathBuf, str::FromStr};
+
 use tonic::transport::Server;
 
 use nebula_common::{
     configuration::registry::get_configuration,
     server::{NebulaPackageQueryMockImpl, NebulaPackageQueryServer},
+    storage::root_folder::RootFolderSource,
 };
 
 #[tokio::main]
@@ -11,7 +14,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let app_conf = config.application;
 
     let addr = format!("{}:{}", app_conf.host, app_conf.port).parse()?;
-    let registry = NebulaPackageQueryMockImpl::default();
+
+    let path = config.root_folder.expect("Root Folder Source expected").path.to_string();
+    let ds = RootFolderSource::new_from_folder(PathBuf::from_str(path.as_str()).unwrap());
+    let registry = NebulaPackageQueryMockImpl::new(ds);
 
     println!("Nebula Registry v0.1.0 - running on: '{}'", addr);
     Server::builder().add_service(NebulaPackageQueryServer::new(registry)).serve(addr).await?;

From 47d9dafe865f832764b923896b275e592275a135 Mon Sep 17 00:00:00 2001
From: Tim Janus <tim@janus.rs>
Date: Sat, 25 Jan 2025 16:05:42 +0100
Subject: [PATCH 3/4] feat: add settings in model layer

---
 nebula_common/proto/nebula.proto              |  4 +-
 nebula_common/src/client/mod.rs               |  2 +-
 nebula_common/src/datapackage/pod.rs          |  1 -
 nebula_common/src/datapackage/validated.rs    |  6 +-
 nebula_common/src/lib.rs                      |  3 +
 nebula_common/src/model/mod.rs                | 62 ++++++++++++
 nebula_common/src/model/pb_mapper.rs          | 98 +++++++++++++++++++
 nebula_common/src/server/endpoints.rs         | 49 +++++++---
 nebula_common/src/server/metadata_fetchers.rs | 69 -------------
 nebula_common/src/server/mod.rs               |  1 -
 nebula_common/src/storage/data_source.rs      | 12 ---
 nebula_common/src/storage/mod.rs              | 33 ++++++-
 nebula_common/src/storage/root_folder.rs      | 39 ++++++--
 nebula_common/src/storage/sql_db.rs           |  3 +
 14 files changed, 272 insertions(+), 110 deletions(-)
 create mode 100644 nebula_common/src/model/mod.rs
 create mode 100644 nebula_common/src/model/pb_mapper.rs
 delete mode 100644 nebula_common/src/server/metadata_fetchers.rs
 delete mode 100644 nebula_common/src/storage/data_source.rs
 create mode 100644 nebula_common/src/storage/sql_db.rs

diff --git a/nebula_common/proto/nebula.proto b/nebula_common/proto/nebula.proto
index 91d8a65..b23fc07 100644
--- a/nebula_common/proto/nebula.proto
+++ b/nebula_common/proto/nebula.proto
@@ -54,11 +54,13 @@ enum SortOption {
     CREATION_DATE   = 0;    // Sort by creation date
     DOWNLOADS       = 1;    // Sort by download count descending
     NAME            = 2;    // Sort Alphabetical by name
+    AUTHOR          = 3;    // Sort Alphabetical by auhtor
 }
 
 message SortParameter {
     SortOption sort_by = 1;         // Sort option
     optional bool descending = 2;   // Sort direction, default: ascending
+    repeated bytes params = 3;        // interpretation based on SortOption, e.g. DateRange for DOWNLOADs  
 }
 
 message FieldOptions {
@@ -70,7 +72,7 @@ message FieldOptions {
 // message returns complete package info
 message PackageRequest {
     string search_query = 1;                // searches for EXACT package-name
-    PackageType package_type = 2;           // filters by dataset, model or both
+    optional PackageType package_type = 2;  // filters by dataset, model or both
 }
 
 message ListPackagesRequest {
diff --git a/nebula_common/src/client/mod.rs b/nebula_common/src/client/mod.rs
index 4a96b9c..e8cea27 100644
--- a/nebula_common/src/client/mod.rs
+++ b/nebula_common/src/client/mod.rs
@@ -35,7 +35,7 @@ pub async fn get_package_info(
     name: String,
 ) -> Result<PackageInfo, Box<dyn std::error::Error>> {
     let request =
-        Request::new(PackageRequest { search_query: name, package_type: PackageType::Both as i32 });
+        Request::new(PackageRequest { search_query: name, package_type: None });
     let response = client.get_package_info(request).await?;
 
     Ok(response.into_inner())
diff --git a/nebula_common/src/datapackage/pod.rs b/nebula_common/src/datapackage/pod.rs
index 8b33160..a6aab0f 100644
--- a/nebula_common/src/datapackage/pod.rs
+++ b/nebula_common/src/datapackage/pod.rs
@@ -63,7 +63,6 @@ pub struct DataPackageNotValidated {
 
     pub delta: Option<DeltaDataPackageNotValidated>,
 }
-
 /// A mapping for the Data Resource json format that is not validated in respect to the schema.
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct DataResourceNotValidated {
diff --git a/nebula_common/src/datapackage/validated.rs b/nebula_common/src/datapackage/validated.rs
index f4735cb..86e7983 100644
--- a/nebula_common/src/datapackage/validated.rs
+++ b/nebula_common/src/datapackage/validated.rs
@@ -36,8 +36,8 @@ pub trait ValidateData {
 
 /// A wrapper typ that marks input data as validated
 #[derive(Debug, Clone, PartialEq)]
-pub struct Validated<T: Sized>(T);
-impl<T: Sized> Validated<T> {
+pub struct Validated<T: Sized + Sync + Send>(T);
+impl<T: Sized + Sync + Send> Validated<T> {
     pub fn into_inner(self) -> T {
         self.0
     }
@@ -48,7 +48,7 @@ impl<T: Sized> Validated<T> {
     }
 }
 
-impl<T> Deref for Validated<T> {
+impl<T: Sized + Sync + Send> Deref for Validated<T> {
     type Target = T;
 
     fn deref(&self) -> &T {
diff --git a/nebula_common/src/lib.rs b/nebula_common/src/lib.rs
index 5ba5f3a..6f6c44b 100644
--- a/nebula_common/src/lib.rs
+++ b/nebula_common/src/lib.rs
@@ -1,6 +1,9 @@
+//! Nebula common library crate with functionality for both registry and cli
+
 pub mod client;
 pub mod configuration;
 pub mod datapackage;
+pub mod model;
 pub mod server;
 pub mod storage;
 
diff --git a/nebula_common/src/model/mod.rs b/nebula_common/src/model/mod.rs
new file mode 100644
index 0000000..22c138e
--- /dev/null
+++ b/nebula_common/src/model/mod.rs
@@ -0,0 +1,62 @@
+//! Contains the model internally used by nebula
+//! 
+//! 
+
+pub mod pb_mapper;
+
+pub type PackageType = super::server::PackageType;
+
+/// Optional MetaData Fields
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum MetaDataField {
+    PreviewImages,
+
+    DataPackage,
+
+    // todo more?
+}
+
+
+/// Settings to select additional (heavy) fields
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct FieldSettings {
+    optional_fields: Vec<MetaDataField>, // todo: on stack? set semantic...
+}
+
+/// Pagation Settings
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct PagationSettings {
+    pub limit: u32,
+
+    pub offset: u32,
+}
+
+impl Default for PagationSettings {
+    fn default() -> Self {
+        Self { limit: 30, offset: Default::default() }
+    }
+}
+
+/// multi level sort settinggs, not implemented
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SortSettings {
+
+}
+
+impl Default for SortSettings {
+    fn default() -> Self {
+        Self {  }
+    }
+}
+
+/// Filter Settings, not implemnted
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct FilterSettings {
+    package_type: PackageType,
+}
+
+impl Default for FilterSettings {
+    fn default() -> Self {
+        Self { package_type: crate::server::PackageType::Both }
+    }
+}
diff --git a/nebula_common/src/model/pb_mapper.rs b/nebula_common/src/model/pb_mapper.rs
new file mode 100644
index 0000000..549d6cf
--- /dev/null
+++ b/nebula_common/src/model/pb_mapper.rs
@@ -0,0 +1,98 @@
+//! Contains functionality to map protobuf related types to nebulas internal model
+
+use crate::{datapackage::DataPackage, server::PackageInfo};
+
+use super::{FilterSettings, PackageType, PagationSettings, SortSettings};
+
+/// Maps self to Pagation Settings
+pub trait PagationMapper {
+    fn as_pagation(&self) -> Result<PagationSettings, Box<dyn std::error::Error>>;
+}
+
+/// Maps self to Filter Settings
+pub trait FilterMapper {
+    fn as_filter(&self) -> Result<FilterSettings, Box<dyn std::error::Error>>;
+    fn into_filter(self) -> Result<FilterSettings, Box<dyn std::error::Error>>;
+}
+
+pub trait SortMapper {
+    fn as_sort(&self) -> Result<SortSettings, Box<dyn std::error::Error>>;
+}
+
+impl PagationMapper for super::super::server::ListPackagesRequest {
+    fn as_pagation(&self) -> Result<PagationSettings, Box<dyn std::error::Error>> {
+        let mut reval = PagationSettings::default();
+        if let Some(limit) = self.limit {reval.limit = limit as u32;}
+        if let Some(offset) = self.offset {reval.offset = offset as u32;}
+        Ok(reval)
+    }
+}
+
+impl PagationMapper for super::super::server::SearchPackagesRequest {
+    fn as_pagation(&self) -> Result<PagationSettings, Box<dyn std::error::Error>> {
+        let mut reval = PagationSettings::default();
+        if let Some(limit) = self.limit {reval.limit = limit as u32;}
+        if let Some(offset) = self.offset {reval.offset = offset as u32;}
+        Ok(reval)
+    }
+}
+
+impl FilterMapper for super::super::server::PackageRequest {
+    fn as_filter(&self) -> Result<FilterSettings, Box<dyn std::error::Error>> {
+        let mut reval = FilterSettings::default();
+        if let Some(pt) = self.package_type {reval.package_type = PackageType::try_from(pt).unwrap();}
+        Ok(reval)
+    }
+
+    fn into_filter(self) -> Result<FilterSettings, Box<dyn std::error::Error>> {
+        self.as_filter()
+    }
+}
+
+impl FilterMapper for super::super::server::SearchPackagesRequest {
+    fn as_filter(&self) -> Result<FilterSettings, Box<dyn std::error::Error>> {
+        Ok(FilterSettings::default())
+    }
+    
+    fn into_filter(self) -> Result<FilterSettings, Box<dyn std::error::Error>> {
+        Ok(FilterSettings::default())
+    }
+    
+}
+
+impl SortMapper for super::super::server::SearchPackagesRequest {
+    fn as_sort(&self) -> Result<SortSettings, Box<dyn std::error::Error>> {
+        Ok(SortSettings::default())
+    }
+}
+
+impl Into<PackageInfo> for DataPackage {
+    fn into(self) -> PackageInfo {
+        let mut inner = self.into_inner();
+        PackageInfo {
+            name: match inner.name.take() {
+                Some(v) => v,
+                None => "No name".to_string(),
+            },
+            version: match inner.version.take() {
+                Some(v) => v,
+                None => "0.1.0".to_string(),
+            },
+            description: match inner.description.take() {
+                Some(v) => v,
+                None => "No Information".to_string(),
+            },
+            license: {
+                let mut reval = String::new();
+                for lic in &inner.licenses {
+                    reval += lic.name.as_str();
+                }
+                if reval.is_empty() {
+                    reval = "UKNOWN".to_string()
+                }
+                reval
+            },
+            ..Default::default()
+        }
+    }
+}
\ No newline at end of file
diff --git a/nebula_common/src/server/endpoints.rs b/nebula_common/src/server/endpoints.rs
index d3b8a25..22b317f 100644
--- a/nebula_common/src/server/endpoints.rs
+++ b/nebula_common/src/server/endpoints.rs
@@ -2,25 +2,28 @@
 //!
 //! An endpoint has to implement an autogenerated trait of grpc.
 
-use crate::server::metadata_fetchers::{fetch_list_packages, fetch_packages};
-use crate::storage::data_source::DataSource;
+use crate::model::pb_mapper::FilterMapper as _;
+use crate::model::pb_mapper::PagationMapper as _;
+use crate::model::{FieldSettings, FilterSettings, SortSettings};
+use crate::storage::MetaDataSource;
 
 use super::nebula_package_query_server::NebulaPackageQuery;
 use super::{ListPackagesRequest, PackageInfo, PackageList, PackageRequest, SearchPackagesRequest};
 
+use tonic::body;
 use tonic::{Request, Response, Status};
 
 #[derive(Debug)]
 pub struct NebulaPackageQueryMockImpl<T>
 where
-    T: DataSource + Send + Sync,
+    T: MetaDataSource + Send + Sync,
 {
     inner_ds: T,
 }
 
 impl<T> NebulaPackageQueryMockImpl<T>
 where
-    T: DataSource + Send + Sync,
+    T: MetaDataSource + Send + Sync,
 {
     pub fn new(ds: T) -> Self {
         Self { inner_ds: ds }
@@ -30,7 +33,7 @@ where
 #[tonic::async_trait]
 impl<T> NebulaPackageQuery for NebulaPackageQueryMockImpl<T>
 where
-    T: DataSource + Send + Sync + 'static,
+    T: MetaDataSource + Send + Sync + 'static,
 {
     async fn get_package_info(
         &self,
@@ -38,14 +41,21 @@ where
     ) -> Result<Response<PackageInfo>, Status> {
         println!("Got a request: {:?}", request);
 
-        match fetch_packages(
-            &self.inner_ds,
-            request.get_ref().search_query.as_str(),
-            request.get_ref().package_type,
-        ) {
-            Ok(body) => Ok(Response::new(body)),
-            Err(_) => Err(Status::internal("Error handling not implemented")),
+        let mut package = self
+            .inner_ds
+            .get_package(&request.get_ref().search_query, request.get_ref().as_filter().unwrap())
+            .await;
+
+
+        let pi: PackageInfo = package.take().unwrap().into();
+        Ok(Response::new(pi))
+
+        /* 
+        match package {
+            Some(package) => Ok(Response::new()),
+            None => Err(Status::internal("Error handling not implemented")),
         }
+        */
     }
 
     async fn list_packages(
@@ -54,8 +64,19 @@ where
     ) -> Result<Response<PackageList>, Status> {
         println!("Got a request: {:?}", request);
 
-        // todo: params from request
-        let body = fetch_list_packages(&self.inner_ds, 0, 30, 0);
+        let pagation = request.get_ref().as_pagation().unwrap();
+        let fields = FieldSettings::default();
+        let filter = FilterSettings::default();
+        let sort = SortSettings::default();
+
+        let body = self.inner_ds.list_packages(sort, filter, pagation, fields).await;
+        let len = body.len();
+        let body = PackageList {
+            packages: body.into_iter().map(|el| el.into()).collect(),
+            total_count: len as i32,
+            limit: None,
+            offset: None,
+        };
 
         Ok(Response::new(body))
     }
diff --git a/nebula_common/src/server/metadata_fetchers.rs b/nebula_common/src/server/metadata_fetchers.rs
deleted file mode 100644
index c73a7b5..0000000
--- a/nebula_common/src/server/metadata_fetchers.rs
+++ /dev/null
@@ -1,69 +0,0 @@
-//! Contains functions that use [crate::storage::DataSource] to fetch data from a storage backend
-
-use super::{PackageInfo, PackageList};
-use crate::{datapackage::DataPackage, storage::data_source::DataSource};
-
-#[derive(Debug, strum_macros::Display)]
-pub enum FetchError {
-    NotFound,
-    Internal,
-}
-
-impl std::error::Error for FetchError {}
-
-pub(super) fn fetch_packages<T>(
-    _ds: &T,
-    _search_query: &str,
-    _package_type: i32,
-) -> Result<PackageInfo, FetchError> {
-    todo!();
-}
-
-fn convert_to_package_info(data: DataPackage) -> Result<PackageInfo, FetchError> {
-    let info = PackageInfo {
-        name: data.name.clone().unwrap(),
-        version: match &data.version {
-            Some(v) => v.clone(),
-            None => "0.1.0".to_string(),
-        },
-        description: match &data.description {
-            Some(v) => v.clone(),
-            None => "No Information".to_string(),
-        },
-        license: {
-            let mut reval = String::new();
-            for lic in &data.licenses {
-                reval += lic.name.as_str();
-            }
-            if reval.is_empty() {
-                reval = "UKNOWN".to_string()
-            }
-            reval
-        },
-        ..Default::default()
-    };
-
-    Ok(info)
-}
-
-pub(super) fn fetch_list_packages<T>(
-    ds: &T,
-    _package_type: i32,
-    _limit: i32,
-    _offset: i32,
-) -> PackageList
-where
-    T: DataSource,
-{
-    let ids = ds.list_packages();
-    let mut packages = Vec::with_capacity(ids.len());
-    for id in ids {
-        if let Some(package) = ds.get_package(id) {
-            packages
-                .push(convert_to_package_info(package).expect("valid packages in data sources"));
-        }
-    }
-
-    let len = packages.len();
-    PackageList { packages, total_count: len as i32, limit: None, offset: None }
-}
diff --git a/nebula_common/src/server/mod.rs b/nebula_common/src/server/mod.rs
index eb198a9..12731c4 100644
--- a/nebula_common/src/server/mod.rs
+++ b/nebula_common/src/server/mod.rs
@@ -6,5 +6,4 @@ pub use super::nebula_proto::nebula_package_query_server::NebulaPackageQueryServ
 pub(crate) use super::nebula_proto::*;
 
 pub mod endpoints;
-pub mod metadata_fetchers;
 pub use endpoints::NebulaPackageQueryMockImpl;
diff --git a/nebula_common/src/storage/data_source.rs b/nebula_common/src/storage/data_source.rs
deleted file mode 100644
index 852fed9..0000000
--- a/nebula_common/src/storage/data_source.rs
+++ /dev/null
@@ -1,12 +0,0 @@
-use uuid::Uuid;
-
-use crate::datapackage::DataPackage;
-
-pub(crate) type PackageId = Uuid;
-
-/// Trait to receive package information from a data source like the filesystem or a database
-pub trait DataSource {
-    fn list_packages(&self) -> Vec<PackageId>;
-
-    fn get_package(&self, id: PackageId) -> Option<DataPackage>;
-}
diff --git a/nebula_common/src/storage/mod.rs b/nebula_common/src/storage/mod.rs
index dfecb9d..5b3dff1 100644
--- a/nebula_common/src/storage/mod.rs
+++ b/nebula_common/src/storage/mod.rs
@@ -1,2 +1,33 @@
-pub mod data_source;
 pub mod root_folder;
+
+use std::future::Future;
+
+use crate::{
+    datapackage::DataPackage,
+    model::{FieldSettings, FilterSettings, PagationSettings, SortSettings},
+};
+
+/// Trait to receive package meta information from a data source like the filesystem or a database
+pub trait MetaDataSource {
+    fn list_packages(
+        &self,
+        sort: SortSettings,
+        filter: FilterSettings,
+        pagation: PagationSettings,
+        fields: FieldSettings,
+    ) -> impl Future<Output = Vec<DataPackage>> + Send;
+
+    fn get_package(
+        &self,
+        query: &str,
+        filter: FilterSettings,
+    ) -> impl Future<Output = Option<DataPackage>> + Send;
+
+    fn search_package(
+        &self,
+        search_query: &str,
+        sort: SortSettings,
+        filter: FilterSettings,
+        pagation: PagationSettings,
+    ) -> impl Future<Output = Vec<DataPackage>> + Send;
+}
diff --git a/nebula_common/src/storage/root_folder.rs b/nebula_common/src/storage/root_folder.rs
index e1dbd2d..acf543a 100644
--- a/nebula_common/src/storage/root_folder.rs
+++ b/nebula_common/src/storage/root_folder.rs
@@ -2,9 +2,12 @@ use std::{collections::HashMap, path::PathBuf};
 
 use uuid::Uuid;
 
-use crate::datapackage::{DataPackage, datapackage_meta_from_file};
+use crate::{
+    datapackage::{DataPackage, datapackage_meta_from_file},
+    model::{FieldSettings, FilterSettings, PagationSettings, SortSettings},
+};
 
-use super::data_source::{DataSource, PackageId};
+use super::MetaDataSource;
 
 /// Reads datapackage.json files from the filesystem
 pub struct RootFolderSource {
@@ -65,16 +68,38 @@ impl RootFolderSource {
     }
 }
 
-impl DataSource for RootFolderSource {
-    fn list_packages(&self) -> Vec<Uuid> {
-        self.buf.values().map(|(id, _)| *id).collect()
+impl MetaDataSource for RootFolderSource {
+    async fn list_packages(
+        &self,
+        _sort: SortSettings,
+        _filter: FilterSettings,
+        _pagation: PagationSettings,
+        _fields: FieldSettings,
+    ) -> Vec<DataPackage> {
+        self.buf.values().map(|(_, v)| v).cloned().collect()
     }
 
-    fn get_package(&self, id: PackageId) -> Option<crate::datapackage::DataPackage> {
+    async fn get_package(
+        &self,
+        query: &str,
+        _filter: FilterSettings,
+    ) -> Option<crate::datapackage::DataPackage> {
         self.buf
             .values()
-            .filter_map(|(inner_id, v)| if inner_id == &id { Some(v) } else { None })
+            .filter_map(|(_, v)| {
+                if v.name.clone().map_or(false, |el| el.contains(query)) { Some(v) } else { None }
+            })
             .nth(0)
             .cloned()
     }
+
+    async fn search_package(
+        &self,
+        _search_query: &str,
+        _sort: SortSettings,
+        _filter: FilterSettings,
+        _pagation: PagationSettings,
+    ) -> Vec<DataPackage> {
+        todo!()
+    }
 }
diff --git a/nebula_common/src/storage/sql_db.rs b/nebula_common/src/storage/sql_db.rs
new file mode 100644
index 0000000..183fbdd
--- /dev/null
+++ b/nebula_common/src/storage/sql_db.rs
@@ -0,0 +1,3 @@
+//! not implemented yet
+//! 
+//! [sqlx](https://github.com/launchbadge/sqlx) with query builder
\ No newline at end of file

From 8e6cf3bc3e4e9f66580679a3f6a8f9dd197d42ae Mon Sep 17 00:00:00 2001
From: Tim Janus <tim@janus.rs>
Date: Sat, 25 Jan 2025 16:08:08 +0100
Subject: [PATCH 4/4] fix: warning

---
 nebula_common/src/server/endpoints.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/nebula_common/src/server/endpoints.rs b/nebula_common/src/server/endpoints.rs
index 22b317f..b3f04b4 100644
--- a/nebula_common/src/server/endpoints.rs
+++ b/nebula_common/src/server/endpoints.rs
@@ -10,7 +10,6 @@ use crate::storage::MetaDataSource;
 use super::nebula_package_query_server::NebulaPackageQuery;
 use super::{ListPackagesRequest, PackageInfo, PackageList, PackageRequest, SearchPackagesRequest};
 
-use tonic::body;
 use tonic::{Request, Response, Status};
 
 #[derive(Debug)]