diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 71dffe7d2..4e1ba75a4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,7 +33,7 @@ jobs: run: cargo build --all-targets --all-features - name: Build no default features run: cargo build --all-targets --no-default-features - - name: Build s3-storage - run: cargo build --all-targets --features s3-storage - - name: Build url-storage - run: cargo build --all-targets --features url-storage + - name: Build aws + run: cargo build --all-targets --features aws + - name: Build url + run: cargo build --all-targets --features url diff --git a/htsget-actix/Cargo.toml b/htsget-actix/Cargo.toml index 1c4239954..4747624dd 100644 --- a/htsget-actix/Cargo.toml +++ b/htsget-actix/Cargo.toml @@ -11,8 +11,8 @@ homepage = "https://github.com/umccr/htsget-rs/blob/main/htsget-actix/README.md" repository = "https://github.com/umccr/htsget-rs" [features] -s3-storage = ["htsget-config/s3-storage", "htsget-search/s3-storage", "htsget-http/s3-storage", "htsget-axum/s3-storage", "htsget-test/s3-storage"] -url-storage = ["htsget-config/url-storage", "htsget-search/url-storage", "htsget-http/url-storage", "htsget-axum/url-storage", "htsget-test/url-storage"] +aws = ["htsget-config/aws", "htsget-search/aws", "htsget-http/aws", "htsget-axum/aws", "htsget-test/aws"] +url = ["htsget-config/url", "htsget-search/url", "htsget-http/url", "htsget-axum/url", "htsget-test/url"] experimental = [ "htsget-config/experimental", "htsget-search/experimental", diff --git a/htsget-actix/README.md b/htsget-actix/README.md index 826c5221e..e8b67f64c 100644 --- a/htsget-actix/README.md +++ b/htsget-actix/README.md @@ -53,8 +53,8 @@ are exposed in the public API. #### Feature flags This crate has the following features: -* `s3-storage`: used to enable `S3` location functionality. -* `url-storage`: used to enable `Url` location functionality. +* `aws`: used to enable `S3` location functionality. +* `url`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## Benchmarks diff --git a/htsget-axum/Cargo.toml b/htsget-axum/Cargo.toml index 2eae66fa5..da998e4b7 100644 --- a/htsget-axum/Cargo.toml +++ b/htsget-axum/Cargo.toml @@ -11,18 +11,18 @@ homepage = "https://github.com/umccr/htsget-rs/blob/main/htsget-axum/README.md" repository = "https://github.com/umccr/htsget-rs" [features] -s3-storage = [ - "htsget-config/s3-storage", - "htsget-search/s3-storage", - "htsget-test/s3-storage", - "htsget-test/aws-mocks", - "htsget-http/s3-storage" +aws = [ + "htsget-config/aws", + "htsget-search/aws", + "htsget-test/aws", + "htsget-test/aws", + "htsget-http/aws" ] -url-storage = [ - "htsget-config/url-storage", - "htsget-search/url-storage", - "htsget-test/url-storage", - "htsget-http/url-storage" +url = [ + "htsget-config/url", + "htsget-search/url", + "htsget-test/url", + "htsget-http/url" ] experimental = [ "htsget-config/experimental", diff --git a/htsget-axum/README.md b/htsget-axum/README.md index 886e3f1d0..cc257e955 100644 --- a/htsget-axum/README.md +++ b/htsget-axum/README.md @@ -42,14 +42,14 @@ This crate uses [htsget-config] for configuration. Using the default configuration, this will start a ticket server on `127.0.0.1:8080` and a data block server on `127.0.0.1:8081` with data accessible from the [`data`][data] directory. This application supports storage backends defined in [htsget-storage]. -To use `S3Storage`, compile with the `s3-storage` feature: +To use `S3Storage`, compile with the `aws` feature: ```sh -cargo run -p htsget-axum --features s3-storage +cargo run -p htsget-axum --features aws ``` This will start a ticket server with `S3Storage` using a bucket called `"data"`. -To use `UrlStorage`, compile with the `url-storage` feature. +To use `UrlStorage`, compile with the `url` feature. See [htsget-search] for details on how to structure files. @@ -178,8 +178,8 @@ htsget-rs. It also contains the data block server which fetches data from a `Loc #### Feature flags This crate has the following features: -* `s3-storage`: used to enable `S3` location functionality. -* `url-storage`: used to enable `Url` location functionality. +* `aws`: used to enable `S3` location functionality and any other AWS features. +* `url`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## License diff --git a/htsget-config/Cargo.toml b/htsget-config/Cargo.toml index c12cf7379..e18f71370 100644 --- a/htsget-config/Cargo.toml +++ b/htsget-config/Cargo.toml @@ -11,8 +11,8 @@ homepage = "https://github.com/umccr/htsget-rs/blob/main/htsget-config/README.md repository = "https://github.com/umccr/htsget-rs" [features] -s3-storage = ["dep:aws-sdk-secretsmanager", "dep:aws-config", "dep:tempfile"] -url-storage = ["dep:reqwest", "dep:cfg-if"] +aws = ["dep:aws-sdk-secretsmanager", "dep:aws-config", "dep:tempfile"] +url = ["dep:reqwest", "dep:cfg-if"] experimental = ["dep:crypt4gh", "dep:tokio", "dep:futures-util"] default = [] diff --git a/htsget-config/README.md b/htsget-config/README.md index 6ed67398b..7c55a8178 100644 --- a/htsget-config/README.md +++ b/htsget-config/README.md @@ -82,7 +82,7 @@ data_server = "None" > [!IMPORTANT] > Some parts of htsget-rs require extra feature flags for conditional compilation, that's why the examples specify -> using `--all-features`. Notably, `--features s3-storage` enables the `S3` location type, and `--features url-storage` +> using `--all-features`. Notably, `--features aws` enables the `S3` location type, and `--features url` > enabled the remote HTTP server location type. If using a subset of features, for example S3 locations only, then > a single feature can be enabled instead of using `--all-features`. @@ -387,7 +387,7 @@ backend.keys.private = "data/c4gh/keys/bob.sec" # pragma: allowlist secret backend.keys.public = "data/c4gh/keys/alice.pub" ``` -Keys can also be retrieved from [AWS Secrets Manager][secrets-manager]. Compile with the `s3-storage` feature flag and specify `keys.kind = "SecretsManager"` under +Keys can also be retrieved from [AWS Secrets Manager][secrets-manager]. Compile with the `aws` feature flag and specify `keys.kind = "SecretsManager"` under `location` to fetch keys from Secrets Manager. When using Secrets Manager, the `private` and `public` correspond to ARNs or secret names in Secrets Manager storing PEM formatted keys. @@ -447,8 +447,8 @@ regex, and changing it by using a substitution string. Advanced configuration op ### Feature flags This crate has the following features: -* `s3-storage`: used to enable `S3` location functionality. -* `url-storage`: used to enable `Url` location functionality. +* `aws`: used to enable `S3` location functionality and any other AWS features. +* `url`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## License diff --git a/htsget-config/examples/config-files/c4gh.toml b/htsget-config/examples/config-files/c4gh.toml index 84645ee16..5d6a3d43d 100644 --- a/htsget-config/examples/config-files/c4gh.toml +++ b/htsget-config/examples/config-files/c4gh.toml @@ -1,5 +1,5 @@ # An example of running htsget-rs with Crypt4GH enabled. -# Run with `cargo run -p htsget-axum --features experimental,s3-storage -- --config htsget-config/examples/config-files/c4gh.toml` +# Run with `cargo run -p htsget-axum --all-features -- --config htsget-config/examples/config-files/c4gh.toml` ticket_server.addr = "127.0.0.1:8080" data_server.addr = "127.0.0.1:8081" diff --git a/htsget-config/examples/config-files/s3_storage.toml b/htsget-config/examples/config-files/s3_storage.toml index 5b7b123ca..adc9ae9f7 100644 --- a/htsget-config/examples/config-files/s3_storage.toml +++ b/htsget-config/examples/config-files/s3_storage.toml @@ -1,5 +1,5 @@ # An example for a server which uses s3 storage with data located in "bucket". -# Run with `cargo run -p htsget-axum --features s3-storage -- --config htsget-config/examples/config-files/s3_storage.toml` +# Run with `cargo run -p htsget-axum --all-features -- --config htsget-config/examples/config-files/s3_storage.toml` ticket_server.cors.allow_headers = "All" ticket_server.cors.allow_methods = "All" diff --git a/htsget-config/examples/config-files/tls_data_server.toml b/htsget-config/examples/config-files/tls_data_server.toml index e7274323c..d39a8768f 100644 --- a/htsget-config/examples/config-files/tls_data_server.toml +++ b/htsget-config/examples/config-files/tls_data_server.toml @@ -1,5 +1,5 @@ # An example config file for a TLS data server that uses a local storage backend. -# Run with `cargo run -p htsget-axum -- --config htsget-config/examples/config-files/tls_data_server.toml` +# Run with `cargo run -p htsget-axum --all-features -- --config htsget-config/examples/config-files/tls_data_server.toml` ticket_server.addr = "0.0.0.0:8080" data_server.addr = "0.0.0.0:8081" diff --git a/htsget-config/examples/config-files/tls_ticket_server.toml b/htsget-config/examples/config-files/tls_ticket_server.toml index d5600d7bc..c634dcc59 100644 --- a/htsget-config/examples/config-files/tls_ticket_server.toml +++ b/htsget-config/examples/config-files/tls_ticket_server.toml @@ -1,5 +1,5 @@ # An example config file for a TLS ticket server that uses S3 as a storage backend. -# Run with `cargo run -p htsget-axum --features s3-storage -- --config htsget-config/examples/config-files/tls_ticket_server.toml` +# Run with `cargo run -p htsget-axum --all-features -- --config htsget-config/examples/config-files/tls_ticket_server.toml` ticket_server.addr = "0.0.0.0:8080" ticket_server.cors_allow_origins = "All" diff --git a/htsget-config/examples/config-files/url_storage.toml b/htsget-config/examples/config-files/url_storage.toml index b25a16ef6..be654d2ec 100644 --- a/htsget-config/examples/config-files/url_storage.toml +++ b/htsget-config/examples/config-files/url_storage.toml @@ -1,6 +1,6 @@ # An example for a server which uses url storage with data located at "http://localhost:3000". # Run with -# `cargo run -p htsget-axum --features url-storage -- --config htsget-config/examples/config-files/url_storage.toml` +# `cargo run -p htsget-axum --all-features -- --config htsget-config/examples/config-files/url_storage.toml` # in the project directory. ticket_server.addr = "127.0.0.1:8082" diff --git a/htsget-config/src/config/advanced/allow_guard.rs b/htsget-config/src/config/advanced/allow_guard.rs index 3394ce938..08cdd57a5 100644 --- a/htsget-config/src/config/advanced/allow_guard.rs +++ b/htsget-config/src/config/advanced/allow_guard.rs @@ -150,7 +150,7 @@ impl QueryAllowed for AllowGuard { mod tests { use super::*; use crate::config::tests::test_serialize_and_deserialize; - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] use crate::config::Config; use crate::types::Class::Header; @@ -265,7 +265,7 @@ mod tests { ); } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[test] fn allow_guard() { test_serialize_and_deserialize( diff --git a/htsget-config/src/config/advanced/mod.rs b/htsget-config/src/config/advanced/mod.rs index 7ab2b0742..ecd3fec35 100644 --- a/htsget-config/src/config/advanced/mod.rs +++ b/htsget-config/src/config/advanced/mod.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; pub mod allow_guard; pub mod cors; pub mod regex_location; -#[cfg(feature = "url-storage")] +#[cfg(feature = "url")] pub mod url; /// Determines which tracing formatting style to use. diff --git a/htsget-config/src/config/advanced/regex_location.rs b/htsget-config/src/config/advanced/regex_location.rs index d8bff741e..69830ef20 100644 --- a/htsget-config/src/config/advanced/regex_location.rs +++ b/htsget-config/src/config/advanced/regex_location.rs @@ -98,7 +98,7 @@ mod tests { ); } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[test] fn regex_location_s3() { test_serialize_and_deserialize( @@ -127,7 +127,7 @@ mod tests { ); } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] #[test] fn regex_location_url() { test_serialize_and_deserialize( @@ -158,7 +158,7 @@ mod tests { ); } - #[cfg(all(feature = "url-storage", feature = "s3-storage"))] + #[cfg(all(feature = "url", feature = "aws"))] #[test] fn regex_location_multiple() { test_serialize_and_deserialize( diff --git a/htsget-config/src/config/location.rs b/htsget-config/src/config/location.rs index 7ca847f41..095ed864e 100644 --- a/htsget-config/src/config/location.rs +++ b/htsget-config/src/config/location.rs @@ -10,7 +10,7 @@ use crate::types::Scheme; use serde::de::Error; use serde::{Deserialize, Deserializer, Serialize}; use std::result; -#[cfg(feature = "url-storage")] +#[cfg(feature = "url")] use {crate::config::advanced::url::Url, crate::error, http::Uri}; /// The locations of data. @@ -214,7 +214,7 @@ impl<'de> Deserialize<'de> for StringLocation { }); } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] if let Some(s) = s.strip_prefix("s3://") { let (bucket, prefix) = split(s)?; return Ok(StringLocation { @@ -223,7 +223,7 @@ impl<'de> Deserialize<'de> for StringLocation { }); } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] if let Some(s_stripped) = s .strip_prefix("http://") .or_else(|| s.strip_prefix("https://")) @@ -330,7 +330,7 @@ mod tests { ); } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[test] fn location_s3() { test_serialize_and_deserialize( @@ -364,7 +364,7 @@ mod tests { ); } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] #[test] fn location_url() { test_serialize_and_deserialize( diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index 087bb4ca0..7b12bf5a4 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -221,7 +221,7 @@ pub(crate) mod tests { use crate::types::Scheme; use figment::Jail; use http::uri::Authority; - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] use http::Uri; use serde::de::DeserializeOwned; use serde_json::json; @@ -572,7 +572,7 @@ pub(crate) mod tests { ); } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[test] fn simple_locations_s3() { test_config_from_file( @@ -590,7 +590,7 @@ pub(crate) mod tests { ); } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] #[test] fn simple_locations_url() { test_config_from_file( @@ -621,7 +621,7 @@ pub(crate) mod tests { ); } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[test] fn simple_locations_multiple_mixed() { test_config_from_file( diff --git a/htsget-config/src/resolver.rs b/htsget-config/src/resolver.rs index 4638814ca..625f8d9cb 100644 --- a/htsget-config/src/resolver.rs +++ b/htsget-config/src/resolver.rs @@ -22,11 +22,11 @@ pub trait ResolveResponse { async fn from_file(file_storage: &storage::file::File, query: &Query) -> Result; /// Convert from `S3`. - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] async fn from_s3(s3_storage: &storage::s3::S3, query: &Query) -> Result; /// Convert from `Url`. - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] async fn from_url(url_storage: &storage::url::Url, query: &Query) -> Result; } @@ -116,7 +116,7 @@ impl StorageResolver for LocationEither { match self.backend() { Backend::File(file) => Some(T::from_file(file, query).await), - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] Backend::S3(s3) => { let s3 = if let Self::Regex(regex_location) = self { if s3.bucket().is_empty() { @@ -136,7 +136,7 @@ impl StorageResolver for LocationEither { Some(T::from_s3(s3, query).await) } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] Backend::Url(url_storage) => Some(T::from_url(url_storage, query).await), } } @@ -194,9 +194,9 @@ mod tests { use crate::types::Scheme::Http; use crate::types::Url; use http::uri::Authority; - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] use reqwest::ClientBuilder; - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] use { crate::config::advanced::allow_guard::{AllowGuard, ReferenceNames}, crate::types::{Class, Fields, Interval, Tags}, @@ -214,7 +214,7 @@ mod tests { )) } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] async fn from_s3(s3_storage: &storage::s3::S3, query: &Query) -> Result { Ok(Response::new( Bam, @@ -222,7 +222,7 @@ mod tests { )) } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] async fn from_url(url: &storage::url::Url, query: &Query) -> Result { Ok(Response::new( Bam, @@ -257,7 +257,7 @@ mod tests { expected_resolved_request(vec![location.into()], "127.0.0.1:8080/id-1").await; } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[tokio::test] async fn resolver_resolve_s3_request_tagged() { let s3_storage = storage::s3::S3::new("id2".to_string(), None, false); @@ -273,7 +273,7 @@ mod tests { expected_resolved_request(vec![location.into()], "id2/id-1").await; } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[tokio::test] async fn resolver_resolve_s3_request() { let regex_location = RegexLocation::new( @@ -299,7 +299,7 @@ mod tests { expected_resolved_request(vec![location.into()], "bucket/id-1").await; } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] #[tokio::test] async fn resolver_resolve_url_request() { let client = ClientBuilder::new().build().unwrap(); @@ -425,7 +425,7 @@ mod tests { }); } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[test] fn config_resolvers_all_options_env() { test_config_from_env( diff --git a/htsget-config/src/storage/c4gh/local.rs b/htsget-config/src/storage/c4gh/local.rs index 696abc8b7..0aba877de 100644 --- a/htsget-config/src/storage/c4gh/local.rs +++ b/htsget-config/src/storage/c4gh/local.rs @@ -98,7 +98,7 @@ mod tests { }); } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[tokio::test] async fn config_s3_storage_c4gh() { test_c4gh_storage_config( @@ -115,7 +115,7 @@ mod tests { ); } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] #[tokio::test] async fn config_url_storage_c4gh() { test_c4gh_storage_config( diff --git a/htsget-config/src/storage/c4gh/mod.rs b/htsget-config/src/storage/c4gh/mod.rs index 1ef593293..8c688d7ae 100644 --- a/htsget-config/src/storage/c4gh/mod.rs +++ b/htsget-config/src/storage/c4gh/mod.rs @@ -4,7 +4,7 @@ use crate::error::Error::{IoError, ParseError}; use crate::error::{Error, Result}; use crate::storage::c4gh::local::C4GHLocal; -#[cfg(feature = "s3-storage")] +#[cfg(feature = "aws")] use crate::storage::c4gh::secrets_manager::C4GHSecretsManager; use crypt4gh::error::Crypt4GHError; use futures_util::future::{BoxFuture, Shared}; @@ -14,7 +14,7 @@ use tokio::task::{JoinError, JoinHandle}; pub mod local; -#[cfg(feature = "s3-storage")] +#[cfg(feature = "aws")] pub mod secrets_manager; /// Config for Crypt4GH keys. @@ -66,7 +66,7 @@ impl TryFrom for C4GHKeys { fn try_from(location: C4GHKeyLocation) -> Result { match location { C4GHKeyLocation::File(file) => file.try_into(), - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] C4GHKeyLocation::SecretsManager(secrets_manager) => secrets_manager.try_into(), } } @@ -79,7 +79,7 @@ impl TryFrom for C4GHKeys { pub enum C4GHKeyLocation { #[serde(alias = "file", alias = "FILE")] File(C4GHLocal), - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[serde(alias = "secretsmanager", alias = "SECRETSMANAGER")] SecretsManager(C4GHSecretsManager), } diff --git a/htsget-config/src/storage/mod.rs b/htsget-config/src/storage/mod.rs index b8dba455b..7e33e9b9a 100644 --- a/htsget-config/src/storage/mod.rs +++ b/htsget-config/src/storage/mod.rs @@ -1,22 +1,22 @@ //! Storage backends. //! -#[cfg(any(feature = "url-storage", feature = "s3-storage"))] +#[cfg(any(feature = "url", feature = "aws"))] use crate::error::Error; use crate::error::Result; use crate::storage::file::File; -#[cfg(feature = "s3-storage")] +#[cfg(feature = "aws")] use crate::storage::s3::S3; -#[cfg(feature = "url-storage")] +#[cfg(feature = "url")] use crate::storage::url::Url; use serde::{Deserialize, Serialize}; #[cfg(feature = "experimental")] pub mod c4gh; pub mod file; -#[cfg(feature = "s3-storage")] +#[cfg(feature = "aws")] pub mod s3; -#[cfg(feature = "url-storage")] +#[cfg(feature = "url")] pub mod url; /// A new type representing a resolved id. @@ -42,10 +42,10 @@ impl ResolvedId { pub enum Backend { #[serde(alias = "file", alias = "FILE")] File(File), - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[serde(alias = "s3")] S3(S3), - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] #[serde(alias = "url", alias = "URL")] Url(Url), } @@ -55,15 +55,15 @@ impl Backend { pub fn as_file(&self) -> Result<&File> { match self { Backend::File(file) => Ok(file), - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] Backend::S3(_) => Err(Error::ParseError("not a `File` variant".to_string())), - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] Backend::Url(_) => Err(Error::ParseError("not a `File` variant".to_string())), } } /// Get the file variant and error if it is not `S3`. - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] pub fn as_s3(&self) -> Result<&S3> { if let Backend::S3(s3) = self { Ok(s3) @@ -73,7 +73,7 @@ impl Backend { } /// Get the url variant and error if it is not `Url`. - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] pub fn as_url(&self) -> Result<&Url> { if let Backend::Url(url) = self { Ok(url) @@ -124,7 +124,7 @@ pub(crate) mod tests { ); } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[test] fn config_storage_tagged_s3_file() { test_config_from_file( @@ -142,7 +142,7 @@ pub(crate) mod tests { ); } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[test] fn config_storage_tagged_s3_env() { test_config_from_env( diff --git a/htsget-config/src/tls/mod.rs b/htsget-config/src/tls/mod.rs index a65d54859..3470b73af 100644 --- a/htsget-config/src/tls/mod.rs +++ b/htsget-config/src/tls/mod.rs @@ -1,7 +1,7 @@ //! Configuration related to TLS. //! -#[cfg(feature = "url-storage")] +#[cfg(feature = "url")] pub mod client; use std::fs::File; diff --git a/htsget-http/Cargo.toml b/htsget-http/Cargo.toml index 6603f12f3..20f834ca6 100644 --- a/htsget-http/Cargo.toml +++ b/htsget-http/Cargo.toml @@ -11,8 +11,8 @@ homepage = "https://github.com/umccr/htsget-rs/blob/main/htsget-http/README.md" repository = "https://github.com/umccr/htsget-rs" [features] -s3-storage = ["htsget-config/s3-storage", "htsget-search/s3-storage", "htsget-test/s3-storage"] -url-storage = ["htsget-config/url-storage", "htsget-search/url-storage", "htsget-test/url-storage"] +aws = ["htsget-config/aws", "htsget-search/aws", "htsget-test/aws"] +url = ["htsget-config/url", "htsget-search/url", "htsget-test/url"] experimental = ["htsget-config/experimental", "htsget-search/experimental", "htsget-test/experimental"] default = [] diff --git a/htsget-http/README.md b/htsget-http/README.md index a3cc059de..1fdc6c995 100644 --- a/htsget-http/README.md +++ b/htsget-http/README.md @@ -35,8 +35,8 @@ and process it using [htsget-search] to return JSON HTTP responses. #### Feature flags This crate has the following features: -* `s3-storage`: used to enable `S3` location functionality. -* `url-storage`: used to enable `Url` location functionality. +* `aws`: used to enable `S3` location functionality and any other AWS features. +* `url`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. [warp]: https://github.com/seanmonstar/warp diff --git a/htsget-lambda/Cargo.toml b/htsget-lambda/Cargo.toml index 8f3b479ab..965d3e609 100644 --- a/htsget-lambda/Cargo.toml +++ b/htsget-lambda/Cargo.toml @@ -11,8 +11,8 @@ homepage = "https://github.com/umccr/htsget-rs/blob/main/htsget-lambda/README.md repository = "https://github.com/umccr/htsget-rs" [features] -s3-storage = ["htsget-axum/s3-storage", "htsget-config/s3-storage", "htsget-search/s3-storage", "htsget-http/s3-storage", "htsget-test/s3-storage"] -url-storage = ["htsget-axum/url-storage", "htsget-config/url-storage", "htsget-search/url-storage", "htsget-http/url-storage", "htsget-test/url-storage"] +aws = ["htsget-axum/aws", "htsget-config/aws", "htsget-search/aws", "htsget-http/aws", "htsget-test/aws"] +url = ["htsget-axum/url", "htsget-config/url", "htsget-search/url", "htsget-http/url", "htsget-test/url"] experimental = [ "htsget-axum/experimental", "htsget-config/experimental", diff --git a/htsget-lambda/README.md b/htsget-lambda/README.md index d2322781b..7775dc1c7 100644 --- a/htsget-lambda/README.md +++ b/htsget-lambda/README.md @@ -42,7 +42,7 @@ library code, and it instead uses `htsget-axum`. Please use that crate for funct #### Feature flags This crate has the following features: -* `s3`: used to enable `S3` location functionality. +* `s3`: used to enable `S3` location functionality and any other AWS features. * `url`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. diff --git a/htsget-search/Cargo.toml b/htsget-search/Cargo.toml index 9388ef061..3db0f4c2e 100644 --- a/htsget-search/Cargo.toml +++ b/htsget-search/Cargo.toml @@ -11,16 +11,16 @@ homepage = "https://github.com/umccr/htsget-rs/blob/main/htsget-search/README.md repository = "https://github.com/umccr/htsget-rs" [features] -s3-storage = [ - "htsget-storage/s3-storage", - "htsget-config/s3-storage", - "htsget-test/s3-storage", - "htsget-test/aws-mocks" +aws = [ + "htsget-storage/aws", + "htsget-config/aws", + "htsget-test/aws", + "htsget-test/aws" ] -url-storage = [ - "htsget-storage/url-storage", - "htsget-config/url-storage", - "htsget-test/url-storage" +url = [ + "htsget-storage/url", + "htsget-config/url", + "htsget-test/url" ] experimental = [ "htsget-storage/experimental", diff --git a/htsget-search/README.md b/htsget-search/README.md index 27149f166..5ce4b52ba 100644 --- a/htsget-search/README.md +++ b/htsget-search/README.md @@ -53,8 +53,8 @@ used to process requests. #### Feature flags This crate has the following features: -* `s3-storage`: used to enable `S3` location functionality. -* `url-storage`: used to enable `Url` location functionality. +* `aws`: used to enable `S3` location functionality and any other AWS features. +* `url`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## Minimising Byte Ranges diff --git a/htsget-search/src/bam_search.rs b/htsget-search/src/bam_search.rs index 4d80587aa..b2d76679f 100644 --- a/htsget-search/src/bam_search.rs +++ b/htsget-search/src/bam_search.rs @@ -147,7 +147,7 @@ impl BamSearch { #[cfg(test)] pub(crate) mod tests { use super::*; - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] use crate::from_storage::tests::with_aws_storage_fn; use crate::from_storage::tests::with_local_storage_fn; use crate::{Class::Body, Class::Header, Headers, HtsGetError::NotFound, Response, Url}; @@ -535,7 +535,7 @@ pub(crate) mod tests { .await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[tokio::test] async fn search_non_existent_id_reference_name_aws() { with_aws_storage_fn( @@ -553,7 +553,7 @@ pub(crate) mod tests { .await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[tokio::test] async fn search_non_existent_id_all_reads_aws() { with_aws_storage_fn( @@ -572,7 +572,7 @@ pub(crate) mod tests { .await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[tokio::test] async fn search_non_existent_id_header_aws() { with_aws_storage_fn( diff --git a/htsget-search/src/bcf_search.rs b/htsget-search/src/bcf_search.rs index d44318393..f435f9137 100644 --- a/htsget-search/src/bcf_search.rs +++ b/htsget-search/src/bcf_search.rs @@ -113,7 +113,7 @@ mod tests { use std::future::Future; use super::*; - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] use crate::from_storage::tests::with_aws_storage_fn; use crate::from_storage::tests::with_local_storage_fn; use crate::search::SearchAll; @@ -328,7 +328,7 @@ mod tests { .await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[tokio::test] async fn search_non_existent_id_reference_name_aws() { with_aws_storage_fn( @@ -346,7 +346,7 @@ mod tests { .await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[tokio::test] async fn search_non_existent_id_all_reads_aws() { with_aws_storage_fn( @@ -365,7 +365,7 @@ mod tests { .await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[tokio::test] async fn search_non_existent_id_header_aws() { with_aws_storage_fn( diff --git a/htsget-search/src/cram_search.rs b/htsget-search/src/cram_search.rs index 861210854..4a6698d00 100644 --- a/htsget-search/src/cram_search.rs +++ b/htsget-search/src/cram_search.rs @@ -277,7 +277,7 @@ mod tests { use htsget_test::http::concat::ConcatResponse; use super::*; - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] use crate::from_storage::tests::with_aws_storage_fn; use crate::from_storage::tests::with_local_storage_fn; use crate::{Class::Header, Headers, HtsGetError::NotFound, Response, Url}; @@ -542,7 +542,7 @@ mod tests { .await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[tokio::test] async fn search_non_existent_id_reference_name_aws() { with_aws_storage_fn( @@ -560,7 +560,7 @@ mod tests { .await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[tokio::test] async fn search_non_existent_id_all_reads_aws() { with_aws_storage_fn( @@ -579,7 +579,7 @@ mod tests { .await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[tokio::test] async fn search_non_existent_id_header_aws() { with_aws_storage_fn( diff --git a/htsget-search/src/from_storage.rs b/htsget-search/src/from_storage.rs index ca72580b8..c8082f3da 100644 --- a/htsget-search/src/from_storage.rs +++ b/htsget-search/src/from_storage.rs @@ -56,14 +56,14 @@ impl ResolveResponse for HtsGetFromStorage { searcher.search(query.clone()).await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] async fn from_s3(s3_storage: &storage::s3::S3, query: &Query) -> Result { let storage = Storage::from_s3(s3_storage).await; let searcher = HtsGetFromStorage::new(storage?); searcher.search(query.clone()).await } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] async fn from_url(url_storage_config: &storage::url::Url, query: &Query) -> Result { let storage = Storage::from_url(url_storage_config).await; let searcher = HtsGetFromStorage::new(storage?); @@ -90,7 +90,7 @@ pub(crate) mod tests { use std::fs; use std::future::Future; use std::path::{Path, PathBuf}; - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] use { htsget_storage::s3::S3Storage, htsget_test::aws_mocks::with_s3_test_server, std::fs::create_dir, }; @@ -318,7 +318,7 @@ pub(crate) mod tests { .await; } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] pub(crate) async fn with_aws_storage_fn(test: F, path: &str, copy_files: &[&str]) where F: FnOnce(Storage) -> Fut, @@ -341,7 +341,7 @@ pub(crate) mod tests { .await; } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] pub(crate) async fn with_aws_s3_storage_fn(test: F, folder_name: String, base_path: &Path) where F: FnOnce(Storage) -> Fut, diff --git a/htsget-search/src/vcf_search.rs b/htsget-search/src/vcf_search.rs index d83209e15..9d331f9f6 100644 --- a/htsget-search/src/vcf_search.rs +++ b/htsget-search/src/vcf_search.rs @@ -124,7 +124,7 @@ pub(crate) mod tests { use std::future::Future; use super::*; - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] use crate::from_storage::tests::with_aws_storage_fn; use crate::from_storage::tests::with_local_storage_fn; use crate::search::SearchAll; @@ -339,7 +339,7 @@ pub(crate) mod tests { .await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[tokio::test] async fn search_non_existent_id_reference_name_aws() { with_aws_storage_fn( @@ -357,7 +357,7 @@ pub(crate) mod tests { .await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[tokio::test] async fn search_non_existent_id_all_reads_aws() { with_aws_storage_fn( @@ -376,7 +376,7 @@ pub(crate) mod tests { .await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[tokio::test] async fn search_non_existent_id_header_aws() { with_aws_storage_fn( diff --git a/htsget-storage/Cargo.toml b/htsget-storage/Cargo.toml index a9d914bcc..fd68f30b9 100644 --- a/htsget-storage/Cargo.toml +++ b/htsget-storage/Cargo.toml @@ -11,19 +11,19 @@ homepage = "https://github.com/umccr/htsget-rs/blob/main/htsget-storage/README.m repository = "https://github.com/umccr/htsget-rs" [features] -s3-storage = [ +aws = [ "dep:bytes", "dep:aws-sdk-s3", "dep:aws-config", - "htsget-config/s3-storage", - "htsget-test/s3-storage", - "htsget-test/aws-mocks" + "htsget-config/aws", + "htsget-test/aws", + "htsget-test/aws" ] -url-storage = [ +url = [ "dep:bytes", "dep:reqwest", - "htsget-config/url-storage", - "htsget-test/url-storage" + "htsget-config/url", + "htsget-test/url" ] experimental = ["dep:crypt4gh", "dep:bincode", "htsget-config/experimental", "htsget-test/experimental"] default = [] diff --git a/htsget-storage/README.md b/htsget-storage/README.md index 5e035db79..b3b58e12b 100644 --- a/htsget-storage/README.md +++ b/htsget-storage/README.md @@ -46,8 +46,8 @@ and [url] modules implement the `Storage` functionality. #### Feature flags This crate has the following features: -* `s3-storage`: used to enable `S3` location functionality. -* `url-storage`: used to enable `Url` location functionality. +* `aws`: used to enable `S3` location functionality and any other AWS features. +* `url`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. [local]: src/local.rs diff --git a/htsget-storage/src/c4gh/storage.rs b/htsget-storage/src/c4gh/storage.rs index d450425b3..aad9d060f 100644 --- a/htsget-storage/src/c4gh/storage.rs +++ b/htsget-storage/src/c4gh/storage.rs @@ -315,9 +315,9 @@ impl From for StorageError { mod tests { use super::*; use crate::local::tests::with_local_storage; - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] use crate::s3::tests::with_aws_s3_storage; - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] use crate::url::tests::{test_headers, with_url_test_server}; use htsget_config::types::Headers; use htsget_test::c4gh::{encrypt_data, get_decryption_keys}; @@ -335,7 +335,7 @@ mod tests { .await; } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[tokio::test] async fn test_preprocess_s3_storage() { with_s3_c4gh_storage(|mut storage| async move { @@ -344,7 +344,7 @@ mod tests { .await; } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] #[tokio::test] async fn test_preprocess_url_storage() { with_url_c4gh_storage(|mut storage, _| async move { @@ -363,7 +363,7 @@ mod tests { .await; } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[tokio::test] async fn test_get_s3_storage() { with_s3_c4gh_storage(|mut storage| async move { @@ -372,7 +372,7 @@ mod tests { .await; } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] #[tokio::test] async fn test_get_url_storage() { with_url_c4gh_storage(|mut storage, _| async move { @@ -391,7 +391,7 @@ mod tests { .await; } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[tokio::test] async fn test_head_s3_storage() { with_s3_c4gh_storage(|mut storage| async move { @@ -400,7 +400,7 @@ mod tests { .await; } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] #[tokio::test] async fn test_head_url_storage() { with_url_c4gh_storage(|mut storage, _| async move { @@ -419,7 +419,7 @@ mod tests { .await; } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[tokio::test] async fn test_postprocess_s3_storage() { with_s3_c4gh_storage(|mut storage| async move { @@ -428,7 +428,7 @@ mod tests { .await; } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] #[tokio::test] async fn test_postprocess_url_storage() { with_url_c4gh_storage(|mut storage, _| async move { @@ -453,7 +453,7 @@ mod tests { .await; } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[tokio::test] async fn test_range_s3_storage() { with_s3_c4gh_storage(|mut storage| async move { @@ -468,7 +468,7 @@ mod tests { .await; } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] #[tokio::test] async fn test_range_url_storage() { with_url_c4gh_storage(|mut storage, url| async move { @@ -608,7 +608,7 @@ mod tests { .await; } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] pub(crate) async fn with_s3_c4gh_storage(test: F) where F: FnOnce(C4GHStorage) -> Fut, @@ -621,7 +621,7 @@ mod tests { .await; } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] pub(crate) async fn with_url_c4gh_storage(test: F) where F: FnOnce(C4GHStorage, String) -> Fut, diff --git a/htsget-storage/src/error.rs b/htsget-storage/src/error.rs index 2f02d7884..c526f5bc8 100644 --- a/htsget-storage/src/error.rs +++ b/htsget-storage/src/error.rs @@ -40,7 +40,7 @@ pub enum StorageError { #[error("response error: {0}")] ResponseError(String), - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] #[error("aws error: {0}, with key: `{1}`")] AwsS3Error(String, String), @@ -60,7 +60,7 @@ impl From for HtsGetError { | StorageError::InvalidUri(_) | StorageError::InvalidAddress(_) | StorageError::InternalError(_)) => Self::InternalError(err.to_string()), - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] err @ StorageError::AwsS3Error(_, _) => Self::IoError(err.to_string()), err @ StorageError::UrlParseError(_) => Self::ParseError(err.to_string()), } diff --git a/htsget-storage/src/lib.rs b/htsget-storage/src/lib.rs index bb985df83..124065f78 100644 --- a/htsget-storage/src/lib.rs +++ b/htsget-storage/src/lib.rs @@ -12,10 +12,10 @@ use crate::error::Result; use crate::error::StorageError; use crate::error::StorageError::InvalidKey; use crate::local::FileStorage; -#[cfg(feature = "s3-storage")] +#[cfg(feature = "aws")] use crate::s3::S3Storage; use crate::types::{BytesPositionOptions, DataBlock, GetOptions, HeadOptions, RangeUrlOptions}; -#[cfg(feature = "url-storage")] +#[cfg(feature = "url")] use crate::url::UrlStorage; use async_trait::async_trait; use base64::engine::general_purpose; @@ -38,10 +38,10 @@ use tokio::io::{AsyncRead, ReadBuf}; pub mod c4gh; pub mod error; pub mod local; -#[cfg(feature = "s3-storage")] +#[cfg(feature = "aws")] pub mod s3; pub mod types; -#[cfg(feature = "url-storage")] +#[cfg(feature = "url")] pub mod url; pin_project! { @@ -163,7 +163,7 @@ impl Storage { } /// Create from s3 config. - #[cfg(feature = "s3-storage")] + #[cfg(feature = "aws")] pub async fn from_s3(s3: &storage::s3::S3) -> Result { let storage = Storage::new( S3Storage::new_with_default_config( @@ -184,7 +184,7 @@ impl Storage { } /// Create from url config. - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] pub async fn from_url(url: &storage::url::Url) -> Result { let storage = Storage::new(UrlStorage::new( url.client_cloned(), diff --git a/htsget-test/Cargo.toml b/htsget-test/Cargo.toml index 8f3dffe75..b54bc5417 100644 --- a/htsget-test/Cargo.toml +++ b/htsget-test/Cargo.toml @@ -24,18 +24,17 @@ http = [ "dep:mime", "dep:base64" ] -aws-mocks = [ - "s3-storage", +aws = [ "dep:tempfile", "dep:aws-sdk-s3", "dep:aws-config", "dep:aws-credential-types", "dep:s3s", "dep:s3s-fs", - "dep:s3s-aws" + "dep:s3s-aws", + "htsget-config?/aws" ] -s3-storage = ["htsget-config?/s3-storage"] -url-storage = ["htsget-config?/url-storage"] +url = ["htsget-config?/url"] experimental = ["dep:crypt4gh", "dep:htsget-config", "htsget-config/experimental"] default = [] diff --git a/htsget-test/README.md b/htsget-test/README.md index b3503ae8a..91274094c 100644 --- a/htsget-test/README.md +++ b/htsget-test/README.md @@ -33,9 +33,8 @@ This library is intended to be used as a [development dependency][dev-dependenci This crate has the following features: * `http`: used to enable common functionality for HTTP tests. -* `aws-mocks`: used to enable AWS mocking for tests. -* `s3-storage`: used to enable `S3` location functionality. -* `url-storage`: used to enable `Url` location functionality. +* `aws`: used to enable AWS mocking for tests, `S3` location functionality and any other AWS features. +* `url`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. [dev-dependencies]: https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#development-dependencies diff --git a/htsget-test/src/lib.rs b/htsget-test/src/lib.rs index 817ad8efe..e12b2febe 100644 --- a/htsget-test/src/lib.rs +++ b/htsget-test/src/lib.rs @@ -1,4 +1,4 @@ -#[cfg(feature = "aws-mocks")] +#[cfg(feature = "aws")] pub mod aws_mocks; #[cfg(feature = "experimental")] pub mod c4gh;