feat: podman support (apache#489)

* feat: improve docker/podman get OsArch compatibility * fix: replace deprecated 'links' keyword in docker compose * refactor: use IpAddr and SocketAddr when able * docs: add podman documentation * address PR documentation comments * address pr comments on tests * Address pr comments, properly handle result with match * address pr comments, refactor get_cmd_output * move podman instr to docs/contributing * address pr comment, show detailed err msg --------- Co-authored-by: Alex Yin <[email protected]>
matanolabs · Dec 9, 2024 · 061d68f · 061d68f
1 parent 544e504
commit 061d68f
Show file tree

Hide file tree

Showing 12 changed files with 191 additions and 67 deletions.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -110,14 +110,16 @@ $ cargo version
 cargo 1.69.0 (6e9a83356 2023-04-12)
 ```
 
-#### Install docker
+#### Install Docker or Podman
 
-Currently, iceberg-rust uses docker to set up environment for integration tests.
+Currently, iceberg-rust uses Docker to set up environment for integration tests. Podman is also supported.
 
-You can learn how to install docker from [here](https://docs.docker.com/get-docker/).
+You can learn how to install Docker from [here](https://docs.docker.com/get-docker/).
 
 For macos users, you can install [OrbStack](https://orbstack.dev/) as a docker alternative.
 
+For podman users, refer to [Using Podman instead of Docker](docs/contributing/podman.md)
+
 ## Build
 
 * To compile the project: `make build`

diff --git a/crates/catalog/glue/tests/glue_catalog_test.rs b/crates/catalog/glue/tests/glue_catalog_test.rs
@@ -18,6 +18,7 @@
 //! Integration tests for glue catalog.
 
 use std::collections::HashMap;
+use std::net::SocketAddr;
 use std::sync::RwLock;
 
 use ctor::{ctor, dtor};
@@ -64,14 +65,11 @@ async fn get_catalog() -> GlueCatalog {
             docker_compose.get_container_ip("minio"),
         )
     };
-    let read_port = format!("{}:{}", glue_catalog_ip, GLUE_CATALOG_PORT);
-    loop {
-        if !scan_port_addr(&read_port) {
-            log::info!("Waiting for 1s glue catalog to ready...");
-            sleep(std::time::Duration::from_millis(1000)).await;
-        } else {
-            break;
-        }
+    let glue_socket_addr = SocketAddr::new(glue_catalog_ip, GLUE_CATALOG_PORT);
+    let minio_socket_addr = SocketAddr::new(minio_ip, MINIO_PORT);
+    while !scan_port_addr(glue_socket_addr) {
+        log::info!("Waiting for 1s glue catalog to ready...");
+        sleep(std::time::Duration::from_millis(1000)).await;
     }
 
     let props = HashMap::from([
@@ -83,15 +81,15 @@ async fn get_catalog() -> GlueCatalog {
         (AWS_REGION_NAME.to_string(), "us-east-1".to_string()),
         (
             S3_ENDPOINT.to_string(),
-            format!("http://{}:{}", minio_ip, MINIO_PORT),
+            format!("http://{}", minio_socket_addr),
         ),
         (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()),
         (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()),
         (S3_REGION.to_string(), "us-east-1".to_string()),
     ]);
 
     let config = GlueCatalogConfig::builder()
-        .uri(format!("http://{}:{}", glue_catalog_ip, GLUE_CATALOG_PORT))
+        .uri(format!("http://{}", glue_socket_addr))
         .warehouse("s3a://warehouse/hive".to_string())
         .props(props.clone())
         .build();

diff --git a/crates/catalog/hms/tests/hms_catalog_test.rs b/crates/catalog/hms/tests/hms_catalog_test.rs
@@ -18,6 +18,7 @@
 //! Integration tests for hms catalog.
 
 use std::collections::HashMap;
+use std::net::SocketAddr;
 use std::sync::RwLock;
 
 use ctor::{ctor, dtor};
@@ -63,30 +64,26 @@ async fn get_catalog() -> HmsCatalog {
             docker_compose.get_container_ip("minio"),
         )
     };
-
-    let read_port = format!("{}:{}", hms_catalog_ip, HMS_CATALOG_PORT);
-    loop {
-        if !scan_port_addr(&read_port) {
-            log::info!("scan read_port {} check", read_port);
-            log::info!("Waiting for 1s hms catalog to ready...");
-            sleep(std::time::Duration::from_millis(1000)).await;
-        } else {
-            break;
-        }
+    let hms_socket_addr = SocketAddr::new(hms_catalog_ip, HMS_CATALOG_PORT);
+    let minio_socket_addr = SocketAddr::new(minio_ip, MINIO_PORT);
+    while !scan_port_addr(hms_socket_addr) {
+        log::info!("scan hms_socket_addr {} check", hms_socket_addr);
+        log::info!("Waiting for 1s hms catalog to ready...");
+        sleep(std::time::Duration::from_millis(1000)).await;
     }
 
     let props = HashMap::from([
         (
             S3_ENDPOINT.to_string(),
-            format!("http://{}:{}", minio_ip, MINIO_PORT),
+            format!("http://{}", minio_socket_addr),
         ),
         (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()),
         (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()),
         (S3_REGION.to_string(), "us-east-1".to_string()),
     ]);
 
     let config = HmsCatalogConfig::builder()
-        .address(format!("{}:{}", hms_catalog_ip, HMS_CATALOG_PORT))
+        .address(hms_socket_addr.to_string())
         .thrift_transport(HmsThriftTransport::Buffered)
         .warehouse("s3a://warehouse/hive".to_string())
         .props(props)

diff --git a/crates/catalog/rest/testdata/rest_catalog/docker-compose.yaml b/crates/catalog/rest/testdata/rest_catalog/docker-compose.yaml
@@ -15,6 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+networks:
+  rest_bridge:
+
 services:
   rest:
     image: tabulario/iceberg-rest:0.10.0
@@ -29,8 +32,10 @@ services:
       - CATALOG_S3_ENDPOINT=http://minio:9000
     depends_on:
       - minio
-    links:
-      - minio:icebergdata.minio
+    networks:
+      rest_bridge:
+        aliases:
+          - icebergdata.minio
     expose:
       - 8181
 
@@ -40,10 +45,13 @@ services:
       - MINIO_ROOT_USER=admin
       - MINIO_ROOT_PASSWORD=password
       - MINIO_DOMAIN=minio
+    hostname: icebergdata.minio
+    networks:
+      rest_bridge:
     expose:
       - 9001
       - 9000
-    command: [ "server", "/data", "--console-address", ":9001" ]
+    command: ["server", "/data", "--console-address", ":9001"]
 
   mc:
     depends_on:
@@ -55,3 +63,5 @@ services:
       - AWS_REGION=us-east-1
     entrypoint: >
       /bin/sh -c " until (/usr/bin/mc config host add minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done; /usr/bin/mc rm -r --force minio/icebergdata; /usr/bin/mc mb minio/icebergdata; /usr/bin/mc policy set public minio/icebergdata; tail -f /dev/null "
+    networks:
+      rest_bridge:
diff --git a/crates/catalog/rest/tests/rest_catalog_test.rs b/crates/catalog/rest/tests/rest_catalog_test.rs
@@ -18,6 +18,7 @@
 //! Integration tests for rest catalog.
 
 use std::collections::HashMap;
+use std::net::SocketAddr;
 use std::sync::RwLock;
 
 use ctor::{ctor, dtor};
@@ -59,18 +60,14 @@ async fn get_catalog() -> RestCatalog {
         docker_compose.get_container_ip("rest")
     };
 
-    let read_port = format!("{}:{}", rest_catalog_ip, REST_CATALOG_PORT);
-    loop {
-        if !scan_port_addr(&read_port) {
-            log::info!("Waiting for 1s rest catalog to ready...");
-            sleep(std::time::Duration::from_millis(1000)).await;
-        } else {
-            break;
-        }
+    let rest_socket_addr = SocketAddr::new(rest_catalog_ip, REST_CATALOG_PORT);
+    while !scan_port_addr(rest_socket_addr) {
+        log::info!("Waiting for 1s rest catalog to ready...");
+        sleep(std::time::Duration::from_millis(1000)).await;
     }
 
     let config = RestCatalogConfig::builder()
-        .uri(format!("http://{}:{}", rest_catalog_ip, REST_CATALOG_PORT))
+        .uri(format!("http://{}", rest_socket_addr))
         .build();
     RestCatalog::new(config)
 }

diff --git a/crates/iceberg/testdata/file_io_s3/docker-compose.yaml b/crates/iceberg/testdata/file_io_s3/docker-compose.yaml
@@ -18,7 +18,7 @@
 services:
   minio:
     image: minio/minio:RELEASE.2024-02-26T09-33-48Z
-    ports:
+    expose:
       - 9000
       - 9001
     environment:

diff --git a/crates/iceberg/tests/file_io_s3_test.rs b/crates/iceberg/tests/file_io_s3_test.rs
@@ -17,6 +17,7 @@
 
 //! Integration tests for FileIO S3.
 
+use std::net::SocketAddr;
 use std::sync::RwLock;
 
 use ctor::{ctor, dtor};
@@ -26,6 +27,7 @@ use iceberg::io::{
 use iceberg_test_utils::docker::DockerCompose;
 use iceberg_test_utils::{normalize_test_name, set_up};
 
+const MINIO_PORT: u16 = 9000;
 static DOCKER_COMPOSE_ENV: RwLock<Option<DockerCompose>> = RwLock::new(None);
 
 #[ctor]
@@ -51,11 +53,11 @@ async fn get_file_io() -> FileIO {
     let guard = DOCKER_COMPOSE_ENV.read().unwrap();
     let docker_compose = guard.as_ref().unwrap();
     let container_ip = docker_compose.get_container_ip("minio");
-    let read_port = format!("{}:{}", container_ip, 9000);
+    let minio_socket_addr = SocketAddr::new(container_ip, MINIO_PORT);
 
     FileIOBuilder::new("s3")
         .with_props(vec![
-            (S3_ENDPOINT, format!("http://{}", read_port)),
+            (S3_ENDPOINT, format!("http://{}", minio_socket_addr)),
             (S3_ACCESS_KEY_ID, "admin".to_string()),
             (S3_SECRET_ACCESS_KEY, "password".to_string()),
             (S3_REGION, "us-east-1".to_string()),

diff --git a/crates/integrations/datafusion/tests/integration_datafusion_hms_test.rs b/crates/integrations/datafusion/tests/integration_datafusion_hms_test.rs
@@ -18,6 +18,7 @@
 //! Integration tests for Iceberg Datafusion with Hive Metastore.
 
 use std::collections::HashMap;
+use std::net::SocketAddr;
 use std::sync::{Arc, RwLock};
 
 use ctor::{ctor, dtor};
@@ -40,7 +41,7 @@ static DOCKER_COMPOSE_ENV: RwLock<Option<DockerCompose>> = RwLock::new(None);
 struct TestFixture {
     hms_catalog: HmsCatalog,
     props: HashMap<String, String>,
-    hms_catalog_ip: String,
+    hms_catalog_socket_addr: SocketAddr,
 }
 
 #[ctor]
@@ -63,7 +64,7 @@ fn after_all() {
 impl TestFixture {
     fn get_catalog(&self) -> HmsCatalog {
         let config = HmsCatalogConfig::builder()
-            .address(format!("{}:{}", self.hms_catalog_ip, HMS_CATALOG_PORT))
+            .address(self.hms_catalog_socket_addr.to_string())
             .thrift_transport(HmsThriftTransport::Buffered)
             .warehouse("s3a://warehouse/hive".to_string())
             .props(self.props.clone())
@@ -85,28 +86,25 @@ async fn get_test_fixture() -> TestFixture {
         )
     };
 
-    let read_port = format!("{}:{}", hms_catalog_ip, HMS_CATALOG_PORT);
-    loop {
-        if !scan_port_addr(&read_port) {
-            log::info!("Waiting for 1s hms catalog to ready...");
-            sleep(std::time::Duration::from_millis(1000)).await;
-        } else {
-            break;
-        }
+    let hms_catalog_socket_addr = SocketAddr::new(hms_catalog_ip, HMS_CATALOG_PORT);
+    let minio_socket_addr = SocketAddr::new(minio_ip, MINIO_PORT);
+    while !scan_port_addr(hms_catalog_socket_addr) {
+        log::info!("Waiting for 1s hms catalog to ready...");
+        sleep(std::time::Duration::from_millis(1000)).await;
     }
 
     let props = HashMap::from([
         (
             S3_ENDPOINT.to_string(),
-            format!("http://{}:{}", minio_ip, MINIO_PORT),
+            format!("http://{}", minio_socket_addr),
         ),
         (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()),
         (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()),
         (S3_REGION.to_string(), "us-east-1".to_string()),
     ]);
 
     let config = HmsCatalogConfig::builder()
-        .address(format!("{}:{}", hms_catalog_ip, HMS_CATALOG_PORT))
+        .address(hms_catalog_socket_addr.to_string())
         .thrift_transport(HmsThriftTransport::Buffered)
         .warehouse("s3a://warehouse/hive".to_string())
         .props(props.clone())
@@ -117,7 +115,7 @@ async fn get_test_fixture() -> TestFixture {
     TestFixture {
         hms_catalog,
         props,
-        hms_catalog_ip,
+        hms_catalog_socket_addr,
     }
 }
 

diff --git a/crates/test_utils/src/cmd.rs b/crates/test_utils/src/cmd.rs
@@ -28,14 +28,27 @@ pub fn run_command(mut cmd: Command, desc: impl ToString) {
     }
 }
 
-pub fn get_cmd_output(mut cmd: Command, desc: impl ToString) -> String {
+pub fn get_cmd_output_result(mut cmd: Command, desc: impl ToString) -> Result<String, String> {
     let desc = desc.to_string();
     log::info!("Starting to {}, command: {:?}", &desc, cmd);
-    let output = cmd.output().unwrap();
-    if output.status.success() {
-        log::info!("{} succeed!", desc);
-        String::from_utf8(output.stdout).unwrap()
-    } else {
-        panic!("{} failed: {:?}", desc, output.status);
+    let result = cmd.output();
+    match result {
+        Ok(output) => {
+            if output.status.success() {
+                log::info!("{} succeed!", desc);
+                Ok(String::from_utf8(output.stdout).unwrap())
+            } else {
+                Err(format!("{} failed with rc: {:?}", desc, output.status))
+            }
+        }
+        Err(err) => Err(format!("{} failed with error: {}", desc, { err })),
+    }
+}
+
+pub fn get_cmd_output(cmd: Command, desc: impl ToString) -> String {
+    let result = get_cmd_output_result(cmd, desc);
+    match result {
+        Ok(output_str) => output_str,
+        Err(err) => panic!("{}", err),
     }
 }
diff --git a/crates/test_utils/src/docker.rs b/crates/test_utils/src/docker.rs
@@ -15,9 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use std::net::IpAddr;
 use std::process::Command;
 
-use crate::cmd::{get_cmd_output, run_command};
+use crate::cmd::{get_cmd_output, get_cmd_output_result, run_command};
 
 /// A utility to manage the lifecycle of `docker compose`.
 ///
@@ -46,9 +47,22 @@ impl DockerCompose {
             .arg("--format")
             .arg("{{.OSType}}/{{.Architecture}}");
 
-        get_cmd_output(cmd, "Get os arch".to_string())
-            .trim()
-            .to_string()
+        let result = get_cmd_output_result(cmd, "Get os arch".to_string());
+        match result {
+            Ok(value) => value.trim().to_string(),
+            Err(_err) => {
+                // docker/podman do not consistently place OSArch info in the same json path across OS and versions
+                // Below tries an alternative path if the above path fails
+                let mut alt_cmd = Command::new("docker");
+                alt_cmd
+                    .arg("info")
+                    .arg("--format")
+                    .arg("{{.Version.OsArch}}");
+                get_cmd_output(alt_cmd, "Get os arch".to_string())
+                    .trim()
+                    .to_string()
+            }
+        }
     }
 
     pub fn run(&self) {
@@ -77,17 +91,24 @@ impl DockerCompose {
         )
     }
 
-    pub fn get_container_ip(&self, service_name: impl AsRef<str>) -> String {
+    pub fn get_container_ip(&self, service_name: impl AsRef<str>) -> IpAddr {
         let container_name = format!("{}-{}-1", self.project_name, service_name.as_ref());
         let mut cmd = Command::new("docker");
         cmd.arg("inspect")
             .arg("-f")
             .arg("{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}")
             .arg(&container_name);
 
-        get_cmd_output(cmd, format!("Get container ip of {container_name}"))
+        let ip_result = get_cmd_output(cmd, format!("Get container ip of {container_name}"))
             .trim()
-            .to_string()
+            .parse::<IpAddr>();
+        match ip_result {
+            Ok(ip) => ip,
+            Err(e) => {
+                log::error!("Invalid IP, {e}");
+                panic!("Failed to parse IP for {container_name}")
+            }
+        }
     }
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -18,7 +18,7 @@ @@
     services:
       minio:
         image: minio/minio:RELEASE.2024-02-26T09-33-48Z
-        ports:
+        expose:
           - 9000
           - 9001
         environment:
@@ Expand Down @@