From 65af4406fa8aa6b316bcaf6df86af6745dab5ccf Mon Sep 17 00:00:00 2001 From: Chengxu Bian Date: Sun, 25 Feb 2024 18:41:10 -0500 Subject: [PATCH 1/3] add file io s3 test --- crates/iceberg/Cargo.toml | 2 + .../testdata/file_io_s3/docker-compose.yaml | 14 +++ crates/iceberg/tests/file_io_s3_test.rs | 100 ++++++++++++++++++ 3 files changed, 116 insertions(+) create mode 100644 crates/iceberg/testdata/file_io_s3/docker-compose.yaml create mode 100644 crates/iceberg/tests/file_io_s3_test.rs diff --git a/crates/iceberg/Cargo.toml b/crates/iceberg/Cargo.toml index 6af68a106..981b5caea 100644 --- a/crates/iceberg/Cargo.toml +++ b/crates/iceberg/Cargo.toml @@ -62,6 +62,8 @@ urlencoding = { workspace = true } uuid = { workspace = true } [dev-dependencies] +iceberg_test_utils = { path = "../test_utils", features = ["tests"] } +port_scanner = { workspace = true } pretty_assertions = { workspace = true } tempfile = { workspace = true } tera = { workspace = true } diff --git a/crates/iceberg/testdata/file_io_s3/docker-compose.yaml b/crates/iceberg/testdata/file_io_s3/docker-compose.yaml new file mode 100644 index 000000000..20437992b --- /dev/null +++ b/crates/iceberg/testdata/file_io_s3/docker-compose.yaml @@ -0,0 +1,14 @@ +version: '3' +services: + minio: + image: quay.io/minio/minio:latest + ports: + - 9000 + - 9001 + environment: + MINIO_ROOT_USER: 'admin' + MINIO_ROOT_PASSWORD: 'password' + MINIO_ADDRESS: ':9000' + MINIO_CONSOLE_ADDRESS: ':9001' + entrypoint: sh + command: -c 'mkdir -p /data/bucket1 && /usr/bin/minio server /data' diff --git a/crates/iceberg/tests/file_io_s3_test.rs b/crates/iceberg/tests/file_io_s3_test.rs new file mode 100644 index 000000000..66923f416 --- /dev/null +++ b/crates/iceberg/tests/file_io_s3_test.rs @@ -0,0 +1,100 @@ +use futures::{AsyncReadExt, AsyncWriteExt}; +use iceberg::io::{ + FileIO, FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY, +}; +use iceberg_test_utils::docker::DockerCompose; +use port_scanner::scan_port_addr; +use tokio::time::sleep; + +struct MinIOFixture { + _docker_compose: DockerCompose, + file_io: FileIO, +} +impl MinIOFixture { + async fn new(project_name: impl ToString) -> Self { + // Start the Docker container for the test fixture + let docker = DockerCompose::new( + project_name.to_string(), + format!("{}/testdata/file_io_s3", env!("CARGO_MANIFEST_DIR")), + ); + docker.run(); + let container_ip = docker.get_container_ip("minio"); + let read_port = format!("{}:{}", container_ip, 9000); + loop { + if !scan_port_addr(&read_port) { + log::info!("Waiting for 1s rest catalog to ready..."); + sleep(std::time::Duration::from_millis(1000)).await; + } else { + break; + } + } + MinIOFixture { + _docker_compose: docker, + file_io: FileIOBuilder::new("s3") + .with_props(vec![ + (S3_ENDPOINT, format!("http://{}", read_port)), + (S3_ACCESS_KEY_ID, "admin".to_string()), + (S3_SECRET_ACCESS_KEY, "password".to_string()), + (S3_REGION, "us-east-1".to_string()), + ]) + .build() + .unwrap(), + } + } +} + +#[tokio::test] +async fn test_file_io_s3_is_exist() { + let fixture = MinIOFixture::new("test_file_io_s3_is_exist").await; + assert!(!fixture.file_io.is_exist("s3://bucket2/any").await.unwrap()); + assert!(fixture.file_io.is_exist("s3://bucket1/").await.unwrap()); +} + +#[tokio::test] +async fn test_file_io_s3_output() { + // Start the Docker container for the test fixture + let fixture = MinIOFixture::new("test_file_io_s3_output").await; + assert!(!fixture + .file_io + .is_exist("s3://bucket1/test_output") + .await + .unwrap()); + let output_file = fixture + .file_io + .new_output("s3://bucket1/test_output") + .unwrap(); + { + let mut writer = output_file.writer().await.unwrap(); + writer.write_all("123".as_bytes()).await.unwrap(); + writer.close().await.unwrap(); + } + assert!(fixture + .file_io + .is_exist("s3://bucket1/test_output") + .await + .unwrap()); +} + +#[tokio::test] +async fn test_file_io_s3_input() { + let fixture = MinIOFixture::new("test_file_io_s3_input").await; + let output_file = fixture + .file_io + .new_output("s3://bucket1/test_input") + .unwrap(); + { + let mut writer = output_file.writer().await.unwrap(); + writer.write_all("test_input".as_bytes()).await.unwrap(); + writer.close().await.unwrap(); + } + let input_file = fixture + .file_io + .new_input("s3://bucket1/test_input") + .unwrap(); + { + let mut reader = input_file.reader().await.unwrap(); + let mut buffer = vec![]; + reader.read_to_end(&mut buffer).await.unwrap(); + assert_eq!(buffer, "test_input".as_bytes()); + } +} From 456fad4bd60f5ff1a1534b5d65fa414d144c21af Mon Sep 17 00:00:00 2001 From: Chengxu Bian Date: Sun, 25 Feb 2024 19:13:36 -0500 Subject: [PATCH 2/3] add license --- .../testdata/file_io_s3/docker-compose.yaml | 17 +++++++++++++++ crates/iceberg/tests/file_io_s3_test.rs | 21 ++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/crates/iceberg/testdata/file_io_s3/docker-compose.yaml b/crates/iceberg/testdata/file_io_s3/docker-compose.yaml index 20437992b..72408cb6b 100644 --- a/crates/iceberg/testdata/file_io_s3/docker-compose.yaml +++ b/crates/iceberg/testdata/file_io_s3/docker-compose.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + version: '3' services: minio: diff --git a/crates/iceberg/tests/file_io_s3_test.rs b/crates/iceberg/tests/file_io_s3_test.rs index 66923f416..1407c4e5e 100644 --- a/crates/iceberg/tests/file_io_s3_test.rs +++ b/crates/iceberg/tests/file_io_s3_test.rs @@ -1,3 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Integration tests for FileIO S3. + use futures::{AsyncReadExt, AsyncWriteExt}; use iceberg::io::{ FileIO, FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY, @@ -22,7 +41,7 @@ impl MinIOFixture { let read_port = format!("{}:{}", container_ip, 9000); loop { if !scan_port_addr(&read_port) { - log::info!("Waiting for 1s rest catalog to ready..."); + log::info!("Waiting 1s for MinIO to ready..."); sleep(std::time::Duration::from_millis(1000)).await; } else { break; From 740d899bf626d2b01694929871243efea4ba5284 Mon Sep 17 00:00:00 2001 From: Chengxu Bian Date: Mon, 26 Feb 2024 10:52:41 -0500 Subject: [PATCH 3/3] fixed version & rm port scanner --- crates/iceberg/Cargo.toml | 1 - .../iceberg/testdata/file_io_s3/docker-compose.yaml | 2 +- crates/iceberg/tests/file_io_s3_test.rs | 11 +---------- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/crates/iceberg/Cargo.toml b/crates/iceberg/Cargo.toml index 981b5caea..32288ee81 100644 --- a/crates/iceberg/Cargo.toml +++ b/crates/iceberg/Cargo.toml @@ -63,7 +63,6 @@ uuid = { workspace = true } [dev-dependencies] iceberg_test_utils = { path = "../test_utils", features = ["tests"] } -port_scanner = { workspace = true } pretty_assertions = { workspace = true } tempfile = { workspace = true } tera = { workspace = true } diff --git a/crates/iceberg/testdata/file_io_s3/docker-compose.yaml b/crates/iceberg/testdata/file_io_s3/docker-compose.yaml index 72408cb6b..3c1cfcc78 100644 --- a/crates/iceberg/testdata/file_io_s3/docker-compose.yaml +++ b/crates/iceberg/testdata/file_io_s3/docker-compose.yaml @@ -18,7 +18,7 @@ version: '3' services: minio: - image: quay.io/minio/minio:latest + image: minio/minio:RELEASE.2024-02-26T09-33-48Z ports: - 9000 - 9001 diff --git a/crates/iceberg/tests/file_io_s3_test.rs b/crates/iceberg/tests/file_io_s3_test.rs index 1407c4e5e..7553bcd2e 100644 --- a/crates/iceberg/tests/file_io_s3_test.rs +++ b/crates/iceberg/tests/file_io_s3_test.rs @@ -22,8 +22,6 @@ use iceberg::io::{ FileIO, FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY, }; use iceberg_test_utils::docker::DockerCompose; -use port_scanner::scan_port_addr; -use tokio::time::sleep; struct MinIOFixture { _docker_compose: DockerCompose, @@ -39,14 +37,7 @@ impl MinIOFixture { docker.run(); let container_ip = docker.get_container_ip("minio"); let read_port = format!("{}:{}", container_ip, 9000); - loop { - if !scan_port_addr(&read_port) { - log::info!("Waiting 1s for MinIO to ready..."); - sleep(std::time::Duration::from_millis(1000)).await; - } else { - break; - } - } + MinIOFixture { _docker_compose: docker, file_io: FileIOBuilder::new("s3")