From 9810afddc55ecd88cee006ccc0768f3034df6d27 Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Mon, 6 Jan 2025 11:44:09 +0800
Subject: [PATCH 01/29] Add testers for fileset fs

---
 .../filesystem-fuse/src/gravitino_client.rs   | 24 ++++++
 .../src/gravitino_fileset_filesystem.rs       | 73 ++++++++++++++++++-
 clients/filesystem-fuse/src/gvfs_creator.rs   |  2 +-
 clients/filesystem-fuse/src/s3_filesystem.rs  | 28 +++----
 clients/filesystem-fuse/src/utils.rs          | 14 ++++
 5 files changed, 120 insertions(+), 21 deletions(-)

diff --git a/clients/filesystem-fuse/src/gravitino_client.rs b/clients/filesystem-fuse/src/gravitino_client.rs
index 9bdfbb2c288..5a16a23358c 100644
--- a/clients/filesystem-fuse/src/gravitino_client.rs
+++ b/clients/filesystem-fuse/src/gravitino_client.rs
@@ -35,6 +35,18 @@ pub(crate) struct Fileset {
     properties: HashMap<String, String>,
 }
 
+impl Fileset {
+    pub fn new(name: &str, storage_location: &str) -> Fileset {
+        Self {
+            name: name.to_string(),
+            fileset_type: "managed".to_string(),
+            comment: "".to_string(),
+            storage_location: storage_location.to_string(),
+            properties: HashMap::default(),
+        }
+    }
+}
+
 #[derive(Debug, Deserialize)]
 struct FilesetResponse {
     code: u32,
@@ -58,6 +70,18 @@ pub(crate) struct Catalog {
     pub(crate) properties: HashMap<String, String>,
 }
 
+impl Catalog {
+    pub fn new(name: &str, properties: HashMap<String, String>) -> Catalog {
+        Self {
+            name: name.to_string(),
+            catalog_type: "fileset".to_string(),
+            provider: "s3".to_string(),
+            comment: "".to_string(),
+            properties: properties,
+        }
+    }
+}
+
 #[derive(Debug, Deserialize)]
 struct CatalogResponse {
     code: u32,
diff --git a/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs b/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs
index 7da2f572dcc..b7b69d5cc7b 100644
--- a/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs
+++ b/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs
@@ -140,16 +140,23 @@ impl PathFileSystem for GravitinoFilesetFileSystem {
 
 #[cfg(test)]
 mod tests {
-    use crate::config::GravitinoConfig;
+    use crate::config::{AppConfig, GravitinoConfig};
+    use crate::default_raw_filesystem::DefaultRawFileSystem;
+    use crate::filesystem::tests::{TestPathFileSystem, TestRawFileSystem};
+    use crate::filesystem::{FileSystemContext, PathFileSystem, RawFileSystem};
+    use crate::gravitino_client::{Catalog, Fileset, GravitinoClient};
     use crate::gravitino_fileset_filesystem::GravitinoFilesetFileSystem;
+    use crate::gvfs_creator::create_fs_with_fileset;
     use crate::memory_filesystem::MemoryFileSystem;
+    use crate::s3_filesystem::tests::cleanup_s3_fs;
+    use std::collections::HashMap;
     use std::path::Path;
 
     #[tokio::test]
     async fn test_map_fileset_path_to_raw_path() {
         let fs = GravitinoFilesetFileSystem {
             physical_fs: Box::new(MemoryFileSystem::new().await),
-            client: super::GravitinoClient::new(&GravitinoConfig::default()),
+            client: GravitinoClient::new(&GravitinoConfig::default()),
             location: "/c1/fileset1".into(),
         };
         let path = fs.gvfs_path_to_raw_path(Path::new("/a"));
@@ -162,7 +169,7 @@ mod tests {
     async fn test_map_raw_path_to_fileset_path() {
         let fs = GravitinoFilesetFileSystem {
             physical_fs: Box::new(MemoryFileSystem::new().await),
-            client: super::GravitinoClient::new(&GravitinoConfig::default()),
+            client: GravitinoClient::new(&GravitinoConfig::default()),
             location: "/c1/fileset1".into(),
         };
         let path = fs
@@ -172,4 +179,64 @@ mod tests {
         let path = fs.raw_path_to_gvfs_path(Path::new("/c1/fileset1")).unwrap();
         assert_eq!(path, Path::new("/"));
     }
+
+    async fn create_fileset_fs(path: &Path) -> GravitinoFilesetFileSystem {
+        cleanup_s3_fs(path).await;
+
+        let catalog = Catalog::new(
+            "c1",
+            vec![
+                ("location".to_string(), "s3a://trino-test-ice".to_string()),
+                (
+                    "s3-endpoint".to_string(),
+                    "http://s3.ap-southeast-2.amazonaws.com".to_string(),
+                ),
+            ]
+            .into_iter()
+            .collect::<HashMap<String, String>>(),
+        );
+        let file_set_location = format!("s3a://trino-test-ice{}", path.to_string_lossy());
+        let file_set = Fileset::new("fileset1", &file_set_location);
+        let config = AppConfig::from_file(Some("tests/conf/gvfs_fuse_s3.toml")).unwrap();
+        let fs_context = FileSystemContext::default();
+        let inner_fs = create_fs_with_fileset(&catalog, &file_set, &config, &fs_context).unwrap();
+        GravitinoFilesetFileSystem::new(
+            inner_fs,
+            path,
+            GravitinoClient::new(&config.gravitino),
+            &config,
+            &fs_context,
+        )
+        .await
+    }
+
+    #[tokio::test]
+    async fn test_fileset_file_system() {
+        if std::env::var("RUN_S3_TESTS").is_err() {
+            return;
+        }
+        let cwd = Path::new("/gvfs_test3");
+        let fs = create_fileset_fs(cwd).await;
+        let _ = fs.init().await;
+        let mut tester = TestPathFileSystem::new(Path::new("/"), fs);
+        tester.test_path_file_system().await;
+    }
+
+    #[tokio::test]
+    async fn test_fileset_with_raw_file_system() {
+        if std::env::var("RUN_S3_TESTS").is_err() {
+            return;
+        }
+
+        let cwd = Path::new("/gvfs_test4");
+        let fileset_fs = create_fileset_fs(cwd).await;
+        let raw_fs = DefaultRawFileSystem::new(
+            fileset_fs,
+            &AppConfig::default(),
+            &FileSystemContext::default(),
+        );
+        let _ = raw_fs.init().await;
+        let mut tester = TestRawFileSystem::new(Path::new("/"), raw_fs);
+        tester.test_raw_file_system().await;
+    }
 }
diff --git a/clients/filesystem-fuse/src/gvfs_creator.rs b/clients/filesystem-fuse/src/gvfs_creator.rs
index aac88ad9d08..f9bdfc85674 100644
--- a/clients/filesystem-fuse/src/gvfs_creator.rs
+++ b/clients/filesystem-fuse/src/gvfs_creator.rs
@@ -95,7 +95,7 @@ pub async fn create_gvfs_filesystem(
     Ok(CreateFileSystemResult::Gvfs(fs))
 }
 
-fn create_fs_with_fileset(
+pub(crate) fn create_fs_with_fileset(
     catalog: &Catalog,
     fileset: &Fileset,
     config: &AppConfig,
diff --git a/clients/filesystem-fuse/src/s3_filesystem.rs b/clients/filesystem-fuse/src/s3_filesystem.rs
index e0ca69b4ccf..e883f560f86 100644
--- a/clients/filesystem-fuse/src/s3_filesystem.rs
+++ b/clients/filesystem-fuse/src/s3_filesystem.rs
@@ -181,11 +181,12 @@ pub fn extract_s3_config(config: &AppConfig) -> HashMap<String, String> {
 }
 
 #[cfg(test)]
-mod tests {
+pub(crate) mod tests {
     use super::*;
     use crate::default_raw_filesystem::DefaultRawFileSystem;
     use crate::filesystem::tests::{TestPathFileSystem, TestRawFileSystem};
     use crate::filesystem::RawFileSystem;
+    use crate::utils::delete_dir;
     use opendal::layers::TimeoutLayer;
     use std::time::Duration;
 
@@ -205,25 +206,10 @@ mod tests {
         assert_eq!(result.unwrap(), "ap-southeast-2");
     }
 
-    async fn delete_dir(op: &Operator, dir_name: &str) {
-        let childs = op.list(dir_name).await.expect("list dir failed");
-        for child in childs {
-            let child_name = dir_name.to_string() + child.name();
-            if child.metadata().is_dir() {
-                Box::pin(delete_dir(op, &child_name)).await;
-            } else {
-                op.delete(&child_name).await.expect("delete file failed");
-            }
-        }
-        op.delete(dir_name).await.expect("delete dir failed");
-    }
-
-    async fn create_s3_fs(cwd: &Path) -> S3FileSystem {
+    pub(crate) async fn cleanup_s3_fs(cwd: &Path) -> Operator {
         let config = AppConfig::from_file(Some("tests/conf/gvfs_fuse_s3.toml")).unwrap();
         let opendal_config = extract_s3_config(&config);
 
-        let fs_context = FileSystemContext::default();
-
         let builder = S3::from_map(opendal_config);
         let op = Operator::new(builder)
             .expect("opendal create failed")
@@ -241,8 +227,16 @@ mod tests {
         op.create_dir(&file_name)
             .await
             .expect("create test dir failed");
+        op
+    }
+
+    async fn create_s3_fs(cwd: &Path) -> S3FileSystem {
+        let op = cleanup_s3_fs(cwd).await;
 
+        let config = AppConfig::from_file(Some("tests/conf/gvfs_fuse_s3.toml")).unwrap();
+        let fs_context = FileSystemContext::default();
         let open_dal_fs = OpenDalFileSystem::new(op, &config, &fs_context);
+
         S3FileSystem { open_dal_fs }
     }
 
diff --git a/clients/filesystem-fuse/src/utils.rs b/clients/filesystem-fuse/src/utils.rs
index 53eb9179d71..73af044ee4c 100644
--- a/clients/filesystem-fuse/src/utils.rs
+++ b/clients/filesystem-fuse/src/utils.rs
@@ -18,6 +18,7 @@
  */
 use crate::error::ErrorCode::InvalidConfig;
 use crate::error::GvfsError;
+use opendal::Operator;
 use reqwest::Url;
 use std::path::PathBuf;
 
@@ -36,6 +37,19 @@ pub(crate) fn extract_root_path(location: &str) -> GvfsResult<PathBuf> {
     Ok(PathBuf::from(url.path()))
 }
 
+pub(crate) async fn delete_dir(op: &Operator, dir_name: &str) {
+    let childs = op.list(dir_name).await.expect("list dir failed");
+    for child in childs {
+        let child_name = dir_name.to_string() + child.name();
+        if child.metadata().is_dir() {
+            Box::pin(delete_dir(op, &child_name)).await;
+        } else {
+            op.delete(&child_name).await.expect("delete file failed");
+        }
+    }
+    op.delete(dir_name).await.expect("delete dir failed");
+}
+
 #[cfg(test)]
 mod tests {
     use crate::utils::extract_root_path;

From 4350b437504079afc842c0184df9815487aad563 Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Mon, 6 Jan 2025 15:03:20 +0800
Subject: [PATCH 02/29] Add integration test script

---
 .../filesystem-fuse/tests/start_backend.sh    | 104 ++++++++++++++++++
 1 file changed, 104 insertions(+)
 create mode 100755 clients/filesystem-fuse/tests/start_backend.sh

diff --git a/clients/filesystem-fuse/tests/start_backend.sh b/clients/filesystem-fuse/tests/start_backend.sh
new file mode 100755
index 00000000000..a5d55671ae0
--- /dev/null
+++ b/clients/filesystem-fuse/tests/start_backend.sh
@@ -0,0 +1,104 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ set -euo pipefail
+
+s3-access_key_id=${s3-access_key_id:-}
+s3-secret_access=${s3-secret_access:-}
+s3-region=${s3-region:-}
+s3-bucket=${s3-bucket:-}
+
+# Check required environment variables
+if [[ -z "$s3-access_key_id" || -z "$s3-secret_access" || -z "$s3-region" || -z "$s3-bucket" ]]; then
+  echo "Error: One or more required S3 environment variables are not set."
+  echo "Please set: s3-access_key_id, s3-secret_access, s3-region, s3-bucket."
+  exit 1
+fi
+
+GRAVITINO_SERVER_HOME=../../..
+GRAVITINO_SERVER_DIR=$GRAVITINO_SERVER_HOME/distribution/package
+CLIENT_FUSE_DIR=$GRAVITINO_SERVER_HOME/clients/filesystem-fuse
+
+echo "Start the Gravitino server"
+$GRAVITINO_SERVER_DIR/bin/start_gravitino_server.sh
+
+GRAVITINO_SERVER_URL=http://localhost:8090
+
+curl $GRAVITINO_SERVER_URL/api/metalakes
+
+# create metalake
+curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \
+-H "Content-Type: application/json" -d '{
+  "name":"test","comment":"comment","properties":{}
+}' $GRAVITINO_SERVER_URL/api/metalakes
+
+# create catalog
+curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \
+-H "Content-Type: application/json" -d '{
+  "name": "c1",
+  "type": "FILESET",
+  "comment": "comment",
+  "provider": "hadoop",
+  "properties": {
+    "location": "s3a://'"$s3-bucket"'",
+    "s3-access-key-id": "'"$s3-access_key_id"'",
+    "s3-secret-access-key": "'"$s3-secret_access"'",
+    "s3-endpoint": "http://s3.'"$s3-region"'.amazonaws.com",
+    "filesystem-providers": "s3"
+  }
+}' $GRAVITINO_SERVER_URL/api/metalakes/test/catalogs
+
+# create schema
+curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \
+-H "Content-Type: application/json" -d '{
+  "name":"s1","comment":"comment","properties":{}
+}' $GRAVITINO_SERVER_URL/api/metalakes/test/catalogs/catalog/schemas
+
+# create fileset
+curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \
+-H "Content-Type: application/json" -d '{
+  "name":"fileset1","comment":"comment","properties":{}
+}' $GRAVITINO_SERVER_URL/api/metalakes/test/catalogs/c1/schemas/s1/filesets
+
+
+echo "Start the Gvfs fuse client"
+
+mount_dir=$CLIENT_FUSE_DIR/target/gvfs
+if [ -d "$mount_dir" ]; then
+  echo "Unmount the existing mount point"
+  fusermount -u $mount_dir
+else
+  echo "Create the mount point"
+  mkdir -p $mount_dir
+fi
+
+fileset=gvfs://fileset/test/c1/s1/fileset1
+
+config_file=$CLIENT_FUSE_DIR/target/debug/gvfs-fuse.toml
+cp $CLIENT_FUSE_DIR/test/conf/gvfs_fuse-s3.toml $config_file
+
+
+sed -i 's|s3-access_key_id = ".*"|s3-access_key_id = "$s3-access_key_id"|' "$config_file"
+sed -i 's|s3-secret_access_key = ".*"|s3-secret_access_key = "$s3-secret_access"|' "$config_file"
+sed -i 's|s3-region = ".*"|s3-region = "$s3-region"|' "$config_file"
+sed -i 's|s3-bucket = ".*"|s3-bucket = "$s3-bucket"|' "$config_file"
+
+$CLIENT_FUSE_DIR/target/debug/gvfs-fuse $mount_dir $fileset $config_file
+
+

From 81bbf63ddf7930d61bb231f3a4223296f4a2cacc Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Mon, 6 Jan 2025 15:04:59 +0800
Subject: [PATCH 03/29] Add integration testers script

---
 clients/filesystem-fuse/tests/{ => bin}/start_backend.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename clients/filesystem-fuse/tests/{ => bin}/start_backend.sh (100%)

diff --git a/clients/filesystem-fuse/tests/start_backend.sh b/clients/filesystem-fuse/tests/bin/start_backend.sh
similarity index 100%
rename from clients/filesystem-fuse/tests/start_backend.sh
rename to clients/filesystem-fuse/tests/bin/start_backend.sh

From 6b686508f55908ebc0db1fe08262741a7fc534b8 Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Mon, 6 Jan 2025 15:17:17 +0800
Subject: [PATCH 04/29] Fix

---
 .../tests/bin/start_backend.sh                | 66 ++++++++++++-------
 1 file changed, 42 insertions(+), 24 deletions(-)

diff --git a/clients/filesystem-fuse/tests/bin/start_backend.sh b/clients/filesystem-fuse/tests/bin/start_backend.sh
index a5d55671ae0..624d7a8fa45 100755
--- a/clients/filesystem-fuse/tests/bin/start_backend.sh
+++ b/clients/filesystem-fuse/tests/bin/start_backend.sh
@@ -19,15 +19,15 @@
 
  set -euo pipefail
 
-s3-access_key_id=${s3-access_key_id:-}
-s3-secret_access=${s3-secret_access:-}
-s3-region=${s3-region:-}
-s3-bucket=${s3-bucket:-}
+S3-ACCESS_KEY_ID=${S3-ACCESS_KEY_ID:-}
+S3-SECRET_ACCESS=${S3-SECRET_ACCESS:-}
+S3-REGION=${S3-REGION:-}
+S3-BUCKET=${S3-BUCKET:-}
 
 # Check required environment variables
-if [[ -z "$s3-access_key_id" || -z "$s3-secret_access" || -z "$s3-region" || -z "$s3-bucket" ]]; then
+if [[ -z "$S3-ACCESS_KEY_ID" || -z "$S3-SECRET_ACCESS" || -z "$S3-REGION" || -z "$S3-BUCKET" ]]; then
   echo "Error: One or more required S3 environment variables are not set."
-  echo "Please set: s3-access_key_id, s3-secret_access, s3-region, s3-bucket."
+  echo "Please set: S3-ACCESS_KEY_ID, S3-SECRET_ACCESS, S3-REGION, S3-BUCKET."
   exit 1
 fi
 
@@ -40,7 +40,7 @@ $GRAVITINO_SERVER_DIR/bin/start_gravitino_server.sh
 
 GRAVITINO_SERVER_URL=http://localhost:8090
 
-curl $GRAVITINO_SERVER_URL/api/metalakes
+check_server_ready "$GRAVITINO_SERVER_URL/api/metalakes"
 
 # create metalake
 curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \
@@ -56,10 +56,10 @@ curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \
   "comment": "comment",
   "provider": "hadoop",
   "properties": {
-    "location": "s3a://'"$s3-bucket"'",
-    "s3-access-key-id": "'"$s3-access_key_id"'",
-    "s3-secret-access-key": "'"$s3-secret_access"'",
-    "s3-endpoint": "http://s3.'"$s3-region"'.amazonaws.com",
+    "location": "s3a://'"$S3-BUCKET"'",
+    "s3-access-key-id": "'"$S3-ACCESS_KEY_ID"'",
+    "s3-secret-access-key": "'"$S3-SECRET_ACCESS"'",
+    "s3-endpoint": "http://s3.'"$S3-REGION"'.amazonaws.com",
     "filesystem-providers": "s3"
   }
 }' $GRAVITINO_SERVER_URL/api/metalakes/test/catalogs
@@ -70,7 +70,7 @@ curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \
   "name":"s1","comment":"comment","properties":{}
 }' $GRAVITINO_SERVER_URL/api/metalakes/test/catalogs/catalog/schemas
 
-# create fileset
+# create FILESET
 curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \
 -H "Content-Type: application/json" -d '{
   "name":"fileset1","comment":"comment","properties":{}
@@ -79,26 +79,44 @@ curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \
 
 echo "Start the Gvfs fuse client"
 
-mount_dir=$CLIENT_FUSE_DIR/target/gvfs
-if [ -d "$mount_dir" ]; then
+MOUNT_DIR=$CLIENT_FUSE_DIR/target/gvfs
+if [ -d "$MOUNT_DIR" ]; then
   echo "Unmount the existing mount point"
-  fusermount -u $mount_dir
+  fusermount -u $MOUNT_DIR
 else
   echo "Create the mount point"
-  mkdir -p $mount_dir
+  mkdir -p $MOUNT_DIR
 fi
 
-fileset=gvfs://fileset/test/c1/s1/fileset1
+FILESET=gvfs://fileset/test/c1/s1/fileset1
 
-config_file=$CLIENT_FUSE_DIR/target/debug/gvfs-fuse.toml
-cp $CLIENT_FUSE_DIR/test/conf/gvfs_fuse-s3.toml $config_file
+CONF_FILE=$CLIENT_FUSE_DIR/target/debug/gvfs-fuse.toml
+cp $CLIENT_FUSE_DIR/test/conf/gvfs_fuse-s3.toml $CONF_FILE
 
 
-sed -i 's|s3-access_key_id = ".*"|s3-access_key_id = "$s3-access_key_id"|' "$config_file"
-sed -i 's|s3-secret_access_key = ".*"|s3-secret_access_key = "$s3-secret_access"|' "$config_file"
-sed -i 's|s3-region = ".*"|s3-region = "$s3-region"|' "$config_file"
-sed -i 's|s3-bucket = ".*"|s3-bucket = "$s3-bucket"|' "$config_file"
+sed -i 's|S3-ACCESS_KEY_ID = ".*"|S3-ACCESS_KEY_ID = "$S3-ACCESS_KEY_ID"|' "$CONF_FILE"
+sed -i 's|S3-SECRET_ACCESS_key = ".*"|S3-SECRET_ACCESS_key = "$S3-SECRET_ACCESS"|' "$CONF_FILE"
+sed -i 's|S3-REGION = ".*"|S3-REGION = "$S3-REGION"|' "$CONF_FILE"
+sed -i 's|S3-BUCKET = ".*"|S3-BUCKET = "$S3-BUCKET"|' "$CONF_FILE"
 
-$CLIENT_FUSE_DIR/target/debug/gvfs-fuse $mount_dir $fileset $config_file
+$CLIENT_FUSE_DIR/target/debug/gvfs-fuse $MOUNT_DIR $FILESET $CONF_FILE
 
+check_server_ready() {
+  local url=$1
+  local retries=10  # Number of retries
+  local wait_time=3 # Wait time between retries (seconds)
+
+  for ((i=1; i<=retries; i++)); do
+    if curl --silent --head --fail "$url" >/dev/null; then
+      echo "Gravitino server is ready."
+      return 0
+    else
+      echo "Attempt $i/$retries: Server not ready. Retrying in $wait_time seconds..."
+      sleep "$wait_time"
+    fi
+  done
+
+  echo "Error: Gravitino server did not become ready after $((retries * wait_time)) seconds."
+  exit 1
+}
 

From 922ed0a380e761c62e825504b44e9c1be3ca09c2 Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Mon, 6 Jan 2025 15:23:15 +0800
Subject: [PATCH 05/29] Fix

---
 clients/filesystem-fuse/tests/bin/start_backend.sh | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/clients/filesystem-fuse/tests/bin/start_backend.sh b/clients/filesystem-fuse/tests/bin/start_backend.sh
index 624d7a8fa45..1ab188bb87c 100755
--- a/clients/filesystem-fuse/tests/bin/start_backend.sh
+++ b/clients/filesystem-fuse/tests/bin/start_backend.sh
@@ -93,11 +93,13 @@ FILESET=gvfs://fileset/test/c1/s1/fileset1
 CONF_FILE=$CLIENT_FUSE_DIR/target/debug/gvfs-fuse.toml
 cp $CLIENT_FUSE_DIR/test/conf/gvfs_fuse-s3.toml $CONF_FILE
 
-
-sed -i 's|S3-ACCESS_KEY_ID = ".*"|S3-ACCESS_KEY_ID = "$S3-ACCESS_KEY_ID"|' "$CONF_FILE"
-sed -i 's|S3-SECRET_ACCESS_key = ".*"|S3-SECRET_ACCESS_key = "$S3-SECRET_ACCESS"|' "$CONF_FILE"
-sed -i 's|S3-REGION = ".*"|S3-REGION = "$S3-REGION"|' "$CONF_FILE"
-sed -i 's|S3-BUCKET = ".*"|S3-BUCKET = "$S3-BUCKET"|' "$CONF_FILE"
+awk '{
+    if ($0 ~ /S3-ACCESS_KEY_ID/) $0 = "S3-ACCESS_KEY_ID = \"" ENVIRON["S3_ACCESS_KEY_ID"] "\"";
+    if ($0 ~ /S3-SECRET_ACCESS_KEY/) $0 = "S3-SECRET_ACCESS_KEY = \"" ENVIRON["S3_SECRET_ACCESS"] "\"";
+    if ($0 ~ /S3-REGION/) $0 = "S3-REGION = \"" ENVIRON["S3_REGION"] "\"";
+    if ($0 ~ /S3-BUCKET/) $0 = "S3-BUCKET = \"" ENVIRON["S3_BUCKET"] "\"";
+    print
+}' $CLIENT_FUSE_DIR/test/conf/gvfs_fuse-s3.toml > "$CONF_FILE"
 
 $CLIENT_FUSE_DIR/target/debug/gvfs-fuse $MOUNT_DIR $FILESET $CONF_FILE
 

From 3435810274a8893c45f32b0e34e399ab215b2d07 Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Mon, 6 Jan 2025 17:08:31 +0800
Subject: [PATCH 06/29] Fix

---
 .../tests/bin/start_backend.sh                | 101 ++++++++++--------
 .../tests/conf/gvfs_fuse_s3.toml              |   2 +-
 2 files changed, 58 insertions(+), 45 deletions(-)

diff --git a/clients/filesystem-fuse/tests/bin/start_backend.sh b/clients/filesystem-fuse/tests/bin/start_backend.sh
index 1ab188bb87c..e1f8245aff6 100755
--- a/clients/filesystem-fuse/tests/bin/start_backend.sh
+++ b/clients/filesystem-fuse/tests/bin/start_backend.sh
@@ -17,28 +17,56 @@
 # specific language governing permissions and limitations
 # under the License.
 
- set -euo pipefail
+set -euo pipefail
 
-S3-ACCESS_KEY_ID=${S3-ACCESS_KEY_ID:-}
-S3-SECRET_ACCESS=${S3-SECRET_ACCESS:-}
-S3-REGION=${S3-REGION:-}
-S3-BUCKET=${S3-BUCKET:-}
+S3_ACCESS_KEY_ID=${S3_ACCESS_KEY_ID:-}
+S3_SECRET_ACCESS=${S3_SECRET_ACCESS:-}
+S3_REGION=${S3_REGION:-}
+S3_BUCKET=${S3_BUCKET:-}
 
 # Check required environment variables
-if [[ -z "$S3-ACCESS_KEY_ID" || -z "$S3-SECRET_ACCESS" || -z "$S3-REGION" || -z "$S3-BUCKET" ]]; then
+if [[ -z "$S3_ACCESS_KEY_ID" || -z "$S3_SECRET_ACCESS" || -z "$S3_REGION" || -z "$S3_BUCKET" ]]; then
   echo "Error: One or more required S3 environment variables are not set."
-  echo "Please set: S3-ACCESS_KEY_ID, S3-SECRET_ACCESS, S3-REGION, S3-BUCKET."
+  echo "Please set: S3_ACCESS_KEY_ID, S3_SECRET_ACCESS, S3_REGION, S3_BUCKET."
   exit 1
 fi
 
-GRAVITINO_SERVER_HOME=../../..
-GRAVITINO_SERVER_DIR=$GRAVITINO_SERVER_HOME/distribution/package
-CLIENT_FUSE_DIR=$GRAVITINO_SERVER_HOME/clients/filesystem-fuse
+GRAVITINO_HOME=../../../..
+GRAVITINO_SERVER_DIR=$GRAVITINO_HOME/distribution/package
+CLIENT_FUSE_DIR=$GRAVITINO_HOME/clients/filesystem-fuse
+GRAVITINO_SERVER_URL=http://localhost:8090
+
+# copy the aws-bundle to the server
+if ls $GRAVITINO_SERVER_DIR/catalogs/hadoop/libs/gravitino-aws-bundle-*-incubating-SNAPSHOT.jar 1>/dev/null 2>&1; then
+  echo "File exists, skipping copy."
+else
+  cp $GRAVITINO_HOME/bundles/aws-bundle/build/libs/gravitino-aws-bundle-*-incubating-SNAPSHOT.jar \
+    $GRAVITINO_SERVER_DIR/catalogs/hadoop/libs
+fi
+
 
 echo "Start the Gravitino server"
-$GRAVITINO_SERVER_DIR/bin/start_gravitino_server.sh
+rm -rf $GRAVITINO_SERVER_DIR/data
+$GRAVITINO_SERVER_DIR/bin/gravitino.sh restart
 
-GRAVITINO_SERVER_URL=http://localhost:8090
+check_server_ready() {
+  local url=$1
+  local retries=10  # Number of retries
+  local wait_time=1 # Wait time between retries (seconds)
+
+  for ((i=1; i<=retries; i++)); do
+    if curl --silent --head --fail "$url" >/dev/null; then
+      echo "Gravitino server is ready."
+      return 0
+    else
+      echo "Attempt $i/$retries: Server not ready. Retrying in $wait_time seconds..."
+      sleep "$wait_time"
+    fi
+  done
+
+  echo "Error: Gravitino server did not become ready after $((retries * wait_time)) seconds."
+  exit 1
+}
 
 check_server_ready "$GRAVITINO_SERVER_URL/api/metalakes"
 
@@ -56,10 +84,10 @@ curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \
   "comment": "comment",
   "provider": "hadoop",
   "properties": {
-    "location": "s3a://'"$S3-BUCKET"'",
-    "s3-access-key-id": "'"$S3-ACCESS_KEY_ID"'",
-    "s3-secret-access-key": "'"$S3-SECRET_ACCESS"'",
-    "s3-endpoint": "http://s3.'"$S3-REGION"'.amazonaws.com",
+    "location": "s3a://'"$S3_BUCKET"'",
+    "s3-access-key-id": "'"$S3_ACCESS_KEY_ID"'",
+    "s3-secret-access-key": "'"$S3_SECRET_ACCESS"'",
+    "s3-endpoint": "http://s3.'"$S3_REGION"'.amazonaws.com",
     "filesystem-providers": "s3"
   }
 }' $GRAVITINO_SERVER_URL/api/metalakes/test/catalogs
@@ -68,7 +96,7 @@ curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \
 curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \
 -H "Content-Type: application/json" -d '{
   "name":"s1","comment":"comment","properties":{}
-}' $GRAVITINO_SERVER_URL/api/metalakes/test/catalogs/catalog/schemas
+}' $GRAVITINO_SERVER_URL/api/metalakes/test/catalogs/c1/schemas
 
 # create FILESET
 curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \
@@ -82,7 +110,7 @@ echo "Start the Gvfs fuse client"
 MOUNT_DIR=$CLIENT_FUSE_DIR/target/gvfs
 if [ -d "$MOUNT_DIR" ]; then
   echo "Unmount the existing mount point"
-  fusermount -u $MOUNT_DIR
+  umount -l $MOUNT_DIR > /dev/null 2>&1 || true
 else
   echo "Create the mount point"
   mkdir -p $MOUNT_DIR
@@ -91,34 +119,19 @@ fi
 FILESET=gvfs://fileset/test/c1/s1/fileset1
 
 CONF_FILE=$CLIENT_FUSE_DIR/target/debug/gvfs-fuse.toml
-cp $CLIENT_FUSE_DIR/test/conf/gvfs_fuse-s3.toml $CONF_FILE
 
-awk '{
-    if ($0 ~ /S3-ACCESS_KEY_ID/) $0 = "S3-ACCESS_KEY_ID = \"" ENVIRON["S3_ACCESS_KEY_ID"] "\"";
-    if ($0 ~ /S3-SECRET_ACCESS_KEY/) $0 = "S3-SECRET_ACCESS_KEY = \"" ENVIRON["S3_SECRET_ACCESS"] "\"";
-    if ($0 ~ /S3-REGION/) $0 = "S3-REGION = \"" ENVIRON["S3_REGION"] "\"";
-    if ($0 ~ /S3-BUCKET/) $0 = "S3-BUCKET = \"" ENVIRON["S3_BUCKET"] "\"";
-    print
-}' $CLIENT_FUSE_DIR/test/conf/gvfs_fuse-s3.toml > "$CONF_FILE"
+awk -v access_key="$S3_ACCESS_KEY_ID" \
+    -v secret_key="$S3_SECRET_ACCESS" \
+    -v region="$S3_REGION" \
+    -v bucket="$S3_BUCKET" \
+    'BEGIN { in_extend_config = 0 }
+    /^\[extend_config\]/ { in_extend_config = 1 }
+    in_extend_config && /s3-access_key_id/ { $0 = "s3-access_key_id = \"" access_key "\"" }
+    in_extend_config && /s3-secret_access_key/ { $0 = "s3-secret_access_key = \"" secret_key "\"" }
+    in_extend_config && /s3-region/ { $0 = "s3-region = \"" region "\"" }
+    in_extend_config && /s3-bucket/ { $0 = "s3-bucket = \"" bucket "\"" }
+    { print }' $CLIENT_FUSE_DIR/tests/conf/gvfs_fuse_s3.toml > "$CONF_FILE"
 
 $CLIENT_FUSE_DIR/target/debug/gvfs-fuse $MOUNT_DIR $FILESET $CONF_FILE
 
-check_server_ready() {
-  local url=$1
-  local retries=10  # Number of retries
-  local wait_time=3 # Wait time between retries (seconds)
-
-  for ((i=1; i<=retries; i++)); do
-    if curl --silent --head --fail "$url" >/dev/null; then
-      echo "Gravitino server is ready."
-      return 0
-    else
-      echo "Attempt $i/$retries: Server not ready. Retrying in $wait_time seconds..."
-      sleep "$wait_time"
-    fi
-  done
-
-  echo "Error: Gravitino server did not become ready after $((retries * wait_time)) seconds."
-  exit 1
-}
 
diff --git a/clients/filesystem-fuse/tests/conf/gvfs_fuse_s3.toml b/clients/filesystem-fuse/tests/conf/gvfs_fuse_s3.toml
index 7d182cd40df..cf61f6f1265 100644
--- a/clients/filesystem-fuse/tests/conf/gvfs_fuse_s3.toml
+++ b/clients/filesystem-fuse/tests/conf/gvfs_fuse_s3.toml
@@ -19,7 +19,7 @@
 [fuse]
 file_mask= 0o600
 dir_mask= 0o700
-fs_type = "memory"
+fs_type = "gvfs"
 
 [fuse.properties]
 key1 = "value1"

From 4ad62ed58316b800695f35d473b332c27e959939 Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Mon, 6 Jan 2025 17:24:18 +0800
Subject: [PATCH 07/29] Fix

---
 clients/filesystem-fuse/tests/bin/start_backend.sh | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/clients/filesystem-fuse/tests/bin/start_backend.sh b/clients/filesystem-fuse/tests/bin/start_backend.sh
index e1f8245aff6..115777e3d2c 100755
--- a/clients/filesystem-fuse/tests/bin/start_backend.sh
+++ b/clients/filesystem-fuse/tests/bin/start_backend.sh
@@ -108,10 +108,9 @@ curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \
 echo "Start the Gvfs fuse client"
 
 MOUNT_DIR=$CLIENT_FUSE_DIR/target/gvfs
-if [ -d "$MOUNT_DIR" ]; then
-  echo "Unmount the existing mount point"
-  umount -l $MOUNT_DIR > /dev/null 2>&1 || true
-else
+
+umount $MOUNT_DIR > /dev/null 2>&1 || true
+if [ ! -d "$MOUNT_DIR" ]; then
   echo "Create the mount point"
   mkdir -p $MOUNT_DIR
 fi

From 7900190394d844b97be9ed543746286b1edf4f89 Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Tue, 7 Jan 2025 09:58:41 +0800
Subject: [PATCH 08/29] Update test script

---
 clients/filesystem-fuse/tests/bin/start_backend.sh | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/clients/filesystem-fuse/tests/bin/start_backend.sh b/clients/filesystem-fuse/tests/bin/start_backend.sh
index 115777e3d2c..b12b0a56333 100755
--- a/clients/filesystem-fuse/tests/bin/start_backend.sh
+++ b/clients/filesystem-fuse/tests/bin/start_backend.sh
@@ -131,6 +131,18 @@ awk -v access_key="$S3_ACCESS_KEY_ID" \
     in_extend_config && /s3-bucket/ { $0 = "s3-bucket = \"" bucket "\"" }
     { print }' $CLIENT_FUSE_DIR/tests/conf/gvfs_fuse_s3.toml > "$CONF_FILE"
 
-$CLIENT_FUSE_DIR/target/debug/gvfs-fuse $MOUNT_DIR $FILESET $CONF_FILE
+# Start the gvfs-fuse process in the background
+$CLIENT_FUSE_DIR/target/debug/gvfs-fuse $MOUNT_DIR $FILESET $CONF_FILE &
+FUSE_PID=$!
 
+# run the integration test
+cd $CLIENT_FUSE_DIR
+make integration_test
+cd -
 
+# Stop the gvfs-fuse process after the test completes
+echo "Stopping the Gvfs fuse client..."
+kill -INT $FUSE_PID
+
+# Stop the Gravitino server
+$GRAVITINO_SERVER_DIR/bin/gravitino.sh stop

From 4e34bd2449ee6579236da5d3b97dd046c2255612 Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Tue, 7 Jan 2025 14:32:40 +0800
Subject: [PATCH 09/29] Add integration test

---
 .../src/gravitino_fileset_filesystem.rs       |  5 +--
 clients/filesystem-fuse/src/lib.rs            |  3 ++
 .../src/open_dal_filesystem.rs                |  5 +++
 clients/filesystem-fuse/src/s3_filesystem.rs  |  6 ++--
 .../tests/bin/start_backend.sh                | 36 ++++++++++++-------
 clients/filesystem-fuse/tests/fuse_test.rs    | 12 +++++--
 6 files changed, 48 insertions(+), 19 deletions(-)

diff --git a/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs b/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs
index b7b69d5cc7b..e52cc26ca45 100644
--- a/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs
+++ b/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs
@@ -149,6 +149,7 @@ mod tests {
     use crate::gvfs_creator::create_fs_with_fileset;
     use crate::memory_filesystem::MemoryFileSystem;
     use crate::s3_filesystem::tests::cleanup_s3_fs;
+    use crate::TEST_ENV_WITH_S3;
     use std::collections::HashMap;
     use std::path::Path;
 
@@ -212,7 +213,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_fileset_file_system() {
-        if std::env::var("RUN_S3_TESTS").is_err() {
+        if std::env::var(TEST_ENV_WITH_S3).is_err() {
             return;
         }
         let cwd = Path::new("/gvfs_test3");
@@ -224,7 +225,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_fileset_with_raw_file_system() {
-        if std::env::var("RUN_S3_TESTS").is_err() {
+        if std::env::var(TEST_ENV_WITH_S3).is_err() {
             return;
         }
 
diff --git a/clients/filesystem-fuse/src/lib.rs b/clients/filesystem-fuse/src/lib.rs
index 31e7c7fd8e1..4643b5dd9d4 100644
--- a/clients/filesystem-fuse/src/lib.rs
+++ b/clients/filesystem-fuse/src/lib.rs
@@ -36,6 +36,9 @@ mod opened_file_manager;
 mod s3_filesystem;
 mod utils;
 
+pub const TEST_ENV_WITH_S3: &str = "RUN_TEST_WITH_S3";
+pub const TEST_ENV_WITH_BACKGROUND: &str = "RUN_TEST_WITH_BACKGROUND";
+
 pub async fn gvfs_mount(mount_to: &str, mount_from: &str, config: &AppConfig) -> GvfsResult<()> {
     gvfs_fuse::mount(mount_to, mount_from, config).await
 }
diff --git a/clients/filesystem-fuse/src/open_dal_filesystem.rs b/clients/filesystem-fuse/src/open_dal_filesystem.rs
index e53fbaf6032..f9f34584bf6 100644
--- a/clients/filesystem-fuse/src/open_dal_filesystem.rs
+++ b/clients/filesystem-fuse/src/open_dal_filesystem.rs
@@ -261,11 +261,16 @@ fn opendal_filemode_to_filetype(mode: EntryMode) -> FileType {
 mod test {
     use crate::config::AppConfig;
     use crate::s3_filesystem::extract_s3_config;
+    use crate::TEST_ENV_WITH_S3;
     use opendal::layers::LoggingLayer;
     use opendal::{services, Builder, Operator};
 
     #[tokio::test]
     async fn test_s3_stat() {
+        if std::env::var(TEST_ENV_WITH_S3).is_err() {
+            return;
+        }
+
         let config = AppConfig::from_file(Some("tests/conf/gvfs_fuse_s3.toml")).unwrap();
         let opendal_config = extract_s3_config(&config);
 
diff --git a/clients/filesystem-fuse/src/s3_filesystem.rs b/clients/filesystem-fuse/src/s3_filesystem.rs
index e883f560f86..3161755a41c 100644
--- a/clients/filesystem-fuse/src/s3_filesystem.rs
+++ b/clients/filesystem-fuse/src/s3_filesystem.rs
@@ -187,6 +187,7 @@ pub(crate) mod tests {
     use crate::filesystem::tests::{TestPathFileSystem, TestRawFileSystem};
     use crate::filesystem::RawFileSystem;
     use crate::utils::delete_dir;
+    use crate::TEST_ENV_WITH_S3;
     use opendal::layers::TimeoutLayer;
     use std::time::Duration;
 
@@ -242,9 +243,10 @@ pub(crate) mod tests {
 
     #[tokio::test]
     async fn test_s3_file_system() {
-        if std::env::var("RUN_S3_TESTS").is_err() {
+        if std::env::var(TEST_ENV_WITH_S3).is_err() {
             return;
         }
+
         let cwd = Path::new("/gvfs_test1");
         let fs = create_s3_fs(cwd).await;
 
@@ -255,7 +257,7 @@ pub(crate) mod tests {
 
     #[tokio::test]
     async fn test_s3_file_system_with_raw_file_system() {
-        if std::env::var("RUN_S3_TESTS").is_err() {
+        if std::env::var(TEST_ENV_WITH_S3).is_err() {
             return;
         }
 
diff --git a/clients/filesystem-fuse/tests/bin/start_backend.sh b/clients/filesystem-fuse/tests/bin/start_backend.sh
index b12b0a56333..1c472665d58 100755
--- a/clients/filesystem-fuse/tests/bin/start_backend.sh
+++ b/clients/filesystem-fuse/tests/bin/start_backend.sh
@@ -32,6 +32,7 @@ if [[ -z "$S3_ACCESS_KEY_ID" || -z "$S3_SECRET_ACCESS" || -z "$S3_REGION" || -z
 fi
 
 GRAVITINO_HOME=../../../..
+GRAVITINO_HOME=$(cd $GRAVITINO_HOME && pwd)
 GRAVITINO_SERVER_DIR=$GRAVITINO_HOME/distribution/package
 CLIENT_FUSE_DIR=$GRAVITINO_HOME/clients/filesystem-fuse
 GRAVITINO_SERVER_URL=http://localhost:8090
@@ -49,7 +50,7 @@ echo "Start the Gravitino server"
 rm -rf $GRAVITINO_SERVER_DIR/data
 $GRAVITINO_SERVER_DIR/bin/gravitino.sh restart
 
-check_server_ready() {
+check_gravitino_server_ready() {
   local url=$1
   local retries=10  # Number of retries
   local wait_time=1 # Wait time between retries (seconds)
@@ -68,7 +69,7 @@ check_server_ready() {
   exit 1
 }
 
-check_server_ready "$GRAVITINO_SERVER_URL/api/metalakes"
+check_gravitino_server_ready "$GRAVITINO_SERVER_URL/api/metalakes"
 
 # create metalake
 curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \
@@ -115,7 +116,7 @@ if [ ! -d "$MOUNT_DIR" ]; then
   mkdir -p $MOUNT_DIR
 fi
 
-FILESET=gvfs://fileset/test/c1/s1/fileset1
+MOUNT_FROM_LOCATION=gvfs://fileset/test/c1/s1/fileset1
 
 CONF_FILE=$CLIENT_FUSE_DIR/target/debug/gvfs-fuse.toml
 
@@ -131,18 +132,27 @@ awk -v access_key="$S3_ACCESS_KEY_ID" \
     in_extend_config && /s3-bucket/ { $0 = "s3-bucket = \"" bucket "\"" }
     { print }' $CLIENT_FUSE_DIR/tests/conf/gvfs_fuse_s3.toml > "$CONF_FILE"
 
+cleanup() {
+    # Stop the gvfs-fuse process if it's running
+    if [ -n "$FUSE_PID" ] && ps -p $FUSE_PID > /dev/null; then
+        echo "Stopping gvfs-fuse..."
+        kill -INT $FUSE_PID
+    else
+        echo "gvfs-fuse process not found or already stopped."
+    fi
+
+    # Stop the Gravitino server
+    echo "Stopping Gravitino server..."
+    $GRAVITINO_SERVER_DIR/bin/gravitino.sh stop || echo "Failed to stop Gravitino server."
+}
+trap cleanup EXIT
+
 # Start the gvfs-fuse process in the background
-$CLIENT_FUSE_DIR/target/debug/gvfs-fuse $MOUNT_DIR $FILESET $CONF_FILE &
+$CLIENT_FUSE_DIR/target/debug/gvfs-fuse $MOUNT_DIR $MOUNT_FROM_LOCATION $CONF_FILE > $CLIENT_FUSE_DIR/target/debug/fuse.log 2>&1 &
 FUSE_PID=$!
+echo "Gvfs fuse started with PID: $FUSE_PID"
 
 # run the integration test
 cd $CLIENT_FUSE_DIR
-make integration_test
-cd -
-
-# Stop the gvfs-fuse process after the test completes
-echo "Stopping the Gvfs fuse client..."
-kill -INT $FUSE_PID
-
-# Stop the Gravitino server
-$GRAVITINO_SERVER_DIR/bin/gravitino.sh stop
+export RUN_TEST_WITH_BACKGROUND=1
+cargo test --test fuse_test test_fuse_system_with_manual -- --exact
diff --git a/clients/filesystem-fuse/tests/fuse_test.rs b/clients/filesystem-fuse/tests/fuse_test.rs
index d06199d782e..adee45a3ec2 100644
--- a/clients/filesystem-fuse/tests/fuse_test.rs
+++ b/clients/filesystem-fuse/tests/fuse_test.rs
@@ -19,7 +19,7 @@
 
 use fuse3::Errno;
 use gvfs_fuse::config::AppConfig;
-use gvfs_fuse::{gvfs_mount, gvfs_unmount};
+use gvfs_fuse::{gvfs_mount, gvfs_unmount, TEST_ENV_WITH_BACKGROUND};
 use log::{error, info};
 use std::fs::File;
 use std::path::Path;
@@ -106,14 +106,22 @@ fn test_fuse_system_with_auto() {
     test_fuse_filesystem(mount_point);
 }
 
+#[test]
 fn test_fuse_system_with_manual() {
-    test_fuse_filesystem("build/gvfs");
+    if std::env::var(TEST_ENV_WITH_BACKGROUND).is_err() {
+        return;
+    }
+    test_fuse_filesystem("target/gvfs/gvfs_test");
 }
 
 fn test_fuse_filesystem(mount_point: &str) {
     info!("Test startup");
     let base_path = Path::new(mount_point);
 
+    if !file_exists(base_path) {
+        fs::create_dir_all(base_path).expect("Failed to create test dir");
+    }
+
     //test create file
     let test_file = base_path.join("test_create");
     let file = File::create(&test_file).expect("Failed to create file");

From 1f014f6e1f0869507ee031e3e10d71a9f4c04ae5 Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Tue, 7 Jan 2025 16:10:40 +0800
Subject: [PATCH 10/29] fix some logs

---
 .../tests/bin/start_backend.sh                | 34 ++++++++++++++++---
 clients/filesystem-fuse/tests/fuse_test.rs    |  6 ++--
 2 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/clients/filesystem-fuse/tests/bin/start_backend.sh b/clients/filesystem-fuse/tests/bin/start_backend.sh
index 1c472665d58..7118f8e6496 100755
--- a/clients/filesystem-fuse/tests/bin/start_backend.sh
+++ b/clients/filesystem-fuse/tests/bin/start_backend.sh
@@ -71,14 +71,15 @@ check_gravitino_server_ready() {
 
 check_gravitino_server_ready "$GRAVITINO_SERVER_URL/api/metalakes"
 
+echo "Create the metalake, catalog, schema, and fileset"
 # create metalake
-curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \
+curl -s -o /dev/null -X POST -H "Accept: application/vnd.gravitino.v1+json" \
 -H "Content-Type: application/json" -d '{
   "name":"test","comment":"comment","properties":{}
 }' $GRAVITINO_SERVER_URL/api/metalakes
 
 # create catalog
-curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \
+curl -s -o /dev/null -X POST -H "Accept: application/vnd.gravitino.v1+json" \
 -H "Content-Type: application/json" -d '{
   "name": "c1",
   "type": "FILESET",
@@ -94,18 +95,17 @@ curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \
 }' $GRAVITINO_SERVER_URL/api/metalakes/test/catalogs
 
 # create schema
-curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \
+curl -s -o /dev/null -X POST -H "Accept: application/vnd.gravitino.v1+json" \
 -H "Content-Type: application/json" -d '{
   "name":"s1","comment":"comment","properties":{}
 }' $GRAVITINO_SERVER_URL/api/metalakes/test/catalogs/c1/schemas
 
 # create FILESET
-curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \
+curl -s -o /dev/null -X POST -H "Accept: application/vnd.gravitino.v1+json" \
 -H "Content-Type: application/json" -d '{
   "name":"fileset1","comment":"comment","properties":{}
 }' $GRAVITINO_SERVER_URL/api/metalakes/test/catalogs/c1/schemas/s1/filesets
 
-
 echo "Start the Gvfs fuse client"
 
 MOUNT_DIR=$CLIENT_FUSE_DIR/target/gvfs
@@ -152,7 +152,31 @@ $CLIENT_FUSE_DIR/target/debug/gvfs-fuse $MOUNT_DIR $MOUNT_FROM_LOCATION $CONF_FI
 FUSE_PID=$!
 echo "Gvfs fuse started with PID: $FUSE_PID"
 
+#check the gvfs-fuse is ready
+check_gvfs_fuse_ready() {
+  local retries=10
+  local wait_time=1
+
+  for ((i=1; i<=retries; i++)); do
+    # check the $MOUNT_DIR/.gvfs_meta is exist
+    if [ -f "$MOUNT_DIR/.gvfs_meta" ]; then
+      echo "Gvfs fuse is ready."
+      return 0
+    else
+      echo "Attempt $i/$retries: Gvfs fuse not ready. Retrying in $wait_time seconds..."
+      sleep "$wait_time"
+    fi
+  done
+
+  echo "Error: Gvfs fuse did not become ready after $((retries * wait_time)) seconds."
+  exit 1
+}
+
+check_gvfs_fuse_ready
+
 # run the integration test
 cd $CLIENT_FUSE_DIR
 export RUN_TEST_WITH_BACKGROUND=1
 cargo test --test fuse_test test_fuse_system_with_manual -- --exact
+
+sleep 3
diff --git a/clients/filesystem-fuse/tests/fuse_test.rs b/clients/filesystem-fuse/tests/fuse_test.rs
index adee45a3ec2..85cb678a433 100644
--- a/clients/filesystem-fuse/tests/fuse_test.rs
+++ b/clients/filesystem-fuse/tests/fuse_test.rs
@@ -132,12 +132,12 @@ fn test_fuse_filesystem(mount_point: &str) {
     fs::write(&test_file, "read test").expect("Failed to write file");
 
     //test read file
-    let content = fs::read_to_string(test_file.clone()).expect("Failed to read file");
+    let content = fs::read_to_string(&test_file).expect("Failed to read file");
     assert_eq!(content, "read test", "File content mismatch");
 
     //test delete file
-    fs::remove_file(test_file.clone()).expect("Failed to delete file");
-    assert!(!file_exists(test_file));
+    fs::remove_file(&test_file).expect("Failed to delete file");
+    assert!(!file_exists(&test_file));
 
     //test create directory
     let test_dir = base_path.join("test_dir");

From 0960c0883893e1f14059a86fc3ab180d80c770f4 Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Tue, 7 Jan 2025 19:53:16 +0800
Subject: [PATCH 11/29] Test s3

---
 .../src/default_raw_filesystem.rs             | 14 ++++--
 .../src/open_dal_filesystem.rs                | 46 +++++++++++++++----
 .../{start_backend.sh => s3_fileset_it.sh}    | 13 ++++--
 3 files changed, 59 insertions(+), 14 deletions(-)
 rename clients/filesystem-fuse/tests/bin/{start_backend.sh => s3_fileset_it.sh} (97%)

diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs
index 944181246d5..922af121197 100644
--- a/clients/filesystem-fuse/src/default_raw_filesystem.rs
+++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs
@@ -334,13 +334,21 @@ impl<T: PathFileSystem> RawFileSystem for DefaultRawFileSystem<T> {
         file.flush().await
     }
 
-    async fn close_file(&self, _file_id: u64, fh: u64) -> Result<()> {
+    async fn close_file(&self, file_id: u64, fh: u64) -> Result<()> {
+        let file_entry = self.get_file_entry(file_id).await;
+
         let opened_file = self
             .opened_file_manager
             .remove(fh)
             .ok_or(Errno::from(libc::EBADF))?;
-        let mut file = opened_file.lock().await;
-        file.close().await
+
+        // todo: handle race condition
+        if file_entry.is_ok() {
+            let mut file = opened_file.lock().await;
+            file.close().await
+        } else {
+            Ok(())
+        }
     }
 
     async fn read(&self, file_id: u64, fh: u64, offset: u64, size: u32) -> Result<Bytes> {
diff --git a/clients/filesystem-fuse/src/open_dal_filesystem.rs b/clients/filesystem-fuse/src/open_dal_filesystem.rs
index f9f34584bf6..d3207d98f99 100644
--- a/clients/filesystem-fuse/src/open_dal_filesystem.rs
+++ b/clients/filesystem-fuse/src/open_dal_filesystem.rs
@@ -265,23 +265,26 @@ mod test {
     use opendal::layers::LoggingLayer;
     use opendal::{services, Builder, Operator};
 
-    #[tokio::test]
-    async fn test_s3_stat() {
-        if std::env::var(TEST_ENV_WITH_S3).is_err() {
-            return;
-        }
-
+    fn create_opendal() -> Operator {
         let config = AppConfig::from_file(Some("tests/conf/gvfs_fuse_s3.toml")).unwrap();
         let opendal_config = extract_s3_config(&config);
 
         let builder = services::S3::from_map(opendal_config);
 
         // Init an operator
-        let op = Operator::new(builder)
+        Operator::new(builder)
             .expect("opendal create failed")
             .layer(LoggingLayer::default())
-            .finish();
+            .finish()
+    }
 
+    #[tokio::test]
+    async fn test_s3_stat() {
+        if std::env::var(TEST_ENV_WITH_S3).is_err() {
+            return;
+        }
+
+        let op = create_opendal();
         let path = "/";
         let list = op.list(path).await;
         if let Ok(l) = list {
@@ -299,4 +302,31 @@ mod test {
             println!("stat error: {:?}", meta.err());
         }
     }
+
+    #[tokio::test]
+    async fn test_s3_delete() {
+        if std::env::var(TEST_ENV_WITH_S3).is_err() {
+            return;
+        }
+
+        let op = create_opendal();
+        let path = "/s1/fileset1/gvfs_test/test_dir/test_file";
+
+        let meta = op.stat(path).await;
+        if let Ok(m) = meta {
+            println!("stat result: {:?}", m);
+        } else {
+            println!("stat error: {:?}", meta.err());
+        }
+
+        let result = op.remove(vec![path.to_string()]).await;
+        match result {
+            Ok(_) => {
+                println!("Delete successful (or no-op).");
+            }
+            Err(e) => {
+                println!("Delete failed: {:?}", e);
+            }
+        }
+    }
 }
diff --git a/clients/filesystem-fuse/tests/bin/start_backend.sh b/clients/filesystem-fuse/tests/bin/s3_fileset_it.sh
similarity index 97%
rename from clients/filesystem-fuse/tests/bin/start_backend.sh
rename to clients/filesystem-fuse/tests/bin/s3_fileset_it.sh
index 7118f8e6496..82ffef807ab 100755
--- a/clients/filesystem-fuse/tests/bin/start_backend.sh
+++ b/clients/filesystem-fuse/tests/bin/s3_fileset_it.sh
@@ -19,6 +19,9 @@
 
 set -euo pipefail
 
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR"
+
 S3_ACCESS_KEY_ID=${S3_ACCESS_KEY_ID:-}
 S3_SECRET_ACCESS=${S3_SECRET_ACCESS:-}
 S3_REGION=${S3_REGION:-}
@@ -147,7 +150,13 @@ cleanup() {
 }
 trap cleanup EXIT
 
+cd $CLIENT_FUSE_DIR
+
 # Start the gvfs-fuse process in the background
+echo "Starting gvfs-fuse"
+# Build the gvfs-fuse
+make build
+
 $CLIENT_FUSE_DIR/target/debug/gvfs-fuse $MOUNT_DIR $MOUNT_FROM_LOCATION $CONF_FILE > $CLIENT_FUSE_DIR/target/debug/fuse.log 2>&1 &
 FUSE_PID=$!
 echo "Gvfs fuse started with PID: $FUSE_PID"
@@ -175,8 +184,6 @@ check_gvfs_fuse_ready() {
 check_gvfs_fuse_ready
 
 # run the integration test
-cd $CLIENT_FUSE_DIR
+
 export RUN_TEST_WITH_BACKGROUND=1
 cargo test --test fuse_test test_fuse_system_with_manual -- --exact
-
-sleep 3

From 8f9bea8994ba97d70f105287629de5a985a09e18 Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Tue, 7 Jan 2025 19:56:45 +0800
Subject: [PATCH 12/29] Update

---
 clients/filesystem-fuse/src/default_raw_filesystem.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs
index 922af121197..d69022c0eaa 100644
--- a/clients/filesystem-fuse/src/default_raw_filesystem.rs
+++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs
@@ -342,7 +342,7 @@ impl<T: PathFileSystem> RawFileSystem for DefaultRawFileSystem<T> {
             .remove(fh)
             .ok_or(Errno::from(libc::EBADF))?;
 
-        // todo: handle race condition
+        // todo: need to handle racing condition
         if file_entry.is_ok() {
             let mut file = opened_file.lock().await;
             file.close().await

From 536ff25f26b49e88c2188baed8f85493f1344a7d Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Tue, 7 Jan 2025 20:03:16 +0800
Subject: [PATCH 13/29] Update

---
 clients/filesystem-fuse/src/s3_filesystem.rs | 14 +++++++++++++-
 clients/filesystem-fuse/src/utils.rs         | 14 --------------
 2 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/clients/filesystem-fuse/src/s3_filesystem.rs b/clients/filesystem-fuse/src/s3_filesystem.rs
index 3161755a41c..711e86dcf96 100644
--- a/clients/filesystem-fuse/src/s3_filesystem.rs
+++ b/clients/filesystem-fuse/src/s3_filesystem.rs
@@ -186,7 +186,6 @@ pub(crate) mod tests {
     use crate::default_raw_filesystem::DefaultRawFileSystem;
     use crate::filesystem::tests::{TestPathFileSystem, TestRawFileSystem};
     use crate::filesystem::RawFileSystem;
-    use crate::utils::delete_dir;
     use crate::TEST_ENV_WITH_S3;
     use opendal::layers::TimeoutLayer;
     use std::time::Duration;
@@ -207,6 +206,19 @@ pub(crate) mod tests {
         assert_eq!(result.unwrap(), "ap-southeast-2");
     }
 
+    pub(crate) async fn delete_dir(op: &Operator, dir_name: &str) {
+        let childs = op.list(dir_name).await.expect("list dir failed");
+        for child in childs {
+            let child_name = dir_name.to_string() + child.name();
+            if child.metadata().is_dir() {
+                Box::pin(delete_dir(op, &child_name)).await;
+            } else {
+                op.delete(&child_name).await.expect("delete file failed");
+            }
+        }
+        op.delete(dir_name).await.expect("delete dir failed");
+    }
+
     pub(crate) async fn cleanup_s3_fs(cwd: &Path) -> Operator {
         let config = AppConfig::from_file(Some("tests/conf/gvfs_fuse_s3.toml")).unwrap();
         let opendal_config = extract_s3_config(&config);
diff --git a/clients/filesystem-fuse/src/utils.rs b/clients/filesystem-fuse/src/utils.rs
index 73af044ee4c..53eb9179d71 100644
--- a/clients/filesystem-fuse/src/utils.rs
+++ b/clients/filesystem-fuse/src/utils.rs
@@ -18,7 +18,6 @@
  */
 use crate::error::ErrorCode::InvalidConfig;
 use crate::error::GvfsError;
-use opendal::Operator;
 use reqwest::Url;
 use std::path::PathBuf;
 
@@ -37,19 +36,6 @@ pub(crate) fn extract_root_path(location: &str) -> GvfsResult<PathBuf> {
     Ok(PathBuf::from(url.path()))
 }
 
-pub(crate) async fn delete_dir(op: &Operator, dir_name: &str) {
-    let childs = op.list(dir_name).await.expect("list dir failed");
-    for child in childs {
-        let child_name = dir_name.to_string() + child.name();
-        if child.metadata().is_dir() {
-            Box::pin(delete_dir(op, &child_name)).await;
-        } else {
-            op.delete(&child_name).await.expect("delete file failed");
-        }
-    }
-    op.delete(dir_name).await.expect("delete dir failed");
-}
-
 #[cfg(test)]
 mod tests {
     use crate::utils::extract_root_path;

From 52649d7f82f4f5c7a083c8ed4480638c59b3d547 Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Wed, 8 Jan 2025 16:25:08 +0800
Subject: [PATCH 14/29] refector s3 testcases

---
 .../src/gravitino_fileset_filesystem.rs       |  47 ++++----
 clients/filesystem-fuse/src/gvfs_creator.rs   |  10 +-
 clients/filesystem-fuse/src/lib.rs            |  14 ++-
 .../src/open_dal_filesystem.rs                |  24 ++---
 clients/filesystem-fuse/src/s3_filesystem.rs  | 101 +++++++++---------
 .../tests/conf/gvfs_fuse_s3.toml              |   1 +
 clients/filesystem-fuse/tests/fuse_test.rs    |   8 +-
 7 files changed, 111 insertions(+), 94 deletions(-)

diff --git a/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs b/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs
index e52cc26ca45..0860def6c5f 100644
--- a/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs
+++ b/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs
@@ -148,8 +148,10 @@ mod tests {
     use crate::gravitino_fileset_filesystem::GravitinoFilesetFileSystem;
     use crate::gvfs_creator::create_fs_with_fileset;
     use crate::memory_filesystem::MemoryFileSystem;
-    use crate::s3_filesystem::tests::cleanup_s3_fs;
-    use crate::TEST_ENV_WITH_S3;
+    use crate::s3_filesystem::extract_s3_config;
+    use crate::s3_filesystem::tests::{cleanup_s3_fs, s3_test_config};
+    use crate::test_enable_with;
+    use crate::RUN_TEST_WITH_S3;
     use std::collections::HashMap;
     use std::path::Path;
 
@@ -181,31 +183,35 @@ mod tests {
         assert_eq!(path, Path::new("/"));
     }
 
-    async fn create_fileset_fs(path: &Path) -> GravitinoFilesetFileSystem {
-        cleanup_s3_fs(path).await;
+    async fn create_fileset_fs(path: &Path, config: &AppConfig) -> GravitinoFilesetFileSystem {
+        let opendal_config = extract_s3_config(config);
+
+        cleanup_s3_fs(path, &opendal_config).await;
+
+        let bucket = opendal_config.get("bucket").expect("Bucket must exist");
+        let endpoint = opendal_config.get("endpoint").expect("Endpoint must exist");
 
         let catalog = Catalog::new(
             "c1",
             vec![
-                ("location".to_string(), "s3a://trino-test-ice".to_string()),
-                (
-                    "s3-endpoint".to_string(),
-                    "http://s3.ap-southeast-2.amazonaws.com".to_string(),
-                ),
+                ("location".to_string(), format!("s3a://{}", bucket)),
+                ("s3-endpoint".to_string(), endpoint.to_string()),
             ]
             .into_iter()
             .collect::<HashMap<String, String>>(),
         );
-        let file_set_location = format!("s3a://trino-test-ice{}", path.to_string_lossy());
+        let file_set_location = format!("s3a://{}{}", bucket, path.to_string_lossy());
         let file_set = Fileset::new("fileset1", &file_set_location);
-        let config = AppConfig::from_file(Some("tests/conf/gvfs_fuse_s3.toml")).unwrap();
+
         let fs_context = FileSystemContext::default();
-        let inner_fs = create_fs_with_fileset(&catalog, &file_set, &config, &fs_context).unwrap();
+        let inner_fs = create_fs_with_fileset(&catalog, &file_set, config, &fs_context)
+            .await
+            .unwrap();
         GravitinoFilesetFileSystem::new(
             inner_fs,
             path,
             GravitinoClient::new(&config.gravitino),
-            &config,
+            config,
             &fs_context,
         )
         .await
@@ -213,11 +219,11 @@ mod tests {
 
     #[tokio::test]
     async fn test_fileset_file_system() {
-        if std::env::var(TEST_ENV_WITH_S3).is_err() {
-            return;
-        }
+        test_enable_with!(RUN_TEST_WITH_S3);
+
+        let config = s3_test_config();
         let cwd = Path::new("/gvfs_test3");
-        let fs = create_fileset_fs(cwd).await;
+        let fs = create_fileset_fs(cwd, &config).await;
         let _ = fs.init().await;
         let mut tester = TestPathFileSystem::new(Path::new("/"), fs);
         tester.test_path_file_system().await;
@@ -225,12 +231,11 @@ mod tests {
 
     #[tokio::test]
     async fn test_fileset_with_raw_file_system() {
-        if std::env::var(TEST_ENV_WITH_S3).is_err() {
-            return;
-        }
+        test_enable_with!(RUN_TEST_WITH_S3);
 
+        let config = s3_test_config();
         let cwd = Path::new("/gvfs_test4");
-        let fileset_fs = create_fileset_fs(cwd).await;
+        let fileset_fs = create_fileset_fs(cwd, &config).await;
         let raw_fs = DefaultRawFileSystem::new(
             fileset_fs,
             &AppConfig::default(),
diff --git a/clients/filesystem-fuse/src/gvfs_creator.rs b/clients/filesystem-fuse/src/gvfs_creator.rs
index f9bdfc85674..88bc8a1b422 100644
--- a/clients/filesystem-fuse/src/gvfs_creator.rs
+++ b/clients/filesystem-fuse/src/gvfs_creator.rs
@@ -87,7 +87,7 @@ pub async fn create_gvfs_filesystem(
         .get_fileset(&catalog_name, &schema_name, &fileset_name)
         .await?;
 
-    let inner_fs = create_fs_with_fileset(&catalog, &fileset, config, fs_context)?;
+    let inner_fs = create_fs_with_fileset(&catalog, &fileset, config, fs_context).await?;
 
     let target_path = extract_root_path(fileset.storage_location.as_str())?;
     let fs =
@@ -95,7 +95,7 @@ pub async fn create_gvfs_filesystem(
     Ok(CreateFileSystemResult::Gvfs(fs))
 }
 
-pub(crate) fn create_fs_with_fileset(
+pub(crate) async fn create_fs_with_fileset(
     catalog: &Catalog,
     fileset: &Fileset,
     config: &AppConfig,
@@ -104,9 +104,9 @@ pub(crate) fn create_fs_with_fileset(
     let schema = extract_filesystem_scheme(&fileset.storage_location)?;
 
     match schema {
-        FileSystemSchema::S3 => Ok(Box::new(S3FileSystem::new(
-            catalog, fileset, config, fs_context,
-        )?)),
+        FileSystemSchema::S3 => Ok(Box::new(
+            S3FileSystem::new(catalog, fileset, config, fs_context).await?,
+        )),
     }
 }
 
diff --git a/clients/filesystem-fuse/src/lib.rs b/clients/filesystem-fuse/src/lib.rs
index 4643b5dd9d4..7132f1c9471 100644
--- a/clients/filesystem-fuse/src/lib.rs
+++ b/clients/filesystem-fuse/src/lib.rs
@@ -36,8 +36,18 @@ mod opened_file_manager;
 mod s3_filesystem;
 mod utils;
 
-pub const TEST_ENV_WITH_S3: &str = "RUN_TEST_WITH_S3";
-pub const TEST_ENV_WITH_BACKGROUND: &str = "RUN_TEST_WITH_BACKGROUND";
+#[macro_export]
+macro_rules! test_enable_with {
+    ($env_var:expr) => {
+        if std::env::var($env_var).is_err() {
+            println!("Test skipped because {} is not set", $env_var);
+            return;
+        }
+    };
+}
+
+pub const RUN_TEST_WITH_S3: &str = "RUN_TEST_WITH_S3";
+pub const RUN_TEST_WITH_BACKEND: &str = "RUN_TEST_WITH_BACKEND";
 
 pub async fn gvfs_mount(mount_to: &str, mount_from: &str, config: &AppConfig) -> GvfsResult<()> {
     gvfs_fuse::mount(mount_to, mount_from, config).await
diff --git a/clients/filesystem-fuse/src/open_dal_filesystem.rs b/clients/filesystem-fuse/src/open_dal_filesystem.rs
index d3207d98f99..b116f8fec5f 100644
--- a/clients/filesystem-fuse/src/open_dal_filesystem.rs
+++ b/clients/filesystem-fuse/src/open_dal_filesystem.rs
@@ -261,14 +261,14 @@ fn opendal_filemode_to_filetype(mode: EntryMode) -> FileType {
 mod test {
     use crate::config::AppConfig;
     use crate::s3_filesystem::extract_s3_config;
-    use crate::TEST_ENV_WITH_S3;
+    use crate::s3_filesystem::tests::s3_test_config;
+    use crate::test_enable_with;
+    use crate::RUN_TEST_WITH_S3;
     use opendal::layers::LoggingLayer;
     use opendal::{services, Builder, Operator};
 
-    fn create_opendal() -> Operator {
-        let config = AppConfig::from_file(Some("tests/conf/gvfs_fuse_s3.toml")).unwrap();
-        let opendal_config = extract_s3_config(&config);
-
+    fn create_opendal(config: &AppConfig) -> Operator {
+        let opendal_config = extract_s3_config(config);
         let builder = services::S3::from_map(opendal_config);
 
         // Init an operator
@@ -280,11 +280,10 @@ mod test {
 
     #[tokio::test]
     async fn test_s3_stat() {
-        if std::env::var(TEST_ENV_WITH_S3).is_err() {
-            return;
-        }
+        test_enable_with!(RUN_TEST_WITH_S3);
 
-        let op = create_opendal();
+        let config = s3_test_config();
+        let op = create_opendal(&config);
         let path = "/";
         let list = op.list(path).await;
         if let Ok(l) = list {
@@ -305,11 +304,10 @@ mod test {
 
     #[tokio::test]
     async fn test_s3_delete() {
-        if std::env::var(TEST_ENV_WITH_S3).is_err() {
-            return;
-        }
+        test_enable_with!(RUN_TEST_WITH_S3);
+        let config = s3_test_config();
 
-        let op = create_opendal();
+        let op = create_opendal(&config);
         let path = "/s1/fileset1/gvfs_test/test_dir/test_file";
 
         let meta = op.stat(path).await;
diff --git a/clients/filesystem-fuse/src/s3_filesystem.rs b/clients/filesystem-fuse/src/s3_filesystem.rs
index 711e86dcf96..668ffa4da5d 100644
--- a/clients/filesystem-fuse/src/s3_filesystem.rs
+++ b/clients/filesystem-fuse/src/s3_filesystem.rs
@@ -40,7 +40,7 @@ impl S3FileSystem {}
 impl S3FileSystem {
     const S3_CONFIG_PREFIX: &'static str = "s3-";
 
-    pub(crate) fn new(
+    pub(crate) async fn new(
         catalog: &Catalog,
         fileset: &Fileset,
         config: &AppConfig,
@@ -48,10 +48,20 @@ impl S3FileSystem {
     ) -> GvfsResult<Self> {
         let mut opendal_config = extract_s3_config(config);
         let bucket = extract_bucket(&fileset.storage_location)?;
-        opendal_config.insert("bucket".to_string(), bucket);
+        opendal_config.insert("bucket".to_string(), bucket.to_string());
 
-        let region = Self::get_s3_region(catalog)?;
-        opendal_config.insert("region".to_string(), region);
+        let endpoint = catalog.properties.get("s3-endpoint");
+        if endpoint.is_none() {
+            return Err(InvalidConfig.to_error("s3-endpoint is required".to_string()));
+        }
+        let endpoint = endpoint.unwrap();
+        opendal_config.insert("endpoint".to_string(), endpoint.clone());
+
+        let region = Self::get_s3_region(catalog, &bucket).await;
+        if region.is_none() {
+            return Err(InvalidConfig.to_error("s3-region is required".to_string()));
+        }
+        opendal_config.insert("region".to_string(), region.unwrap());
 
         let builder = S3::from_map(opendal_config);
 
@@ -67,16 +77,13 @@ impl S3FileSystem {
         })
     }
 
-    fn get_s3_region(catalog: &Catalog) -> GvfsResult<String> {
+    async fn get_s3_region(catalog: &Catalog, bucket: &str) -> Option<String> {
         if let Some(region) = catalog.properties.get("s3-region") {
-            Ok(region.clone())
+            Some(region.clone())
         } else if let Some(endpoint) = catalog.properties.get("s3-endpoint") {
-            extract_region(endpoint)
+            S3::detect_region(endpoint, bucket).await
         } else {
-            Err(InvalidConfig.to_error(format!(
-                "Cant not retrieve region in the Catalog {}",
-                catalog.name
-            )))
+            None
         }
     }
 }
@@ -139,25 +146,11 @@ pub(crate) fn extract_bucket(location: &str) -> GvfsResult<String> {
     }
 }
 
-pub(crate) fn extract_region(location: &str) -> GvfsResult<String> {
-    let url = parse_location(location)?;
-    match url.host_str() {
-        Some(host) => {
-            let parts: Vec<&str> = host.split('.').collect();
-            if parts.len() > 1 {
-                Ok(parts[1].to_string())
-            } else {
-                Err(InvalidConfig.to_error(format!(
-                    "Invalid location: expected region in host, got {}",
-                    location
-                )))
-            }
-        }
-        None => Err(InvalidConfig.to_error(format!(
-            "Invalid fileset location without bucket: {}",
-            location
-        ))),
-    }
+pub(crate) fn extract_region(location: &str) -> Option<String> {
+    parse_location(location).ok().and_then(|url| {
+        url.host_str()
+            .and_then(|host| host.split('.').nth(1).map(|part| part.to_string()))
+    })
 }
 
 pub fn extract_s3_config(config: &AppConfig) -> HashMap<String, String> {
@@ -186,7 +179,8 @@ pub(crate) mod tests {
     use crate::default_raw_filesystem::DefaultRawFileSystem;
     use crate::filesystem::tests::{TestPathFileSystem, TestRawFileSystem};
     use crate::filesystem::RawFileSystem;
-    use crate::TEST_ENV_WITH_S3;
+    use crate::test_enable_with;
+    use crate::RUN_TEST_WITH_S3;
     use opendal::layers::TimeoutLayer;
     use std::time::Duration;
 
@@ -202,7 +196,7 @@ pub(crate) mod tests {
     fn test_extract_region() {
         let location = "http://s3.ap-southeast-2.amazonaws.com";
         let result = extract_region(location);
-        assert!(result.is_ok());
+        assert!(result.is_some());
         assert_eq!(result.unwrap(), "ap-southeast-2");
     }
 
@@ -219,11 +213,11 @@ pub(crate) mod tests {
         op.delete(dir_name).await.expect("delete dir failed");
     }
 
-    pub(crate) async fn cleanup_s3_fs(cwd: &Path) -> Operator {
-        let config = AppConfig::from_file(Some("tests/conf/gvfs_fuse_s3.toml")).unwrap();
-        let opendal_config = extract_s3_config(&config);
-
-        let builder = S3::from_map(opendal_config);
+    pub(crate) async fn cleanup_s3_fs(
+        cwd: &Path,
+        opendal_config: &HashMap<String, String>,
+    ) -> Operator {
+        let builder = S3::from_map(opendal_config.clone());
         let op = Operator::new(builder)
             .expect("opendal create failed")
             .layer(LoggingLayer::default())
@@ -243,24 +237,34 @@ pub(crate) mod tests {
         op
     }
 
-    async fn create_s3_fs(cwd: &Path) -> S3FileSystem {
-        let op = cleanup_s3_fs(cwd).await;
+    async fn create_s3_fs(cwd: &Path, config: &AppConfig) -> S3FileSystem {
+        let opendal_config = extract_s3_config(config);
+        let op = cleanup_s3_fs(cwd, &opendal_config).await;
 
-        let config = AppConfig::from_file(Some("tests/conf/gvfs_fuse_s3.toml")).unwrap();
         let fs_context = FileSystemContext::default();
-        let open_dal_fs = OpenDalFileSystem::new(op, &config, &fs_context);
+        let open_dal_fs = OpenDalFileSystem::new(op, config, &fs_context);
 
         S3FileSystem { open_dal_fs }
     }
 
+    pub(crate) fn s3_test_config() -> AppConfig {
+        let mut config_file_name = "target/conf/gvfs_fuse_s3.toml";
+        let source_file_name = "tests/conf/gvfs_fuse_s3.toml";
+
+        if !Path::new(config_file_name).exists() {
+            config_file_name = source_file_name;
+        }
+
+        AppConfig::from_file(Some(config_file_name)).unwrap()
+    }
+
     #[tokio::test]
     async fn test_s3_file_system() {
-        if std::env::var(TEST_ENV_WITH_S3).is_err() {
-            return;
-        }
+        test_enable_with!(RUN_TEST_WITH_S3);
 
+        let config = s3_test_config();
         let cwd = Path::new("/gvfs_test1");
-        let fs = create_s3_fs(cwd).await;
+        let fs = create_s3_fs(cwd, &config).await;
 
         let _ = fs.init().await;
         let mut tester = TestPathFileSystem::new(cwd, fs);
@@ -269,12 +273,11 @@ pub(crate) mod tests {
 
     #[tokio::test]
     async fn test_s3_file_system_with_raw_file_system() {
-        if std::env::var(TEST_ENV_WITH_S3).is_err() {
-            return;
-        }
+        test_enable_with!(RUN_TEST_WITH_S3);
 
+        let config = s3_test_config();
         let cwd = Path::new("/gvfs_test2");
-        let s3_fs = create_s3_fs(cwd).await;
+        let s3_fs = create_s3_fs(cwd, &config).await;
         let raw_fs =
             DefaultRawFileSystem::new(s3_fs, &AppConfig::default(), &FileSystemContext::default());
         let _ = raw_fs.init().await;
diff --git a/clients/filesystem-fuse/tests/conf/gvfs_fuse_s3.toml b/clients/filesystem-fuse/tests/conf/gvfs_fuse_s3.toml
index cf61f6f1265..d0ff8e5ddec 100644
--- a/clients/filesystem-fuse/tests/conf/gvfs_fuse_s3.toml
+++ b/clients/filesystem-fuse/tests/conf/gvfs_fuse_s3.toml
@@ -40,4 +40,5 @@ s3-access_key_id = "XXX_access_key"
 s3-secret_access_key = "XXX_secret_key"
 s3-region = "XXX_region"
 s3-bucket = "XXX_bucket"
+s3-endpoint = "XXX_endpoint"
 
diff --git a/clients/filesystem-fuse/tests/fuse_test.rs b/clients/filesystem-fuse/tests/fuse_test.rs
index 85cb678a433..de1d64d9b9a 100644
--- a/clients/filesystem-fuse/tests/fuse_test.rs
+++ b/clients/filesystem-fuse/tests/fuse_test.rs
@@ -19,7 +19,8 @@
 
 use fuse3::Errno;
 use gvfs_fuse::config::AppConfig;
-use gvfs_fuse::{gvfs_mount, gvfs_unmount, TEST_ENV_WITH_BACKGROUND};
+use gvfs_fuse::RUN_TEST_WITH_BACKEND;
+use gvfs_fuse::{gvfs_mount, gvfs_unmount, test_enable_with};
 use log::{error, info};
 use std::fs::File;
 use std::path::Path;
@@ -108,9 +109,8 @@ fn test_fuse_system_with_auto() {
 
 #[test]
 fn test_fuse_system_with_manual() {
-    if std::env::var(TEST_ENV_WITH_BACKGROUND).is_err() {
-        return;
-    }
+    test_enable_with!(RUN_TEST_WITH_BACKEND);
+
     test_fuse_filesystem("target/gvfs/gvfs_test");
 }
 

From 00bb21f9f90b290600695b8218eb8ed84d2d7467 Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Wed, 8 Jan 2025 16:37:52 +0800
Subject: [PATCH 15/29] Fix

---
 clients/filesystem-fuse/tests/bin/s3_fileset_it.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clients/filesystem-fuse/tests/bin/s3_fileset_it.sh b/clients/filesystem-fuse/tests/bin/s3_fileset_it.sh
index 82ffef807ab..0cef0544336 100755
--- a/clients/filesystem-fuse/tests/bin/s3_fileset_it.sh
+++ b/clients/filesystem-fuse/tests/bin/s3_fileset_it.sh
@@ -92,7 +92,7 @@ curl -s -o /dev/null -X POST -H "Accept: application/vnd.gravitino.v1+json" \
     "location": "s3a://'"$S3_BUCKET"'",
     "s3-access-key-id": "'"$S3_ACCESS_KEY_ID"'",
     "s3-secret-access-key": "'"$S3_SECRET_ACCESS"'",
-    "s3-endpoint": "http://s3.'"$S3_REGION"'.amazonaws.com",
+    "s3-endpoint": "https://s3.amazonaws.com",
     "filesystem-providers": "s3"
   }
 }' $GRAVITINO_SERVER_URL/api/metalakes/test/catalogs
@@ -185,5 +185,5 @@ check_gvfs_fuse_ready
 
 # run the integration test
 
-export RUN_TEST_WITH_BACKGROUND=1
+export RUN_TEST_WITH_BACKEND=1
 cargo test --test fuse_test test_fuse_system_with_manual -- --exact

From 6c29810ad993b5d862e8ab6f395b4598a1388eb3 Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Thu, 9 Jan 2025 11:28:32 +0800
Subject: [PATCH 16/29] Integration with localstack

---
 clients/filesystem-fuse/Makefile              |   6 +
 .../src/gravitino_fileset_filesystem.rs       |   4 +-
 clients/filesystem-fuse/src/lib.rs            |   2 +-
 .../src/open_dal_filesystem.rs                |   4 +-
 clients/filesystem-fuse/src/s3_filesystem.rs  |   4 +-
 clients/filesystem-fuse/tests/bin/env.sh      |  45 +++++
 .../tests/bin/gravitino_server.sh             | 116 +++++++++++
 .../filesystem-fuse/tests/bin/gvfs_fuse.sh    |  80 ++++++++
 .../filesystem-fuse/tests/bin/localstatck.sh  |  46 +++++
 .../tests/bin/s3_fileset_it.sh                | 190 +++---------------
 clients/filesystem-fuse/tests/bin/s3_test.sh  |  62 ++++++
 clients/filesystem-fuse/tests/fuse_test.rs    |   8 +-
 12 files changed, 399 insertions(+), 168 deletions(-)
 create mode 100644 clients/filesystem-fuse/tests/bin/env.sh
 create mode 100644 clients/filesystem-fuse/tests/bin/gravitino_server.sh
 create mode 100644 clients/filesystem-fuse/tests/bin/gvfs_fuse.sh
 create mode 100644 clients/filesystem-fuse/tests/bin/localstatck.sh
 create mode 100644 clients/filesystem-fuse/tests/bin/s3_test.sh

diff --git a/clients/filesystem-fuse/Makefile b/clients/filesystem-fuse/Makefile
index f4a4cef20ae..f30206155ad 100644
--- a/clients/filesystem-fuse/Makefile
+++ b/clients/filesystem-fuse/Makefile
@@ -62,6 +62,12 @@ doc-test:
 unit-test: doc-test
 	cargo test --no-fail-fast --lib --all-features --workspace
 
+test-fuse:
+	@bash ./tests/bin/s3_fileset_it.sh test
+
+test-s3:
+	@bash ./tests/bin/s3_test.sh test
+
 test: doc-test
 	cargo test --no-fail-fast --all-targets --all-features --workspace
 
diff --git a/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs b/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs
index 0860def6c5f..75bbc8329db 100644
--- a/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs
+++ b/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs
@@ -218,7 +218,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_fileset_file_system() {
+    async fn s3_ut_test_fileset_file_system() {
         test_enable_with!(RUN_TEST_WITH_S3);
 
         let config = s3_test_config();
@@ -230,7 +230,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_fileset_with_raw_file_system() {
+    async fn s3_ut_test_fileset_with_raw_file_system() {
         test_enable_with!(RUN_TEST_WITH_S3);
 
         let config = s3_test_config();
diff --git a/clients/filesystem-fuse/src/lib.rs b/clients/filesystem-fuse/src/lib.rs
index 7132f1c9471..41a9a5335d5 100644
--- a/clients/filesystem-fuse/src/lib.rs
+++ b/clients/filesystem-fuse/src/lib.rs
@@ -47,7 +47,7 @@ macro_rules! test_enable_with {
 }
 
 pub const RUN_TEST_WITH_S3: &str = "RUN_TEST_WITH_S3";
-pub const RUN_TEST_WITH_BACKEND: &str = "RUN_TEST_WITH_BACKEND";
+pub const RUN_TEST_WITH_FUSE: &str = "RUN_TEST_WITH_FUSE";
 
 pub async fn gvfs_mount(mount_to: &str, mount_from: &str, config: &AppConfig) -> GvfsResult<()> {
     gvfs_fuse::mount(mount_to, mount_from, config).await
diff --git a/clients/filesystem-fuse/src/open_dal_filesystem.rs b/clients/filesystem-fuse/src/open_dal_filesystem.rs
index b116f8fec5f..d32b014d1f0 100644
--- a/clients/filesystem-fuse/src/open_dal_filesystem.rs
+++ b/clients/filesystem-fuse/src/open_dal_filesystem.rs
@@ -279,7 +279,7 @@ mod test {
     }
 
     #[tokio::test]
-    async fn test_s3_stat() {
+    async fn s3_ut_test_s3_stat() {
         test_enable_with!(RUN_TEST_WITH_S3);
 
         let config = s3_test_config();
@@ -303,7 +303,7 @@ mod test {
     }
 
     #[tokio::test]
-    async fn test_s3_delete() {
+    async fn s3_ut_test_s3_delete() {
         test_enable_with!(RUN_TEST_WITH_S3);
         let config = s3_test_config();
 
diff --git a/clients/filesystem-fuse/src/s3_filesystem.rs b/clients/filesystem-fuse/src/s3_filesystem.rs
index 668ffa4da5d..35a091b3fe1 100644
--- a/clients/filesystem-fuse/src/s3_filesystem.rs
+++ b/clients/filesystem-fuse/src/s3_filesystem.rs
@@ -259,7 +259,7 @@ pub(crate) mod tests {
     }
 
     #[tokio::test]
-    async fn test_s3_file_system() {
+    async fn s3_ut_test_s3_file_system() {
         test_enable_with!(RUN_TEST_WITH_S3);
 
         let config = s3_test_config();
@@ -272,7 +272,7 @@ pub(crate) mod tests {
     }
 
     #[tokio::test]
-    async fn test_s3_file_system_with_raw_file_system() {
+    async fn s3_ut_test_s3_file_system_with_raw_file_system() {
         test_enable_with!(RUN_TEST_WITH_S3);
 
         let config = s3_test_config();
diff --git a/clients/filesystem-fuse/tests/bin/env.sh b/clients/filesystem-fuse/tests/bin/env.sh
new file mode 100644
index 00000000000..5fad9514c9b
--- /dev/null
+++ b/clients/filesystem-fuse/tests/bin/env.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR"
+
+S3_ACCESS_KEY_ID=${S3_ACCESS_KEY_ID:-test}
+S3_SECRET_ACCESS=${S3_SECRET_ACCESS:-test}
+S3_REGION=${S3_REGION:-ap-southeast-2}
+S3_BUCKET=${S3_BUCKET:-my-bucket}
+S3_ENDPOINT=${S3_ENDPOINT:-http://127.0.0.1:4566}
+
+# Check required environment variables
+if [[ -z "$S3_ACCESS_KEY_ID" || -z "$S3_SECRET_ACCESS" || -z "$S3_REGION" || -z "$S3_BUCKET" || -z "$S3_ENDPOINT" ]]; then
+  echo "Error: One or more required S3 environment variables are not set."
+  echo "Please set: S3_ACCESS_KEY_ID, S3_SECRET_ACCESS, S3_REGION, S3_BUCKET, S3_ENDPOINT."
+  exit 1
+fi
+
+DISABLE_LOCALSTACK=${DISABLE_LOCALSTACK:-0}
+# if S3 endpoint is an AWS endpoint, disable localstack
+if [[ "$S3_ENDPOINT" =~ amazonaws\.com ]]; then
+  echo "AWS S3 endpoint detected, disabling localstack"
+  DISABLE_LOCALSTACK=1
+fi
+
+GRAVITINO_HOME=../../../..
+GRAVITINO_HOME=$(cd $GRAVITINO_HOME && pwd)
+GRAVITINO_SERVER_DIR=$GRAVITINO_HOME/distribution/package
+CLIENT_FUSE_DIR=$GRAVITINO_HOME/clients/filesystem-fuse
\ No newline at end of file
diff --git a/clients/filesystem-fuse/tests/bin/gravitino_server.sh b/clients/filesystem-fuse/tests/bin/gravitino_server.sh
new file mode 100644
index 00000000000..026f25ac378
--- /dev/null
+++ b/clients/filesystem-fuse/tests/bin/gravitino_server.sh
@@ -0,0 +1,116 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+GRAVITINO_SERVER_URL="http://localhost:8090"
+
+check_gravitino_server_ready() {
+  local url=$1
+  local retries=10  # Number of retries
+  local wait_time=1 # Wait time between retries (seconds)
+
+  for ((i=1; i<=retries; i++)); do
+    if curl --silent --head --fail "$url" >/dev/null; then
+      echo "Gravitino server is ready."
+      return 0
+    else
+      echo "Attempt $i/$retries: Server not ready. Retrying in $wait_time seconds..."
+      sleep "$wait_time"
+    fi
+  done
+
+  echo "Error: Gravitino server did not become ready after $((retries * wait_time)) seconds."
+  exit 1
+}
+
+create_resource() {
+  local url=$1
+  local data=$2
+
+  response=$(curl -s -w "\n%{http_code}" -X POST -H "Accept: application/vnd.gravitino.v1+json" \
+    -H "Content-Type: application/json" -d "$data" "$url")
+
+  body=$(echo "$response" | head -n -1)
+  response_code=$(echo "$response" | tail -n 1)
+
+  # Check if the response code is not 2xx
+  if [[ "$response_code" -lt 200 || "$response_code" -ge 300 ]]; then
+    echo "Error: Failed to create resource. Status code: $response_code"
+    echo "Response body: $body"
+    exit 1
+  fi
+}
+
+
+
+start_gravitino_server() {
+  echo "Starting Gravitino Server"
+  # copy the aws-bundle to the server
+  if ls $GRAVITINO_SERVER_DIR/catalogs/hadoop/libs/gravitino-aws-bundle-*-incubating-SNAPSHOT.jar 1>/dev/null 2>&1; then
+     echo "File exists, skipping copy."
+  else
+    echo "Copying the aws-bundle to the server"
+    cp $GRAVITINO_HOME/bundles/aws-bundle/build/libs/gravitino-aws-bundle-*-incubating-SNAPSHOT.jar \
+      $GRAVITINO_SERVER_DIR/catalogs/hadoop/libs
+  fi
+
+  rm -rf $GRAVITINO_SERVER_DIR/data
+  $GRAVITINO_SERVER_DIR/bin/gravitino.sh restart
+
+  check_gravitino_server_ready $GRAVITINO_SERVER_URL
+
+  # Create metalake
+  create_resource "$GRAVITINO_SERVER_URL/api/metalakes" '{
+    "name":"test",
+    "comment":"comment",
+    "properties":{}
+  }'
+
+  # Create catalog
+  create_resource "$GRAVITINO_SERVER_URL/api/metalakes/test/catalogs" '{
+    "name": "c1",
+    "type": "FILESET",
+    "comment": "comment",
+    "provider": "hadoop",
+    "properties": {
+      "location": "s3a://'"$S3_BUCKET"'",
+      "s3-access-key-id": "'"$S3_ACCESS_KEY_ID"'",
+      "s3-secret-access-key": "'"$S3_SECRET_ACCESS"'",
+      "s3-endpoint": "'"$S3_ENDPOINT"'",
+      "filesystem-providers": "s3"
+    }
+  }'
+
+  # Create schema
+  create_resource "$GRAVITINO_SERVER_URL/api/metalakes/test/catalogs/c1/schemas" '{
+    "name":"s1",
+    "comment":"comment",
+    "properties":{}
+  }'
+
+  # Create FILESET
+  create_resource "$GRAVITINO_SERVER_URL/api/metalakes/test/catalogs/c1/schemas/s1/filesets" '{
+    "name":"fileset1",
+    "comment":"comment",
+    "properties":{}
+  }'
+}
+
+stop_gravitino_server() {
+  $GRAVITINO_SERVER_DIR/bin/gravitino.sh stop
+  echo "Gravitino Server stopped"
+}
\ No newline at end of file
diff --git a/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh b/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh
new file mode 100644
index 00000000000..e8c6deec271
--- /dev/null
+++ b/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+check_gvfs_fuse_ready() {
+  local retries=10
+  local wait_time=1
+
+  for ((i=1; i<=retries; i++)); do
+    # check the $MOUNT_DIR/.gvfs_meta is exist
+    if [ -f "$MOUNT_DIR/.gvfs_meta" ]; then
+      echo "Gvfs fuse is ready."
+      return 0
+    else
+      echo "Attempt $i/$retries: Gvfs fuse not ready. Retrying in $wait_time seconds..."
+      sleep "$wait_time"
+    fi
+  done
+
+  echo "Error: Gvfs fuse did not become ready after $((retries * wait_time)) seconds."
+  tail -n 100 $CLIENT_FUSE_DIR/target/debug/fuse.log
+  exit 1
+}
+
+start_gvfs_fuse() {
+  MOUNT_DIR=$CLIENT_FUSE_DIR/target/gvfs
+
+  umount $MOUNT_DIR > /dev/null 2>&1 || true
+  if [ ! -d "$MOUNT_DIR" ]; then
+    echo "Create the mount point"
+    mkdir -p $MOUNT_DIR
+  fi
+
+  MOUNT_FROM_LOCATION=gvfs://fileset/test/c1/s1/fileset1
+
+  CONF_FILE=$CLIENT_FUSE_DIR/target/debug/gvfs-fuse.toml
+
+  awk -v access_key="$S3_ACCESS_KEY_ID" \
+      -v secret_key="$S3_SECRET_ACCESS" \
+      -v region="$S3_REGION" \
+      -v bucket="$S3_BUCKET" \
+      -v endpoint="$S3_ENDPOINT" \
+      'BEGIN { in_extend_config = 0 }
+      /^\[extend_config\]/ { in_extend_config = 1 }
+      in_extend_config && /s3-access_key_id/ { $0 = "s3-access_key_id = \"" access_key "\"" }
+      in_extend_config && /s3-secret_access_key/ { $0 = "s3-secret_access_key = \"" secret_key "\"" }
+      in_extend_config && /s3-region/ { $0 = "s3-region = \"" region "\"" }
+      in_extend_config && /s3-bucket/ { $0 = "s3-bucket = \"" bucket "\"" }
+      in_extend_config && /s3-endpoint/ { $0 = "s3-endpoint = \"" endpoint "\"" }
+      { print }' $CLIENT_FUSE_DIR/tests/conf/gvfs_fuse_s3.toml > "$CONF_FILE"
+
+  # Build the gvfs-fuse
+  cd $CLIENT_FUSE_DIR
+  make build
+
+  echo "Starting gvfs-fuse-daemon"
+  $CLIENT_FUSE_DIR/target/debug/gvfs-fuse $MOUNT_DIR $MOUNT_FROM_LOCATION $CONF_FILE > $CLIENT_FUSE_DIR/target/debug/fuse.log 2>&1 &
+  check_gvfs_fuse_ready
+  cd -
+}
+
+stop_gvfs_fuse() {
+  # Stop the gvfs-fuse process if it's running
+  pkill -INT gvfs-fuse || true
+  echo "Stopping gvfs-fuse-daemon"
+}
\ No newline at end of file
diff --git a/clients/filesystem-fuse/tests/bin/localstatck.sh b/clients/filesystem-fuse/tests/bin/localstatck.sh
new file mode 100644
index 00000000000..c06c211e0a1
--- /dev/null
+++ b/clients/filesystem-fuse/tests/bin/localstatck.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+start_localstack() {
+if [ "$DISABLE_LOCALSTACK" -eq 1 ]; then
+  return
+fi
+
+  echo "Starting localstack..."
+  docker run -d -p 4566:4566 -p 4571:4571 --name localstack localstack/localstack
+  echo "Localstack started"
+
+  docker exec -it localstack sh -c "\
+    aws configure set aws_access_key_id $S3_ACCESS_KEY_ID && \
+    aws configure set aws_secret_access_key $S3_SECRET_ACCESS && \
+    aws configure set region $S3_REGION && \
+    aws configure set output json"
+
+  docker exec -it localstack awslocal s3 mb s3://$S3_BUCKET
+}
+
+stop_localstack() {
+if [ "$DISABLE_LOCALSTACK" -eq 1 ]; then
+  return
+fi
+
+  echo "Stopping localstack..."
+  docker stop localstack 2>/dev/null || true
+  docker rm localstack 2>/dev/null || true
+  echo "Localstack stopped"
+}
\ No newline at end of file
diff --git a/clients/filesystem-fuse/tests/bin/s3_fileset_it.sh b/clients/filesystem-fuse/tests/bin/s3_fileset_it.sh
index 0cef0544336..f72ca95f07d 100755
--- a/clients/filesystem-fuse/tests/bin/s3_fileset_it.sh
+++ b/clients/filesystem-fuse/tests/bin/s3_fileset_it.sh
@@ -1,5 +1,4 @@
 #!/bin/bash
-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -22,168 +21,45 @@ set -euo pipefail
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 cd "$SCRIPT_DIR"
 
-S3_ACCESS_KEY_ID=${S3_ACCESS_KEY_ID:-}
-S3_SECRET_ACCESS=${S3_SECRET_ACCESS:-}
-S3_REGION=${S3_REGION:-}
-S3_BUCKET=${S3_BUCKET:-}
-
-# Check required environment variables
-if [[ -z "$S3_ACCESS_KEY_ID" || -z "$S3_SECRET_ACCESS" || -z "$S3_REGION" || -z "$S3_BUCKET" ]]; then
-  echo "Error: One or more required S3 environment variables are not set."
-  echo "Please set: S3_ACCESS_KEY_ID, S3_SECRET_ACCESS, S3_REGION, S3_BUCKET."
-  exit 1
-fi
-
-GRAVITINO_HOME=../../../..
-GRAVITINO_HOME=$(cd $GRAVITINO_HOME && pwd)
-GRAVITINO_SERVER_DIR=$GRAVITINO_HOME/distribution/package
-CLIENT_FUSE_DIR=$GRAVITINO_HOME/clients/filesystem-fuse
-GRAVITINO_SERVER_URL=http://localhost:8090
-
-# copy the aws-bundle to the server
-if ls $GRAVITINO_SERVER_DIR/catalogs/hadoop/libs/gravitino-aws-bundle-*-incubating-SNAPSHOT.jar 1>/dev/null 2>&1; then
-  echo "File exists, skipping copy."
-else
-  cp $GRAVITINO_HOME/bundles/aws-bundle/build/libs/gravitino-aws-bundle-*-incubating-SNAPSHOT.jar \
-    $GRAVITINO_SERVER_DIR/catalogs/hadoop/libs
-fi
-
+source ./env.sh
 
-echo "Start the Gravitino server"
-rm -rf $GRAVITINO_SERVER_DIR/data
-$GRAVITINO_SERVER_DIR/bin/gravitino.sh restart
+source ./gravitino_server.sh
+source ./gvfs_fuse.sh
+source ./localstatck.sh
 
-check_gravitino_server_ready() {
-  local url=$1
-  local retries=10  # Number of retries
-  local wait_time=1 # Wait time between retries (seconds)
-
-  for ((i=1; i<=retries; i++)); do
-    if curl --silent --head --fail "$url" >/dev/null; then
-      echo "Gravitino server is ready."
-      return 0
-    else
-      echo "Attempt $i/$retries: Server not ready. Retrying in $wait_time seconds..."
-      sleep "$wait_time"
-    fi
-  done
-
-  echo "Error: Gravitino server did not become ready after $((retries * wait_time)) seconds."
-  exit 1
+start_servers() {
+  start_localstack
+  start_gravitino_server
+  start_gvfs_fuse
 }
 
-check_gravitino_server_ready "$GRAVITINO_SERVER_URL/api/metalakes"
-
-echo "Create the metalake, catalog, schema, and fileset"
-# create metalake
-curl -s -o /dev/null -X POST -H "Accept: application/vnd.gravitino.v1+json" \
--H "Content-Type: application/json" -d '{
-  "name":"test","comment":"comment","properties":{}
-}' $GRAVITINO_SERVER_URL/api/metalakes
-
-# create catalog
-curl -s -o /dev/null -X POST -H "Accept: application/vnd.gravitino.v1+json" \
--H "Content-Type: application/json" -d '{
-  "name": "c1",
-  "type": "FILESET",
-  "comment": "comment",
-  "provider": "hadoop",
-  "properties": {
-    "location": "s3a://'"$S3_BUCKET"'",
-    "s3-access-key-id": "'"$S3_ACCESS_KEY_ID"'",
-    "s3-secret-access-key": "'"$S3_SECRET_ACCESS"'",
-    "s3-endpoint": "https://s3.amazonaws.com",
-    "filesystem-providers": "s3"
-  }
-}' $GRAVITINO_SERVER_URL/api/metalakes/test/catalogs
-
-# create schema
-curl -s -o /dev/null -X POST -H "Accept: application/vnd.gravitino.v1+json" \
--H "Content-Type: application/json" -d '{
-  "name":"s1","comment":"comment","properties":{}
-}' $GRAVITINO_SERVER_URL/api/metalakes/test/catalogs/c1/schemas
-
-# create FILESET
-curl -s -o /dev/null -X POST -H "Accept: application/vnd.gravitino.v1+json" \
--H "Content-Type: application/json" -d '{
-  "name":"fileset1","comment":"comment","properties":{}
-}' $GRAVITINO_SERVER_URL/api/metalakes/test/catalogs/c1/schemas/s1/filesets
-
-echo "Start the Gvfs fuse client"
-
-MOUNT_DIR=$CLIENT_FUSE_DIR/target/gvfs
-
-umount $MOUNT_DIR > /dev/null 2>&1 || true
-if [ ! -d "$MOUNT_DIR" ]; then
-  echo "Create the mount point"
-  mkdir -p $MOUNT_DIR
-fi
-
-MOUNT_FROM_LOCATION=gvfs://fileset/test/c1/s1/fileset1
-
-CONF_FILE=$CLIENT_FUSE_DIR/target/debug/gvfs-fuse.toml
-
-awk -v access_key="$S3_ACCESS_KEY_ID" \
-    -v secret_key="$S3_SECRET_ACCESS" \
-    -v region="$S3_REGION" \
-    -v bucket="$S3_BUCKET" \
-    'BEGIN { in_extend_config = 0 }
-    /^\[extend_config\]/ { in_extend_config = 1 }
-    in_extend_config && /s3-access_key_id/ { $0 = "s3-access_key_id = \"" access_key "\"" }
-    in_extend_config && /s3-secret_access_key/ { $0 = "s3-secret_access_key = \"" secret_key "\"" }
-    in_extend_config && /s3-region/ { $0 = "s3-region = \"" region "\"" }
-    in_extend_config && /s3-bucket/ { $0 = "s3-bucket = \"" bucket "\"" }
-    { print }' $CLIENT_FUSE_DIR/tests/conf/gvfs_fuse_s3.toml > "$CONF_FILE"
-
-cleanup() {
-    # Stop the gvfs-fuse process if it's running
-    if [ -n "$FUSE_PID" ] && ps -p $FUSE_PID > /dev/null; then
-        echo "Stopping gvfs-fuse..."
-        kill -INT $FUSE_PID
-    else
-        echo "gvfs-fuse process not found or already stopped."
-    fi
-
-    # Stop the Gravitino server
-    echo "Stopping Gravitino server..."
-    $GRAVITINO_SERVER_DIR/bin/gravitino.sh stop || echo "Failed to stop Gravitino server."
+stop_servers() {
+  set +e
+  stop_gvfs_fuse
+  stop_gravitino_server
+  stop_localstack
 }
-trap cleanup EXIT
-
-cd $CLIENT_FUSE_DIR
 
-# Start the gvfs-fuse process in the background
-echo "Starting gvfs-fuse"
-# Build the gvfs-fuse
-make build
-
-$CLIENT_FUSE_DIR/target/debug/gvfs-fuse $MOUNT_DIR $MOUNT_FROM_LOCATION $CONF_FILE > $CLIENT_FUSE_DIR/target/debug/fuse.log 2>&1 &
-FUSE_PID=$!
-echo "Gvfs fuse started with PID: $FUSE_PID"
-
-#check the gvfs-fuse is ready
-check_gvfs_fuse_ready() {
-  local retries=10
-  local wait_time=1
-
-  for ((i=1; i<=retries; i++)); do
-    # check the $MOUNT_DIR/.gvfs_meta is exist
-    if [ -f "$MOUNT_DIR/.gvfs_meta" ]; then
-      echo "Gvfs fuse is ready."
-      return 0
-    else
-      echo "Attempt $i/$retries: Gvfs fuse not ready. Retrying in $wait_time seconds..."
-      sleep "$wait_time"
-    fi
-  done
-
-  echo "Error: Gvfs fuse did not become ready after $((retries * wait_time)) seconds."
+# Main logic based on parameters
+if [ "$1" == "test" ]; then
+  trap stop_servers EXIT
+  start_servers
+  # Run the integration test
+  echo "Running tests..."
+  cd $CLIENT_FUSE_DIR
+  export RUN_TEST_WITH_FUSE=1
+  cargo test --test fuse_test fuse_it_
+elif [ "$1" == "start" ]; then
+  # Start the servers
+  echo "Starting servers..."
+  start_servers
+elif [ "$1" == "stop" ]; then
+  # Stop the servers
+  echo "Stopping servers..."
+  stop_servers
+else
+  echo "Usage: $0 {test|start|stop}"
   exit 1
-}
-
-check_gvfs_fuse_ready
+fi
 
-# run the integration test
 
-export RUN_TEST_WITH_BACKEND=1
-cargo test --test fuse_test test_fuse_system_with_manual -- --exact
diff --git a/clients/filesystem-fuse/tests/bin/s3_test.sh b/clients/filesystem-fuse/tests/bin/s3_test.sh
new file mode 100644
index 00000000000..2948aa9ba64
--- /dev/null
+++ b/clients/filesystem-fuse/tests/bin/s3_test.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR"
+
+source ./env.sh
+
+source ./localstatck.sh
+
+start_servers() {
+  start_localstack
+}
+
+stop_servers() {
+  set +e
+  stop_localstack
+}
+
+# Main logic based on parameters
+if [ "$1" == "test" ]; then
+  trap stop_servers EXIT
+  start_servers
+  # Run the integration test
+  echo "Running tests..."
+  cd $CLIENT_FUSE_DIR
+  export RUN_TEST_WITH_S3=1
+  cargo test s3_ut_ --lib
+
+elif [ "$1" == "start" ]; then
+  # Start the servers
+  echo "Starting servers..."
+  start_servers
+
+elif [ "$1" == "stop" ]; then
+  # Stop the servers
+  echo "Stopping servers..."
+  stop_servers
+
+else
+  echo "Usage: $0 {test|start|stop}"
+  exit 1
+fi
+
+
diff --git a/clients/filesystem-fuse/tests/fuse_test.rs b/clients/filesystem-fuse/tests/fuse_test.rs
index de1d64d9b9a..41e385c49f1 100644
--- a/clients/filesystem-fuse/tests/fuse_test.rs
+++ b/clients/filesystem-fuse/tests/fuse_test.rs
@@ -19,7 +19,7 @@
 
 use fuse3::Errno;
 use gvfs_fuse::config::AppConfig;
-use gvfs_fuse::RUN_TEST_WITH_BACKEND;
+use gvfs_fuse::RUN_TEST_WITH_FUSE;
 use gvfs_fuse::{gvfs_mount, gvfs_unmount, test_enable_with};
 use log::{error, info};
 use std::fs::File;
@@ -86,7 +86,7 @@ impl Drop for FuseTest {
 }
 
 #[test]
-fn test_fuse_system_with_auto() {
+fn test_fuse_with_memory_fs() {
     tracing_subscriber::fmt().init();
 
     panic::set_hook(Box::new(|info| {
@@ -108,8 +108,8 @@ fn test_fuse_system_with_auto() {
 }
 
 #[test]
-fn test_fuse_system_with_manual() {
-    test_enable_with!(RUN_TEST_WITH_BACKEND);
+fn fuse_it_test_fuse() {
+    test_enable_with!(RUN_TEST_WITH_FUSE);
 
     test_fuse_filesystem("target/gvfs/gvfs_test");
 }

From 7a32f7f68c9c20d3d3e77627b8eebfb91d9f62e2 Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Thu, 9 Jan 2025 11:33:39 +0800
Subject: [PATCH 17/29] Fix

---
 clients/filesystem-fuse/tests/bin/env.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clients/filesystem-fuse/tests/bin/env.sh b/clients/filesystem-fuse/tests/bin/env.sh
index 5fad9514c9b..ad693328ba9 100644
--- a/clients/filesystem-fuse/tests/bin/env.sh
+++ b/clients/filesystem-fuse/tests/bin/env.sh
@@ -33,8 +33,8 @@ if [[ -z "$S3_ACCESS_KEY_ID" || -z "$S3_SECRET_ACCESS" || -z "$S3_REGION" || -z
 fi
 
 DISABLE_LOCALSTACK=${DISABLE_LOCALSTACK:-0}
-# if S3 endpoint is an AWS endpoint, disable localstack
-if [[ "$S3_ENDPOINT" =~ amazonaws\.com ]]; then
+# if S3 endpoint is not default value. disable localstack
+if [[ "$S3_ENDPOINT" != "http://127.0.0.1:4566" ]]; then
   echo "AWS S3 endpoint detected, disabling localstack"
   DISABLE_LOCALSTACK=1
 fi

From eb23de621868a1c6e3c121ed9f92d57e0b4190f5 Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Thu, 9 Jan 2025 15:28:11 +0800
Subject: [PATCH 18/29] Add pipline

---
 .github/workflows/gvfs-fuse-build-test.yml    |  9 +++++++-
 clients/filesystem-fuse/tests/bin/env.sh      | 22 ++++++++++++++++++-
 .../filesystem-fuse/tests/bin/gvfs_fuse.sh    | 18 +--------------
 .../tests/bin/s3_fileset_it.sh                |  7 +++++-
 clients/filesystem-fuse/tests/bin/s3_test.sh  |  4 +++-
 5 files changed, 39 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/gvfs-fuse-build-test.yml b/.github/workflows/gvfs-fuse-build-test.yml
index 4af01d82da3..0b95459237a 100644
--- a/.github/workflows/gvfs-fuse-build-test.yml
+++ b/.github/workflows/gvfs-fuse-build-test.yml
@@ -71,10 +71,17 @@ jobs:
         run: |
           dev/ci/check_commands.sh
 
-      - name: Build and test Gravitino
+      - name: Build and test Gvfs-fuse
         run: |
            ./gradlew :clients:filesystem-fuse:build -PenableFuse=true
 
+      - name: Integration test
+        run: |
+          ./gradlew bundles:aws-bundle:build -x :clients:client-python:build compileDistribution -x test -x web -PjdkVersion=${{ matrix.java-version }}
+          cd clients/filesystem-fuse
+          make test-s3
+          make test-fuse
+
       - name: Free up disk space
         run: |
           dev/ci/util_free_space.sh
diff --git a/clients/filesystem-fuse/tests/bin/env.sh b/clients/filesystem-fuse/tests/bin/env.sh
index ad693328ba9..c2e0b23be05 100644
--- a/clients/filesystem-fuse/tests/bin/env.sh
+++ b/clients/filesystem-fuse/tests/bin/env.sh
@@ -42,4 +42,24 @@ fi
 GRAVITINO_HOME=../../../..
 GRAVITINO_HOME=$(cd $GRAVITINO_HOME && pwd)
 GRAVITINO_SERVER_DIR=$GRAVITINO_HOME/distribution/package
-CLIENT_FUSE_DIR=$GRAVITINO_HOME/clients/filesystem-fuse
\ No newline at end of file
+CLIENT_FUSE_DIR=$GRAVITINO_HOME/clients/filesystem-fuse
+
+generate_test_config() {
+  local config_dir
+  config_dir=$(dirname "$TEST_CONFIG_FILE")
+  mkdir -p "$config_dir"
+
+  awk -v access_key="$S3_ACCESS_KEY_ID" \
+      -v secret_key="$S3_SECRET_ACCESS" \
+      -v region="$S3_REGION" \
+      -v bucket="$S3_BUCKET" \
+      -v endpoint="$S3_ENDPOINT" \
+      'BEGIN { in_extend_config = 0 }
+      /^\[extend_config\]/ { in_extend_config = 1 }
+      in_extend_config && /s3-access_key_id/ { $0 = "s3-access_key_id = \"" access_key "\"" }
+      in_extend_config && /s3-secret_access_key/ { $0 = "s3-secret_access_key = \"" secret_key "\"" }
+      in_extend_config && /s3-region/ { $0 = "s3-region = \"" region "\"" }
+      in_extend_config && /s3-bucket/ { $0 = "s3-bucket = \"" bucket "\"" }
+      in_extend_config && /s3-endpoint/ { $0 = "s3-endpoint = \"" endpoint "\"" }
+      { print }' $CLIENT_FUSE_DIR/tests/conf/gvfs_fuse_s3.toml > "$TEST_CONFIG_FILE"
+}
diff --git a/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh b/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh
index e8c6deec271..ac1c7fa74d4 100644
--- a/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh
+++ b/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh
@@ -47,28 +47,12 @@ start_gvfs_fuse() {
 
   MOUNT_FROM_LOCATION=gvfs://fileset/test/c1/s1/fileset1
 
-  CONF_FILE=$CLIENT_FUSE_DIR/target/debug/gvfs-fuse.toml
-
-  awk -v access_key="$S3_ACCESS_KEY_ID" \
-      -v secret_key="$S3_SECRET_ACCESS" \
-      -v region="$S3_REGION" \
-      -v bucket="$S3_BUCKET" \
-      -v endpoint="$S3_ENDPOINT" \
-      'BEGIN { in_extend_config = 0 }
-      /^\[extend_config\]/ { in_extend_config = 1 }
-      in_extend_config && /s3-access_key_id/ { $0 = "s3-access_key_id = \"" access_key "\"" }
-      in_extend_config && /s3-secret_access_key/ { $0 = "s3-secret_access_key = \"" secret_key "\"" }
-      in_extend_config && /s3-region/ { $0 = "s3-region = \"" region "\"" }
-      in_extend_config && /s3-bucket/ { $0 = "s3-bucket = \"" bucket "\"" }
-      in_extend_config && /s3-endpoint/ { $0 = "s3-endpoint = \"" endpoint "\"" }
-      { print }' $CLIENT_FUSE_DIR/tests/conf/gvfs_fuse_s3.toml > "$CONF_FILE"
-
   # Build the gvfs-fuse
   cd $CLIENT_FUSE_DIR
   make build
 
   echo "Starting gvfs-fuse-daemon"
-  $CLIENT_FUSE_DIR/target/debug/gvfs-fuse $MOUNT_DIR $MOUNT_FROM_LOCATION $CONF_FILE > $CLIENT_FUSE_DIR/target/debug/fuse.log 2>&1 &
+  $CLIENT_FUSE_DIR/target/debug/gvfs-fuse $MOUNT_DIR $MOUNT_FROM_LOCATION $TEST_CONF_FILE > $CLIENT_FUSE_DIR/target/debug/fuse.log 2>&1 &
   check_gvfs_fuse_ready
   cd -
 }
diff --git a/clients/filesystem-fuse/tests/bin/s3_fileset_it.sh b/clients/filesystem-fuse/tests/bin/s3_fileset_it.sh
index f72ca95f07d..20e10f6ef47 100755
--- a/clients/filesystem-fuse/tests/bin/s3_fileset_it.sh
+++ b/clients/filesystem-fuse/tests/bin/s3_fileset_it.sh
@@ -22,15 +22,17 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 cd "$SCRIPT_DIR"
 
 source ./env.sh
-
 source ./gravitino_server.sh
 source ./gvfs_fuse.sh
 source ./localstatck.sh
 
+TEST_CONFIG_FILE=$CLIENT_FUSE_DIR/target/debug/gvfs-fuse.toml
+
 start_servers() {
   start_localstack
   start_gravitino_server
   start_gvfs_fuse
+  generate_test_config
 }
 
 stop_servers() {
@@ -49,14 +51,17 @@ if [ "$1" == "test" ]; then
   cd $CLIENT_FUSE_DIR
   export RUN_TEST_WITH_FUSE=1
   cargo test --test fuse_test fuse_it_
+
 elif [ "$1" == "start" ]; then
   # Start the servers
   echo "Starting servers..."
   start_servers
+
 elif [ "$1" == "stop" ]; then
   # Stop the servers
   echo "Stopping servers..."
   stop_servers
+
 else
   echo "Usage: $0 {test|start|stop}"
   exit 1
diff --git a/clients/filesystem-fuse/tests/bin/s3_test.sh b/clients/filesystem-fuse/tests/bin/s3_test.sh
index 2948aa9ba64..ea232fe2fc7 100644
--- a/clients/filesystem-fuse/tests/bin/s3_test.sh
+++ b/clients/filesystem-fuse/tests/bin/s3_test.sh
@@ -21,12 +21,14 @@ set -euo pipefail
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 cd "$SCRIPT_DIR"
 
-source ./env.sh
+TEST_CONFIG_FILE=$CLIENT_FUSE_DIR/target/conf/gvfs_fuse_s3.toml
 
+source ./env.sh
 source ./localstatck.sh
 
 start_servers() {
   start_localstack
+  generate_test_config
 }
 
 stop_servers() {

From 5ed1a21553276a2bd084602c8865f1ede1131b89 Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Thu, 9 Jan 2025 15:53:26 +0800
Subject: [PATCH 19/29] Fix

---
 clients/filesystem-fuse/tests/bin/gravitino_server.sh | 2 +-
 clients/filesystem-fuse/tests/bin/gvfs_fuse.sh        | 3 ++-
 clients/filesystem-fuse/tests/bin/s3_test.sh          | 4 ++--
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/clients/filesystem-fuse/tests/bin/gravitino_server.sh b/clients/filesystem-fuse/tests/bin/gravitino_server.sh
index 026f25ac378..0f9b0fdab98 100644
--- a/clients/filesystem-fuse/tests/bin/gravitino_server.sh
+++ b/clients/filesystem-fuse/tests/bin/gravitino_server.sh
@@ -24,7 +24,7 @@ check_gravitino_server_ready() {
   local wait_time=1 # Wait time between retries (seconds)
 
   for ((i=1; i<=retries; i++)); do
-    if curl --silent --head --fail "$url" >/dev/null; then
+    if curl --silent --head --fail "$url/api/metalakes" >/dev/null; then
       echo "Gravitino server is ready."
       return 0
     else
diff --git a/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh b/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh
index ac1c7fa74d4..e706d8e2c0d 100644
--- a/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh
+++ b/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh
@@ -52,7 +52,8 @@ start_gvfs_fuse() {
   make build
 
   echo "Starting gvfs-fuse-daemon"
-  $CLIENT_FUSE_DIR/target/debug/gvfs-fuse $MOUNT_DIR $MOUNT_FROM_LOCATION $TEST_CONF_FILE > $CLIENT_FUSE_DIR/target/debug/fuse.log 2>&1 &
+  $CLIENT_FUSE_DIR/target/debug/gvfs-fuse $MOUNT_DIR $MOUNT_FROM_LOCATION $TEST_CONFIG_FILE > \
+    $CLIENT_FUSE_DIR/target/debug/fuse.log 2>&1 &
   check_gvfs_fuse_ready
   cd -
 }
diff --git a/clients/filesystem-fuse/tests/bin/s3_test.sh b/clients/filesystem-fuse/tests/bin/s3_test.sh
index ea232fe2fc7..ac5f9812c93 100644
--- a/clients/filesystem-fuse/tests/bin/s3_test.sh
+++ b/clients/filesystem-fuse/tests/bin/s3_test.sh
@@ -21,11 +21,11 @@ set -euo pipefail
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 cd "$SCRIPT_DIR"
 
-TEST_CONFIG_FILE=$CLIENT_FUSE_DIR/target/conf/gvfs_fuse_s3.toml
-
 source ./env.sh
 source ./localstatck.sh
 
+TEST_CONFIG_FILE=$CLIENT_FUSE_DIR/target/conf/gvfs_fuse_s3.toml
+
 start_servers() {
   start_localstack
   generate_test_config

From 14cfdc91d0f3b6b01387e68a5d1132439e330707 Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Thu, 9 Jan 2025 16:00:31 +0800
Subject: [PATCH 20/29] Add logs

---
 .github/workflows/gvfs-fuse-build-test.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/gvfs-fuse-build-test.yml b/.github/workflows/gvfs-fuse-build-test.yml
index 0b95459237a..3088e88fee7 100644
--- a/.github/workflows/gvfs-fuse-build-test.yml
+++ b/.github/workflows/gvfs-fuse-build-test.yml
@@ -92,5 +92,7 @@ jobs:
         with:
           name: Gvfs-fuse integrate-test-reports-${{ matrix.java-version }}
           path: |
-            clients/filesystem-fuse/build/test/log/*.log
+            clients/filesystem-fuse/target/debug/fuse.log
+            distribution/package/logs/gravitino-server.out
+            distribution/package/logs/gravitino-server.log
 

From 078a2e9c3419aff75f9755e46e4d2a2977a46ba2 Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Thu, 9 Jan 2025 16:12:43 +0800
Subject: [PATCH 21/29] Fix tty error

---
 clients/filesystem-fuse/tests/bin/localstatck.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clients/filesystem-fuse/tests/bin/localstatck.sh b/clients/filesystem-fuse/tests/bin/localstatck.sh
index c06c211e0a1..fa4552d48a3 100644
--- a/clients/filesystem-fuse/tests/bin/localstatck.sh
+++ b/clients/filesystem-fuse/tests/bin/localstatck.sh
@@ -25,13 +25,13 @@ fi
   docker run -d -p 4566:4566 -p 4571:4571 --name localstack localstack/localstack
   echo "Localstack started"
 
-  docker exec -it localstack sh -c "\
+  docker exec localstack sh -c "\
     aws configure set aws_access_key_id $S3_ACCESS_KEY_ID && \
     aws configure set aws_secret_access_key $S3_SECRET_ACCESS && \
     aws configure set region $S3_REGION && \
     aws configure set output json"
 
-  docker exec -it localstack awslocal s3 mb s3://$S3_BUCKET
+  docker exec localstack awslocal s3 mb s3://$S3_BUCKET
 }
 
 stop_localstack() {

From e278ccfd06c0a3ce28bb03d6b61839fbe0ad1dd2 Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Thu, 9 Jan 2025 16:30:49 +0800
Subject: [PATCH 22/29] Fix ci error

---
 clients/filesystem-fuse/tests/bin/s3_fileset_it.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clients/filesystem-fuse/tests/bin/s3_fileset_it.sh b/clients/filesystem-fuse/tests/bin/s3_fileset_it.sh
index 20e10f6ef47..6dc38c48f07 100755
--- a/clients/filesystem-fuse/tests/bin/s3_fileset_it.sh
+++ b/clients/filesystem-fuse/tests/bin/s3_fileset_it.sh
@@ -31,8 +31,8 @@ TEST_CONFIG_FILE=$CLIENT_FUSE_DIR/target/debug/gvfs-fuse.toml
 start_servers() {
   start_localstack
   start_gravitino_server
-  start_gvfs_fuse
   generate_test_config
+  start_gvfs_fuse
 }
 
 stop_servers() {

From 629d449471ed59e09a9ef5aee08ce7cd5f9fa8a2 Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Fri, 10 Jan 2025 10:08:33 +0800
Subject: [PATCH 23/29] Fix

---
 .../filesystem-fuse/src/gravitino_client.rs   | 46 +++++++++----------
 .../src/gravitino_fileset_filesystem.rs       |  8 ++--
 2 files changed, 26 insertions(+), 28 deletions(-)

diff --git a/clients/filesystem-fuse/src/gravitino_client.rs b/clients/filesystem-fuse/src/gravitino_client.rs
index 5a16a23358c..375d71a6ede 100644
--- a/clients/filesystem-fuse/src/gravitino_client.rs
+++ b/clients/filesystem-fuse/src/gravitino_client.rs
@@ -35,18 +35,6 @@ pub(crate) struct Fileset {
     properties: HashMap<String, String>,
 }
 
-impl Fileset {
-    pub fn new(name: &str, storage_location: &str) -> Fileset {
-        Self {
-            name: name.to_string(),
-            fileset_type: "managed".to_string(),
-            comment: "".to_string(),
-            storage_location: storage_location.to_string(),
-            properties: HashMap::default(),
-        }
-    }
-}
-
 #[derive(Debug, Deserialize)]
 struct FilesetResponse {
     code: u32,
@@ -70,18 +58,6 @@ pub(crate) struct Catalog {
     pub(crate) properties: HashMap<String, String>,
 }
 
-impl Catalog {
-    pub fn new(name: &str, properties: HashMap<String, String>) -> Catalog {
-        Self {
-            name: name.to_string(),
-            catalog_type: "fileset".to_string(),
-            provider: "s3".to_string(),
-            comment: "".to_string(),
-            properties: properties,
-        }
-    }
-}
-
 #[derive(Debug, Deserialize)]
 struct CatalogResponse {
     code: u32,
@@ -223,10 +199,30 @@ impl GravitinoClient {
 }
 
 #[cfg(test)]
-mod tests {
+pub(crate) mod tests {
     use super::*;
     use mockito::mock;
 
+    pub(crate) fn create_test_catalog(name: &str, provider: &str, properties: HashMap<String, String>)-> Catalog {
+        Catalog {
+            name: name.to_string(),
+            catalog_type: "fileset".to_string(),
+            provider: provider.to_string(),
+            comment: "".to_string(),
+            properties: properties,
+        }
+    }
+
+    pub(crate) fn create_test_fileset(name: &str, storage_location: &str) -> Fileset {
+        Fileset {
+            name: name.to_string(),
+            fileset_type: "managed".to_string(),
+            comment: "".to_string(),
+            storage_location: storage_location.to_string(),
+            properties: HashMap::default(),
+        }
+    }
+
     #[tokio::test]
     async fn test_get_fileset_success() {
         let fileset_response = r#"
diff --git a/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs b/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs
index 75bbc8329db..04236dfe841 100644
--- a/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs
+++ b/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs
@@ -144,7 +144,8 @@ mod tests {
     use crate::default_raw_filesystem::DefaultRawFileSystem;
     use crate::filesystem::tests::{TestPathFileSystem, TestRawFileSystem};
     use crate::filesystem::{FileSystemContext, PathFileSystem, RawFileSystem};
-    use crate::gravitino_client::{Catalog, Fileset, GravitinoClient};
+    use crate::gravitino_client::tests::{create_test_catalog, create_test_fileset};
+    use crate::gravitino_client::GravitinoClient;
     use crate::gravitino_fileset_filesystem::GravitinoFilesetFileSystem;
     use crate::gvfs_creator::create_fs_with_fileset;
     use crate::memory_filesystem::MemoryFileSystem;
@@ -191,8 +192,9 @@ mod tests {
         let bucket = opendal_config.get("bucket").expect("Bucket must exist");
         let endpoint = opendal_config.get("endpoint").expect("Endpoint must exist");
 
-        let catalog = Catalog::new(
+        let catalog = create_test_catalog(
             "c1",
+            "s3",
             vec![
                 ("location".to_string(), format!("s3a://{}", bucket)),
                 ("s3-endpoint".to_string(), endpoint.to_string()),
@@ -201,7 +203,7 @@ mod tests {
             .collect::<HashMap<String, String>>(),
         );
         let file_set_location = format!("s3a://{}{}", bucket, path.to_string_lossy());
-        let file_set = Fileset::new("fileset1", &file_set_location);
+        let file_set = create_test_fileset("fileset1", &file_set_location);
 
         let fs_context = FileSystemContext::default();
         let inner_fs = create_fs_with_fileset(&catalog, &file_set, config, &fs_context)

From ed3d1b4a05597a123af454bac00a30838b58c80e Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Fri, 10 Jan 2025 10:21:56 +0800
Subject: [PATCH 24/29] Fix

---
 clients/filesystem-fuse/Makefile                            | 4 ++--
 clients/filesystem-fuse/src/gravitino_client.rs             | 6 +++++-
 .../tests/bin/{s3_fileset_it.sh => run_fuse_testers.sh}     | 0
 .../tests/bin/{s3_test.sh => run_s3fs_testers.sh}           | 0
 4 files changed, 7 insertions(+), 3 deletions(-)
 rename clients/filesystem-fuse/tests/bin/{s3_fileset_it.sh => run_fuse_testers.sh} (100%)
 rename clients/filesystem-fuse/tests/bin/{s3_test.sh => run_s3fs_testers.sh} (100%)

diff --git a/clients/filesystem-fuse/Makefile b/clients/filesystem-fuse/Makefile
index f30206155ad..d5fe8eec753 100644
--- a/clients/filesystem-fuse/Makefile
+++ b/clients/filesystem-fuse/Makefile
@@ -63,10 +63,10 @@ unit-test: doc-test
 	cargo test --no-fail-fast --lib --all-features --workspace
 
 test-fuse:
-	@bash ./tests/bin/s3_fileset_it.sh test
+	@bash ./tests/bin/run_fuse_testers.sh test
 
 test-s3:
-	@bash ./tests/bin/s3_test.sh test
+	@bash ./tests/bin/run_s3fs_testers.sh test
 
 test: doc-test
 	cargo test --no-fail-fast --all-targets --all-features --workspace
diff --git a/clients/filesystem-fuse/src/gravitino_client.rs b/clients/filesystem-fuse/src/gravitino_client.rs
index 375d71a6ede..1e1cd411eac 100644
--- a/clients/filesystem-fuse/src/gravitino_client.rs
+++ b/clients/filesystem-fuse/src/gravitino_client.rs
@@ -203,7 +203,11 @@ pub(crate) mod tests {
     use super::*;
     use mockito::mock;
 
-    pub(crate) fn create_test_catalog(name: &str, provider: &str, properties: HashMap<String, String>)-> Catalog {
+    pub(crate) fn create_test_catalog(
+        name: &str,
+        provider: &str,
+        properties: HashMap<String, String>,
+    ) -> Catalog {
         Catalog {
             name: name.to_string(),
             catalog_type: "fileset".to_string(),
diff --git a/clients/filesystem-fuse/tests/bin/s3_fileset_it.sh b/clients/filesystem-fuse/tests/bin/run_fuse_testers.sh
similarity index 100%
rename from clients/filesystem-fuse/tests/bin/s3_fileset_it.sh
rename to clients/filesystem-fuse/tests/bin/run_fuse_testers.sh
diff --git a/clients/filesystem-fuse/tests/bin/s3_test.sh b/clients/filesystem-fuse/tests/bin/run_s3fs_testers.sh
similarity index 100%
rename from clients/filesystem-fuse/tests/bin/s3_test.sh
rename to clients/filesystem-fuse/tests/bin/run_s3fs_testers.sh

From df7b0dafa76941b64dcab41873caf96c92bf7e7a Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Fri, 10 Jan 2025 16:16:58 +0800
Subject: [PATCH 25/29] Fix

---
 .github/workflows/gvfs-fuse-build-test.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/gvfs-fuse-build-test.yml b/.github/workflows/gvfs-fuse-build-test.yml
index 3088e88fee7..ec65eecea60 100644
--- a/.github/workflows/gvfs-fuse-build-test.yml
+++ b/.github/workflows/gvfs-fuse-build-test.yml
@@ -77,7 +77,8 @@ jobs:
 
       - name: Integration test
         run: |
-          ./gradlew bundles:aws-bundle:build -x :clients:client-python:build compileDistribution -x test -x web -PjdkVersion=${{ matrix.java-version }}
+          ./gradle build -x :clients:client-python:build -x test -x web -PjdkVersion=${{ matrix.java-version }}
+          ./gradlew compileDistribution -x :clients:client-python:build -x test -x web -PjdkVersion=${{ matrix.java-version }}
           cd clients/filesystem-fuse
           make test-s3
           make test-fuse

From 1907c9ea0ebcd23dd70f8fcccaa5922be3de5ea6 Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Fri, 10 Jan 2025 16:35:03 +0800
Subject: [PATCH 26/29] Fix

---
 .github/workflows/gvfs-fuse-build-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/gvfs-fuse-build-test.yml b/.github/workflows/gvfs-fuse-build-test.yml
index ec65eecea60..0ee5b7c0381 100644
--- a/.github/workflows/gvfs-fuse-build-test.yml
+++ b/.github/workflows/gvfs-fuse-build-test.yml
@@ -77,7 +77,7 @@ jobs:
 
       - name: Integration test
         run: |
-          ./gradle build -x :clients:client-python:build -x test -x web -PjdkVersion=${{ matrix.java-version }}
+          ./gradlew build -x :clients:client-python:build -x test -x web -PjdkVersion=${{ matrix.java-version }}
           ./gradlew compileDistribution -x :clients:client-python:build -x test -x web -PjdkVersion=${{ matrix.java-version }}
           cd clients/filesystem-fuse
           make test-s3

From 1b8a949b41ab1cf0f3520445f7ae13aa991132ca Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Fri, 10 Jan 2025 16:41:18 +0800
Subject: [PATCH 27/29] Fix

---
 clients/filesystem-fuse/src/default_raw_filesystem.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs
index d69022c0eaa..c684e8cfb20 100644
--- a/clients/filesystem-fuse/src/default_raw_filesystem.rs
+++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs
@@ -342,7 +342,7 @@ impl<T: PathFileSystem> RawFileSystem for DefaultRawFileSystem<T> {
             .remove(fh)
             .ok_or(Errno::from(libc::EBADF))?;
 
-        // todo: need to handle racing condition
+        // todo: need to handle racing condition and corner case when the file has been deleted.
         if file_entry.is_ok() {
             let mut file = opened_file.lock().await;
             file.close().await

From 016aec224d830968a783450e794ee1ce1ac6d96b Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Mon, 13 Jan 2025 11:36:08 +0800
Subject: [PATCH 28/29] Fix

---
 .github/workflows/gvfs-fuse-build-test.yml            | 2 +-
 clients/filesystem-fuse/Makefile                      | 2 +-
 clients/filesystem-fuse/src/default_raw_filesystem.rs | 1 +
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/gvfs-fuse-build-test.yml b/.github/workflows/gvfs-fuse-build-test.yml
index 0ee5b7c0381..e0fe16fbcf4 100644
--- a/.github/workflows/gvfs-fuse-build-test.yml
+++ b/.github/workflows/gvfs-fuse-build-test.yml
@@ -81,7 +81,7 @@ jobs:
           ./gradlew compileDistribution -x :clients:client-python:build -x test -x web -PjdkVersion=${{ matrix.java-version }}
           cd clients/filesystem-fuse
           make test-s3
-          make test-fuse
+          make test-fuse-it
 
       - name: Free up disk space
         run: |
diff --git a/clients/filesystem-fuse/Makefile b/clients/filesystem-fuse/Makefile
index d5fe8eec753..86dd2f22152 100644
--- a/clients/filesystem-fuse/Makefile
+++ b/clients/filesystem-fuse/Makefile
@@ -62,7 +62,7 @@ doc-test:
 unit-test: doc-test
 	cargo test --no-fail-fast --lib --all-features --workspace
 
-test-fuse:
+test-fuse-it:
 	@bash ./tests/bin/run_fuse_testers.sh test
 
 test-s3:
diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs
index c684e8cfb20..d1d8e7605df 100644
--- a/clients/filesystem-fuse/src/default_raw_filesystem.rs
+++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs
@@ -347,6 +347,7 @@ impl<T: PathFileSystem> RawFileSystem for DefaultRawFileSystem<T> {
             let mut file = opened_file.lock().await;
             file.close().await
         } else {
+            // If the file has been deleted, it does not cause a leak even if it has not been closed.
             Ok(())
         }
     }

From df64efda45a5913114b837b4b788905e187a0286 Mon Sep 17 00:00:00 2001
From: yuhui <hui@datastrato.com>
Date: Tue, 14 Jan 2025 10:24:02 +0800
Subject: [PATCH 29/29] Update

---
 .github/workflows/gvfs-fuse-build-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/gvfs-fuse-build-test.yml b/.github/workflows/gvfs-fuse-build-test.yml
index e0fe16fbcf4..4fe7b66e09d 100644
--- a/.github/workflows/gvfs-fuse-build-test.yml
+++ b/.github/workflows/gvfs-fuse-build-test.yml
@@ -71,7 +71,7 @@ jobs:
         run: |
           dev/ci/check_commands.sh
 
-      - name: Build and test Gvfs-fuse
+      - name: Build Gvfs-fuse
         run: |
            ./gradlew :clients:filesystem-fuse:build -PenableFuse=true