test: add FileIO s3 test (#220)

* add file io s3 test

* add license

* fixed version & rm port scanner

Author: odysa

crates/iceberg/Cargo.toml

@@ -62,6 +62,7 @@ urlencoding = { workspace = true }
 uuid = { workspace = true }
 
 [dev-dependencies]
+iceberg_test_utils = { path = "../test_utils", features = ["tests"] }
 pretty_assertions = { workspace = true }
 tempfile = { workspace = true }
 tera = { workspace = true }

crates/iceberg/testdata/file_io_s3/docker-compose.yaml

@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+version: '3'
+services:
+  minio:
+    image: minio/minio:RELEASE.2024-02-26T09-33-48Z
+    ports:
+      - 9000
+      - 9001
+    environment:
+      MINIO_ROOT_USER: 'admin'
+      MINIO_ROOT_PASSWORD: 'password'
+      MINIO_ADDRESS: ':9000'
+      MINIO_CONSOLE_ADDRESS: ':9001'
+    entrypoint: sh
+    command: -c 'mkdir -p /data/bucket1 && /usr/bin/minio server /data'

crates/iceberg/tests/file_io_s3_test.rs

@@ -0,0 +1,110 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Integration tests for FileIO S3.
+
+use futures::{AsyncReadExt, AsyncWriteExt};
+use iceberg::io::{
+    FileIO, FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY,
+};
+use iceberg_test_utils::docker::DockerCompose;
+
+struct MinIOFixture {
+    _docker_compose: DockerCompose,
+    file_io: FileIO,
+}
+impl MinIOFixture {
+    async fn new(project_name: impl ToString) -> Self {
+        // Start the Docker container for the test fixture
+        let docker = DockerCompose::new(
+            project_name.to_string(),
+            format!("{}/testdata/file_io_s3", env!("CARGO_MANIFEST_DIR")),
+        );
+        docker.run();
+        let container_ip = docker.get_container_ip("minio");
+        let read_port = format!("{}:{}", container_ip, 9000);
+
+        MinIOFixture {
+            _docker_compose: docker,
+            file_io: FileIOBuilder::new("s3")
+                .with_props(vec![
+                    (S3_ENDPOINT, format!("http://{}", read_port)),
+                    (S3_ACCESS_KEY_ID, "admin".to_string()),
+                    (S3_SECRET_ACCESS_KEY, "password".to_string()),
+                    (S3_REGION, "us-east-1".to_string()),
+                ])
+                .build()
+                .unwrap(),
+        }
+    }
+}
+
+#[tokio::test]
+async fn test_file_io_s3_is_exist() {
+    let fixture = MinIOFixture::new("test_file_io_s3_is_exist").await;
+    assert!(!fixture.file_io.is_exist("s3://bucket2/any").await.unwrap());
+    assert!(fixture.file_io.is_exist("s3://bucket1/").await.unwrap());
+}
+
+#[tokio::test]
+async fn test_file_io_s3_output() {
+    // Start the Docker container for the test fixture
+    let fixture = MinIOFixture::new("test_file_io_s3_output").await;
+    assert!(!fixture
+        .file_io
+        .is_exist("s3://bucket1/test_output")
+        .await
+        .unwrap());
+    let output_file = fixture
+        .file_io
+        .new_output("s3://bucket1/test_output")
+        .unwrap();
+    {
+        let mut writer = output_file.writer().await.unwrap();
+        writer.write_all("123".as_bytes()).await.unwrap();
+        writer.close().await.unwrap();
+    }
+    assert!(fixture
+        .file_io
+        .is_exist("s3://bucket1/test_output")
+        .await
+        .unwrap());
+}
+
+#[tokio::test]
+async fn test_file_io_s3_input() {
+    let fixture = MinIOFixture::new("test_file_io_s3_input").await;
+    let output_file = fixture
+        .file_io
+        .new_output("s3://bucket1/test_input")
+        .unwrap();
+    {
+        let mut writer = output_file.writer().await.unwrap();
+        writer.write_all("test_input".as_bytes()).await.unwrap();
+        writer.close().await.unwrap();
+    }
+    let input_file = fixture
+        .file_io
+        .new_input("s3://bucket1/test_input")
+        .unwrap();
+    {
+        let mut reader = input_file.reader().await.unwrap();
+        let mut buffer = vec![];
+        reader.read_to_end(&mut buffer).await.unwrap();
+        assert_eq!(buffer, "test_input".as_bytes());
+    }
+}