From 567446628c38e1bd7e7478c1a150382d3d0a928e Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 25 Apr 2025 13:27:42 +0200 Subject: [PATCH 1/3] Test Apache Iceberg 1.9.0 RC2 --- spark/Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/spark/Dockerfile b/spark/Dockerfile index 0a430a6..15ac835 100644 --- a/spark/Dockerfile +++ b/spark/Dockerfile @@ -44,7 +44,7 @@ WORKDIR ${SPARK_HOME} ENV SPARK_VERSION=3.5.5 ENV SPARK_MAJOR_VERSION=3.5 -ENV ICEBERG_VERSION=1.8.1 +ENV ICEBERG_VERSION=1.9.0 # Download spark RUN mkdir -p ${SPARK_HOME} \ @@ -53,16 +53,16 @@ RUN mkdir -p ${SPARK_HOME} \ && rm -rf spark-${SPARK_VERSION}-bin-hadoop3.tgz # Download iceberg spark runtime -RUN curl https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${SPARK_MAJOR_VERSION}_2.12/${ICEBERG_VERSION}/iceberg-spark-runtime-${SPARK_MAJOR_VERSION}_2.12-${ICEBERG_VERSION}.jar -Lo /opt/spark/jars/iceberg-spark-runtime-${SPARK_MAJOR_VERSION}_2.12-${ICEBERG_VERSION}.jar +RUN curl https://repository.apache.org/content/repositories/orgapacheiceberg-1190/org/apache/iceberg/iceberg-spark-runtime-3.5_2.12/1.9.0/iceberg-spark-runtime-3.5_2.12-1.9.0.jar -Lo /opt/spark/jars/iceberg-spark-runtime-${SPARK_MAJOR_VERSION}_2.12-${ICEBERG_VERSION}.jar # Download AWS bundle -RUN curl -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar +RUN curl -s https://repository.apache.org/content/repositories/orgapacheiceberg-1190/org/apache/iceberg/iceberg-aws-bundle/1.9.0/iceberg-aws-bundle-1.9.0.jar -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar # Download GCP bundle -RUN curl -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-gcp-bundle/${ICEBERG_VERSION}/iceberg-gcp-bundle-${ICEBERG_VERSION}.jar -Lo /opt/spark/jars/iceberg-gcp-bundle-${ICEBERG_VERSION}.jar +RUN curl -s https://repository.apache.org/content/repositories/orgapacheiceberg-1190/org/apache/iceberg/iceberg-gcp-bundle/1.9.0/iceberg-gcp-bundle-1.9.0.jar -Lo /opt/spark/jars/iceberg-gcp-bundle-${ICEBERG_VERSION}.jar # Download Azure bundle -RUN curl -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-azure-bundle/${ICEBERG_VERSION}/iceberg-azure-bundle-${ICEBERG_VERSION}.jar -Lo /opt/spark/jars/iceberg-azure-bundle-${ICEBERG_VERSION}.jar +RUN curl -s https://repository.apache.org/content/repositories/orgapacheiceberg-1190/org/apache/iceberg/iceberg-azure-bundle/1.9.0/iceberg-azure-bundle-1.9.0.jar -Lo /opt/spark/jars/iceberg-azure-bundle-${ICEBERG_VERSION}.jar # Install AWS CLI RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \ From 13214b8c79359bd350a54ed8a27eaf2aede4ae68 Mon Sep 17 00:00:00 2001 From: Fokko Date: Fri, 25 Apr 2025 15:10:50 +0200 Subject: [PATCH 2/3] Syntax --- ... - An Introduction to the Iceberg Java API using Scala.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/notebooks/Iceberg - An Introduction to the Iceberg Java API using Scala.ipynb b/spark/notebooks/Iceberg - An Introduction to the Iceberg Java API using Scala.ipynb index 6db9d18..1132194 100644 --- a/spark/notebooks/Iceberg - An Introduction to the Iceberg Java API using Scala.ipynb +++ b/spark/notebooks/Iceberg - An Introduction to the Iceberg Java API using Scala.ipynb @@ -417,7 +417,7 @@ "val dataWriter: DataWriter[GenericRecord] =\n", " Parquet.writeData(file)\n", " .schema(schema)\n", - " .createWriterFunc(GenericParquetWriter.buildWriter)\n", + " .createWriterFunc(GenericParquetWriter.buildWriter(_))\n", " .overwrite()\n", " .withSpec(PartitionSpec.unpartitioned())\n", " .build()\n", From 26b45c600058b7caef27a063f119529213bc170c Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 28 Apr 2025 09:50:17 +0200 Subject: [PATCH 3/3] Bump to release --- spark/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spark/Dockerfile b/spark/Dockerfile index 15ac835..215bc40 100644 --- a/spark/Dockerfile +++ b/spark/Dockerfile @@ -53,16 +53,16 @@ RUN mkdir -p ${SPARK_HOME} \ && rm -rf spark-${SPARK_VERSION}-bin-hadoop3.tgz # Download iceberg spark runtime -RUN curl https://repository.apache.org/content/repositories/orgapacheiceberg-1190/org/apache/iceberg/iceberg-spark-runtime-3.5_2.12/1.9.0/iceberg-spark-runtime-3.5_2.12-1.9.0.jar -Lo /opt/spark/jars/iceberg-spark-runtime-${SPARK_MAJOR_VERSION}_2.12-${ICEBERG_VERSION}.jar +RUN curl https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${SPARK_MAJOR_VERSION}_2.12/${ICEBERG_VERSION}/iceberg-spark-runtime-${SPARK_MAJOR_VERSION}_2.12-${ICEBERG_VERSION}.jar -Lo /opt/spark/jars/iceberg-spark-runtime-${SPARK_MAJOR_VERSION}_2.12-${ICEBERG_VERSION}.jar # Download AWS bundle -RUN curl -s https://repository.apache.org/content/repositories/orgapacheiceberg-1190/org/apache/iceberg/iceberg-aws-bundle/1.9.0/iceberg-aws-bundle-1.9.0.jar -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar +RUN curl -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar # Download GCP bundle -RUN curl -s https://repository.apache.org/content/repositories/orgapacheiceberg-1190/org/apache/iceberg/iceberg-gcp-bundle/1.9.0/iceberg-gcp-bundle-1.9.0.jar -Lo /opt/spark/jars/iceberg-gcp-bundle-${ICEBERG_VERSION}.jar +RUN curl -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-gcp-bundle/${ICEBERG_VERSION}/iceberg-gcp-bundle-${ICEBERG_VERSION}.jar -Lo /opt/spark/jars/iceberg-gcp-bundle-${ICEBERG_VERSION}.jar # Download Azure bundle -RUN curl -s https://repository.apache.org/content/repositories/orgapacheiceberg-1190/org/apache/iceberg/iceberg-azure-bundle/1.9.0/iceberg-azure-bundle-1.9.0.jar -Lo /opt/spark/jars/iceberg-azure-bundle-${ICEBERG_VERSION}.jar +RUN curl -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-azure-bundle/${ICEBERG_VERSION}/iceberg-azure-bundle-${ICEBERG_VERSION}.jar -Lo /opt/spark/jars/iceberg-azure-bundle-${ICEBERG_VERSION}.jar # Install AWS CLI RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \