duckdb · Tishj · Feb 13, 2025 · Feb 13, 2025 · Feb 13, 2025 · Feb 14, 2025
diff --git a/.github/workflows/Rest.yml b/.github/workflows/Rest.yml
@@ -50,10 +50,6 @@ jobs:
         run: |
           make release
 
-      - name: Start Rest Catalog
-        run: |
-          make start-rest-catalog
-
       - name: Generate data
         run: |
           make data

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -18,6 +18,7 @@
     src/common/utils.cpp
     src/common/schema.cpp
     src/common/iceberg.cpp
+    src/iceberg_functions/iceberg_multi_file_reader.cpp
     src/iceberg_functions/iceberg_snapshots.cpp
     src/iceberg_functions/iceberg_scan.cpp
     src/iceberg_functions/iceberg_metadata.cpp

diff --git a/duckdb b/duckdb
diff --git a/extension-ci-tools b/extension-ci-tools
diff --git a/extension_config.cmake b/extension_config.cmake
@@ -6,4 +6,4 @@ duckdb_extension_load(iceberg
     LOAD_TESTS
 )
 
-duckdb_extension_load(tpch)
+duckdb_extension_load(tpch)
diff --git a/...ts/data_generators/generate_spark_local/lineitem_001_deletes/lineitem_001_deletes/q00.sql b/...ts/data_generators/generate_spark_local/lineitem_001_deletes/lineitem_001_deletes/q00.sql
@@ -0,0 +1,6 @@
+CREATE or REPLACE TABLE iceberg_catalog.lineitem_001_deletes
+       TBLPROPERTIES (
+        'format-version'='2',
+        'write.update.mode'='merge-on-read'
+       )
+AS SELECT * FROM parquet_file_view;
diff --git a/...ts/data_generators/generate_spark_local/lineitem_001_deletes/lineitem_001_deletes/q01.sql b/...ts/data_generators/generate_spark_local/lineitem_001_deletes/lineitem_001_deletes/q01.sql
@@ -0,0 +1,11 @@
+update iceberg_catalog.lineitem_001_deletes
+set l_orderkey=NULL,
+    l_partkey=NULL,
+    l_suppkey=NULL,
+    l_linenumber=NULL,
+    l_quantity=NULL,
+    l_extendedprice=NULL,
+    l_discount=NULL,
+    l_shipdate=NULL,
+    l_comment=NULL
+where l_partkey % 2 = 0;
diff --git a/...s/data_generators/generate_spark_local/lineitem_001_deletes/lineitem_001_deletes/setup.py b/...s/data_generators/generate_spark_local/lineitem_001_deletes/lineitem_001_deletes/setup.py
@@ -0,0 +1,8 @@
+import duckdb
+import os
+
+PARQUET_SRC_FILE = os.getenv('PARQUET_SRC_FILE')
+
+duckdb_con = duckdb.connect()
+duckdb_con.execute("call dbgen(sf=0.01)")
+duckdb_con.execute(f"copy lineitem to '{PARQUET_SRC_FILE}' (FORMAT PARQUET)")
diff --git a/scripts/data_generators/generate_spark_local/lineitem_001_deletes/q00.sql b/scripts/data_generators/generate_spark_local/lineitem_001_deletes/q00.sql
@@ -0,0 +1,6 @@
+CREATE or REPLACE TABLE iceberg_catalog.lineitem_001_deletes
+       TBLPROPERTIES (
+        'format-version'='2',
+        'write.update.mode'='merge-on-read'
+       )
+AS SELECT * FROM parquet_file_view;
diff --git a/scripts/data_generators/generate_spark_local/lineitem_001_deletes/q01.sql b/scripts/data_generators/generate_spark_local/lineitem_001_deletes/q01.sql
@@ -0,0 +1,11 @@
+update iceberg_catalog.lineitem_001_deletes
+set l_orderkey=NULL,
+    l_partkey=NULL,
+    l_suppkey=NULL,
+    l_linenumber=NULL,
+    l_quantity=NULL,
+    l_extendedprice=NULL,
+    l_discount=NULL,
+    l_shipdate=NULL,
+    l_comment=NULL
+where l_partkey % 2 = 0;
diff --git a/scripts/data_generators/generate_spark_local/lineitem_001_deletes/setup.py b/scripts/data_generators/generate_spark_local/lineitem_001_deletes/setup.py
@@ -0,0 +1,8 @@
+import duckdb
+import os
+
+PARQUET_SRC_FILE = os.getenv('PARQUET_SRC_FILE')
+
+duckdb_con = duckdb.connect()
+duckdb_con.execute("call dbgen(sf=0.01)")
+duckdb_con.execute(f"copy lineitem to '{PARQUET_SRC_FILE}' (FORMAT PARQUET)")
diff --git a/scripts/data_generators/generate_spark_local/lineitem_partitioned_l_shipmode/q00.sql b/scripts/data_generators/generate_spark_local/lineitem_partitioned_l_shipmode/q00.sql
@@ -0,0 +1,8 @@
+CREATE OR REPLACE TABLE iceberg_catalog.lineitem_partitioned_l_shipmode
+USING iceberg
+PARTITIONED BY (l_shipmode)
+TBLPROPERTIES (
+    'format-version'='2',
+    'write.update.mode'='merge-on-read'
+)
+as select * from parquet_file_view;
diff --git a/scripts/data_generators/generate_spark_local/lineitem_partitioned_l_shipmode/q01.sql b/scripts/data_generators/generate_spark_local/lineitem_partitioned_l_shipmode/q01.sql
@@ -0,0 +1 @@
+delete from iceberg_catalog.lineitem_partitioned_l_shipmode where l_shipmode = 'TRUCK';
diff --git a/scripts/data_generators/generate_spark_local/lineitem_partitioned_l_shipmode/setup.py b/scripts/data_generators/generate_spark_local/lineitem_partitioned_l_shipmode/setup.py
@@ -0,0 +1,8 @@
+import duckdb
+import os
+
+PARQUET_SRC_FILE = os.getenv('PARQUET_SRC_FILE')
+
+duckdb_con = duckdb.connect()
+duckdb_con.execute("call dbgen(sf=0.01)")
+duckdb_con.execute(f"copy lineitem to '{PARQUET_SRC_FILE}' (FORMAT PARQUET)")
diff --git a/scripts/data_generators/generate_spark_local/lineitem_partitioned_l_shipmode_deletes/q00.sql b/scripts/data_generators/generate_spark_local/lineitem_partitioned_l_shipmode_deletes/q00.sql
@@ -0,0 +1,8 @@
+CREATE OR REPLACE TABLE iceberg_catalog.lineitem_partitioned_l_shipmode_deletes
+USING iceberg
+PARTITIONED BY (l_shipmode)
+TBLPROPERTIES (
+    'format-version'='2',
+    'write.update.mode'='merge-on-read'
+)
+as select * from parquet_file_view;
diff --git a/scripts/data_generators/generate_spark_local/lineitem_partitioned_l_shipmode_deletes/q01.sql b/scripts/data_generators/generate_spark_local/lineitem_partitioned_l_shipmode_deletes/q01.sql
@@ -0,0 +1,6 @@
+UPDATE iceberg_catalog.lineitem_partitioned_l_shipmode_deletes
+Set l_comment=NULL,
+    l_quantity=NULL,
+    l_discount=NULL,
+    l_linestatus=NULL
+where l_linenumber = 3 or l_linenumber = 4 or l_linenumber = 5;
diff --git a/...pts/data_generators/generate_spark_local/lineitem_partitioned_l_shipmode_deletes/setup.py b/...pts/data_generators/generate_spark_local/lineitem_partitioned_l_shipmode_deletes/setup.py
@@ -0,0 +1,8 @@
+import duckdb
+import os
+
+PARQUET_SRC_FILE = os.getenv('PARQUET_SRC_FILE')
+
+duckdb_con = duckdb.connect()
+duckdb_con.execute("call dbgen(sf=0.01)")
+duckdb_con.execute(f"copy lineitem to '{PARQUET_SRC_FILE}' (FORMAT PARQUET)")
diff --git a/scripts/data_generators/generate_spark_local/lineitem_sf1_deletes/q00.sql b/scripts/data_generators/generate_spark_local/lineitem_sf1_deletes/q00.sql
@@ -0,0 +1,6 @@
+CREATE or REPLACE TABLE iceberg_catalog.lineitem_sf1_deletes
+       TBLPROPERTIES (
+        'format-version'='2',
+        'write.update.mode'='merge-on-read'
+       )
+AS SELECT * FROM parquet_file_view;
diff --git a/scripts/data_generators/generate_spark_local/lineitem_sf1_deletes/q01.sql b/scripts/data_generators/generate_spark_local/lineitem_sf1_deletes/q01.sql
@@ -0,0 +1,11 @@
+update iceberg_catalog.lineitem_sf1_deletes
+set l_orderkey=NULL,
+    l_partkey=NULL,
+    l_suppkey=NULL,
+    l_linenumber=NULL,
+    l_quantity=NULL,
+    l_extendedprice=NULL,
+    l_discount=NULL,
+    l_shipdate=NULL,
+    l_comment=NULL
+where l_partkey % 2 = 0;
diff --git a/scripts/data_generators/generate_spark_local/lineitem_sf1_deletes/setup.py b/scripts/data_generators/generate_spark_local/lineitem_sf1_deletes/setup.py
@@ -0,0 +1,8 @@
+import duckdb
+import os
+
+PARQUET_SRC_FILE = os.getenv('PARQUET_SRC_FILE')
+
+duckdb_con = duckdb.connect()
+duckdb_con.execute("call dbgen(sf=1)")
+duckdb_con.execute(f"copy lineitem to '{PARQUET_SRC_FILE}' (FORMAT PARQUET)")
diff --git a/scripts/data_generators/generate_spark_local/lineitem_sf_01_1_delete/q00.sql b/scripts/data_generators/generate_spark_local/lineitem_sf_01_1_delete/q00.sql
@@ -0,0 +1,6 @@
+CREATE or REPLACE TABLE iceberg_catalog.lineitem_sf_01_1_delete
+       TBLPROPERTIES (
+        'format-version'='2',
+        'write.update.mode'='merge-on-read'
+       )
+AS SELECT * FROM parquet_file_view;
diff --git a/scripts/data_generators/generate_spark_local/lineitem_sf_01_1_delete/q01.sql b/scripts/data_generators/generate_spark_local/lineitem_sf_01_1_delete/q01.sql
@@ -0,0 +1 @@
+delete from iceberg_catalog.lineitem_sf_01_1_delete where l_orderkey=10053 and l_partkey = 77;
diff --git a/scripts/data_generators/generate_spark_local/lineitem_sf_01_1_delete/setup.py b/scripts/data_generators/generate_spark_local/lineitem_sf_01_1_delete/setup.py
@@ -0,0 +1,8 @@
+import duckdb
+import os
+
+PARQUET_SRC_FILE = os.getenv('PARQUET_SRC_FILE')
+
+duckdb_con = duckdb.connect()
+duckdb_con.execute("call dbgen(sf=0.01)")
+duckdb_con.execute(f"copy lineitem to '{PARQUET_SRC_FILE}' (FORMAT PARQUET)")
diff --git a/scripts/data_generators/generate_spark_local/lineitem_sf_01_no_deletes/q00.sql b/scripts/data_generators/generate_spark_local/lineitem_sf_01_no_deletes/q00.sql
@@ -0,0 +1,6 @@
+CREATE or REPLACE TABLE iceberg_catalog.lineitem_sf_01_no_deletes
+       TBLPROPERTIES (
+        'format-version'='2',
+        'write.update.mode'='merge-on-read'
+       )
+AS SELECT * FROM parquet_file_view;
diff --git a/scripts/data_generators/generate_spark_local/lineitem_sf_01_no_deletes/setup.py b/scripts/data_generators/generate_spark_local/lineitem_sf_01_no_deletes/setup.py
@@ -0,0 +1,8 @@
+import duckdb
+import os
+
+PARQUET_SRC_FILE = os.getenv('PARQUET_SRC_FILE')
+
+duckdb_con = duckdb.connect()
+duckdb_con.execute("call dbgen(sf=0.01)")
+duckdb_con.execute(f"copy lineitem to '{PARQUET_SRC_FILE}' (FORMAT PARQUET)")
diff --git a/scripts/data_generators/generate_spark_local/table_with_deletes/q00.sql b/scripts/data_generators/generate_spark_local/table_with_deletes/q00.sql
@@ -0,0 +1,6 @@
+CREATE or REPLACE TABLE iceberg_catalog.table_with_deletes
+       TBLPROPERTIES (
+        'format-version'='2',
+        'write.update.mode'='merge-on-read'
+       )
+AS SELECT * FROM parquet_file_view;
diff --git a/scripts/data_generators/generate_spark_local/table_with_deletes/q01.sql b/scripts/data_generators/generate_spark_local/table_with_deletes/q01.sql
@@ -0,0 +1,11 @@
+update iceberg_catalog.table_with_deletes
+set l_orderkey=NULL,
+    l_partkey=NULL,
+    l_suppkey=NULL,
+    l_linenumber=NULL,
+    l_quantity=NULL,
+    l_extendedprice=NULL,
+    l_discount=NULL,
+    l_shipdate=NULL,
+    l_comment=NULL
+where l_partkey % 2 = 0;
diff --git a/scripts/data_generators/generate_spark_local/table_with_deletes/setup.py b/scripts/data_generators/generate_spark_local/table_with_deletes/setup.py
@@ -0,0 +1,8 @@
+import duckdb
+import os
+
+PARQUET_SRC_FILE = os.getenv('PARQUET_SRC_FILE')
+
+duckdb_con = duckdb.connect()
+duckdb_con.execute("call dbgen(sf=0.01)")
+duckdb_con.execute(f"copy lineitem to '{PARQUET_SRC_FILE}' (FORMAT PARQUET)")
diff --git a/scripts/data_generators/generate_spark_rest/lineitem_001_deletes/q00.sql b/scripts/data_generators/generate_spark_rest/lineitem_001_deletes/q00.sql
@@ -0,0 +1,6 @@
+CREATE or REPLACE TABLE default.lineitem_001_deletes
+       TBLPROPERTIES (
+        'format-version'='2',
+        'write.update.mode'='merge-on-read'
+       )
+AS SELECT * FROM parquet_file_view;
diff --git a/scripts/data_generators/generate_spark_rest/lineitem_001_deletes/q01.sql b/scripts/data_generators/generate_spark_rest/lineitem_001_deletes/q01.sql
@@ -0,0 +1,11 @@
+update default.lineitem_001_deletes
+set l_orderkey=NULL,
+    l_partkey=NULL,
+    l_suppkey=NULL,
+    l_linenumber=NULL,
+    l_quantity=NULL,
+    l_extendedprice=NULL,
+    l_discount=NULL,
+    l_shipdate=NULL,
+    l_comment=NULL
+where l_partkey % 2 = 0;
diff --git a/scripts/data_generators/generate_spark_rest/lineitem_001_deletes/setup.py b/scripts/data_generators/generate_spark_rest/lineitem_001_deletes/setup.py
@@ -0,0 +1,8 @@
+import duckdb
+import os
+
+PARQUET_SRC_FILE = os.getenv('PARQUET_SRC_FILE')
+
+duckdb_con = duckdb.connect()
+duckdb_con.execute("call dbgen(sf=0.01)")
+duckdb_con.execute(f"copy lineitem to '{PARQUET_SRC_FILE}' (FORMAT PARQUET)")
diff --git a/scripts/data_generators/generate_spark_rest/lineitem_partitioned_l_shipmode/q00.sql b/scripts/data_generators/generate_spark_rest/lineitem_partitioned_l_shipmode/q00.sql
@@ -0,0 +1,8 @@
+CREATE OR REPLACE TABLE default.lineitem_partitioned_l_shipmode
+USING iceberg
+PARTITIONED BY (l_shipmode)
+TBLPROPERTIES (
+    'format-version'='2',
+    'write.update.mode'='merge-on-read'
+)
+as select * from parquet_file_view;
diff --git a/scripts/data_generators/generate_spark_rest/lineitem_partitioned_l_shipmode/q01.sql b/scripts/data_generators/generate_spark_rest/lineitem_partitioned_l_shipmode/q01.sql
@@ -0,0 +1 @@
+delete from default.lineitem_partitioned_l_shipmode where l_shipmode = 'TRUCK';
diff --git a/scripts/data_generators/generate_spark_rest/lineitem_partitioned_l_shipmode/setup.py b/scripts/data_generators/generate_spark_rest/lineitem_partitioned_l_shipmode/setup.py
@@ -0,0 +1,8 @@
+import duckdb
+import os
+
+PARQUET_SRC_FILE = os.getenv('PARQUET_SRC_FILE')
+
+duckdb_con = duckdb.connect()
+duckdb_con.execute("call dbgen(sf=0.01)")
+duckdb_con.execute(f"copy lineitem to '{PARQUET_SRC_FILE}' (FORMAT PARQUET)")
diff --git a/scripts/data_generators/generate_spark_rest/lineitem_partitioned_l_shipmode_deletes/q00.sql b/scripts/data_generators/generate_spark_rest/lineitem_partitioned_l_shipmode_deletes/q00.sql
@@ -0,0 +1,8 @@
+CREATE OR REPLACE TABLE default.lineitem_partitioned_l_shipmode_deletes
+USING iceberg
+PARTITIONED BY (l_shipmode)
+TBLPROPERTIES (
+    'format-version'='2',
+    'write.update.mode'='merge-on-read'
+)
+as select * from parquet_file_view;
diff --git a/scripts/data_generators/generate_spark_rest/lineitem_partitioned_l_shipmode_deletes/q01.sql b/scripts/data_generators/generate_spark_rest/lineitem_partitioned_l_shipmode_deletes/q01.sql
@@ -0,0 +1,6 @@
+UPDATE default.lineitem_partitioned_l_shipmode_deletes
+Set l_comment=NULL,
+    l_quantity=NULL,
+    l_discount=NULL,
+    l_linestatus=NULL
+where l_linenumber = 3 or l_linenumber = 4 or l_linenumber = 5;
diff --git a/scripts/data_generators/generate_spark_rest/lineitem_partitioned_l_shipmode_deletes/setup.py b/scripts/data_generators/generate_spark_rest/lineitem_partitioned_l_shipmode_deletes/setup.py
@@ -0,0 +1,8 @@
+import duckdb
+import os
+
+PARQUET_SRC_FILE = os.getenv('PARQUET_SRC_FILE')
+
+duckdb_con = duckdb.connect()
+duckdb_con.execute("call dbgen(sf=0.01)")
+duckdb_con.execute(f"copy lineitem to '{PARQUET_SRC_FILE}' (FORMAT PARQUET)")
diff --git a/scripts/data_generators/generate_spark_rest/lineitem_sf1_deletes/q00.sql b/scripts/data_generators/generate_spark_rest/lineitem_sf1_deletes/q00.sql
@@ -0,0 +1,6 @@
+CREATE or REPLACE TABLE default.lineitem_sf1_deletes
+       TBLPROPERTIES (
+        'format-version'='2',
+        'write.update.mode'='merge-on-read'
+       )
+AS SELECT * FROM parquet_file_view;
diff --git a/scripts/data_generators/generate_spark_rest/lineitem_sf1_deletes/q01.sql b/scripts/data_generators/generate_spark_rest/lineitem_sf1_deletes/q01.sql
@@ -0,0 +1,11 @@
+update default.lineitem_sf1_deletes
+set l_orderkey=NULL,
+    l_partkey=NULL,
+    l_suppkey=NULL,
+    l_linenumber=NULL,
+    l_quantity=NULL,
+    l_extendedprice=NULL,
+    l_discount=NULL,
+    l_shipdate=NULL,
+    l_comment=NULL
+where l_partkey % 2 = 0;
diff --git a/scripts/data_generators/generate_spark_rest/lineitem_sf1_deletes/setup.py b/scripts/data_generators/generate_spark_rest/lineitem_sf1_deletes/setup.py
@@ -0,0 +1,8 @@
+import duckdb
+import os
+
+PARQUET_SRC_FILE = os.getenv('PARQUET_SRC_FILE')
+
+duckdb_con = duckdb.connect()
+duckdb_con.execute("call dbgen(sf=1)")
+duckdb_con.execute(f"copy lineitem to '{PARQUET_SRC_FILE}' (FORMAT PARQUET)")
diff --git a/scripts/data_generators/generate_spark_rest/lineitem_sf_01_1_delete/q00.sql b/scripts/data_generators/generate_spark_rest/lineitem_sf_01_1_delete/q00.sql
@@ -0,0 +1,6 @@
+CREATE or REPLACE TABLE default.lineitem_sf_01_1_delete
+       TBLPROPERTIES (
+        'format-version'='2',
+        'write.update.mode'='merge-on-read'
+       )
+AS SELECT * FROM parquet_file_view;
diff --git a/scripts/data_generators/generate_spark_rest/lineitem_sf_01_1_delete/q01.sql b/scripts/data_generators/generate_spark_rest/lineitem_sf_01_1_delete/q01.sql
@@ -0,0 +1 @@
+delete from default.lineitem_sf_01_1_delete where l_orderkey=10053 and l_partkey = 77;
diff --git a/scripts/data_generators/generate_spark_rest/lineitem_sf_01_1_delete/setup.py b/scripts/data_generators/generate_spark_rest/lineitem_sf_01_1_delete/setup.py
@@ -0,0 +1,8 @@
+import duckdb
+import os
+
+PARQUET_SRC_FILE = os.getenv('PARQUET_SRC_FILE')
+
+duckdb_con = duckdb.connect()
+duckdb_con.execute("call dbgen(sf=0.01)")
+duckdb_con.execute(f"copy lineitem to '{PARQUET_SRC_FILE}' (FORMAT PARQUET)")
diff --git a/scripts/data_generators/generate_spark_rest/lineitem_sf_01_no_deletes/q00.sql b/scripts/data_generators/generate_spark_rest/lineitem_sf_01_no_deletes/q00.sql
@@ -0,0 +1,6 @@
+CREATE or REPLACE TABLE default.lineitem_sf_01_no_deletes
+       TBLPROPERTIES (
+        'format-version'='2',
+        'write.update.mode'='merge-on-read'
+       )
+AS SELECT * FROM parquet_file_view;
diff --git a/scripts/data_generators/generate_spark_rest/lineitem_sf_01_no_deletes/setup.py b/scripts/data_generators/generate_spark_rest/lineitem_sf_01_no_deletes/setup.py
@@ -0,0 +1,8 @@
+import duckdb
+import os
+
+PARQUET_SRC_FILE = os.getenv('PARQUET_SRC_FILE')
+
+duckdb_con = duckdb.connect()
+duckdb_con.execute("call dbgen(sf=0.01)")
+duckdb_con.execute(f"copy lineitem to '{PARQUET_SRC_FILE}' (FORMAT PARQUET)")
diff --git a/scripts/data_generators/tmp_data/tmp.parquet b/scripts/data_generators/tmp_data/tmp.parquet
+4 −29		.github/workflows/TestCITools.yml
+10 −17		.github/workflows/_extension_deploy.yml
+71 −55		.github/workflows/_extension_distribution.yml
+2 −3		README.md
+0 −5		config/distribution_matrix.json
+9 −9		docker/linux_amd64/Dockerfile
+0 −84		docker/linux_amd64_musl/Dockerfile
+9 −9		docker/linux_arm64/Dockerfile
+0 −259		makefiles/c_api_extensions/base.Makefile
+0 −129		makefiles/c_api_extensions/c_cpp.Makefile
+0 −47		makefiles/c_api_extensions/rust.Makefile
+7 −31		makefiles/duckdb_extension.Makefile
+6 −44		scripts/append_extension_metadata.py
+0 −45		scripts/configure_helper.py
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		delete from iceberg_catalog.lineitem_partitioned_l_shipmode where l_shipmode = 'TRUCK';
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		delete from iceberg_catalog.lineitem_sf_01_1_delete where l_orderkey=10053 and l_partkey = 77;
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		delete from default.lineitem_partitioned_l_shipmode where l_shipmode = 'TRUCK';
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		delete from default.lineitem_sf_01_1_delete where l_orderkey=10053 and l_partkey = 77;