duckdb · samansmink · Feb 20, 2025 · Feb 11, 2025 · Feb 11, 2025 · Feb 11, 2025
diff --git a/.github/workflows/Rest.yml b/.github/workflows/Rest.yml
@@ -39,6 +39,10 @@ jobs:
         with:
           vcpkgGitCommitId: 5e5d0e1cd7785623065e77eff011afdeec1a3574
 
+      - name: Setup Ccache
+        uses: hendrikmuhs/ccache-action@main
+        continue-on-error: true
+
       - name: Build extension
         env:
           GEN: ninja
@@ -47,12 +51,16 @@ jobs:
           make release
 
       - name: Start Rest Catalog
-        working-directory: scripts/
         run: |
-          ./start-rest-catalog.sh
+          make start-rest-catalog
+
+      - name: Generate data
+        run: |
+          make data
 
-      - name: Test With rest catalog
+      - name: Test with rest catalog
         env:
           ICEBERG_SERVER_AVAILABLE: 1
+          DUCKDB_ICEBERG_HAVE_GENERATED_DATA: 1
         run: |
-          make test_release
+          make test_release
diff --git a/.gitignore b/.gitignore
@@ -12,3 +12,7 @@ data/iceberg/generated_*
 scripts/metastore_db/
 scripts/derby.log
 scripts/test-script-with-path.sql
+scripts/data_generators/__pycache__/
+scripts/data_generators/*/__pycache__/
+scripts/data_generators/*/*/*.parquet
+data/generated/*
diff --git a/Makefile b/Makefile
@@ -5,18 +5,23 @@ EXT_NAME=iceberg
 EXT_CONFIG=${PROJ_DIR}extension_config.cmake
 
 # We need this for testing
-CORE_EXTENSIONS='httpfs'
+CORE_EXTENSIONS='parquet;httpfs'
 
 # Include the Makefile from extension-ci-tools
 include extension-ci-tools/makefiles/duckdb_extension.Makefile
 
+start-rest-catalog: install_requirements
+	./scripts/start-rest-catalog.sh
+
+install_requirements:
+	python3 -m pip install -r scripts/requirements.txt
+
 # Custom makefile targets
-data: data_clean
-	python3 scripts/test_data_generator/generate_iceberg.py 0.001 data/iceberg/generated_spec1_0_001 1
-	python3 scripts/test_data_generator/generate_iceberg.py 0.001 data/iceberg/generated_spec2_0_001 2
+data: data_clean start-rest-catalog
+	python3 scripts/data_generators/generate_data.py
 
 data_large: data data_clean
-	python3 scripts/test_data_generator/generate_iceberg.py 1 data/iceberg/generated_spec2_1 2
+	python3 scripts/data_generators/generate_data.py
 
 data_clean:
-	rm -rf data/iceberg/generated_*
+	rm -rf data/generated
diff --git a/data/iceberg/generated_spec1_0_001/expected_results/last/count.csv b/data/iceberg/generated_spec1_0_001/expected_results/last/count.csv
diff --git a/data/iceberg/generated_spec1_0_001/expected_results/last/data/._SUCCESS.crc b/data/iceberg/generated_spec1_0_001/expected_results/last/data/._SUCCESS.crc
diff --git a/...esults/last/data/.part-00000-90484277-8f62-41f7-b073-a4c8999e393e-c000.snappy.parquet.crc b/...esults/last/data/.part-00000-90484277-8f62-41f7-b073-a4c8999e393e-c000.snappy.parquet.crc
diff --git a/...ted_results/last/data/part-00000-90484277-8f62-41f7-b073-a4c8999e393e-c000.snappy.parquet b/...ted_results/last/data/part-00000-90484277-8f62-41f7-b073-a4c8999e393e-c000.snappy.parquet
diff --git a/data/iceberg/generated_spec1_0_001/expected_results/last/query.sql b/data/iceberg/generated_spec1_0_001/expected_results/last/query.sql
diff --git a/...001/pyspark_iceberg_table/data/00000-1-bcb5bbb9-a993-41f7-95e1-09e0c2475f4a-00001.parquet b/...001/pyspark_iceberg_table/data/00000-1-bcb5bbb9-a993-41f7-95e1-09e0c2475f4a-00001.parquet
diff --git a/...01/pyspark_iceberg_table/data/00000-14-db28d572-b7de-4568-9b00-8c55c69cb179-00001.parquet b/...01/pyspark_iceberg_table/data/00000-14-db28d572-b7de-4568-9b00-8c55c69cb179-00001.parquet
diff --git a/...01/pyspark_iceberg_table/data/00000-26-f6481588-9ba4-4a7b-b3dd-f188d41fa5b8-00001.parquet b/...01/pyspark_iceberg_table/data/00000-26-f6481588-9ba4-4a7b-b3dd-f188d41fa5b8-00001.parquet
diff --git a/...01/pyspark_iceberg_table/data/00000-36-cf35a788-d8c2-4ded-a9f7-5239797e80b8-00001.parquet b/...01/pyspark_iceberg_table/data/00000-36-cf35a788-d8c2-4ded-a9f7-5239797e80b8-00001.parquet
diff --git a/...001/pyspark_iceberg_table/data/00000-5-bd5417f5-f28c-46b1-b1ab-39ee9c191368-00001.parquet b/...001/pyspark_iceberg_table/data/00000-5-bd5417f5-f28c-46b1-b1ab-39ee9c191368-00001.parquet
diff --git a/...001/pyspark_iceberg_table/data/00000-9-6cdc0135-4256-4772-8c3e-3f4803ded842-00001.parquet b/...001/pyspark_iceberg_table/data/00000-9-6cdc0135-4256-4772-8c3e-3f4803ded842-00001.parquet
diff --git a/...d_spec1_0_001/pyspark_iceberg_table/metadata/0f120788-1926-4605-a1ab-450f4cf3ccee-m0.avro b/...d_spec1_0_001/pyspark_iceberg_table/metadata/0f120788-1926-4605-a1ab-450f4cf3ccee-m0.avro
diff --git a/...d_spec1_0_001/pyspark_iceberg_table/metadata/0f120788-1926-4605-a1ab-450f4cf3ccee-m1.avro b/...d_spec1_0_001/pyspark_iceberg_table/metadata/0f120788-1926-4605-a1ab-450f4cf3ccee-m1.avro
diff --git a/...d_spec1_0_001/pyspark_iceberg_table/metadata/4aa40041-ccc6-4e64-a9ab-366875aafd63-m0.avro b/...d_spec1_0_001/pyspark_iceberg_table/metadata/4aa40041-ccc6-4e64-a9ab-366875aafd63-m0.avro
diff --git a/...d_spec1_0_001/pyspark_iceberg_table/metadata/4aa40041-ccc6-4e64-a9ab-366875aafd63-m1.avro b/...d_spec1_0_001/pyspark_iceberg_table/metadata/4aa40041-ccc6-4e64-a9ab-366875aafd63-m1.avro
diff --git a/...d_spec1_0_001/pyspark_iceberg_table/metadata/7723fb1b-ae48-49de-9e77-cd7945667cb9-m0.avro b/...d_spec1_0_001/pyspark_iceberg_table/metadata/7723fb1b-ae48-49de-9e77-cd7945667cb9-m0.avro
diff --git a/...d_spec1_0_001/pyspark_iceberg_table/metadata/c091e891-ac3a-4429-be9a-e63f1ed63b99-m0.avro b/...d_spec1_0_001/pyspark_iceberg_table/metadata/c091e891-ac3a-4429-be9a-e63f1ed63b99-m0.avro
diff --git a/...d_spec1_0_001/pyspark_iceberg_table/metadata/c091e891-ac3a-4429-be9a-e63f1ed63b99-m1.avro b/...d_spec1_0_001/pyspark_iceberg_table/metadata/c091e891-ac3a-4429-be9a-e63f1ed63b99-m1.avro
diff --git a/...d_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m0.avro b/...d_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m0.avro
diff --git a/...d_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m1.avro b/...d_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m1.avro
diff --git a/...d_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m2.avro b/...d_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m2.avro
diff --git a/...d_spec1_0_001/pyspark_iceberg_table/metadata/e3febcc2-7f11-44b9-80af-571fb1c0463a-m0.avro b/...d_spec1_0_001/pyspark_iceberg_table/metadata/e3febcc2-7f11-44b9-80af-571fb1c0463a-m0.avro
diff --git a/...eberg_table/metadata/snap-1692767036460164714-1-0f120788-1926-4605-a1ab-450f4cf3ccee.avro b/...eberg_table/metadata/snap-1692767036460164714-1-0f120788-1926-4605-a1ab-450f4cf3ccee.avro
diff --git a/...eberg_table/metadata/snap-2276968461870063565-1-5ee46b42-10e4-401d-8f61-2bd3b5ebb548.avro b/...eberg_table/metadata/snap-2276968461870063565-1-5ee46b42-10e4-401d-8f61-2bd3b5ebb548.avro
diff --git a/...eberg_table/metadata/snap-4407328776463037310-1-c091e891-ac3a-4429-be9a-e63f1ed63b99.avro b/...eberg_table/metadata/snap-4407328776463037310-1-c091e891-ac3a-4429-be9a-e63f1ed63b99.avro
diff --git a/...eberg_table/metadata/snap-4543110679664799316-1-e3febcc2-7f11-44b9-80af-571fb1c0463a.avro b/...eberg_table/metadata/snap-4543110679664799316-1-e3febcc2-7f11-44b9-80af-571fb1c0463a.avro
diff --git a/...eberg_table/metadata/snap-6238750566879819059-1-d76c5203-4f0a-46ef-a293-268e0afec64b.avro b/...eberg_table/metadata/snap-6238750566879819059-1-d76c5203-4f0a-46ef-a293-268e0afec64b.avro
diff --git a/...eberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro b/...eberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro
diff --git a/...eberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro b/...eberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json