Merge pull request #106 from Tmonster/robust-data-generation

Cleaner Data Generation

.github/workflows/Rest.yml

@@ -39,6 +39,10 @@ jobs:
         with:
           vcpkgGitCommitId: 5e5d0e1cd7785623065e77eff011afdeec1a3574
 
+      - name: Setup Ccache
+        uses: hendrikmuhs/ccache-action@main
+        continue-on-error: true
+
       - name: Build extension
         env:
           GEN: ninja
@@ -47,12 +51,16 @@ jobs:
           make release
 
       - name: Start Rest Catalog
-        working-directory: scripts/
         run: |
-          ./start-rest-catalog.sh
+          make start-rest-catalog
+
+      - name: Generate data
+        run: |
+          make data
 
-      - name: Test With rest catalog
+      - name: Test with rest catalog
         env:
           ICEBERG_SERVER_AVAILABLE: 1
+          DUCKDB_ICEBERG_HAVE_GENERATED_DATA: 1
         run: |
-          make test_release
+          make test_release

.gitignore

@@ -12,3 +12,7 @@ data/iceberg/generated_*
 scripts/metastore_db/
 scripts/derby.log
 scripts/test-script-with-path.sql
+scripts/data_generators/__pycache__/
+scripts/data_generators/*/__pycache__/
+scripts/data_generators/*/*/*.parquet
+data/generated/*

Makefile

@@ -5,18 +5,23 @@ EXT_NAME=iceberg
 EXT_CONFIG=${PROJ_DIR}extension_config.cmake
 
 # We need this for testing
-CORE_EXTENSIONS='httpfs'
+CORE_EXTENSIONS='parquet;httpfs'
 
 # Include the Makefile from extension-ci-tools
 include extension-ci-tools/makefiles/duckdb_extension.Makefile
 
+start-rest-catalog: install_requirements
+	./scripts/start-rest-catalog.sh
+
+install_requirements:
+	python3 -m pip install -r scripts/requirements.txt
+
 # Custom makefile targets
-data: data_clean
-	python3 scripts/test_data_generator/generate_iceberg.py 0.001 data/iceberg/generated_spec1_0_001 1
-	python3 scripts/test_data_generator/generate_iceberg.py 0.001 data/iceberg/generated_spec2_0_001 2
+data: data_clean start-rest-catalog
+	python3 scripts/data_generators/generate_data.py
 
 data_large: data data_clean
-	python3 scripts/test_data_generator/generate_iceberg.py 1 data/iceberg/generated_spec2_1 2
+	python3 scripts/data_generators/generate_data.py
 
 data_clean:
-	rm -rf data/iceberg/generated_*
+	rm -rf data/generated