Check for partition stats

apache · Jan 8, 2025 · e58b461 · e58b461
1 parent aa5693c
commit e58b461
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 0 deletions.
diff --git a/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java b/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java
@@ -127,6 +127,7 @@ public static TableMetadata replacePaths(
         metadataLogEntries,
         metadata.refs(),
         updatePathInStatisticsFiles(metadata.statisticsFiles(), sourcePrefix, targetPrefix),
+        // TODO: update partition statistics file paths
         metadata.partitionStatisticsFiles(),
         metadata.changes());
   }

diff --git a/...3.5/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java b/...3.5/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java
@@ -265,6 +265,11 @@ private String rebuildMetadata() {
     TableMetadata endMetadata =
         ((HasTableOperations) newStaticTable(endVersionName, table.io())).operations().current();
 
+    Preconditions.checkArgument(
+        endMetadata.partitionStatisticsFiles() == null
+            || endMetadata.partitionStatisticsFiles().isEmpty(),
+        "Partition statistics files are not supported yet.");
+
     // rebuild version files
     RewriteResult<Snapshot> rewriteVersionResult = rewriteVersionFiles(endMetadata);
     Set<Snapshot> deltaSnapshots = deltaSnapshots(startMetadata, rewriteVersionResult.toRewrite());

diff --git a/...3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java b/...3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java
@@ -37,11 +37,13 @@
 import org.apache.iceberg.DataFile;
 import org.apache.iceberg.DeleteFile;
 import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
 import org.apache.iceberg.PartitionSpec;
 import org.apache.iceberg.Schema;
 import org.apache.iceberg.StaticTableOperations;
 import org.apache.iceberg.Table;
 import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableMetadataParser;
 import org.apache.iceberg.TableProperties;
 import org.apache.iceberg.TestHelpers;
 import org.apache.iceberg.actions.ActionsProvider;
@@ -728,6 +730,39 @@ public void testInvalidArgs() {
         .hasMessageContaining("End version('null') cannot be empty");
   }
 
+  @Test
+  public void testPartitionStatisticFile() throws IOException {
+    String sourceTableLocation = newTableLocation();
+    Map<String, String> properties = Maps.newHashMap();
+    properties.put("format-version", "2");
+    String tableName = "v2tblwithPartStats";
+    Table sourceTable =
+        createMetastoreTable(sourceTableLocation, properties, "default", tableName, 0);
+
+    TableMetadata metadata = currentMetadata(sourceTable);
+    TableMetadata withPartStatistics =
+        TableMetadata.buildFrom(metadata)
+            .setPartitionStatistics(
+                ImmutableGenericPartitionStatisticsFile.builder()
+                    .snapshotId(11L)
+                    .path("/some/partition/stats/file.parquet")
+                    .fileSizeInBytes(42L)
+                    .build())
+            .build();
+
+    OutputFile file = sourceTable.io().newOutputFile(metadata.metadataFileLocation());
+    TableMetadataParser.overwrite(withPartStatistics, file);
+
+    assertThatThrownBy(
+            () ->
+                actions()
+                    .rewriteTablePath(sourceTable)
+                    .rewriteLocationPrefix(sourceTableLocation, targetTableLocation())
+                    .execute())
+        .isInstanceOf(IllegalArgumentException.class)
+        .hasMessageContaining("Partition statistics files are not supported yet");
+  }
+
   @Test
   public void testTableWithManyStatisticFiles() throws IOException {
     String sourceTableLocation = newTableLocation();