Improve the basemap schema and workflow

.gitignore

@@ -26,6 +26,7 @@ basemap/cache_*/
 basemap/local.*
 basemap/tiles.mbtiles
 basemap/tiles/
+basemap/cache/
 
 # Examples
 examples/ip-to-location/archives/

.run/basemap-serve.run.xml

@@ -2,7 +2,7 @@
   <configuration default="false" name="basemap-serve" type="Application" factoryName="Application">
     <option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" />
     <module name="baremaps-cli" />
-    <option name="PROGRAM_PARAMETERS" value="map serve --tileset tileset.js --style style.js" />
+    <option name="PROGRAM_PARAMETERS" value="map serve --tileset tileset.js --style style.js --log-level DEBUG" />
     <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/basemap" />
     <method v="2">
       <option name="Make" enabled="true" />

baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOsm.java

@@ -18,7 +18,6 @@
 package org.apache.baremaps.cli.database;
 
 
-
 import java.nio.file.Path;
 import java.util.concurrent.Callable;
 import org.apache.baremaps.cli.Options;
@@ -55,6 +54,7 @@ public Integer call() throws Exception {
         file.toAbsolutePath(),
         database,
         srid,
+        true,
         true).execute(new WorkflowContext());
     return 0;
   }

baremaps-core/src/main/java/org/apache/baremaps/tasks/ImportOsmPbf.java

@@ -44,6 +44,7 @@ public class ImportOsmPbf implements Task {
   private Object database;
   private Integer databaseSrid;
   private Boolean replaceExisting;
+  private Boolean truncateTables;
 
   /**
    * Constructs a {@code ImportOsmPbf}.
@@ -61,11 +62,12 @@ public ImportOsmPbf() {
    * @param replaceExisting whether to replace the existing tables
    */
   public ImportOsmPbf(Path file, Object database,
-      Integer databaseSrid, Boolean replaceExisting) {
+      Integer databaseSrid, Boolean replaceExisting, Boolean truncateTables) {
     this.file = file;
     this.database = database;
     this.databaseSrid = databaseSrid;
     this.replaceExisting = replaceExisting;
+    this.truncateTables = truncateTables;
   }
 
   /**
@@ -82,8 +84,8 @@ public void execute(WorkflowContext context) throws Exception {
     var wayRepository = new WayRepository(datasource);
     var relationRepository = new RelationRepository(datasource);
 
+    // Drop the existing tables
     if (TRUE.equals(replaceExisting)) {
-      // Drop the existing tables
       headerRepository.drop();
       nodeRepository.drop();
       wayRepository.drop();
@@ -96,6 +98,14 @@ public void execute(WorkflowContext context) throws Exception {
       relationRepository.create();
     }
 
+    // Truncate the existing tables
+    if (TRUE.equals(truncateTables)) {
+      headerRepository.truncate();
+      nodeRepository.truncate();
+      wayRepository.truncate();
+      relationRepository.truncate();
+    }
+
     var coordinateMap = context.getCoordinateMap();
     var referenceMap = context.getReferenceMap();
 

baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java

@@ -43,7 +43,7 @@ public void test() throws IOException {
             new Step("import", List.of("download"),
                 List.of(new ImportOsmPbf(Paths.get("liechtenstein-latest.osm.pbf"),
                     "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps",
-                    3857, true)))));
+                    3857, true, true)))));
     var json = mapper.writeValueAsString(workflow1);
     assertTrue(json.contains(DownloadUrl.class.getSimpleName()));
     assertTrue(json.contains(ImportOsmPbf.class.getSimpleName()));

baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java

@@ -103,7 +103,7 @@ void execute() {
         new Step("import-osmpbf", List.of("fetch-osmpbf"),
             List.of(new ImportOsmPbf(Paths.get("downloads/liechtenstein.osm.pbf"),
                 jdbcUrl(),
-                3857, true))),
+                3857, true, true))),
         new Step("fetch-shapefile", List.of(), List.of(new DownloadUrl(
             "https://osmdata.openstreetmap.de/download/simplified-water-polygons-split-3857.zip",
             Paths.get("downloads/simplified-water-polygons-split-3857.zip"), false))),

baremaps-postgres/src/main/java/org/apache/baremaps/postgres/openstreetmap/CoordinateMap.java

@@ -53,7 +53,7 @@ public class CoordinateMap extends PostgresMap<Long, Coordinate> {
    * Constructs a {@link CoordinateMap}.
    */
   public CoordinateMap(DataSource dataSource) {
-    this(dataSource, "public", "osm_nodes");
+    this(dataSource, "public", "osm_node");
   }
 
   /**

baremaps-postgres/src/main/java/org/apache/baremaps/postgres/openstreetmap/HeaderRepository.java

@@ -67,7 +67,7 @@ public HeaderRepository(DataSource dataSource) {
     this(
         dataSource,
         "public",
-        "osm_headers",
+        "osm_header",
         "replication_sequence_number",
         "replication_timestamp",
         "replication_url",

baremaps-postgres/src/main/java/org/apache/baremaps/postgres/openstreetmap/NodeRepository.java

@@ -74,7 +74,7 @@ public NodeRepository(DataSource dataSource) {
     this(
         dataSource,
         "public",
-        "osm_nodes",
+        "osm_node",
         "id",
         "version",
         "uid",

baremaps-postgres/src/main/java/org/apache/baremaps/postgres/openstreetmap/ReferenceMap.java

@@ -55,7 +55,7 @@ public class ReferenceMap extends PostgresMap<Long, List<Long>> {
    * Constructs a {@code PostgresReferenceMap}.
    */
   public ReferenceMap(DataSource dataSource) {
-    this(dataSource, "public", "osm_ways");
+    this(dataSource, "public", "osm_way");
   }
 
   /**

baremaps-postgres/src/main/java/org/apache/baremaps/postgres/openstreetmap/RelationRepository.java

@@ -68,7 +68,7 @@ public RelationRepository(DataSource dataSource) {
     this(
         dataSource,
         "public",
-        "osm_relations",
+        "osm_relation",
         "id",
         "version",
         "uid",

baremaps-postgres/src/main/java/org/apache/baremaps/postgres/openstreetmap/WayRepository.java

@@ -76,7 +76,7 @@ public class WayRepository implements Repository<Long, Way> {
   public WayRepository(DataSource dataSource) {
     this(dataSource,
         "public",
-        "osm_ways",
+        "osm_way",
         "id",
         "version",
         "uid",

baremaps-postgres/src/main/java/org/apache/baremaps/postgres/store/PostgresDataStore.java

@@ -121,20 +121,20 @@ public void add(String name, DataTable table) {
 
       var schema = new DataSchemaImpl(name, properties);
 
-      // Drop the table if it exists
-      var dropQuery = dropTable(schema);
-      logger.debug(dropQuery);
-      try (var dropStatement = connection.prepareStatement(dropQuery)) {
-        dropStatement.execute();
-      }
-
       // Create the table
       var createQuery = createTable(schema);
       logger.debug(createQuery);
       try (var createStatement = connection.prepareStatement(createQuery)) {
         createStatement.execute();
       }
 
+      // Truncate the table
+      var truncateQuery = truncateTable(schema);
+      logger.debug(truncateQuery);
+      try (var truncateStatement = connection.prepareStatement(truncateQuery)) {
+        truncateStatement.execute();
+      }
+
       // Copy the data
       var pgConnection = connection.unwrap(PGConnection.class);
       var copyQuery = copy(schema);
@@ -206,6 +206,16 @@ protected String dropTable(DataSchema schema) {
     return String.format("DROP TABLE IF EXISTS \"%s\" CASCADE", schema.name());
   }
 
+  /**
+   * Generate a truncate table query.
+   *
+   * @param schema the schema
+   * @return the query
+   */
+  protected String truncateTable(DataSchema schema) {
+    return String.format("TRUNCATE TABLE \"%s\" CASCADE", schema.name());
+  }
+
   /**
    * Generate a create table query.
    *
@@ -214,7 +224,7 @@ protected String dropTable(DataSchema schema) {
    */
   protected String createTable(DataSchema schema) {
     StringBuilder builder = new StringBuilder();
-    builder.append("CREATE TABLE \"");
+    builder.append("CREATE TABLE IF NOT EXISTS \"");
     builder.append(schema.name());
     builder.append("\" (");
     builder.append(schema.columns().stream()

baremaps-postgres/src/test/java/org/apache/baremaps/postgres/metadata/PostgresMetadataTest.java

@@ -40,10 +40,10 @@ void resetDatabase() throws SQLException, IOException {
       PostgresUtils.executeResource(connection, "queries/osm_drop_tables.sql");
       PostgresUtils.executeResource(connection, "queries/osm_drop_tables.sql");
       PostgresUtils.executeResource(connection, "queries/osm_create_tables.sql");
-      assertTrue(tableExists("osm_headers"));
-      assertTrue(tableExists("osm_nodes"));
-      assertTrue(tableExists("osm_ways"));
-      assertTrue(tableExists("osm_relations"));
+      assertTrue(tableExists("osm_header"));
+      assertTrue(tableExists("osm_node"));
+      assertTrue(tableExists("osm_way"));
+      assertTrue(tableExists("osm_relation"));
     }
   }
 

baremaps-postgres/src/test/resources/queries/hello-world.sql

@@ -13,8 +13,5 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 
-DO $$
-    BEGIN
-        PERFORM 'Hello, World!';
-    END
-$$;
+DO $$ BEGIN PERFORM 'Hello, World!';
+END $$;

baremaps-postgres/src/test/resources/queries/osm_create_extensions.sql

@@ -12,5 +12,8 @@
 -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
-CREATE EXTENSION IF NOT EXISTS hstore;
-CREATE EXTENSION IF NOT EXISTS postgis;
+CREATE
+    EXTENSION IF NOT EXISTS hstore;
+
+CREATE
+    EXTENSION IF NOT EXISTS postgis;

baremaps-postgres/src/test/resources/queries/osm_create_gin_indexes.sql

@@ -12,5 +12,12 @@
 -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
-CREATE INDEX CONCURRENTLY IF NOT EXISTS osm_ways_gin ON osm_ways USING gin (nodes);
-CREATE INDEX CONCURRENTLY IF NOT EXISTS osm_relations_gin ON osm_relations USING gin (member_refs);
+CREATE
+    INDEX IF NOT EXISTS osm_way_gin ON
+    osm_way
+        USING gin(nodes);
+
+CREATE
+    INDEX IF NOT EXISTS osm_relation_gin ON
+    osm_relation
+        USING gin(member_refs);

baremaps-postgres/src/test/resources/queries/osm_create_gist_indexes.sql

@@ -12,6 +12,17 @@
 -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
-CREATE INDEX CONCURRENTLY IF NOT EXISTS osm_nodes_gix ON osm_nodes USING GIST (geom);
-CREATE INDEX CONCURRENTLY IF NOT EXISTS osm_ways_gix ON osm_ways USING GIST (geom);
-CREATE INDEX CONCURRENTLY IF NOT EXISTS osm_relations_gix ON osm_relations USING GIST (geom);
+CREATE
+    INDEX IF NOT EXISTS osm_node_gix ON
+    osm_node
+        USING GIST(geom);
+
+CREATE
+    INDEX IF NOT EXISTS osm_way_gix ON
+    osm_way
+        USING GIST(geom);
+
+CREATE
+    INDEX IF NOT EXISTS osm_relation_gix ON
+    osm_relation
+        USING GIST(geom);

baremaps-postgres/src/test/resources/queries/osm_create_spgist_indexes.sql

@@ -12,6 +12,17 @@
 -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
-CREATE INDEX CONCURRENTLY IF NOT EXISTS osm_nodes_gix ON osm_nodes USING SPGIST (geom);
-CREATE INDEX CONCURRENTLY IF NOT EXISTS osm_ways_gix ON osm_ways USING SPGIST (geom);
-CREATE INDEX CONCURRENTLY IF NOT EXISTS osm_relations_gix ON osm_relations USING SPGIST (geom);
+CREATE
+    INDEX IF NOT EXISTS osm_node_gix ON
+    osm_node
+        USING SPGIST(geom);
+
+CREATE
+    INDEX IF NOT EXISTS osm_way_gix ON
+    osm_way
+        USING SPGIST(geom);
+
+CREATE
+    INDEX IF NOT EXISTS osm_relation_gix ON
+    osm_relation
+        USING SPGIST(geom);