Utilities to reduce tile size (#669)

planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java

@@ -87,6 +87,53 @@ public static List<VectorTile.Feature> mergeLineStrings(List<VectorTile.Feature>
     return mergeLineStrings(features, minLength, tolerance, buffer, false);
   }
 
+  /** Merges points with the same attributes into multipoints. */
+  public static List<VectorTile.Feature> mergeMultiPoint(List<VectorTile.Feature> features) {
+    return mergeGeometries(features, GeometryType.POINT);
+  }
+
+  /**
+   * Merges polygons with the same attributes into multipolygons.
+   * <p>
+   * NOTE: This does not attempt to combine overlapping geometries, see {@link #mergeOverlappingPolygons(List, double)}
+   * or {@link #mergeNearbyPolygons(List, double, double, double, double)} for that.
+   */
+  public static List<VectorTile.Feature> mergeMultiPolygon(List<VectorTile.Feature> features) {
+    return mergeGeometries(features, GeometryType.POLYGON);
+  }
+
+  /**
+   * Merges linestrings with the same attributes into multilinestrings.
+   * <p>
+   * NOTE: This does not attempt to connect linestrings that intersect at endpoints, see
+   * {@link #mergeLineStrings(List, double, double, double, boolean)} for that. Also, this removes extra detail that was
+   * preserved to improve connected-linestring merging, so you should only use one or the other.
+   */
+  public static List<VectorTile.Feature> mergeMultiLineString(List<VectorTile.Feature> features) {
+    return mergeGeometries(features, GeometryType.LINE);
+  }
+
+  private static List<VectorTile.Feature> mergeGeometries(
+    List<VectorTile.Feature> features,
+    GeometryType geometryType
+  ) {
+    List<VectorTile.Feature> result = new ArrayList<>(features.size());
+    var groupedByAttrs = groupByAttrs(features, result, geometryType);
+    for (List<VectorTile.Feature> groupedFeatures : groupedByAttrs) {
+      VectorTile.Feature feature1 = groupedFeatures.get(0);
+      if (groupedFeatures.size() == 1) {
+        result.add(feature1);
+      } else {
+        VectorTile.VectorGeometryMerger combined = VectorTile.newMerger(geometryType);
+        for (var feature : groupedFeatures) {
+          combined.accept(feature.geometry());
+        }
+        result.add(feature1.copyWithNewGeometry(combined.finish()));
+      }
+    }
+    return result;
+  }
+
   /**
    * Merges linestrings with the same attributes as {@link #mergeLineStrings(List, Function, double, double, boolean)}
    * except sets {@code resimplify=false} by default.
@@ -485,4 +532,27 @@ private static void depthFirstSearch(int startNode, IntArrayList group, IntObjec
       }
     }
   }
+
+  /**
+   * Returns a new list of features with points that are more than {@code buffer} pixels outside the tile boundary
+   * removed, assuming a 256x256px tile.
+   */
+  public static List<VectorTile.Feature> removePointsOutsideBuffer(List<VectorTile.Feature> features, double buffer) {
+    if (!Double.isFinite(buffer)) {
+      return features;
+    }
+    List<VectorTile.Feature> result = new ArrayList<>(features.size());
+    for (var feature : features) {
+      var geometry = feature.geometry();
+      if (geometry.geomType() == GeometryType.POINT) {
+        var newGeometry = geometry.filterPointsOutsideBuffer(buffer);
+        if (!newGeometry.isEmpty()) {
+          result.add(feature.copyWithNewGeometry(newGeometry));
+        }
+      } else {
+        result.add(feature);
+      }
+    }
+    return result;
+  }
 }

planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java

@@ -31,7 +31,9 @@
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.util.function.Consumer;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import javax.annotation.concurrent.NotThreadSafe;
@@ -176,7 +178,7 @@ private static int[] unscale(int[] commands, int scale, GeometryType geomType) {
     return result.toArray();
   }
 
-  private static int zigZagEncode(int n) {
+  static int zigZagEncode(int n) {
     // https://developers.google.com/protocol-buffers/docs/encoding#types
     return (n << 1) ^ (n >> 31);
   }
@@ -206,6 +208,11 @@ private static Geometry decodeCommands(GeometryType geomType, int[] commands, in
           length = commands[i++];
           command = length & ((1 << 3) - 1);
           length = length >> 3;
+          assert geomType != GeometryType.POINT || i == 1 : "Invalid multipoint, command found at index %d, expected 0"
+            .formatted(i);
+          assert geomType != GeometryType.POINT ||
+            (length * 2 + 1 == geometryCount) : "Invalid multipoint: int[%d] length=%d".formatted(geometryCount,
+              length);
         }
 
         if (length > 0) {
@@ -404,14 +411,22 @@ public static VectorGeometry encodeGeometry(Geometry geometry, int scale) {
     return new VectorGeometry(getCommands(geometry, scale), GeometryType.typeOf(geometry), scale);
   }
 
+  /**
+   * Returns a new {@link VectorGeometryMerger} that combines encoded geometries of the same type into a merged
+   * multipoint, multilinestring, or multipolygon.
+   */
+  public static VectorGeometryMerger newMerger(GeometryType geometryType) {
+    return new VectorGeometryMerger(geometryType);
+  }
+
   /**
    * Adds features in a layer to this tile.
    *
    * @param layerName name of the layer in this tile to add the features to
    * @param features  features to add to the tile
    * @return this encoder for chaining
    */
-  public VectorTile addLayerFeatures(String layerName, List<? extends Feature> features) {
+  public VectorTile addLayerFeatures(String layerName, List<Feature> features) {
     if (features.isEmpty()) {
       return this;
     }
@@ -548,7 +563,7 @@ public boolean likelyToBeDuplicated() {
     return layers.values().stream().allMatch(v -> v.encodedFeatures.isEmpty()) || containsOnlyFillsOrEdges();
   }
 
-  private enum Command {
+  enum Command {
     MOVE_TO(1),
     LINE_TO(2),
     CLOSE_PATH(7);
@@ -560,6 +575,85 @@ private enum Command {
     }
   }
 
+  /**
+   * Utility that combines encoded geometries of the same type into a merged multipoint, multilinestring, or
+   * multipolygon.
+   */
+  public static class VectorGeometryMerger implements Consumer<VectorGeometry> {
+    // For the most part this just concatenates the individual command arrays together
+    // EXCEPT we need to adjust the first coordinate of each subsequent linestring to
+    // be an offset from the end of the previous linestring.
+    // AND we need to combine all multipoint "move to" commands into one at the start of
+    // the sequence
+
+    private final GeometryType geometryType;
+    private int overallX = 0;
+    private int overallY = 0;
+    private final IntArrayList result = new IntArrayList();
+
+    private VectorGeometryMerger(GeometryType geometryType) {
+      this.geometryType = geometryType;
+    }
+
+    @Override
+    public void accept(VectorGeometry vectorGeometry) {
+      if (vectorGeometry.geomType != geometryType) {
+        throw new IllegalArgumentException(
+          "Cannot merge a " + vectorGeometry.geomType.name().toLowerCase(Locale.ROOT) + " geometry into a multi" +
+            vectorGeometry.geomType.name().toLowerCase(Locale.ROOT));
+      }
+      if (vectorGeometry.isEmpty()) {
+        return;
+      }
+      var commands = vectorGeometry.unscale().commands();
+      int x = 0;
+      int y = 0;
+
+      int geometryCount = commands.length;
+      int length = 0;
+      int command = 0;
+      int i = 0;
+
+      result.ensureCapacity(result.elementsCount + commands.length);
+      // and multipoints will end up with only one command ("move to" with length=# points)
+      if (geometryType != GeometryType.POINT || result.isEmpty()) {
+        result.add(commands[0]);
+      }
+      result.add(zigZagEncode(zigZagDecode(commands[1]) - overallX));
+      result.add(zigZagEncode(zigZagDecode(commands[2]) - overallY));
+      if (commands.length > 3) {
+        result.add(commands, 3, commands.length - 3);
+      }
+
+      while (i < geometryCount) {
+        if (length <= 0) {
+          length = commands[i++];
+          command = length & ((1 << 3) - 1);
+          length = length >> 3;
+        }
+
+        if (length > 0) {
+          length--;
+          if (command != Command.CLOSE_PATH.value) {
+            x += zigZagDecode(commands[i++]);
+            y += zigZagDecode(commands[i++]);
+          }
+        }
+      }
+      overallX = x;
+      overallY = y;
+    }
+
+    /** Returns the merged multi-geometry. */
+    public VectorGeometry finish() {
+      // set the correct "move to" length for multipoints based on how many points were actually added
+      if (geometryType == GeometryType.POINT) {
+        result.buffer[0] = Command.MOVE_TO.value | (((result.size() - 1) / 2) << 3);
+      }
+      return new VectorGeometry(result.toArray(), geometryType, 0);
+    }
+  }
+
   /**
    * A vector geometry encoded as a list of commands according to the
    * <a href="https://github.com/mapbox/vector-tile-spec/tree/master/2.1#43-geometry-encoding">vector tile
@@ -578,6 +672,7 @@ public record VectorGeometry(int[] commands, GeometryType geomType, int scale) {
     private static final int BOTTOM = 1 << 3;
     private static final int INSIDE = 0;
     private static final int ALL = TOP | LEFT | RIGHT | BOTTOM;
+    private static final VectorGeometry EMPTY_POINT = new VectorGeometry(new int[0], GeometryType.POINT, 0);
 
     public VectorGeometry {
       if (scale < 0) {
@@ -759,6 +854,75 @@ public boolean isFillOrEdge(boolean allowEdges) {
       return visitedEnoughSides(allowEdges, visited);
     }
 
+    /** Returns true if there are no commands in this geometry. */
+    public boolean isEmpty() {
+      return commands.length == 0;
+    }
+
+    /**
+     * If this is a point, returns an empty geometry if more than {@code buffer} pixels outside the tile bounds, or if
+     * it is a multipoint than removes all points outside the buffer.
+     */
+    public VectorGeometry filterPointsOutsideBuffer(double buffer) {
+      if (geomType != GeometryType.POINT) {
+        return this;
+      }
+      IntArrayList result = null;
+
+      int extent = (EXTENT << scale);
+      int bufferInt = (int) Math.ceil(buffer * extent / 256);
+      int min = -bufferInt;
+      int max = extent + bufferInt;
+
+      int x = 0;
+      int y = 0;
+      int lastX = 0;
+      int lastY = 0;
+
+      int geometryCount = commands.length;
+      int length = 0;
+      int i = 0;
+
+      while (i < geometryCount) {
+        if (length <= 0) {
+          length = commands[i++] >> 3;
+          assert i <= 1 : "Bad index " + i;
+        }
+
+        if (length > 0) {
+          length--;
+          x += zigZagDecode(commands[i++]);
+          y += zigZagDecode(commands[i++]);
+          if (x < min || y < min || x > max || y > max) {
+            if (result == null) {
+              // short-circuit the common case of only a single point that gets filtered-out
+              if (commands.length == 3) {
+                return EMPTY_POINT;
+              }
+              result = new IntArrayList(commands.length);
+              result.add(commands, 0, i - 2);
+            }
+          } else {
+            if (result != null) {
+              result.add(zigZagEncode(x - lastX), zigZagEncode(y - lastY));
+            }
+            lastX = x;
+            lastY = y;
+          }
+        }
+      }
+      if (result != null) {
+        if (result.size() < 3) {
+          result.elementsCount = 0;
+        } else {
+          result.set(0, Command.MOVE_TO.value | (((result.size() - 1) / 2) << 3));
+        }
+
+        return new VectorGeometry(result.toArray(), geomType, scale);
+      } else {
+        return this;
+      }
+    }
   }
 
   /**
@@ -807,7 +971,7 @@ public Feature copyWithNewGeometry(Geometry newGeometry) {
      * Returns a copy of this feature with {@code geometry} replaced with {@code newGeometry}.
      */
     public Feature copyWithNewGeometry(VectorGeometry newGeometry) {
-      return new Feature(
+      return newGeometry == geometry ? this : new Feature(
         layer,
         id,
         newGeometry,

planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java

@@ -277,13 +277,13 @@ private void tileEncoderSink(Iterable<TileBatch> prev) throws IOException {
           layerStats = lastLayerStats;
           memoizedTiles.inc();
         } else {
-          VectorTile en = tileFeatures.getVectorTileEncoder();
-          if (skipFilled && (lastIsFill = en.containsOnlyFills())) {
+          VectorTile tile = tileFeatures.getVectorTile();
+          if (skipFilled && (lastIsFill = tile.containsOnlyFills())) {
             encoded = null;
             layerStats = null;
             bytes = null;
           } else {
-            var proto = en.toProto();
+            var proto = tile.toProto();
             encoded = proto.toByteArray();
             bytes = switch (config.tileCompression()) {
               case GZIP -> gzip(encoded);
@@ -301,7 +301,7 @@ private void tileEncoderSink(Iterable<TileBatch> prev) throws IOException {
           lastEncoded = encoded;
           lastBytes = bytes;
           last = tileFeatures;
-          if (archive.deduplicates() && en.likelyToBeDuplicated() && bytes != null) {
+          if (archive.deduplicates() && tile.likelyToBeDuplicated() && bytes != null) {
             tileDataHash = generateContentHash(bytes);
           } else {
             tileDataHash = null;