clipper2 + less allocations

.pkg

@@ -6,6 +6,10 @@
   url=git@github.com:motis-project/clipper.git
   branch=master
   commit=2abf72015b85f1cfd38706b21d4f28b8c359c7d5
+[Clipper2]
+  url=git@github.com:motis-project/Clipper2.git
+  branch=main
+  commit=f9c5eb6e14a59f6f5d65fbfb3564519a561cf4fd
 [concurrentqueue]
   url=git@github.com:motis-project/concurrentqueue.git
   branch=master

CMakeLists.txt

@@ -23,6 +23,13 @@ create_resource("${CMAKE_CURRENT_SOURCE_DIR}/profile/" "${tiles-profiles-res}" t
 
 include(cmake/clang-tidy.cmake)
 
+# Clipper2 (fast polygon / rectangle clipping). Wired explicitly here because
+# the motis pkg tool does not (yet) emit an add_subdirectory() for it.
+if (NOT TARGET Clipper2)
+  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../Clipper2
+                   ${CMAKE_CURRENT_BINARY_DIR}/Clipper2 EXCLUDE_FROM_ALL)
+endif()
+
 # --- tiles lib
 file(GLOB_RECURSE tiles-files
   src/db/*.cc
@@ -39,7 +46,7 @@ target_compile_options(tiles PRIVATE ${TILES_WARNINGS})
 target_include_directories(tiles PUBLIC include)
 target_link_libraries(tiles
   boost
-  clipper
+  Clipper2
   fmt
   geo
   lmdb
@@ -94,6 +101,12 @@ target_compile_options(tiles-benchmark PRIVATE ${TILES_WARNINGS})
 target_include_directories(tiles-benchmark PUBLIC include)
 target_link_libraries(tiles-benchmark boost tiles lmdb geo)
 
+# --- tier-index experiment exe
+add_executable(tiles-index-experiment EXCLUDE_FROM_ALL src/index_experiment.cc)
+set_property(TARGET tiles-index-experiment PROPERTY CXX_STANDARD 23)
+target_include_directories(tiles-index-experiment PUBLIC include)
+target_link_libraries(tiles-index-experiment boost tiles lmdb geo)
+
 # --- test exe
 file(GLOB_RECURSE tiles-test-files
   test/*_test.cc

src/fixed/algo/clip.cc

@@ -1,21 +1,30 @@
 #include "tiles/fixed/algo/clip.h"
 
+#include <cmath>
+#include <limits>
+#include <vector>
+
 #include "boost/geometry.hpp"
 
-#include "clipper/clipper.hpp"
+#include "clipper2/clipper.h"
 
 #include "utl/erase_if.h"
 #include "utl/verify.h"
 
 #include "tiles/util.h"
 
-namespace cl = ClipperLib;
+namespace c2 = Clipper2Lib;
 
 namespace tiles {
 
-fixed_geometry clip(fixed_null const&, fixed_box const&) {
-  return fixed_null{};
-}
+// Polygon-clip backend:
+//   1 = Clipper2 RectClip64  (specialized, fast rectangle clipping)
+//   0 = Clipper2 Clipper64 + PolyTree64  (general boolean, faithful to v1)
+#ifndef TILES_CLIP_RECTCLIP
+#define TILES_CLIP_RECTCLIP 1
+#endif
+
+fixed_geometry clip(fixed_null const&, fixed_box const&) { return fixed_null{}; }
 
 fixed_geometry clip(fixed_point const& in, fixed_box const& box) {
   fixed_point out;
@@ -45,69 +54,163 @@ fixed_geometry clip(fixed_polyline const& in, fixed_box const& box) {
   }
 }
 
-void to_fixed_polygon2(fixed_polygon& polygon, cl::PolyNodes const& nodes) {
-  auto const path_to_ring = [](auto const& path) {
-    utl::verify(!path.empty(), "path empty");
-    fixed_ring ring;
-    ring.reserve(path.size() + 1);
-    for (auto const& pt : path) {
-      ring.emplace_back(pt.X, pt.Y);
-    }
-    ring.emplace_back(path[0].X, path[0].Y);
-    return ring;
-  };
-
-  for (auto const* outer : nodes) {
-    utl::verify(!outer->IsHole(), "outer ring is hole");
-    fixed_simple_polygon simple;
-    simple.outer() = path_to_ring(outer->Contour);
+namespace {
 
-    for (auto const* inner : outer->Childs) {
-      utl::verify(inner->IsHole(), "inner ring is no hole");
-      simple.inners().emplace_back(path_to_ring(inner->Contour));
-
-      to_fixed_polygon2(polygon, inner->Childs);
-    }
-
-    polygon.emplace_back(std::move(simple));
+fixed_ring to_ring(c2::Path64 const& path) {
+  fixed_ring ring;
+  if (path.empty()) {
+    return ring;
+  }
+  ring.reserve(path.size() + 1);
+  for (auto const& pt : path) {
+    ring.emplace_back(pt.x, pt.y);
   }
+  ring.emplace_back(path.front().x, path.front().y);  // close ring
+  return ring;
 }
 
-fixed_geometry clip(fixed_polygon const& in, fixed_box const& box) {
-  auto const clip = cl::Path{{box.min_corner().x(), box.min_corner().y()},
-                             {box.max_corner().x(), box.min_corner().y()},
-                             {box.max_corner().x(), box.max_corner().y()},
-                             {box.min_corner().x(), box.max_corner().y()}};
-  cl::Paths subject;
+c2::Paths64 to_subject(fixed_polygon const& in) {
+  c2::Paths64 subject;
   for (auto const& poly : in) {
     subject.emplace_back();
-
     for (auto const& pt : poly.outer()) {
       subject.back().emplace_back(pt.x(), pt.y());
     }
-    subject.back().pop_back();
+    if (!subject.back().empty()) {
+      subject.back().pop_back();  // Clipper2 paths are implicitly closed
+    }
 
     for (auto const& inner : poly.inners()) {
       subject.emplace_back();
       for (auto const& pt : inner) {
         subject.back().emplace_back(pt.x(), pt.y());
       }
-      subject.back().pop_back();
+      if (!subject.back().empty()) {
+        subject.back().pop_back();
+      }
     }
   }
+  return subject;
+}
 
-  cl::Clipper clpr;
-  utl::verify(clpr.AddPaths(subject, cl::ptSubject, true), "AddPath1 failed");
-  utl::verify(clpr.AddPath(clip, cl::ptClip, true), "AddPath2 failed");
+#if TILES_CLIP_RECTCLIP
+// Rebuild the boost multi-polygon hierarchy from the flat ring set returned by
+// RectClip. Ring role is determined by containment depth (even depth = outer
+// ring, odd depth = hole of its immediate parent). This is orientation
+// agnostic; boost::geometry::correct() normalizes winding afterwards.
+void to_fixed_polygon(fixed_polygon& out, c2::Paths64 const& paths) {
+  auto const n = paths.size();
+
+  std::vector<double> area(n, 0.0);
+  for (auto i = 0U; i < n; ++i) {
+    area[i] = std::abs(c2::Area(paths[i]));
+  }
 
-  cl::PolyTree solution;
-  clpr.Execute(cl::ctIntersection, solution, cl::pftEvenOdd, cl::pftEvenOdd);
-  if (solution.Childs.empty()) {
-    return fixed_null{};
+  // parent[i] = index of the smallest-area ring strictly containing ring i.
+  std::vector<int> parent(n, -1);
+  for (auto i = 0U; i < n; ++i) {
+    if (paths[i].empty()) {
+      continue;
+    }
+    auto const pt = paths[i].front();
+    auto best = std::numeric_limits<double>::max();
+    for (auto j = 0U; j < n; ++j) {
+      if (i == j || paths[j].size() < 3 || area[j] <= area[i]) {
+        continue;
+      }
+      if (c2::PointInPolygon(pt, paths[j]) !=
+          c2::PointInPolygonResult::IsInside) {
+        continue;
+      }
+      if (area[j] < best) {
+        best = area[j];
+        parent[i] = static_cast<int>(j);
+      }
+    }
   }
 
+  auto const depth = [&](std::size_t i) {
+    auto d = 0;
+    for (auto p = parent[i]; p != -1; p = parent[static_cast<std::size_t>(p)]) {
+      ++d;
+    }
+    return d;
+  };
+
+  std::vector<int> out_idx(n, -1);
+  for (auto i = 0U; i < n; ++i) {  // outer rings (even depth)
+    if (paths[i].size() < 3 || (depth(i) & 1) != 0) {
+      continue;
+    }
+    fixed_simple_polygon simple;
+    simple.outer() = to_ring(paths[i]);
+    out_idx[i] = static_cast<int>(out.size());
+    out.emplace_back(std::move(simple));
+  }
+  for (auto i = 0U; i < n; ++i) {  // holes (odd depth) -> attach to parent outer
+    if (paths[i].size() < 3 || (depth(i) & 1) != 1) {
+      continue;
+    }
+    auto const p = parent[i];
+    if (p >= 0 && out_idx[static_cast<std::size_t>(p)] >= 0) {
+      out[static_cast<std::size_t>(out_idx[static_cast<std::size_t>(p)])]
+          .inners()
+          .emplace_back(to_ring(paths[i]));
+    }
+  }
+}
+#else
+void to_fixed_polygon(fixed_polygon& polygon, c2::PolyPath64 const& parent) {
+  for (auto const& child : parent) {  // top-level children are outer rings
+    auto const& outer = *child;
+    fixed_simple_polygon simple;
+    simple.outer() = to_ring(outer.Polygon());
+
+    for (auto const& grand_child : outer) {  // children of an outer are holes
+      auto const& inner = *grand_child;
+      simple.inners().emplace_back(to_ring(inner.Polygon()));
+      to_fixed_polygon(polygon, inner);  // islands nested inside holes
+    }
+
+    polygon.emplace_back(std::move(simple));
+  }
+}
+#endif
+
+}  // namespace
+
+fixed_geometry clip(fixed_polygon const& in, fixed_box const& box) {
+  auto const subject = to_subject(in);
+
+  auto const rect = c2::Rect64{box.min_corner().x(), box.min_corner().y(),
+                               box.max_corner().x(), box.max_corner().y()};
+
   fixed_polygon out;
-  to_fixed_polygon2(out, solution.Childs);
+
+#if TILES_CLIP_RECTCLIP
+  auto const solution = c2::RectClip(rect, subject);
+  if (solution.empty()) {
+    return fixed_null{};
+  }
+  to_fixed_polygon(out, solution);
+#else
+  c2::Clipper64 clpr;
+  clpr.AddSubject(subject);
+  clpr.AddClip(c2::Paths64{c2::Path64{{rect.left, rect.top},
+                                      {rect.right, rect.top},
+                                      {rect.right, rect.bottom},
+                                      {rect.left, rect.bottom}}});
+  c2::PolyTree64 solution;
+  clpr.Execute(c2::ClipType::Intersection, c2::FillRule::EvenOdd, solution);
+  if (solution.Count() == 0) {
+    return fixed_null{};
+  }
+  to_fixed_polygon(out, solution);
+#endif
+
+  if (out.empty()) {
+    return fixed_null{};
+  }
 
   boost::geometry::correct(out);
   return out;

src/import.cc

@@ -4,6 +4,8 @@
 #include "conf/configuration.h"
 #include "conf/options_parser.h"
 
+#include "utl/progress_tracker.h"
+
 #include "tiles/db/clear_database.h"
 #include "tiles/db/database_stats.h"
 #include "tiles/db/feature_pack.h"
@@ -64,6 +66,10 @@ int run_tiles_import(int argc, char const** argv) {
     return 1;
   }
 
+  // standalone tiles-import lacks the progress tracker that motis' importer
+  // sets up; prepare_tiles/coastlines require an active (global) tracker.
+  utl::activate_progress_tracker("tiles-import");
+
   if (opt.has_any_task({"features"})) {
     check_profile(opt.osm_profile_);
   }

src/mvt/tile_builder.cc

@@ -82,30 +82,31 @@ struct layer_builder {
     has_geometry_ = true;
     ++features_written_;
 
-    std::string feature_buf;
-    pbf_builder<ttm::Feature> feature_pb(feature_buf);
+    feature_buf_.clear();
+    pbf_builder<ttm::Feature> feature_pb(feature_buf_);
 
     encode_geometry(feature_pb, f.geometry_, spec_);
 
     feature_pb.add_uint64(ttm::Feature::optional_uint64_id, f.id_);
     write_metadata(feature_pb, f.meta_);
-    pb_.add_message(ttm::Layer::repeated_Feature_features, feature_buf);
+    pb_.add_message(ttm::Layer::repeated_Feature_features, feature_buf_);
   }
 
   void write_metadata(pbf_builder<ttm::Feature>& pb,
                       std::vector<metadata> const& meta) {
-    std::vector<uint32_t> t;
+    tags_.clear();
 
     for (auto const& m : meta) {
       if (m.key_ == "layer" || boost::starts_with(m.key_, "__")) {
         continue;
       }
 
-      t.emplace_back(utl::get_or_create_index(meta_key_cache_, m.key_));
-      t.emplace_back(utl::get_or_create_index(meta_value_cache_, m.value_));
+      tags_.emplace_back(utl::get_or_create_index(meta_key_cache_, m.key_));
+      tags_.emplace_back(utl::get_or_create_index(meta_value_cache_, m.value_));
     }
 
-    pb.add_packed_uint32(ttm::Feature::packed_uint32_tags, begin(t), end(t));
+    pb.add_packed_uint32(ttm::Feature::packed_uint32_tags, begin(tags_),
+                         end(tags_));
   }
 
   void aggregate_geometry() {
@@ -203,6 +204,10 @@ struct layer_builder {
   std::string buf_;
   pbf_builder<ttm::Layer> pb_;
 
+  // per-feature scratch, reused across features (cleared, capacity retained)
+  std::string feature_buf_;
+  std::vector<uint32_t> tags_;
+
   std::map<std::string, size_t> meta_key_cache_;
   std::map<std::string, size_t> meta_value_cache_;