Build OSRM with 64-bit IDs so we can extract planet.osm.pbf foot profile

CHANGELOG.md

@@ -2,6 +2,7 @@
   - Changes from 5.27.1
     - Features
       - ADDED: Add support for a default_radius flag. [#6575](https://github.com/Project-OSRM/osrm-backend/pull/6575)
+      - ADDED: Allow building OSRM with 64-bit node and edge IDs[#1](https://github.com/wanderlog/osrm-backend/pull/1)
     - Build:
       - ADDED: Add CI job which builds OSRM with gcc 12. [#6455](https://github.com/Project-OSRM/osrm-backend/pull/6455)
       - CHANGED: Upgrade to clang-tidy 15. [#6439](https://github.com/Project-OSRM/osrm-backend/pull/6439)

CMakeLists.txt

@@ -35,6 +35,7 @@ option(ENABLE_LTO "Use LTO if available" OFF)
 option(ENABLE_FUZZING "Fuzz testing using LLVM's libFuzzer" OFF)
 option(ENABLE_NODE_BINDINGS "Build NodeJs bindings" OFF)
 option(ENABLE_CLANG_TIDY "Enables clang-tidy checks" OFF)
+option(USE_64BIT_IDS "Whether to use 64-bit node and edge IDs" OFF)
 
 if (ENABLE_CLANG_TIDY)
   find_program(CLANG_TIDY_COMMAND NAMES clang-tidy)
@@ -89,6 +90,13 @@ else()
   add_definitions(-DOSRM_PROJECT_DIR="${CMAKE_CURRENT_SOURCE_DIR}")
 endif()
 
+if(USE_64BIT_IDS)
+  add_definitions(-DUSE_64BIT_IDS)
+  message(STATUS "Using 64-bit node and edge IDs")
+else()
+  message(STATUS "Using 32-bit node and edge IDs")
+endif()
+
 # these two functions build up custom variables:
 #   DEPENDENCIES_INCLUDE_DIRS and OSRM_DEFINES
 # These variables we want to pass to
@@ -151,7 +159,7 @@ add_library(UPDATER OBJECT ${UpdaterGlob})
 add_library(STORAGE OBJECT ${StorageGlob})
 add_library(ENGINE OBJECT ${EngineGlob})
 
-if (BUILD_ROUTED) 
+if (BUILD_ROUTED)
   add_library(SERVER OBJECT ${ServerGlob})
   add_executable(osrm-routed src/tools/routed.cpp $<TARGET_OBJECTS:SERVER> $<TARGET_OBJECTS:UTIL>)
 endif()
@@ -309,7 +317,7 @@ add_subdirectory(${FLATBUFFERS_SRC_DIR}
 set(FMT_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/fmt-9.1.0/include")
 add_compile_definitions(FMT_HEADER_ONLY)
 include_directories(SYSTEM ${FMT_INCLUDE_DIR})
-        
+
 
 # see https://stackoverflow.com/questions/70898030/boost-link-error-using-conan-find-package
 if (MSVC)
@@ -319,7 +327,7 @@ endif()
 if(ENABLE_CONAN)
   message(STATUS "Installing dependencies via Conan")
 
-  # Conan will generate Find*.cmake files to build directory, so we use them with the highest priority 
+  # Conan will generate Find*.cmake files to build directory, so we use them with the highest priority
   list(INSERT CMAKE_MODULE_PATH 0 ${CMAKE_BINARY_DIR})
   list(INSERT CMAKE_PREFIX_PATH 0 ${CMAKE_BINARY_DIR})
 
@@ -330,26 +338,26 @@ if(ENABLE_CONAN)
   set(CONAN_EXPAT_VERSION "2.2.10#916908d4a570ad839edd25322c3268cd")
   set(CONAN_LUA_VERSION "5.4.4#3ec62efc37cd0a5d80b9e5cb35277360")
   set(CONAN_TBB_VERSION "2021.3.0#507ec17cbd51a84167e143b20d170eea")
-  
+
   set(CONAN_SYSTEM_INCLUDES ON)
 
-  # TODO: 
-  # if we link TBB dynamically osrm-extract.exe finishes on the first access to any TBB symbol 
+  # TODO:
+  # if we link TBB dynamically osrm-extract.exe finishes on the first access to any TBB symbol
   # with exit code = -1073741515, which means that program cannot load required DLL.
   if (MSVC)
     set(TBB_SHARED False)
   else()
     set(TBB_SHARED True)
   endif()
 
-  set(CONAN_ARGS       
-    REQUIRES 
+  set(CONAN_ARGS
+    REQUIRES
     "boost/${CONAN_BOOST_VERSION}"
     "bzip2/${CONAN_BZIP2_VERSION}"
     "expat/${CONAN_EXPAT_VERSION}"
     "lua/${CONAN_LUA_VERSION}"
     "onetbb/${CONAN_TBB_VERSION}"
-    BASIC_SETUP 
+    BASIC_SETUP
     GENERATORS cmake_find_package json  # json generator generates a conanbuildinfo.json in the build folder so (non-CMake) projects can easily parse OSRM's dependencies
     KEEP_RPATHS
     NO_OUTPUT_DIRS
@@ -386,7 +394,7 @@ if(ENABLE_CONAN)
   set(Boost_ZLIB_LIBRARY "${Boost_zlib_LIB_TARGETS}")
   set(Boost_REGEX_LIBRARY "${Boost_regex_LIB_TARGETS}")
   set(Boost_UNIT_TEST_FRAMEWORK_LIBRARY "${Boost_unit_test_framework_LIB_TARGETS}")
-  
+
 
   find_package(BZip2 REQUIRED)
   find_package(EXPAT REQUIRED)
@@ -730,23 +738,23 @@ if (ENABLE_FUZZING)
   add_subdirectory(fuzz)
 endif ()
 
-# add headers sanity check target that includes all headers independently	
-set(check_headers_dir "${PROJECT_BINARY_DIR}/check-headers")	
-file(GLOB_RECURSE headers_to_check	
-  ${PROJECT_BINARY_DIR}/*.hpp	
-  ${PROJECT_SOURCE_DIR}/include/*.hpp)	
-foreach(header ${headers_to_check})	
+# add headers sanity check target that includes all headers independently
+set(check_headers_dir "${PROJECT_BINARY_DIR}/check-headers")
+file(GLOB_RECURSE headers_to_check
+  ${PROJECT_BINARY_DIR}/*.hpp
+  ${PROJECT_SOURCE_DIR}/include/*.hpp)
+foreach(header ${headers_to_check})
   if ("${header}" MATCHES ".*/include/nodejs/.*")
     # we do not check NodeJS bindings headers
     continue()
   endif()
-  get_filename_component(filename ${header} NAME_WE)	
-  set(filename "${check_headers_dir}/${filename}.cpp")	
-  if (NOT EXISTS ${filename})	
-    file(WRITE ${filename} "#include \"${header}\"\n")	
-  endif()	
-  list(APPEND sources ${filename})	
-endforeach()	
-add_library(check-headers STATIC EXCLUDE_FROM_ALL ${sources})	
-set_target_properties(check-headers PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${check_headers_dir})	
+  get_filename_component(filename ${header} NAME_WE)
+  set(filename "${check_headers_dir}/${filename}.cpp")
+  if (NOT EXISTS ${filename})
+    file(WRITE ${filename} "#include \"${header}\"\n")
+  endif()
+  list(APPEND sources ${filename})
+endforeach()
+add_library(check-headers STATIC EXCLUDE_FROM_ALL ${sources})
+set_target_properties(check-headers PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${check_headers_dir})
 

include/contractor/graph_contractor_adaptors.hpp

@@ -145,7 +145,7 @@ template <class Edge, typename GraphT> inline std::vector<Edge> toEdges(GraphT g
         util::Percent p(log, graph.GetNumberOfNodes());
         const NodeID number_of_nodes = graph.GetNumberOfNodes();
         std::size_t edge_index = 0;
-        for (const auto node : util::irange(0u, number_of_nodes))
+        for (const auto node : util::irange(MIN_NODEID, number_of_nodes))
         {
             p.PrintStatus(node);
             for (auto edge : graph.GetAdjacentEdgeRange(node))

include/contractor/query_edge.hpp

@@ -45,7 +45,11 @@ struct QueryEdge
         // this ID is either the middle node of the shortcut, or the ID of the edge based node (node
         // based edge) storing the appropriate data. If `shortcut` is set to true, we get the middle
         // node. Otherwise we see the edge based node to access node data.
+        #ifdef USE_64BIT_IDS
+        NodeID turn_id : 63;
+        #else
         NodeID turn_id : 31;
+        #endif
         bool shortcut : 1;
         EdgeWeight weight;
         EdgeDuration::value_type duration : 30;

include/engine/datafacade/algorithm_datafacade.hpp

@@ -30,9 +30,9 @@ template <> class AlgorithmDataFacade<CH>
     virtual ~AlgorithmDataFacade() = default;
 
     // search graph access
-    virtual unsigned GetNumberOfNodes() const = 0;
+    virtual NodeID GetNumberOfNodes() const = 0;
 
-    virtual unsigned GetNumberOfEdges() const = 0;
+    virtual EdgeID GetNumberOfEdges() const = 0;
 
     virtual unsigned GetOutDegree(const NodeID edge_based_node_id) const = 0;
 
@@ -67,11 +67,11 @@ template <> class AlgorithmDataFacade<MLD>
     virtual ~AlgorithmDataFacade() = default;
 
     // search graph access
-    virtual unsigned GetNumberOfNodes() const = 0;
+    virtual NodeID GetNumberOfNodes() const = 0;
 
-    virtual unsigned GetMaxBorderNodeID() const = 0;
+    virtual NodeID GetMaxBorderNodeID() const = 0;
 
-    virtual unsigned GetNumberOfEdges() const = 0;
+    virtual EdgeID GetNumberOfEdges() const = 0;
 
     virtual unsigned GetOutDegree(const NodeID edge_based_node_id) const = 0;
 

include/engine/datafacade/contiguous_internalmem_datafacade.hpp

@@ -65,9 +65,9 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
     }
 
     // search graph access
-    unsigned GetNumberOfNodes() const override final { return m_query_graph.GetNumberOfNodes(); }
+    NodeID GetNumberOfNodes() const override final { return m_query_graph.GetNumberOfNodes(); }
 
-    unsigned GetNumberOfEdges() const override final { return m_query_graph.GetNumberOfEdges(); }
+    EdgeID GetNumberOfEdges() const override final { return m_query_graph.GetNumberOfEdges(); }
 
     unsigned GetOutDegree(const NodeID edge_based_node_id) const override final
     {
@@ -609,11 +609,11 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
     const customizer::CellMetricView &GetCellMetric() const override { return mld_cell_metric; }
 
     // search graph access
-    unsigned GetNumberOfNodes() const override final { return query_graph.GetNumberOfNodes(); }
+    NodeID GetNumberOfNodes() const override final { return query_graph.GetNumberOfNodes(); }
 
-    unsigned GetMaxBorderNodeID() const override final { return query_graph.GetMaxBorderNodeID(); }
+    NodeID GetMaxBorderNodeID() const override final { return query_graph.GetMaxBorderNodeID(); }
 
-    unsigned GetNumberOfEdges() const override final { return query_graph.GetNumberOfEdges(); }
+    EdgeID GetNumberOfEdges() const override final { return query_graph.GetNumberOfEdges(); }
 
     unsigned GetOutDegree(const NodeID edge_based_node_id) const override final
     {

include/engine/guidance/assemble_leg.hpp

@@ -32,7 +32,7 @@ struct NamedSegment
 {
     EdgeDuration duration;
     std::uint32_t position;
-    std::uint32_t name_id;
+    NameID name_id;
 };
 
 template <std::size_t SegmentNumber>

include/engine/guidance/route_step.hpp

@@ -56,7 +56,7 @@ inline IntermediateIntersection getInvalidIntersection()
 struct RouteStep
 {
     NodeID from_id;
-    unsigned name_id;
+    NameID name_id;
     bool is_segregated;
     std::string name;
     std::string ref;

include/engine/hint.hpp

@@ -76,11 +76,16 @@ struct Hint
     static Hint FromBase64(const std::string &base64Hint);
 };
 
+#ifdef USE_64BIT_IDS
+static_assert(sizeof(SegmentHint) == 96 + 8, "Hint is bigger than expected");
+// 140 == ceil(104 / 3) * 4 since we use integer division
+constexpr std::size_t ENCODED_SEGMENT_HINT_SIZE = 140;
+#else
 static_assert(sizeof(SegmentHint) == 80 + 4, "Hint is bigger than expected");
 constexpr std::size_t ENCODED_SEGMENT_HINT_SIZE = 112;
+#endif
 static_assert(ENCODED_SEGMENT_HINT_SIZE / 4 * 3 >= sizeof(SegmentHint),
               "ENCODED_SEGMENT_HINT_SIZE does not match size of SegmentHint");
-
 } // namespace osrm::engine
 
 #endif

include/engine/phantom_node.hpp

@@ -222,7 +222,11 @@ struct PhantomNode
     unsigned short bearing : 12;
 };
 
+#ifdef USE_64BIT_IDS
+static_assert(sizeof(PhantomNode) == 96, "PhantomNode has more padding then expected");
+#else
 static_assert(sizeof(PhantomNode) == 80, "PhantomNode has more padding then expected");
+#endif
 
 using PhantomNodeCandidates = std::vector<PhantomNode>;
 using PhantomCandidateAlternatives = std::pair<PhantomNodeCandidates, PhantomNodeCandidates>;

include/extractor/compressed_edge_container.hpp

@@ -46,7 +46,7 @@ class CompressedEdgeContainer
                              const EdgeDuration duration);
 
     void InitializeBothwayVector();
-    unsigned ZipEdges(const unsigned f_edge_pos, const unsigned r_edge_pos);
+    unsigned ZipEdges(const EdgeID f_edge_pos, const EdgeID r_edge_pos);
 
     bool HasEntryForID(const EdgeID edge_id) const;
     bool HasZippedEntryForForwardID(const EdgeID edge_id) const;

include/extractor/edge_based_edge.hpp

@@ -57,10 +57,18 @@ struct EdgeBasedEdge
     NodeID target;
     EdgeData data;
 };
-static_assert(sizeof(extractor::EdgeBasedEdge) == 24,
-              "Size of extractor::EdgeBasedEdge type is "
-              "bigger than expected. This will influence "
-              "memory consumption.");
+
+#ifdef USE_64BIT_IDS
+    static_assert(sizeof(extractor::EdgeBasedEdge) == 40,
+                  "Size of extractor::EdgeBasedEdge type is "
+                  "bigger than expected. This will influence "
+                  "memory consumption.");
+#else
+    static_assert(sizeof(extractor::EdgeBasedEdge) == 24,
+                  "Size of extractor::EdgeBasedEdge type is "
+                  "bigger than expected. This will influence "
+                  "memory consumption.");
+#endif
 
 // Impl.
 

include/extractor/edge_based_graph_factory.hpp

@@ -52,7 +52,11 @@ struct TurnIndexBlock
 };
 #pragma pack(pop)
 static_assert(std::is_trivial<TurnIndexBlock>::value, "TurnIndexBlock is not trivial");
-static_assert(sizeof(TurnIndexBlock) == 12, "TurnIndexBlock is not packed correctly");
+#ifdef USE_64BIT_IDS
+    static_assert(sizeof(TurnIndexBlock) == 24, "TurnIndexBlock is not packed correctly");
+#else
+    static_assert(sizeof(TurnIndexBlock) == 12, "TurnIndexBlock is not packed correctly");
+#endif
 } // namespace lookup
 
 struct NodeBasedGraphToEdgeBasedGraphMappingWriter; // fwd. decl

include/extractor/edge_based_node.hpp

@@ -10,7 +10,11 @@ struct EdgeBasedNode
 {
     GeometryID geometry_id;
     ComponentID component_id;
-    std::uint32_t annotation_id : 31;
+    #ifdef USE_64BIT_IDS
+    AnnotationID annotation_id : 63;
+    #else
+    AnnotationID annotation_id : 31;
+    #endif
     std::uint32_t segregated : 1;
 };
 

include/extractor/extractor_callbacks.hpp

@@ -61,7 +61,7 @@ class ExtractorCallbacks
     // used to deduplicate street names, refs, destinations, pronunciation, exits:
     // actually maps to name ids
     using MapKey = std::tuple<std::string, std::string, std::string, std::string, std::string>;
-    using MapVal = unsigned;
+    using MapVal = NameID;
     using StringMap = std::unordered_map<MapKey, MapVal>;
     StringMap string_map;
     ExtractionContainers &external_memory;

include/extractor/graph_compressor.hpp

@@ -33,8 +33,8 @@ class GraphCompressor
                   CompressedEdgeContainer &geometry_compressor);
 
   private:
-    void PrintStatistics(unsigned original_number_of_nodes,
-                         unsigned original_number_of_edges,
+    void PrintStatistics(NodeID original_number_of_nodes,
+                         EdgeID original_number_of_edges,
                          const util::NodeBasedDynamicGraph &graph) const;
 };
 } // namespace osrm::extractor

include/extractor/node_based_edge.hpp

@@ -102,13 +102,13 @@ struct NodeBasedEdge
 
     bool operator<(const NodeBasedEdge &other) const;
 
-    NodeID source;                     // 32 4
-    NodeID target;                     // 32 4
+    NodeID source;                     // 64 8
+    NodeID target;                     // 64 8
     EdgeWeight weight;                 // 32 4
     EdgeDuration duration;             // 32 4
     EdgeDistance distance;             // 32 4
-    GeometryID geometry_id;            // 32 4
-    AnnotationID annotation_data;      // 32 4
+    GeometryID geometry_id;            // 64 8
+    AnnotationID annotation_data;      // 64 8
     NodeBasedEdgeClassification flags; // 32 4
 };
 
@@ -200,10 +200,17 @@ inline NodeBasedEdgeWithOSM::NodeBasedEdgeWithOSM()
 {
 }
 
+#ifdef USE_64BIT_IDS
+static_assert(sizeof(extractor::NodeBasedEdge) == 56,
+              "Size of extractor::NodeBasedEdge type is "
+              "bigger than expected. This will influence "
+              "memory consumption.");
+#else
 static_assert(sizeof(extractor::NodeBasedEdge) == 32,
               "Size of extractor::NodeBasedEdge type is "
               "bigger than expected. This will influence "
               "memory consumption.");
+#endif
 
 } // namespace osrm::extractor
 

include/extractor/node_data_container.hpp

@@ -98,7 +98,7 @@ template <storage::Ownership Ownership> class EdgeBasedNodeDataContainerImpl
                                     const EdgeBasedNodeDataContainerImpl &ebn_data_container);
 
     template <typename = std::enable_if<Ownership == storage::Ownership::Container>>
-    void Renumber(const std::vector<std::uint32_t> &permutation)
+    void Renumber(const std::vector<NodeID> &permutation)
     {
         util::inplacePermutation(nodes.begin(), nodes.end(), permutation);
     }

include/partitioner/cell_statistics.hpp

@@ -19,7 +19,7 @@ void printCellStatistics(const Partition &partition, const CellStorage &storage)
         std::size_t source = 0, destination = 0;
         std::size_t boundary_nodes = 0;
         std::size_t entries = 0;
-        for (std::uint32_t cell_id = 0; cell_id < num_cells; ++cell_id)
+        for (CellID cell_id = 0; cell_id < num_cells; ++cell_id)
         {
             std::unordered_set<NodeID> boundary;
             const auto &cell = storage.GetUnfilledCell(level, cell_id);