RAPTOR (#28)

* add MOTIS_GET_TIMING

* fix warning

* remove unused timing var

* initial raptor module

* some clang tidy fixes

* clang tidy fixes

* fix windows ci

* add destination meta station capabilites to raptor reconstructor

* fix domination for pretrip

* remove debug output

* add max travel duration

* fix ci

* remove debug info

* add contains util

* add print raptor route from station evas

* add source meta station capabilities

* add departure events for start meta stations

* fix source meta station for pretrip

* fix valid check for departure times in reconstructor

* remove unused

* adjust reconstructor for start metas

* fix earliest arrival update

* remove already completed todo

* initial graptor

* clean up

* add print_footpaths for debuggin

* remove unnecessary files

* fix reconstructor

* remove unused

* remove streams from devices

* add multiprocessor per query config

* add memory store wip

* remove utils

* add debug util

* clean up gpu raptor

* finish memory store

* refactor

* delete devices

* fix include

* fix warning

* remove global timetable on device

* fix switchup

* fix warnings

* make memory store safer

* switch to c++17

* rename cc to cuda_check to avoid name conflict

* fix ub

* fix ci

* fix enter/exit flags in journey

* fix journey reconstruction

* fix enter/exit flags in journey

* fix max queries per device for negative values

* fix timekeeping in cpu raptor

* raptor: update cmake file

* raptor: refactor

* raptor: remove for_each

* raptor: append_vector -> utl::concat

* update utl

* raptor: fix build

* raptor: get_raptor_schedule refactor

move intermediate data structures to cc file

* csa: rename gpucsa to gpu-csa

* raptor: refactor (remove unused code)

* refactor gpu raptor

* add additional starts to gpu raptor

* split gpu raptor into multiple files

* add ontrip flag to raptor query

* raptor: fix ub in departure range calculation

* initialize ontrip flag

* add gpu workflow

* gpu ci: run tests, save deps

* gpu: disable avx

* always move

* move code to .cc files

* tidy up includes

* remove unnecessary include

* add routing test, fix bug in raptor

* only compile if MOTIS_CUDA

* check journey equal

* refactor raptor module according to the pimpl idiom

* remove designated initializers

* fix format

* fix raptor module destructor

* fix raptor module destructor

* fix segfault

* remove unused includes

* raptor: remove backward code, simplify departure track computation

* add raptor print test

* tuple -> pair

* remove unused includes

* rename raptor_schedule to raptor_meta_info

* fix remaining renames

* routing itest -> --tripbased.use_data_file=false

* fix clang tidy errors

* Merge branch 'raptor' of github.com:motis-project/motis into raptor

* check formatting of cuda files

* refactor

* refactor

* reorganize includes

* always set footpath length to maximum of transfertimes or duration

* add minct to set transfer time to 1

Co-authored-by: Felix Gündling <[email protected]>

.github/workflows/gpu.yml

@@ -0,0 +1,56 @@
+name: Linux GPU Build
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+  release:
+    types:
+      - published
+
+jobs:
+  build:
+    runs-on: [self-hosted, linux, x64, gpu]
+    env:
+      DEBIAN_FRONTEND: noninteractive
+      BUILDCACHE_COMPRESS: true
+      BUILDCACHE_DIRECT_MODE: true
+      BUILDCACHE_ACCURACY: SLOPPY
+      UBSAN_OPTIONS: halt_on_error=1:abort_on_error=1
+      CUDACXX: /usr/local/cuda/bin/nvcc
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Get deps
+        run: mkdir -p ~/deps && mv ~/deps .
+
+      - name: CMake
+        run: |
+          cmake ${{ matrix.config.cross }} \
+            -G Ninja -S . -B build \
+            -DCMAKE_C_COMPILER=gcc-10 \
+            -DCMAKE_CXX_COMPILER=g++-10 \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DMOTIS_CUDA=On \
+            -DMOTIS_AVX=Off \
+            -DMOTIS_AVX2=Off \
+            -DMOTIS_WITH_WEBUI=${{ matrix.config.webui }}
+
+      - name: Build
+        run: |
+          ./build/buildcache/bin/buildcache -z
+          cmake --build build --target \
+            motis \
+            motis-test \
+            motis-itest
+          ./build/buildcache/bin/buildcache -s
+
+      - name: Save deps
+        run: mv deps ~
+
+      - name: Run Tests
+        run: ${{ matrix.config.emulator }} ./build/motis-test
+
+      - name: Run Integration Tests
+        run: ${{ matrix.config.emulator }} ./build/motis-itest

.github/workflows/unix.yml

@@ -2,7 +2,7 @@ name: Unix Build
 
 on:
   schedule:
-    - cron:  '0 3 * * 2,6'
+    - cron: '0 3 * * 2,6'
   push:
     branches: [ master ]
   pull_request:
@@ -28,7 +28,10 @@ jobs:
           sudo apt-get install -y --no-install-recommends clang-format-12
 
       - name: Format files
-        run: find base libs modules test -type f -a \( -name "*.cc" -o -name "*.h" \) -print0 | xargs -0 clang-format-12 -i
+        run: |
+          find base libs modules test \
+            -type f -a \( -name "*.cc" -o -name "*.h" -o -name ".cuh" -o -name ".cu" \) \
+            -print0 | xargs -0 clang-format-12 -i
 
       - name: Check for differences
         run: |
@@ -228,9 +231,9 @@ jobs:
           path: ${{ github.workspace }}/.buildcache
           key: buildcache-${{ matrix.config.name }}-${{ hashFiles('.pkg') }}-${{ hashFiles('**/*.h') }}-${{ hashFiles('**/*.cc') }}
           restore-keys: |
-               buildcache-${{ matrix.config.name }}-${{ hashFiles('.pkg') }}-${{ hashFiles('**/*.h') }}-
-               buildcache-${{ matrix.config.name }}-${{ hashFiles('.pkg') }}-
-               buildcache-${{ matrix.config.name }}-
+            buildcache-${{ matrix.config.name }}-${{ hashFiles('.pkg') }}-${{ hashFiles('**/*.h') }}-
+            buildcache-${{ matrix.config.name }}-${{ hashFiles('.pkg') }}-
+            buildcache-${{ matrix.config.name }}-
 
       - name: Dependencies Cache
         uses: actions/cache@v2

.pkg

@@ -77,7 +77,7 @@
 [utl]
   [email protected]:motis-project/utl.git
   branch=master
-  commit=c7be2bfe0b81702078b3aaf9c620c658d8d1f658
+  commit=a2375f6e2b20b6754335649857923e8b4a149ed9
 [guess]
   [email protected]:motis-project/guess.git
   branch=master

.pkg.lock

@@ -1,4 +1,4 @@
-14695981039346656037
+5276275784285909422
 cista eb1b0199eef401493db0d2dd735a14481b823083
 zlib 1e1dfdedddb54a2e2cb8fec3b67f925233c495aa
 boost bca212ca286121db1e7eb2efa9ee8de626633481
@@ -10,7 +10,7 @@ libosmium e35f4f63facbc87a0a5bf388bce19e6c4ed1dca7
 protozero 8c9f3fa97c2cfdceef86d0b61818ae98e9328f29
 Catch2 e5c9a58d66ff0780e956b5447573af9d6b9b2ca3
 fmt c68ab4be8f3cb0e5c6eb181b3f419622e15e02bd
-utl c7be2bfe0b81702078b3aaf9c620c658d8d1f658
+utl a2375f6e2b20b6754335649857923e8b4a149ed9
 address-typeahead 9b33a191c05ee3f489492ea9a89253eaa26b91d4
 conf 4c809244b10de534f4423ce4b27aa1a371610e96
 deboost.context e586729432e76f97eccb2cb75d8287cab47198f5

CMakeLists.txt

@@ -40,23 +40,20 @@ set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
 option(MOTIS_MIMALLOC "use mimalloc" OFF)
 option(MOTIS_AVX "enable AVX functions" ON)
 option(MOTIS_AVX2 "enable AVX2 + FMA functions" ON)
-
 option(MOTIS_CUDA "enable CUDA functions" OFF)
+option(MOTIS_LINT "enable lint (clang-tidy) target" OFF)
+option(MOTIS_COV "enable coverage (coverage) target" OFF)
+option(MOTIS_WITH_WEBUI "enable motis-web-ui target" OFF)
+
 if (MOTIS_CUDA)
-  set(MOTIS_CUDA_ARCH "-gencode arch=compute_75,code=sm_75 -gencode arch=compute_61,code=sm_61")
   if (MSVC)
-    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${MOTIS_CUDA_ARCH} -lcudadevrt")
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -lcudadevrt")
   else()
-    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${MOTIS_CUDA_ARCH} --compiler-options -static")
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --compiler-options -static")
   endif()
   enable_language(CUDA)
 endif()
 
-option(MOTIS_LINT "enable lint (clang-tidy) target" OFF)
-option(MOTIS_COV "enable coverage (coverage) target" OFF)
-
-option(MOTIS_WITH_WEBUI "enable motis-web-ui target" OFF)
-
 add_library(motis-generated INTERFACE)
 target_include_directories(motis-generated INTERFACE ${CMAKE_BINARY_DIR}/generated)
 
@@ -238,7 +235,6 @@ endif()
 target_compile_options(motis-test PRIVATE ${MOTIS_CXX_FLAGS})
 target_compile_definitions(motis-test PRIVATE ${MOTIS_COMPILE_DEFINITIONS})
 target_link_libraries(motis-test
-  ${module-targets}
   ${motis-test-extra-dependencies}
   motis-bootstrap
   motis-core

base/core/CMakeLists.txt

@@ -6,7 +6,7 @@ add_library(motis-core STATIC ${motis-core-files})
 target_compile_features(motis-core PUBLIC cxx_std_17)
 target_include_directories(motis-core PUBLIC include)
 target_compile_definitions(motis-core PUBLIC ${MOTIS_COMPILE_DEFINITIONS})
-target_compile_options(motis-core PUBLIC ${MOTIS_CXX_FLAGS})
+target_compile_options(motis-core PRIVATE ${MOTIS_CXX_FLAGS})
 target_link_libraries(motis-core
   utl
   date

base/core/include/motis/core/common/timing.h

@@ -3,8 +3,8 @@
 #include <chrono>
 
 #define MOTIS_START_TIMING(_X) \
-  auto _X##_start = std::chrono::steady_clock::now(), _X##_stop = _X##_start
-#define MOTIS_STOP_TIMING(_X) _X##_stop = std::chrono::steady_clock::now()
+  auto _X##_start = std::chrono::steady_clock::now()
+#define MOTIS_STOP_TIMING(_X) auto _X##_stop = std::chrono::steady_clock::now()
 #define MOTIS_TIMING_MS(_X)                                          \
   (std::chrono::duration_cast<std::chrono::milliseconds>(_X##_stop - \
                                                          _X##_start) \
@@ -13,3 +13,13 @@
   (std::chrono::duration_cast<std::chrono::microseconds>(_X##_stop - \
                                                          _X##_start) \
        .count())
+
+#define MOTIS_GET_TIMING_MS(_X)                           \
+  (std::chrono::duration_cast<std::chrono::milliseconds>( \
+       std::chrono::steady_clock::now() - _X##_start)     \
+       .count())
+
+#define MOTIS_GET_TIMING_US(_X)                           \
+  (std::chrono::duration_cast<std::chrono::microseconds>( \
+       std::chrono::steady_clock::now() - _X##_start)     \
+       .count())

base/core/include/motis/core/journey/journey.h

@@ -4,6 +4,8 @@
 #include <string>
 #include <vector>
 
+#include "cista/reflection/comparable.h"
+
 #include "motis/core/schedule/attribute.h"
 #include "motis/core/schedule/free_text.h"
 #include "motis/core/schedule/timestamp_reason.h"
@@ -12,6 +14,8 @@
 namespace motis {
 
 struct journey {
+  CISTA_COMPARABLE()
+
   enum class connection_status : uint8_t {
     OK,
     INTERCHANGE_INVALID,
@@ -26,6 +30,7 @@ struct journey {
   };
 
   struct transport {
+    CISTA_COMPARABLE()
     unsigned from_{0}, to_{0};
     bool is_walk_{false};
     std::string name_;
@@ -44,18 +49,20 @@ struct journey {
   };
 
   struct trip {
+    CISTA_COMPARABLE()
     unsigned from_{0}, to_{0};
     extern_trip extern_trip_;
     std::string debug_;
   };
 
   struct stop {
+    CISTA_COMPARABLE()
     bool exit_{false}, enter_{false};
     std::string name_;
     std::string eva_no_;
     double lat_{0}, lng_{0};
-
     struct event_info {
+      CISTA_COMPARABLE()
       bool valid_{false};
       unixtime timestamp_{0};
       unixtime schedule_timestamp_{0};
@@ -66,16 +73,19 @@ struct journey {
   };
 
   struct ranged_attribute {
+    CISTA_COMPARABLE()
     unsigned from_{0}, to_{0};
     attribute attr_;
   };
 
   struct ranged_free_text {
+    CISTA_COMPARABLE()
     unsigned from_{0}, to_{0};
     free_text text_;
   };
 
   struct problem {
+    CISTA_COMPARABLE()
     problem_type type_{problem_type::NO_PROBLEM};
     unsigned from_{0}, to_{0};
   };

base/core/include/motis/core/schedule/attribute.h

@@ -1,12 +1,15 @@
 #pragma once
 
+#include "cista/reflection/comparable.h"
+
 #include "motis/string.h"
 
 namespace motis {
 
 struct attribute {
-  mcd::string text_;
+  CISTA_COMPARABLE()
   mcd::string code_;
+  mcd::string text_;
 };
 
 }  // namespace motis

base/core/src/print_journey.cc

@@ -173,6 +173,15 @@ bool print_journey(journey const& j, std::ostream& out, bool local_time,
         << trp.line_id_ << "  " << j.trips_[i].debug_ << std::endl;
   }
 
+  out << "\nAttributes:" << std::endl;
+  for (auto i = 0UL; i < j.attributes_.size(); ++i) {
+    auto const& attribute = j.attributes_[i];
+    out << std::right << std::setw(2) << i << ": " << std::left << std::setw(2)
+        << attribute.from_ << " -> " << std::left << std::setw(2)
+        << attribute.to_ << " {" << attribute.attr_.code_ << "  "
+        << attribute.attr_.text_ << "}" << std::endl;
+  }
+
   auto const report_error = [&](bool first_error) -> std::ostream& {
     if (first_error) {
       out << "\nWARNING: Journey is broken:" << std::endl;

base/loader/src/build_footpaths.cc

@@ -58,11 +58,13 @@ struct footpath_builder {
         continue;
       }
 
-      auto duration = static_cast<int32_t>(footpath->duration());
       auto const from_node = get_station("from", footpath->from());
       auto const to_node = get_station("to", footpath->to());
       auto& from_station = sched_.stations_.at(from_node->id_);
       auto& to_station = sched_.stations_.at(to_node->id_);
+      auto duration =
+          std::max({from_station->transfer_time_, to_station->transfer_time_,
+                    static_cast<int32_t>(footpath->duration())});
 
       if (from_node == to_node) {
         LOG(ml::warn) << "Footpath loop at station " << from_station->eva_nr_
@@ -71,8 +73,6 @@ struct footpath_builder {
       }
 
       if (opt_.adjust_footpaths_) {
-        duration = std::max({from_station->transfer_time_,
-                             to_station->transfer_time_, duration});
         auto const distance = get_distance(*from_station, *to_station) * 1000;
 
         auto adjusted_duration = adjust_footpath_duration(duration, distance);

base/loader/src/build_stations.cc

@@ -65,7 +65,7 @@ struct stations_builder {
     s->length_ = fbs_station->lng();
     s->eva_nr_ = std::string{sched_.prefixes_[source_schedule]} +
                  fbs_station->id()->str();
-    s->transfer_time_ = std::max(2, fbs_station->interchange_time());
+    s->transfer_time_ = std::max(1, fbs_station->interchange_time());
     s->timez_ = fbs_station->timezone() != nullptr
                     ? get_or_create_timezone(fbs_station->timezone())
                     : nullptr;

modules/csa/CMakeLists.txt

@@ -1,8 +1,6 @@
 cmake_minimum_required(VERSION 3.10)
 project(motis)
 
-include_directories(include)
-
 file(GLOB_RECURSE motis-csa-files src/*.cc)
 add_library(motis-csa STATIC ${motis-csa-files})
 target_include_directories(motis-csa PUBLIC include)
@@ -12,20 +10,20 @@ target_link_libraries(motis-csa
   motis-routing
   motis-module
   motis-core
-)
+  )
 target_compile_options(motis-csa PRIVATE ${MOTIS_CXX_FLAGS})
 target_compile_definitions(motis-csa PRIVATE ${MOTIS_COMPILE_DEFINITIONS})
 
 if (MOTIS_CUDA)
   add_library(gpucsa SHARED src/gpu/gpu_csa.cu)
   set_target_properties(gpucsa PROPERTIES
-      WINDOWS_EXPORT_ALL_SYMBOLS ON
-      CUDA_SEPARABLE_COMPILATION ON
-      CUDA_STANDARD 14
-      RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}
-      LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}
-      INSTALL_RPATH "$ORIGIN/../lib:$ORIGIN/")
+    WINDOWS_EXPORT_ALL_SYMBOLS ON
+    CUDA_SEPARABLE_COMPILATION ON
+    CUDA_STANDARD 17
+    RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}
+    LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}
+    INSTALL_RPATH "$ORIGIN/../lib:$ORIGIN/")
   target_include_directories(gpucsa PUBLIC include)
   set_property(TARGET gpucsa PROPERTY CUDA_ARCHITECTURES 75 61)
   target_link_libraries(motis-csa gpucsa)
-endif()
+endif ()