Ci edits

.github/workflows/main.yml

@@ -103,9 +103,14 @@ jobs:
       - name: Load Docker image
         run: docker load --input /tmp/optimizer-api.tar
       - name: Launch Stack
-        run: timeout 4m docker compose -f ./docker/docker-compose.test.yml up --wait
+        run: timeout 4m docker compose -f ./docker/docker-compose.test.yml up --wait api resque-default redis-resque
+      - name: Verify async stack
+        run: |
+          docker compose -f ./docker/docker-compose.test.yml ps
+          docker compose -f ./docker/docker-compose.test.yml exec -T redis-resque redis-cli ping
+          docker compose -f ./docker/docker-compose.test.yml exec -T resque-default pgrep -af resque:work
       - name: Starting tests
-        run: timeout ${TIMEOUT} docker compose -f ./docker/docker-compose.test.yml exec api rake test TESTOPTS="${TESTOPTS}" ${OPTIONS}
+        run: timeout ${TIMEOUT} docker compose -f ./docker/docker-compose.test.yml exec -T api rake test TESTOPTS="${TESTOPTS}" ${OPTIONS}
 
   test_dicho:
     needs: build

Dockerfile

@@ -3,22 +3,39 @@ FROM ghcr.io/cartoway/optimizer-ortools:master
 ARG BUNDLE_WITHOUT="test development"
 # Install Vroom
 ARG VROOM_RELEASE=v1.14.0
+ARG VROOM_GIT_URL=https://github.com/braktar/vroom.git
+ARG VROOM_BRANCH=waiting-time
 RUN apt update -y && \
+    echo "deb http://deb.debian.org/debian trixie main" > /etc/apt/sources.list.d/trixie.list && \
+    apt update -y && \
     apt install -y \
         git-core \
         build-essential \
         g++ \
+        gcc-13 \
+        g++-13 \
         libssl-dev \
         libasio-dev \
         libglpk-dev \
         pkg-config \
         netcat-traditional
-RUN git clone --recurse-submodules https://github.com/VROOM-Project/vroom.git && \
-    cd vroom/src && \
-    git fetch --tags && \
-    git checkout -q $VROOM_RELEASE && \
-    make -j$(nproc) && \
-    cp ../bin/vroom /usr/local/bin && \
+RUN git clone "$VROOM_GIT_URL" vroom && \
+    cd vroom && \
+    if [ -n "$VROOM_BRANCH" ]; then \
+      git fetch origin "$VROOM_BRANCH" && \
+      git checkout -q "$VROOM_BRANCH"; \
+    else \
+      git fetch --tags && \
+      git checkout -q "$VROOM_RELEASE"; \
+    fi && \
+    git submodule sync --recursive && \
+    git submodule update --init --recursive && \
+    cd src && \
+    CC=gcc-13 CXX=g++-13 \
+    make -j"$(nproc)" \
+      CXXFLAGS='-MMD -MP -I. -std=c++20 -Wextra -Wpedantic -Wall -O3 -DNDEBUG -DASIO_STANDALONE -DUSE_ROUTING=true' && \
+    cp ../bin/vroom /usr/local/bin/vroom && \
+    strip /usr/local/bin/vroom && \
     cd /
 
 RUN apt update -y && apt install -y \
@@ -48,11 +65,29 @@ WORKDIR /srv/app
 
 RUN apt update && \
     libgeos=$(apt-cache search 'libgeos-' | grep -P 'libgeos-\d.*' | awk '{print $1}') && \
-    apt install -y git libgeos-dev ${libgeos} libicu-dev libglpk-dev nano
+    apt install -y --no-install-recommends \
+        git \
+        libgeos-dev \
+        ${libgeos} \
+        libicu-dev \
+        libglpk-dev \
+        nano \
+        rustc \
+        cargo \
+        pkg-config \
+        libssl-dev \
+        libclang-dev \
+        clang \
+    && rm -rf /var/lib/apt/lists/*
+
+# balanced_vrp_clustering (bump): Rust native ext via rb_sys — https://github.com/cartoway/balanced_vrp_clustering/tree/bump
+# SOURCE_DATE_EPOCH: RubyGems sets it during native compile; helps rb_sys/cargo detection during bundle install.
+ENV SOURCE_DATE_EPOCH=1
 
 ADD ./Gemfile /srv/app/
 ADD ./Gemfile.lock /srv/app/
-RUN bundle install --full-index --without ${BUNDLE_WITHOUT}
+RUN bundle config set --local without "${BUNDLE_WITHOUT}" && \
+    bundle install --full-index
 
 ADD . /srv/app
 

Gemfile

@@ -43,7 +43,7 @@ gem 'rest-client'
 
 # AI
 gem 'ai4r'
-gem 'balanced_vrp_clustering', github: 'mapotempo/balanced_vrp_clustering', branch: 'dev'
+gem 'balanced_vrp_clustering', github: 'cartoway/balanced_vrp_clustering', branch: 'bump'
 
 # Geo
 gem 'polylines'

Gemfile.lock

@@ -1,3 +1,17 @@
+GIT
+  remote: https://github.com/cartoway/balanced_vrp_clustering.git
+  revision: b489d73bd64f205fd9c16f2b3953d8fb454a141e
+  branch: bump
+  specs:
+    balanced_vrp_clustering (0.3.0)
+      activesupport
+      ai4r
+      awesome_print
+      color-generator
+      geojson2image
+      rake
+      rb_sys (~> 0.9.128)
+
 GIT
   remote: https://github.com/cartoway/rubocop-policy.git
   revision: 209bde9a8088d4204b22829b2a9150860c16dcac
@@ -7,16 +21,6 @@ GIT
       rubocop-minitest (~> 0.31.0)
       rubocop-performance (~> 1.19.0)
 
-GIT
-  remote: https://github.com/mapotempo/balanced_vrp_clustering.git
-  revision: c537ae70519888438c965a7c3f966e38bd41582d
-  branch: dev
-  specs:
-    balanced_vrp_clustering (0.2.0)
-      awesome_print
-      color-generator
-      geojson2image
-
 GEM
   remote: https://rubygems.org/
   specs:
@@ -52,13 +56,13 @@ GEM
     base64 (0.1.1)
     benchmark (0.2.1)
     benchmark-ips (2.12.0)
-    bigdecimal (3.1.8)
+    bigdecimal (4.1.2)
     builder (3.2.4)
     byebug (11.1.3)
     charlock_holmes (0.7.9)
     chunky_png (1.4.0)
     color-generator (0.0.4)
-    concurrent-ruby (1.3.5)
+    concurrent-ruby (1.3.6)
     crack (0.4.5)
       rexml
     crass (1.0.6)
@@ -112,7 +116,7 @@ GEM
     http-cookie (1.0.5)
       domain_name (~> 0.5)
     http_accept_language (2.1.1)
-    i18n (1.14.7)
+    i18n (1.14.8)
       concurrent-ruby (~> 1.0)
     jaro_winkler (1.5.6)
     json (2.7.1)
@@ -127,7 +131,7 @@ GEM
     mime-types (3.5.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2023.0808)
-    minitest (5.26.0)
+    minitest (5.26.1)
     minitest-around (0.5.0)
       minitest (~> 5.0)
     minitest-bisect (1.7.0)
@@ -155,7 +159,10 @@ GEM
     nio4r (2.5.9)
     nokogiri (1.15.4-x86_64-linux)
       racc (~> 1.4)
-    oj (3.16.1)
+    oj (3.17.3)
+      bigdecimal (>= 3.0)
+      ostruct (>= 0.2)
+    ostruct (0.6.3)
     parallel (1.24.0)
     parser (3.3.0.5)
       ast (~> 2.4.1)
@@ -187,7 +194,10 @@ GEM
       loofah (~> 2.21)
       nokogiri (~> 1.14)
     rainbow (3.1.1)
-    rake (13.0.6)
+    rake (13.4.2)
+    rake-compiler-dock (1.12.0)
+    rb_sys (0.9.128)
+      rake-compiler-dock (= 1.12.0)
     redis (3.3.5)
     redis-activesupport (5.3.0)
       activesupport (>= 3, < 8)

config/environments/test.rb

@@ -45,7 +45,7 @@ module OptimizerWrapper
 
   PARAMS_LIMIT = { points: 150, vehicles: 10 }.freeze
   QUOTAS = [{ daily: 100000, monthly: 1000000, yearly: 10000000 }].freeze # Only taken into account if REDIS_COUNT
-  REDIS_COUNT = ENV['REDIS_COUNT_HOST'] && Redis.new(host: ENV['REDIS_COUNT_HOST']) || Redis.new
+  REDIS_COUNT = Redis.new(host: ENV['REDIS_COUNT_HOST'] || ENV['REDIS_RESQUE_HOST'] || 'localhost')
 
   DUMP_DIR = File.join(Dir.tmpdir, 'optimizer-api', 'test', 'dump')
   FileUtils.mkdir_p(DUMP_DIR) unless File.directory?(DUMP_DIR)

core/strategies/orchestration.rb

@@ -77,10 +77,18 @@ def define_process(service_vrp, job = nil, &block)
         shipment_size = vrp.relations.count{ |r| r.type == :shipment }
 
         # Repopulate Objects which are referenced by others using ids but deleted by the multiple sub problem creations
+        reinsert_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         vrp.units.each{ |unit| Models::Unit.insert(unit) } if Models::Unit.all.empty?
         vrp.points.each{ |point| Models::Point.insert(point) } if Models::Point.all.empty?
         vrp.reload_depots.each{ |reload_depot| Models::ReloadDepot.insert(reload_depot) } if Models::ReloadDepot.all.empty?
         vrp.services.each{ |s| Models::Service.insert(s) } if Models::Service.all.empty?
+        if service_vrp.dicho_data.is_a?(Hash) && service_vrp.dicho_data[:construction_timings]
+          Interpreters::DichoConstructionTimings.add!(
+            service_vrp.dicho_data,
+            :define_process_reinsert_ms,
+            (Process.clock_gettime(Process::CLOCK_MONOTONIC) - reinsert_start) * 1000
+          )
+        end
 
         log "--> define_process VRP (service: #{vrp.services.size} including #{shipment_size} shipment relations, "\
             "vehicle: #{vrp.vehicles.size}, v_limit: #{vrp.configuration.resolution.vehicle_limit}) "\
@@ -147,7 +155,14 @@ def solve(service_vrp, job = nil, block = nil)
             )
           else
             # TODO: Eliminate the points which has no feasible vehicle or service
-            vrp.compute_matrix(job, &block)
+            if service_vrp.dicho_data.is_a?(Hash)
+              Interpreters::DichoResolutionTimings.measure(service_vrp.dicho_data, :compute_matrix_ms) {
+                Interpreters::DichoResolutionTimings.increment!(service_vrp.dicho_data, :compute_matrix_calls)
+                vrp.compute_matrix(job, &block)
+              }
+            else
+              vrp.compute_matrix(job, &block)
+            end
 
             optim_wrapper_config.check_distances(vrp, unfeasible_services)
 
@@ -199,7 +214,18 @@ def solve(service_vrp, job = nil, block = nil)
                   end
 
                   # TODO: Move select best heuristic in each solver
-                  Interpreters::SeveralSolutions.custom_heuristics(service, vrp, block)
+                  Interpreters::SeveralSolutions.custom_heuristics(service, vrp, block, service_vrp: service_vrp)
+
+                  solve_options = {}
+                  if service == :ortools && service_vrp.dicho_data.is_a?(Hash)
+                    timings_target = Interpreters::OrtoolsTimings.dicho_data_target(service_vrp)
+                    if timings_target
+                      solve_options[:timings] = timings_target
+                      if timings_target[:level_timings] && !service_vrp.dicho_level.nil?
+                        solve_options[:timings_level] = service_vrp.dicho_level
+                      end
+                    end
+                  end
 
                   cliqued_solution =
                     optim_wrapper_config.solve(
@@ -211,7 +237,8 @@ def solve(service_vrp, job = nil, block = nil)
                         result_object = OptimizerWrapper::Result.get(job) || { pids: [] }
                         result_object[:pids] = pids
                         OptimizerWrapper::Result.set(job, result_object)
-                      }
+                      },
+                      **solve_options
                     ) { |wrapper, avancement, total, _message, cost, _time, solution|
                       solution =
                         if solution.is_a?(Models::Solution)

docker-compose.yml

@@ -13,7 +13,8 @@ x-default-service: &default-service
     ROUTER_API_KEY: ${ROUTER_API_KEY:-demo}
     SENTRY_DSN: ${SENTRY_DSN:-}
   depends_on:
-    - redis-resque
+    redis-resque:
+      condition: service_healthy
     # - redis-cache
   volumes:
     - .:/srv/app
@@ -34,10 +35,15 @@ services:
     environment:
       <<: *default-environment
       COUNT: 5
-      QUEUES: DEFAULT
+      QUEUE: DEFAULT
     command: bundle exec rake resque:workers --trace
     healthcheck:
-      disable: true
+      test: ["CMD-SHELL", "pgrep -f 'resque:work' >/dev/null || exit 1"]
+      start_interval: 1s
+      start_period: 60s
+      interval: 30s
+      timeout: 10s
+      retries: 5
     restart: unless-stopped
 
   redis-resque:

docker/docker-compose.test.yml

@@ -3,14 +3,17 @@ x-default-service: &default-service
   environment: &default-environment
     APP_ENV: test
     BUNDLE_WITH: "test development"
+    DOCKER: 'true'
     LOG_LEVEL: fatal
     OPTIM_DEFAULT_MAX_LATENESS_RATIO: 1
     OPTIM_GENERATE_GEOJSON_POLYLINES: 'true'
     # REDIS_CACHE_HOST: redis-cache
+    REDIS_COUNT_HOST: redis-resque
     REDIS_RESQUE_HOST: redis-resque
     REDIS_RESULT_TTL_DAYS: ${REDIS_RESULT_TTL_DAYS:-1}
   depends_on:
-    - redis-resque
+    redis-resque:
+      condition: service_healthy
     # - redis-cache
 
 services:
@@ -25,10 +28,15 @@ services:
     environment:
       <<: *default-environment
       COUNT: 5
-      QUEUES: DEFAULT
+      QUEUE: DEFAULT
     command: bundle exec rake resque:workers --trace
     healthcheck:
-      test: echo ok
+      test: ["CMD-SHELL", "pgrep -f 'resque:work' >/dev/null || exit 1"]
+      start_interval: 1s
+      start_period: 60s
+      interval: 10s
+      timeout: 5s
+      retries: 5
 
   redis-resque:
     image: redis:${REDIS_VERSION:-7-alpine}

lib/heuristics/concerns/periodic_end_phase.rb

@@ -147,6 +147,9 @@ def compute_first_best_insertion_data
   #### CORRECT POORLY POPULATED ROUTES PROCESS ####
 
   def correct_poorly_populated_routes
+    has_exclusion_costs = @services_data.any?{ |_id, data| data[:raw].exclusion_cost.to_f.positive? }
+    return unless @remove_poorly_populated_routes || has_exclusion_costs
+
     @output_tool&.add_comment('REMOVE_POORLY_POPULATED_ROUTES PHASE')
     @still_removed = {}