Add request sparks

app/controllers/priv/analytics/sparks/requests_controller.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Priv::Analytics::Sparks
+  class RequestsController < Priv::Analytics::BaseController
+    use_clickhouse
+
+    CACHE_TTL      = 10.minutes
+    CACHE_RACE_TTL = 1.minute
+
+    typed_query {
+      param :date, type: :hash, optional: true, collapse: { format: :child_parent } do
+        param :start, type: :date, coerce: true
+        param :end, type: :date, coerce: true
+      end
+    }
+    def show
+      authorize! with: Accounts::AnalyticsPolicy
+
+      series = Analytics::Series.new(
+        :requests,
+        **request_query,
+      )
+
+      unless series.valid?
+        render_bad_request *series.errors.as_jsonapi(
+          title: 'Bad request',
+          source: :parameter,
+          sources: {
+            parameters: {
+              start_date: 'date[start]',
+              end_date: 'date[end]',
+            },
+          },
+        )
+
+        return
+      end
+
+      data = Rails.cache.fetch series.cache_key, expires_in: CACHE_TTL, race_condition_ttl: CACHE_RACE_TTL do
+        series.as_json
+      end
+
+      render json: { data: }
+    end
+  end
+end

app/controllers/priv/analytics/usage_controller.rb

@@ -17,7 +17,7 @@ def show
       authorize! with: Accounts::AnalyticsPolicy
 
       series = Analytics::Series.new(
-        :requests,
+        :usage,
         **usage_query,
       )
 

app/models/analytics/gauge.rb

@@ -13,6 +13,7 @@ class Gauge
       alus: ActiveLicensedUsers,
       licenses: Licenses,
       machines: Machines,
+      requests: Requests,
       users: Users,
       validations: Validations,
     }

app/models/analytics/gauge/requests.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Analytics
+  class Gauge
+    class Requests
+      METRICS = %w[requests.2xx requests.3xx requests.4xx requests.5xx].freeze
+
+      def initialize(account:, environment:)
+        @account     = account
+        @environment = environment
+      end
+
+      def metrics = METRICS
+      def count
+        rows = RequestLog::Clickhouse.for_account(account)
+                                     .for_environment(environment)
+                                     .where(created_date: Date.current)
+                                     .pluck(
+                                       Arel.sql(%{countIf(status IN ('200', '201', '202', '204')) AS "2xx"}),
+                                       Arel.sql(%{countIf(status IN ('301', '302', '303', '304', '307', '308')) AS "3xx"}),
+                                       Arel.sql(%{countIf(status IN ('400', '401', '402', '403', '404', '405', '406', '409', '410', '413', '422', '429')) AS "4xx"}),
+                                       Arel.sql(%{countIf(status IN ('500', '501', '502', '503', '504')) AS "5xx"}),
+                                     )
+
+        rows.each_with_object({}) do |counts, hash|
+          METRICS.zip counts do |metric, count|
+            hash[metric] = count
+          end
+        end
+      end
+
+      private
+
+      attr_reader :account,
+                  :environment
+    end
+  end
+end

app/models/analytics/series.rb

@@ -8,9 +8,10 @@ class Series
 
     COUNTERS = {
       events: Events,
-      requests: Requests,
+      usage: Requests,
       sparks: Sparks,
       validations: Sparks::Validations,
+      requests: Sparks::Requests,
     }
 
     include ActiveModel::Model

app/models/analytics/series/requests.rb

@@ -2,6 +2,7 @@
 
 module Analytics
   class Series
+    # TODO(ezekg) remove after migrating to requests spark
     class Requests
       METRICS = %w[requests.2xx requests.3xx requests.4xx requests.5xx].freeze
 

app/models/analytics/series/sparks/requests.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Analytics
+  class Series
+    class Sparks
+      class Requests
+        METRICS = %w[requests.2xx requests.3xx requests.4xx requests.5xx].freeze
+
+        def initialize(account:, environment:, realtime: true, **)
+          @account     = account
+          @environment = environment
+          @realtime    = realtime
+        end
+
+        def metrics = METRICS
+        def count(start_date:, end_date:)
+          rows = RequestSpark.for_account(account)
+                             .for_environment(environment)
+                             .where(created_date: start_date..end_date)
+                             .group(:created_date)
+                             .pluck(
+                               :created_date,
+                               Arel.sql(%{sumIf(count, status >= 200 AND status < 300) AS "2xx"}),
+                               Arel.sql(%{sumIf(count, status >= 300 AND status < 400) AS "3xx"}),
+                               Arel.sql(%{sumIf(count, status >= 400 AND status < 500) AS "4xx"}),
+                               Arel.sql(%{sumIf(count, status >= 500 AND status < 600) AS "5xx"}),
+                             )
+
+          counts = rows.each_with_object({}) do |(date, *counts), hash|
+            METRICS.zip counts do |metric, count|
+              hash[[metric, date]] = count
+            end
+          end
+
+          # defer to gauge for a realtime count since sparks are nightly
+          if realtime? && end_date.today?
+            gauge = Analytics::Gauge.new(:requests, account:, environment:)
+
+            gauge.measurements.each do |measurement|
+              counts[[measurement.metric, end_date]] = measurement.count
+            end
+          end
+
+          counts
+        end
+
+        private
+
+        attr_reader :account,
+                    :environment,
+                    :realtime
+
+        def realtime? = !!realtime
+      end
+    end
+  end
+end

app/models/request_spark.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class RequestSpark < ClickhouseRecord
+  include Accountable, Environmental
+
+  has_environment
+  has_account
+end

app/workers/record_request_spark_worker.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+class RecordRequestSparkWorker < BaseWorker
+  sidekiq_options queue: :cron,
+                  cronitor_enabled: true
+
+  def perform(account_id)
+    logs_cte = RequestLog::Clickhouse.where(account_id:, created_date: Date.yesterday)
+                                     .where(is_deleted: 0)
+                                     .where.not(status: nil)
+                                     .select(
+                                       :account_id,
+                                       :environment_id,
+                                       :created_date,
+                                       :created_at,
+                                       'toUInt16OrZero(status) AS status',
+                                     )
+
+    agg_cte = RequestLog::Clickhouse.from('request_logs')
+                                    .select(
+                                      :account_id,
+                                      :environment_id,
+                                      :created_date,
+                                      'max(created_at) AS created_at',
+                                      :status,
+                                      'count() AS count',
+                                    )
+                                    .group(
+                                      :account_id,
+                                      :environment_id,
+                                      :created_date,
+                                      :status,
+                                    )
+
+    RequestSpark.connection.execute(<<~SQL.squish)
+      WITH
+        request_logs    AS (#{logs_cte.to_sql}),
+        request_log_agg AS (#{agg_cte.to_sql})
+      INSERT INTO request_sparks (
+        account_id,
+        environment_id,
+        created_date,
+        created_at,
+        status,
+        count
+      )
+      SELECT
+        account_id,
+        environment_id,
+        created_date,
+        created_at,
+        status,
+        count
+      FROM
+        request_log_agg
+    SQL
+  end
+end

app/workers/record_request_sparks_worker.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class RecordRequestSparksWorker < BaseWorker
+  sidekiq_options queue: :cron,
+                  cronitor_enabled: true
+
+  def perform
+    Account.unordered.paid.find_each do |account|
+      jitter = rand(0..30.minutes) # prevent a thundering herd effect
+
+      RecordRequestSparkWorker.perform_in(jitter, account.id)
+    end
+  end
+end

config/routes.rb

@@ -479,6 +479,7 @@
         get 'gauges/validations',        to: 'gauges/validations#show', as: :validation_gauge # specialized gauge
         get 'gauges/:metric',            to: 'gauges#show',             as: :gauge
         get 'sparks/validations',        to: 'sparks/validations#show', as: :validation_spark # specialized spark
+        get 'sparks/requests',           to: 'sparks/requests#show',    as: :request_spark
         get 'sparks/:metric',            to: 'sparks#show',             as: :spark
         get 'usage',                     to: 'usage#show',              as: :usage
       end

config/schedule.yml

@@ -83,3 +83,8 @@ record_license_validation_sparks:
   cron: "0 0 * * *" # Every day at midnight (6pm CST)
   class: "RecordLicenseValidationSparksWorker"
   status: <%= ENV.true?('CLICKHOUSE_DATABASE_ENABLED') ? 'enabled' : 'disabled' %>
+
+record_request_sparks:
+  cron: "0 0 * * *" # Every day at midnight (6pm CST)
+  class: "RecordRequestSparksWorker"
+  status: <%= ENV.true?('CLICKHOUSE_DATABASE_ENABLED') ? 'enabled' : 'disabled' %>

db/clickhouse/20260316104304_create_request_sparks.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class CreateRequestSparks < ActiveRecord::Migration[8.1]
+  verbose!
+
+  def up
+    create_table :request_sparks, id: false,
+      options: "MergeTree PARTITION BY toYYYYMM(created_date) ORDER BY (account_id, created_date)",
+      force: :cascade do |t|
+      t.uuid :account_id, null: false
+      t.uuid :environment_id, null: true
+      t.column :status, "UInt16", null: false
+      t.column :count, "UInt64", null: false, default: 0
+      t.date :created_date, null: false
+      t.datetime :created_at, precision: 3, null: false
+
+      t.index :environment_id, name: "idx_environment", type: "bloom_filter", granularity: 4
+      t.index :status, name: "idx_status", type: "set(100)", granularity: 4
+    end
+  end
+
+  def down
+    drop_table :request_sparks
+  end
+end

db/clickhouse_schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[8.1].define(version: 2026_03_10_212409) do
+ActiveRecord::Schema[8.1].define(version: 2026_03_16_104304) do
   # TABLE: active_licensed_user_sparks
   # SQL: CREATE TABLE active_licensed_user_sparks ( `account_id` UUID, `environment_id` Nullable(UUID), `count` UInt64 DEFAULT 0, `created_date` Date, `created_at` DateTime64(3), INDEX idx_environment environment_id TYPE bloom_filter GRANULARITY 4 ) ENGINE = MergeTree PARTITION BY toYYYYMM(created_date) ORDER BY (account_id, created_date) SETTINGS index_granularity = 8192
   create_table "active_licensed_user_sparks", id: false, options: "MergeTree PARTITION BY toYYYYMM(created_date) ORDER BY (account_id, created_date) SETTINGS index_granularity = 8192", force: :cascade do |t|
@@ -198,6 +198,20 @@
     t.index "id", name: "idx_id", type: "bloom_filter", granularity: 4
   end
 
+  # TABLE: request_sparks
+  # SQL: CREATE TABLE request_sparks ( `account_id` UUID, `environment_id` Nullable(UUID), `status` UInt16, `count` UInt64 DEFAULT 0, `created_date` Date, `created_at` DateTime64(3), INDEX idx_environment environment_id TYPE bloom_filter GRANULARITY 4, INDEX idx_status status TYPE set(100) GRANULARITY 4 ) ENGINE = MergeTree PARTITION BY toYYYYMM(created_date) ORDER BY (account_id, created_date) SETTINGS index_granularity = 8192
+  create_table "request_sparks", id: false, options: "MergeTree PARTITION BY toYYYYMM(created_date) ORDER BY (account_id, created_date) SETTINGS index_granularity = 8192", force: :cascade do |t|
+    t.uuid "account_id", null: false
+    t.uuid "environment_id"
+    t.integer "status", limit: 2, null: false
+    t.integer "count", limit: 8, default: 0, null: false
+    t.date "created_date", null: false
+    t.datetime "created_at", precision: 3, null: false
+
+    t.index "environment_id", name: "idx_environment", type: "bloom_filter", granularity: 4
+    t.index "status", name: "idx_status", type: "set(100)", granularity: 4
+  end
+
   # TABLE: user_sparks
   # SQL: CREATE TABLE user_sparks ( `account_id` UUID, `environment_id` Nullable(UUID), `count` UInt64 DEFAULT 0, `created_date` Date, `created_at` DateTime64(3), INDEX idx_environment environment_id TYPE bloom_filter GRANULARITY 4 ) ENGINE = MergeTree PARTITION BY toYYYYMM(created_date) ORDER BY (account_id, created_date) SETTINGS index_granularity = 8192
   create_table "user_sparks", id: false, options: "MergeTree PARTITION BY toYYYYMM(created_date) ORDER BY (account_id, created_date) SETTINGS index_granularity = 8192", force: :cascade do |t|

features/priv/analytics/gauges.feature

@@ -221,6 +221,57 @@ Feature: Gauge analytics
     And sidekiq should have 0 "request-log" jobs
     And sidekiq should have 0 "event-log" jobs
 
+  @clickhouse
+  Scenario: Admin retrieves requests gauge for their account
+    Given I am an admin of account "test1"
+    And the current account is "test1"
+    And the current account has the following "request_log" rows:
+      | id                                   | status |
+      | d00998f9-d224-4ee7-ac4e-f1e5fe318ff7 | 200    |
+      | 96faacd6-16e6-4661-8e16-9e8064fbeb0a | 200    |
+      | 31e30cc1-d454-40dc-b4ae-93ad683ddf33 | 201    |
+      | d1e6f594-7bcb-455f-971b-1e8b3ea63fd7 | 301    |
+      | 99e87418-ade4-460f-a5aa-a856a0059397 | 404    |
+      | 19a9aefc-00b9-4905-b236-ff3cca788b3e | 404    |
+      | 09d7a1f9-3c4a-401f-b6a9-839f4e35d493 | 500    |
+    And I use an authentication token
+    When I send a GET request to "/accounts/test1/analytics/gauges/requests"
+    Then the response status should be "200"
+    And the response body should be a JSON document with the following content:
+      """
+      {
+        "data": [
+          { "metric": "requests.2xx", "count": 3 },
+          { "metric": "requests.3xx", "count": 1 },
+          { "metric": "requests.4xx", "count": 2 },
+          { "metric": "requests.5xx", "count": 1 }
+        ]
+      }
+      """
+    And sidekiq should have 0 "request-log" jobs
+    And sidekiq should have 0 "event-log" jobs
+
+  @clickhouse
+  Scenario: Admin retrieves requests gauge with no data
+    Given I am an admin of account "test1"
+    And the current account is "test1"
+    And I use an authentication token
+    When I send a GET request to "/accounts/test1/analytics/gauges/requests"
+    Then the response status should be "200"
+    And the response body should be a JSON document with the following content:
+      """
+      {
+        "data": [
+          { "metric": "requests.2xx", "count": 0 },
+          { "metric": "requests.3xx", "count": 0 },
+          { "metric": "requests.4xx", "count": 0 },
+          { "metric": "requests.5xx", "count": 0 }
+        ]
+      }
+      """
+    And sidekiq should have 0 "request-log" jobs
+    And sidekiq should have 0 "event-log" jobs
+
   Scenario: Admin retrieves invalid gauge
     Given I am an admin of account "test1"
     And the current account is "test1"