Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,23 @@ Here's an overview of the different options:
It is recommended to set this value less than or equal to the queue database's connection pool size minus 2, as each worker thread uses one connection, and two additional connections are reserved for polling and heartbeat.
- `processes`: this is the number of worker processes that will be forked by the supervisor with the settings given. By default, this is `1`, just a single process. This setting is useful if you want to dedicate more than one CPU core to a queue or queues with the same configuration. Only workers have this setting. **Note**: this option will be ignored if [running in `async` mode](#fork-vs-async-mode).
- `concurrency_maintenance`: whether the dispatcher will perform the concurrency maintenance work. This is `true` by default, and it's useful if you don't use any [concurrency controls](#concurrency-controls) and want to disable it or if you run multiple dispatchers and want some of them to just dispatch jobs without doing anything else.
- `name`: an optional, human-readable name for the process. By default, processes are named with their kind and a random hex string, such as `worker-5a3b1c0d9e8f7a6b5c4d`, which makes it hard to tell them apart in `ps` output or in the processes table of the dashboard. Setting a name replaces that random name, and it's also reflected in the process title, for example `solid-queue-worker[pipeline-1](1.3.1): waiting for jobs in background`. When a worker also sets `processes` to a value greater than `1`, each of the forked processes gets a numbered suffix, so `name: pipeline` with `processes: 3` yields `pipeline-1`, `pipeline-2` and `pipeline-3`. With a single process, the name is used as-is. For example:

```yml
production:
dispatchers:
- polling_interval: 1
batch_size: 500
name: main-dispatcher
workers:
- queues: background
threads: 3
processes: 3
name: pipeline
- queues: default
threads: 1
name: housekeeping
```


### Optional scheduler configuration
Expand Down
12 changes: 11 additions & 1 deletion lib/solid_queue/configuration.rb
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,17 @@ def workers
1
end

processes.times.map { Process.new(:worker, worker_options.with_defaults(WORKER_DEFAULTS)) }
base_name = worker_options[:name]

processes.times.map do |index|
options = worker_options.with_defaults(WORKER_DEFAULTS)

if base_name
options = options.merge(name: processes > 1 ? "#{base_name}-#{index + 1}" : base_name)
end

Process.new(:worker, options)
end
end
end

Expand Down
9 changes: 7 additions & 2 deletions lib/solid_queue/processes/base.rb
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,16 @@ class Base

attr_reader :name

def initialize(*)
@name = generate_name
def initialize(name: nil, **)
@name = name || generate_name
@custom_name = name.present?
@stopped = false
end

def custom_name?
@custom_name
end

def kind
self.class.name.demodulize
end
Expand Down
5 changes: 4 additions & 1 deletion lib/solid_queue/processes/procline.rb
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,11 @@ module SolidQueue::Processes
module Procline
# Sets the procline ($0)
# solid-queue-supervisor(0.1.0): <string>
# solid-queue-worker[pipeline-1](0.1.0): <string>
def procline(string)
$0 = "solid-queue-#{self.class.name.demodulize.underscore.dasherize}(#{SolidQueue::VERSION}): #{string}"
process_kind = self.class.name.demodulize.underscore.dasherize
label = custom_name? ? "#{process_kind}[#{name}]" : process_kind
$0 = "solid-queue-#{label}(#{SolidQueue::VERSION}): #{string}"
end
end
end
2 changes: 1 addition & 1 deletion lib/solid_queue/supervisor.rb
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ def initialize(configuration)
@configured_processes = {}
@process_instances = {}

super
super()
end

def start
Expand Down
19 changes: 19 additions & 0 deletions test/dummy/config/named_workers_configuration.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
default: &default
workers:
- queues: background
threads: 3
processes: 3
name: pipeline
- queues: default
threads: 1
name: housekeeping
dispatchers:
- polling_interval: 1
batch_size: 500
name: main-dispatcher

development:
<<: *default

test:
<<: *default
52 changes: 52 additions & 0 deletions test/unit/configuration_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,58 @@ class ConfigurationTest < ActiveSupport::TestCase
end
end

test "named workers with multiple processes get sequential names" do
configuration = SolidQueue::Configuration.new(
workers: [ { queues: "background", threads: 3, processes: 3, name: "pipeline" } ],
skip_recurring: true
)

assert_processes configuration, :worker, 3,
queues: "background",
name: [ "pipeline-1", "pipeline-2", "pipeline-3" ]
end

test "named worker with single process gets name without suffix" do
configuration = SolidQueue::Configuration.new(
workers: [ { queues: "default", threads: 1, name: "housekeeping" } ],
skip_recurring: true
)

assert_processes configuration, :worker, 1, name: "housekeeping"
end

test "named dispatcher gets name passed through" do
configuration = SolidQueue::Configuration.new(
dispatchers: [ { batch_size: 500, name: "main-dispatcher" } ],
skip_recurring: true
)

assert_processes configuration, :dispatcher, 1, name: "main-dispatcher"
end

test "workers without name still get no name in config attributes" do
configuration = SolidQueue::Configuration.new(
workers: [ { queues: "background", processes: 2 } ],
skip_recurring: true
)

assert_processes configuration, :worker, 2
configuration.configured_processes.select { |p| p.kind == :worker }.each do |p|
assert_nil p.attributes[:name]
end
end

test "read named configuration from provided file" do
configuration = SolidQueue::Configuration.new(
config_file: config_file_path(:named_workers_configuration),
skip_recurring: true
)

assert_processes configuration, :worker, 4,
name: [ "pipeline-1", "pipeline-2", "pipeline-3", "housekeeping" ]
assert_processes configuration, :dispatcher, 1, name: "main-dispatcher"
end

test "validate configuration" do
# Valid and invalid recurring tasks
configuration = SolidQueue::Configuration.new(recurring_schedule_file: config_file_path(:recurring_with_invalid))
Expand Down
21 changes: 21 additions & 0 deletions test/unit/dispatcher_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,27 @@ class DispatcherTest < ActiveSupport::TestCase
assert_metadata process, { polling_interval: 0.1, batch_size: 10, concurrency_maintenance_interval: 600 }
end

test "dispatcher with custom name registers with that name" do
dispatcher = SolidQueue::Dispatcher.new(polling_interval: 0.1, batch_size: 10, name: "main-dispatcher")
assert_equal "main-dispatcher", dispatcher.name
assert dispatcher.custom_name?

dispatcher.start
wait_for_registered_processes(1, timeout: 1.second)

process = SolidQueue::Process.first
assert_equal "Dispatcher", process.kind
assert_equal "main-dispatcher", process.name
ensure
dispatcher&.stop
wait_for_registered_processes(0, timeout: 1.second)
end

test "dispatcher without custom name generates a random name" do
assert_match /\Adispatcher-[0-9a-f]{20}\z/, @dispatcher.name
assert_not @dispatcher.custom_name?
end

test "concurrency maintenance is optional" do
no_concurrency_maintenance_dispatcher = SolidQueue::Dispatcher.new(polling_interval: 0.1, batch_size: 10, concurrency_maintenance: false)
no_concurrency_maintenance_dispatcher.start
Expand Down
21 changes: 21 additions & 0 deletions test/unit/worker_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,27 @@ class WorkerTest < ActiveSupport::TestCase
assert_metadata process, { queues: "background", polling_interval: 0.2, thread_pool_size: 3 }
end

test "worker with custom name registers with that name" do
worker = SolidQueue::Worker.new(queues: "background", threads: 3, polling_interval: 0.2, name: "pipeline-1")
assert_equal "pipeline-1", worker.name
assert worker.custom_name?

worker.start
wait_for_registered_processes(1, timeout: 1.second)

process = SolidQueue::Process.first
assert_equal "Worker", process.kind
assert_equal "pipeline-1", process.name
ensure
worker&.stop
wait_for_registered_processes(0, timeout: 1.second)
end

test "worker without custom name generates a random name" do
assert_match /\Aworker-[0-9a-f]{20}\z/, @worker.name
assert_not @worker.custom_name?
end

test "errors on polling are passed to on_thread_error and re-raised" do
errors = Concurrent::Array.new

Expand Down
Loading