Merge pull request #30 from Lytro/resque

Add recipes for monitoring resque workers and a scheduler

recipes/resque.rb

@@ -0,0 +1,9 @@
+include_recipe "monit"
+
+node['monit']['resque']['queues'].each do |queue|
+  if queue['worker_count'] <= 0
+    raise ArgumentError, 'The number of workers for a given queue must be a positive integer.'
+  end
+end
+
+monitrc "resque"

recipes/resque_scheduler.rb

@@ -0,0 +1,3 @@
+include_recipe "monit"
+
+monitrc "resque_scheduler"

spec/unit/recipes/resque_scheduler_spec.rb

@@ -0,0 +1,18 @@
+require 'spec_helper'
+
+describe 'monit::resque_scheduler' do
+  let(:chef_run) do
+    ChefSpec::Runner.new do |node|
+      node.set[:monit][:resque][:app_root] = '/www/myapp-test/current'
+    end.converge(described_recipe)
+  end
+
+  it "includes the default recipe" do
+    expect(chef_run).to include_recipe 'monit::resque_scheduler'
+  end
+
+  it "creates the resque_scheduler.conf" do
+    expect(chef_run).to render_file("/etc/monit/conf.d/resque_scheduler.conf")
+                        .with_content("check process resque_scheduler with pidfile /www/myapp-test/current")
+  end
+end

spec/unit/recipes/resque_spec.rb

@@ -0,0 +1,53 @@
+require 'spec_helper'
+
+describe 'monit::resque' do
+  def chef_run(monit_resque_attrs = {})
+    @chef_run ||= ChefSpec::Runner.new do |node|
+      node.set[:monit][:resque] = monit_resque_attrs
+    end.converge described_recipe
+  end
+
+  it 'includes the default recipe' do
+    chef_run({ queues: [{ worker_count: 1, queue_list: %w(apple) }] })
+
+    expect(chef_run).to include_recipe 'monit::resque'
+  end
+
+  it 'creates the resque.conf' do
+    chef_run({ queues: [{ queue_list: %w(apple banana peanut), worker_count: 1 }]})
+
+    expect(chef_run).to render_file('/etc/monit/conf.d/resque.conf').with_content('check process resque_worker_0')
+    expect(chef_run).to render_file('/etc/monit/conf.d/resque.conf').with_content('QUEUE=apple,banana,peanut')
+  end
+
+  it 'groups queues appropriately' do
+    queues = [
+      { queue_list: %w(tomato), worker_count: 2 },
+      { queue_list: %w(apple banana carrot), worker_count: 3 },
+      { queue_list: %w(plum orange), worker_count: 1 }
+    ]
+    chef_run({ queues: queues })
+
+    expect(chef_run).to render_file('/etc/monit/conf.d/resque.conf').with_content('QUEUE=tomato')
+    expect(chef_run).to render_file('/etc/monit/conf.d/resque.conf').with_content('QUEUE=plum,orange')
+    expect(chef_run).to render_file('/etc/monit/conf.d/resque.conf').with_content('QUEUE=apple,banana,carrot')
+  end
+
+  it 'gives each worker a unique name' do
+    queues = [
+      { queue_list: %w(tomato), worker_count: 2 }
+    ]
+    chef_run({ queues: queues })
+
+    expect(chef_run).to render_file('/etc/monit/conf.d/resque.conf').with_content('check process resque_worker_0')
+    expect(chef_run).to render_file('/etc/monit/conf.d/resque.conf').with_content('check process resque_worker_1')
+  end
+
+  [0, -1].each do |worker_count|
+    it 'should guard against an invalid number of workers' do
+      lambda {
+        chef_run({ queues: [{ queue_list: %w(apple), worker_count: worker_count }] })
+      }.should raise_error ArgumentError, 'The number of workers for a given queue must be a positive integer.'
+    end
+  end
+end

templates/default/resque.conf.erb

@@ -0,0 +1,21 @@
+<% counter = 0 %>
+
+<% node['monit']['resque']['queues'].each do |queue| %>
+  <% queue['worker_count'].times do %>
+check process resque_worker_<%= counter %> with pidfile <%= @node['monit']['resque']['app_root'] %>/tmp/pids/resque_worker.<%= counter %>.pid
+
+  group resque_workers
+
+  # Note: the priority of queues is determined by their order in the following statement:
+
+  start program = "/usr/bin/env PATH=/usr/local/bin:/usr/local/ruby/bin:/usr/bin:/bin:$PATH /bin/sh -l -c 'cd <%= @node['monit']['resque']['app_root'] %> && nohup bundle exec rake environment resque'work' RAILS_ENV=<%= node.chef_environment %> QUEUE=<%=
+  queue['queue_list'].join(',')
+  %> VERBOSE=1 PIDFILE=<%= @node['monit']['resque']['app_root'] %>/tmp/pids/resque_worker.<%= counter %>.pid >> log/resque_worker.<%= counter %>.log 2>&1 &'" with timeout 90 seconds
+
+  stop program = "/bin/sh -c 'cd <%= @node['monit']['resque']['app_root'] %> && kill -TERM `cat tmp/pids/resque_worker.<%= counter %>.pid` && rm -f tmp/pids/resque_worker.<%= counter %>.pid; exit 0;'"
+
+  if totalmem is greater than 1500 MB for 10 cycles then restart
+
+  <% counter += 1 %>
+  <% end %>
+<% end %>

templates/default/resque_scheduler.conf.erb

@@ -0,0 +1,5 @@
+check process resque_scheduler with pidfile <%= @node[:monit][:resque][:app_root] %>/tmp/pids/resque_scheduler.pid
+  group resque_scheduler
+  start program = "/usr/bin/env PATH=/usr/local/bin:/usr/local/ruby/bin:/usr/bin:/bin:$PATH RACK_ENV=<%= node.chef_environment %> /bin/sh -l -c 'cd <%= @node[:monit][:resque][:app_root] %> && nohup bundle exec rake environment resque:scheduler RAILS_ENV=<%= node.chef_environment %> QUEUE=* VERBOSE=1 INITIALIZER_PATH=<%= @node[:monit][:resque][:app_root] %>/config/initializers/resque.rb PIDFILE=<%= @node[:monit][:resque][:app_root] %>/tmp/pids/resque_scheduler.pid & >> log/resque_scheduler.log 2>&1'" with timeout 90 seconds
+  stop program = "/bin/sh -c 'cd <%= @node[:monit][:resque][:app_root] %> && kill -9 `cat tmp/pids/resque_scheduler.pid` && rm -f tmp/pids/resque_scheduler.pid; exit 0;'"
+  if totalmem is greater than 300 MB for 10 cycles then restart