I, [2008-01-29T12:56:55.580572 #70]  INFO -- : backgroundrb move
'unmonitored' to 'init'
I, [2008-01-29T12:56:56.122592 #70]  INFO -- : backgroundrb [trigger]
backgroundrb God::Conditions::ProcessRunning: no such pid file: /
Volumes/OtherInboxDev/otherinbox/shared/log/backgroundrb.pid
(ProcessRunning)
I, [2008-01-29T12:56:56.191987 #70]  INFO -- : backgroundrb move
'init' to 'start'
I, [2008-01-29T12:56:56.196883 #70]  INFO -- : backgroundrb
before_start: no pid file to delete (CleanPidFile)
I, [2008-01-29T12:56:56.202412 #70]  INFO -- : backgroundrb start: /
Volumes/OtherInboxDev/otherinbox/current/script/backgroundrb start
I, [2008-01-29T12:57:13.414195 #70]  INFO -- : backgroundrb [ok]
backgroundrb God::Conditions::ProcessRunning: no such pid file: /
Volumes/OtherInboxDev/otherinbox/shared/log/backgroundrb.pid
(ProcessRunning)
I, [2008-01-29T12:57:13.665885 #70]  INFO -- : backgroundrb [ok] tries
within bounds [1/5] (Tries)
I, [2008-01-29T12:57:43.172296 #70]  INFO -- : backgroundrb [ok]
backgroundrb God::Conditions::ProcessRunning: no such pid file: /
Volumes/OtherInboxDev/otherinbox/shared/log/backgroundrb.pid
(ProcessRunning)
I, [2008-01-29T12:57:43.424154 #70]  INFO -- : backgroundrb [ok] tries
within bounds [2/5] (Tries)
I, [2008-01-29T12:58:13.205674 #70]  INFO -- : backgroundrb [ok]
backgroundrb God::Conditions::ProcessRunning: no such pid file: /
Volumes/OtherInboxDev/otherinbox/shared/log/backgroundrb.pid
(ProcessRunning)
I, [2008-01-29T12:58:13.457650 #70]  INFO -- : backgroundrb [ok] tries
within bounds [3/5] (Tries)
I, [2008-01-29T12:58:43.099112 #70]  INFO -- : backgroundrb [ok]
backgroundrb God::Conditions::ProcessRunning: no such pid file: /
Volumes/OtherInboxDev/otherinbox/shared/log/backgroundrb.pid
(ProcessRunning)
I, [2008-01-29T12:58:43.351051 #70]  INFO -- : backgroundrb [ok] tries
within bounds [4/5] (Tries)
I, [2008-01-29T12:59:13.135258 #70]  INFO -- : backgroundrb [ok]
backgroundrb God::Conditions::ProcessRunning: no such pid file: /
Volumes/OtherInboxDev/otherinbox/shared/log/backgroundrb.pid
(ProcessRunning)
I, [2008-01-29T12:59:13.386950 #70]  INFO -- : backgroundrb [trigger]
tries exceeded [5/5] (Tries)

The relevant watch is:

God.watch do |w|
  w.name = "backgroundrb"
  w.group = "support-services"

  w.interval = 30.seconds
  w.grace = 15.seconds

  w.start = "#{RAILS_ROOT}/script/backgroundrb start"
  w.stop = "#{RAILS_ROOT}/script/backgroundrb stop"

  w.pid_file = "#{APP_ROOT}/shared/log/backgroundrb.pid"

  # clean pid files before start if necessary
  w.behavior(:clean_pid_file)

  # determine the state on startup
  w.transition(:init, { true => :up, false => :start }) do |on|
    on.condition(:process_running) do |c|
      c.running = true
    end
  end

  # determine when process has finished starting
  w.transition([:start, :restart], :up) do |on|
    on.condition(:process_running) do |c|
      c.running = true
    end

    # failsafe
    on.condition(:tries) do |c|
      c.times = 5
      c.transition = :start
    end
  end

  # start if process is not running
  w.transition(:up, :start) do |on|
    on.condition(:process_exits) { |c| c.notify = 'developers' }
  end

  w.lifecycle do |on|
    on.condition(:flapping) do |c|
      c.to_state = [:start, :restart]
      c.times = 5
      c.within = 5.minute
      c.transition = :unmonitored
      c.retry_in = 10.minutes
      c.retry_times = 5
      c.retry_within = 2.hours
      c.notify = 'developers'
    end
  end

end

I'm using rev 305 of backgroundrb on edge rails, and I've verified
that the above listed location for the PID file is correct.

Thanks for any help you might be able to provide.  I love this
software!

Also, for testing purposes, it works very well on Tiger -- I even have
launchd starting it.  It also works on Leopard, but I couldn't get
launchd to play nice, so on my Leopard system I start god from /etc/
rc.local

-Mike

James

On Jan 29, 2:35 pm, Mike Subelsky <subel...@gmail.com> wrote:

Here is a watch and script I use inside a rails app.
It is using the 'daemons' gem and works wonders :)

Cheers,
Tim

## START WATCH ##
God.watch do |w|
   base_path = "/home/deploy/foobar"
   w.name = "foobar-daemon"
   w.uid = "deploy"
   w.gid = "nogroup"
   w.pid_file = "#{base_path}/shared/log/foobar.pid"
   w.interval = 30.seconds
   w.start   = "#{base_path}/current/script/daemon start    --  
production"
   w.stop    = "#{base_path}/current/script/daemon stop     --  
production"
   w.restart = "#{base_path}/current/script/daemon restart  --  
production"
   w.start_grace = 10.seconds
   w.restart_grace = 10.seconds

   w.behavior(:clean_pid_file)

   w.start_if do |start|
     start.condition(:process_running) do |c|
       c.interval = 5.seconds
       c.running = false
     end
   end
end
## END WATCH ##

## START DAEMON ##
#!/usr/bin/env ruby
require 'rubygems'
require 'daemons'

code = File.dirname(__FILE__) + "/../lib/scripts/daemon.rb"
options = {
   :app_name   => "foobar-daemon",
   :ARGV       => ARGV,
   :dir_mode   => :normal,
   :dir        => File.dirname(__FILE__) + '/../log',
   :multiple   => false,
   :backtrace  => true,
   :monitor    => false

Now isn't that clever!  Thanks Tim!

James

One of the biggest things was that my config wasn't giving it enough
time to try to start up, and would start trying again before it was
started.  It would work only intermitently.

So set a longer start_grace(and restart to be safe).  I changed mine
to 20 seconds alla:

w.start_grace = 15.seconds
w.restart_grace = 15.seconds

Also, borrowing from my capistrano recipes, I use a slightly different
set of start/stop/restart commands (which avoid an error when trying
to stop when there is no pid and uses nohup, as it dies when
capistrano exits otherwise).  Starting the grace periods may be enough
for you, but if not you might give these a shot.
  w.start = "nohup #{RAILS_ROOT}/script/backgroundrb start -d"
  w.stop = "sh -c 'if [ -a #{RAILS_ROOT}/log/backgroundrb.pid ]; then
#{RAILS_ROOT}/script/backgroundrb stop; fi;'"
  w.restart = "sh -c 'if [ -a #{RAILS_ROOT}/log/backgroundrb.pid ];
then #{RAILS_ROOT}/script/backgroundrb stop; fi;' && nohup
#{RAILS_ROOT}/script/backgroundrb start -d"

Let us know what you come up with.

On Jan 29, 1:35 pm, Mike Subelsky <subel...@gmail.com> wrote:

So we added this to our god.conf file:

  God::Watch.class_eval do
    attr_accessor :unix_socket
  end

  module God
    module Behaviors
      class CleanUnixSocket < Behavior
        def valid?
          valid = true
          if self.watch.unix_socket.nil?
            valid &= complain("Attribute 'unix_socket' must be
specified", self)
          end
          valid
        end

        def before_start
          File.delete(self.watch.unix_socket)
          "deleted unix socket"
        rescue
          "no unix socket to delete"
        end
      end
    end
  end

and then used it like this (inside a God.watch block):

  w.unix_socket = "/tmp/backgroundrbunix_localhost_#{port}"

Now god cleans up stale socket files when starting backgroundrb.

Graham

  w.behaviour(:clean_unix_socket)

Graham

-Mike

Mike Subelsky
Hacker & Improvisor // subelsky.com

On Mar 14, 2008, at 6:47 PM, Mike Subelsky wrote:

I tried setting up the unix socket cleaning stuff, and though the
cleaning appears to work I'm still not getting backgroundrb to play
nice.

When it does crash, it ends up restarting but doesn't appear to
actually be running tasks.  If I check the log file it will show
evidence of having crashed rather than showing something that would
represent a currently running server.  If I do 'ps aux' all of the
processes do appear to be there though.  I'm not really sure where
this discrepancy cropped up from, but any assistance you could provide
would be great (perhaps you could provide more/all of your working
watch block for backgroundrb?).

thanks,
wes

Have you tried running backgroundrb manually, as the same user that
god starts it as, using exactly the same command line as god uses? You
can get the command line from the output of "ps aux | grep
backgroundrb". Maybe it'll still crash...

Ours is all setup inside one function, so you can certainly have a
look at that. I can't give you the entire file without sanitising it
to protect the innocent, but I don't think the rest of it will be
relevant anyway.

def configure_backgroundrb(w, rails_root, app_name, port)
  w.uid = 'app'
  w.gid = 'app'
  w.name = "#{app_name}-backgroundrb"
  w.interval = 30.seconds # default
  w.start = "#{rails_root}/script/backgroundrb start"
  w.stop = "#{rails_root}/script/backgroundrb stop"
  w.start_grace = 10.seconds
  w.restart_grace = 10.seconds
  w.pid_file = "#{rails_root}/log/backgroundrb.pid"
  w.unix_socket = "/tmp/backgroundrbunix_localhost_#{port}"

  w.behavior(:clean_pid_file)
  w.behavior(:clean_unix_socket)
end

The function is called like this:

God.watch do |w|
  configure_backgroundrb(w, rails_root, app_name,
app.backgroundrb_port)
end if app.respond_to? :backgroundrb_port

The app object is just a little config object that knows what each of
our apps is capable of (they don't all need backgroundrb); you can
essentially ignore the app object and just call configure_backgroundrb
from within your watch block.

Graham

I had everything set up pretty similarly, but although I had set a uid
& gid on my mongrel watch I somehow forgot to do so on the
backgroundrb watch.  So it now appears to be working, and was almost
certainly an issue of user permissions.  So thanks much for point me
in the right direction.  My watch looks like this, for others that
might be looking to have another point of reference(You would also
need to include Graham's unix socket cleaning stuff to get that
behavior to work).

God.watch do |w|
  w.name = "feedish-backgroundrb"
  w.gid = "app_feedish"
  w.uid = "backgroundrb_feedish"
  w.interval = 30.seconds # default
  w.start = "#{RAILS_ROOT}/script/backgroundrb start"
  w.stop = "#{RAILS_ROOT}/script/backgroundrb stop"
  w.restart = "#{RAILS_ROOT}/script/backgroundrb stop && #{RAILS_ROOT}/
script/backgroundrb start"
  w.start_grace = 10.seconds
  w.restart_grace = 10.seconds
  w.pid_file = "#{RAILS_ROOT}/log/backgroundrb_2000.pid"
  w.unix_socket = "/tmp/backgroundrbunix_localhost_#{port}"
  w.behaviour(:clean_pid_file)
  w.behaviour(:clean_unix_socket)

  w.start_if do |start|
    start.condition(:process_running) do |c|
      c.interval = 5.seconds
      c.running = false
    end
  end

  w.restart_if do |restart|
    restart.condition(:memory_usage) do |c|
      c.above = 150.megabytes
      c.times = [3, 5] # 3 out of 5 intervals
    end

    restart.condition(:cpu_usage) do |c|
      c.above = 50.percent
      c.times = 5
    end
  end

  # lifecycle
  w.lifecycle do |on|
    on.condition(:flapping) do |c|
      c.to_state = [:start, :restart]
      c.times = 5
      c.within = 5.minute
      c.transition = :unmonitored
      c.retry_in = 10.minutes
      c.retry_times = 5
      c.retry_within = 2.hours
    end
  end
end

At some point I had replaced the backgroundrb script from the
repository to get the newest version and forgot to update the pid
location.

So it was set to:
pid_file = "#{RAILS_HOME}/tmp/pids/
backgroundrb_#{CONFIG_FILE[:backgroundrb][:port]}.pid"

even though god thought it was this:
pid_file = "#{RAILS_HOME}/log/backgroundrb_#{CONFIG_FILE[:backgroundrb]
[:port]}.pid"

So I changed that and it should probably cooperate now.

I just blogged about the unix socket clean up trick, and I think it'll
be clearer what I did from there:

http://effectif.com/2008/4/8/monitoring-backgroundrb-with-god

Cheers,

Graham