#! /usr/bin/env ruby require "bj" Main { usage["description"] = <<-txt ________________________________ Overview -------------------------------- Backgroundjob (Bj) is a simple to use background priority queue for rails. Although not yet tested on windows, the design of bj is such that operation should be possible on any operating system, including M$. Jobs can be submitted to the queue directly using the api or from the commandline using the 'bj' script. For example code: Bj.submit 'cat /etc/password' cli: bj submit cat /etc/password When used from inside a rails application bj arranges that another process will always be running in the background to process the jobs that you submit. By using a separate process to run jobs bj does not impact the resource utilization of your rails application at all and enables several very cool features: 1) Bj allows you to sumbit jobs to any of your configured databases and, in each case, spawns a separate background process to run jobs from that queue Bj.in :production do Bj.submit 'production_job.exe' end Bj.in :development do Bj.submit 'development_job.exe' end 2) Although bj ensures that a process is always running to process your jobs, you can start a proces manually. This means that any machine capable of seeing your RAILS_ROOT can run jobs for your application, allowing one to setup a cluster of machines doing the work of a single front end rails applicaiton. ________________________________ Install -------------------------------- Bj can be installed two ways: as a gem or as a plugin. gem: 1) $sudo gem install bj 2) bj setup ./rails_root/ plugin: 1) ./script/plugin install http://codeforpeople.rubyforge.org/svn/rails/plugins/bj 2) ./script/bj setup ________________________________ Api -------------------------------- submit jobs for background processing. 'jobs' can be a string or array of strings. options are applied to each job in the 'jobs', and the list of submitted jobs is always returned. options (string or symbol) can be :rails_env => production|development|key_in_database_yml when given this keyword causes bj to submit jobs to the specified database. default is RAILS_ENV. :priority => any number, including negative ones. default is zero. :tag => a tag added to the job. simply makes searching easier. :env => a hash specifying any additional environment vars the background process should have. :stdin => any stdin the background process should have. eg: jobs = Bj.submit 'echo foobar', :tag => 'simple job' jobs = Bj.submit '/bin/cat', :stdin => 'in the hat' jobs = Bj.submit './script/runner ./scripts/a.rb', :rails_env => 'production' when jobs are run, they are run in RAILS_ROOT. various attributes are available *only* once the job has finished. you can check whether or not a job is finished by using the #finished method, which simple does a reload and checks to see if the exit_status is non-nil. eg: jobs = Bj.submit list_of_jobs, :tag => 'important' ... jobs.each do |job| if job.finished? p job.exit_status p job.stdout p job.stderr end end See lib/bj/api.rb for more details. ________________________________ Sponsors -------------------------------- http://www.engineyard.com/ http://quintess.com/ http://eparklabs.com/ txt usage["uris"] = <<-txt http://codeforpeople.com/lib/ruby/ http://rubyforge.org/projects/codeforpeople/ http://codeforpeople.rubyforge.org/svn/rails/plugins/ txt author "ara.t.howard@gmail.com" option("rails_root", "R"){ description "the rails_root will be guessed unless you set this" argument_required default RAILS_ROOT } option("rails_env", "E"){ description "set the rails_env" argument_required default RAILS_ENV } option("log", "l"){ description "set the logfile" argument_required default STDERR } mode "migration_code" do description "dump migration code on stdout" def run puts Bj.table.migration_code end end mode "generate_migration" do description "generate a migration" def run Bj.generate_migration end end mode "migrate" do description "migrate the db" def run Bj.migrate end end mode "setup" do description "generate a migration and migrate" def run Bj.setup end end mode "run" do description "start a job runnner, possibly as a daemon" option("--forever"){} option("--daemon"){} option("--ppid"){ argument :required cast :integer } option("--redirect"){ argument :required } def run options = {} %w[ forever daemon ppid ].each do |key| options[key.to_sym] = true if param[key].given? end if param["redirect"].given? open(param["redirect"].value, "a+") do |fd| STDERR.reopen fd STDOUT.reopen fd end STDERR.sync = true STDOUT.sync = true end trap("SIGTERM"){ info{ "SIGTERM" } exit } if param["daemon"].given? daemon{ Bj.run options } else Bj.run options end end end mode "submit" do keyword("file"){ argument :required attr } def run joblist = Bj.joblist.for argv.join(' ') case file when "-" joblist.push(Bj.joblist.jobs_from_io(STDIN)) when "--", "---" joblist.push(Bj.joblist.jobs_from_yaml(STDIN)) else open(file){|io| joblist.push(Bj.joblist.jobs_from_io(io)) } end jobs = Bj.submit joblist, :no_tickle => true oh = lambda{|job| OrderedHash["id", job.id, "command", job.command]} y jobs.map{|job| oh[job]} end end mode "list" do def run Bj.transaction do y Bj::Table::Job.find(:all).map(&:to_hash) end end end mode "set" do argument("key"){ attr } argument("value"){ attr } option("hostname", "H"){ argument :required default Bj.hostname attr } option("cast", "c"){ argument :required default "to_s" attr } def run Bj.transaction do Bj.config.set(key, value, :hostname => hostname, :cast => cast) y Bj.table.config.for(:hostname => hostname) end end end mode "config" do option("hostname", "H"){ argument :required default Bj.hostname } def run Bj.transaction do y Bj.table.config.for(:hostname => param["hostname"].value) end end end mode "pid" do option("hostname", "H"){ argument :required default Bj.hostname } def run Bj.transaction do config = Bj.table.config.for(:hostname => param["hostname"].value) puts config[ "#{ RAILS_ENV }.pid" ] if config end end end def run help! end def before_run self.logger = param["log"].value Bj.logger = logger if param["rails_root"].given? rails_root = param["rails_root"].value ENV["RAILS_ROOT"] = rails_root ::Object.instance_eval do remove_const :RAILS_ROOT const_set :RAILS_ROOT, rails_root end end if param["rails_env"].given? rails_env = param["rails_env"].value ENV["RAILS_ENV"] = rails_env ::Object.instance_eval do remove_const :RAILS_ENV const_set :RAILS_ENV, rails_env end end end def daemon ra, wa = IO.pipe rb, wb = IO.pipe if fork at_exit{ exit! } wa.close r = ra rb.close w = wb pid = r.gets w.puts pid Integer pid.strip else ra.close w = wa wb.close r = rb open("/dev/null", "r+") do |fd| STDIN.reopen fd STDOUT.reopen fd STDERR.reopen fd end Process::setsid rescue nil pid = fork do Dir::chdir RAILS_ROOT File::umask 0 $DAEMON = true yield exit! end w.puts pid r.gets exit! end end } # # we setup a few things so the script works regardless of whether it was # called out of /usr/local/bin, ./script, or wherever. note that the script # does *not* require the entire rails application to be loaded into memory! # we could just load boot.rb and environment.rb, but this method let's # submitting and running jobs be infinitely more lightweight. # BEGIN { # # see if we're running out of RAILS_ROOT/script/ # unless defined? BJ_IS_SCRIPT #{{{ BJ_IS_SCRIPT = if %w[ script config app ].map{|d| test ?d, "#{ File.dirname __FILE__ }/../#{ d }"}.all? __FILE__ else nil end end #}}} # # setup RAILS_ROOT # unless defined? RAILS_ROOT #{{{ ### grab env var first rails_root = ENV["RAILS_ROOT"] ### commandline usage clobbers kv = nil ARGV.delete_if{|arg| arg =~ %r/^RAILS_ROOT=/ and kv = arg} rails_root = kv.split(%r/=/,2).last if kv ### we know the rails_root if we are in RAILS_ROOT/script/ unless rails_root if BJ_IS_SCRIPT rails_root = File.expand_path "#{ File.dirname __FILE__ }/.." end end ### perhaps the current directory is a rails_root? unless rails_root if %w[ script config app ].map{|d| test(?d, d)}.all? rails_root = File.expand_path "." end end ### bootstrap RAILS_ROOT = rails_root end #}}} # # setup RAILS_ENV # unless defined? RAILS_ENV #{{{ ### grab env var first rails_env = ENV["RAILS_ENV"] ### commandline usage clobbers kv = nil ARGV.delete_if{|arg| arg =~ %r/^RAILS_ENV=/ and kv = arg} rails_env = kv.split(%r/=/,2).last if kv ### fallback to development unless rails_env rails_env = "development" end ### bootstrap RAILS_ENV = rails_env end #}}} # # setup $LOAD_PATH to detect plugin - iff it is installed and running out of # RAILS_ROOT/script # if RAILS_ROOT #{{{ if BJ_IS_SCRIPT and test(?d, "#{ RAILS_ROOT }/vendor/plugins/bj/lib") $LOAD_PATH.unshift "#{ RAILS_ROOT }/vendor/plugins/bj/lib" end end #}}} # # ensure that rubygems is loaded # begin require "rubygems" rescue 42 end # # hack to_s of STDERR/STDOUT for nice help messages # class << STDERR def to_s() 'STDERR' end end class << STDOUT def to_s() 'STDOUT' end end }