diff --git a/Gemfile b/Gemfile index 2ed39890..dbe59003 100644 --- a/Gemfile +++ b/Gemfile @@ -6,6 +6,7 @@ gem 'gorillib', ">= 0.4", :github => 'infochimps-labs/gorillib', :branch = gem 'multi_json', ">= 1.1" gem 'extlib' +gem 'right_aws' gem 'addressable' gem 'htmlentities' gem 'home_run', :platform => [:ruby], :require=>'date' diff --git a/lib/wukong/script/emr_command.rb b/lib/wukong/script/emr_command.rb index b07d8ee5..396c6e61 100644 --- a/lib/wukong/script/emr_command.rb +++ b/lib/wukong/script/emr_command.rb @@ -9,8 +9,9 @@ Settings.define :emr_runner, :description => 'Path to the elastic-mapreduce command (~ etc will be expanded)' Settings.define :emr_root, :description => 'S3 bucket and path to use as the base for Elastic MapReduce storage, organized by job name' Settings.define :emr_data_root, :description => 'Optional ' +Settings.define :emr_region, :description => 'Optional. Which EMR region to run the job. Defaults to US East (Virginia)' Settings.define :emr_bootstrap_script, :description => 'Bootstrap actions for Elastic Map Reduce machine provisioning', :default => EMR_CONFIG_DIR+'/emr_bootstrap.sh', :type => :filename, :finally => lambda{ Settings.emr_bootstrap_script = File.expand_path(Settings.emr_bootstrap_script) } -Settings.define :emr_extra_args, :description => 'kludge: allows you to stuff extra args into the elastic-mapreduce invocation', :type => Array, :wukong => true +Settings.define :emr_extra_args, :description => 'kludge: allows you to stuff extra args into the elastic-mapreduce invocation', :wukong => true Settings.define :alive, :description => 'Whether to keep machine running after job invocation', :type => :boolean # Settings.define :key_pair_file, :description => 'AWS Key pair file', :type => :filename @@ -69,11 +70,12 @@ def execute_emr_runner "--stream", "--mapper=#{mapper_s3_uri} ", "--reducer=#{reducer_s3_uri} ", - "--input=#{input_paths.join(",")} --output=#{output_path}", + "--input=#{input_paths.join(",")} --output=#{output_path}" ] # eg to specify zero reducers: # Settings[:emr_extra_args] = "--arg '-D mapred.reduce.tasks=0'" - command_args += Settings[:emr_extra_args] unless Settings[:emr_extra_args].blank? + command_args << "--region #{Settings[:emr_region]}" unless Settings[:emr_region].blank? + command_args << Settings[:emr_extra_args] unless Settings[:emr_extra_args].blank? command_args += hadoop_options_for_emr_runner Log.info 'Follow along at http://localhost:9000/job' execute_command!( File.expand_path(Settings.emr_runner), *command_args ) diff --git a/wukong.gemspec b/wukong.gemspec index 71e005f8..47561ead 100644 --- a/wukong.gemspec +++ b/wukong.gemspec @@ -152,13 +152,10 @@ Gem::Specification.new do |s| "examples/pagerank/run_pagerank.sh", "examples/sample_records.rb", "examples/server_logs/apache_log_parser.rb", - "examples/server_logs/breadcrumb_edges.rb", "examples/server_logs/breadcrumbs.rb", - "examples/server_logs/histograms.rb", "examples/server_logs/logline.rb", "examples/server_logs/nook.rb", "examples/server_logs/nook/faraday_dummy_adapter.rb", - "examples/server_logs/page_counts.pig", "examples/server_logs/user_agent.rb", "examples/simple_word_count.rb", "examples/size.rb", @@ -241,18 +238,10 @@ Gem::Specification.new do |s| "lib/wukong/streamer/summing_reducer.rb", "lib/wukong/streamer/uniq_by_last_reducer.rb", "lib/wukong/typed_struct.rb", - "notes/away/icss/icss_specification.textile", - "notes/away/swineherd/swineherd-old-readme.textile", - "notes/away/wukong-fs/wukong-fs-old-readme.textile", - "notes/away/wukong-transform/wukong-processor-old-readme.textile", "old/cassandra_streaming/berlitz_for_cassandra.textile", "old/cassandra_streaming/client_interface_notes.textile", "old/cassandra_streaming/client_schema.textile", "old/cassandra_streaming/tuning.textile", - "spec/data/a_atsigns_b.tsv", - "spec/data/a_follows_b.tsv", - "spec/data/tweet.tsv", - "spec/data/twitter_user.tsv", "spec/spec.opts", "spec/spec_helper.rb", "spec/wukong/encoding_spec.rb", @@ -270,4 +259,6 @@ Gem::Specification.new do |s| s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, [">= 0"]) + s.add_dependency(%q, [">= 0"]) + s.add_dependency(%q, [">= 0"]) end