subcorpus_counter.conf

runners:
  emr:
    ## you must edit these to point to your AWS EMR key
    ec2_key_pair: kba-aws-us-east-1
    ec2_key_pair_file: ~/.ssh/kba-aws-us-east-1.pem # ~/ and $ENV_VARS allowed here
    ## turn on logging and tracker
    ssh_tunnel_to_job_tracker: true
    s3_log_uri: s3://trec-kba-emr/example/logs
    ## do not bootstrap mrjob, because we install it using our private
    ## python27 install below.
    bootstrap_mrjob: false

    ## must include this tarball created by running `make`
    python_archives:
    - kba_corpus.tar.gz

    ## these bootstrap actions happen during cluster setup
    bootstrap_actions:
    - s3://elasticmapreduce/bootstrap-actions/configure-hadoop -m mapred.tasktracker.map.tasks.maximum=5 -m mapred.tasktracker.reduce.tasks.maximum=5
    ## install our own python27 built on the m1.large instance class;
    ## this script also installs xz-utils and bootstraps mrjob
    - s3://trec-kba-emr/emr-setup/install_python27.m1.large.sh

    ## could enable skipping -- see also jobconf for more skipping related params
    #- s3://elasticmapreduce/bootstrap-actions/configure-hadoop -m mapred.skip.map.auto.incr.proc.count=false  -m mapred.skip.mode.enabled=true
    #- s3://elasticmapreduce/bootstrap-actions/configure-hadoop -m io.map.index.skip=1

    ## could use spot instances by uncommenting the bid prices
    #ec2_instance_type: m1.large
    #num_ec2_instances: 1
    ec2_master_instance_type: m1.large
    #ec2_master_instance_bid_price: '0.154'
    #ec2_core_instance_type: m2.xlarge
    #ec2_core_instance_bid_price: '0.154'
    #num_ec2_core_instances: 1
    #ec2_task_instance_type: m2.xlarge
    #ec2_task_instance_bid_price: '0.154'
    #num_ec2_task_instances: 1

    enable_emr_debugging: true
    jobconf:
      # enable skip mode
      #mapred.skip.map.auto.incr.proc.count: false
      #mapred.skip.mode.enabled: true
      #mapred.skip.map.max.skip.records: 1
      #mapred.skip.attempts.to.start.skipping: 0
      #mapred.map.max.attempts: 10
      # limit total tasks
      mapred.map.tasks: 40
      mapred.reduce.tasks: 3
      mapred.task.timeout: 300000  ## five minutes

  local:
    ## to run locally, all we need is the tarball, which includes the
    ## decryption key
    python_archives:
    - kba_corpus.tar.gz