Start line:  
End line:  

Snippet Preview

Snippet HTML Code

Stack Overflow Questions
  /*
   * Copyright 2011-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
   *
   * Licensed under the Apache License, Version 2.0 (the "License").
   * You may not use this file except in compliance with the License.
   * A copy of the License is located at
   *
   *  http://aws.amazon.com/apache2.0
   *
  * or in the "license" file accompanying this file. This file is distributed
  * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
  * express or implied. See the License for the specific language governing
  * permissions and limitations under the License.
  */
 package com.amazonaws.services.elasticmapreduce.util;
 
 import java.util.List;
 
Class that provides helper methods for constructing predefined bootstrap actions.
 AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
 AmazonElasticMapReduce emr = new AmazonElasticMapReduceClient(credentials);

 BootstrapActions bootstrapActions = new BootstrapActions();

 RunJobFlowRequest request = new RunJobFlowRequest()
       .withName("Job Flow With Bootstrap Actions")
       .withBootstrapActions(
             bootstrapActions.newRunIf(
                 "instance.isMaster=true",
                 bootstrapActions.newConfigureDaemons()
                     .withHeapSize(Daemon.JobTracker, 2048)
                     .build()))
       .withLogUri("s3://log-bucket/")
       .withInstances(new JobFlowInstancesConfig()
            .withEc2KeyName("keypair")
            .withHadoopVersion("0.20")
            .withInstanceCount(5)
            .withKeepJobFlowAliveWhenNoSteps(true)
            .withMasterInstanceType("m1.small")
            .withSlaveInstanceType("m1.small"));

 RunJobFlowResult result = emr.runJobFlow(request);
 
 
 public class BootstrapActions {
   private final String bucket;

  
Creates a new default BootstrapActions for us in us-east-1.
 
   public BootstrapActions() {
     this("us-east-1.elasticmapreduce");
   }

  
Creates a new BootstrapActions.

Parameters:
bucket the bucket from which to download the bootstrap actions.
 
   public BootstrapActions(String bucket) {
     this. = bucket;
   }

  
Create a new run-if bootstrap action which lets you conditionally run bootstrap actions.

Parameters:
condition The condition to evaluate, if true the bootstrap action executes.
config The bootstrap action to execute in case of successful evaluation.
Returns:
A BootstrapActionConfig to be provided when running a job flow.
 
   public BootstrapActionConfig newRunIf(String conditionBootstrapActionConfig config) {
     List<Stringargs = config.getScriptBootstrapAction().getArgs();
     args.add(0, condition);
     args.add(1, config.getScriptBootstrapAction().getPath());
 
     return new BootstrapActionConfig()
       .withName("Run If, " + config.getName())
         .withPath("s3://" +  + "/bootstrap-actions/run-if")
         .withArgs(args));
   }

  
Enum specifying all valid config files.
 
   public enum ConfigFile {
     Site,
     Default,
     Core,
     Hdfs,
     Mapred
   }

  
Create a new bootstrap action which lets you configure Hadoop's XML files.
    return new ConfigureHadoop();
  }
  public class ConfigureHadoop {
    List<Stringargs = new ArrayList<String>();
    private ConfigureHadoop() {
    }

    
Specify an XML file in S3 to merge with Hadoop's default configuration.

Parameters:
file The config file to merge with.
xmlPath The path in S3 of the XML file.
Returns:
A reference to this updated object so that method calls can be chained together.
    public ConfigureHadoop withXml(ConfigFile fileString xmlPath) {
      String arg = "";
      switch (file) {
      case arg = "-S"break;
      case arg = "-D"break;
      case arg = "-C"break;
      case arg = "-H"break;
      case arg = "-M"break;
      }
      .add(arg);
      .add(xmlPath);
      return this;
    }

    
Specify a key-value pair to merge with Hadoop's default configuration.

Parameters:
file The config file to merge with.
key The config key.
value The config value.
Returns:
A reference to this updated object so that method calls can be chained together.
    public ConfigureHadoop withKeyValue(ConfigFile fileString keyString value) {
      String arg = "";
      switch (file) {
      case arg = "-s"break;
      case arg = "-d"break;
      case arg = "-c"break;
      case arg = "-h"break;
      case arg = "-m"break;
      }
      .add(arg);
      .add(key + "=" + value);
      return this;
    }

    
Returns an object which can be used in a RunJobflow call.

Returns:
an object which can be used in a RunJobflow call.
    public BootstrapActionConfig build() {
      return new BootstrapActionConfig()
        .withName("Configure Hadoop")
          .withPath("s3://" +  + "/bootstrap-actions/configure-hadoop")
          .withArgs());
    }
  }

  
List of Hadoop daemons which can be configured.
  public enum Daemon {
    NameNode,
    DataNode,
    JobTracker,
    TaskTracker,
    Client;
  }

  
Create a new bootstrap action which lets you configure Hadoop's daemons. The options are written to the hadoop-user-env.sh file.
    return new ConfigureDaemons();
  }
  public class ConfigureDaemons {
    List<Stringargs = new ArrayList<String>();
    boolean replace = false;
    private ConfigureDaemons() {
    }

    
Set the heap size of a daemon.

Parameters:
daemon The deamon to configure.
megabytes The requested heap size of the daemon.
Returns:
A reference to this updated object so that method calls can be chained together.
    public ConfigureDaemons withHeapSize(Daemon daemonint megabytes) {
      .add("--" + daemon.name().toLowerCase() + "-heap-size=" + megabytes);
      return this;
    }

    
Specify additional Java opts to be included when the daemon starts.

Parameters:
daemon The daemon to add opts to.
opts Additional Java command line arguments.
Returns:
A reference to this updated object so that method calls can be chained together.
    public ConfigureDaemons withOpts(Daemon daemonString opts) {
      .add("--" + daemon.name().toLowerCase() + "-opts=\"" + opts + "\"");
      return this;
    }

    
Replace the existing hadoop-user-env.sh file if it exists.

Parameters:
replace whether the file should be replaced.
Returns:
A reference to this updated object so that method calls can be chained together.
    public ConfigureDaemons withReplace(boolean replace) {
      this. = replace;
      return this;
    }

    
Returns an object which can be used in a RunJobflow call.

Returns:
an object which can be used in a RunJobflow call.
    public BootstrapActionConfig build() {
      if () {
        .add("--replace");
      }
      return new BootstrapActionConfig()
        .withName("Configure Daemons")
          .withPath("s3://" +  + "/bootstrap-actions/configure-daemons")
          .withArgs());
    }
  }
New to GrepCode? Check out our FAQ X