Start line:  
End line:  

Snippet Preview

Snippet HTML Code

Stack Overflow Questions
Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
 
 
 package org.apache.tez.dag.api;
 
 
 public class TezConfiguration extends Configuration {
 
   public final static String TEZ_SITE_XML = "tez-site.xml";
 
   static {
   }
 
   public TezConfiguration() {
     super();
   }
 
   public TezConfiguration(Configuration conf) {
     super(conf);
   }
 
   public static final String TEZ_PREFIX = "tez.";
   public static final String TEZ_AM_PREFIX =  + "am.";
   public static final String TEZ_TASK_PREFIX =  + "task.";

  
The staging dir used while submitting DAGs
 
   public static final String TEZ_AM_STAGING_DIR =  + "staging-dir";
   public static final String TEZ_AM_STAGING_DIR_DEFAULT = "/tmp/tez/staging";
 
   public static final String TEZ_APPLICATION_MASTER_CLASS =
       "org.apache.tez.dag.app.DAGAppMaster";

  
Root Logging level passed to the Tez app master
 
   public static final String TEZ_AM_LOG_LEVEL = +"log.level";
   public static final String TEZ_AM_LOG_LEVEL_DEFAULT = "INFO";
 
   public static final String TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS =
        + "commit-all-outputs-on-dag-success";
   public static final boolean TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS_DEFAULT = true;

  
Java options for the Tez AppMaster process.
 
   public static final String TEZ_AM_JAVA_OPTS = 
       + "java.opts";
   public static final String TEZ_AM_JAVA_OPTS_DEFAULT = " -Xmx1024m ";
 
   public static final String TEZ_AM_CANCEL_DELEGATION_TOKEN =  +
       "am.complete.cancel.delegation.tokens";
   public static final boolean TEZ_AM_CANCEL_DELEGATION_TOKEN_DEFAULT = true;
 
   public static final String TEZ_AM_TASK_LISTENER_THREAD_COUNT =
        + "task.listener.thread-count";
   public static final int TEZ_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT = 30;
 
   /*
    * MR AM Service Authorization
    * These are the same as MR which allows Tez to run in secure
    * mode without configuring service ACLs
    */
   public static final String   
       "security.job.task.protocol.acl";
   public static final String   
       "security.job.client.protocol.acl";

  
Upper limit on the number of threads user to launch containers in the app master. Expect level config, you shouldn't be needing it in most cases.
 
   public static final String TEZ_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT =
     +"containerlauncher.thread-count-limit";
 
   public static final int TEZ_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT_DEFAULT = 
     500;
 
   
   // TODO Some of the DAG properties are job specific and not AM specific. Rename accordingly.
   // TODO Are any of these node blacklisting properties required. (other than for MR compat)
   public static final String TEZ_AM_MAX_TASK_FAILURES_PER_NODE = 
       + "maxtaskfailures.per.node";
  public static final int TEZ_AM_MAX_TASK_FAILURES_PER_NODE_DEFAULT = 3;
  public static final String TEZ_AM_MAX_APP_ATTEMPTS =  + 
      "max.app.attempts";
  public static int TEZ_AM_MAX_APP_ATTEMPTS_DEFAULT = 2;
  
  public static final String TEZ_AM_MAX_TASK_ATTEMPTS =
       + "max.task.attempts";
  public static final int TEZ_AM_MAX_TASK_ATTEMPTS_DEFAULT = 4;
      + "node-blacklisting.enabled";
  public static final boolean TEZ_AM_NODE_BLACKLISTING_ENABLED_DEFAULT = true;
      + "node-blacklisting.ignore-threshold-node-percent";
  public static final int TEZ_AM_NODE_BLACKLISTING_IGNORE_THRESHOLD_DEFAULT = 33;

  
Number of threads to handle job client RPC requests
  public static final String TEZ_AM_CLIENT_THREAD_COUNT =
       + "client.am.thread-count";
  public static final int TEZ_AM_CLIENT_THREAD_COUNT_DEFAULT = 1;
  
Range of ports that the AM can use when binding. Leave blank if you want all possible ports.
  public static final String TEZ_AM_CLIENT_AM_PORT_RANGE =
       + "client.am.port-range";


  
The amount of memory to be used by the AppMaster
  public static final String TEZ_AM_RESOURCE_MEMORY_MB = 
      + "resource.memory.mb";
  public static final int TEZ_AM_RESOURCE_MEMORY_MB_DEFAULT = 1536;
  public static final String TEZ_AM_RESOURCE_CPU_VCORES = 
      + "resource.cpu.vcores";
  public static final int TEZ_AM_RESOURCE_CPU_VCORES_DEFAULT = 1;
  public static final String
          + "slowstart-dag-scheduler.min-resource-fraction";
  public static final float
The complete path to the serialized dag plan file TEZ_AM_PLAN_PB_BINARY. Used to make the plan available to individual tasks if needed. This will typically be a path in the job submit directory.
  public static final String TEZ_AM_PLAN_REMOTE_PATH = 
      + "dag-am-plan.remote.path";

  
The maximum heartbeat interval between the AM and RM in milliseconds
      + "am-rm.heartbeat.interval-ms.max";
  public static final int TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX_DEFAULT = 1000;

  
The maximum amount of time, in milliseconds, to wait before a task asks an AM for another task.
      + "get-task.sleep.interval-ms.max";
  public static final int TEZ_TASK_GET_TASK_SLEEP_INTERVAL_MS_MAX_DEFAULT = 200;
      + "am.heartbeat.interval-ms.max";
  public static final int TEZ_TASK_AM_HEARTBEAT_INTERVAL_MS_DEFAULT = 100;
      + "max-events-per-heartbeat.max";
  public static final int TEZ_TASK_MAX_EVENTS_PER_HEARTBEAT_DEFAULT = 100;
  
  
Whether to scale down memory requested by each component if the total exceeds the available JVM memory
      + "scale.memory.enabled";
  public static final boolean TEZ_TASK_SCALE_MEMORY_ENABLED_DEFAULT = true;
  
  
The fraction of the JVM memory which will not be considered for allocation. No defaults, since there are pre-existing defaults based on different scenarios.
      + "scale.memory.reserve-fraction";
  public static final String TASK_TIMEOUT =  + "timeout";
  public static final String TASK_HEARTBEAT_TIMEOUT_MS =  + "heartbeat.timeout-ms";
  
Configuration to specify whether container should be reused.
      + "container.reuse.enabled";
  public static final boolean TEZ_AM_CONTAINER_REUSE_ENABLED_DEFAULT = true;

  
Whether to reuse containers for rack local tasks. Active only if reuse is enabled.
       + "container.reuse.rack-fallback.enabled";
  public static final boolean
Whether to reuse containers for non-local tasks. Active only if reuse is enabled.
       + "container.reuse.non-local-fallback.enabled";
  public static final boolean
The amount of time to wait before assigning a container to the next level of locality. NODE - RACK - NON_LOCAL
  public static final String
       + "container.reuse.locality.delay-allocation-millis";
  public static final long
The amount of time to hold on to a container if no task can be assigned to it immediately. Only active when reuse is enabled. Set to -1 to never release a container in a session.
     + "container.session.delay-allocation-millis";
  public static final long
  public static final String TEZ_PB_BINARY_CONF_NAME = "tez-conf.pb";
  public static final String TEZ_PB_PLAN_BINARY_NAME = "tez-dag.pb";
  public static final String TEZ_PB_PLAN_TEXT_NAME = "tez-dag.pb.txt";
  /*
   * Logger properties
   */
  public static final String TEZ_CONTAINER_LOG4J_PROPERTIES_FILE = "tez-container-log4j.properties";
  public static final String TEZ_CONTAINER_LOGGER_NAME = "CLA";
  public static final String TEZ_ROOT_LOGGER_NAME = "tez.root.logger";
  public static final String TEZ_CONTAINER_LOG_FILE_NAME = "syslog";
  public static final String TEZ_CONTAINER_ERR_FILE_NAME = "stderr";
  public static final String TEZ_CONTAINER_OUT_FILE_NAME = "stdout";


  
The location of the Tez libraries which will be localized for DAGs
  public static final String TEZ_LIB_URIS =  + "lib.uris";
  public static final String TEZ_APPLICATION_TYPE = "TEZ";
  public static final String TEZ_AM_GROUPING_SPLIT_COUNT =  +
      "grouping.split-count";
  public static final String TEZ_AM_GROUPING_SPLIT_BY_LENGTH =  + 
      "grouping.by-length";
  public static final boolean TEZ_AM_GROUPING_SPLIT_BY_LENGTH_DEFAULT = true;
  public static final String TEZ_AM_GROUPING_SPLIT_BY_COUNT =  + 
      "grouping.by-count";
  public static final boolean TEZ_AM_GROUPING_SPLIT_BY_COUNT_DEFAULT = false;
  
  
The multiplier for available queue capacity when determining number of tasks for a Vertex. 1.7 with 100% queue available implies generating a number of tasks roughly equal to 170% of the available containers on the queue
  public static final String TEZ_AM_GROUPING_SPLIT_WAVES =  +
      "grouping.split-waves";
  public static float TEZ_AM_GROUPING_SPLIT_WAVES_DEFAULT = 1.5f;
  
  
Upper bound on the size (in bytes) of a grouped split, to avoid generating excessively large splits.
  public static final String TEZ_AM_GROUPING_SPLIT_MAX_SIZE =  +
      "grouping.max-size";
  public static long TEZ_AM_GROUPING_SPLIT_MAX_SIZE_DEFAULT = 
      1024*1024*1024L;

  
Lower bound on the size (in bytes) of a grouped split, to avoid generating too many splits.
  public static final String TEZ_AM_GROUPING_SPLIT_MIN_SIZE =  +
      "grouping.min-size";
  public static long TEZ_AM_GROUPING_SPLIT_MIN_SIZE_DEFAULT = 
      50*1024*1024L;
  public static final String TEZ_AM_GROUPING_RACK_SPLIT_SIZE_REDUCTION = 
       + "grouping.rack-split-reduction";
  public static final float TEZ_AM_GROUPING_RACK_SPLIT_SIZE_REDUCTION_DEFAULT = 0.75f;


  
Session-related properties
  public static final String TEZ_SESSION_PREFIX =
       + "session.";
     + "local-resources.pb.file-name";

  
Time (in seconds) to wait for AM to come up when trying to submit a DAG from the client.
  public static final String TEZ_SESSION_CLIENT_TIMEOUT_SECS =
       + "client.timeout.secs";
  public static final int TEZ_SESSION_CLIENT_TIMEOUT_SECS_DEFAULT =
      120;

  
Time (in seconds) for which the Tez AM should wait for a DAG to be submitted before shutting down.
  public static final String TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS =
       + "am.dag.submit.timeout.secs";
  public static final int TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS_DEFAULT =
      300;

  
The queue name for all jobs being submitted as part of a session, or for non session jobs.
  public static final String TEZ_QUEUE_NAME = 
       + "queue.name";
  public static final String TEZ_GENERATE_DAG_VIZ =
       + "generate.dag.viz";
  public static final boolean TEZ_GENERATE_DAG_VIZ_DEFAULT = false;
  
  
Comma separated list of containers which should be profiled.
  public static final String TEZ_PROFILE_CONTAINER_LIST =  + "profile.container.list";
  
  
The string to be added to the JVM command line for containers being profiled.
  public static final String TEZ_PROFILE_JVM_OPTS =  + "profile.jvm.opts";

  
The service id for the NodeManager plugin used to share intermediate data between vertices.
  public static final String TEZ_SHUFFLE_HANDLER_SERVICE_ID = "mapreduce_shuffle";
  public static final String TEZ_PREWARM_DAG_NAME_PREFIX = "TezPreWarmDAG";
  public static final String YARN_ATS_ENABLED =
       + "yarn.ats.enabled";
  public static final boolean YARN_ATS_ENABLED_DEFAULT = false;
  public static final String DAG_RECOVERY_ENABLED =
       + "dag.recovery.enabled";
  public static final boolean DAG_RECOVERY_ENABLED_DEFAULT = false;
  public static final String DAG_RECOVERY_FILE_IO_BUFFER_SIZE =
       + "dag.recovery.io.buffer.size";
  public static final int DAG_RECOVERY_FILE_IO_BUFFER_SIZE_DEFAULT = 8192;
  public static final String DAG_RECOVERY_DATA_DIR_NAME = "recovery";
  public static final String DAG_RECOVERY_SUMMARY_FILE_SUFFIX = ".summary";
  public static final String DAG_RECOVERY_RECOVER_FILE_SUFFIX = ".recovery";
New to GrepCode? Check out our FAQ X