001/** 002* Licensed to the Apache Software Foundation (ASF) under one 003* or more contributor license agreements. See the NOTICE file 004* distributed with this work for additional information 005* regarding copyright ownership. The ASF licenses this file 006* to you under the Apache License, Version 2.0 (the 007* "License"); you may not use this file except in compliance 008* with the License. You may obtain a copy of the License at 009* 010* http://www.apache.org/licenses/LICENSE-2.0 011* 012* Unless required by applicable law or agreed to in writing, software 013* distributed under the License is distributed on an "AS IS" BASIS, 014* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015* See the License for the specific language governing permissions and 016* limitations under the License. 017*/ 018 019package org.apache.hadoop.yarn.conf; 020 021import java.net.InetSocketAddress; 022import java.util.Arrays; 023import java.util.Collections; 024import java.util.List; 025 026import org.apache.hadoop.HadoopIllegalArgumentException; 027import org.apache.hadoop.classification.InterfaceAudience.Private; 028import org.apache.hadoop.classification.InterfaceAudience.Public; 029import org.apache.hadoop.classification.InterfaceStability.Evolving; 030import org.apache.hadoop.classification.InterfaceStability.Unstable; 031import org.apache.hadoop.conf.Configuration; 032import org.apache.hadoop.http.HttpConfig; 033import org.apache.hadoop.net.NetUtils; 034import org.apache.hadoop.security.authorize.ProxyUsers; 035import org.apache.hadoop.util.StringUtils; 036import org.apache.hadoop.yarn.api.ApplicationConstants; 037 038@Public 039@Evolving 040public class YarnConfiguration extends Configuration { 041 042 @Private 043 public static final String CS_CONFIGURATION_FILE= "capacity-scheduler.xml"; 044 045 @Private 046 public static final String HADOOP_POLICY_CONFIGURATION_FILE = 047 "hadoop-policy.xml"; 048 049 @Private 050 public static final String YARN_SITE_CONFIGURATION_FILE = "yarn-site.xml"; 051 052 private static final String YARN_DEFAULT_CONFIGURATION_FILE = 053 "yarn-default.xml"; 054 055 @Private 056 public static final String CORE_SITE_CONFIGURATION_FILE = "core-site.xml"; 057 058 @Private 059 public static final List<String> RM_CONFIGURATION_FILES = 060 Collections.unmodifiableList(Arrays.asList( 061 CS_CONFIGURATION_FILE, 062 HADOOP_POLICY_CONFIGURATION_FILE, 063 YARN_SITE_CONFIGURATION_FILE, 064 CORE_SITE_CONFIGURATION_FILE)); 065 066 @Evolving 067 public static final int APPLICATION_MAX_TAGS = 10; 068 069 @Evolving 070 public static final int APPLICATION_MAX_TAG_LENGTH = 100; 071 072 static { 073 addDeprecatedKeys(); 074 Configuration.addDefaultResource(YARN_DEFAULT_CONFIGURATION_FILE); 075 Configuration.addDefaultResource(YARN_SITE_CONFIGURATION_FILE); 076 } 077 078 private static void addDeprecatedKeys() { 079 Configuration.addDeprecations(new DeprecationDelta[] { 080 new DeprecationDelta("yarn.client.max-nodemanagers-proxies", 081 NM_CLIENT_MAX_NM_PROXIES) 082 }); 083 } 084 085 //Configurations 086 087 public static final String YARN_PREFIX = "yarn."; 088 089 /** Delay before deleting resource to ease debugging of NM issues */ 090 public static final String DEBUG_NM_DELETE_DELAY_SEC = 091 YarnConfiguration.NM_PREFIX + "delete.debug-delay-sec"; 092 093 //////////////////////////////// 094 // IPC Configs 095 //////////////////////////////// 096 public static final String IPC_PREFIX = YARN_PREFIX + "ipc."; 097 098 /** Factory to create client IPC classes.*/ 099 public static final String IPC_CLIENT_FACTORY_CLASS = 100 IPC_PREFIX + "client.factory.class"; 101 public static final String DEFAULT_IPC_CLIENT_FACTORY_CLASS = 102 "org.apache.hadoop.yarn.factories.impl.pb.RpcClientFactoryPBImpl"; 103 104 /** Factory to create server IPC classes.*/ 105 public static final String IPC_SERVER_FACTORY_CLASS = 106 IPC_PREFIX + "server.factory.class"; 107 public static final String DEFAULT_IPC_SERVER_FACTORY_CLASS = 108 "org.apache.hadoop.yarn.factories.impl.pb.RpcServerFactoryPBImpl"; 109 110 /** Factory to create serializeable records.*/ 111 public static final String IPC_RECORD_FACTORY_CLASS = 112 IPC_PREFIX + "record.factory.class"; 113 public static final String DEFAULT_IPC_RECORD_FACTORY_CLASS = 114 "org.apache.hadoop.yarn.factories.impl.pb.RecordFactoryPBImpl"; 115 116 /** RPC class implementation*/ 117 public static final String IPC_RPC_IMPL = 118 IPC_PREFIX + "rpc.class"; 119 public static final String DEFAULT_IPC_RPC_IMPL = 120 "org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC"; 121 122 //////////////////////////////// 123 // Resource Manager Configs 124 //////////////////////////////// 125 public static final String RM_PREFIX = "yarn.resourcemanager."; 126 127 public static final String RM_CLUSTER_ID = RM_PREFIX + "cluster-id"; 128 129 public static final String RM_HOSTNAME = RM_PREFIX + "hostname"; 130 131 /** The address of the applications manager interface in the RM.*/ 132 public static final String RM_ADDRESS = 133 RM_PREFIX + "address"; 134 public static final int DEFAULT_RM_PORT = 8032; 135 public static final String DEFAULT_RM_ADDRESS = 136 "0.0.0.0:" + DEFAULT_RM_PORT; 137 138 /** The actual bind address for the RM.*/ 139 public static final String RM_BIND_HOST = 140 RM_PREFIX + "bind-host"; 141 142 /** The number of threads used to handle applications manager requests.*/ 143 public static final String RM_CLIENT_THREAD_COUNT = 144 RM_PREFIX + "client.thread-count"; 145 public static final int DEFAULT_RM_CLIENT_THREAD_COUNT = 50; 146 147 /** The Kerberos principal for the resource manager.*/ 148 public static final String RM_PRINCIPAL = 149 RM_PREFIX + "principal"; 150 151 /** The address of the scheduler interface.*/ 152 public static final String RM_SCHEDULER_ADDRESS = 153 RM_PREFIX + "scheduler.address"; 154 public static final int DEFAULT_RM_SCHEDULER_PORT = 8030; 155 public static final String DEFAULT_RM_SCHEDULER_ADDRESS = "0.0.0.0:" + 156 DEFAULT_RM_SCHEDULER_PORT; 157 158 /** Miniumum request grant-able by the RM scheduler. */ 159 public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_MB = 160 YARN_PREFIX + "scheduler.minimum-allocation-mb"; 161 public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB = 1024; 162 public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES = 163 YARN_PREFIX + "scheduler.minimum-allocation-vcores"; 164 public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES = 1; 165 166 /** Maximum request grant-able by the RM scheduler. */ 167 public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_MB = 168 YARN_PREFIX + "scheduler.maximum-allocation-mb"; 169 public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB = 8192; 170 public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES = 171 YARN_PREFIX + "scheduler.maximum-allocation-vcores"; 172 public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES = 4; 173 174 /** Number of threads to handle scheduler interface.*/ 175 public static final String RM_SCHEDULER_CLIENT_THREAD_COUNT = 176 RM_PREFIX + "scheduler.client.thread-count"; 177 public static final int DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT = 50; 178 179 /** If the port should be included or not in the node name. The node name 180 * is used by the scheduler for resource requests allocation location 181 * matching. Typically this is just the hostname, using the port is needed 182 * when using minicluster and specific NM are required.*/ 183 public static final String RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME = 184 YARN_PREFIX + "scheduler.include-port-in-node-name"; 185 public static final boolean DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME = 186 false; 187 188 /** Whether the RM should enable Reservation System */ 189 public static final String RM_RESERVATION_SYSTEM_ENABLE = RM_PREFIX 190 + "reservation-system.enable"; 191 public static final boolean DEFAULT_RM_RESERVATION_SYSTEM_ENABLE = false; 192 193 /** The class to use as the Reservation System. */ 194 public static final String RM_RESERVATION_SYSTEM_CLASS = RM_PREFIX 195 + "reservation-system.class"; 196 197 /** The PlanFollower for the Reservation System. */ 198 public static final String RM_RESERVATION_SYSTEM_PLAN_FOLLOWER = RM_PREFIX 199 + "reservation-system.plan.follower"; 200 201 /** The step size of the Reservation System. */ 202 public static final String RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP = 203 RM_PREFIX + "reservation-system.planfollower.time-step"; 204 public static final long DEFAULT_RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP = 205 1000L; 206 207 /** 208 * Enable periodic monitor threads. 209 * @see #RM_SCHEDULER_MONITOR_POLICIES 210 */ 211 public static final String RM_SCHEDULER_ENABLE_MONITORS = 212 RM_PREFIX + "scheduler.monitor.enable"; 213 public static final boolean DEFAULT_RM_SCHEDULER_ENABLE_MONITORS = false; 214 215 /** List of SchedulingEditPolicy classes affecting the scheduler. */ 216 public static final String RM_SCHEDULER_MONITOR_POLICIES = 217 RM_PREFIX + "scheduler.monitor.policies"; 218 219 /** The address of the RM web application.*/ 220 public static final String RM_WEBAPP_ADDRESS = 221 RM_PREFIX + "webapp.address"; 222 223 public static final int DEFAULT_RM_WEBAPP_PORT = 8088; 224 public static final String DEFAULT_RM_WEBAPP_ADDRESS = "0.0.0.0:" + 225 DEFAULT_RM_WEBAPP_PORT; 226 227 /** The https address of the RM web application.*/ 228 public static final String RM_WEBAPP_HTTPS_ADDRESS = 229 RM_PREFIX + "webapp.https.address"; 230 public static final boolean YARN_SSL_CLIENT_HTTPS_NEED_AUTH_DEFAULT = false; 231 public static final String YARN_SSL_SERVER_RESOURCE_DEFAULT = "ssl-server.xml"; 232 233 public static final int DEFAULT_RM_WEBAPP_HTTPS_PORT = 8090; 234 public static final String DEFAULT_RM_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:" 235 + DEFAULT_RM_WEBAPP_HTTPS_PORT; 236 237 public static final String RM_RESOURCE_TRACKER_ADDRESS = 238 RM_PREFIX + "resource-tracker.address"; 239 public static final int DEFAULT_RM_RESOURCE_TRACKER_PORT = 8031; 240 public static final String DEFAULT_RM_RESOURCE_TRACKER_ADDRESS = 241 "0.0.0.0:" + DEFAULT_RM_RESOURCE_TRACKER_PORT; 242 243 /** The expiry interval for application master reporting.*/ 244 public static final String RM_AM_EXPIRY_INTERVAL_MS = 245 YARN_PREFIX + "am.liveness-monitor.expiry-interval-ms"; 246 public static final int DEFAULT_RM_AM_EXPIRY_INTERVAL_MS = 600000; 247 248 /** How long to wait until a node manager is considered dead.*/ 249 public static final String RM_NM_EXPIRY_INTERVAL_MS = 250 YARN_PREFIX + "nm.liveness-monitor.expiry-interval-ms"; 251 public static final int DEFAULT_RM_NM_EXPIRY_INTERVAL_MS = 600000; 252 253 /** Are acls enabled.*/ 254 public static final String YARN_ACL_ENABLE = 255 YARN_PREFIX + "acl.enable"; 256 public static final boolean DEFAULT_YARN_ACL_ENABLE = false; 257 258 /** ACL of who can be admin of YARN cluster.*/ 259 public static final String YARN_ADMIN_ACL = 260 YARN_PREFIX + "admin.acl"; 261 public static final String DEFAULT_YARN_ADMIN_ACL = "*"; 262 263 /** ACL used in case none is found. Allows nothing. */ 264 public static final String DEFAULT_YARN_APP_ACL = " "; 265 266 /** The address of the RM admin interface.*/ 267 public static final String RM_ADMIN_ADDRESS = 268 RM_PREFIX + "admin.address"; 269 public static final int DEFAULT_RM_ADMIN_PORT = 8033; 270 public static final String DEFAULT_RM_ADMIN_ADDRESS = "0.0.0.0:" + 271 DEFAULT_RM_ADMIN_PORT; 272 273 /**Number of threads used to handle RM admin interface.*/ 274 public static final String RM_ADMIN_CLIENT_THREAD_COUNT = 275 RM_PREFIX + "admin.client.thread-count"; 276 public static final int DEFAULT_RM_ADMIN_CLIENT_THREAD_COUNT = 1; 277 278 /** 279 * The maximum number of application attempts. 280 * It's a global setting for all application masters. 281 */ 282 public static final String RM_AM_MAX_ATTEMPTS = 283 RM_PREFIX + "am.max-attempts"; 284 public static final int DEFAULT_RM_AM_MAX_ATTEMPTS = 2; 285 286 /** The keytab for the resource manager.*/ 287 public static final String RM_KEYTAB = 288 RM_PREFIX + "keytab"; 289 290 /**The kerberos principal to be used for spnego filter for RM.*/ 291 public static final String RM_WEBAPP_SPNEGO_USER_NAME_KEY = 292 RM_PREFIX + "webapp.spnego-principal"; 293 294 /**The kerberos keytab to be used for spnego filter for RM.*/ 295 public static final String RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY = 296 RM_PREFIX + "webapp.spnego-keytab-file"; 297 298 /** 299 * Flag to enable override of the default kerberos authentication filter with 300 * the RM authentication filter to allow authentication using delegation 301 * tokens(fallback to kerberos if the tokens are missing). Only applicable 302 * when the http authentication type is kerberos. 303 */ 304 public static final String RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER = RM_PREFIX 305 + "webapp.delegation-token-auth-filter.enabled"; 306 public static final boolean DEFAULT_RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER = 307 true; 308 309 /** How long to wait until a container is considered dead.*/ 310 public static final String RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = 311 RM_PREFIX + "rm.container-allocation.expiry-interval-ms"; 312 public static final int DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = 600000; 313 314 /** Path to file with nodes to include.*/ 315 public static final String RM_NODES_INCLUDE_FILE_PATH = 316 RM_PREFIX + "nodes.include-path"; 317 public static final String DEFAULT_RM_NODES_INCLUDE_FILE_PATH = ""; 318 319 /** Path to file with nodes to exclude.*/ 320 public static final String RM_NODES_EXCLUDE_FILE_PATH = 321 RM_PREFIX + "nodes.exclude-path"; 322 public static final String DEFAULT_RM_NODES_EXCLUDE_FILE_PATH = ""; 323 324 /** Number of threads to handle resource tracker calls.*/ 325 public static final String RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT = 326 RM_PREFIX + "resource-tracker.client.thread-count"; 327 public static final int DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT = 50; 328 329 /** The class to use as the resource scheduler.*/ 330 public static final String RM_SCHEDULER = 331 RM_PREFIX + "scheduler.class"; 332 333 public static final String DEFAULT_RM_SCHEDULER = 334 "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"; 335 336 /** RM set next Heartbeat interval for NM */ 337 public static final String RM_NM_HEARTBEAT_INTERVAL_MS = 338 RM_PREFIX + "nodemanagers.heartbeat-interval-ms"; 339 public static final long DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS = 1000; 340 341 /** Number of worker threads that write the history data. */ 342 public static final String RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE = 343 RM_PREFIX + "history-writer.multi-threaded-dispatcher.pool-size"; 344 public static final int DEFAULT_RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE = 345 10; 346 347 /** 348 * The setting that controls whether yarn system metrics is published on the 349 * timeline server or not by RM. 350 */ 351 public static final String RM_SYSTEM_METRICS_PUBLISHER_ENABLED = 352 RM_PREFIX + "system-metrics-publisher.enabled"; 353 public static final boolean DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED = false; 354 355 public static final String RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE = 356 RM_PREFIX + "system-metrics-publisher.dispatcher.pool-size"; 357 public static final int DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE = 358 10; 359 360 //Delegation token related keys 361 public static final String DELEGATION_KEY_UPDATE_INTERVAL_KEY = 362 RM_PREFIX + "delegation.key.update-interval"; 363 public static final long DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT = 364 24*60*60*1000; // 1 day 365 public static final String DELEGATION_TOKEN_RENEW_INTERVAL_KEY = 366 RM_PREFIX + "delegation.token.renew-interval"; 367 public static final long DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = 368 24*60*60*1000; // 1 day 369 public static final String DELEGATION_TOKEN_MAX_LIFETIME_KEY = 370 RM_PREFIX + "delegation.token.max-lifetime"; 371 public static final long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 372 7*24*60*60*1000; // 7 days 373 374 public static final String RECOVERY_ENABLED = RM_PREFIX + "recovery.enabled"; 375 public static final boolean DEFAULT_RM_RECOVERY_ENABLED = false; 376 377 @Private 378 public static final String RM_WORK_PRESERVING_RECOVERY_ENABLED = RM_PREFIX 379 + "work-preserving-recovery.enabled"; 380 @Private 381 public static final boolean DEFAULT_RM_WORK_PRESERVING_RECOVERY_ENABLED = 382 false; 383 384 public static final String RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS = 385 RM_PREFIX + "work-preserving-recovery.scheduling-wait-ms"; 386 public static final long DEFAULT_RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS = 387 10000; 388 389 /** Zookeeper interaction configs */ 390 public static final String RM_ZK_PREFIX = RM_PREFIX + "zk-"; 391 392 public static final String RM_ZK_ADDRESS = RM_ZK_PREFIX + "address"; 393 394 public static final String RM_ZK_NUM_RETRIES = RM_ZK_PREFIX + "num-retries"; 395 public static final int DEFAULT_ZK_RM_NUM_RETRIES = 1000; 396 397 public static final String RM_ZK_RETRY_INTERVAL_MS = 398 RM_ZK_PREFIX + "retry-interval-ms"; 399 public static final long DEFAULT_RM_ZK_RETRY_INTERVAL_MS = 1000; 400 401 public static final String RM_ZK_TIMEOUT_MS = RM_ZK_PREFIX + "timeout-ms"; 402 public static final int DEFAULT_RM_ZK_TIMEOUT_MS = 10000; 403 404 public static final String RM_ZK_ACL = RM_ZK_PREFIX + "acl"; 405 public static final String DEFAULT_RM_ZK_ACL = "world:anyone:rwcda"; 406 407 public static final String RM_ZK_AUTH = RM_ZK_PREFIX + "auth"; 408 409 public static final String ZK_STATE_STORE_PREFIX = 410 RM_PREFIX + "zk-state-store."; 411 412 /** Parent znode path under which ZKRMStateStore will create znodes */ 413 public static final String ZK_RM_STATE_STORE_PARENT_PATH = 414 ZK_STATE_STORE_PREFIX + "parent-path"; 415 public static final String DEFAULT_ZK_RM_STATE_STORE_PARENT_PATH = "/rmstore"; 416 417 /** Root node ACLs for fencing */ 418 public static final String ZK_RM_STATE_STORE_ROOT_NODE_ACL = 419 ZK_STATE_STORE_PREFIX + "root-node.acl"; 420 421 /** HA related configs */ 422 public static final String RM_HA_PREFIX = RM_PREFIX + "ha."; 423 public static final String RM_HA_ENABLED = RM_HA_PREFIX + "enabled"; 424 public static final boolean DEFAULT_RM_HA_ENABLED = false; 425 426 public static final String RM_HA_IDS = RM_HA_PREFIX + "rm-ids"; 427 public static final String RM_HA_ID = RM_HA_PREFIX + "id"; 428 429 /** Store the related configuration files in File System */ 430 public static final String FS_BASED_RM_CONF_STORE = RM_PREFIX 431 + "configuration.file-system-based-store"; 432 public static final String DEFAULT_FS_BASED_RM_CONF_STORE = "/yarn/conf"; 433 434 public static final String RM_CONFIGURATION_PROVIDER_CLASS = RM_PREFIX 435 + "configuration.provider-class"; 436 public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS = 437 "org.apache.hadoop.yarn.LocalConfigurationProvider"; 438 439 private static final List<String> RM_SERVICES_ADDRESS_CONF_KEYS_HTTP = 440 Collections.unmodifiableList(Arrays.asList( 441 RM_ADDRESS, 442 RM_SCHEDULER_ADDRESS, 443 RM_ADMIN_ADDRESS, 444 RM_RESOURCE_TRACKER_ADDRESS, 445 RM_WEBAPP_ADDRESS)); 446 447 private static final List<String> RM_SERVICES_ADDRESS_CONF_KEYS_HTTPS = 448 Collections.unmodifiableList(Arrays.asList( 449 RM_ADDRESS, 450 RM_SCHEDULER_ADDRESS, 451 RM_ADMIN_ADDRESS, 452 RM_RESOURCE_TRACKER_ADDRESS, 453 RM_WEBAPP_HTTPS_ADDRESS)); 454 455 public static final String AUTO_FAILOVER_PREFIX = 456 RM_HA_PREFIX + "automatic-failover."; 457 458 public static final String AUTO_FAILOVER_ENABLED = 459 AUTO_FAILOVER_PREFIX + "enabled"; 460 public static final boolean DEFAULT_AUTO_FAILOVER_ENABLED = true; 461 462 public static final String AUTO_FAILOVER_EMBEDDED = 463 AUTO_FAILOVER_PREFIX + "embedded"; 464 public static final boolean DEFAULT_AUTO_FAILOVER_EMBEDDED = true; 465 466 public static final String AUTO_FAILOVER_ZK_BASE_PATH = 467 AUTO_FAILOVER_PREFIX + "zk-base-path"; 468 public static final String DEFAULT_AUTO_FAILOVER_ZK_BASE_PATH = 469 "/yarn-leader-election"; 470 471 public static final String CLIENT_FAILOVER_PREFIX = 472 YARN_PREFIX + "client.failover-"; 473 public static final String CLIENT_FAILOVER_PROXY_PROVIDER = 474 CLIENT_FAILOVER_PREFIX + "proxy-provider"; 475 public static final String DEFAULT_CLIENT_FAILOVER_PROXY_PROVIDER = 476 "org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider"; 477 478 public static final String CLIENT_FAILOVER_MAX_ATTEMPTS = 479 CLIENT_FAILOVER_PREFIX + "max-attempts"; 480 481 public static final String CLIENT_FAILOVER_SLEEPTIME_BASE_MS = 482 CLIENT_FAILOVER_PREFIX + "sleep-base-ms"; 483 484 public static final String CLIENT_FAILOVER_SLEEPTIME_MAX_MS = 485 CLIENT_FAILOVER_PREFIX + "sleep-max-ms"; 486 487 public static final String CLIENT_FAILOVER_RETRIES = 488 CLIENT_FAILOVER_PREFIX + "retries"; 489 public static final int DEFAULT_CLIENT_FAILOVER_RETRIES = 0; 490 491 public static final String CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS = 492 CLIENT_FAILOVER_PREFIX + "retries-on-socket-timeouts"; 493 public static final int 494 DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS = 0; 495 496 //////////////////////////////// 497 // RM state store configs 498 //////////////////////////////// 499 /** The class to use as the persistent store.*/ 500 public static final String RM_STORE = RM_PREFIX + "store.class"; 501 502 /** URI for FileSystemRMStateStore */ 503 public static final String FS_RM_STATE_STORE_URI = RM_PREFIX 504 + "fs.state-store.uri"; 505 public static final String FS_RM_STATE_STORE_RETRY_POLICY_SPEC = RM_PREFIX 506 + "fs.state-store.retry-policy-spec"; 507 public static final String DEFAULT_FS_RM_STATE_STORE_RETRY_POLICY_SPEC = 508 "2000, 500"; 509 510 /** The maximum number of completed applications RM keeps. */ 511 public static final String RM_MAX_COMPLETED_APPLICATIONS = 512 RM_PREFIX + "max-completed-applications"; 513 public static final int DEFAULT_RM_MAX_COMPLETED_APPLICATIONS = 10000; 514 515 /** 516 * The maximum number of completed applications RM state store keeps, by 517 * default equals to DEFAULT_RM_MAX_COMPLETED_APPLICATIONS 518 */ 519 public static final String RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS = 520 RM_PREFIX + "state-store.max-completed-applications"; 521 public static final int DEFAULT_RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS = 522 DEFAULT_RM_MAX_COMPLETED_APPLICATIONS; 523 524 /** Default application name */ 525 public static final String DEFAULT_APPLICATION_NAME = "N/A"; 526 527 /** Default application type */ 528 public static final String DEFAULT_APPLICATION_TYPE = "YARN"; 529 530 /** Default application type length */ 531 public static final int APPLICATION_TYPE_LENGTH = 20; 532 533 /** Default queue name */ 534 public static final String DEFAULT_QUEUE_NAME = "default"; 535 536 /** 537 * Buckets (in minutes) for the number of apps running in each queue. 538 */ 539 public static final String RM_METRICS_RUNTIME_BUCKETS = 540 RM_PREFIX + "metrics.runtime.buckets"; 541 542 /** 543 * Default sizes of the runtime metric buckets in minutes. 544 */ 545 public static final String DEFAULT_RM_METRICS_RUNTIME_BUCKETS = 546 "60,300,1440"; 547 548 public static final String RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = RM_PREFIX 549 + "am-rm-tokens.master-key-rolling-interval-secs"; 550 551 public static final long DEFAULT_RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 552 24 * 60 * 60; 553 554 public static final String RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 555 RM_PREFIX + "container-tokens.master-key-rolling-interval-secs"; 556 557 public static final long DEFAULT_RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 558 24 * 60 * 60; 559 560 public static final String RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 561 RM_PREFIX + "nm-tokens.master-key-rolling-interval-secs"; 562 563 public static final long DEFAULT_RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 564 24 * 60 * 60; 565 566 public static final String RM_NODEMANAGER_MINIMUM_VERSION = 567 RM_PREFIX + "nodemanager.minimum.version"; 568 569 public static final String DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION = 570 "NONE"; 571 572 /** 573 * RM proxy users' prefix 574 */ 575 public static final String RM_PROXY_USER_PREFIX = RM_PREFIX + "proxyuser."; 576 577 //////////////////////////////// 578 // Node Manager Configs 579 //////////////////////////////// 580 581 /** Prefix for all node manager configs.*/ 582 public static final String NM_PREFIX = "yarn.nodemanager."; 583 584 /** Environment variables that will be sent to containers.*/ 585 public static final String NM_ADMIN_USER_ENV = NM_PREFIX + "admin-env"; 586 public static final String DEFAULT_NM_ADMIN_USER_ENV = "MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX"; 587 588 /** Environment variables that containers may override rather than use NodeManager's default.*/ 589 public static final String NM_ENV_WHITELIST = NM_PREFIX + "env-whitelist"; 590 public static final String DEFAULT_NM_ENV_WHITELIST = StringUtils.join(",", 591 Arrays.asList(ApplicationConstants.Environment.JAVA_HOME.key(), 592 ApplicationConstants.Environment.HADOOP_COMMON_HOME.key(), 593 ApplicationConstants.Environment.HADOOP_HDFS_HOME.key(), 594 ApplicationConstants.Environment.HADOOP_CONF_DIR.key(), 595 ApplicationConstants.Environment.HADOOP_YARN_HOME.key())); 596 597 /** address of node manager IPC.*/ 598 public static final String NM_ADDRESS = NM_PREFIX + "address"; 599 public static final int DEFAULT_NM_PORT = 0; 600 public static final String DEFAULT_NM_ADDRESS = "0.0.0.0:" 601 + DEFAULT_NM_PORT; 602 603 /** The actual bind address or the NM.*/ 604 public static final String NM_BIND_HOST = 605 NM_PREFIX + "bind-host"; 606 607 /** who will execute(launch) the containers.*/ 608 public static final String NM_CONTAINER_EXECUTOR = 609 NM_PREFIX + "container-executor.class"; 610 611 /** 612 * Adjustment to make to the container os scheduling priority. 613 * The valid values for this could vary depending on the platform. 614 * On Linux, higher values mean run the containers at a less 615 * favorable priority than the NM. 616 * The value specified is an int. 617 */ 618 public static final String NM_CONTAINER_EXECUTOR_SCHED_PRIORITY = 619 NM_PREFIX + "container-executor.os.sched.priority.adjustment"; 620 public static final int DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY = 0; 621 622 /** Number of threads container manager uses.*/ 623 public static final String NM_CONTAINER_MGR_THREAD_COUNT = 624 NM_PREFIX + "container-manager.thread-count"; 625 public static final int DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT = 20; 626 627 /** Number of threads used in cleanup.*/ 628 public static final String NM_DELETE_THREAD_COUNT = 629 NM_PREFIX + "delete.thread-count"; 630 public static final int DEFAULT_NM_DELETE_THREAD_COUNT = 4; 631 632 /** Keytab for NM.*/ 633 public static final String NM_KEYTAB = NM_PREFIX + "keytab"; 634 635 /**List of directories to store localized files in.*/ 636 public static final String NM_LOCAL_DIRS = NM_PREFIX + "local-dirs"; 637 public static final String DEFAULT_NM_LOCAL_DIRS = "/tmp/nm-local-dir"; 638 639 /** 640 * Number of files in each localized directories 641 * Avoid tuning this too low. 642 */ 643 public static final String NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY = 644 NM_PREFIX + "local-cache.max-files-per-directory"; 645 public static final int DEFAULT_NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY = 8192; 646 647 /** Address where the localizer IPC is.*/ 648 public static final String NM_LOCALIZER_ADDRESS = 649 NM_PREFIX + "localizer.address"; 650 public static final int DEFAULT_NM_LOCALIZER_PORT = 8040; 651 public static final String DEFAULT_NM_LOCALIZER_ADDRESS = "0.0.0.0:" + 652 DEFAULT_NM_LOCALIZER_PORT; 653 654 /** Interval in between cache cleanups.*/ 655 public static final String NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS = 656 NM_PREFIX + "localizer.cache.cleanup.interval-ms"; 657 public static final long DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS = 658 10 * 60 * 1000; 659 660 /** 661 * Target size of localizer cache in MB, per nodemanager. It is a target 662 * retention size that only includes resources with PUBLIC and PRIVATE 663 * visibility and excludes resources with APPLICATION visibility 664 */ 665 public static final String NM_LOCALIZER_CACHE_TARGET_SIZE_MB = 666 NM_PREFIX + "localizer.cache.target-size-mb"; 667 public static final long DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB = 10 * 1024; 668 669 /** Number of threads to handle localization requests.*/ 670 public static final String NM_LOCALIZER_CLIENT_THREAD_COUNT = 671 NM_PREFIX + "localizer.client.thread-count"; 672 public static final int DEFAULT_NM_LOCALIZER_CLIENT_THREAD_COUNT = 5; 673 674 /** Number of threads to use for localization fetching.*/ 675 public static final String NM_LOCALIZER_FETCH_THREAD_COUNT = 676 NM_PREFIX + "localizer.fetch.thread-count"; 677 public static final int DEFAULT_NM_LOCALIZER_FETCH_THREAD_COUNT = 4; 678 679 /** Where to store container logs.*/ 680 public static final String NM_LOG_DIRS = NM_PREFIX + "log-dirs"; 681 public static final String DEFAULT_NM_LOG_DIRS = "/tmp/logs"; 682 683 /** The number of threads to handle log aggregation in node manager. */ 684 public static final String NM_LOG_AGGREGATION_THREAD_POOL_SIZE = 685 NM_PREFIX + "logaggregation.threadpool-size-max"; 686 public static final int DEFAULT_NM_LOG_AGGREGATION_THREAD_POOL_SIZE = 100; 687 688 /** Default permissions for container logs. */ 689 public static final String NM_DEFAULT_CONTAINER_EXECUTOR_PREFIX = 690 NM_PREFIX + "default-container-executor."; 691 public static final String NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS = 692 NM_DEFAULT_CONTAINER_EXECUTOR_PREFIX + "log-dirs.permissions"; 693 public static final String NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS_DEFAULT = "710"; 694 695 public static final String NM_RESOURCEMANAGER_MINIMUM_VERSION = 696 NM_PREFIX + "resourcemanager.minimum.version"; 697 public static final String DEFAULT_NM_RESOURCEMANAGER_MINIMUM_VERSION = "NONE"; 698 699 /** Interval at which the delayed token removal thread runs */ 700 public static final String RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS = 701 RM_PREFIX + "delayed.delegation-token.removal-interval-ms"; 702 public static final long DEFAULT_RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS = 703 30000l; 704 705 /** Delegation Token renewer thread count */ 706 public static final String RM_DELEGATION_TOKEN_RENEWER_THREAD_COUNT = 707 RM_PREFIX + "delegation-token-renewer.thread-count"; 708 public static final int DEFAULT_RM_DELEGATION_TOKEN_RENEWER_THREAD_COUNT = 50; 709 710 public static final String RM_PROXY_USER_PRIVILEGES_ENABLED = RM_PREFIX 711 + "proxy-user-privileges.enabled"; 712 public static boolean DEFAULT_RM_PROXY_USER_PRIVILEGES_ENABLED = false; 713 714 /** 715 * How many diagnostics/failure messages can be saved in RM for 716 * log aggregation. It also defines the number of diagnostics/failure 717 * messages can be shown in log aggregation web ui. 718 */ 719 public static final String RM_MAX_LOG_AGGREGATION_DIAGNOSTICS_IN_MEMORY = 720 RM_PREFIX + "max-log-aggregation-diagnostics-in-memory"; 721 public static final int DEFAULT_RM_MAX_LOG_AGGREGATION_DIAGNOSTICS_IN_MEMORY = 722 10; 723 724 /** Whether to enable log aggregation */ 725 public static final String LOG_AGGREGATION_ENABLED = YARN_PREFIX 726 + "log-aggregation-enable"; 727 public static final boolean DEFAULT_LOG_AGGREGATION_ENABLED = false; 728 729 /** 730 * How long to wait before deleting aggregated logs, -1 disables. 731 * Be careful set this too small and you will spam the name node. 732 */ 733 public static final String LOG_AGGREGATION_RETAIN_SECONDS = YARN_PREFIX 734 + "log-aggregation.retain-seconds"; 735 public static final long DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS = -1; 736 737 /** 738 * How long to wait between aggregated log retention checks. If set to 739 * a value <= 0 then the value is computed as one-tenth of the log retention 740 * setting. Be careful set this too small and you will spam the name node. 741 */ 742 public static final String LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS = 743 YARN_PREFIX + "log-aggregation.retain-check-interval-seconds"; 744 public static final long DEFAULT_LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS = -1; 745 746 /** 747 * How long for ResourceManager to wait for NodeManager to report its 748 * log aggregation status. If waiting time of which the log aggregation status 749 * is reported from NodeManager exceeds the configured value, RM will report 750 * log aggregation status for this NodeManager as TIME_OUT 751 */ 752 public static final String LOG_AGGREGATION_STATUS_TIME_OUT_MS = 753 YARN_PREFIX + "log-aggregation-status.time-out.ms"; 754 public static final long DEFAULT_LOG_AGGREGATION_STATUS_TIME_OUT_MS 755 = 10 * 60 * 1000; 756 757 /** 758 * Number of seconds to retain logs on the NodeManager. Only applicable if Log 759 * aggregation is disabled 760 */ 761 public static final String NM_LOG_RETAIN_SECONDS = NM_PREFIX 762 + "log.retain-seconds"; 763 public static final long DEFAULT_NM_LOG_RETAIN_SECONDS = 3 * 60 * 60; 764 765 /** 766 * Define how often NMs wake up and upload log files 767 */ 768 public static final String NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS = 769 NM_PREFIX + "log-aggregation.roll-monitoring-interval-seconds"; 770 public static final long 771 DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS = -1; 772 /** 773 * Number of threads used in log cleanup. Only applicable if Log aggregation 774 * is disabled 775 */ 776 public static final String NM_LOG_DELETION_THREADS_COUNT = 777 NM_PREFIX + "log.deletion-threads-count"; 778 public static final int DEFAULT_NM_LOG_DELETE_THREAD_COUNT = 4; 779 780 /** Where to aggregate logs to.*/ 781 public static final String NM_REMOTE_APP_LOG_DIR = 782 NM_PREFIX + "remote-app-log-dir"; 783 public static final String DEFAULT_NM_REMOTE_APP_LOG_DIR = "/tmp/logs"; 784 785 /** 786 * The remote log dir will be created at 787 * NM_REMOTE_APP_LOG_DIR/${user}/NM_REMOTE_APP_LOG_DIR_SUFFIX/${appId} 788 */ 789 public static final String NM_REMOTE_APP_LOG_DIR_SUFFIX = 790 NM_PREFIX + "remote-app-log-dir-suffix"; 791 public static final String DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX="logs"; 792 793 public static final String YARN_LOG_SERVER_URL = 794 YARN_PREFIX + "log.server.url"; 795 796 public static final String YARN_TRACKING_URL_GENERATOR = 797 YARN_PREFIX + "tracking.url.generator"; 798 799 /** Amount of memory in GB that can be allocated for containers.*/ 800 public static final String NM_PMEM_MB = NM_PREFIX + "resource.memory-mb"; 801 public static final int DEFAULT_NM_PMEM_MB = 8 * 1024; 802 803 /** Specifies whether physical memory check is enabled. */ 804 public static final String NM_PMEM_CHECK_ENABLED = NM_PREFIX 805 + "pmem-check-enabled"; 806 public static final boolean DEFAULT_NM_PMEM_CHECK_ENABLED = true; 807 808 /** Specifies whether physical memory check is enabled. */ 809 public static final String NM_VMEM_CHECK_ENABLED = NM_PREFIX 810 + "vmem-check-enabled"; 811 public static final boolean DEFAULT_NM_VMEM_CHECK_ENABLED = false; 812 813 /** Conversion ratio for physical memory to virtual memory. */ 814 public static final String NM_VMEM_PMEM_RATIO = 815 NM_PREFIX + "vmem-pmem-ratio"; 816 public static final float DEFAULT_NM_VMEM_PMEM_RATIO = 2.1f; 817 818 /** Number of Virtual CPU Cores which can be allocated for containers.*/ 819 public static final String NM_VCORES = NM_PREFIX + "resource.cpu-vcores"; 820 public static final int DEFAULT_NM_VCORES = 8; 821 822 /** Percentage of overall CPU which can be allocated for containers. */ 823 public static final String NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT = 824 NM_PREFIX + "resource.percentage-physical-cpu-limit"; 825 public static final int DEFAULT_NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT = 826 100; 827 828 /** 829 * Prefix for disk configurations. Work in progress: This configuration 830 * parameter may be changed/removed in the future. 831 */ 832 @Private 833 public static final String NM_DISK_RESOURCE_PREFIX = NM_PREFIX 834 + "resource.disk."; 835 /** 836 * This setting controls if resource handling for disk operations is enabled. 837 * Work in progress: This configuration parameter may be changed/removed in 838 * the future 839 */ 840 @Private 841 public static final String NM_DISK_RESOURCE_ENABLED = NM_DISK_RESOURCE_PREFIX 842 + "enabled"; 843 /** Disk as a resource is disabled by default. **/ 844 @Private 845 public static final boolean DEFAULT_NM_DISK_RESOURCE_ENABLED = false; 846 847 public static final String NM_NETWORK_RESOURCE_PREFIX = NM_PREFIX 848 + "resource.network."; 849 850 /** 851 * This setting controls if resource handling for network bandwidth is 852 * enabled. Work in progress: This configuration parameter may be 853 * changed/removed in the future 854 */ 855 @Private 856 public static final String NM_NETWORK_RESOURCE_ENABLED = 857 NM_NETWORK_RESOURCE_PREFIX + "enabled"; 858 /** Network as a resource is disabled by default. **/ 859 @Private 860 public static final boolean DEFAULT_NM_NETWORK_RESOURCE_ENABLED = false; 861 862 /** 863 * Specifies the interface to be used for applying network throttling rules. 864 * Work in progress: This configuration parameter may be changed/removed in 865 * the future 866 */ 867 @Private 868 public static final String NM_NETWORK_RESOURCE_INTERFACE = 869 NM_NETWORK_RESOURCE_PREFIX + "interface"; 870 @Private 871 public static final String DEFAULT_NM_NETWORK_RESOURCE_INTERFACE = "eth0"; 872 873 /** 874 * Specifies the total available outbound bandwidth on the node. Work in 875 * progress: This configuration parameter may be changed/removed in the future 876 */ 877 @Private 878 public static final String NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT = 879 NM_NETWORK_RESOURCE_PREFIX + "outbound-bandwidth-mbit"; 880 @Private 881 public static final int DEFAULT_NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT = 882 1000; 883 884 /** 885 * Specifies the total outbound bandwidth available to YARN containers. 886 * defaults to NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT if not specified. 887 * Work in progress: This configuration parameter may be changed/removed in 888 * the future 889 */ 890 @Private 891 public static final String NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_YARN_MBIT = 892 NM_NETWORK_RESOURCE_PREFIX + "outbound-bandwidth-yarn-mbit"; 893 894 /** NM Webapp address.**/ 895 public static final String NM_WEBAPP_ADDRESS = NM_PREFIX + "webapp.address"; 896 public static final int DEFAULT_NM_WEBAPP_PORT = 8042; 897 public static final String DEFAULT_NM_WEBAPP_ADDRESS = "0.0.0.0:" + 898 DEFAULT_NM_WEBAPP_PORT; 899 900 /** NM Webapp https address.**/ 901 public static final String NM_WEBAPP_HTTPS_ADDRESS = NM_PREFIX 902 + "webapp.https.address"; 903 public static final int DEFAULT_NM_WEBAPP_HTTPS_PORT = 8044; 904 public static final String DEFAULT_NM_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:" 905 + DEFAULT_NM_WEBAPP_HTTPS_PORT; 906 907 /** How often to monitor containers.*/ 908 public final static String NM_CONTAINER_MON_INTERVAL_MS = 909 NM_PREFIX + "container-monitor.interval-ms"; 910 public final static int DEFAULT_NM_CONTAINER_MON_INTERVAL_MS = 3000; 911 912 /** Class that calculates containers current resource utilization.*/ 913 public static final String NM_CONTAINER_MON_RESOURCE_CALCULATOR = 914 NM_PREFIX + "container-monitor.resource-calculator.class"; 915 /** Class that calculates process tree resource utilization.*/ 916 public static final String NM_CONTAINER_MON_PROCESS_TREE = 917 NM_PREFIX + "container-monitor.process-tree.class"; 918 public static final String PROCFS_USE_SMAPS_BASED_RSS_ENABLED = NM_PREFIX + 919 "container-monitor.procfs-tree.smaps-based-rss.enabled"; 920 public static final boolean DEFAULT_PROCFS_USE_SMAPS_BASED_RSS_ENABLED = 921 false; 922 923 /** Enable/disable container metrics. */ 924 @Private 925 public static final String NM_CONTAINER_METRICS_ENABLE = 926 NM_PREFIX + "container-metrics.enable"; 927 @Private 928 public static final boolean DEFAULT_NM_CONTAINER_METRICS_ENABLE = true; 929 930 /** Container metrics flush period. -1 for flush on completion. */ 931 @Private 932 public static final String NM_CONTAINER_METRICS_PERIOD_MS = 933 NM_PREFIX + "container-metrics.period-ms"; 934 @Private 935 public static final int DEFAULT_NM_CONTAINER_METRICS_PERIOD_MS = -1; 936 937 /** The delay time ms to unregister container metrics after completion. */ 938 @Private 939 public static final String NM_CONTAINER_METRICS_UNREGISTER_DELAY_MS = 940 NM_PREFIX + "container-metrics.unregister-delay-ms"; 941 @Private 942 public static final int DEFAULT_NM_CONTAINER_METRICS_UNREGISTER_DELAY_MS = 943 10000; 944 945 /** Prefix for all node manager disk health checker configs. */ 946 private static final String NM_DISK_HEALTH_CHECK_PREFIX = 947 "yarn.nodemanager.disk-health-checker."; 948 /** 949 * Enable/Disable disks' health checker. Default is true. An expert level 950 * configuration property. 951 */ 952 public static final String NM_DISK_HEALTH_CHECK_ENABLE = 953 NM_DISK_HEALTH_CHECK_PREFIX + "enable"; 954 /** Frequency of running disks' health checker. */ 955 public static final String NM_DISK_HEALTH_CHECK_INTERVAL_MS = 956 NM_DISK_HEALTH_CHECK_PREFIX + "interval-ms"; 957 /** By default, disks' health is checked every 2 minutes. */ 958 public static final long DEFAULT_NM_DISK_HEALTH_CHECK_INTERVAL_MS = 959 2 * 60 * 1000; 960 961 /** 962 * The minimum fraction of number of disks to be healthy for the nodemanager 963 * to launch new containers. This applies to nm-local-dirs and nm-log-dirs. 964 */ 965 public static final String NM_MIN_HEALTHY_DISKS_FRACTION = 966 NM_DISK_HEALTH_CHECK_PREFIX + "min-healthy-disks"; 967 /** 968 * By default, at least 25% of disks are to be healthy to say that the node is 969 * healthy in terms of disks. 970 */ 971 public static final float DEFAULT_NM_MIN_HEALTHY_DISKS_FRACTION = 0.25F; 972 973 /** 974 * The maximum percentage of disk space that can be used after which a disk is 975 * marked as offline. Values can range from 0.0 to 100.0. If the value is 976 * greater than or equal to 100, NM will check for full disk. This applies to 977 * nm-local-dirs and nm-log-dirs. 978 */ 979 public static final String NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE = 980 NM_DISK_HEALTH_CHECK_PREFIX + "max-disk-utilization-per-disk-percentage"; 981 /** 982 * By default, 90% of the disk can be used before it is marked as offline. 983 */ 984 public static final float DEFAULT_NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE = 985 90.0F; 986 987 /** 988 * The low threshold percentage of disk space used when an offline disk is 989 * marked as online. Values can range from 0.0 to 100.0. The value shouldn't 990 * be more than NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE. If its value is 991 * more than NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE or not set, it will be 992 * set to the same value as NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE. 993 * This applies to nm-local-dirs and nm-log-dirs. 994 */ 995 public static final String NM_WM_LOW_PER_DISK_UTILIZATION_PERCENTAGE = 996 NM_DISK_HEALTH_CHECK_PREFIX + 997 "disk-utilization-watermark-low-per-disk-percentage"; 998 999 /** 1000 * The minimum space that must be available on a local dir for it to be used. 1001 * This applies to nm-local-dirs and nm-log-dirs. 1002 */ 1003 public static final String NM_MIN_PER_DISK_FREE_SPACE_MB = 1004 NM_DISK_HEALTH_CHECK_PREFIX + "min-free-space-per-disk-mb"; 1005 /** 1006 * By default, all of the disk can be used before it is marked as offline. 1007 */ 1008 public static final long DEFAULT_NM_MIN_PER_DISK_FREE_SPACE_MB = 0; 1009 1010 /** Frequency of running node health script.*/ 1011 public static final String NM_HEALTH_CHECK_INTERVAL_MS = 1012 NM_PREFIX + "health-checker.interval-ms"; 1013 public static final long DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS = 10 * 60 * 1000; 1014 1015 /** Health check script time out period.*/ 1016 public static final String NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS = 1017 NM_PREFIX + "health-checker.script.timeout-ms"; 1018 public static final long DEFAULT_NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS = 1019 2 * DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS; 1020 1021 /** The health check script to run.*/ 1022 public static final String NM_HEALTH_CHECK_SCRIPT_PATH = 1023 NM_PREFIX + "health-checker.script.path"; 1024 1025 /** The arguments to pass to the health check script.*/ 1026 public static final String NM_HEALTH_CHECK_SCRIPT_OPTS = 1027 NM_PREFIX + "health-checker.script.opts"; 1028 1029 /** The JVM options used on forking ContainerLocalizer process 1030 by container executor. */ 1031 public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY = 1032 NM_PREFIX + "container-localizer.java.opts"; 1033 public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT = 1034 "-Xmx256m"; 1035 1036 /** The Docker image name(For DockerContainerExecutor).*/ 1037 public static final String NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME = 1038 NM_PREFIX + "docker-container-executor.image-name"; 1039 1040 /** The name of the docker executor (For DockerContainerExecutor).*/ 1041 public static final String NM_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME = 1042 NM_PREFIX + "docker-container-executor.exec-name"; 1043 1044 /** The default docker executor (For DockerContainerExecutor).*/ 1045 public static final String NM_DEFAULT_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME = 1046 "/usr/bin/docker"; 1047 1048 /** The path to the Linux container executor.*/ 1049 public static final String NM_LINUX_CONTAINER_EXECUTOR_PATH = 1050 NM_PREFIX + "linux-container-executor.path"; 1051 1052 /** 1053 * The UNIX group that the linux-container-executor should run as. 1054 * This is intended to be set as part of container-executor.cfg. 1055 */ 1056 public static final String NM_LINUX_CONTAINER_GROUP = 1057 NM_PREFIX + "linux-container-executor.group"; 1058 1059 /** 1060 * If linux-container-executor should limit itself to one user 1061 * when running in non-secure mode. 1062 */ 1063 public static final String NM_NONSECURE_MODE_LIMIT_USERS= NM_PREFIX + 1064 "linux-container-executor.nonsecure-mode.limit-users"; 1065 1066 public static final boolean DEFAULT_NM_NONSECURE_MODE_LIMIT_USERS = true; 1067 1068 /** 1069 * The UNIX user that containers will run as when Linux-container-executor 1070 * is used in nonsecure mode (a use case for this is using cgroups). 1071 */ 1072 public static final String NM_NONSECURE_MODE_LOCAL_USER_KEY = NM_PREFIX + 1073 "linux-container-executor.nonsecure-mode.local-user"; 1074 1075 public static final String DEFAULT_NM_NONSECURE_MODE_LOCAL_USER = "nobody"; 1076 1077 /** 1078 * The allowed pattern for UNIX user names enforced by 1079 * Linux-container-executor when used in nonsecure mode (use case for this 1080 * is using cgroups). The default value is taken from /usr/sbin/adduser 1081 */ 1082 public static final String NM_NONSECURE_MODE_USER_PATTERN_KEY = NM_PREFIX + 1083 "linux-container-executor.nonsecure-mode.user-pattern"; 1084 1085 public static final String DEFAULT_NM_NONSECURE_MODE_USER_PATTERN = 1086 "^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$"; 1087 1088 /** The type of resource enforcement to use with the 1089 * linux container executor. 1090 */ 1091 public static final String NM_LINUX_CONTAINER_RESOURCES_HANDLER = 1092 NM_PREFIX + "linux-container-executor.resources-handler.class"; 1093 1094 /** The path the linux container executor should use for cgroups */ 1095 public static final String NM_LINUX_CONTAINER_CGROUPS_HIERARCHY = 1096 NM_PREFIX + "linux-container-executor.cgroups.hierarchy"; 1097 1098 /** Whether the linux container executor should mount cgroups if not found */ 1099 public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT = 1100 NM_PREFIX + "linux-container-executor.cgroups.mount"; 1101 1102 /** Where the linux container executor should mount cgroups if not found */ 1103 public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH = 1104 NM_PREFIX + "linux-container-executor.cgroups.mount-path"; 1105 1106 /** 1107 * Whether the apps should run in strict resource usage mode(not allowed to 1108 * use spare CPU) 1109 */ 1110 public static final String NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE = 1111 NM_PREFIX + "linux-container-executor.cgroups.strict-resource-usage"; 1112 public static final boolean DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE = 1113 false; 1114 1115 1116 1117 /** 1118 * Interval of time the linux container executor should try cleaning up 1119 * cgroups entry when cleaning up a container. This is required due to what 1120 * it seems a race condition because the SIGTERM/SIGKILL is asynch. 1121 */ 1122 public static final String NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT = 1123 NM_PREFIX + "linux-container-executor.cgroups.delete-timeout-ms"; 1124 1125 public static final long DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT = 1126 1000; 1127 1128 /** 1129 * Delay between attempts to remove linux cgroup. 1130 */ 1131 public static final String NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY = 1132 NM_PREFIX + "linux-container-executor.cgroups.delete-delay-ms"; 1133 1134 public static final long DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY = 1135 20; 1136 1137 /** 1138 /* The Windows group that the windows-secure-container-executor should run as. 1139 */ 1140 public static final String NM_WINDOWS_SECURE_CONTAINER_GROUP = 1141 NM_PREFIX + "windows-secure-container-executor.group"; 1142 1143 /** T-file compression types used to compress aggregated logs.*/ 1144 public static final String NM_LOG_AGG_COMPRESSION_TYPE = 1145 NM_PREFIX + "log-aggregation.compression-type"; 1146 public static final String DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE = "none"; 1147 1148 /** The kerberos principal for the node manager.*/ 1149 public static final String NM_PRINCIPAL = 1150 NM_PREFIX + "principal"; 1151 1152 public static final String NM_AUX_SERVICES = 1153 NM_PREFIX + "aux-services"; 1154 1155 public static final String NM_AUX_SERVICE_FMT = 1156 NM_PREFIX + "aux-services.%s.class"; 1157 1158 public static final String NM_USER_HOME_DIR = 1159 NM_PREFIX + "user-home-dir"; 1160 1161 /**The kerberos principal to be used for spnego filter for NM.*/ 1162 public static final String NM_WEBAPP_SPNEGO_USER_NAME_KEY = 1163 NM_PREFIX + "webapp.spnego-principal"; 1164 1165 /**The kerberos keytab to be used for spnego filter for NM.*/ 1166 public static final String NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY = 1167 NM_PREFIX + "webapp.spnego-keytab-file"; 1168 1169 public static final String DEFAULT_NM_USER_HOME_DIR= "/home/"; 1170 1171 public static final String NM_RECOVERY_PREFIX = NM_PREFIX + "recovery."; 1172 public static final String NM_RECOVERY_ENABLED = 1173 NM_RECOVERY_PREFIX + "enabled"; 1174 public static final boolean DEFAULT_NM_RECOVERY_ENABLED = false; 1175 1176 public static final String NM_RECOVERY_DIR = NM_RECOVERY_PREFIX + "dir"; 1177 1178 //////////////////////////////// 1179 // Web Proxy Configs 1180 //////////////////////////////// 1181 public static final String PROXY_PREFIX = "yarn.web-proxy."; 1182 1183 /** The kerberos principal for the proxy.*/ 1184 public static final String PROXY_PRINCIPAL = 1185 PROXY_PREFIX + "principal"; 1186 1187 /** Keytab for Proxy.*/ 1188 public static final String PROXY_KEYTAB = PROXY_PREFIX + "keytab"; 1189 1190 /** The address for the web proxy.*/ 1191 public static final String PROXY_ADDRESS = 1192 PROXY_PREFIX + "address"; 1193 public static final int DEFAULT_PROXY_PORT = 9099; 1194 public static final String DEFAULT_PROXY_ADDRESS = 1195 "0.0.0.0:" + DEFAULT_PROXY_PORT; 1196 1197 /** 1198 * YARN Service Level Authorization 1199 */ 1200 public static final String 1201 YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL = 1202 "security.resourcetracker.protocol.acl"; 1203 public static final String 1204 YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONCLIENT_PROTOCOL = 1205 "security.applicationclient.protocol.acl"; 1206 public static final String 1207 YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCEMANAGER_ADMINISTRATION_PROTOCOL = 1208 "security.resourcemanager-administration.protocol.acl"; 1209 public static final String 1210 YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_PROTOCOL = 1211 "security.applicationmaster.protocol.acl"; 1212 1213 public static final String 1214 YARN_SECURITY_SERVICE_AUTHORIZATION_CONTAINER_MANAGEMENT_PROTOCOL = 1215 "security.containermanagement.protocol.acl"; 1216 public static final String 1217 YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCE_LOCALIZER = 1218 "security.resourcelocalizer.protocol.acl"; 1219 1220 public static final String 1221 YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONHISTORY_PROTOCOL = 1222 "security.applicationhistory.protocol.acl"; 1223 1224 /** No. of milliseconds to wait between sending a SIGTERM and SIGKILL 1225 * to a running container */ 1226 public static final String NM_SLEEP_DELAY_BEFORE_SIGKILL_MS = 1227 NM_PREFIX + "sleep-delay-before-sigkill.ms"; 1228 public static final long DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS = 1229 250; 1230 1231 /** Max time to wait for a process to come up when trying to cleanup 1232 * container resources */ 1233 public static final String NM_PROCESS_KILL_WAIT_MS = 1234 NM_PREFIX + "process-kill-wait.ms"; 1235 public static final long DEFAULT_NM_PROCESS_KILL_WAIT_MS = 1236 2000; 1237 1238 /** Max time to wait to establish a connection to RM */ 1239 public static final String RESOURCEMANAGER_CONNECT_MAX_WAIT_MS = 1240 RM_PREFIX + "connect.max-wait.ms"; 1241 public static final long DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS = 1242 15 * 60 * 1000; 1243 1244 /** Time interval between each attempt to connect to RM */ 1245 public static final String RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS = 1246 RM_PREFIX + "connect.retry-interval.ms"; 1247 public static final long DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS 1248 = 30 * 1000; 1249 1250 /** 1251 * CLASSPATH for YARN applications. A comma-separated list of CLASSPATH 1252 * entries 1253 */ 1254 public static final String YARN_APPLICATION_CLASSPATH = YARN_PREFIX 1255 + "application.classpath"; 1256 1257 /** 1258 * Default platform-agnostic CLASSPATH for YARN applications. A 1259 * comma-separated list of CLASSPATH entries. The parameter expansion marker 1260 * will be replaced with real parameter expansion marker ('%' for Windows and 1261 * '$' for Linux) by NodeManager on container launch. For example: {{VAR}} 1262 * will be replaced as $VAR on Linux, and %VAR% on Windows. 1263 */ 1264 @Public 1265 @Unstable 1266 public static final String[] DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH= { 1267 ApplicationConstants.Environment.HADOOP_CONF_DIR.$$(), 1268 ApplicationConstants.Environment.HADOOP_COMMON_HOME.$$() 1269 + "/share/hadoop/common/*", 1270 ApplicationConstants.Environment.HADOOP_COMMON_HOME.$$() 1271 + "/share/hadoop/common/lib/*", 1272 ApplicationConstants.Environment.HADOOP_HDFS_HOME.$$() 1273 + "/share/hadoop/hdfs/*", 1274 ApplicationConstants.Environment.HADOOP_HDFS_HOME.$$() 1275 + "/share/hadoop/hdfs/lib/*", 1276 ApplicationConstants.Environment.HADOOP_YARN_HOME.$$() 1277 + "/share/hadoop/yarn/*", 1278 ApplicationConstants.Environment.HADOOP_YARN_HOME.$$() 1279 + "/share/hadoop/yarn/lib/*" }; 1280 /** 1281 * <p> 1282 * Default platform-specific CLASSPATH for YARN applications. A 1283 * comma-separated list of CLASSPATH entries constructed based on the client 1284 * OS environment expansion syntax. 1285 * </p> 1286 * <p> 1287 * Note: Use {@link DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH} for 1288 * cross-platform practice i.e. submit an application from a Windows client to 1289 * a Linux/Unix server or vice versa. 1290 * </p> 1291 */ 1292 public static final String[] DEFAULT_YARN_APPLICATION_CLASSPATH = { 1293 ApplicationConstants.Environment.HADOOP_CONF_DIR.$(), 1294 ApplicationConstants.Environment.HADOOP_COMMON_HOME.$() 1295 + "/share/hadoop/common/*", 1296 ApplicationConstants.Environment.HADOOP_COMMON_HOME.$() 1297 + "/share/hadoop/common/lib/*", 1298 ApplicationConstants.Environment.HADOOP_HDFS_HOME.$() 1299 + "/share/hadoop/hdfs/*", 1300 ApplicationConstants.Environment.HADOOP_HDFS_HOME.$() 1301 + "/share/hadoop/hdfs/lib/*", 1302 ApplicationConstants.Environment.HADOOP_YARN_HOME.$() 1303 + "/share/hadoop/yarn/*", 1304 ApplicationConstants.Environment.HADOOP_YARN_HOME.$() 1305 + "/share/hadoop/yarn/lib/*" }; 1306 1307 /** Container temp directory */ 1308 public static final String DEFAULT_CONTAINER_TEMP_DIR = "./tmp"; 1309 1310 public static final String IS_MINI_YARN_CLUSTER = YARN_PREFIX 1311 + "is.minicluster"; 1312 1313 public static final String YARN_MC_PREFIX = YARN_PREFIX + "minicluster."; 1314 1315 /** Whether to use fixed ports with the minicluster. */ 1316 public static final String YARN_MINICLUSTER_FIXED_PORTS = 1317 YARN_MC_PREFIX + "fixed.ports"; 1318 1319 /** 1320 * Default is false to be able to run tests concurrently without port 1321 * conflicts. 1322 */ 1323 public static final boolean DEFAULT_YARN_MINICLUSTER_FIXED_PORTS = false; 1324 1325 /** 1326 * Whether the NM should use RPC to connect to the RM. Default is false. 1327 * Can be set to true only when using fixed ports. 1328 */ 1329 public static final String YARN_MINICLUSTER_USE_RPC = YARN_MC_PREFIX + "use-rpc"; 1330 public static final boolean DEFAULT_YARN_MINICLUSTER_USE_RPC = false; 1331 1332 /** 1333 * Whether users are explicitly trying to control resource monitoring 1334 * configuration for the MiniYARNCluster. Disabled by default. 1335 */ 1336 public static final String YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING = 1337 YARN_MC_PREFIX + "control-resource-monitoring"; 1338 public static final boolean 1339 DEFAULT_YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING = false; 1340 1341 /** The log directory for the containers */ 1342 public static final String YARN_APP_CONTAINER_LOG_DIR = 1343 YARN_PREFIX + "app.container.log.dir"; 1344 1345 public static final String YARN_APP_CONTAINER_LOG_SIZE = 1346 YARN_PREFIX + "app.container.log.filesize"; 1347 1348 public static final String YARN_APP_CONTAINER_LOG_BACKUPS = 1349 YARN_PREFIX + "app.container.log.backups"; 1350 1351 /** Directory for fail flag files */ 1352 @Private 1353 public static final String YARN_AM_FAILURE_FLAG_DIR = 1354 YARN_PREFIX + "am-failure.flag.dir"; 1355 @Private 1356 public static final String DEFAULT_YARN_AM_FAILURE_FLAG_DIR = 1357 "/tmp/hadoop-yarn/fail"; 1358 1359 //////////////////////////////// 1360 // Timeline Service Configs 1361 //////////////////////////////// 1362 1363 public static final String TIMELINE_SERVICE_PREFIX = 1364 YARN_PREFIX + "timeline-service."; 1365 1366 1367 // mark app-history related configs @Private as application history is going 1368 // to be integrated into the timeline service 1369 @Private 1370 public static final String APPLICATION_HISTORY_PREFIX = 1371 TIMELINE_SERVICE_PREFIX + "generic-application-history."; 1372 1373 /** 1374 * The setting that controls whether application history service is 1375 * enabled or not. 1376 */ 1377 @Private 1378 public static final String APPLICATION_HISTORY_ENABLED = 1379 APPLICATION_HISTORY_PREFIX + "enabled"; 1380 @Private 1381 public static final boolean DEFAULT_APPLICATION_HISTORY_ENABLED = false; 1382 1383 /** Application history store class */ 1384 @Private 1385 public static final String APPLICATION_HISTORY_STORE = 1386 APPLICATION_HISTORY_PREFIX + "store-class"; 1387 1388 /** URI for FileSystemApplicationHistoryStore */ 1389 @Private 1390 public static final String FS_APPLICATION_HISTORY_STORE_URI = 1391 APPLICATION_HISTORY_PREFIX + "fs-history-store.uri"; 1392 1393 /** T-file compression types used to compress history data.*/ 1394 @Private 1395 public static final String FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE = 1396 APPLICATION_HISTORY_PREFIX + "fs-history-store.compression-type"; 1397 @Private 1398 public static final String DEFAULT_FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE = 1399 "none"; 1400 1401 /** The setting that controls whether timeline service is enabled or not. */ 1402 public static final String TIMELINE_SERVICE_ENABLED = 1403 TIMELINE_SERVICE_PREFIX + "enabled"; 1404 public static final boolean DEFAULT_TIMELINE_SERVICE_ENABLED = false; 1405 1406 /** host:port address for timeline service RPC APIs. */ 1407 public static final String TIMELINE_SERVICE_ADDRESS = 1408 TIMELINE_SERVICE_PREFIX + "address"; 1409 public static final int DEFAULT_TIMELINE_SERVICE_PORT = 10200; 1410 public static final String DEFAULT_TIMELINE_SERVICE_ADDRESS = "0.0.0.0:" 1411 + DEFAULT_TIMELINE_SERVICE_PORT; 1412 1413 /** The listening endpoint for the timeline service application.*/ 1414 public static final String TIMELINE_SERVICE_BIND_HOST = 1415 TIMELINE_SERVICE_PREFIX + "bind-host"; 1416 1417 /** The number of threads to handle client RPC API requests. */ 1418 public static final String TIMELINE_SERVICE_HANDLER_THREAD_COUNT = 1419 TIMELINE_SERVICE_PREFIX + "handler-thread-count"; 1420 public static final int DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT = 10; 1421 1422 1423 /** The address of the timeline service web application.*/ 1424 public static final String TIMELINE_SERVICE_WEBAPP_ADDRESS = 1425 TIMELINE_SERVICE_PREFIX + "webapp.address"; 1426 1427 public static final int DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT = 8188; 1428 public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS = 1429 "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT; 1430 1431 /** The https address of the timeline service web application.*/ 1432 public static final String TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS = 1433 TIMELINE_SERVICE_PREFIX + "webapp.https.address"; 1434 1435 public static final int DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT = 8190; 1436 public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS = 1437 "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT; 1438 1439 /** Timeline service store class */ 1440 public static final String TIMELINE_SERVICE_STORE = 1441 TIMELINE_SERVICE_PREFIX + "store-class"; 1442 1443 /** Timeline service enable data age off */ 1444 public static final String TIMELINE_SERVICE_TTL_ENABLE = 1445 TIMELINE_SERVICE_PREFIX + "ttl-enable"; 1446 1447 /** Timeline service length of time to retain data */ 1448 public static final String TIMELINE_SERVICE_TTL_MS = 1449 TIMELINE_SERVICE_PREFIX + "ttl-ms"; 1450 1451 public static final long DEFAULT_TIMELINE_SERVICE_TTL_MS = 1452 1000 * 60 * 60 * 24 * 7; 1453 1454 public static final String TIMELINE_SERVICE_LEVELDB_PREFIX = 1455 TIMELINE_SERVICE_PREFIX + "leveldb-timeline-store."; 1456 1457 /** Timeline service leveldb path */ 1458 public static final String TIMELINE_SERVICE_LEVELDB_PATH = 1459 TIMELINE_SERVICE_LEVELDB_PREFIX + "path"; 1460 1461 /** Timeline service leveldb read cache (uncompressed blocks) */ 1462 public static final String TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE = 1463 TIMELINE_SERVICE_LEVELDB_PREFIX + "read-cache-size"; 1464 1465 public static final long DEFAULT_TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE = 1466 100 * 1024 * 1024; 1467 1468 /** Timeline service leveldb start time read cache (number of entities) */ 1469 public static final String 1470 TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE = 1471 TIMELINE_SERVICE_LEVELDB_PREFIX + "start-time-read-cache-size"; 1472 1473 public static final int 1474 DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE = 10000; 1475 1476 /** Timeline service leveldb start time write cache (number of entities) */ 1477 public static final String 1478 TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE = 1479 TIMELINE_SERVICE_LEVELDB_PREFIX + "start-time-write-cache-size"; 1480 1481 public static final int 1482 DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE = 10000; 1483 1484 /** Timeline service leveldb interval to wait between deletion rounds */ 1485 public static final String TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS = 1486 TIMELINE_SERVICE_LEVELDB_PREFIX + "ttl-interval-ms"; 1487 1488 public static final long DEFAULT_TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS = 1489 1000 * 60 * 5; 1490 1491 /** The Kerberos principal for the timeline server.*/ 1492 public static final String TIMELINE_SERVICE_PRINCIPAL = 1493 TIMELINE_SERVICE_PREFIX + "principal"; 1494 1495 /** The Kerberos keytab for the timeline server.*/ 1496 public static final String TIMELINE_SERVICE_KEYTAB = 1497 TIMELINE_SERVICE_PREFIX + "keytab"; 1498 1499 /** Enables cross origin support for timeline server.*/ 1500 public static final String TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED = 1501 TIMELINE_SERVICE_PREFIX + "http-cross-origin.enabled"; 1502 1503 /** Default value for cross origin support for timeline server.*/ 1504 public static final boolean 1505 TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED_DEFAULT = false; 1506 1507 /** Timeline client settings */ 1508 public static final String TIMELINE_SERVICE_CLIENT_PREFIX = 1509 TIMELINE_SERVICE_PREFIX + "client."; 1510 1511 /** Timeline client call, max retries (-1 means no limit) */ 1512 public static final String TIMELINE_SERVICE_CLIENT_MAX_RETRIES = 1513 TIMELINE_SERVICE_CLIENT_PREFIX + "max-retries"; 1514 1515 public static final int DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES = 30; 1516 1517 /** Timeline client call, retry interval */ 1518 public static final String TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS = 1519 TIMELINE_SERVICE_CLIENT_PREFIX + "retry-interval-ms"; 1520 1521 public static final long 1522 DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS = 1000; 1523 1524 //////////////////////////////// 1525 // Other Configs 1526 //////////////////////////////// 1527 1528 /** 1529 * Use YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS instead. 1530 * The interval of the yarn client's querying application state after 1531 * application submission. The unit is millisecond. 1532 */ 1533 @Deprecated 1534 public static final String YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS = 1535 YARN_PREFIX + "client.app-submission.poll-interval"; 1536 1537 /** 1538 * The interval that the yarn client library uses to poll the completion 1539 * status of the asynchronous API of application client protocol. 1540 */ 1541 public static final String YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS = 1542 YARN_PREFIX + "client.application-client-protocol.poll-interval-ms"; 1543 public static final long DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS = 1544 200; 1545 1546 /** 1547 * The duration that the yarn client library waits, cumulatively across polls, 1548 * for an expected state change to occur. Defaults to -1, which indicates no 1549 * limit. 1550 */ 1551 public static final String YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS = 1552 YARN_PREFIX + "client.application-client-protocol.poll-timeout-ms"; 1553 public static final long DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS = 1554 -1; 1555 1556 /** 1557 * Max number of threads in NMClientAsync to process container management 1558 * events 1559 */ 1560 public static final String NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE = 1561 YARN_PREFIX + "client.nodemanager-client-async.thread-pool-max-size"; 1562 public static final int DEFAULT_NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE = 500; 1563 1564 /** 1565 * Maximum number of proxy connections to cache for node managers. If set 1566 * to a value greater than zero then the cache is enabled and the NMClient 1567 * and MRAppMaster will cache the specified number of node manager proxies. 1568 * There will be at max one proxy per node manager. Ex. configuring it to a 1569 * value of 5 will make sure that client will at max have 5 proxies cached 1570 * with 5 different node managers. These connections for these proxies will 1571 * be timed out if idle for more than the system wide idle timeout period. 1572 * Note that this could cause issues on large clusters as many connections 1573 * could linger simultaneously and lead to a large number of connection 1574 * threads. The token used for authentication will be used only at 1575 * connection creation time. If a new token is received then the earlier 1576 * connection should be closed in order to use the new token. This and 1577 * {@link YarnConfiguration#NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE} are related 1578 * and should be in sync (no need for them to be equal). 1579 * If the value of this property is zero then the connection cache is 1580 * disabled and connections will use a zero idle timeout to prevent too 1581 * many connection threads on large clusters. 1582 */ 1583 public static final String NM_CLIENT_MAX_NM_PROXIES = 1584 YARN_PREFIX + "client.max-cached-nodemanagers-proxies"; 1585 public static final int DEFAULT_NM_CLIENT_MAX_NM_PROXIES = 0; 1586 1587 /** Max time to wait to establish a connection to NM */ 1588 public static final String CLIENT_NM_CONNECT_MAX_WAIT_MS = 1589 YARN_PREFIX + "client.nodemanager-connect.max-wait-ms"; 1590 public static final long DEFAULT_CLIENT_NM_CONNECT_MAX_WAIT_MS = 1591 15 * 60 * 1000; 1592 1593 /** Time interval between each attempt to connect to NM */ 1594 public static final String CLIENT_NM_CONNECT_RETRY_INTERVAL_MS = 1595 YARN_PREFIX + "client.nodemanager-connect.retry-interval-ms"; 1596 public static final long DEFAULT_CLIENT_NM_CONNECT_RETRY_INTERVAL_MS 1597 = 10 * 1000; 1598 1599 public static final String YARN_HTTP_POLICY_KEY = YARN_PREFIX + "http.policy"; 1600 public static final String YARN_HTTP_POLICY_DEFAULT = HttpConfig.Policy.HTTP_ONLY 1601 .name(); 1602 1603 public static final String NODE_LABELS_PREFIX = YARN_PREFIX + "node-labels."; 1604 1605 /** 1606 * Class for RMNodeLabelsManager Please note this value should be consistent 1607 * in client nodes and RM node(s) 1608 */ 1609 public static final String RM_NODE_LABELS_MANAGER_CLASS = NODE_LABELS_PREFIX 1610 + "manager-class"; 1611 1612 /** URI for NodeLabelManager */ 1613 public static final String FS_NODE_LABELS_STORE_ROOT_DIR = NODE_LABELS_PREFIX 1614 + "fs-store.root-dir"; 1615 public static final String FS_NODE_LABELS_STORE_RETRY_POLICY_SPEC = 1616 NODE_LABELS_PREFIX + "fs-store.retry-policy-spec"; 1617 public static final String DEFAULT_FS_NODE_LABELS_STORE_RETRY_POLICY_SPEC = 1618 "2000, 500"; 1619 1620 public static final String AM_BLACKLISTING_ENABLED = 1621 YARN_PREFIX + "am.blacklisting.enabled"; 1622 public static final boolean DEFAULT_AM_BLACKLISTING_ENABLED = true; 1623 1624 public static final String AM_BLACKLISTING_DISABLE_THRESHOLD = 1625 YARN_PREFIX + "am.blacklisting.disable-failure-threshold"; 1626 public static final float DEFAULT_AM_BLACKLISTING_DISABLE_THRESHOLD = 0.8f; 1627 1628 1629 public YarnConfiguration() { 1630 super(); 1631 } 1632 1633 public YarnConfiguration(Configuration conf) { 1634 super(conf); 1635 if (! (conf instanceof YarnConfiguration)) { 1636 this.reloadConfiguration(); 1637 } 1638 } 1639 1640 @Private 1641 public static List<String> getServiceAddressConfKeys(Configuration conf) { 1642 return useHttps(conf) ? RM_SERVICES_ADDRESS_CONF_KEYS_HTTPS 1643 : RM_SERVICES_ADDRESS_CONF_KEYS_HTTP; 1644 } 1645 1646 /** 1647 * Get the socket address for <code>name</code> property as a 1648 * <code>InetSocketAddress</code>. On a HA cluster, 1649 * this fetches the address corresponding to the RM identified by 1650 * {@link #RM_HA_ID}. 1651 * @param name property name. 1652 * @param defaultAddress the default value 1653 * @param defaultPort the default port 1654 * @return InetSocketAddress 1655 */ 1656 @Override 1657 public InetSocketAddress getSocketAddr( 1658 String name, String defaultAddress, int defaultPort) { 1659 String address; 1660 if (HAUtil.isHAEnabled(this) && getServiceAddressConfKeys(this).contains(name)) { 1661 address = HAUtil.getConfValueForRMInstance(name, defaultAddress, this); 1662 } else { 1663 address = get(name, defaultAddress); 1664 } 1665 return NetUtils.createSocketAddr(address, defaultPort, name); 1666 } 1667 1668 @Override 1669 public InetSocketAddress updateConnectAddr(String name, 1670 InetSocketAddress addr) { 1671 String prefix = name; 1672 if (HAUtil.isHAEnabled(this)) { 1673 prefix = HAUtil.addSuffix(prefix, HAUtil.getRMHAId(this)); 1674 } 1675 return super.updateConnectAddr(prefix, addr); 1676 } 1677 1678 @Private 1679 public static int getRMDefaultPortNumber(String addressPrefix, 1680 Configuration conf) { 1681 if (addressPrefix.equals(YarnConfiguration.RM_ADDRESS)) { 1682 return YarnConfiguration.DEFAULT_RM_PORT; 1683 } else if (addressPrefix.equals(YarnConfiguration.RM_SCHEDULER_ADDRESS)) { 1684 return YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT; 1685 } else if (addressPrefix.equals(YarnConfiguration.RM_WEBAPP_ADDRESS)) { 1686 return YarnConfiguration.DEFAULT_RM_WEBAPP_PORT; 1687 } else if (addressPrefix.equals(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS)) { 1688 return YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT; 1689 } else if (addressPrefix 1690 .equals(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS)) { 1691 return YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT; 1692 } else if (addressPrefix.equals(YarnConfiguration.RM_ADMIN_ADDRESS)) { 1693 return YarnConfiguration.DEFAULT_RM_ADMIN_PORT; 1694 } else { 1695 throw new HadoopIllegalArgumentException( 1696 "Invalid RM RPC address Prefix: " + addressPrefix 1697 + ". The valid value should be one of " 1698 + getServiceAddressConfKeys(conf)); 1699 } 1700 } 1701 1702 public static boolean useHttps(Configuration conf) { 1703 return HttpConfig.Policy.HTTPS_ONLY == HttpConfig.Policy.fromString(conf 1704 .get(YARN_HTTP_POLICY_KEY, 1705 YARN_HTTP_POLICY_DEFAULT)); 1706 } 1707 1708 @Private 1709 public static String getClusterId(Configuration conf) { 1710 String clusterId = conf.get(YarnConfiguration.RM_CLUSTER_ID); 1711 if (clusterId == null) { 1712 throw new HadoopIllegalArgumentException("Configuration doesn't specify " + 1713 YarnConfiguration.RM_CLUSTER_ID); 1714 } 1715 return clusterId; 1716 } 1717}