Executor (VM) x Node = ((total number of Node's core) / 5) - 1
numExecutors (Number of executorns to launch for this session) = number of Nodes * Executor (VM) x Node
executorCores (Number of cores to use for each executor) = (total number of Node's core - 5 ) / Executor x Node
executorMemory (Amount of memory to use per executor process) = (yarn.nodemanager.resource.memory-mb - 1024) / (Executor (VM) x Node + 1)
Runtime.getRuntime.maxMemory * memoryFraction * safetyFraction.
executorMemory = ((yarn.nodemanager.resource.memory-mb - 1024) / (Executor (VM) x Node + 1)) * memoryFraction * safetyFraction.
driverCores = executorCores
driverMemory = executorMemory
yarn.nodemanager.resource.memory-mb = (52 - 2) * 1024 = 51200 MB
yarn.scheduler.maximum-allocation-mb = 20830 MB (Must be greater than executorMemory)
Executor x Node = (16) / 5 = 2
numExecutors = 2 * 4 = 8
executorCores = (16 - 5) / 2 = 5
executorMemory = ((51200 - 1024) / 3) * 0.6 * 0.9 = 16725,33 MB * 0.6 * 0.9 = 9031,68 MB
driverCores = 5
driverMemory = 16725,33 MB * 0.6 * 0.9 = 9031,68 MB
Partition = 5.722/64 = 89
yarn.nodemanager.resource.memory-mb = (52 - 2) * 1024 = 51200 MB
yarn.scheduler.maximum-allocation-mb = 20830 MB (Must be greater than executorMemory)
Executor x Node = ((16) / 5) - 1 = 2
numExecutors = 2 * 4 = 8
executorCores = (16 - 5) / 2 = 5
executorMemory = ((51200 - 1024) / 3) * 0.6 * 0.9 = 16725,33 MB * 0.6 * 0.9 = 9031,68 MB
driverCores = 5
driverMemory = 16725,33 MB * 0.6 * 0.9 = 9031,68 MB