/hudson-core/src/main/java/hudson/slaves/NodeProvisioner.java

http://github.com/hudson/hudson · Java · 306 lines · 134 code · 29 blank · 143 comment · 12 complexity · 71eef451350bbfde78ac062b31c71235 MD5 · raw file

  1. /*
  2. * The MIT License
  3. *
  4. * Copyright (c) 2004-2009, Sun Microsystems, Inc., Kohsuke Kawaguchi
  5. *
  6. * Permission is hereby granted, free of charge, to any person obtaining a copy
  7. * of this software and associated documentation files (the "Software"), to deal
  8. * in the Software without restriction, including without limitation the rights
  9. * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10. * copies of the Software, and to permit persons to whom the Software is
  11. * furnished to do so, subject to the following conditions:
  12. *
  13. * The above copyright notice and this permission notice shall be included in
  14. * all copies or substantial portions of the Software.
  15. *
  16. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  19. * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21. * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22. * THE SOFTWARE.
  23. */
  24. package hudson.slaves;
  25. import hudson.model.LoadStatistics;
  26. import hudson.model.Node;
  27. import hudson.model.Hudson;
  28. import hudson.model.MultiStageTimeSeries;
  29. import hudson.model.Label;
  30. import hudson.model.PeriodicWork;
  31. import static hudson.model.LoadStatistics.DECAY;
  32. import hudson.model.MultiStageTimeSeries.TimeScale;
  33. import hudson.Extension;
  34. import java.awt.Color;
  35. import java.util.concurrent.Future;
  36. import java.util.concurrent.ExecutionException;
  37. import java.util.List;
  38. import java.util.Collection;
  39. import java.util.ArrayList;
  40. import java.util.Iterator;
  41. import java.util.logging.Logger;
  42. import java.util.logging.Level;
  43. import java.io.IOException;
  44. /**
  45. * Uses the {@link LoadStatistics} and determines when we need to allocate
  46. * new {@link Node}s through {@link Cloud}.
  47. *
  48. * @author Kohsuke Kawaguchi
  49. */
  50. public class NodeProvisioner {
  51. /**
  52. * The node addition activity in progress.
  53. */
  54. public static final class PlannedNode {
  55. /**
  56. * Used to display this planned node to UI. Should ideally include the identifier unique to the node
  57. * being provisioned (like the instance ID), but if such an identifier doesn't readily exist, this
  58. * can be just a name of the template being provisioned (like the machine image ID.)
  59. */
  60. //TODO: review and check whether we can do it private
  61. public final String displayName;
  62. public final Future<Node> future;
  63. public final int numExecutors;
  64. public String getDisplayName() {
  65. return displayName;
  66. }
  67. public Future<Node> getFuture() {
  68. return future;
  69. }
  70. public int getNumExecutors() {
  71. return numExecutors;
  72. }
  73. public PlannedNode(String displayName, Future<Node> future, int numExecutors) {
  74. if(displayName==null || future==null || numExecutors<1) throw new IllegalArgumentException();
  75. this.displayName = displayName;
  76. this.future = future;
  77. this.numExecutors = numExecutors;
  78. }
  79. }
  80. /**
  81. * Load for the label.
  82. */
  83. private final LoadStatistics stat;
  84. /**
  85. * For which label are we working?
  86. * Null if this {@link NodeProvisioner} is working for the entire Hudson,
  87. * for jobs that are unassigned to any particular node.
  88. */
  89. private final Label label;
  90. private List<PlannedNode> pendingLaunches = new ArrayList<PlannedNode>();
  91. /**
  92. * Exponential moving average of the "planned capacity" over time, which is the number of
  93. * additional executors being brought up.
  94. *
  95. * This is used to filter out high-frequency components from the planned capacity, so that
  96. * the comparison with other low-frequency only variables won't leave spikes.
  97. */
  98. private final MultiStageTimeSeries plannedCapacitiesEMA =
  99. new MultiStageTimeSeries(Messages._NodeProvisioner_EmptyString(),Color.WHITE,0,DECAY);
  100. public NodeProvisioner(Label label, LoadStatistics loadStatistics) {
  101. this.label = label;
  102. this.stat = loadStatistics;
  103. }
  104. /**
  105. * Periodically invoked to keep track of the load.
  106. * Launches additional nodes if necessary.
  107. */
  108. private void update() {
  109. Hudson hudson = Hudson.getInstance();
  110. // clean up the cancelled launch activity, then count the # of executors that we are about to bring up.
  111. float plannedCapacity = 0;
  112. for (Iterator<PlannedNode> itr = pendingLaunches.iterator(); itr.hasNext();) {
  113. PlannedNode f = itr.next();
  114. if(f.future.isDone()) {
  115. try {
  116. hudson.addNode(f.future.get());
  117. LOGGER.info(f.displayName+" provisioning successfully completed. We have now "+hudson.getComputers().length+" computer(s)");
  118. } catch (InterruptedException e) {
  119. throw new AssertionError(e); // since we confirmed that the future is already done
  120. } catch (ExecutionException e) {
  121. LOGGER.log(Level.WARNING, "Provisioned slave "+f.displayName+" failed to launch",e.getCause());
  122. } catch (IOException e) {
  123. LOGGER.log(Level.WARNING, "Provisioned slave "+f.displayName+" failed to launch",e);
  124. }
  125. itr.remove();
  126. } else
  127. plannedCapacity += f.numExecutors;
  128. }
  129. plannedCapacitiesEMA.update(plannedCapacity);
  130. /*
  131. Here we determine how many additional slaves we need to keep up with the load (if at all),
  132. which involves a simple math.
  133. Broadly speaking, first we check that all the executors are fully utilized before attempting
  134. to start any new slave (this also helps to ignore the temporary gap between different numbers,
  135. as changes in them are not necessarily synchronized --- for example, there's a time lag between
  136. when a slave launches (thus bringing the planned capacity down) and the time when its executors
  137. pick up builds (thus bringing the queue length down.)
  138. Once we confirm that, we compare the # of buildable items against the additional slaves
  139. that are being brought online. If we have more jobs than our executors can handle, we'll launch a new slave.
  140. So this computation involves three stats:
  141. 1. # of idle executors
  142. 2. # of jobs that are starving for executors
  143. 3. # of additional slaves being provisioned (planned capacities.)
  144. To ignore a temporary surge/drop, we make conservative estimates on each one of them. That is,
  145. we take the current snapshot value, and we take the current exponential moving average (EMA) value,
  146. and use the max/min.
  147. This is another measure to be robust against temporary surge/drop in those indicators, and helps
  148. us avoid over-reacting to stats.
  149. If we only use the snapshot value or EMA value, tests confirmed that the gap creates phantom
  150. excessive loads and Hudson ends up firing excessive capacities. In a static system, over the time
  151. EMA and the snapshot value becomes the same, so this makes sure that in a long run this conservative
  152. estimate won't create a starvation.
  153. */
  154. int idleSnapshot = stat.computeIdleExecutors();
  155. int totalSnapshot = stat.computeTotalExecutors();
  156. float idle = Math.max(stat.getLatestIdleExecutors(TIME_SCALE), idleSnapshot);
  157. if(idle<MARGIN) {
  158. // make sure the system is fully utilized before attempting any new launch.
  159. // this is the amount of work left to be done
  160. float qlen = Math.min(stat.queueLength.getLatest(TIME_SCALE), stat.computeQueueLength());
  161. // ... and this is the additional executors we've already provisioned.
  162. plannedCapacity = Math.max(plannedCapacitiesEMA.getLatest(TIME_SCALE),plannedCapacity);
  163. float excessWorkload = qlen - plannedCapacity;
  164. float m = calcThresholdMargin(totalSnapshot);
  165. if(excessWorkload>1-m) {// and there's more work to do...
  166. LOGGER.fine("Excess workload "+excessWorkload+" detected. (planned capacity="+plannedCapacity+",Qlen="+qlen+",idle="+idle+"&"+idleSnapshot+",total="+totalSnapshot+"m,="+m+")");
  167. for( Cloud c : hudson.clouds ) {
  168. if(excessWorkload<0) break; // enough slaves allocated
  169. // provisioning a new node should be conservative --- for example if exceeWorkload is 1.4,
  170. // we don't want to allocate two nodes but just one.
  171. // OTOH, because of the exponential decay, even when we need one slave, excess workload is always
  172. // something like 0.95, in which case we want to allocate one node.
  173. // so the threshold here is 1-MARGIN, and hence floor(excessWorkload+MARGIN) is needed to handle this.
  174. Collection<PlannedNode> additionalCapacities = c.provision(label, (int)Math.round(Math.floor(excessWorkload+m)));
  175. for (PlannedNode ac : additionalCapacities) {
  176. excessWorkload -= ac.numExecutors;
  177. LOGGER.info("Started provisioning "+ac.displayName+" from "+c.name+" with "+ac.numExecutors+" executors. Remaining excess workload:"+excessWorkload);
  178. }
  179. pendingLaunches.addAll(additionalCapacities);
  180. }
  181. }
  182. }
  183. }
  184. /**
  185. * Computes the threshold for triggering an allocation.
  186. *
  187. * <p>
  188. * Because the excessive workload value is EMA, even when the snapshot value of the excessive
  189. * workload is 1, the value never really gets to 1. So we need to introduce a notion of the margin M,
  190. * where we provision a new node if the EMA of the excessive workload goes beyond 1-M (where M is a small value
  191. * in the (0,1) range.)
  192. *
  193. * <p>
  194. * M effectively controls how long Hudson waits until allocating a new node, in the face of workload.
  195. * This delay is justified for absorbing temporary ups and downs, and can be interpreted as Hudson
  196. * holding off provisioning in the hope that one of the existing nodes will become available.
  197. *
  198. * <p>
  199. * M can be a constant value, but there's a benefit in adjusting M based on the total current capacity,
  200. * based on the above justification; that is, if there's no existing capacity at all, holding off
  201. * an allocation doesn't make much sense, as there won't be any executors available no matter how long we wait.
  202. * On the other hand, if we have a large number of existing executors, chances are good that some
  203. * of them become available &mdash; the chance gets better and better as the number of current total
  204. * capacity increases.
  205. *
  206. * <p>
  207. * Therefore, we compute the threshold margin as follows:
  208. *
  209. * <pre>
  210. * M(t) = M* + (M0 - M*) alpha ^ t
  211. * </pre>
  212. *
  213. * ... where:
  214. *
  215. * <ul>
  216. * <li>M* is the ultimate margin value that M(t) converges to with t->inf,
  217. * <li>M0 is the value of M(0), the initial value.
  218. * <li>alpha is the decay factor in (0,1). M(t) converges to M* faster if alpha is smaller.
  219. * </ul>
  220. */
  221. private float calcThresholdMargin(int totalSnapshot) {
  222. float f = (float) (MARGIN + (MARGIN0 - MARGIN) * Math.pow(MARGIN_DECAY, totalSnapshot));
  223. // defensively ensure that the threshold margin is in (0,1)
  224. f = Math.max(f,0);
  225. f = Math.min(f,1);
  226. return f;
  227. }
  228. /**
  229. * Periodically invoke NodeProvisioners
  230. */
  231. @Extension
  232. public static class NodeProvisionerInvoker extends PeriodicWork {
  233. /**
  234. * Give some initial warm up time so that statically connected slaves
  235. * can be brought online before we start allocating more.
  236. */
  237. public static int INITIALDELAY = Integer.getInteger(NodeProvisioner.class.getName()+".initialDelay",LoadStatistics.CLOCK*10);
  238. public static int RECURRENCEPERIOD = Integer.getInteger(NodeProvisioner.class.getName()+".recurrencePeriod",LoadStatistics.CLOCK);
  239. @Override
  240. public long getInitialDelay() {
  241. return INITIALDELAY;
  242. }
  243. public long getRecurrencePeriod() {
  244. return RECURRENCEPERIOD;
  245. }
  246. @Override
  247. protected void doRun() {
  248. Hudson h = Hudson.getInstance();
  249. h.overallNodeProvisioner.update();
  250. for( Label l : h.getLabels() )
  251. l.nodeProvisioner.update();
  252. }
  253. }
  254. private static final Logger LOGGER = Logger.getLogger(NodeProvisioner.class.getName());
  255. private static final float MARGIN = Integer.getInteger(NodeProvisioner.class.getName()+".MARGIN",10)/100f;
  256. private static final float MARGIN0 = Math.max(MARGIN, getFloatSystemProperty(NodeProvisioner.class.getName()+".MARGIN0",0.5f));
  257. private static final float MARGIN_DECAY = getFloatSystemProperty(NodeProvisioner.class.getName()+".MARGIN_DECAY",0.5f);
  258. // TODO: picker should be selectable
  259. private static final TimeScale TIME_SCALE = TimeScale.SEC10;
  260. private static float getFloatSystemProperty(String propName, float defaultValue) {
  261. String v = System.getProperty(propName);
  262. if (v!=null)
  263. try {
  264. return Float.parseFloat(v);
  265. } catch (NumberFormatException e) {
  266. LOGGER.warning("Failed to parse a float value from system property "+propName+". value was "+v);
  267. }
  268. return defaultValue;
  269. }
  270. }