/hudson-core/src/main/java/hudson/slaves/NodeProvisioner.java

  1/*
  2 * The MIT License
  3 * 
  4 * Copyright (c) 2004-2009, Sun Microsystems, Inc., Kohsuke Kawaguchi
  5 * 
  6 * Permission is hereby granted, free of charge, to any person obtaining a copy
  7 * of this software and associated documentation files (the "Software"), to deal
  8 * in the Software without restriction, including without limitation the rights
  9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 10 * copies of the Software, and to permit persons to whom the Software is
 11 * furnished to do so, subject to the following conditions:
 12 * 
 13 * The above copyright notice and this permission notice shall be included in
 14 * all copies or substantial portions of the Software.
 15 * 
 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 22 * THE SOFTWARE.
 23 */
 24package hudson.slaves;
 25
 26import hudson.model.LoadStatistics;
 27import hudson.model.Node;
 28import hudson.model.Hudson;
 29import hudson.model.MultiStageTimeSeries;
 30import hudson.model.Label;
 31import hudson.model.PeriodicWork;
 32import static hudson.model.LoadStatistics.DECAY;
 33import hudson.model.MultiStageTimeSeries.TimeScale;
 34import hudson.Extension;
 35
 36import java.awt.Color;
 37import java.util.concurrent.Future;
 38import java.util.concurrent.ExecutionException;
 39import java.util.List;
 40import java.util.Collection;
 41import java.util.ArrayList;
 42import java.util.Iterator;
 43import java.util.logging.Logger;
 44import java.util.logging.Level;
 45import java.io.IOException;
 46
 47/**
 48 * Uses the {@link LoadStatistics} and determines when we need to allocate
 49 * new {@link Node}s through {@link Cloud}.
 50 *
 51 * @author Kohsuke Kawaguchi
 52 */
 53public class NodeProvisioner {
 54    /**
 55     * The node addition activity in progress.
 56     */
 57    public static final class PlannedNode {
 58        /**
 59         * Used to display this planned node to UI. Should ideally include the identifier unique to the node
 60         * being provisioned (like the instance ID), but if such an identifier doesn't readily exist, this
 61         * can be just a name of the template being provisioned (like the machine image ID.)
 62         */
 63        //TODO: review and check whether we can do it private
 64        public final String displayName;
 65        public final Future<Node> future;
 66        public final int numExecutors;
 67
 68        public String getDisplayName() {
 69            return displayName;
 70        }
 71
 72        public Future<Node> getFuture() {
 73            return future;
 74        }
 75
 76        public int getNumExecutors() {
 77            return numExecutors;
 78        }
 79
 80        public PlannedNode(String displayName, Future<Node> future, int numExecutors) {
 81            if(displayName==null || future==null || numExecutors<1)  throw new IllegalArgumentException();
 82            this.displayName = displayName;
 83            this.future = future;
 84            this.numExecutors = numExecutors;
 85        }
 86    }
 87
 88    /**
 89     * Load for the label.
 90     */
 91    private final LoadStatistics stat;
 92
 93    /**
 94     * For which label are we working?
 95     * Null if this {@link NodeProvisioner} is working for the entire Hudson,
 96     * for jobs that are unassigned to any particular node.
 97     */
 98    private final Label label;
 99
100    private List<PlannedNode> pendingLaunches = new ArrayList<PlannedNode>();
101
102    /**
103     * Exponential moving average of the "planned capacity" over time, which is the number of
104     * additional executors being brought up.
105     *
106     * This is used to filter out high-frequency components from the planned capacity, so that
107     * the comparison with other low-frequency only variables won't leave spikes.
108     */
109    private final MultiStageTimeSeries plannedCapacitiesEMA =
110            new MultiStageTimeSeries(Messages._NodeProvisioner_EmptyString(),Color.WHITE,0,DECAY);
111
112    public NodeProvisioner(Label label, LoadStatistics loadStatistics) {
113        this.label = label;
114        this.stat = loadStatistics;
115    }
116
117    /**
118     * Periodically invoked to keep track of the load.
119     * Launches additional nodes if necessary.
120     */
121    private void update() {
122        Hudson hudson = Hudson.getInstance();
123
124        // clean up the cancelled launch activity, then count the # of executors that we are about to bring up.
125        float plannedCapacity = 0;
126        for (Iterator<PlannedNode> itr = pendingLaunches.iterator(); itr.hasNext();) {
127            PlannedNode f = itr.next();
128            if(f.future.isDone()) {
129                try {
130                    hudson.addNode(f.future.get());
131                    LOGGER.info(f.displayName+" provisioning successfully completed. We have now "+hudson.getComputers().length+" computer(s)");
132                } catch (InterruptedException e) {
133                    throw new AssertionError(e); // since we confirmed that the future is already done
134                } catch (ExecutionException e) {
135                    LOGGER.log(Level.WARNING, "Provisioned slave "+f.displayName+" failed to launch",e.getCause());
136                } catch (IOException e) {
137                    LOGGER.log(Level.WARNING, "Provisioned slave "+f.displayName+" failed to launch",e);
138                }
139                itr.remove();
140            } else
141                plannedCapacity += f.numExecutors;
142        }
143        plannedCapacitiesEMA.update(plannedCapacity);
144
145        /*
146            Here we determine how many additional slaves we need to keep up with the load (if at all),
147            which involves a simple math.
148
149            Broadly speaking, first we check that all the executors are fully utilized before attempting
150            to start any new slave (this also helps to ignore the temporary gap between different numbers,
151            as changes in them are not necessarily synchronized --- for example, there's a time lag between
152            when a slave launches (thus bringing the planned capacity down) and the time when its executors
153            pick up builds (thus bringing the queue length down.)
154
155            Once we confirm that, we compare the # of buildable items against the additional slaves
156            that are being brought online. If we have more jobs than our executors can handle, we'll launch a new slave.
157
158            So this computation involves three stats:
159
160              1. # of idle executors
161              2. # of jobs that are starving for executors
162              3. # of additional slaves being provisioned (planned capacities.)
163
164            To ignore a temporary surge/drop, we make conservative estimates on each one of them. That is,
165            we take the current snapshot value, and we take the current exponential moving average (EMA) value,
166            and use the max/min.
167
168            This is another measure to be robust against temporary surge/drop in those indicators, and helps
169            us avoid over-reacting to stats.
170
171            If we only use the snapshot value or EMA value, tests confirmed that the gap creates phantom
172            excessive loads and Hudson ends up firing excessive capacities. In a static system, over the time
173            EMA and the snapshot value becomes the same, so this makes sure that in a long run this conservative
174            estimate won't create a starvation.
175         */
176
177        int idleSnapshot = stat.computeIdleExecutors();
178        int totalSnapshot = stat.computeTotalExecutors();
179        float idle = Math.max(stat.getLatestIdleExecutors(TIME_SCALE), idleSnapshot);
180        if(idle<MARGIN) {
181            // make sure the system is fully utilized before attempting any new launch.
182
183            // this is the amount of work left to be done
184            float qlen = Math.min(stat.queueLength.getLatest(TIME_SCALE), stat.computeQueueLength());
185
186            // ... and this is the additional executors we've already provisioned.
187            plannedCapacity = Math.max(plannedCapacitiesEMA.getLatest(TIME_SCALE),plannedCapacity);
188
189            float excessWorkload = qlen - plannedCapacity;
190            float m = calcThresholdMargin(totalSnapshot);
191            if(excessWorkload>1-m) {// and there's more work to do...
192                LOGGER.fine("Excess workload "+excessWorkload+" detected. (planned capacity="+plannedCapacity+",Qlen="+qlen+",idle="+idle+"&"+idleSnapshot+",total="+totalSnapshot+"m,="+m+")");
193                for( Cloud c : hudson.clouds ) {
194                    if(excessWorkload<0)    break;  // enough slaves allocated
195
196                    // provisioning a new node should be conservative --- for example if exceeWorkload is 1.4,
197                    // we don't want to allocate two nodes but just one.
198                    // OTOH, because of the exponential decay, even when we need one slave, excess workload is always
199                    // something like 0.95, in which case we want to allocate one node.
200                    // so the threshold here is 1-MARGIN, and hence floor(excessWorkload+MARGIN) is needed to handle this.
201
202                    Collection<PlannedNode> additionalCapacities = c.provision(label, (int)Math.round(Math.floor(excessWorkload+m)));
203                    for (PlannedNode ac : additionalCapacities) {
204                        excessWorkload -= ac.numExecutors;
205                        LOGGER.info("Started provisioning "+ac.displayName+" from "+c.name+" with "+ac.numExecutors+" executors. Remaining excess workload:"+excessWorkload);
206                    }
207                    pendingLaunches.addAll(additionalCapacities);
208                }
209            }
210        }
211    }
212
213    /**
214     * Computes the threshold for triggering an allocation.
215     *
216     * <p>
217     * Because the excessive workload value is EMA, even when the snapshot value of the excessive
218     * workload is 1, the value never really gets to 1. So we need to introduce a notion of the margin M,
219     * where we provision a new node if the EMA of the excessive workload goes beyond 1-M (where M is a small value
220     * in the (0,1) range.)
221     *
222     * <p>
223     * M effectively controls how long Hudson waits until allocating a new node, in the face of workload.
224     * This delay is justified for absorbing temporary ups and downs, and can be interpreted as Hudson
225     * holding off provisioning in the hope that one of the existing nodes will become available.
226     *
227     * <p>
228     * M can be a constant value, but there's a benefit in adjusting M based on the total current capacity,
229     * based on the above justification; that is, if there's no existing capacity at all, holding off
230     * an allocation doesn't make much sense, as there won't be any executors available no matter how long we wait.
231     * On the other hand, if we have a large number of existing executors, chances are good that some
232     * of them become available &mdash; the chance gets better and better as the number of current total
233     * capacity increases.
234     *
235     * <p>
236     * Therefore, we compute the threshold margin as follows:
237     *
238     * <pre>
239     *   M(t) = M* + (M0 - M*) alpha ^ t
240     * </pre>
241     *
242     * ... where:
243     *
244     * <ul>
245     * <li>M* is the ultimate margin value that M(t) converges to with t->inf,
246     * <li>M0 is the value of M(0), the initial value.
247     * <li>alpha is the decay factor in (0,1). M(t) converges to M* faster if alpha is smaller.
248     * </ul>
249     */
250    private float calcThresholdMargin(int totalSnapshot) {
251        float f = (float) (MARGIN + (MARGIN0 - MARGIN) * Math.pow(MARGIN_DECAY, totalSnapshot));
252        // defensively ensure that the threshold margin is in (0,1)
253        f = Math.max(f,0);
254        f = Math.min(f,1);
255        return f;
256    }
257
258    /**
259     * Periodically invoke NodeProvisioners
260     */
261    @Extension
262    public static class NodeProvisionerInvoker extends PeriodicWork {
263        /**
264         * Give some initial warm up time so that statically connected slaves
265         * can be brought online before we start allocating more.
266         */
267    	 public static int INITIALDELAY = Integer.getInteger(NodeProvisioner.class.getName()+".initialDelay",LoadStatistics.CLOCK*10);
268    	 public static int RECURRENCEPERIOD = Integer.getInteger(NodeProvisioner.class.getName()+".recurrencePeriod",LoadStatistics.CLOCK);
269    	 
270        @Override
271        public long getInitialDelay() {
272            return INITIALDELAY;
273        }
274
275        public long getRecurrencePeriod() {
276            return RECURRENCEPERIOD;
277        }
278
279        @Override
280        protected void doRun() {
281            Hudson h = Hudson.getInstance();
282            h.overallNodeProvisioner.update();
283            for( Label l : h.getLabels() )
284                l.nodeProvisioner.update();
285        }
286    }
287
288    private static final Logger LOGGER = Logger.getLogger(NodeProvisioner.class.getName());
289    private static final float MARGIN = Integer.getInteger(NodeProvisioner.class.getName()+".MARGIN",10)/100f;
290    private static final float MARGIN0 = Math.max(MARGIN, getFloatSystemProperty(NodeProvisioner.class.getName()+".MARGIN0",0.5f));
291    private static final float MARGIN_DECAY = getFloatSystemProperty(NodeProvisioner.class.getName()+".MARGIN_DECAY",0.5f);
292
293    // TODO: picker should be selectable
294    private static final TimeScale TIME_SCALE = TimeScale.SEC10;
295
296    private static float getFloatSystemProperty(String propName, float defaultValue) {
297        String v = System.getProperty(propName);
298        if (v!=null)
299            try {
300                return Float.parseFloat(v);
301            } catch (NumberFormatException e) {
302                LOGGER.warning("Failed to parse a float value from system property "+propName+". value was "+v);
303            }
304        return defaultValue;
305    }
306}