/atlassian-plugins-core/src/main/java/com/atlassian/plugin/manager/DefaultPluginManager.java

package com.atlassian.plugin.manager;

import com.atlassian.annotations.ExperimentalApi;
import com.atlassian.annotations.Internal;
import com.atlassian.instrumentation.operations.OpTimer;
import com.atlassian.plugin.ModuleCompleteKey;
import com.atlassian.plugin.ModuleDescriptor;
import com.atlassian.plugin.ModuleDescriptorFactory;
import com.atlassian.plugin.Plugin;
import com.atlassian.plugin.PluginAccessor;
import com.atlassian.plugin.PluginArtifact;
import com.atlassian.plugin.PluginController;
import com.atlassian.plugin.PluginDependencies;
import com.atlassian.plugin.PluginException;
import com.atlassian.plugin.PluginInformation;
import com.atlassian.plugin.PluginInstaller;
import com.atlassian.plugin.PluginInternal;
import com.atlassian.plugin.PluginParseException;
import com.atlassian.plugin.PluginRegistry;
import com.atlassian.plugin.PluginRestartState;
import com.atlassian.plugin.PluginState;
import com.atlassian.plugin.RevertablePluginInstaller;
import com.atlassian.plugin.SplitStartupPluginSystemLifecycle;
import com.atlassian.plugin.StateAware;
import com.atlassian.plugin.classloader.PluginsClassLoader;
import com.atlassian.plugin.descriptors.CannotDisable;
import com.atlassian.plugin.descriptors.UnloadableModuleDescriptor;
import com.atlassian.plugin.descriptors.UnloadableModuleDescriptorFactory;
import com.atlassian.plugin.event.NotificationException;
import com.atlassian.plugin.event.PluginEventListener;
import com.atlassian.plugin.event.PluginEventManager;
import com.atlassian.plugin.event.events.PluginContainerUnavailableEvent;
import com.atlassian.plugin.event.events.PluginDependentsChangedEvent;
import com.atlassian.plugin.event.events.PluginDisabledEvent;
import com.atlassian.plugin.event.events.PluginDisablingEvent;
import com.atlassian.plugin.event.events.PluginEnabledEvent;
import com.atlassian.plugin.event.events.PluginEnablingEvent;
import com.atlassian.plugin.event.events.PluginFrameworkDelayedEvent;
import com.atlassian.plugin.event.events.PluginFrameworkResumingEvent;
import com.atlassian.plugin.event.events.PluginFrameworkShutdownEvent;
import com.atlassian.plugin.event.events.PluginFrameworkShuttingDownEvent;
import com.atlassian.plugin.event.events.PluginFrameworkStartedEvent;
import com.atlassian.plugin.event.events.PluginFrameworkStartingEvent;
import com.atlassian.plugin.event.events.PluginFrameworkWarmRestartedEvent;
import com.atlassian.plugin.event.events.PluginFrameworkWarmRestartingEvent;
import com.atlassian.plugin.event.events.PluginInstalledEvent;
import com.atlassian.plugin.event.events.PluginInstallingEvent;
import com.atlassian.plugin.event.events.PluginModuleAvailableEvent;
import com.atlassian.plugin.event.events.PluginModuleDisabledEvent;
import com.atlassian.plugin.event.events.PluginModuleDisablingEvent;
import com.atlassian.plugin.event.events.PluginModuleEnabledEvent;
import com.atlassian.plugin.event.events.PluginModuleEnablingEvent;
import com.atlassian.plugin.event.events.PluginModuleUnavailableEvent;
import com.atlassian.plugin.event.events.PluginRefreshedEvent;
import com.atlassian.plugin.event.events.PluginUninstalledEvent;
import com.atlassian.plugin.event.events.PluginUninstallingEvent;
import com.atlassian.plugin.event.events.PluginUpgradedEvent;
import com.atlassian.plugin.event.events.PluginUpgradingEvent;
import com.atlassian.plugin.exception.NoOpPluginExceptionInterception;
import com.atlassian.plugin.exception.PluginExceptionInterception;
import com.atlassian.plugin.impl.UnloadablePlugin;
import com.atlassian.plugin.impl.UnloadablePluginFactory;
import com.atlassian.plugin.instrumentation.PluginSystemInstrumentation;
import com.atlassian.plugin.instrumentation.Timer;
import com.atlassian.plugin.loaders.DiscardablePluginLoader;
import com.atlassian.plugin.loaders.DynamicPluginLoader;
import com.atlassian.plugin.loaders.PermissionCheckingPluginLoader;
import com.atlassian.plugin.loaders.PluginLoader;
import com.atlassian.plugin.metadata.ClasspathFilePluginMetadata;
import com.atlassian.plugin.metadata.DefaultRequiredPluginValidator;
import com.atlassian.plugin.metadata.RequiredPluginValidator;
import com.atlassian.plugin.predicate.EnabledModulePredicate;
import com.atlassian.plugin.predicate.EnabledPluginPredicate;
import com.atlassian.plugin.predicate.ModuleOfClassPredicate;
import com.atlassian.plugin.scope.ScopeManager;
import com.atlassian.plugin.util.PluginUtils;
import com.atlassian.plugin.util.VersionStringComparator;
import com.google.common.annotations.VisibleForTesting;
import org.dom4j.Element;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nullable;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Predicate;
import java.util.stream.Stream;

import static com.atlassian.plugin.PluginDependencies.Type.DYNAMIC;
import static com.atlassian.plugin.PluginDependencies.Type.MANDATORY;
import static com.atlassian.plugin.PluginDependencies.Type.OPTIONAL;
import static com.atlassian.plugin.impl.AbstractPlugin.cleanVersionString;
import static com.atlassian.plugin.manager.SafeModeManager.START_ALL_PLUGINS;
import static com.atlassian.plugin.util.Assertions.notNull;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.Maps.filterKeys;
import static java.util.Arrays.asList;
import static java.util.Collections.emptyList;
import static java.util.Collections.max;
import static java.util.Collections.reverse;
import static java.util.Collections.singleton;
import static java.util.Collections.singletonList;
import static java.util.Collections.unmodifiableMap;
import static java.util.Collections.unmodifiableSet;
import static java.util.stream.Collectors.collectingAndThen;
import static java.util.stream.Collectors.partitioningBy;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;
import static java.util.stream.StreamSupport.stream;

/**
 * This implementation delegates the initiation and classloading of plugins to a
 * list of {@link com.atlassian.plugin.loaders.PluginLoader}s and records the
 * state of plugins in a
 * {@link com.atlassian.plugin.manager.PluginPersistentStateStore}.
 * <p>
 * This class is responsible for enabling and disabling plugins and plugin
 * modules and reflecting these state changes in the PluginPersistentStateStore.
 * <p>
 * An interesting quirk in the design is that
 * {@link #installPlugins(com.atlassian.plugin.PluginArtifact[])} explicitly stores
 * the plugin via a {@link com.atlassian.plugin.PluginInstaller}, whereas
 * {@link #uninstall(Plugin)} relies on the underlying
 * {@link com.atlassian.plugin.loaders.PluginLoader} to remove the plugin if
 * necessary.
 */
public class DefaultPluginManager implements PluginController, PluginAccessor, SplitStartupPluginSystemLifecycle {
    private static final Logger log = LoggerFactory.getLogger(DefaultPluginManager.class);

    @Internal
    public static String getStartupOverrideFileProperty() {
        return DefaultPluginManager.class.getName() + ".startupOverrideFile";
    }

    @Internal
    public static String getLateStartupEnableRetryProperty() {
        return DefaultPluginManager.class.getName() + ".lateStartupEnableRetry";
    }

    @Internal
    public static String getMinimumPluginVersionsFileProperty() {
        return DefaultPluginManager.class.getName() + ".minimumPluginVersionsFile";
    }

    private final List<DiscardablePluginLoader> pluginLoaders;
    private final PluginPersistentStateModifier persistentStateModifier;
    private final ModuleDescriptorFactory moduleDescriptorFactory;
    private final PluginEventManager pluginEventManager;

    private final PluginRegistry.ReadWrite pluginRegistry;
    private final PluginsClassLoader classLoader;
    private final PluginEnabler pluginEnabler;
    private final StateTracker tracker;

    private final boolean verifyRequiredPlugins;

    /**
     * Predicate for identify plugins which are not loaded at initialization.
     * <p>
     * This can be used to support two-phase "tenant aware" startup.
     */
    private final Predicate<Plugin> delayLoadOf;

    /**
     * Installer used for storing plugins. Used by
     * {@link #installPlugins(PluginArtifact[])}.
     */
    private RevertablePluginInstaller pluginInstaller;

    /**
     * Stores {@link Plugin}s as a key and {@link PluginLoader} as a value.
     */
    private final Map<Plugin, PluginLoader> installedPluginsToPluginLoader;

    /**
     * A map to pass information from {@link #earlyStartup} to {@link #addPlugins} without an API change.
     * <p>
     * This map allows earlyStartup to specify a plugin-specific loader for the list of plugins provided to addPlugins. It is only
     * valid when addPlugins is called from earlyStartup or lateStartup. Products may override addPlugins (to add clustering
     * behaviour for example), so fixing the API here is a more involved change.
     */
    private final Map<Plugin, DiscardablePluginLoader> candidatePluginsToPluginLoader;

    /**
     * A list of plugins to be re-enabled when adding plugins.
     * <p>
     * As with {@link #candidatePluginsToPluginLoader}, this is passing data from {@link #earlyStartup} and {@link #lateStartup} to
     * {@link #addPlugins} without an API change.
     */
    private final Collection<Plugin> additionalPluginsToEnable;

    private final DefaultPluginManagerJmxBridge defaultPluginManagerJmxBridge;

    /**
     * The list of plugins whose load was delayed.
     */
    private final List<Plugin> delayedPlugins;

    /**
     * A map of plugins which need to be removed during lateStartup.
     * <p>
     * The plugins which need to be removed on restart are discovered during earlyStartup so we can avoid installing them
     * when not required, but cannot be removed until persistence is available during late startup.
     */
    private final Map<Plugin, DiscardablePluginLoader> delayedPluginRemovalsToLoader;

    private final SafeModuleExtractor safeModuleExtractor;
    private final SafeModeManager safeModeManager;
    private final PluginTransactionContext pluginTransactionContext;

    public DefaultPluginManager(
            final PluginPersistentStateStore store,
            final List<PluginLoader> pluginLoaders,
            final ModuleDescriptorFactory moduleDescriptorFactory,
            final PluginEventManager pluginEventManager) {
        this(newBuilder()
                .withStore(store)
                .withPluginLoaders(pluginLoaders)
                .withModuleDescriptorFactory(moduleDescriptorFactory)
                .withPluginEventManager(pluginEventManager));
    }

    public DefaultPluginManager(
            final PluginPersistentStateStore store,
            final List<PluginLoader> pluginLoaders,
            final ModuleDescriptorFactory moduleDescriptorFactory,
            final PluginEventManager pluginEventManager,
            final PluginExceptionInterception pluginExceptionInterception) {
        this(newBuilder()
                .withStore(store)
                .withPluginLoaders(pluginLoaders)
                .withModuleDescriptorFactory(moduleDescriptorFactory)
                .withPluginEventManager(pluginEventManager)
                .withPluginExceptionInterception(pluginExceptionInterception));
    }

    public DefaultPluginManager(
            final PluginPersistentStateStore store,
            final List<PluginLoader> pluginLoaders,
            final ModuleDescriptorFactory moduleDescriptorFactory,
            final PluginEventManager pluginEventManager,
            final boolean verifyRequiredPlugins) {
        this(newBuilder()
                .withStore(store)
                .withPluginLoaders(pluginLoaders)
                .withModuleDescriptorFactory(moduleDescriptorFactory)
                .withPluginEventManager(pluginEventManager)
                .withVerifyRequiredPlugins(verifyRequiredPlugins));
    }

    @ExperimentalApi
    public DefaultPluginManager(
            final PluginPersistentStateStore store,
            final List<PluginLoader> pluginLoaders,
            final ModuleDescriptorFactory moduleDescriptorFactory,
            final PluginEventManager pluginEventManager,
            final Predicate<Plugin> delayLoadOf) {
        this(newBuilder()
                .withStore(store)
                .withPluginLoaders(pluginLoaders)
                .withModuleDescriptorFactory(moduleDescriptorFactory)
                .withPluginEventManager(pluginEventManager)
                .withDelayLoadOf(delayLoadOf));
    }

    @ExperimentalApi
    public DefaultPluginManager(
            final PluginPersistentStateStore store,
            final List<PluginLoader> pluginLoaders,
            final ModuleDescriptorFactory moduleDescriptorFactory,
            final PluginEventManager pluginEventManager,
            final PluginExceptionInterception pluginExceptionInterception,
            final Predicate<Plugin> delayLoadOf) {
        this(newBuilder()
                .withStore(store)
                .withPluginLoaders(pluginLoaders)
                .withModuleDescriptorFactory(moduleDescriptorFactory)
                .withPluginEventManager(pluginEventManager)
                .withPluginExceptionInterception(pluginExceptionInterception)
                .withDelayLoadOf(delayLoadOf));
    }

    public DefaultPluginManager(
            final PluginPersistentStateStore store,
            final List<PluginLoader> pluginLoaders,
            final ModuleDescriptorFactory moduleDescriptorFactory,
            final PluginEventManager pluginEventManager,
            final PluginExceptionInterception pluginExceptionInterception,
            final boolean verifyRequiredPlugins) {
        this(newBuilder()
                .withStore(store)
                .withPluginLoaders(pluginLoaders)
                .withModuleDescriptorFactory(moduleDescriptorFactory)
                .withPluginEventManager(pluginEventManager)
                .withPluginExceptionInterception(pluginExceptionInterception)
                .withVerifyRequiredPlugins(verifyRequiredPlugins));
    }

    public DefaultPluginManager(
            final PluginPersistentStateStore store,
            final List<PluginLoader> pluginLoaders,
            final ModuleDescriptorFactory moduleDescriptorFactory,
            final PluginEventManager pluginEventManager,
            final PluginExceptionInterception pluginExceptionInterception,
            final boolean verifyRequiredPlugins,
            final Predicate<Plugin> delayLoadOf) {
        this(newBuilder()
                .withStore(store)
                .withPluginLoaders(pluginLoaders)
                .withModuleDescriptorFactory(moduleDescriptorFactory)
                .withPluginEventManager(pluginEventManager)
                .withPluginExceptionInterception(pluginExceptionInterception)
                .withVerifyRequiredPlugins(verifyRequiredPlugins)
                .withDelayLoadOf(delayLoadOf));
    }

    protected DefaultPluginManager(Builder<? extends Builder> builder) {
        this.safeModeManager = builder.safeModeManager;
        this.pluginLoaders = toPermissionCheckingPluginLoaders(notNull("Plugin Loaders list", builder.pluginLoaders));
        this.persistentStateModifier = new PluginPersistentStateModifier(notNull("PluginPersistentStateStore", builder.store));
        this.moduleDescriptorFactory = notNull("ModuleDescriptorFactory", builder.moduleDescriptorFactory);
        this.pluginEventManager = notNull("PluginEventManager", builder.pluginEventManager);
        this.pluginEnabler = new PluginEnabler(this, this, notNull("PluginExceptionInterception", builder.pluginExceptionInterception));
        this.verifyRequiredPlugins = builder.verifyRequiredPlugins;
        this.delayLoadOf = wrapDelayPredicateWithOverrides(builder.delayLoadOf);
        this.pluginRegistry = builder.pluginRegistry;
        this.classLoader = builder.pluginAccessor.map(pa -> PluginsClassLoader.class.cast(pa.getClassLoader()))
                .orElseGet(() -> new PluginsClassLoader(null, this, this.pluginEventManager));
        this.tracker = new StateTracker();
        this.pluginInstaller = new NoOpRevertablePluginInstaller(new UnsupportedPluginInstaller());
        this.installedPluginsToPluginLoader = new HashMap<>();
        this.candidatePluginsToPluginLoader = new HashMap<>();
        this.additionalPluginsToEnable = new ArrayList<>();
        this.delayedPlugins = new ArrayList<>();
        this.delayedPluginRemovalsToLoader = new HashMap<>();

        this.pluginEventManager.register(this);
        this.defaultPluginManagerJmxBridge = new DefaultPluginManagerJmxBridge(this);
        this.safeModuleExtractor = new SafeModuleExtractor(this);
        this.pluginTransactionContext = new PluginTransactionContext(this.pluginEventManager);
    }

    @SuppressWarnings("unchecked")
    public static class Builder<T extends Builder<?>> {
        private PluginPersistentStateStore store;
        private List<PluginLoader> pluginLoaders = new ArrayList<>();
        private ModuleDescriptorFactory moduleDescriptorFactory;
        private PluginEventManager pluginEventManager;
        private PluginExceptionInterception pluginExceptionInterception = NoOpPluginExceptionInterception.NOOP_INTERCEPTION;
        private boolean verifyRequiredPlugins = false;
        private Predicate<Plugin> delayLoadOf = p -> false;
        private PluginRegistry.ReadWrite pluginRegistry = new PluginRegistryImpl();
        private Optional<PluginAccessor> pluginAccessor = Optional.empty();
        private SafeModeManager safeModeManager = START_ALL_PLUGINS;

        public T withSafeModeManager(final SafeModeManager safeModeManager) {
            this.safeModeManager = safeModeManager;
            return (T) this;
        }

        public T withStore(final PluginPersistentStateStore store) {
            this.store = store;
            return (T) this;
        }

        public T withPluginLoaders(final List<PluginLoader> pluginLoaders) {
            this.pluginLoaders.addAll(pluginLoaders);
            return (T) this;
        }

        public T withPluginLoader(final PluginLoader pluginLoader) {
            this.pluginLoaders.add(pluginLoader);
            return (T) this;
        }

        public T withModuleDescriptorFactory(final ModuleDescriptorFactory moduleDescriptorFactory) {
            this.moduleDescriptorFactory = moduleDescriptorFactory;
            return (T) this;
        }

        public T withPluginEventManager(final PluginEventManager pluginEventManager) {
            this.pluginEventManager = pluginEventManager;
            return (T) this;
        }

        public T withPluginExceptionInterception(final PluginExceptionInterception pluginExceptionInterception) {
            this.pluginExceptionInterception = pluginExceptionInterception;
            return (T) this;
        }

        public T withVerifyRequiredPlugins(final boolean verifyRequiredPlugins) {
            this.verifyRequiredPlugins = verifyRequiredPlugins;
            return (T) this;
        }

        public T withDelayLoadOf(final Predicate<Plugin> delayLoadOf) {
            this.delayLoadOf = delayLoadOf;
            return (T) this;
        }

        public T withPluginRegistry(final PluginRegistry.ReadWrite pluginRegistry) {
            this.pluginRegistry = pluginRegistry;
            return (T) this;
        }

        public T withPluginAccessor(PluginAccessor pluginAccessor) {
            this.pluginAccessor = Optional.of(pluginAccessor);
            return (T) this;
        }

        /**
         * @deprecated in 5.0 for removal in 6.0 when {@link ScopeManager} will be removed.
         */
        @Deprecated
        public T withScopeManager(ScopeManager ignored) {
            return (T) this;
        }

        public DefaultPluginManager build() {
            return new DefaultPluginManager(this);
        }
    }

    public static Builder<? extends Builder<?>> newBuilder() {
        return new Builder<>();
    }

    private static Iterable<String> toPluginKeys(Iterable<Plugin> plugins) {
        return stream(plugins.spliterator(), false).map(Plugin::getKey).collect(toList());
    }

    private List<DiscardablePluginLoader> toPermissionCheckingPluginLoaders(final List<PluginLoader> fromIterable) {
        return fromIterable.stream().map(PermissionCheckingPluginLoader::new).collect(toList());
    }

    private Predicate<Plugin> wrapDelayPredicateWithOverrides(final Predicate<Plugin> pluginPredicate) {
        final Map<String, String> startupOverridesMap = parseFileNamedByPropertyAsMap(getStartupOverrideFileProperty());

        return new Predicate<Plugin>() {
            @Override
            public boolean test(final Plugin plugin) {
                final String pluginKey = plugin.getKey();

                // Check startup overrides for plugin startup declaration
                final String stringFromFile = startupOverridesMap.get(pluginKey);
                final Optional<Boolean> parsedFromFile = parseStartupToDelay(stringFromFile, pluginKey, "override file");
                if (parsedFromFile.isPresent()) {
                    return parsedFromFile.get();
                }

                // Check PluginInformation for plugin startup declaration
                final PluginInformation pluginInformation = plugin.getPluginInformation();
                final String stringFromInformation = (null != pluginInformation) ? pluginInformation.getStartup() : null;
                final Optional<Boolean> parsedFromInformation = parseStartupToDelay(stringFromInformation, pluginKey, "PluginInformation");
                // If no plugin startup information, use product supplied predicate
                return parsedFromInformation.orElseGet(() -> pluginPredicate.test(plugin));
            }

            private Optional<Boolean> parseStartupToDelay(final String startup, final String pluginKey, final String source) {
                if (null != startup) {
                    if ("early".equals(startup)) {
                        return Optional.of(Boolean.FALSE);
                    }
                    if ("late".equals(startup)) {
                        return Optional.of(Boolean.TRUE);
                    }

                    log.warn("Unknown startup '{}' for plugin '{}' from {}", startup, pluginKey, source);
                    // and fall through
                }

                return Optional.empty();
            }
        };
    }

    private Map<String, String> parseFileNamedByPropertyAsMap(final String property) {
        final Properties properties = new Properties();
        final String fileName = System.getProperty(property);
        if (null != fileName) {
            try (FileInputStream inStream = new FileInputStream(fileName)) {
                properties.load(inStream);
            } catch (final IOException eio) {
                log.warn("Failed to load file named by property {}, that is '{}': {}",
                        property, fileName, eio);
            }
        }
        return unmodifiableMap(propertiesToMap(properties));
    }

    private Map<String, String> propertiesToMap(final Properties properties) {
        final Map<String, String> propertiesMap = new HashMap<>();
        properties.forEach((key, value) -> propertiesMap.put((String) key, (String) value));
        return propertiesMap;
    }

    @Override
    public void init() {
        pluginTransactionContext.wrap(() -> {
            earlyStartup();
            lateStartup();
        });
    }

    @ExperimentalApi
    @Override
    public void earlyStartup() {
        pluginTransactionContext.wrap(() -> {
            try (Timer timer = PluginSystemInstrumentation.instance().pullSingleTimer("earlyStartup")) {
                log.info("Plugin system earlyStartup begun");
                tracker.setState(StateTracker.State.STARTING);

                defaultPluginManagerJmxBridge.register();
                broadcastIgnoreError(new PluginFrameworkStartingEvent(this, this));
                pluginInstaller.clearBackups();
                final PluginPersistentState pluginPersistentState = getState();
                final Map<String, List<Plugin>> candidatePluginKeyToVersionedPlugins = new TreeMap<>();
                for (final DiscardablePluginLoader loader : pluginLoaders) {
                    if (loader == null) {
                        continue;
                    }

                    final Iterable<Plugin> possiblePluginsToLoad = loader.loadAllPlugins(moduleDescriptorFactory);

                    if (log.isDebugEnabled()) {
                        log.debug("Found {} plugins to possibly load: {}",
                                stream(possiblePluginsToLoad.spliterator(), false).count(), toPluginKeys(possiblePluginsToLoad));
                    }

                    for (final Plugin plugin : possiblePluginsToLoad) {
                        if (pluginPersistentState.getPluginRestartState(plugin.getKey()) == PluginRestartState.REMOVE) {
                            log.info("Plugin {} was marked to be removed on restart. Removing now.", plugin);
                            // We need to remove the plugin and clear its state, but we don't want to do any persistence until
                            // late startup. We may as well delay PluginLoader#removePlugin also, as it doesn't hurt,
                            // and it makes fewer assumptions about how the product persists the plugins themselves.
                            delayedPluginRemovalsToLoader.put(plugin, loader);
                        } else {
                            // We need the loaders for installed plugins so we can issue removePlugin() when
                            // the plugin is unloaded. So anything we didn't remove above, we put into the
                            // candidatePluginsToPluginLoader map. All of these need either removePlugin()
                            // or discardPlugin() for resource management.
                            candidatePluginsToPluginLoader.put(plugin, loader);
                            List<Plugin> plugins = candidatePluginKeyToVersionedPlugins.computeIfAbsent(plugin.getKey(), key -> new ArrayList<>());
                            plugins.add(plugin);
                        }
                    }
                }

                final List<Plugin> pluginsToInstall = new ArrayList<>();
                for (final List<Plugin> plugins : candidatePluginKeyToVersionedPlugins.values()) {
                    final Plugin plugin = max(plugins, Comparator.naturalOrder());
                    if (plugins.size() > 1) {
                        log.debug("Plugin {} contained multiple versions. installing version {}.", plugin.getKey(), plugin.getPluginInformation().getVersion());
                    }
                    pluginsToInstall.add(plugin);
                }

                // Partition pluginsToInstall into immediatePlugins that we install now, and delayedPlugins
                // that we install when instructed by a call to lateStartup.
                // In later versions of Guava, ImmutableListMultimap.index is another way to slice this if
                // you convert the PluginPredicate to a function. Whether or this is cleaner is a bit moot
                // since we can't use it yet anyway.
                final List<Plugin> immediatePlugins = new ArrayList<>();
                for (final Plugin plugin : pluginsToInstall) {
                    if (delayLoadOf.test(plugin)) {
                        delayedPlugins.add(plugin);
                    } else {
                        immediatePlugins.add(plugin);
                    }
                }

                // Install the non-delayed plugins
                addPlugins(null, immediatePlugins);

                // For each immediatePlugins, addPlugins has either called removePlugin()/discardPlugin()
                // for its loader, or is tracking it in installedPluginsToPluginLoader for a removePlugin()
                // when it is uninstalled (either via upgrade or shutdown).
                for (final Plugin plugin : immediatePlugins) {
                    candidatePluginsToPluginLoader.remove(plugin);
                }

                if (Boolean.getBoolean(getLateStartupEnableRetryProperty())) {
                    for (final Plugin plugin : immediatePlugins) {
                        // For each plugin that didn't enable but should have, make a note so we can try them again later. It's a little
                        // bit vexing that we are checking the persistent state here again just after addPlugins did it, but refactoring this
                        // stuff is fraught with API danger, so this can wait. PLUG-1116 seems like a time this might be worth revisiting.
                        if ((PluginState.ENABLED != plugin.getPluginState()) && pluginPersistentState.isEnabled(plugin)) {
                            additionalPluginsToEnable.add(plugin);
                        }
                    }
                    if (!additionalPluginsToEnable.isEmpty()) {
                        // Let people know we're going to retry them, so there is information in the logs about this near the resolution errors
                        log.warn("Failed to enable some ({}) early plugins, will fallback during lateStartup. Plugins: {}",
                                additionalPluginsToEnable.size(), additionalPluginsToEnable);
                    }
                }

                // We need to keep candidatePluginsToPluginLoader populated with the delayedPlugins so that
                // addPlugins can do the right thing when called by lateStartup. However, we want to
                // discardPlugin anything we don't need so that loaders can release resources. So move what
                // we need later from candidatePluginsToPluginLoader to delayedPluginsLoaders.
                final Map<Plugin, DiscardablePluginLoader> delayedPluginsLoaders = new HashMap<>();
                for (final Plugin plugin : delayedPlugins) {
                    final DiscardablePluginLoader loader = candidatePluginsToPluginLoader.remove(plugin);
                    delayedPluginsLoaders.put(plugin, loader);
                }
                // Now candidatePluginsToPluginLoader contains exactly Plugins returned by loadAllPlugins
                // for which we didn't removePlugin() above, didn't pass on addPlugins(), and won't handle
                // in loadDelayedPlugins. So loaders can release resources, we discardPlugin() these.
                for (final Map.Entry<Plugin, DiscardablePluginLoader> entry : candidatePluginsToPluginLoader.entrySet()) {
                    final Plugin plugin = entry.getKey();
                    final DiscardablePluginLoader loader = entry.getValue();
                    loader.discardPlugin(plugin);
                }
                // Finally, make candidatePluginsToPluginLoader contain what loadDelayedPlugins needs.
                candidatePluginsToPluginLoader.clear();
                candidatePluginsToPluginLoader.putAll(delayedPluginsLoaders);
                tracker.setState(StateTracker.State.DELAYED);

                logTime(timer, "Plugin system earlyStartup ended");

                broadcastIgnoreError(new PluginFrameworkDelayedEvent(this, this));
            }
        });
    }

    @ExperimentalApi
    @Override
    public void lateStartup() {
        pluginTransactionContext.wrap(() -> {
            try (Timer timer = PluginSystemInstrumentation.instance().pullSingleTimer("lateStartup")) {
                log.info("Plugin system lateStartup begun");

                tracker.setState(StateTracker.State.RESUMING);
                broadcastIgnoreError(new PluginFrameworkResumingEvent(this, this));

                addPlugins(null, delayedPlugins);
                delayedPlugins.clear();
                candidatePluginsToPluginLoader.clear();

                persistentStateModifier.clearPluginRestartState();
                for (final Map.Entry<Plugin, DiscardablePluginLoader> entry : delayedPluginRemovalsToLoader.entrySet()) {
                    final Plugin plugin = entry.getKey();
                    final DiscardablePluginLoader loader = entry.getValue();
                    // Remove the plugin from the loader, and discard saved state (see PLUG-13).
                    loader.removePlugin(plugin);
                    persistentStateModifier.removeState(plugin);
                }
                delayedPluginRemovalsToLoader.clear();

                logTime(timer, "Plugin system lateStartup ended");

                tracker.setState(StateTracker.State.STARTED);
                if (verifyRequiredPlugins) {
                    validateRequiredPlugins();
                }
                broadcastIgnoreError(new PluginFrameworkStartedEvent(this, this));
            }
        });
    }

    private void validateRequiredPlugins() {
        final RequiredPluginValidator validator = new DefaultRequiredPluginValidator(this, new ClasspathFilePluginMetadata());
        final Collection<String> errors = validator.validate();
        if (errors.size() > 0) {
            log.error("Unable to validate required plugins or modules - plugin system shutting down");
            log.error("Failures:");
            for (final String error : errors) {
                log.error("\t{}", error);
            }
            shutdown();
            throw new PluginException("Unable to validate required plugins or modules");
        }
    }

    /**
     * @param timer the timer
     * @param message Message to log as info
     */
    private void logTime(Timer timer, String message) {
        Optional<OpTimer> opTimer = timer.getOpTimer();
        if (opTimer.isPresent()) {
            long elapsedSeconds = opTimer.get().snapshot().getElapsedTotalTime(TimeUnit.SECONDS);
            log.info("{} in {}s", message, elapsedSeconds);
        } else {
            log.info(message);
        }
    }

    /**
     * Fires the shutdown event
     *
     * @throws IllegalStateException if already shutdown or already in the
     *                               process of shutting down.
     * @since 2.0.0
     */
    @Override
    public void shutdown() {
        pluginTransactionContext.wrap(() -> {
            try (Timer ignored = PluginSystemInstrumentation.instance().pullSingleTimer("shutdown")) {
                tracker.setState(StateTracker.State.SHUTTING_DOWN);

                log.info("Preparing to shut down the plugin system");
                broadcastIgnoreError(new PluginFrameworkShuttingDownEvent(DefaultPluginManager.this, DefaultPluginManager.this));

                log.info("Shutting down the plugin system");
                broadcastIgnoreError(new PluginFrameworkShutdownEvent(DefaultPluginManager.this, DefaultPluginManager.this));

                pluginRegistry.clear();
                pluginEventManager.unregister(this);
                tracker.setState(StateTracker.State.SHUTDOWN);
                defaultPluginManagerJmxBridge.unregister();
            }
        });
    }

    @Override
    public final void warmRestart() {
        pluginTransactionContext.wrap(() -> {
            tracker.setState(StateTracker.State.WARM_RESTARTING);
            log.info("Initiating a warm restart of the plugin system");
            broadcastIgnoreError(new PluginFrameworkWarmRestartingEvent(DefaultPluginManager.this, DefaultPluginManager.this));

            // Make sure we reload plugins in order
            final List<Plugin> restartedPlugins = new ArrayList<>();
            final List<PluginLoader> loaders = new ArrayList<>(pluginLoaders);
            reverse(loaders);
            for (final PluginLoader loader : pluginLoaders) {
                for (final Map.Entry<Plugin, PluginLoader> entry : installedPluginsToPluginLoader.entrySet()) {
                    if (entry.getValue() == loader) {
                        final Plugin plugin = entry.getKey();
                        if (isPluginEnabled(plugin.getKey())) {
                            disablePluginModules(plugin);
                            restartedPlugins.add(plugin);
                        }
                    }
                }
            }

            // then enable them in reverse order
            reverse(restartedPlugins);
            for (final Plugin plugin : restartedPlugins) {
                enableConfiguredPluginModules(plugin);
            }

            broadcastIgnoreError(new PluginFrameworkWarmRestartedEvent(DefaultPluginManager.this, DefaultPluginManager.this));
            tracker.setState(StateTracker.State.STARTED);
        });

    }

    @PluginEventListener
    public void onPluginModuleAvailable(final PluginModuleAvailableEvent event) {
        pluginTransactionContext.wrap(() -> enableConfiguredPluginModule(event.getModule().getPlugin(), event.getModule(), new HashSet<>()));

    }

    @PluginEventListener
    public void onPluginModuleUnavailable(final PluginModuleUnavailableEvent event) {
        pluginTransactionContext.wrap(() -> disablePluginModuleNoPersist(event.getModule()));
    }

    @PluginEventListener
    public void onPluginContainerUnavailable(final PluginContainerUnavailableEvent event) {
        pluginTransactionContext.wrap(() -> disablePluginWithoutPersisting(event.getPluginKey()));
    }

    @PluginEventListener
    public void onPluginRefresh(final PluginRefreshedEvent event) {
        pluginTransactionContext.wrap(() -> {
            final Plugin plugin = event.getPlugin();

            disablePluginModules(plugin);

            // It would be nice to fire this earlier, but doing it earlier than the disable of the plugin modules seems too early.
            // We should probably hook methods on NonValidatingOsgiBundleXmlApplicationContext (such as prepareRefresh ?) to
            // move this and the disable earlier if it makes sense.
            broadcastIgnoreError(new PluginEnablingEvent(plugin));

            if (enableConfiguredPluginModules(plugin)) {
                broadcastPluginEnabled(plugin);
            }
        });
    }

    /**
     * Set the plugin installation strategy for this manager
     *
     * @param pluginInstaller the plugin installation strategy to use
     * @see PluginInstaller
     */
    public void setPluginInstaller(final PluginInstaller pluginInstaller) {
        if (pluginInstaller instanceof RevertablePluginInstaller) {
            this.pluginInstaller = (RevertablePluginInstaller) pluginInstaller;
        } else {
            this.pluginInstaller = new NoOpRevertablePluginInstaller(pluginInstaller);
        }
    }

    @Override
    public Set<String> installPlugins(final PluginArtifact... pluginArtifacts) {
        final Map<String, PluginArtifact> validatedArtifacts = new LinkedHashMap<>();
        pluginTransactionContext.wrap(
                () -> {
                    try {
                        for (final PluginArtifact pluginArtifact : pluginArtifacts) {
                            validatedArtifacts.put(validatePlugin(pluginArtifact), pluginArtifact);
                        }
                    } catch (final PluginParseException ex) {
                        throw new PluginParseException("All plugins could not be validated", ex);
                    }

                    for (final Map.Entry<String, PluginArtifact> entry : validatedArtifacts.entrySet()) {
                        pluginInstaller.installPlugin(entry.getKey(), entry.getValue());
                    }

                    scanForNewPlugins();
                }
        );
        return validatedArtifacts.keySet();
    }

    /**
     * Validate a plugin jar. Looks through all plugin loaders for ones that can
     * load the plugin and extract the plugin key as proof.
     *
     * @param pluginArtifact the jar file representing the plugin
     * @return The plugin key
     * @throws PluginParseException if the plugin cannot be parsed
     * @throws NullPointerException if <code>pluginJar</code> is null.
     */
    String validatePlugin(final PluginArtifact pluginArtifact) {
        boolean foundADynamicPluginLoader = false;
        for (final PluginLoader loader : pluginLoaders) {
            if (loader.isDynamicPluginLoader()) {
                foundADynamicPluginLoader = true;
                final String key = ((DynamicPluginLoader) loader).canLoad(pluginArtifact);
                if (key != null) {
                    return key;
                }
            }
        }

        if (!foundADynamicPluginLoader) {
            throw new IllegalStateException("Should be at least one DynamicPluginLoader in the plugin loader list");
        }
        throw new PluginParseException("Jar " + pluginArtifact.getName() + " is not a valid plugin!");
    }

    @Override
    public int scanForNewPlugins() {
        final StateTracker.State state = tracker.get();
        checkState((StateTracker.State.RESUMING == state) || (StateTracker.State.STARTED == state),
                "Cannot scanForNewPlugins in state %s", state);

        final AtomicInteger numberFound = new AtomicInteger(0);
        pluginTransactionContext.wrap(() -> {
            for (final PluginLoader loader : pluginLoaders) {
                if (loader != null && loader.supportsAddition()) {
                    final List<Plugin> pluginsToAdd = new ArrayList<>();
                    for (Plugin plugin : loader.loadFoundPlugins(moduleDescriptorFactory)) {
                        final Plugin oldPlugin = pluginRegistry.get(plugin.getKey());
                        // Only actually install the plugin if its module
                        // descriptors support it. Otherwise, mark it as
                        // unloadable.
                        if (!(plugin instanceof UnloadablePlugin)) {
                            if (PluginUtils.doesPluginRequireRestart(plugin)) {
                                if (oldPlugin == null) {
                                    markPluginInstallThatRequiresRestart(plugin);

                                    final UnloadablePlugin unloadablePlugin = UnloadablePluginFactory.createUnloadablePlugin(plugin);
                                    unloadablePlugin.setErrorText("Plugin requires a restart of the application due "
                                            + "to the following modules: " + PluginUtils.getPluginModulesThatRequireRestart(plugin));
                                    plugin = unloadablePlugin;
                                } else {
                                    // If a plugin has been installed but is waiting for restart then we do not want to
                                    // put the plugin into the update state, we want to keep it in the install state.
                                    if (!PluginRestartState.INSTALL.equals(getPluginRestartState(plugin.getKey()))) {
                                        markPluginUpgradeThatRequiresRestart(plugin);
                                    }
                                    continue;
                                }
                            }
                            // If the new plugin does not require restart we need to check what the restart state of
                            // the old plugin was and act accordingly
                            else if (oldPlugin != null && PluginUtils.doesPluginRequireRestart(oldPlugin)) {
                                // If you have installed the plugin that requires restart and before restart you have
                                // reinstalled a version of that plugin that does not require restart then you should
                                // just go ahead and install that plugin. This means reverting the previous install
                                // and letting the plugin fall into the plugins to add list
                                if (PluginRestartState.INSTALL.equals(getPluginRestartState(oldPlugin.getKey()))) {
                                    revertRestartRequiredChange(oldPlugin.getKey());
                                } else {
                                    markPluginUpgradeThatRequiresRestart(plugin);
                                    continue;
                                }
                            }
                            pluginsToAdd.add(plugin);
                        }
                    }
                    addPlugins(loader, pluginsToAdd);
                    numberFound.addAndGet(pluginsToAdd.size());
                }
            }
        });
        return numberFound.get();
    }

    private void markPluginInstallThatRequiresRestart(final Plugin plugin) {
        log.info("Installed plugin '{}' requires a restart due to the following modules: {}", plugin, PluginUtils.getPluginModulesThatRequireRestart(plugin));
        updateRequiresRestartState(plugin.getKey(), PluginRestartState.INSTALL);
    }

    private void markPluginUpgradeThatRequiresRestart(final Plugin plugin) {
        log.info("Upgraded plugin '{}' requires a restart due to the following modules: {}", plugin, PluginUtils.getPluginModulesThatRequireRestart(plugin));
        updateRequiresRestartState(plugin.getKey(), PluginRestartState.UPGRADE);
    }

    private void markPluginUninstallThatRequiresRestart(final Plugin plugin) {
        log.info("Uninstalled plugin '{}' requires a restart due to the following modules: {}", plugin, PluginUtils.getPluginModulesThatRequireRestart(plugin));
        updateRequiresRestartState(plugin.getKey(), PluginRestartState.REMOVE);
    }

    private void updateRequiresRestartState(final String pluginKey, final PluginRestartState pluginRestartState) {
        persistentStateModifier.setPluginRestartState(pluginKey, pluginRestartState);
        onUpdateRequiresRestartState(pluginKey, pluginRestartState);
    }

    @SuppressWarnings("UnusedParameters")
    protected void onUpdateRequiresRestartState(final String pluginKey, final PluginRestartState pluginRestartState) {
        // nothing to do in this implementation
    }

    /**
     * Uninstalls the given plugin, emitting disabled and uninstalled events as it does so.
     *
     * @param plugin the plugin to uninstall.
     * @throws PluginException If the plugin or loader doesn't support uninstallation
     */
    @Override
    public void uninstall(final Plugin plugin) {
        pluginTransactionContext.wrap(() ->
                uninstallPlugins(singletonList(plugin))
        );
    }

    @Override
    public void uninstallPlugins(Collection<Plugin> plugins) {
        pluginTransactionContext.wrap(() -> {
            Map<Boolean, Set<Plugin>> requireRestart = plugins.stream()
                    .collect(partitioningBy(PluginUtils::doesPluginRequireRestart, toSet()));

        // Plugins that require application restart will be uninstalled on the next application start
        // (see com.atlassian.plugin.descriptors.RequiresRestart).
        requireRestart.get(true)
                .forEach(plugin -> {
                    ensurePluginAndLoaderSupportsUninstall(plugin);
                    markPluginUninstallThatRequiresRestart(plugin);
                });

            Set<Plugin> pluginsToDisable = requireRestart.get(false);

        if (!pluginsToDisable.isEmpty()) {
            final DependentPlugins disabledPlugins = disablePluginsAndTheirDependencies(
                    pluginsToDisable.stream().map(Plugin::getKey).collect(toList()),
                    unmodifiableSet(new HashSet<>(asList(MANDATORY, OPTIONAL, DYNAMIC)))
            );
            disabledPlugins.getPluginsByTypes(singleton(MANDATORY), true).forEach(persistentStateModifier::disable);

                pluginsToDisable.forEach(p -> broadcastIgnoreError(new PluginUninstallingEvent(p)));

                pluginsToDisable.forEach(this::uninstallNoEvent);

                pluginsToDisable.forEach(p -> broadcastIgnoreError(new PluginUninstalledEvent(p)));

                reenableDependent(pluginsToDisable, disabledPlugins, PluginState.UNINSTALLED);
            }
        });
    }

    /**
     * Preforms an uninstallation without broadcasting the uninstallation event.
     *
     * @param plugin The plugin to uninstall
     * @since 2.5.0
     */
    protected void uninstallNoEvent(final Plugin plugin) {
        unloadPlugin(plugin);

        // PLUG-13: Plugins should not save state across uninstalls.
        persistentStateModifier.removeState(plugin);
    }

    /**
     * @param pluginKey The plugin key to revert
     * @throws PluginException If the revert cannot be completed
     */
    @Override
    public void revertRestartRequiredChange(final String pluginKey) {
        pluginTransactionContext.wrap(() -> {
            notNull("pluginKey", pluginKey);
            final PluginRestartState restartState = getState().getPluginRestartState(pluginKey);
            if (restartState == PluginRestartState.UPGRADE) {
                pluginInstaller.revertInstalledPlugin(pluginKey);
            } else if (restartState == PluginRestartState.INSTALL) {
                pluginInstaller.revertInstalledPlugin(pluginKey);
                pluginRegistry.remove(pluginK