/src/main/java/org/nrg/xnat/services/cache/DefaultGroupsAndPermissionsCache.java

Large files files are truncated, but you can click here to view the full file

/*
 * web: org.nrg.xnat.services.cache.DefaultUserProjectCache
 * XNAT http://www.xnat.org
 * Copyright (c) 2017, Washington University School of Medicine
 * All Rights Reserved
 *
 * Released under the Simplified BSD.
 */

package org.nrg.xnat.services.cache;

import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
import com.google.common.collect.*;
import lombok.extern.slf4j.Slf4j;
import net.sf.ehcache.CacheException;
import net.sf.ehcache.Ehcache;
import net.sf.ehcache.Element;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DurationFormatUtils;
import org.nrg.framework.exceptions.NrgServiceRuntimeException;
import org.nrg.framework.orm.DatabaseHelper;
import org.nrg.framework.utilities.LapStopWatch;
import org.nrg.xdat.XDAT;
import org.nrg.xdat.display.ElementDisplay;
import org.nrg.xdat.om.*;
import org.nrg.xdat.schema.SchemaElement;
import org.nrg.xdat.security.SecurityManager;
import org.nrg.xdat.security.*;
import org.nrg.xdat.security.helpers.Permissions;
import org.nrg.xdat.security.helpers.Users;
import org.nrg.xdat.security.user.exceptions.UserInitException;
import org.nrg.xdat.security.user.exceptions.UserNotFoundException;
import org.nrg.xdat.services.Initializing;
import org.nrg.xdat.services.cache.GroupsAndPermissionsCache;
import org.nrg.xdat.servlet.XDATServlet;
import org.nrg.xft.db.PoolDBUtils;
import org.nrg.xft.event.XftItemEventI;
import org.nrg.xft.event.methods.XftItemEventCriteria;
import org.nrg.xft.exception.ElementNotFoundException;
import org.nrg.xft.exception.FieldNotFoundException;
import org.nrg.xft.exception.ItemNotFoundException;
import org.nrg.xft.exception.XFTInitException;
import org.nrg.xft.schema.XFTManager;
import org.nrg.xft.security.UserI;
import org.nrg.xnat.services.cache.jms.InitializeGroupRequest;
import org.slf4j.event.Level;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.CacheManager;
import org.springframework.dao.DataAccessException;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.ResultSetExtractor;
import org.springframework.jdbc.core.RowCallbackHandler;
import org.springframework.jdbc.core.namedparam.EmptySqlParameterSource;
import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
import org.springframework.jms.core.JmsTemplate;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.AsyncResult;
import org.springframework.stereotype.Service;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.text.DateFormat;
import java.text.NumberFormat;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static org.nrg.framework.exceptions.NrgServiceError.ConfigurationError;
import static org.nrg.xapi.rest.users.DataAccessApi.*;
import static org.nrg.xdat.security.PermissionCriteria.dumpCriteriaList;
import static org.nrg.xdat.security.helpers.Groups.*;
import static org.nrg.xft.event.XftItemEventI.*;

@SuppressWarnings("Duplicates")
@Service
@Slf4j
public class DefaultGroupsAndPermissionsCache extends AbstractXftItemAndCacheEventHandlerMethod implements GroupsAndPermissionsCache, Initializing, GroupsAndPermissionsCache.Provider {

    @Autowired
    public DefaultGroupsAndPermissionsCache(final CacheManager cacheManager, final NamedParameterJdbcTemplate template, final JmsTemplate jmsTemplate) throws SQLException {
        super(cacheManager,
              XftItemEventCriteria.builder().xsiType(XnatProjectdata.SCHEMA_ELEMENT_NAME).actions(CREATE, UPDATE, DELETE).build(),
              XftItemEventCriteria.builder().xsiType(XnatSubjectdata.SCHEMA_ELEMENT_NAME).xsiType(XnatExperimentdata.SCHEMA_ELEMENT_NAME).actions(CREATE, XftItemEventI.DELETE, SHARE).build(),
              XftItemEventCriteria.getXsiTypeCriteria(XdatUsergroup.SCHEMA_ELEMENT_NAME),
              XftItemEventCriteria.getXsiTypeCriteria(XdatElementSecurity.SCHEMA_ELEMENT_NAME));

        _template = template;
        _jmsTemplate = jmsTemplate;
        _helper = new DatabaseHelper((JdbcTemplate) _template.getJdbcOperations());
        _totalCounts = new HashMap<>();
        _missingElements = new HashMap<>();
        _userChecks = new ConcurrentHashMap<>();
        _initialized = new AtomicBoolean(false);
        if (_helper.tableExists("xnat_projectdata")) {
            resetTotalCounts();
        }
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void notifyElementRemoved(final Ehcache cache, final Element element) throws CacheException {
        handleCacheRemoveEvent(cache, element, "removed");
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void notifyElementExpired(final Ehcache cache, final Element element) {
        handleCacheRemoveEvent(cache, element, "expired");
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void notifyElementEvicted(final Ehcache cache, final Element element) {
        handleCacheRemoveEvent(cache, element, "evicted");
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void notifyRemoveAll(final Ehcache cache) {
        handleCacheRemoveEvent(cache, null, "removed");
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public String getCacheName() {
        return CACHE_NAME;
    }

    /**
     * Gets the specified project if the user has any access to it. Returns null otherwise.
     *
     * @param groupId The ID or alias of the project to retrieve.
     *
     * @return The project object if the user can access it, null otherwise.
     */
    @Override
    public UserGroupI get(final String groupId) {
        if (StringUtils.isBlank(groupId)) {
            throw new IllegalArgumentException("Can not request a group with a blank group ID");
        }

        // Check that the group is cached and, if so, return it.
        log.trace("Retrieving group through cache ID {}", groupId);
        final UserGroupI cachedGroup = getCachedGroup(groupId);
        if (cachedGroup != null) {
            log.debug("Found cached group entry for cache ID '{}'", groupId);
            return cachedGroup;
        }

        try {
            log.trace("Initializing group entry for cache ID '{}'", groupId);
            return initGroup(groupId);
        } catch (ItemNotFoundException e) {
            log.info("Can't find a group with the group ID '{}', returning null", groupId);
            return null;
        }
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public Map<String, Long> getReadableCounts(final UserI user) {
        if (user == null) {
            return Collections.emptyMap();
        }

        final String username = user.getUsername();
        final String cacheId  = getCacheIdForUserElements(username, READABLE);

        // Check whether the element types are cached and, if so, return that.
        log.trace("Retrieving readable counts for user {} through cache ID {}", username, cacheId);
        final Map<String, Long> cachedReadableCounts = getCachedMap(cacheId);
        if (cachedReadableCounts != null) {
            log.debug("Found cached readable counts entry for user '{}' with cache ID '{}' containing {} entries", username, cacheId, cachedReadableCounts.size());
            return cachedReadableCounts;
        }

        return initReadableCountsForUser(cacheId, user);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public Map<String, ElementDisplay> getBrowseableElementDisplays(final UserI user) {
        if (user == null) {
            return Collections.emptyMap();
        }

        final Map<String, ElementDisplay> guestBrowseableElementDisplays = getGuestBrowseableElementDisplays();
        if (user.isGuest()) {
            log.debug("Got a request for browseable element displays for the guest user, returning {} entries", guestBrowseableElementDisplays.size());
            return guestBrowseableElementDisplays;
        }

        final String username = user.getUsername();
        final String cacheId  = getCacheIdForUserElements(username, BROWSEABLE);

        // Check whether the element types are cached and, if so, return that.
        log.trace("Retrieving browseable element displays for user {} through cache ID {}", username, cacheId);
        final Map<String, ElementDisplay> cachedUserEntry = getCachedMap(cacheId);
        if (cachedUserEntry != null) {
            @SuppressWarnings("unchecked") final Map<String, ElementDisplay> browseables = buildImmutableMap(cachedUserEntry, guestBrowseableElementDisplays);
            log.debug("Found a cached entry for user '{}' browseable element displays under cache ID '{}' with {} entries", username, cacheId, browseables.size());
            return browseables;
        }

        log.trace("Initializing browseable element displays for user '{}' with cache ID '{}'", username, cacheId);
        @SuppressWarnings("unchecked") final Map<String, ElementDisplay> browseables = buildImmutableMap(initBrowseableElementDisplaysForUser(cacheId, user), guestBrowseableElementDisplays);
        log.debug("Initialized browseable element displays for user '{}' with cache ID '{}' with {} entries (including guest browseable element displays)", username, cacheId, browseables.size());
        return browseables;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public List<ElementDisplay> getSearchableElementDisplays(final UserI user) {
        if (user == null) {
            return Collections.emptyList();
        }

        final String username = user.getUsername();
        final String cacheId  = getCacheIdForUserElements(username, SEARCHABLE);
        log.debug("Retrieving searchable element displays for user {} through cache ID {}", username, cacheId);

        final Map<String, Long> counts = getReadableCounts(user);
        try {
            return Lists.newArrayList(Iterables.filter(getActionElementDisplays(username, SecurityManager.READ), new Predicate<ElementDisplay>() {
                @Override
                public boolean apply(@Nullable final ElementDisplay elementDisplay) {
                    if (elementDisplay == null) {
                        return false;
                    }
                    final String elementName = elementDisplay.getElementName();
                    try {
                        return ElementSecurity.IsSearchable(elementName) && counts.containsKey(elementName) && counts.get(elementName) > 0;
                    } catch (Exception e) {
                        return false;
                    }
                }
            }));
        } catch (Exception e) {
            log.error("An unknown error occurred", e);
        }

        return Collections.emptyList();
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public List<ElementDisplay> getActionElementDisplays(final UserI user, final String action) {
        return getActionElementDisplays(user.getUsername(), action);
    }

    @Override
    public List<ElementDisplay> getActionElementDisplays(final String username, final String action) {
        if (!ACTIONS.contains(action)) {
            throw new NrgServiceRuntimeException(ConfigurationError, "The action '" + action + "' is invalid, must be one of: " + StringUtils.join(ACTIONS, ", "));
        }
        final List<ElementDisplay> elementDisplays = getActionElementDisplays(username).get(action);
        if (log.isTraceEnabled()) {
            log.trace("Found {} element displays for user {} action {}: {}", elementDisplays.size(), username, action, formatElementDisplays(elementDisplays));
        }
        return elementDisplays;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public List<PermissionCriteriaI> getPermissionCriteria(final UserI user, final String dataType) {
        return getPermissionCriteria(user.getUsername(), dataType);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public List<PermissionCriteriaI> getPermissionCriteria(final String username, final String dataType) {
        try {
            PoolDBUtils.CheckSpecialSQLChars(dataType);
        } catch (Exception e) {
            return null;
        }

        try {
            final List<PermissionCriteriaI> criteria = new ArrayList<>();

            final Map<String, ElementAccessManager> managers = getElementAccessManagers(username);
            if (managers.isEmpty()) {
                log.info("Couldn't find element access managers for user {} trying to retrieve permissions for data type {}", username, dataType);
            } else {
                final ElementAccessManager manager = managers.get(dataType);
                if (manager == null) {
                    log.info("Couldn't find element access manager for data type {} for user {} while trying to retrieve permissions ", dataType, username);
                } else {
                    criteria.addAll(manager.getCriteria());
                    if (criteria.isEmpty()) {
                        log.debug("Couldn't find any permission criteria for data type {} for user {} while trying to retrieve permissions ", dataType, username);
                    }
                }
            }

            final Map<String, UserGroupI> userGroups = getMutableGroupsForUser(username);
            if (log.isDebugEnabled()) {
                log.debug("Found {} user groups for the user {}", userGroups.size(), username, userGroups.isEmpty() ? "" : ": " + Joiner.on(", ").join(userGroups.keySet()));
            }
            final Set<String> groups = userGroups.keySet();
            if (CollectionUtils.containsAny(groups, ALL_DATA_GROUPS)) {
                if (groups.contains(ALL_DATA_ADMIN_GROUP)) {
                    _template.query(QUERY_GET_ALL_MEMBER_GROUPS, new RowCallbackHandler() {
                        @Override
                        public void processRow(final ResultSet resultSet) throws SQLException {
                            final String projectId = resultSet.getString("project_id");
                            final String groupId   = resultSet.getString("group_id");

                            // If the user is a collaborator on a project, we're going to upgrade them to member,
                            // so remove that collaborator nonsense, this is the big time.
                            userGroups.remove(projectId + "_collaborator");

                            // If the user is already a member of owner of a project, then don't bother: they already have
                            // sufficient access to the project.
                            if (!userGroups.containsKey(projectId + "_owner") && !userGroups.containsKey(projectId + "_member")) {
                                userGroups.put(groupId, get(groupId));
                            }
                        }
                    });
                } else if (userGroups.containsKey(ALL_DATA_ACCESS_GROUP)) {
                    _template.query(QUERY_GET_ALL_COLLAB_GROUPS, new RowCallbackHandler() {
                        @Override
                        public void processRow(final ResultSet resultSet) throws SQLException {
                            final String projectId = resultSet.getString("project_id");
                            final String groupId   = resultSet.getString("group_id");

                            // If the user has no group membership, then add as a collaborator.
                            if (!CollectionUtils.containsAny(groups, Arrays.asList(groupId, projectId + "_member", projectId + "_owner"))) {
                                userGroups.put(groupId, get(groupId));
                            }
                        }
                    });
                }
            }

            for (final UserGroupI group : userGroups.values()) {
                final List<PermissionCriteriaI> permissions = group.getPermissionsByDataType(dataType);
                if (permissions != null) {
                    if (log.isTraceEnabled()) {
                        log.trace("Searched for permission criteria for user {} on type {} in group {}: {}", username, dataType, group.getId(), dumpCriteriaList(permissions));
                    } else {
                        log.debug("Searched for permission criteria for user {} on type {} in group {}: {} permissions found", username, dataType, group.getId(), permissions.size());
                    }
                    criteria.addAll(permissions);
                } else {
                    log.warn("Tried to retrieve permissions for data type {} for user {} in group {}, but this returned null.", dataType, username, group.getId());
                }
            }

            if (!isGuest(username)) {
                try {
                    final List<PermissionCriteriaI> permissions = getPermissionCriteria(getGuest().getUsername(), dataType);
                    if (permissions != null) {
                        criteria.addAll(permissions);
                    } else {
                        log.warn("Tried to retrieve permissions for data type {} for the guest user, but this returned null.", dataType);
                    }
                } catch (Exception e) {
                    log.error("An error occurred trying to retrieve the guest user", e);
                }
            }

            if (log.isTraceEnabled()) {
                log.trace("Retrieved permission criteria for user {} on the data type {}: {}", username, dataType, dumpCriteriaList(criteria));
            } else {
                log.debug("Retrieved permission criteria for user {} on the data type {}: {} criteria found", username, dataType, criteria.size());
            }

            return ImmutableList.copyOf(criteria);
        } catch (UserNotFoundException e) {
            log.error("Couldn't find the indicated user");
            return Collections.emptyList();
        }
    }

    @Override
    public Map<String, Long> getTotalCounts() {
        if (_totalCounts.isEmpty()) {
            resetTotalCounts();
        }

        return ImmutableMap.copyOf(_totalCounts);
    }

    @Nonnull
    @Override
    public List<String> getProjectsForUser(final String username, final String access) {
        log.info("Getting projects with {} access for user {}", access, username);
        final String cacheId = getCacheIdForUserProjectAccess(username, access);

        final List<String> cachedUserProjects = getCachedList(cacheId);
        if (cachedUserProjects != null) {
            log.debug("Found a cache entry for user '{}' '{}' access with ID '{}' and {} elements", username, access, cacheId, cachedUserProjects.size());
            return cachedUserProjects;
        }

        return updateUserProjectAccess(username, access, cacheId);
    }

    /**
     * {@inheritDoc}
     */
    @Nonnull
    @Override
    public List<UserGroupI> getGroupsForTag(final String tag) {
        // Get the group IDs associated with the tag.
        log.info("Getting groups for tag {}", tag);
        final List<String> groupIds = getTagGroups(tag);
        return getUserGroupList(groupIds);
    }

    @Nonnull
    @Override
    public Map<String, UserGroupI> getGroupsForUser(final String username) throws UserNotFoundException {
        return ImmutableMap.copyOf(getMutableGroupsForUser(username));
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void refreshGroupsForUser(final String username) throws UserNotFoundException {
        initUserGroupIds(getCacheIdForUserGroups(username), username);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    @Nullable
    public UserGroupI getGroupForUserAndTag(final String username, final String tag) throws UserNotFoundException {
        final String groupId = _template.query(QUERY_GET_GROUP_FOR_USER_AND_TAG, checkUser(username).addValue("tag", tag), new ResultSetExtractor<String>() {
            @Override
            public String extractData(final ResultSet results) throws DataAccessException, SQLException {
                return results.next() ? results.getString("id") : null;
            }
        });
        return StringUtils.isNotBlank(groupId) ? get(groupId) : null;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public List<String> getUserIdsForGroup(final String groupId) {
        final UserGroupI userGroup = get(groupId);
        if (userGroup == null) {
            return Collections.emptyList();
        }
        return ImmutableList.copyOf(userGroup.getUsernames());
    }

    @Override
    public void refreshGroup(final String groupId) throws ItemNotFoundException {
        final UserGroupI group = initGroup(groupId);
        for (final String username : group.getUsernames()) {
            try {
                if (!getGroupIdsForUser(username).contains(groupId)) {
                    refreshGroupsForUser(username);
                }
            } catch (UserNotFoundException ignored) {
                //
            }
        }
    }

    @Override
    public Date getUserLastUpdateTime(final UserI user) {
        return getUserLastUpdateTime(user.getUsername());
    }

    @Override
    public Date getUserLastUpdateTime(final String username) {
        try {
            @SuppressWarnings("unchecked") final List<String> cacheIds = new ArrayList<>(buildImmutableSet(getGroupIdsForUser(username), getCacheIdsForUsername(username)));
            if (cacheIds.isEmpty()) {
                return new Date();
            }
            if (log.isDebugEnabled()) {
                log.debug("Found {} cache entries related to user {}: {}", cacheIds.size(), username, StringUtils.join(cacheIds, ", "));
            }
            final long lastUpdateTime = Collections.max(Lists.transform(cacheIds, new Function<String, Long>() {
                @Override
                public Long apply(@Nullable final String cacheId) {
                    final Date lastUpdateTime = getCacheEntryLastUpdateTime(cacheId);
                    log.trace("User {} cache entry '{}' last updated: {}", username, cacheId, lastUpdateTime == null ? "null" : lastUpdateTime.getTime());
                    return lastUpdateTime == null ? 0L : lastUpdateTime.getTime();
                }
            }));
            log.debug("Found latest cache entry last updated time for user {}: {}", username, lastUpdateTime);
            return new Date(lastUpdateTime);
        } catch (UserNotFoundException ignored) {
            log.warn("Someone requested the cache entry last updated time for user {} but that user wasn't found", username);
            return new Date();
        }
    }

    /**
     * Finds all user element cache IDs for the specified user and evicts them from the cache.
     *
     * @param username The username to be cleared.
     */
    @Override
    public void clearUserCache(final String username) {
        final List<String> cacheIds = getCacheIdsForUserElements(username);
        if (log.isDebugEnabled()) {
            log.debug("Clearing caches for user '{}': {}", username, StringUtils.join(cacheIds, ", "));
        }
        evict(cacheIds);
    }

    @Override
    public boolean canInitialize() {
        try {
            if (_listener == null) {
                return false;
            }
            final boolean doesUserGroupTableExists            = _helper.tableExists("xdat_usergroup");
            final boolean isXftManagerComplete                = XFTManager.isComplete();
            final boolean isDatabasePopulateOrUpdateCompleted = XDATServlet.isDatabasePopulateOrUpdateCompleted();
            log.info("User group table {}, XFTManager initialization completed {}, database populate or updated completed {}", doesUserGroupTableExists, isXftManagerComplete, isDatabasePopulateOrUpdateCompleted);
            return doesUserGroupTableExists && isXftManagerComplete && isDatabasePopulateOrUpdateCompleted;
        } catch (SQLException e) {
            log.info("Got an SQL exception checking for xdat_usergroup table", e);
            return false;
        }
    }

    @Async
    @Override
    public Future<Boolean> initialize() {
        final LapStopWatch stopWatch = LapStopWatch.createStarted(log, Level.INFO);

        // This clears out any group initialization requests that may be left in the database from earlier starts.
        _template.update("DELETE FROM activemq_msgs WHERE container LIKE '%initializeGroupRequest'", EmptySqlParameterSource.INSTANCE);

        final int tags = initializeTags();
        stopWatch.lap("Processed {} tags", tags);

        final List<String> groupIds = _template.queryForList(QUERY_ALL_GROUPS, EmptySqlParameterSource.INSTANCE, String.class);
        _listener.setGroupIds(groupIds);
        stopWatch.lap("Initialized listener of type {} with {} tags", _listener.getClass().getName(), tags);

        try {
            final UserI adminUser = Users.getAdminUser();
            assert adminUser != null;

            stopWatch.lap("Found {} group IDs to run through, initializing cache with these as user {}", groupIds.size(), adminUser.getUsername());
            for (final String groupId : groupIds) {
                stopWatch.lap(Level.DEBUG, "Creating queue entry for group {}", groupId);
                XDAT.sendJmsRequest(_jmsTemplate, new InitializeGroupRequest(groupId));
            }
        } finally {
            if (stopWatch.isStarted()) {
                stopWatch.stop();
            }
            log.info("Total time to queue {} groups was {} ms", groupIds.size(), NUMBER_FORMAT.format(stopWatch.getTime()));
            if (log.isInfoEnabled()) {
                log.info(stopWatch.toTable());
            }
        }

        resetGuestBrowseableElementDisplays();
        _initialized.set(true);
        return new AsyncResult<>(true);
    }

    @Override
    public boolean isInitialized() {
        return _initialized.get();
    }

    @Override
    public Map<String, String> getInitializationStatus() {
        final Map<String, String> status = new HashMap<>();
        if (_listener == null) {
            status.put("message", "No listener registered, so no status to report.");
            return status;
        }

        final Set<String> processed      = _listener.getProcessed();
        final int         processedCount = processed.size();
        final Set<String> unprocessed    = _listener.getUnprocessed();
        final Date        start          = _listener.getStart();

        status.put("start", DATE_FORMAT.format(start));
        status.put("processedCount", Integer.toString(processedCount));
        status.put("processed", StringUtils.join(processed, ", "));

        if (unprocessed.isEmpty()) {
            final Date   completed = _listener.getCompleted();
            final String duration  = DurationFormatUtils.formatPeriodISO(start.getTime(), completed.getTime());

            status.put("completed", DATE_FORMAT.format(completed));
            status.put("duration", duration);
            status.put("message", "Cache initialization is complete. Processed " + processedCount + " groups in " + duration);
            return status;
        }

        final Date   now              = new Date();
        final String duration         = DurationFormatUtils.formatPeriodISO(start.getTime(), now.getTime());
        final int    unprocessedCount = unprocessed.size();

        status.put("unprocessedCount", Integer.toString(unprocessedCount));
        status.put("unprocessed", StringUtils.join(unprocessed, ", "));
        status.put("current", DATE_FORMAT.format(now));
        status.put("duration", duration);
        status.put("message", "Cache initialization is on-going, with " + processedCount + " groups processed and " + unprocessedCount + " groups remaining, time elapsed so far is " + duration);
        return status;
    }

    @Override
    public void registerListener(final Listener listener) {
        _listener = listener;
    }

    @Override
    public Listener getListener() {
        return _listener;
    }

    @Override
    protected boolean handleEventImpl(final XftItemEventI event) {
        switch (event.getXsiType()) {
            case XnatProjectdata.SCHEMA_ELEMENT_NAME:
                return handleProjectEvents(event);

            case XnatSubjectdata.SCHEMA_ELEMENT_NAME:
                return handleSubjectEvents(event);

            case XdatUsergroup.SCHEMA_ELEMENT_NAME:
                return handleGroupRelatedEvents(event);

            case XdatElementSecurity.SCHEMA_ELEMENT_NAME:
                return handleElementSecurityEvents(event);

            default:
                // This is always some type of experiment.
                return handleExperimentEvents(event);
        }
    }

    private boolean handleProjectEvents(final XftItemEventI event) {
        final String         xsiType    = event.getXsiType();
        final String         id         = event.getId();
        final String         action     = event.getAction();
        final Map<String, ?> properties = event.getProperties();

        try {
            switch (action) {
                case CREATE:
                    log.debug("New project created with ID {}, caching new instance", xsiType, id);
                    for (final String owner : getProjectOwners(id)) {
                        updateUserProjectAccess(owner);
                        if (!Iterables.any(getActionElementDisplays(owner).get(SecurityManager.CREATE), CONTAINS_MR_SESSION)) {
                            initActionElementDisplays(owner, true);
                        }
                    }

                    final boolean created = !initGroups(getGroups(xsiType, id)).isEmpty();
                    final String access = Permissions.getProjectAccess(_template, id);
                    if (StringUtils.isNotBlank(access)) {
                        switch (access) {
                            case "private":
                                clearAllDataUserProjectAccess();
                                break;

                            case "public":
                                if (!Iterables.any(getActionElementDisplays(GUEST_USERNAME).get(SecurityManager.READ), CONTAINS_MR_SESSION)) {
                                    initActionElementDisplays(GUEST_USERNAME, true);
                                }

                            case "protected":
                                updateProjectRelatedCaches(xsiType, id, false);
                                break;
                        }
                    }
                    return created;

                case UPDATE:
                    log.debug("The {} object {} was updated, caching updated instance", xsiType, id);
                    if (properties.containsKey("accessibility")) {
                        final String accessibility = (String) properties.get("accessibility");
                        switch (accessibility) {
                            case "private":
                                return updateProjectRelatedCaches(xsiType, id, true);

                            case "public":
                                if (!Iterables.any(getActionElementDisplays(GUEST_USERNAME).get(SecurityManager.READ), CONTAINS_MR_SESSION)) {
                                    initActionElementDisplays(GUEST_USERNAME, true);
                                }

                            case "protected":
                                return updateProjectRelatedCaches(xsiType, id, true);

                            default:
                                log.warn("The project {}'s accessibility setting was updated to an invalid value: {}. Must be one of private, protected, or public.", id, accessibility);
                        }
                    }
                    break;

                case XftItemEventI.DELETE:
                    log.debug("The {} {} was deleted, removing related instances from cache", xsiType, id);
                    final String cacheId = getCacheIdForProject(id);
                    evict(cacheId);
                    for (final String accessCacheId : Iterables.filter(getCacheIdsForUserElements(), Predicates.contains(REGEX_USER_PROJECT_ACCESS_CACHE_ID))) {
                        final List<String> projectIds = getCachedList(accessCacheId);
                        if (projectIds != null && projectIds.contains(id)) {
                            final List<String> updated = new ArrayList<>(projectIds);
                            updated.remove(id);
                            forceCacheObject(accessCacheId, updated);
                        }
                    }
                    resetGuestBrowseableElementDisplays();
                    initReadableCountsForUsers(this.<String>getCachedSet(cacheId));
                    resetTotalCounts();
                    return true;

                default:
                    log.warn("I was informed that the '{}' action happened to the project with ID '{}'. I don't know what to do with this action.", action, xsiType, id);
                    break;
            }
        } catch (ItemNotFoundException e) {
            log.warn("While handling action {}, I couldn't find a group for type {} ID {}.", action, xsiType, id);
        }

        return false;
    }

    private void clearAllDataUserProjectAccess() {
        for (final String groupId : ALL_DATA_GROUPS) {
            final UserGroupI group = get(groupId);
            if (group != null) {
                for (final String user : group.getUsernames()) {
                    for (final String projectAction : ACTIONS) {
                        evict(getCacheIdForUserProjectAccess(user, projectAction));
                    }
                }
            }
        }
    }

    private boolean updateProjectRelatedCaches(final String xsiType, final String id, final boolean affectsOtherDataTypes) throws ItemNotFoundException {
        final boolean cachedRelatedGroups = !initGroups(getGroups(xsiType, id)).isEmpty();

        evict(GUEST_CACHE_ID);
        evict(GUEST_ACTION_READ);
        resetGuestBrowseableElementDisplays();
        initActionElementDisplays(GUEST_USERNAME, true);

        final Set<String> readableCountCacheIds = new HashSet<>(getCacheIdsForUserReadableCounts());
        if (affectsOtherDataTypes) {
            for (final String cacheId : readableCountCacheIds) {
                evict(cacheId);
            }
            resetTotalCounts();
        } else {
            // Update existing user element displays
            final List<String> cacheIds = getCacheIdsForActions();
            cacheIds.addAll(getCacheIdsForUserElements());
            clearAllUserProjectAccess();
            initReadableCountsForUsers(Sets.newHashSet(Iterables.filter(Lists.transform(cacheIds, FUNCTION_CACHE_IDS_TO_USERNAMES), Predicates.notNull())));
            resetProjectCount();
        }

        return cachedRelatedGroups;
    }

    private boolean handleSubjectEvents(final XftItemEventI event) {
        final String action = event.getAction();
        log.debug("Handling subject {} event for {} {}", XftItemEventI.ACTIONS.get(action), event.getXsiType(), event.getId());
        final Set<String> projectIds = new HashSet<>();
        switch (action) {
            case CREATE:
                projectIds.add(_template.queryForObject(QUERY_GET_SUBJECT_PROJECT, new MapSqlParameterSource("subjectId", event.getId()), String.class));
                resetTotalCounts();
                break;

            case SHARE:
                projectIds.add((String) event.getProperties().get("target"));
                break;

            case MOVE:
                projectIds.add((String) event.getProperties().get("origin"));
                projectIds.add((String) event.getProperties().get("target"));
                break;

            case XftItemEventI.DELETE:
                projectIds.add((String) event.getProperties().get("target"));
                handleGroupRelatedEvents(event);
                resetTotalCounts();
                break;

            default:
                log.warn("I was informed that the '{}' action happened to subject '{}'. I don't know what to do with this action.", action, event.getId());
        }
        if (projectIds.isEmpty()) {
            return false;
        }
        final Set<String> users = getProjectUsers(projectIds);
        for (final String username : users) {
            initReadableCountsForUser(username);
        }
        return true;
    }

    private boolean handleGroupRelatedEvents(final XftItemEventI event) {
        final String         xsiType    = event.getXsiType();
        final String         id         = event.getId();
        final String         action     = event.getAction();
        final Map<String, ?> properties = event.getProperties();
        final Set<String>    usernames  = new HashSet<>();

        try {
            final List<UserGroupI> groups = getGroups(xsiType, id);
            switch (action) {
                case CREATE:
                    log.debug("New {} created with ID {}, caching new instance", xsiType, id);
                    for (final UserGroupI group : groups) {
                        usernames.addAll(group.getUsernames());
                    }
                    log.debug("Handling create group event with ID '{}' for users: {}", id);
                    return !initGroups(groups).isEmpty();

                case UPDATE:
                    log.debug("The {} object {} was updated, caching updated instance", xsiType, id);
                    for (final UserGroupI group : groups) {
                        usernames.addAll(group.getUsernames());
                        evict(group.getId());
                    }
                    if (properties.containsKey(OPERATION) && StringUtils.equals((String) properties.get(OPERATION), OPERATION_REMOVE_USERS)) {
                        //noinspection unchecked
                        usernames.addAll((Collection<? extends String>) properties.get(USERS));
                    }
                    log.debug("Handling update group event with ID '{}' for users: {}", id);
                    return !initGroups(groups).isEmpty();

                case XftItemEventI.DELETE:
                    if (StringUtils.equals(XnatProjectdata.SCHEMA_ELEMENT_NAME, xsiType)) {
                        final List<String> groupIds = getTagGroups(id);
                        if (CollectionUtils.isNotEmpty(groupIds)) {
                            log.info("Found {} groups cached for deleted project {}", groupIds.size(), id);
                            for (final String groupId : groupIds) {
                                evictGroup(groupId, usernames);
                            }
                        }
                    } else {
                        evictGroup(id, usernames);
                    }
                    break;

                default:
                    log.warn("I was informed that the '{}' action happened to the {} object with ID '{}'. I don't know what to do with this action.", action, xsiType, id);
            }
        } catch (ItemNotFoundException e) {
            log.warn("While handling action {}, I couldn't find a group for type {} ID {}.", action, xsiType, id);
        } finally {
            for (final String username : usernames) {
                try {
                    final String cacheId = getCacheIdForUserGroups(username);
                    initUserGroupIds(cacheId, username);
                    initReadableCountsForUser(username);
                } catch (UserNotFoundException e) {
                    log.warn("While handling action {} for type {} ID {}, I couldn't find a user with username {}.", action, xsiType, id, username);
                }
            }
        }
        return false;
    }

    private boolean handleElementSecurityEvents(final XftItemEventI event) {
        log.debug("Handling {} event for '{}' IDs {}. Updating guest browseable element displays...", event.getAction(), event.getXsiType(), event.getIds());
        final Map<String, ElementDisplay> displays = resetGuestBrowseableElementDisplays();

        if (log.isTraceEnabled()) {
            log.trace("Got back {} browseable element displays for guest user after refresh: {}", displays.size(), StringUtils.join(displays.keySet(), ", "));
        }

        log.debug("Evicting all action and user element cache IDs");
        for (final String cacheId : Iterables.concat(getCacheIdsForActions(), getCacheIdsForUserElements())) {
            log.trace("Evicting cache entry with ID '{}'", cacheId);
            evict(cacheId);
        }

        for (final String dataType : event.getIds()) {
            final List<String> groupIds = getGroupIdsForDataType(dataType);
            log.debug("Found {} groups that reference the '{}' data type, updating cache entries for: {}", groupIds.size(), dataType, StringUtils.join(groupIds, ", "));
            for (final String groupId : groupIds) {
                log.trace("Evicting group '{}' due to change in element securities for data type {}", groupId, dataType);
                evict(groupId);
            }
        }

        return true;
    }

    private boolean handleExperimentEvents(final XftItemEventI event) {
        final String action  = event.getAction();
        final String xsiType = event.getXsiType();
        log.debug("Handling experiment {} event for {} {}", XftItemEventI.ACTIONS.get(action), xsiType, event.getId());
        final String target, origin;
        switch (action) {
            case CREATE:
                target = _template.queryForObject(QUERY_GET_EXPERIMENT_PROJECT, new MapSqlParameterSource("experimentId", event.getId()), String.class);
                origin = null;
                break;

            case SHARE:
                target = (String) event.getProperties().get("target");
                origin = null;
                break;

            case MOVE:
                origin = (String) event.getProperties().get("origin");
                target = (String) event.getProperties().get("target");
                break;

            case XftItemEventI.DELETE:
                target = (String) event.getProperties().get("target");
                origin = null;
                break;

            default:
                log.warn("I was informed that the '{}' action happened to experiment '{}' with ID '{}'. I don't know what to do with this action.", action, xsiType, event.getId());
                return false;
        }

        final Map<String, ElementDisplay> displays = getGuestBrowseableElementDisplays();
        log.debug("Found {} elements for guest user: {}", displays.size(), StringUtils.join(displays.keySet(), ", "));

        // If the data type of the experiment isn't in the guest list AND the target project is public,
        // OR if the origin project is both specified and public (meaning the data type might be REMOVED
        // from the guest browseable element displays), then we update the guest browseable element displays.
        final boolean hasEventXsiType        = displays.containsKey(xsiType);
        final boolean isTargetProjectPublic  = Permissions.isProjectPublic(_template, target);
        final boolean hasOriginProject       = StringUtils.isNotBlank(origin);
        final boolean isMovedFromPublicToNon = !isTargetProjectPublic && hasOriginProject && Permissions.isProjectPublic(_template, origin);

        // We need to add the XSI type if guest doesn't already have it and the target project is public.
        final boolean needsPublicXsiTypeAdded = !hasEventXsiType && isTargetProjectPublic;

        // We need to check if the XSI type should be removed if guest has XSI type and item was moved from public to non-public.
        final boolean needsXsiTypeChecked = hasEventXsiType && isMovedFromPublicToNon;

        if (needsPublicXsiTypeAdded || needsXsiTypeChecked) {
            if (needsPublicXsiTypeAdded) {
                log.debug("Updating guest browseable element displays: guest doesn't have the event XSI type '{}' and the target project {} is public.", xsiType, target);
            } else {
                log.debug("Updating guest browseable element displays: guest has the event XSI type '{}' and item was moved from public project {} to non-public project {}.", xsiType, origin, target);
            }
            resetGuestBrowseableElementDisplays();
        } else {
            log.debug("Not updating guest browseable element displays: guest {} '{}' and {}",
                      hasEventXsiType ? "already has the event XSI type " : "doesn't have the event XSI type",
                      xsiType,
                      isTargetProjectPublic ? "target project is public" : "target project is not public");
        }
        initReadableCountsForUsers(hasOriginProject ? getProjectUsers(target) : getProjectUsers(target, origin));
        if (StringUtils.equalsAny(action, CREATE, XftItemEventI.DELETE)) {
            resetTotalCounts();
        }
        return true;
    }

    private void handleCacheRemoveEvent(final Ehcache cache, final Element element, final String event) {
        if (isGroupsAndPermissionsCacheEvent(cache)) {
            if (element == null) {
                log.debug("Got a {} event for cache {}, no specific element affected", event, cache.getName());
                return;
            }
            final Object objectValue = element.getObjectValue();
            log.debug("Got a {} event for cache {} on ID {} with value of type {}", event, cache.getName(), element.getObjectKey(), objectValue != null ? objectValue.getClass().getName() : "<null>");
        }
    }

    @SuppressWarnings({"UnusedReturnValue", "SameParameterValue"})
    private synchronized List<String> updateUserProjectAccess(final String username) {
        final List<String> projectIds = new ArrayList<>();
        for (final String access : Arrays.asList(SecurityManager.READ, SecurityManager.EDIT, SecurityManager.DELETE)) {
            projectIds.addAll(updateUserProjectAccess(username, access));
        }
        return projectIds;
    }

    @SuppressWarnings({"UnusedReturnValue", "SameParameterValue"})
    private synchronized List<String> updateUserProjectAccess(final String username, final String access) {
        return updateUserProjectAccess(username, access, getCacheIdForUserProjectAccess(username, access));
    }

    private synchronized List<String> updateUserProjectAccess(final String username, final String access, final String cacheId) {
        final List<String> projectIds;
        switch (access) {
            case SecurityManager.READ:
                projectIds = getUserReadableProjects(username);
                break;
            case SecurityManager.EDIT:
                projectIds = getUserEditableProjects(username);
                break;
            case SecurityManager.DELETE:
                projectIds = getUserOwnedProjects(username);
                break;
            default:
                throw new RuntimeException("Unknown access level '" + access + "'. Must be one of " + SecurityManager.READ + ", " + SecurityManager.EDIT + ", or " + SecurityManager.DELETE + ".");
        }
        cacheObject(cacheId, projectIds);
        return ImmutableList.copyOf(projectIds);
    }

    private List<String> getProjectOwners(final String projectId) {
        return _template.queryForList(QUERY_PROJECT_OWNERS, new MapSqlParameterSource("projectId", projectId), String.class);
    }

    private ListMultimap<String, ElementDisplay> getActionElementDisplays(final String username) {
        final String cacheId = getCacheIdForActionElements(username);

        // Check whether the action elements are cached and, if so, return that.
        final ListMultimap<String, ElementDisplay> cachedActions = getCachedListMultimap(cacheId);
        if (cachedActions != null) {
            log.debug("Found a cache entry for user '{}' action elements by ID '{}'", username, cacheId);
            return cachedActions;
        }

        return initActionElementDisplays(username);
    }

    private Long getUserReadableWorkflowCount(final UserI user) {
        return _template.queryForObject(QUERY_USER_READABLE_WORKFLOW_COUNT, new MapSqlParameterSource("username", user.getUsername()), Long.class);
    }

    @Nonnull
    private Map<String, UserGroupI> getMutableGroupsForUser(final String username) throws UserNotFoundException {
        final List<String>            groupIds = getGroupIdsForUser(username);
        final Map<String, UserGroupI> groups   = new HashMap<>();
        for (final String groupId : groupIds) {
            final UserGroupI group = get(groupId);
            if (group != null) {
                log.trace("Adding group {} to groups for user {}", groupId, username);
                groups.put(groupId, group);
            } else {
                log.info("User '{}' is associated with the group ID '{}', but I couldn't find that actual group", username, groupId);
            }
        }
        return groups;
    }

    @Nonnull
    private Map<String, ElementAccessManager> getElementAccessManagers(final String username) {
        if (StringUtils.isBlank(username)) {
            return Collections.emptyMap();
        }
        final String                            cacheId                     = getCacheIdForUserElementAccessManagers(username);
        final Map<String, ElementAccessManager> cachedElementAccessManagers = getCachedMap(cache…