/notification/src/main/java/com/gooddata/processor/GdcNotification.java

/*
 * Copyright (c) 2009, GoodData Corporation. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification, are permitted provided
 * that the following conditions are met:
 *
 *     * Redistributions of source code must retain the above copyright notice, this list of conditions and
 *        the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright notice, this list of conditions
 *        and the following disclaimer in the documentation and/or other materials provided with the distribution.
 *     * Neither the name of the GoodData Corporation nor the names of its contributors may be used to endorse
 *        or promote products derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS
 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
 * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package com.gooddata.processor;

import com.gooddata.config.Metric;
import com.gooddata.config.NotificationConfig;
import com.gooddata.config.NotificationMessage;
import com.gooddata.config.Report;
import com.gooddata.exception.*;
import com.gooddata.filter.DuplicateMessageFilter;
import com.gooddata.filter.MessageFilter;
import com.gooddata.integration.rest.GdcRESTApiWrapper;
import com.gooddata.integration.rest.configuration.NamePasswordConfiguration;
import com.gooddata.transport.NotificationTransport;
import com.gooddata.transport.SfdcChatterTransport;
import com.gooddata.util.FileUtil;
import com.sforce.ws.ConnectionException;
import org.apache.commons.cli.*;
import org.apache.commons.jexl2.Expression;
import org.apache.commons.jexl2.JexlContext;
import org.apache.commons.jexl2.JexlEngine;
import org.apache.commons.jexl2.MapContext;
import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.sql.SQLException;
import java.text.DecimalFormat;
import java.util.List;
import java.util.Properties;

/**
 * The GoodData Data Integration CLI processor.
 *
 * @author jiri.zaloudek
 * @author Zdenek Svoboda <zd@gooddata.org>
 * @version 1.0
 */
public class GdcNotification {

    private static Logger l = Logger.getLogger(GdcNotification.class);

    //Options data
    public static String[] CLI_PARAM_GDC_USERNAME = {"username", "u"};
    public static String[] CLI_PARAM_GDC_PASSWORD = {"password", "p"};

    public static String[] CLI_PARAM_TRANSPORT_USERNAME = {"transportusername", "d"};
    public static String[] CLI_PARAM_TRANSPORT_PASSWORD = {"transportpassword", "c"};

    public static String[] CLI_PARAM_GDC_HOST = {"host", "h"};
    public static String[] CLI_PARAM_VERSION = {"version", "V"};
    public static String CLI_PARAM_CONFIG = "config";

    private static String DEFAULT_PROPERTIES = "gdi.properties";

    // mandatory options
    public static Option[] mandatoryOptions = {};

    // optional options
    public static Option[] optionalOptions = {
            new Option(CLI_PARAM_GDC_USERNAME[1], CLI_PARAM_GDC_USERNAME[0], true, "GoodData username"),
            new Option(CLI_PARAM_GDC_PASSWORD[1], CLI_PARAM_GDC_PASSWORD[0], true, "GoodData password"),
            new Option(CLI_PARAM_TRANSPORT_USERNAME[1], CLI_PARAM_TRANSPORT_USERNAME[0], true, "Salesforce username"),
            new Option(CLI_PARAM_TRANSPORT_PASSWORD[1], CLI_PARAM_TRANSPORT_PASSWORD[0], true, "Salesforce password"),
            new Option(CLI_PARAM_GDC_HOST[1], CLI_PARAM_GDC_HOST[0], true, "GoodData host"),
            new Option(CLI_PARAM_VERSION[1], CLI_PARAM_VERSION[0], false, "Prints the tool version."),
    };

    private CliParams cliParams = null;
    private boolean finishedSucessfuly = false;

    private final static String BUILD_NUMBER = "";

    private GdcNotification(CommandLine ln, Properties defaults) {
        try {
            cliParams = parse(ln, defaults);
            cliParams.setHttpConfig(new NamePasswordConfiguration("https",
                    cliParams.get(CLI_PARAM_GDC_HOST[0]),
                    cliParams.get(CLI_PARAM_GDC_USERNAME[0]), cliParams.get(CLI_PARAM_GDC_PASSWORD[0])));
            String config = cliParams.get(CLI_PARAM_CONFIG);
            if (config != null && config.length() > 0) {
                execute(config);
            } else {
                l.error("No config file given.");
                commandsHelp();
                System.exit(1);
            }
            finishedSucessfuly = true;
        } catch (ConnectionException e) {
            l.error("Can't connect to SFDC: " + e.getMessage());
            Throwable c = e.getCause();
            while (c != null) {
                l.error("Caused by: " + c.getMessage());
                c = c.getCause();
            }
            l.debug("Can't connect to SFDC:", e);
        } catch (InvalidArgumentException e) {
            l.error("Invalid command line argument: " + e.getMessage());
            Throwable c = e.getCause();
            while (c != null) {
                l.error("Caused by: " + c.getMessage());
                c = c.getCause();
            }
            l.debug("Invalid command line argument:", e);
            l.info(commandsHelp());
        } catch (SfdcException e) {
            l.error("Error communicating with SalesForce: " + e.getMessage());
            Throwable c = e.getCause();
            while (c != null) {
                l.error("Caused by: " + c.getMessage());
                c = c.getCause();
            }
            l.debug("Error communicating with SalesForce.", e);
        } catch (IOException e) {
            l.error("Encountered an IO problem. Please check that all files that you use in your command line arguments and commands exist. More info: '" + e.getMessage() + "'");
            Throwable c = e.getCause();
            while (c != null) {
                l.error("Caused by: " + c.getMessage());
                c = c.getCause();
            }
            l.debug("Encountered an IO problem. Please check that all files that you use in your command line arguments and commands exist. More info: '" + e.getMessage() + "'", e);
        } catch (InternalErrorException e) {
            Throwable c = e.getCause();
            if (c != null && c instanceof SQLException) {
                l.error("Error extracting data. Can't process the incoming data. Please check the CSV file " +
                        "separator and consistency (same number of columns in each row). Also, please make sure " +
                        "that the number of columns in your XML config file matches the number of rows in your " +
                        "data source. Make sure that your file is readable by other users (particularly the mysql user). " +
                        "More info: '" + c.getMessage() + "'");
                l.debug("Error extracting data. Can't process the incoming data. Please check the CSV file " +
                        "separator and consistency (same number of columns in each row). Also, please make sure " +
                        "that the number of columns in your XML config file matches the number of rows in your " +
                        "data source. Make sure that your file is readable by other users (particularly the mysql user). " +
                        "More info: '" + c.getMessage() + "'", c);
            } else {
                l.error("Internal error: " + e.getMessage());
                c = e.getCause();
                while (c != null) {
                    l.error("Caused by: " + c.getMessage());
                    c = c.getCause();
                }
                l.debug("REST API invocation error: ", e);
            }
        } catch (HttpMethodException e) {
            l.error("Error executing GoodData REST API: " + e.getMessage());
            Throwable c = e.getCause();
            while (c != null) {
                l.error("Caused by: " + c.getMessage());
                c = c.getCause();
            }
            l.debug("Error executing GoodData REST API.", e);
        } catch (GdcRestApiException e) {
            l.error("REST API invocation error: " + e.getMessage());
            Throwable c = e.getCause();
            while (c != null) {
                if (c instanceof HttpMethodException) {
                    HttpMethodException ex = (HttpMethodException) c;
                    String msg = ex.getMessage();
                    if (msg != null && msg.length() > 0 && msg.indexOf("/ldm/manage") > 0) {
                        l.error("Error creating/updating logical data model (executing MAQL DDL).");
                        if (msg.indexOf(".date") > 0) {
                            l.error("Bad time dimension schemaReference.");
                        } else {
                            l.error("You are either trying to create a data object that already exists " +
                                    "(executing the same MAQL multiple times) or providing a wrong reference " +
                                    "or schemaReference in your XML configuration.");
                        }
                    }
                }
                l.error("Caused by: " + c.getMessage());
                c = c.getCause();
            }
            l.debug("REST API invocation error: ", e);
        } catch (GdcException e) {
            l.error("Unrecognized error: " + e.getMessage());
            Throwable c = e.getCause();
            while (c != null) {
                l.error("Caused by: " + c.getMessage());
                c = c.getCause();
            }
            l.debug("Unrecognized error: ", e);
        }
    }

    private boolean decide(Object result) {
        if (result == null)
            return false;
        else if (result instanceof Boolean)
            return ((Boolean) result).booleanValue();
        else if (result instanceof Number)
            return ((Number) result).doubleValue() != 0;
        else if (result instanceof String)
            return ((String) result).length() > 0;
        else
            return false;
    }


    private NotificationTransport selectTransport(String uri) {
        if (uri.startsWith("sfdc")) {
            return SfdcChatterTransport.createTransport(cliParams.get(CLI_PARAM_TRANSPORT_USERNAME[0]),
                    cliParams.get(CLI_PARAM_TRANSPORT_PASSWORD[0]));
        }
        throw new InvalidParameterException("Can't find transport for uri " + uri);
    }

    private final static String DEFAULT_FORMAT = "#,###.00";

    private void execute(String config) throws ConnectionException, IOException {

        NotificationConfig c = NotificationConfig.fromXml(new File(config));

        MessageFilter dupFilter = DuplicateMessageFilter.createFilter();

        GdcRESTApiWrapper rest = null;
        try {
            for (NotificationMessage m : c.getMessages()) {
                String dupFilterKind = m.getDupFilterKind();
                if (dupFilterKind != null && dupFilterKind.length() > 0) {
                    if (!dupFilter.filter(m.getMessage(), dupFilterKind)) {
                        l.debug("Message filtered out by the dup kind filter.");
                        l.info("Message filtered out by the dup kind filter.");
                        continue;
                    }
                }
                rest = new GdcRESTApiWrapper(cliParams.getHttpConfig());
                rest.login();
                Expression e = null;
                JexlEngine jexl = new JexlEngine();
                e = jexl.createExpression(m.getCondition());
                JexlContext jc = new MapContext();
                List<Metric> metrics = m.getMetrics();
                double[] values = null;
                if (metrics != null && metrics.size() > 0) {
                    values = new double[metrics.size()];
                    for (int i = 0; i < metrics.size(); i++) {
                        values[i] = rest.computeMetric(metrics.get(i).getUri());
                        jc.set(metrics.get(i).getAlias(), new Double(values[i]));
                    }
                }
                String[] texts = null;
                List<Report> reports = m.getReports();
                if (reports != null && reports.size() > 0) {
                    texts = new String[reports.size()];
                    for (int i = 0; i < reports.size(); i++) {
                        texts[i] = rest.computeReport(reports.get(i).getUri());
                    }
                }
                boolean result = decide(e.evaluate(jc));
                if (result) {
                    NotificationTransport t = selectTransport(m.getUri());
                    String msg = m.getMessage();
                    if (values != null && values.length > 0 && metrics != null && metrics.size() > 0) {
                        for (int i = 0; i < metrics.size(); i++) {
                            String fmt = metrics.get(i).getFormat();
                            if (fmt == null || fmt.length() <= 0)
                                fmt = DEFAULT_FORMAT;
                            DecimalFormat df = new DecimalFormat(fmt);
                            msg = msg.replace("%" + metrics.get(i).getAlias() + "%", df.format(values[i]));
                        }
                    }
                    if (texts != null && texts.length > 0 && reports != null && reports.size() > 0) {
                        for (int i = 0; i < reports.size(); i++) {
                            msg = msg.replace("%" + reports.get(i).getAlias() + "%", texts[i]);
                        }
                    }
                    String dupFilterExact = m.getDupFilterExact();
                    if (dupFilterExact != null && dupFilterExact.length() > 0) {
                        if (!dupFilter.filter(msg, dupFilterExact)) {
                            l.debug("Message filtered out by the dup exact filter.");
                            l.info("Message filtered out by the dup exact filter.");
                            continue;
                        }
                    }
                    String fmt = m.getMessageTimestampFormat();
                    if (fmt != null && fmt.length() > 0)
                        t.send(msg + " (at " + getTimestamp(fmt) + ")");
                    else
                        t.send(msg);
                    dupFilter.update(msg);
                    dupFilter.update(m.getMessage());
                    l.info("Notification sent.");
                }
            }
            dupFilter.save();
            rest.logout();
        } catch (Exception e) {
            throw new IOException(e);
        } finally {
            if (rest != null)
                rest.logout();
        }
    }

    private String getTimestamp(String fmt) {
        DateTimeFormatter f = DateTimeFormat.forPattern(fmt);
        return f.print(new DateTime());
    }

    /**
     * Returns all cli options
     *
     * @return all cli options
     */
    public static Options getOptions() {
        Options ops = new Options();
        for (Option o : mandatoryOptions)
            ops.addOption(o);
        for (Option o : optionalOptions)
            ops.addOption(o);
        return ops;
    }

    /**
     * Parse and validate the cli arguments
     *
     * @param ln parsed command line
     * @return parsed cli parameters wrapped in the CliParams
     * @throws InvalidArgumentException in case of nonexistent or incorrect cli args
     */
    protected CliParams parse(CommandLine ln, Properties defaults) throws InvalidArgumentException {
        l.debug("Parsing cli " + ln);
        CliParams cp = new CliParams();

        for (Option o : mandatoryOptions) {
            String name = o.getLongOpt();
            if (ln.hasOption(name))
                cp.put(name, ln.getOptionValue(name));
            else if (defaults.getProperty(name) != null) {
                cp.put(name, defaults.getProperty(name));
            } else {
                throw new InvalidArgumentException("Missing the '" + name + "' commandline parameter.");
            }

        }

        for (Option o : optionalOptions) {
            String name = o.getLongOpt();
            if (ln.hasOption(name)) {
                cp.put(name, ln.getOptionValue(name));
            } else if (defaults.getProperty(name) != null) {
                cp.put(name, defaults.getProperty(name));
            }
        }

        if (cp.containsKey(CLI_PARAM_VERSION[0])) {
            l.info("GoodData Notification Tool version 1.2.67" +
                    ((BUILD_NUMBER.length() > 0) ? ", build " + BUILD_NUMBER : "."));
            System.exit(0);

        }


        // use default host if there is no host in the CLI params
        if (!cp.containsKey(CLI_PARAM_GDC_HOST[0])) {
            cp.put(CLI_PARAM_GDC_HOST[0], Defaults.DEFAULT_HOST);
        }

        l.debug("Using host " + cp.get(CLI_PARAM_GDC_HOST[0]));

        if (ln.getArgs().length == 0) {
            throw new InvalidArgumentException("No config file has been given, quitting.");
        }

        String configs = "";
        for (final String arg : ln.getArgs()) {
            if (configs.length() > 0)
                configs += "," + arg;
            else
                configs += arg;
        }
        cp.put(CLI_PARAM_CONFIG, configs);

        return cp;
    }


    /**
     * Returns the help for commands
     *
     * @return help text
     */
    public static String commandsHelp() {
        try {
            final InputStream is = CliParams.class.getResourceAsStream("/com/gooddata/processor/COMMANDS.txt");
            if (is == null)
                throw new IOException();
            return FileUtil.readStringFromStream(is);
        } catch (IOException e) {
            l.error("Could not read com/gooddata/processor/COMMANDS.txt");
        }
        return "";
    }


    private static boolean checkJavaVersion() {
        String version = System.getProperty("java.version");
        if (version.startsWith("1.8") || version.startsWith("1.7") || version.startsWith("1.6") || version.startsWith("1.5"))
            return true;
        l.error("You're running Java " + version + ". Please use Java 1.5 or higher for running this tool. " +
                "Please refer to http://java.sun.com/javase/downloads/index.jsp for a more recent Java version.");
        throw new InternalErrorException("You're running Java " + version + ". Please use use Java 1.5 or higher for running this tool. " +
                "Please refer to http://java.sun.com/javase/downloads/index.jsp for a more recent Java version.");
    }

    /**
     * The main CLI processor
     *
     * @param args command line argument
     */
    public static void main(String[] args) {

        checkJavaVersion();
        String logConfig = System.getProperty("log4j.configuration");
        if (logConfig != null && logConfig.length() > 0) {
            File lc = new File(logConfig);
            if (lc.exists()) {
                PropertyConfigurator.configure(logConfig);
                Properties defaults = loadDefaults();
                try {
                    Options o = getOptions();
                    CommandLineParser parser = new GnuParser();
                    CommandLine cmdline = parser.parse(o, args);
                    GdcNotification gdi = new GdcNotification(cmdline, defaults);
                    if (!gdi.finishedSucessfuly) {
                        System.exit(1);
                    }
                } catch (org.apache.commons.cli.ParseException e) {
                    l.error("Error parsing command line parameters: " + e.getMessage());
                    l.debug("Error parsing command line parameters", e);
                }
            } else {
                l.error("Can't find the logging config. Please configure the logging via the log4j.configuration.");
            }
        } else {
            l.error("Can't find the logging config. Please configure the logging via the log4j.configuration.");
        }
    }


    /**
     * Loads default values of common parameters from a properties file searching
     * the working directory and user's home.
     *
     * @return default configuration
     */
    private static Properties loadDefaults() {
        final String[] dirs = new String[]{"user.dir", "user.home"};
        final Properties props = new Properties();
        for (final String d : dirs) {
            String path = System.getProperty(d) + File.separator + DEFAULT_PROPERTIES;
            File f = new File(path);
            if (f.exists() && f.canRead()) {
                try {
                    FileInputStream is = new FileInputStream(f);
                    props.load(is);
                    l.debug("Successfully red the gdi configuration from '" + f.getAbsolutePath() + "'.");
                    return props;
                } catch (IOException e) {
                    l.warn("Readable gdi configuration '" + f.getAbsolutePath() + "' found be error occurred reading it.");
                    l.debug("Error reading gdi configuration '" + f.getAbsolutePath() + "': ", e);
                }
            }
        }
        return props;
    }

}