/trunk/flume-core/src/main/java/com/cloudera/flume/reporter/ganglia/GangliaSink.java

/**
 * Licensed to Cloudera, Inc. under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  Cloudera, Inc. licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.cloudera.flume.reporter.ganglia;

import java.io.IOException;
import java.net.DatagramPacket;
import java.net.DatagramSocket;
import java.net.SocketAddress;
import java.net.SocketException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.hadoop.metrics.spi.Util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.cloudera.flume.conf.Context;
import com.cloudera.flume.conf.FlumeConfiguration;
import com.cloudera.flume.conf.SinkFactory.SinkBuilder;
import com.cloudera.flume.core.Attributes;
import com.cloudera.flume.core.Event;
import com.cloudera.flume.core.EventSink;
import com.cloudera.flume.core.Attributes.Type;
import com.cloudera.util.NetUtils;
import com.google.common.base.Preconditions;

/**
 * This sink takes a specified list of ganglia gmond servers, a event attribute
 * (from a report/event), a units name, and a Flume attribute type, and emits
 * XDR formatted UDP packets so that the metric can be seen in Ganglia. This
 * specifically supports the ganglia >=3.1.x wire format.
 * 
 * Originally based on o.a.h.metrics.ganglia.GangliaContext31 / HADOOP-4675
 * 
 * XDR spec http://www.faqs.org/rfcs/rfc1832.html
 * 
 * This is not thread safe.
 */
public class GangliaSink extends EventSink.Base {

  private static final String DEFAULT_GROUP = "flume";
  private static final int DEFAULT_TMAX = 60;
  private static final int DEFAULT_DMAX = 0;
  private static final int DEFAULT_PORT = 8649;
  private static final int BUFFER_SIZE = 1500; // as per libgmond.c

  private static final Logger LOG = LoggerFactory.getLogger(GangliaSink.class);
  private final String servers;

  private static final Map<Type, String> typeTable = new HashMap<Type, String>() {
    private static final long serialVersionUID = 1L;
    {
      put(Type.STRING, "string");
      put(Type.INT, "int32");
      put(Type.LONG, "float"); // Conversion happens here, originally from
      // hadoop source
      put(Type.DOUBLE, "double");
    }
  };

  private byte[] buffer = new byte[BUFFER_SIZE];
  private int offset;

  private List<? extends SocketAddress> metricsServers;

  private DatagramSocket datagramSocket;
  final private String attr; // turns into the metric name.
  final private String units;
  final private Type type;

  /** Creates a new instance of GangliaContext */
  public GangliaSink(String gangliaSvrs, String attr, String units, Type t) {
    this.servers = gangliaSvrs;
    this.attr = attr;
    this.units = units;
    this.type = t;
  }

  /**
   * This currently only outputs one metric from a report, and relies on the
   * sink constructor to give details about the typing information.
   * 
   * This may become moot if typing information is provided by flume's data.
   */
  @Override
  public void append(Event e) throws IOException, InterruptedException {
    String value;
    switch (type) {
    case LONG: {
      Long l = Attributes.readLong(e, attr);
      if (l == null) {
        // attribute not present, drop
        return;
      }
      value = l.toString();
      break;
    }
    case INT: {
      Integer i = Attributes.readInt(e, attr);
      if (i == null) {
        // attribute not present, drop
        return;
      }
      value = i.toString();
      break;
    }
    case STRING: {
      String s = Attributes.readString(e, attr);
      if (s == null) {
        // attribute not present, drop
        return;
      }
      value = s;
      break;
    }
    case DOUBLE: {
      Double d = Attributes.readDouble(e, attr);
      if (d == null) {
        // attribute not present,drop
        return;
      }
      value = d.toString();
      break;
    }
    default:
      return;
    }
    emitMetric(attr, typeTable.get(type), value, units);
    super.append(e);
  }

  @Override
  public void open() throws IOException, InterruptedException {
    // TODO (jon) need to worry about SecurityException and other
    // RuntimeExceptions.

    // From o.a.h.metrics.spi.Util
    // This can throw IllegalArgumentException or SecurityException
    metricsServers = Util.parse(servers, DEFAULT_PORT);

    try {
      datagramSocket = new DatagramSocket();
    } catch (SocketException se) {
      LOG.warn("problem with ganglia socket", se);
    }

  }

  /**
   * Not thread safe
   */
  @Override
  public void close() throws IOException, InterruptedException {
    if (datagramSocket == null) {
      LOG.warn("Double close");
      return;
    }
    datagramSocket.close();
    datagramSocket = null;
  }

  // //////////////////////////////////////////////////////////////////
  // This is ganglia >= 3.1.x wire format. Basically ripped out of
  // HADOOP-4675

  /**
   * This takes a metric and then send it off to the listening ganglia gmond
   * ports via udp.
   * 
   * This is not thread safe.
   */
  private void emitMetric(String name, String type, String value, String units)
      throws IOException {
    int slope = 3; // see gmetric.c
    int tmax = DEFAULT_TMAX;
    int dmax = DEFAULT_DMAX;
    String hostName = NetUtils.localhost();

    // First we send out a metadata message
    xdr_int(128); // metric_id = metadata_msg
    xdr_string(hostName); // hostname
    xdr_string(name); // metric name
    xdr_int(0); // spoof = False
    xdr_string(type); // metric type
    xdr_string(name); // metric name
    xdr_string(units); // units
    xdr_int(slope); // slope
    xdr_int(tmax); // tmax, the maximum time between metrics
    xdr_int(dmax); // dmax, the maximum data value
    xdr_int(1); // ???

    /*
     * Num of the entries in extra_value field for Ganglia 3.1.x
     */
    xdr_string("GROUP"); /* Group attribute */
    xdr_string(DEFAULT_GROUP); /* Group value */

    // send to each ganglia metrics listener/server
    for (SocketAddress socketAddress : metricsServers) {
      DatagramPacket packet = new DatagramPacket(buffer, offset, socketAddress);
      datagramSocket.send(packet);
    }

    // Now we send out a message with the actual value.
    // Technically, we only need to send out the metadata message once for
    // each metric, but I don't want to have to record which metrics we did and
    // did not send.
    offset = 0;
    xdr_int(133); // we are sending a string value
    xdr_string(hostName); // hostName
    xdr_string(name); // metric name
    xdr_int(0); // spoof = False
    xdr_string("%s"); // format field
    xdr_string(value); // metric value

    for (SocketAddress socketAddress : metricsServers) {
      DatagramPacket packet = new DatagramPacket(buffer, offset, socketAddress);
      datagramSocket.send(packet);
    }

  }

  /**
   * Puts a string into the buffer by first writing the size of the string as an
   * int, followed by the bytes of the string, padded if necessary to a multiple
   * of 4.
   */
  private void xdr_string(String s) {
    byte[] bytes = s.getBytes();
    int len = bytes.length;
    xdr_int(len);
    System.arraycopy(bytes, 0, buffer, offset, len);
    offset += len;
    pad();
  }

  /**
   * Pads the buffer with zero bytes up to the nearest multiple of 4.
   * 
   * Not thread safe
   */
  private void pad() {
    int newOffset = ((offset + 3) / 4) * 4;
    while (offset < newOffset) {
      buffer[offset++] = 0;
    }
  }

  /**
   * Puts an integer into the buffer as 4 bytes, big-endian.
   * 
   * Not thread safe
   */
  private void xdr_int(int i) {
    buffer[offset++] = (byte) ((i >> 24) & 0xff);
    buffer[offset++] = (byte) ((i >> 16) & 0xff);
    buffer[offset++] = (byte) ((i >> 8) & 0xff);
    buffer[offset++] = (byte) (i & 0xff);
  }

  // end of rip
  // //////////////////////////////////////////////////////////////////

  public static SinkBuilder builder() {
    return new SinkBuilder() {
      @Override
      public EventSink build(Context context, String... argv) {
        Preconditions.checkArgument(argv.length >= 3 && argv.length <= 4,
            "usage: ganglia(\"attr\", \"units\",\"type\"[, \"gmondservers\"])");

        String ganglias = FlumeConfiguration.get().getGangliaServers(); // default
        String attr = argv[0];
        String units = argv[1];

        String type = argv[2];
        Type t = null;
        // These strings are flume types, not xdr types
        if (type.equals("int")) {
          t = Type.INT;
        } else if (type.equals("long")) {
          t = Type.LONG;
        } else if (type.equals("double")) {
          t = Type.DOUBLE;
        } else if (type.equals("string")) {
          t = Type.STRING;
        } else {
          throw new IllegalArgumentException(
              "Illegal ganglia xdr type: "
                  + type
                  + " != int|long|double|string\n"
                  + "usage: ganglia(\"attr\", \"units\",\"type\"[, \"gmondservers\"]");
        }

        if (argv.length >= 4) {
          ganglias = argv[3];
        }
        EventSink snk = new GangliaSink(ganglias, attr, units, t);
        return snk;

      }
    };
  }
}