/flume-core/src/main/java/com/cloudera/flume/reporter/ganglia/GangliaSink.java
Java | 338 lines | 201 code | 36 blank | 101 comment | 24 complexity | 585563d9ff7f2ba3db1c2685aa717841 MD5 | raw file
Possible License(s): Apache-2.0
- /**
- * Licensed to Cloudera, Inc. under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. Cloudera, Inc. licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
- /*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
- package com.cloudera.flume.reporter.ganglia;
- import java.io.IOException;
- import java.net.DatagramPacket;
- import java.net.DatagramSocket;
- import java.net.SocketAddress;
- import java.net.SocketException;
- import java.util.HashMap;
- import java.util.List;
- import java.util.Map;
- import org.apache.hadoop.metrics.spi.Util;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
- import com.cloudera.flume.conf.Context;
- import com.cloudera.flume.conf.FlumeConfiguration;
- import com.cloudera.flume.conf.SinkFactory.SinkBuilder;
- import com.cloudera.flume.core.Attributes;
- import com.cloudera.flume.core.Event;
- import com.cloudera.flume.core.EventSink;
- import com.cloudera.flume.core.Attributes.Type;
- import com.cloudera.util.NetUtils;
- import com.google.common.base.Preconditions;
- /**
- * This sink takes a specified list of ganglia gmond servers, a event attribute
- * (from a report/event), a units name, and a Flume attribute type, and emits
- * XDR formatted UDP packets so that the metric can be seen in Ganglia. This
- * specifically supports the ganglia >=3.1.x wire format.
- *
- * Originally based on o.a.h.metrics.ganglia.GangliaContext31 / HADOOP-4675
- *
- * XDR spec http://www.faqs.org/rfcs/rfc1832.html
- *
- * This is not thread safe.
- */
- public class GangliaSink extends EventSink.Base {
- private static final String DEFAULT_GROUP = "flume";
- private static final int DEFAULT_TMAX = 60;
- private static final int DEFAULT_DMAX = 0;
- private static final int DEFAULT_PORT = 8649;
- private static final int BUFFER_SIZE = 1500; // as per libgmond.c
- private static final Logger LOG = LoggerFactory.getLogger(GangliaSink.class);
- private final String servers;
- private static final Map<Type, String> typeTable = new HashMap<Type, String>() {
- private static final long serialVersionUID = 1L;
- {
- put(Type.STRING, "string");
- put(Type.INT, "int32");
- put(Type.LONG, "float"); // Conversion happens here, originally from
- // hadoop source
- put(Type.DOUBLE, "double");
- }
- };
- private byte[] buffer = new byte[BUFFER_SIZE];
- private int offset;
- private List<? extends SocketAddress> metricsServers;
- private DatagramSocket datagramSocket;
- final private String attr; // turns into the metric name.
- final private String units;
- final private Type type;
- /** Creates a new instance of GangliaContext */
- public GangliaSink(String gangliaSvrs, String attr, String units, Type t) {
- this.servers = gangliaSvrs;
- this.attr = attr;
- this.units = units;
- this.type = t;
- }
- /**
- * This currently only outputs one metric from a report, and relies on the
- * sink constructor to give details about the typing information.
- *
- * This may become moot if typing information is provided by flume's data.
- */
- @Override
- public void append(Event e) throws IOException, InterruptedException {
- String value;
- switch (type) {
- case LONG: {
- Long l = Attributes.readLong(e, attr);
- if (l == null) {
- // attribute not present, drop
- return;
- }
- value = l.toString();
- break;
- }
- case INT: {
- Integer i = Attributes.readInt(e, attr);
- if (i == null) {
- // attribute not present, drop
- return;
- }
- value = i.toString();
- break;
- }
- case STRING: {
- String s = Attributes.readString(e, attr);
- if (s == null) {
- // attribute not present, drop
- return;
- }
- value = s;
- break;
- }
- case DOUBLE: {
- Double d = Attributes.readDouble(e, attr);
- if (d == null) {
- // attribute not present,drop
- return;
- }
- value = d.toString();
- break;
- }
- default:
- return;
- }
- emitMetric(attr, typeTable.get(type), value, units);
- super.append(e);
- }
- @Override
- public void open() throws IOException, InterruptedException {
- // TODO (jon) need to worry about SecurityException and other
- // RuntimeExceptions.
- // From o.a.h.metrics.spi.Util
- // This can throw IllegalArgumentException or SecurityException
- metricsServers = Util.parse(servers, DEFAULT_PORT);
- try {
- datagramSocket = new DatagramSocket();
- } catch (SocketException se) {
- LOG.warn("problem with ganglia socket", se);
- }
- }
- /**
- * Not thread safe
- */
- @Override
- public void close() throws IOException, InterruptedException {
- if (datagramSocket == null) {
- LOG.warn("Double close");
- return;
- }
- datagramSocket.close();
- datagramSocket = null;
- }
- // //////////////////////////////////////////////////////////////////
- // This is ganglia >= 3.1.x wire format. Basically ripped out of
- // HADOOP-4675
- /**
- * This takes a metric and then send it off to the listening ganglia gmond
- * ports via udp.
- *
- * This is not thread safe.
- */
- private void emitMetric(String name, String type, String value, String units)
- throws IOException {
- int slope = 3; // see gmetric.c
- int tmax = DEFAULT_TMAX;
- int dmax = DEFAULT_DMAX;
- String hostName = NetUtils.localhost();
- // First we send out a metadata message
- xdr_int(128); // metric_id = metadata_msg
- xdr_string(hostName); // hostname
- xdr_string(name); // metric name
- xdr_int(0); // spoof = False
- xdr_string(type); // metric type
- xdr_string(name); // metric name
- xdr_string(units); // units
- xdr_int(slope); // slope
- xdr_int(tmax); // tmax, the maximum time between metrics
- xdr_int(dmax); // dmax, the maximum data value
- xdr_int(1); // ???
- /*
- * Num of the entries in extra_value field for Ganglia 3.1.x
- */
- xdr_string("GROUP"); /* Group attribute */
- xdr_string(DEFAULT_GROUP); /* Group value */
- // send to each ganglia metrics listener/server
- for (SocketAddress socketAddress : metricsServers) {
- DatagramPacket packet = new DatagramPacket(buffer, offset, socketAddress);
- datagramSocket.send(packet);
- }
- // Now we send out a message with the actual value.
- // Technically, we only need to send out the metadata message once for
- // each metric, but I don't want to have to record which metrics we did and
- // did not send.
- offset = 0;
- xdr_int(133); // we are sending a string value
- xdr_string(hostName); // hostName
- xdr_string(name); // metric name
- xdr_int(0); // spoof = False
- xdr_string("%s"); // format field
- xdr_string(value); // metric value
- for (SocketAddress socketAddress : metricsServers) {
- DatagramPacket packet = new DatagramPacket(buffer, offset, socketAddress);
- datagramSocket.send(packet);
- }
- }
- /**
- * Puts a string into the buffer by first writing the size of the string as an
- * int, followed by the bytes of the string, padded if necessary to a multiple
- * of 4.
- */
- private void xdr_string(String s) {
- byte[] bytes = s.getBytes();
- int len = bytes.length;
- xdr_int(len);
- System.arraycopy(bytes, 0, buffer, offset, len);
- offset += len;
- pad();
- }
- /**
- * Pads the buffer with zero bytes up to the nearest multiple of 4.
- *
- * Not thread safe
- */
- private void pad() {
- int newOffset = ((offset + 3) / 4) * 4;
- while (offset < newOffset) {
- buffer[offset++] = 0;
- }
- }
- /**
- * Puts an integer into the buffer as 4 bytes, big-endian.
- *
- * Not thread safe
- */
- private void xdr_int(int i) {
- buffer[offset++] = (byte) ((i >> 24) & 0xff);
- buffer[offset++] = (byte) ((i >> 16) & 0xff);
- buffer[offset++] = (byte) ((i >> 8) & 0xff);
- buffer[offset++] = (byte) (i & 0xff);
- }
- // end of rip
- // //////////////////////////////////////////////////////////////////
- public static SinkBuilder builder() {
- return new SinkBuilder() {
- @Override
- public EventSink build(Context context, String... argv) {
- Preconditions.checkArgument(argv.length >= 3 && argv.length <= 4,
- "usage: ganglia(\"attr\", \"units\",\"type\"[, \"gmondservers\"])");
- String ganglias = FlumeConfiguration.get().getGangliaServers(); // default
- String attr = argv[0];
- String units = argv[1];
- String type = argv[2];
- Type t = null;
- // These strings are flume types, not xdr types
- if (type.equals("int")) {
- t = Type.INT;
- } else if (type.equals("long")) {
- t = Type.LONG;
- } else if (type.equals("double")) {
- t = Type.DOUBLE;
- } else if (type.equals("string")) {
- t = Type.STRING;
- } else {
- throw new IllegalArgumentException(
- "Illegal ganglia xdr type: "
- + type
- + " != int|long|double|string\n"
- + "usage: ganglia(\"attr\", \"units\",\"type\"[, \"gmondservers\"]");
- }
- if (argv.length >= 4) {
- ganglias = argv[3];
- }
- EventSink snk = new GangliaSink(ganglias, attr, units, t);
- return snk;
- }
- };
- }
- }