PageRenderTime 51ms CodeModel.GetById 20ms RepoModel.GetById 0ms app.codeStats 0ms

/src/com/clavain/muninmxcd.java

https://gitlab.com/goolic/MuninMX
Java | 407 lines | 296 code | 44 blank | 67 comment | 23 complexity | e408f10ba704f67154ae1d51188c7293 MD5 | raw file
  1. /*
  2. * MuninMX
  3. * Written by Enrico Kern, kern@clavain.com
  4. * www.clavain.com
  5. *
  6. * Licensed to the Apache Software Foundation (ASF) under one
  7. * or more contributor license agreements. See the NOTICE file
  8. * distributed with this work for additional information
  9. * regarding copyright ownership. The ASF licenses this file
  10. * to you under the Apache License, Version 2.0 (the
  11. * "License"); you may not use this file except in compliance
  12. * with the License. You may obtain a copy of the License at
  13. *
  14. * http://www.apache.org/licenses/LICENSE-2.0
  15. *
  16. * Unless required by applicable law or agreed to in writing,
  17. * software distributed under the License is distributed on an
  18. * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  19. * KIND, either express or implied. See the License for the
  20. * specific language governing permissions and limitations
  21. * under the License.
  22. *
  23. *
  24. *
  25. * special thanks to the unbelievable machine company GmbH
  26. * www.unbelievable-machine.com
  27. */
  28. package com.clavain;
  29. import com.clavain.alerts.Alert;
  30. import com.clavain.alerts.msg.PushOverMessage;
  31. import com.clavain.alerts.msg.ShortTextMessage;
  32. import com.clavain.alerts.msg.TTSMessage;
  33. import com.clavain.alerts.ratelimiter.PushOverLimiter;
  34. import com.clavain.alerts.ratelimiter.SMSLimiter;
  35. import com.clavain.alerts.ratelimiter.TTSLimiter;
  36. import com.clavain.checks.ReturnDebugTrace;
  37. import com.mongodb.BasicDBObject;
  38. import com.mongodb.DB;
  39. import com.mongodb.DBCollection;
  40. import com.mongodb.MongoClient;
  41. import com.mongodb.MongoClientOptions;
  42. import com.mongodb.ServerAddress;
  43. import com.mongodb.WriteConcern;
  44. import com.clavain.executors.MongoExecutor;
  45. import com.clavain.executors.MongoEssentialExecutor;
  46. import com.clavain.handlers.JettyLauncher;
  47. import com.clavain.json.ServiceCheck;
  48. import com.clavain.munin.MuninNode;
  49. import com.clavain.munin.MuninPlugin;
  50. import com.clavain.rca.Analyzer;
  51. import com.clavain.workers.PluginUpdater;
  52. import java.io.FileInputStream;
  53. import java.sql.Connection;
  54. import java.sql.DriverManager;
  55. import java.sql.ResultSet;
  56. import java.util.ArrayList;
  57. import java.util.Properties;
  58. import java.util.concurrent.ExecutorService;
  59. import java.util.concurrent.Executors;
  60. import java.util.concurrent.LinkedBlockingQueue;
  61. import org.apache.log4j.ConsoleAppender;
  62. import org.apache.log4j.FileAppender;
  63. import org.apache.log4j.Level;
  64. import org.apache.log4j.Logger;
  65. import org.apache.log4j.PatternLayout;
  66. import static com.clavain.utils.Database.*;
  67. import static com.clavain.utils.Quartz.*;
  68. import com.clavain.utils.SocketCheck;
  69. import com.clavain.workers.MongoWorker;
  70. import java.util.Collections;
  71. import java.util.concurrent.CopyOnWriteArrayList;
  72. import org.quartz.Scheduler;
  73. import org.quartz.SchedulerFactory;
  74. import org.quartz.impl.StdSchedulerFactory;
  75. import static com.clavain.utils.Generic.getUnixtime;
  76. import com.clavain.workers.NewNodeWatcher;
  77. import static com.clavain.utils.Database.dbScheduleAllCustomJobs;
  78. import com.clavain.workers.DataRetentionWorker;
  79. import java.io.File;
  80. import java.io.ObjectInputStream;
  81. import com.clavain.checks.ReturnServiceCheck;
  82. import com.clavain.executors.MongoCheckExecutor;
  83. import com.clavain.workers.ErrorNotifyExecutor;
  84. import com.google.gson.Gson;
  85. import java.util.List;
  86. /**
  87. *
  88. * @author enricokern
  89. */
  90. public class muninmxcd {
  91. public static Logger logger = Logger.getRootLogger();
  92. public static Properties p = null;
  93. public static LinkedBlockingQueue<BasicDBObject> mongo_queue = new LinkedBlockingQueue<BasicDBObject>();
  94. public static LinkedBlockingQueue<BasicDBObject> mongo_essential_queue = new LinkedBlockingQueue<BasicDBObject>();
  95. public static LinkedBlockingQueue<BasicDBObject> mongo_check_queue = new LinkedBlockingQueue<BasicDBObject>();
  96. public static boolean logMore = false;
  97. public static MongoClient m;
  98. public static DB db;
  99. public static DBCollection col;
  100. public static String version = "1.0 <Codename: Regret>";
  101. public static Connection conn = null;
  102. public static CopyOnWriteArrayList<MuninNode> v_munin_nodes;
  103. public static CopyOnWriteArrayList<MuninPlugin> v_cinterval_plugins;
  104. public static Scheduler sched;
  105. public static Scheduler sched_custom;
  106. public static Scheduler sched_checks;
  107. public static CopyOnWriteArrayList<SocketCheck> v_sockets;
  108. // alerting
  109. public static LinkedBlockingQueue<PushOverMessage> notification_pushover_queue = new LinkedBlockingQueue<PushOverMessage>();
  110. public static LinkedBlockingQueue<ShortTextMessage> notification_sms_queue = new LinkedBlockingQueue<ShortTextMessage>();
  111. public static LinkedBlockingQueue<TTSMessage> notification_tts_queue = new LinkedBlockingQueue<TTSMessage>();
  112. public static LinkedBlockingQueue<ReturnServiceCheck> check_error_queue = new LinkedBlockingQueue<ReturnServiceCheck>();
  113. public static CopyOnWriteArrayList<Alert> v_alerts = new CopyOnWriteArrayList<>();
  114. // RCA
  115. public static CopyOnWriteArrayList<Analyzer> v_analyzer = new CopyOnWriteArrayList<>();
  116. public static int rcajobs_running = 0;
  117. public static int maxnodes = 100000;
  118. public static int maxchecks = 100000;
  119. public static int socketTimeout = 30000;
  120. // CHECKS
  121. public static CopyOnWriteArrayList<ServiceCheck> v_serviceChecks = new CopyOnWriteArrayList<>();;
  122. public static List<Integer> errorProcessing = new CopyOnWriteArrayList<Integer>();
  123. public static String[] initialArgs;
  124. /**
  125. * @param args the command line arguments
  126. */
  127. public static void main(String[] args) {
  128. if(args.length < 1)
  129. {
  130. System.err.println("Usage: java -jar MuninMXcd.jar <full path to config>");
  131. System.exit(1);
  132. }
  133. try
  134. {
  135. initialArgs = args;
  136. p = new Properties();
  137. FileInputStream propInFile = null;
  138. propInFile = new FileInputStream(args[0]);
  139. p.loadFromXML(propInFile);
  140. String logfile = p.getProperty("log.file");
  141. socketTimeout = Integer.parseInt(p.getProperty("socket.timeout"));
  142. PatternLayout layout = new PatternLayout("%d{ISO8601} %-5p %m%n");
  143. ConsoleAppender consoleAppender = new ConsoleAppender( layout );
  144. logger.addAppender( consoleAppender );
  145. FileAppender fileAppender = new FileAppender( layout, logfile, false );
  146. logger.addAppender( fileAppender );
  147. logger.info("MuninMX Collector Daemon - " + version + " starting up...");
  148. logger.info("Loading configuration from <"+args[0]+">" );
  149. String l_strLogLevel = p.getProperty("log.level");
  150. // ALL | DEBUG | INFO | WARN | ERROR | FATAL | OFF:
  151. if(l_strLogLevel.equals("ALL"))
  152. {
  153. logger.setLevel( Level.ALL );
  154. }
  155. else if (l_strLogLevel.equals("DEBUG"))
  156. {
  157. logger.setLevel( Level.DEBUG);
  158. }
  159. else if (l_strLogLevel.equals("INFO"))
  160. {
  161. logger.setLevel( Level.INFO);
  162. }
  163. else if (l_strLogLevel.equals("WARN"))
  164. {
  165. logger.setLevel( Level.WARN);
  166. }
  167. else if (l_strLogLevel.equals("ERROR"))
  168. {
  169. logger.setLevel( Level.ERROR);
  170. }
  171. else if (l_strLogLevel.equals("FATAL"))
  172. {
  173. logger.setLevel( Level.FATAL);
  174. }
  175. else
  176. {
  177. logger.setLevel( Level.OFF);
  178. }
  179. if(p.getProperty("log.more") != null)
  180. {
  181. if(p.getProperty("log.more").equals("true"))
  182. {
  183. logMore = true;
  184. }
  185. }
  186. } catch (Exception ex)
  187. {
  188. System.err.println("Failed to Init basic logging infastructure: " + ex.getLocalizedMessage());
  189. System.exit(1);
  190. }
  191. try
  192. {
  193. // connect to db
  194. logger.info("Connecting to TokuMX");
  195. MongoClientOptions.Builder builder = new MongoClientOptions.Builder();
  196. builder.connectionsPerHost(400);
  197. builder.autoConnectRetry(true);
  198. builder.threadsAllowedToBlockForConnectionMultiplier(10);
  199. // speed up inserts, we dont care if we miss some if the shit hits the fan
  200. builder.writeConcern(WriteConcern.NONE);
  201. m = new MongoClient( new ServerAddress(p.getProperty("mongo.host")), builder.build());
  202. // connect to mysql
  203. connectToDatabase(p);
  204. // PreFilling Nodes, max 100 in concurrent
  205. logger.info("Loading initial MuninNode details. This can take a few minutes...");
  206. v_munin_nodes = new CopyOnWriteArrayList<>();
  207. v_cinterval_plugins = new CopyOnWriteArrayList<>();
  208. v_sockets = new CopyOnWriteArrayList<>();
  209. java.sql.Statement stmt = conn.createStatement();
  210. ResultSet rs = stmt.executeQuery("SELECT * FROM nodes");
  211. while(rs.next())
  212. {
  213. MuninNode mn = new MuninNode();
  214. mn.setHostname(rs.getString("hostname"));
  215. mn.setNodename(rs.getString("hostname"));
  216. mn.setNode_id(rs.getInt("id"));
  217. mn.setPort(rs.getInt("port"));
  218. mn.setUser_id(rs.getInt("user_id"));
  219. mn.setQueryInterval(rs.getInt("query_interval"));
  220. mn.setStr_via(rs.getString("via_host"));
  221. mn.setAuthpw(rs.getString("authpw"));
  222. mn.setGroup(rs.getString("groupname"));
  223. v_munin_nodes.add(mn);
  224. logger.info("* " + mn.getHostname() + " queued for pluginfetch");
  225. }
  226. // launching quartz scheduler
  227. logger.info("Launching Scheduler");
  228. SchedulerFactory sf = new StdSchedulerFactory("quartz.properties");
  229. sched = sf.getScheduler();
  230. sched.start();
  231. // launching quartz scheduler for custom interval
  232. logger.info("Launching Custom Interval Scheduler");
  233. SchedulerFactory sfc = new StdSchedulerFactory("customquartz.properties");
  234. sched_custom = sfc.getScheduler();
  235. sched_custom.start();
  236. // starting API server
  237. new Thread(new JettyLauncher()).start();
  238. int sleepTime = Integer.parseInt(p.getProperty("startup.sleeptime"));
  239. int startupIterations = Integer.parseInt(p.getProperty("startup.iterations"));
  240. // scheduling jobs
  241. int i = 0;
  242. for (MuninNode it_mn : v_munin_nodes) {
  243. if(i == startupIterations)
  244. {
  245. Thread.sleep(sleepTime);
  246. i = 0;
  247. logger.info("Waiting "+sleepTime+"ms for new scheduling slot");
  248. }
  249. scheduleJob(it_mn);
  250. i++;
  251. }
  252. // schedule custom interval jobs
  253. dbScheduleAllCustomJobs();
  254. // add all alerts
  255. dbAddAllAlerts();
  256. // Service Checks
  257. logger.info("Launching Service Check Scheduler");
  258. SchedulerFactory sfsc = new StdSchedulerFactory("checksquartz.properties");
  259. sched_checks = sfsc.getScheduler();
  260. sched_checks.start();
  261. // load service checks from database
  262. stmt = conn.createStatement();
  263. rs = stmt.executeQuery("SELECT * FROM service_checks");
  264. while(rs.next())
  265. {
  266. Gson gson = new Gson();
  267. ServiceCheck tc = gson.fromJson(rs.getString("json"), ServiceCheck.class);
  268. tc.setCid(rs.getInt("id"));
  269. tc.setUser_id(rs.getInt("user_id"));
  270. v_serviceChecks.add(tc);
  271. logger.info("* " + tc.getCheckname() + " Service Check added");
  272. }
  273. // queue service checks
  274. for (ServiceCheck it_sc : v_serviceChecks) {
  275. scheduleServiceCheck(it_sc);
  276. }
  277. // starting MongoExecutor
  278. new Thread(new MongoExecutor()).start();
  279. // starting MongoExecutor for Package Tracking and Essential Informations
  280. new Thread(new MongoEssentialExecutor()).start();
  281. // starting MongoExecutor for Service Checks
  282. new Thread(new MongoCheckExecutor()).start();
  283. // starting newnodewatcher
  284. new Thread(new NewNodeWatcher()).start();
  285. // start pushover sending message
  286. new Thread(new PushOverLimiter()).start();
  287. // SMS Limiter
  288. new Thread(new SMSLimiter()).start();
  289. // TTS Limiter
  290. new Thread(new TTSLimiter()).start();
  291. // start DataRetention Worker
  292. new Thread(new DataRetentionWorker()).start();
  293. // start Error Notify Inspector
  294. new Thread(new ErrorNotifyExecutor()).start();
  295. int curTime;
  296. int toTime;
  297. int mb = 1024*1024;
  298. while(true)
  299. {
  300. Thread.sleep(5000);
  301. System.out.println("Mongo Queue Size: " + mongo_queue.size());
  302. System.out.println("Mongo Check Queue Size: " + mongo_check_queue.size());
  303. System.out.println("Mongo Essential Queue Size: " + mongo_essential_queue.size());
  304. Runtime runtime = Runtime.getRuntime();
  305. //Print used memory
  306. System.out.println("Used Memory:"
  307. + (runtime.totalMemory() - runtime.freeMemory()) / mb);
  308. //Print free memory
  309. System.out.println("Free Memory:"
  310. + runtime.freeMemory() / mb);
  311. //Print total available memory
  312. System.out.println("Total Memory:" + runtime.totalMemory() / mb);
  313. //Print Maximum available memory
  314. System.out.println("Max Memory:" + runtime.maxMemory() / mb);
  315. System.out.println(" ");
  316. if(p.getProperty("kill.sockets").equals("true"))
  317. {
  318. System.out.println("Sockets: " + v_sockets.size());
  319. // check for sockets that we can kill
  320. curTime = getUnixtime();
  321. for(SocketCheck sc : v_sockets)
  322. {
  323. toTime = curTime - 120;
  324. if(sc.getSocketCreated() < toTime)
  325. {
  326. if(!sc.getSocket().isClosed())
  327. {
  328. logger.info("timing out socket... from: " + sc.getHostname());
  329. sc.closeSocket();
  330. v_sockets.remove(sc);
  331. }
  332. else
  333. {
  334. v_sockets.remove(sc);
  335. }
  336. }
  337. }
  338. }
  339. }
  340. } catch (Exception ex)
  341. {
  342. System.err.println("Something went wrong as fuck: " + ex.getLocalizedMessage());
  343. logger.fatal("Something went wrong as fuck. exiting: " + ex.getLocalizedMessage());
  344. ex.printStackTrace();
  345. System.exit(1);
  346. }
  347. }
  348. // Establish a connection to the database
  349. private static boolean connectToDatabase(Properties p)
  350. {
  351. try {
  352. logger.info("Connecting to MySQL");
  353. conn =
  354. DriverManager.getConnection("jdbc:mysql://"+p.getProperty("mysql.host")+":"+p.getProperty("mysql.port")+"/"+p.getProperty("mysql.db")+"?" +
  355. "user="+p.getProperty("mysql.user")+"&password="+p.getProperty("mysql.pass")+"&autoReconnect=true&failOverReadOnly=false&maxReconnects=10");
  356. return(true);
  357. } catch (Exception ex) {
  358. // handle any errors
  359. logger.fatal("Error connecting to database: " + ex.getMessage());
  360. return(false);
  361. }
  362. }
  363. }