PageRenderTime 70ms CodeModel.GetById 32ms RepoModel.GetById 0ms app.codeStats 0ms

/gmonitor/gmonitor-core/src/main/java/com/zdingke/gmonitor/trans/MetricTrans.java

https://gitlab.com/zhengdingke/gmonitor
Java | 535 lines | 463 code | 65 blank | 7 comment | 74 complexity | a6cf501e7077b10eeb7fda560715c492 MD5 | raw file
  1. package com.zdingke.gmonitor.trans;
  2. import java.io.File;
  3. import java.io.FileInputStream;
  4. import java.io.InputStream;
  5. import java.util.ArrayList;
  6. import java.util.Date;
  7. import java.util.HashMap;
  8. import java.util.Iterator;
  9. import java.util.List;
  10. import java.util.Map;
  11. import org.apache.commons.cli.CommandLine;
  12. import org.apache.commons.cli.CommandLineParser;
  13. import org.apache.commons.cli.GnuParser;
  14. import org.apache.commons.cli.HelpFormatter;
  15. import org.apache.commons.cli.Option;
  16. import org.apache.commons.cli.Options;
  17. import org.apache.commons.cli.ParseException;
  18. import org.apache.commons.io.FilenameUtils;
  19. import org.apache.commons.lang.RandomStringUtils;
  20. import org.apache.commons.logging.Log;
  21. import org.apache.commons.logging.LogFactory;
  22. import org.apache.commons.pool.KeyedObjectPool;
  23. import org.apache.commons.pool.impl.GenericKeyedObjectPool;
  24. import org.codehaus.jackson.map.ObjectMapper;
  25. import org.quartz.CronExpression;
  26. import org.quartz.CronTrigger;
  27. import org.quartz.JobDataMap;
  28. import org.quartz.JobDetail;
  29. import org.quartz.Scheduler;
  30. import org.quartz.SchedulerException;
  31. import org.quartz.Trigger;
  32. import org.quartz.TriggerUtils;
  33. import org.quartz.impl.StdSchedulerFactory;
  34. import com.alibaba.fastjson.JSON;
  35. import com.googlecode.jmxtrans.JmxTransformer;
  36. import com.googlecode.jmxtrans.OutputWriter;
  37. import com.googlecode.jmxtrans.jmx.ManagedGenericKeyedObjectPool;
  38. import com.googlecode.jmxtrans.jobs.ServerJob;
  39. import com.googlecode.jmxtrans.model.JmxProcess;
  40. import com.googlecode.jmxtrans.model.Query;
  41. import com.googlecode.jmxtrans.model.Server;
  42. import com.googlecode.jmxtrans.util.JmxUtils;
  43. import com.googlecode.jmxtrans.util.LifecycleException;
  44. import com.googlecode.jmxtrans.util.OptionsException;
  45. import com.googlecode.jmxtrans.util.ValidationException;
  46. import com.googlecode.jmxtrans.util.WatchDir;
  47. import com.googlecode.jmxtrans.util.WatchedCallback;
  48. import com.zdingke.gmonitor.utils.MetricJsonCreateUtil;
  49. public class MetricTrans implements WatchedCallback {
  50. private static final Log log = LogFactory.getLog(MetricTrans.class);
  51. private String quartPropertiesFile;
  52. private int runPeriod;
  53. private File jsonDirOrFile;
  54. private boolean runEndlessly;
  55. private Scheduler serverScheduler;
  56. private WatchDir watcher;
  57. private Map<String, KeyedObjectPool> poolMap;
  58. private Map<String, ManagedGenericKeyedObjectPool> poolMBeans;
  59. private List<Server> masterServersList;
  60. private Thread shutdownHook;
  61. private volatile boolean isRunning;
  62. public MetricTrans() {
  63. this.quartPropertiesFile = null;
  64. this.runPeriod = 60;
  65. this.runEndlessly = false;
  66. this.masterServersList = new ArrayList();
  67. this.shutdownHook = new ShutdownHook();
  68. this.isRunning = false;
  69. }
  70. public static void main(String[] args) throws Exception {
  71. MetricTrans transformer = new MetricTrans();
  72. transformer.doMain(args);
  73. }
  74. private void doMain(String[] args) throws Exception {
  75. if (!parseOptions(args)) {
  76. return;
  77. }
  78. // ManagedJmxTransformerProcess mbean = new
  79. // ManagedJmxTransformerProcess();
  80. // JmxUtils.registerJMX(mbean);
  81. start();
  82. damonReload();
  83. try {
  84. for (;;) {
  85. Thread.sleep(5L);
  86. }
  87. } catch (Exception e) {
  88. // JmxUtils.unregisterJMX(mbean);
  89. }
  90. }
  91. private void damonReload() {
  92. ReloadThread re = new ReloadThread(serverScheduler);
  93. new Thread(re).start();
  94. }
  95. public class ReloadThread implements Runnable {
  96. Scheduler serverScheduler;
  97. public ReloadThread(Scheduler serverScheduler) {
  98. this.serverScheduler = serverScheduler;
  99. }
  100. @Override
  101. public void run() {
  102. while (true) {
  103. try {
  104. Thread.sleep(60 * 60 * 1000L);
  105. if (log.isDebugEnabled()) {
  106. log.debug("reload conf");
  107. }
  108. deleteAllJobs(this.serverScheduler);
  109. startupSystem();
  110. } catch (Exception e) {
  111. log.error("reload error!");
  112. }
  113. }
  114. }
  115. }
  116. public synchronized void start() throws LifecycleException {
  117. if (this.isRunning) {
  118. throw new LifecycleException("Process already started");
  119. }
  120. log.info("Starting Jmxtrans on : " + this.jsonDirOrFile.toString());
  121. try {
  122. startupScheduler();
  123. startupWatchdir();
  124. setupObjectPooling();
  125. startupSystem();
  126. } catch (Exception e) {
  127. log.error(e.getMessage(), e);
  128. throw new LifecycleException(e);
  129. }
  130. Runtime.getRuntime().addShutdownHook(this.shutdownHook);
  131. this.isRunning = true;
  132. }
  133. public synchronized void stop() throws LifecycleException {
  134. if (!this.isRunning) {
  135. throw new LifecycleException("Process already stoped");
  136. }
  137. try {
  138. log.info("Stopping Jmxtrans");
  139. if (this.shutdownHook != null) {
  140. Runtime.getRuntime().removeShutdownHook(this.shutdownHook);
  141. }
  142. stopServices();
  143. this.isRunning = false;
  144. } catch (LifecycleException e) {
  145. log.error(e.getMessage(), e);
  146. throw new LifecycleException(e);
  147. }
  148. }
  149. private synchronized void stopServices() throws LifecycleException {
  150. try {
  151. if (this.serverScheduler != null) {
  152. this.serverScheduler.shutdown(true);
  153. log.debug("Shutdown server scheduler");
  154. try {
  155. Thread.sleep(1500L);
  156. } catch (InterruptedException e) {
  157. log.error(e.getMessage(), e);
  158. }
  159. this.serverScheduler = null;
  160. }
  161. if (this.watcher != null) {
  162. this.watcher.stopService();
  163. this.watcher = null;
  164. log.debug("Shutdown watch service");
  165. }
  166. // for (String key : this.poolMap.keySet()) {
  167. // JmxUtils.unregisterJMX((ManagedObject) this.poolMBeans.get(key));
  168. // }
  169. this.poolMBeans = null;
  170. for (Map.Entry<String, KeyedObjectPool> entry : this.poolMap.entrySet()) {
  171. try {
  172. entry.getValue().close();
  173. log.debug("Closed object pool factory: " + entry.getKey());
  174. } catch (Exception ex) {
  175. log.error("Error closing object pool factory: " + entry.getKey());
  176. }
  177. }
  178. this.poolMap = null;
  179. Iterator i$;
  180. Query query;
  181. for (Server server : this.masterServersList) {
  182. for (i$ = server.getQueries().iterator(); i$.hasNext();) {
  183. query = (Query) i$.next();
  184. for (OutputWriter writer : query.getOutputWriters()) {
  185. try {
  186. writer.stop();
  187. log.debug("Stopped writer: " + writer.getClass().getSimpleName() + " for query: " + query);
  188. } catch (LifecycleException ex) {
  189. log.error("Error stopping writer: " + writer.getClass().getSimpleName() + " for query: " + query);
  190. }
  191. }
  192. }
  193. }
  194. this.masterServersList.clear();
  195. } catch (Exception e) {
  196. log.error(e.getMessage(), e);
  197. throw new LifecycleException(e);
  198. }
  199. }
  200. private void startupWatchdir() throws Exception {
  201. File dirToWatch = null;
  202. if (getJsonDirOrFile().isFile()) {
  203. dirToWatch = new File(FilenameUtils.getFullPath(getJsonDirOrFile().getAbsolutePath()));
  204. } else {
  205. dirToWatch = getJsonDirOrFile();
  206. }
  207. this.watcher = new WatchDir(dirToWatch, this);
  208. this.watcher.start();
  209. }
  210. private void startupScheduler() throws Exception {
  211. StdSchedulerFactory serverSchedFact = new StdSchedulerFactory();
  212. InputStream stream = null;
  213. if (this.quartPropertiesFile == null) {
  214. stream = JmxTransformer.class.getResourceAsStream("/quartz.server.properties");
  215. } else {
  216. stream = new FileInputStream(this.quartPropertiesFile);
  217. }
  218. serverSchedFact.initialize(stream);
  219. this.serverScheduler = serverSchedFact.getScheduler();
  220. this.serverScheduler.start();
  221. }
  222. public void executeStandalone(JmxProcess process) throws Exception {
  223. this.masterServersList = process.getServers();
  224. startupScheduler();
  225. setupObjectPooling();
  226. processServersIntoJobs(this.serverScheduler);
  227. Thread.sleep(10000L);
  228. }
  229. private void startupSystem() throws LifecycleException {
  230. processFilesIntoServers(getJsonFiles());
  231. processServersIntoJobs(this.serverScheduler);
  232. }
  233. protected void setupObjectPooling() throws Exception {
  234. if (this.poolMap == null) {
  235. this.poolMap = JmxUtils.getDefaultPoolMap();
  236. this.poolMBeans = new HashMap();
  237. for (String key : this.poolMap.keySet()) {
  238. ManagedGenericKeyedObjectPool mbean = new ManagedGenericKeyedObjectPool((GenericKeyedObjectPool) this.poolMap.get(key));
  239. mbean.setPoolName(key);
  240. JmxUtils.registerJMX(mbean);
  241. this.poolMBeans.put(key, mbean);
  242. }
  243. }
  244. }
  245. private void validateSetup(List<Query> queries) throws ValidationException {
  246. for (Query q : queries) {
  247. validateSetup(q);
  248. }
  249. }
  250. private void validateSetup(Query query) throws ValidationException {
  251. List<OutputWriter> writers = query.getOutputWriters();
  252. if (writers != null) {
  253. for (OutputWriter w : writers) {
  254. w.validateSetup(query);
  255. }
  256. }
  257. }
  258. private void processFilesIntoServers(List<File> jsonFiles) throws LifecycleException {
  259. MetricJsonCreateUtil.getAllModuleJsonObj(getJsonDirOrFile().getPath().toString()).stream().forEach(m -> {
  260. ObjectMapper mapper = new ObjectMapper();
  261. JmxProcess process = null;
  262. try {
  263. process = mapper.readValue(JSON.toJSONString(m), JmxProcess.class);
  264. } catch (Exception e) {
  265. log.error(e.getMessage(), e);
  266. }
  267. JmxUtils.mergeServerLists(this.masterServersList, process.getServers());
  268. });
  269. }
  270. private void processServersIntoJobs(Scheduler scheduler) throws LifecycleException {
  271. for (Server server : this.masterServersList) {
  272. try {
  273. for (Query query : server.getQueries()) {
  274. query.setServer(server);
  275. for (OutputWriter writer : query.getOutputWriters()) {
  276. writer.setObjectPoolMap(this.poolMap);
  277. writer.start();
  278. }
  279. }
  280. validateSetup(server.getQueries());
  281. scheduleJob(scheduler, server);
  282. } catch (java.text.ParseException ex) {
  283. throw new LifecycleException("Error parsing cron expression: " + server.getCronExpression(), ex);
  284. } catch (SchedulerException ex) {
  285. throw new LifecycleException("Error scheduling job for server: " + server, ex);
  286. } catch (ValidationException ex) {
  287. throw new LifecycleException("Error validating json setup for query", ex);
  288. }
  289. }
  290. }
  291. private void scheduleJob(Scheduler scheduler, Server server) throws java.text.ParseException, SchedulerException {
  292. String name = server.getHost() + ":" + server.getPort() + "-" + System.currentTimeMillis() + "-" + RandomStringUtils.randomNumeric(10);
  293. JobDetail jd = new JobDetail(name, "ServerJob", ServerJob.class);
  294. JobDataMap map = new JobDataMap();
  295. map.put(Server.class.getName(), server);
  296. map.put(Server.JMX_CONNECTION_FACTORY_POOL, this.poolMap.get(Server.JMX_CONNECTION_FACTORY_POOL));
  297. jd.setJobDataMap(map);
  298. Trigger trigger = null;
  299. if ((server.getCronExpression() != null) && (CronExpression.isValidExpression(server.getCronExpression()))) {
  300. trigger = new CronTrigger();
  301. ((CronTrigger) trigger).setCronExpression(server.getCronExpression());
  302. ((CronTrigger) trigger).setName(server.getHost() + ":" + server.getPort() + "-" + Long.valueOf(System.currentTimeMillis()).toString());
  303. ((CronTrigger) trigger).setStartTime(new Date());
  304. } else {
  305. Trigger minuteTrigger = TriggerUtils.makeSecondlyTrigger(this.runPeriod);
  306. minuteTrigger.setName(server.getHost() + ":" + server.getPort() + "-" + Long.valueOf(System.currentTimeMillis()).toString());
  307. minuteTrigger.setStartTime(new Date());
  308. trigger = minuteTrigger;
  309. }
  310. scheduler.scheduleJob(jd, trigger);
  311. if (log.isDebugEnabled()) {
  312. log.debug("Scheduled job: " + jd.getName() + " for server: " + server);
  313. }
  314. }
  315. private void deleteAllJobs(Scheduler scheduler) throws Exception {
  316. List<JobDetail> allJobs = new ArrayList();
  317. String[] jobGroups = scheduler.getJobGroupNames();
  318. for (String jobGroup : jobGroups) {
  319. String[] jobNames = scheduler.getJobNames(jobGroup);
  320. for (String jobName : jobNames) {
  321. allJobs.add(scheduler.getJobDetail(jobName, jobGroup));
  322. }
  323. }
  324. for (JobDetail jd : allJobs) {
  325. scheduler.deleteJob(jd.getName(), jd.getGroup());
  326. if (log.isDebugEnabled()) {
  327. log.debug("Deleted scheduled job: " + jd.getName() + " group: " + jd.getGroup());
  328. }
  329. }
  330. }
  331. public void setRunEndlessly(boolean runEndlessly) {
  332. this.runEndlessly = runEndlessly;
  333. }
  334. public boolean isRunEndlessly() {
  335. return this.runEndlessly;
  336. }
  337. private boolean parseOptions(String[] args) throws OptionsException, ParseException {
  338. CommandLineParser parser = new GnuParser();
  339. CommandLine cl = parser.parse(getOptions(), args);
  340. Option[] options = cl.getOptions();
  341. boolean result = true;
  342. for (Option option : options) {
  343. if (option.getOpt().equals("j")) {
  344. File tmp = new File(option.getValue());
  345. if ((!tmp.exists()) && (!tmp.isDirectory())) {
  346. throw new OptionsException("Path to json directory is invalid: " + tmp);
  347. }
  348. setJsonDirOrFile(tmp);
  349. } else if (option.getOpt().equals("f")) {
  350. File tmp = new File(option.getValue());
  351. if ((!tmp.exists()) && (!tmp.isFile())) {
  352. throw new OptionsException("Path to json file is invalid: " + tmp);
  353. }
  354. setJsonDirOrFile(tmp);
  355. } else if (option.getOpt().equals("e")) {
  356. setRunEndlessly(true);
  357. } else if (option.getOpt().equals("q")) {
  358. setQuartPropertiesFile(option.getValue());
  359. File file = new File(option.getValue());
  360. if (!file.exists()) {
  361. throw new OptionsException("Could not find path to the quartz properties file: " + file.getAbsolutePath());
  362. }
  363. } else if (option.getOpt().equals("s")) {
  364. setRunPeriod(Integer.valueOf(option.getValue()).intValue());
  365. } else if (option.getOpt().equals("h")) {
  366. HelpFormatter formatter = new HelpFormatter();
  367. formatter.printHelp("java -jar jmxtrans-all.jar", getOptions());
  368. result = false;
  369. }
  370. }
  371. if ((result == true) && (getJsonDirOrFile() == null)) {
  372. throw new OptionsException("Please specify either the -f or -j option.");
  373. }
  374. return result;
  375. }
  376. public Options getOptions() {
  377. Options options = new Options();
  378. options.addOption("j", true, "Directory where json configuration is stored. Default is .");
  379. options.addOption("f", true, "A single json file to execute.");
  380. options.addOption("e", false, "Run endlessly. Default false.");
  381. options.addOption("q", true, "Path to quartz configuration file.");
  382. options.addOption("s", true, "Seconds between server job runs (not defined with cron). Default: 60");
  383. options.addOption("h", false, "Help");
  384. return options;
  385. }
  386. public String getQuartPropertiesFile() {
  387. return this.quartPropertiesFile;
  388. }
  389. public void setQuartPropertiesFile(String quartPropertiesFile) {
  390. this.quartPropertiesFile = quartPropertiesFile;
  391. }
  392. public int getRunPeriod() {
  393. return this.runPeriod;
  394. }
  395. public void setRunPeriod(int runPeriod) {
  396. this.runPeriod = runPeriod;
  397. }
  398. public void setJsonDirOrFile(File jsonDirOrFile) {
  399. this.jsonDirOrFile = jsonDirOrFile;
  400. }
  401. public File getJsonDirOrFile() {
  402. return this.jsonDirOrFile;
  403. }
  404. private List<File> getJsonFiles() {
  405. File[] files = null;
  406. if ((getJsonDirOrFile() != null) && (getJsonDirOrFile().isFile())) {
  407. files = new File[1];
  408. files[0] = getJsonDirOrFile();
  409. } else {
  410. files = getJsonDirOrFile().listFiles();
  411. }
  412. List<File> result = new ArrayList();
  413. for (File file : files) {
  414. if (isJsonFile(file)) {
  415. result.add(file);
  416. }
  417. }
  418. return result;
  419. }
  420. private boolean isJsonFile(File file) {
  421. if (getJsonDirOrFile().isFile()) {
  422. return file.equals(getJsonDirOrFile());
  423. }
  424. return file.getName().endsWith(".json");
  425. }
  426. @Override
  427. public void fileModified(File file) throws Exception {
  428. if (isJsonFile(file)) {
  429. Thread.sleep(1000L);
  430. if (log.isDebugEnabled()) {
  431. log.debug("File modified: " + file);
  432. }
  433. deleteAllJobs(this.serverScheduler);
  434. startupSystem();
  435. }
  436. }
  437. @Override
  438. public void fileDeleted(File file) throws Exception {
  439. if (isJsonFile(file)) {
  440. Thread.sleep(1000L);
  441. if (log.isDebugEnabled()) {
  442. log.debug("File deleted: " + file);
  443. }
  444. deleteAllJobs(this.serverScheduler);
  445. startupSystem();
  446. }
  447. }
  448. @Override
  449. public void fileAdded(File file) throws Exception {
  450. if (isJsonFile(file)) {
  451. Thread.sleep(1000L);
  452. if (log.isDebugEnabled()) {
  453. log.debug("File added: " + file);
  454. }
  455. startupSystem();
  456. }
  457. }
  458. protected class ShutdownHook extends Thread {
  459. protected ShutdownHook() {
  460. }
  461. @Override
  462. public void run() {
  463. try {
  464. MetricTrans.this.stopServices();
  465. } catch (LifecycleException e) {
  466. MetricTrans.log.error("Error shutdown hook", e);
  467. }
  468. }
  469. }
  470. }