PageRenderTime 51ms CodeModel.GetById 14ms RepoModel.GetById 1ms app.codeStats 0ms

/src/main/java/com/searchcode/app/service/JobService.java

https://github.com/boyter/searchcode-server
Java | 513 lines | 384 code | 68 blank | 61 comment | 18 complexity | 6b428c84675380618d195d8da21ba229 MD5 | raw file
  1. /*
  2. * Copyright (c) 2016 Boyter Online Services
  3. *
  4. * Use of this software is governed by the Fair Source License included
  5. * in the LICENSE.TXT file, but will be eventually open under GNU General Public License Version 3
  6. * see the README.md for when this clause will take effect
  7. *
  8. * Version 1.3.15
  9. */
  10. package com.searchcode.app.service;
  11. import com.searchcode.app.config.Values;
  12. import com.searchcode.app.dao.IRepo;
  13. import com.searchcode.app.jobs.DeleteRepositoryJob;
  14. import com.searchcode.app.jobs.PopulateSpellingCorrectorJob;
  15. import com.searchcode.app.jobs.enqueue.EnqueueFileRepositoryJob;
  16. import com.searchcode.app.jobs.enqueue.EnqueueRepositoryJob;
  17. import com.searchcode.app.jobs.enqueue.EnqueueSearchcodeRepositoryJob;
  18. import com.searchcode.app.jobs.repository.IndexDocumentsJob;
  19. import com.searchcode.app.jobs.repository.IndexFileRepoJob;
  20. import com.searchcode.app.jobs.repository.IndexGitRepoJob;
  21. import com.searchcode.app.jobs.repository.IndexSvnRepoJob;
  22. import com.searchcode.app.jobs.searchcode.ReindexerJob;
  23. import com.searchcode.app.model.RepoResult;
  24. import com.searchcode.app.service.index.IIndexService;
  25. import com.searchcode.app.util.Helpers;
  26. import com.searchcode.app.util.LoggerWrapper;
  27. import com.searchcode.app.util.Properties;
  28. import org.apache.commons.io.FileUtils;
  29. import org.apache.commons.lang3.SystemUtils;
  30. import org.quartz.Scheduler;
  31. import org.quartz.SchedulerException;
  32. import org.zeroturnaround.exec.ProcessExecutor;
  33. import java.io.File;
  34. import java.io.IOException;
  35. import java.text.DateFormat;
  36. import java.text.SimpleDateFormat;
  37. import java.util.Date;
  38. import static org.quartz.JobBuilder.newJob;
  39. import static org.quartz.SimpleScheduleBuilder.simpleSchedule;
  40. import static org.quartz.TriggerBuilder.newTrigger;
  41. /**
  42. * Starts all of the quartz jobs which perform background tasks such as cloning/updating from GIT/SVN and
  43. * the jobs which delete repositories and which add repositories to the queue to be indexed.
  44. * TODO implement using below for the stopping and starting of jobs
  45. * http://stackoverflow.com/questions/7159080/how-to-interrupt-or-stop-currently-running-quartz-job#7159719
  46. */
  47. public class JobService {
  48. private final Helpers helpers;
  49. private final LoggerWrapper logger;
  50. private final Scheduler scheduler;
  51. private final IRepo repo;
  52. private final DataService dataservice;
  53. private int UPDATETIME;
  54. private int FILEINDEXUPDATETIME;
  55. private int INDEXTIME;
  56. private int NUMBERGITPROCESSORS = Singleton.getHelpers().tryParseInt(Properties.getProperties().getProperty(Values.NUMBER_GIT_PROCESSORS, Values.DEFAULT_NUMBER_GIT_PROCESSORS), Values.DEFAULT_NUMBER_GIT_PROCESSORS);
  57. private int NUMBERSVNPROCESSORS = Singleton.getHelpers().tryParseInt(Properties.getProperties().getProperty(Values.NUMBER_SVN_PROCESSORS, Values.DEFAULT_NUMBER_SVN_PROCESSORS), Values.DEFAULT_NUMBER_SVN_PROCESSORS);
  58. private int NUMBERFILEPROCESSORS = Singleton.getHelpers().tryParseInt(Properties.getProperties().getProperty(Values.NUMBER_FILE_PROCESSORS, Values.DEFAULT_NUMBER_FILE_PROCESSORS), Values.DEFAULT_NUMBER_FILE_PROCESSORS);
  59. private String REPOLOCATION = Properties.getProperties().getProperty(Values.REPOSITORYLOCATION, Values.DEFAULTREPOSITORYLOCATION);
  60. private String TRASHLOCATION = Properties.getProperties().getProperty(Values.TRASH_LOCATION, Values.DEFAULT_TRASH_LOCATION);
  61. private boolean LOWMEMORY = Boolean.parseBoolean(com.searchcode.app.util.Properties.getProperties().getProperty(Values.LOWMEMORY, Values.DEFAULTLOWMEMORY));
  62. private boolean SVNENABLED = Boolean.parseBoolean(com.searchcode.app.util.Properties.getProperties().getProperty(Values.SVNENABLED, Values.DEFAULTSVNENABLED));
  63. private String HIGHLIGHTER_BINARY_LOCATION = Properties.getProperties().getProperty(Values.HIGHLIGHTER_BINARY_LOCATION, Values.DEFAULT_HIGHLIGHTER_BINARY_LOCATION);
  64. private boolean initialJobsRun = false;
  65. public JobService() {
  66. this.scheduler = Singleton.getScheduler();
  67. this.helpers = Singleton.getHelpers();
  68. this.repo = Singleton.getRepo();
  69. this.dataservice = Singleton.getDataService();
  70. this.UPDATETIME = Singleton.getHelpers().tryParseInt(Properties.getProperties().getProperty(Values.CHECKREPOCHANGES, Values.DEFAULTCHECKREPOCHANGES), Values.DEFAULTCHECKREPOCHANGES);
  71. this.FILEINDEXUPDATETIME = Singleton.getHelpers().tryParseInt(Properties.getProperties().getProperty(Values.CHECKFILEREPOCHANGES, Values.DEFAULTCHECKFILEREPOCHANGES), Values.DEFAULTCHECKFILEREPOCHANGES);
  72. this.INDEXTIME = Singleton.getHelpers().tryParseInt(Properties.getProperties().getProperty(Values.INDEXTIME, Values.DEFAULTINDEXTIME), Values.DEFAULTINDEXTIME);
  73. this.logger = Singleton.getLogger();
  74. }
  75. /**
  76. * Starts all of the above jobs as per their unique requirements between searchcode.com
  77. * and local runner
  78. */
  79. public synchronized void initialJobs() {
  80. // Having this run multiple times can be an issue so ensure it can not happen
  81. if (this.initialJobsRun) {
  82. return;
  83. }
  84. this.initialJobsRun = true;
  85. if (Singleton.getHelpers().isStandaloneInstance()) {
  86. this.startDeleteJob();
  87. this.startSpellingJob();
  88. this.startRepositoryJobs();
  89. this.startEnqueueJob();
  90. } else {
  91. // searchcode.com path
  92. this.startHighlighter();
  93. this.startReIndexer();
  94. this.startRepositoryJobs();
  95. this.startSearchcodeEnqueueJob();
  96. }
  97. // This will determine itself what index to use so no need to if condition it
  98. this.startIndexerJob();
  99. }
  100. /**
  101. * Creates a git repo indexer job which will pull from the list of git repositories and start
  102. * indexing them
  103. */
  104. public void startIndexGitRepoJobs(String uniquename) {
  105. try {
  106. var job = newJob(IndexGitRepoJob.class)
  107. .withIdentity("updateindex-git-" + uniquename)
  108. .build();
  109. var trigger = newTrigger()
  110. .withIdentity("updateindex-git-" + uniquename)
  111. .withSchedule(
  112. simpleSchedule()
  113. .withIntervalInSeconds(this.INDEXTIME)
  114. .repeatForever()
  115. )
  116. .withPriority(1)
  117. .build();
  118. job.getJobDataMap().put("REPOLOCATIONS", this.REPOLOCATION);
  119. job.getJobDataMap().put("LOWMEMORY", this.LOWMEMORY);
  120. this.scheduler.scheduleJob(job, trigger);
  121. this.scheduler.start();
  122. } catch (SchedulerException ex) {
  123. this.logger.severe(String.format("93ef44ae::error in class %s exception %s", ex.getClass(), ex.getMessage()));
  124. }
  125. }
  126. /**
  127. * Creates a file repo indexer job which will pull from the file queue and index
  128. */
  129. public void startIndexFileRepoJobs(String uniquename) {
  130. try {
  131. var job = newJob(IndexFileRepoJob.class)
  132. .withIdentity("updateindex-file-" + uniquename)
  133. .build();
  134. var trigger = newTrigger()
  135. .withIdentity("updateindex-file-" + uniquename)
  136. .withSchedule(
  137. simpleSchedule()
  138. .withIntervalInSeconds(this.INDEXTIME)
  139. .repeatForever()
  140. )
  141. .withPriority(1)
  142. .build();
  143. job.getJobDataMap().put("REPOLOCATIONS", this.REPOLOCATION);
  144. job.getJobDataMap().put("LOWMEMORY", this.LOWMEMORY);
  145. this.scheduler.scheduleJob(job, trigger);
  146. this.scheduler.start();
  147. } catch (SchedulerException ex) {
  148. this.logger.severe(String.format("c0f207cd::error in class %s exception %s", ex.getClass(), ex.getMessage()));
  149. }
  150. }
  151. /**
  152. * Creates a svn repo indexer job which will pull from the list of git repositories and start
  153. * indexing them
  154. */
  155. public void startIndexSvnRepoJobs(String uniquename) {
  156. try {
  157. var job = newJob(IndexSvnRepoJob.class)
  158. .withIdentity("updateindex-svn-" + uniquename)
  159. .build();
  160. var trigger = newTrigger()
  161. .withIdentity("updateindex-svn-" + uniquename)
  162. .withSchedule(
  163. simpleSchedule()
  164. .withIntervalInSeconds(this.INDEXTIME)
  165. .repeatForever()
  166. )
  167. .withPriority(1)
  168. .build();
  169. job.getJobDataMap().put("REPOLOCATIONS", this.REPOLOCATION);
  170. job.getJobDataMap().put("LOWMEMORY", this.LOWMEMORY);
  171. this.scheduler.scheduleJob(job, trigger);
  172. this.scheduler.start();
  173. } catch (SchedulerException ex) {
  174. this.logger.severe(String.format("70845099::error in class %s exception %s", ex.getClass(), ex.getMessage()));
  175. }
  176. }
  177. /**
  178. * Starts a background job which pulls all repositories from the database and adds them to the
  179. * queue to be indexed
  180. */
  181. private void startEnqueueJob() {
  182. try {
  183. // Setup the indexer which runs forever adding documents to be indexed
  184. var job = newJob(EnqueueRepositoryJob.class)
  185. .withIdentity("enqueuejob")
  186. .build();
  187. var trigger = newTrigger()
  188. .withIdentity("enqueuejob")
  189. .withSchedule(
  190. simpleSchedule()
  191. .withIntervalInSeconds(this.UPDATETIME)
  192. .repeatForever()
  193. )
  194. .withPriority(2)
  195. .build();
  196. this.scheduler.scheduleJob(job, trigger);
  197. this.scheduler.start();
  198. // Setup the indexer which runs forever adding documents to be indexed
  199. var job2 = newJob(EnqueueFileRepositoryJob.class)
  200. .withIdentity("enqueuefilejob")
  201. .build();
  202. var trigger2 = newTrigger()
  203. .withIdentity("enqueuefilejob")
  204. .withSchedule(
  205. simpleSchedule()
  206. .withIntervalInSeconds(this.FILEINDEXUPDATETIME)
  207. .repeatForever()
  208. )
  209. .withPriority(2)
  210. .build();
  211. this.scheduler.scheduleJob(job2, trigger2);
  212. this.scheduler.start();
  213. } catch (SchedulerException ex) {
  214. this.logger.severe(String.format("40f20408::error in class %s exception %s", ex.getClass(), ex.getMessage()));
  215. }
  216. }
  217. private void startSearchcodeEnqueueJob() {
  218. try {
  219. // Setup the indexer which runs forever adding documents to be indexed
  220. var job = newJob(EnqueueSearchcodeRepositoryJob.class)
  221. .withIdentity("enqueuesearchcodejob")
  222. .build();
  223. var trigger = newTrigger()
  224. .withIdentity("enqueuesearchcodejob")
  225. .withSchedule(
  226. simpleSchedule()
  227. .withIntervalInSeconds(this.UPDATETIME)
  228. .repeatForever()
  229. )
  230. .withPriority(2)
  231. .build();
  232. this.scheduler.scheduleJob(job, trigger);
  233. this.scheduler.start();
  234. } catch (SchedulerException ex) {
  235. this.logger.severe(String.format("9c4b9ccc::error in class %s exception %s", ex.getClass(), ex.getMessage()));
  236. }
  237. }
  238. /**
  239. * Starts a background job which deletes repositories from the database, index and checked out disk
  240. */
  241. private void startDeleteJob() {
  242. try {
  243. var job = newJob(DeleteRepositoryJob.class)
  244. .withIdentity("deletejob")
  245. .build();
  246. var trigger = newTrigger()
  247. .withIdentity("deletejob")
  248. .withSchedule(
  249. simpleSchedule()
  250. .withIntervalInSeconds(1)
  251. .repeatForever()
  252. )
  253. .withPriority(2)
  254. .build();
  255. this.scheduler.scheduleJob(job, trigger);
  256. this.scheduler.start();
  257. } catch (SchedulerException ex) {
  258. this.logger.severe(String.format("703d6d7f::error in class %s exception %s", ex.getClass(), ex.getMessage()));
  259. }
  260. }
  261. /**
  262. * Starts a background job which updates the spelling corrector
  263. */
  264. private void startSpellingJob() {
  265. try {
  266. var job = newJob(PopulateSpellingCorrectorJob.class)
  267. .withIdentity("spellingjob")
  268. .build();
  269. var trigger = newTrigger()
  270. .withIdentity("spellingjob")
  271. .withSchedule(
  272. simpleSchedule()
  273. .withIntervalInSeconds(3600)
  274. .repeatForever()
  275. )
  276. .withPriority(1)
  277. .build();
  278. this.scheduler.scheduleJob(job, trigger);
  279. this.scheduler.start();
  280. } catch (SchedulerException ex) {
  281. this.logger.severe(String.format("6e131da2::error in class %s exception %s", ex.getClass(), ex.getMessage()));
  282. }
  283. }
  284. /**
  285. * This job runs in the background connecting to the searchcode.com database
  286. * pulling out files and adding them to the queue to be indexed
  287. */
  288. public void startReIndexer() {
  289. var job = newJob(ReindexerJob.class)
  290. .withIdentity("reindexer")
  291. .build();
  292. var trigger = newTrigger()
  293. .withIdentity("reindexer")
  294. .withSchedule(simpleSchedule()
  295. .withIntervalInSeconds(this.INDEXTIME)
  296. .repeatForever()
  297. )
  298. .withPriority(15)
  299. .build();
  300. try {
  301. this.scheduler.scheduleJob(job, trigger);
  302. this.scheduler.start();
  303. } catch (SchedulerException ex) {
  304. this.logger.severe(String.format("5e7b51d8::error in class %s exception %s", ex.getClass(), ex.getMessage()));
  305. }
  306. }
  307. /**
  308. * Starts a background process used to highlight code
  309. */
  310. public void startHighlighter() {
  311. try {
  312. if (SystemUtils.IS_OS_LINUX) {
  313. new ProcessExecutor().command(HIGHLIGHTER_BINARY_LOCATION + "/searchcode-server-highlighter-x86_64-unknown-linux").destroyOnExit().start().getFuture();
  314. } else if (SystemUtils.IS_OS_WINDOWS) {
  315. new ProcessExecutor().command(HIGHLIGHTER_BINARY_LOCATION + "/searchcode-server-highlighter-x86_64-pc-windows.exe").destroyOnExit().start().getFuture();
  316. } else if (SystemUtils.IS_OS_MAC) {
  317. new ProcessExecutor().command(HIGHLIGHTER_BINARY_LOCATION + "/searchcode-server-highlighter-x86_64-apple-darwin").destroyOnExit().start().getFuture();
  318. }
  319. } catch (IOException ex) {
  320. this.logger.severe(String.format("947e8a85::error in class %s exception %s", ex.getClass(), ex.getMessage()));
  321. }
  322. }
  323. /**
  324. * Sets up the indexer job which runs in the background forever
  325. * indexing files that are added to the index queue
  326. */
  327. private void startIndexerJob() {
  328. var job = newJob(IndexDocumentsJob.class)
  329. .withIdentity("indexerjob")
  330. .build();
  331. var trigger = newTrigger()
  332. .withIdentity("indexerjob")
  333. .withSchedule(
  334. simpleSchedule()
  335. .withIntervalInSeconds(this.INDEXTIME)
  336. .repeatForever()
  337. )
  338. .withPriority(15)
  339. .build();
  340. try {
  341. this.scheduler.scheduleJob(job, trigger);
  342. this.scheduler.start();
  343. } catch (SchedulerException ex) {
  344. this.logger.severe(String.format("8c3cd302::error in class %s exception %s", ex.getClass(), ex.getMessage()));
  345. }
  346. }
  347. private void startRepositoryJobs() {
  348. // Create a pool of crawlers which read from the queue
  349. for (int i = 0; i < this.NUMBERGITPROCESSORS; i++) {
  350. this.startIndexGitRepoJobs(Values.EMPTYSTRING + i);
  351. }
  352. if (SVNENABLED) {
  353. for (int i = 0; i < this.NUMBERSVNPROCESSORS; i++) {
  354. this.startIndexSvnRepoJobs(Values.EMPTYSTRING + i);
  355. }
  356. }
  357. for (int i = 0; i < this.NUMBERFILEPROCESSORS; i++) {
  358. this.startIndexFileRepoJobs(Values.EMPTYSTRING + i);
  359. }
  360. }
  361. private void shutdownScheduler() {
  362. try {
  363. this.scheduler.shutdown();
  364. } catch (SchedulerException ex) {
  365. this.logger.severe(String.format("12cce757::error in class %s exception %s", ex.getClass(), ex.getMessage()));
  366. }
  367. }
  368. private boolean attemptMoveToTrash(String repoLocation, String indexLocation) {
  369. boolean successful;
  370. this.logger.severe(String.format("e71a5492::searchcode was unable to remove files or folders in the index %s or repository %s they have been moved to trash and must be removed manually", indexLocation, repoLocation));
  371. successful = true;
  372. try {
  373. if (new File(repoLocation).exists()) {
  374. this.moveDirectoryToTrash(repoLocation);
  375. }
  376. } catch (IOException ex) {
  377. successful = false;
  378. this.logger.severe(String.format("dfd26713::error in class %s exception %s it is unlikely that searchcode can recover from this remove all please remove the folder %s manually and restart searchcode", ex.getClass(), ex.getMessage(), repoLocation));
  379. }
  380. try {
  381. if (new File(repoLocation).exists()) {
  382. this.moveDirectoryToTrash(indexLocation);
  383. }
  384. } catch (IOException ex) {
  385. successful = false;
  386. this.logger.severe(String.format("fa274f76::error in class %s exception %s it is unlikely that searchcode can recover from this remove all please remove the folder %s manually and restart searchcode", ex.getClass(), ex.getMessage(), indexLocation));
  387. }
  388. return successful;
  389. }
  390. public void moveDirectoryToTrash(String troublesome) throws IOException {
  391. Date date = new Date();
  392. DateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss");
  393. String newLocation = this.TRASHLOCATION + "/" + dateFormat.format(date);
  394. FileUtils.moveDirectory(new File(troublesome), new File(newLocation));
  395. }
  396. public boolean forceEnqueue() {
  397. // TODO refactor this dependency, because we have circular dependencies
  398. if (Singleton.getIndexService().shouldPause(IIndexService.JobType.REPO_ADDER)) {
  399. return false;
  400. }
  401. var repoResultList = this.helpers.filterRunningAndDeletedRepoJobs(this.repo.getAllRepo());
  402. this.logger.info(String.format("ea4fc311::adding %d repositories to be indexed", repoResultList.size()));
  403. repoResultList.forEach(this::enqueueRepository);
  404. return true;
  405. }
  406. public int forceEnqueueWithCount() {
  407. // Get all of the repositories and enqueue them
  408. var repoResultList = this.helpers.filterRunningAndDeletedRepoJobs(this.repo.getAllRepo());
  409. this.logger.info(String.format("de4d4b59::adding %d repositories to be indexed", repoResultList.size()));
  410. repoResultList.forEach(this::enqueueRepository);
  411. return repoResultList.size();
  412. }
  413. public boolean forceEnqueue(RepoResult repoResult) {
  414. // TODO refactor this dependency, because we have circular dependencies
  415. if (Singleton.getIndexService().shouldPause(IIndexService.JobType.REPO_ADDER)) {
  416. return false;
  417. }
  418. if (this.dataservice.getPersistentDelete().contains(repoResult.getName()) ||
  419. Singleton.getRunningIndexRepoJobs().containsKey(repoResult.getName())) {
  420. return false;
  421. }
  422. this.enqueueRepository(repoResult);
  423. return true;
  424. }
  425. private void enqueueRepository(RepoResult rr) {
  426. var repoGitQueue = Singleton.getUniqueGitRepoQueue();
  427. var repoSvnQueue = Singleton.getUniqueSvnRepoQueue();
  428. var repoFileQueue = Singleton.getUniqueFileRepoQueue();
  429. this.logger.info(String.format("e30a1dca::adding %s to %s queue", rr.getName(), rr.getScm()));
  430. switch (rr.getScm().toLowerCase()) {
  431. case "git":
  432. repoGitQueue.add(rr);
  433. break;
  434. case "svn":
  435. repoSvnQueue.add(rr);
  436. break;
  437. case "file":
  438. repoFileQueue.add(rr);
  439. break;
  440. default:
  441. this.logger.severe(String.format("e9cc3dd6::unknown scm type for %s type %s queue, this should be removed from the list of repositories", rr.getName(), rr.getScm()));
  442. break;
  443. }
  444. }
  445. }