/src/main/java/com/searchcode/app/service/JobService.java
Java | 513 lines | 384 code | 68 blank | 61 comment | 18 complexity | 6b428c84675380618d195d8da21ba229 MD5 | raw file
1/*
2 * Copyright (c) 2016 Boyter Online Services
3 *
4 * Use of this software is governed by the Fair Source License included
5 * in the LICENSE.TXT file, but will be eventually open under GNU General Public License Version 3
6 * see the README.md for when this clause will take effect
7 *
8 * Version 1.3.15
9 */
10
11package com.searchcode.app.service;
12
13import com.searchcode.app.config.Values;
14import com.searchcode.app.dao.IRepo;
15import com.searchcode.app.jobs.DeleteRepositoryJob;
16import com.searchcode.app.jobs.PopulateSpellingCorrectorJob;
17import com.searchcode.app.jobs.enqueue.EnqueueFileRepositoryJob;
18import com.searchcode.app.jobs.enqueue.EnqueueRepositoryJob;
19import com.searchcode.app.jobs.enqueue.EnqueueSearchcodeRepositoryJob;
20import com.searchcode.app.jobs.repository.IndexDocumentsJob;
21import com.searchcode.app.jobs.repository.IndexFileRepoJob;
22import com.searchcode.app.jobs.repository.IndexGitRepoJob;
23import com.searchcode.app.jobs.repository.IndexSvnRepoJob;
24import com.searchcode.app.jobs.searchcode.ReindexerJob;
25import com.searchcode.app.model.RepoResult;
26import com.searchcode.app.service.index.IIndexService;
27import com.searchcode.app.util.Helpers;
28import com.searchcode.app.util.LoggerWrapper;
29import com.searchcode.app.util.Properties;
30import org.apache.commons.io.FileUtils;
31import org.apache.commons.lang3.SystemUtils;
32import org.quartz.Scheduler;
33import org.quartz.SchedulerException;
34import org.zeroturnaround.exec.ProcessExecutor;
35
36import java.io.File;
37import java.io.IOException;
38import java.text.DateFormat;
39import java.text.SimpleDateFormat;
40import java.util.Date;
41
42import static org.quartz.JobBuilder.newJob;
43import static org.quartz.SimpleScheduleBuilder.simpleSchedule;
44import static org.quartz.TriggerBuilder.newTrigger;
45
46/**
47 * Starts all of the quartz jobs which perform background tasks such as cloning/updating from GIT/SVN and
48 * the jobs which delete repositories and which add repositories to the queue to be indexed.
49 * TODO implement using below for the stopping and starting of jobs
50 * http://stackoverflow.com/questions/7159080/how-to-interrupt-or-stop-currently-running-quartz-job#7159719
51 */
52public class JobService {
53
54 private final Helpers helpers;
55 private final LoggerWrapper logger;
56 private final Scheduler scheduler;
57 private final IRepo repo;
58 private final DataService dataservice;
59 private int UPDATETIME;
60 private int FILEINDEXUPDATETIME;
61 private int INDEXTIME;
62 private int NUMBERGITPROCESSORS = Singleton.getHelpers().tryParseInt(Properties.getProperties().getProperty(Values.NUMBER_GIT_PROCESSORS, Values.DEFAULT_NUMBER_GIT_PROCESSORS), Values.DEFAULT_NUMBER_GIT_PROCESSORS);
63 private int NUMBERSVNPROCESSORS = Singleton.getHelpers().tryParseInt(Properties.getProperties().getProperty(Values.NUMBER_SVN_PROCESSORS, Values.DEFAULT_NUMBER_SVN_PROCESSORS), Values.DEFAULT_NUMBER_SVN_PROCESSORS);
64 private int NUMBERFILEPROCESSORS = Singleton.getHelpers().tryParseInt(Properties.getProperties().getProperty(Values.NUMBER_FILE_PROCESSORS, Values.DEFAULT_NUMBER_FILE_PROCESSORS), Values.DEFAULT_NUMBER_FILE_PROCESSORS);
65
66 private String REPOLOCATION = Properties.getProperties().getProperty(Values.REPOSITORYLOCATION, Values.DEFAULTREPOSITORYLOCATION);
67 private String TRASHLOCATION = Properties.getProperties().getProperty(Values.TRASH_LOCATION, Values.DEFAULT_TRASH_LOCATION);
68 private boolean LOWMEMORY = Boolean.parseBoolean(com.searchcode.app.util.Properties.getProperties().getProperty(Values.LOWMEMORY, Values.DEFAULTLOWMEMORY));
69 private boolean SVNENABLED = Boolean.parseBoolean(com.searchcode.app.util.Properties.getProperties().getProperty(Values.SVNENABLED, Values.DEFAULTSVNENABLED));
70 private String HIGHLIGHTER_BINARY_LOCATION = Properties.getProperties().getProperty(Values.HIGHLIGHTER_BINARY_LOCATION, Values.DEFAULT_HIGHLIGHTER_BINARY_LOCATION);
71 private boolean initialJobsRun = false;
72
73 public JobService() {
74 this.scheduler = Singleton.getScheduler();
75 this.helpers = Singleton.getHelpers();
76 this.repo = Singleton.getRepo();
77 this.dataservice = Singleton.getDataService();
78 this.UPDATETIME = Singleton.getHelpers().tryParseInt(Properties.getProperties().getProperty(Values.CHECKREPOCHANGES, Values.DEFAULTCHECKREPOCHANGES), Values.DEFAULTCHECKREPOCHANGES);
79 this.FILEINDEXUPDATETIME = Singleton.getHelpers().tryParseInt(Properties.getProperties().getProperty(Values.CHECKFILEREPOCHANGES, Values.DEFAULTCHECKFILEREPOCHANGES), Values.DEFAULTCHECKFILEREPOCHANGES);
80 this.INDEXTIME = Singleton.getHelpers().tryParseInt(Properties.getProperties().getProperty(Values.INDEXTIME, Values.DEFAULTINDEXTIME), Values.DEFAULTINDEXTIME);
81 this.logger = Singleton.getLogger();
82 }
83
84 /**
85 * Starts all of the above jobs as per their unique requirements between searchcode.com
86 * and local runner
87 */
88 public synchronized void initialJobs() {
89 // Having this run multiple times can be an issue so ensure it can not happen
90 if (this.initialJobsRun) {
91 return;
92 }
93
94 this.initialJobsRun = true;
95
96 if (Singleton.getHelpers().isStandaloneInstance()) {
97 this.startDeleteJob();
98 this.startSpellingJob();
99 this.startRepositoryJobs();
100 this.startEnqueueJob();
101 } else {
102 // searchcode.com path
103 this.startHighlighter();
104 this.startReIndexer();
105 this.startRepositoryJobs();
106 this.startSearchcodeEnqueueJob();
107 }
108
109 // This will determine itself what index to use so no need to if condition it
110 this.startIndexerJob();
111 }
112
113
114 /**
115 * Creates a git repo indexer job which will pull from the list of git repositories and start
116 * indexing them
117 */
118 public void startIndexGitRepoJobs(String uniquename) {
119 try {
120 var job = newJob(IndexGitRepoJob.class)
121 .withIdentity("updateindex-git-" + uniquename)
122 .build();
123
124 var trigger = newTrigger()
125 .withIdentity("updateindex-git-" + uniquename)
126 .withSchedule(
127 simpleSchedule()
128 .withIntervalInSeconds(this.INDEXTIME)
129 .repeatForever()
130 )
131 .withPriority(1)
132 .build();
133
134 job.getJobDataMap().put("REPOLOCATIONS", this.REPOLOCATION);
135 job.getJobDataMap().put("LOWMEMORY", this.LOWMEMORY);
136
137 this.scheduler.scheduleJob(job, trigger);
138 this.scheduler.start();
139 } catch (SchedulerException ex) {
140 this.logger.severe(String.format("93ef44ae::error in class %s exception %s", ex.getClass(), ex.getMessage()));
141 }
142 }
143
144 /**
145 * Creates a file repo indexer job which will pull from the file queue and index
146 */
147 public void startIndexFileRepoJobs(String uniquename) {
148 try {
149 var job = newJob(IndexFileRepoJob.class)
150 .withIdentity("updateindex-file-" + uniquename)
151 .build();
152
153 var trigger = newTrigger()
154 .withIdentity("updateindex-file-" + uniquename)
155 .withSchedule(
156 simpleSchedule()
157 .withIntervalInSeconds(this.INDEXTIME)
158 .repeatForever()
159 )
160 .withPriority(1)
161 .build();
162
163 job.getJobDataMap().put("REPOLOCATIONS", this.REPOLOCATION);
164 job.getJobDataMap().put("LOWMEMORY", this.LOWMEMORY);
165
166 this.scheduler.scheduleJob(job, trigger);
167 this.scheduler.start();
168 } catch (SchedulerException ex) {
169 this.logger.severe(String.format("c0f207cd::error in class %s exception %s", ex.getClass(), ex.getMessage()));
170 }
171 }
172
173 /**
174 * Creates a svn repo indexer job which will pull from the list of git repositories and start
175 * indexing them
176 */
177 public void startIndexSvnRepoJobs(String uniquename) {
178 try {
179 var job = newJob(IndexSvnRepoJob.class)
180 .withIdentity("updateindex-svn-" + uniquename)
181 .build();
182
183 var trigger = newTrigger()
184 .withIdentity("updateindex-svn-" + uniquename)
185 .withSchedule(
186 simpleSchedule()
187 .withIntervalInSeconds(this.INDEXTIME)
188 .repeatForever()
189 )
190 .withPriority(1)
191 .build();
192
193 job.getJobDataMap().put("REPOLOCATIONS", this.REPOLOCATION);
194 job.getJobDataMap().put("LOWMEMORY", this.LOWMEMORY);
195
196 this.scheduler.scheduleJob(job, trigger);
197 this.scheduler.start();
198 } catch (SchedulerException ex) {
199 this.logger.severe(String.format("70845099::error in class %s exception %s", ex.getClass(), ex.getMessage()));
200 }
201 }
202
203 /**
204 * Starts a background job which pulls all repositories from the database and adds them to the
205 * queue to be indexed
206 */
207 private void startEnqueueJob() {
208 try {
209 // Setup the indexer which runs forever adding documents to be indexed
210 var job = newJob(EnqueueRepositoryJob.class)
211 .withIdentity("enqueuejob")
212 .build();
213
214 var trigger = newTrigger()
215 .withIdentity("enqueuejob")
216 .withSchedule(
217 simpleSchedule()
218 .withIntervalInSeconds(this.UPDATETIME)
219 .repeatForever()
220 )
221 .withPriority(2)
222 .build();
223
224 this.scheduler.scheduleJob(job, trigger);
225 this.scheduler.start();
226
227 // Setup the indexer which runs forever adding documents to be indexed
228 var job2 = newJob(EnqueueFileRepositoryJob.class)
229 .withIdentity("enqueuefilejob")
230 .build();
231
232 var trigger2 = newTrigger()
233 .withIdentity("enqueuefilejob")
234 .withSchedule(
235 simpleSchedule()
236 .withIntervalInSeconds(this.FILEINDEXUPDATETIME)
237 .repeatForever()
238 )
239 .withPriority(2)
240 .build();
241
242 this.scheduler.scheduleJob(job2, trigger2);
243 this.scheduler.start();
244 } catch (SchedulerException ex) {
245 this.logger.severe(String.format("40f20408::error in class %s exception %s", ex.getClass(), ex.getMessage()));
246 }
247 }
248
249 private void startSearchcodeEnqueueJob() {
250 try {
251 // Setup the indexer which runs forever adding documents to be indexed
252 var job = newJob(EnqueueSearchcodeRepositoryJob.class)
253 .withIdentity("enqueuesearchcodejob")
254 .build();
255
256 var trigger = newTrigger()
257 .withIdentity("enqueuesearchcodejob")
258 .withSchedule(
259 simpleSchedule()
260 .withIntervalInSeconds(this.UPDATETIME)
261 .repeatForever()
262 )
263 .withPriority(2)
264 .build();
265
266 this.scheduler.scheduleJob(job, trigger);
267 this.scheduler.start();
268 } catch (SchedulerException ex) {
269 this.logger.severe(String.format("9c4b9ccc::error in class %s exception %s", ex.getClass(), ex.getMessage()));
270 }
271 }
272
273 /**
274 * Starts a background job which deletes repositories from the database, index and checked out disk
275 */
276 private void startDeleteJob() {
277 try {
278 var job = newJob(DeleteRepositoryJob.class)
279 .withIdentity("deletejob")
280 .build();
281
282 var trigger = newTrigger()
283 .withIdentity("deletejob")
284 .withSchedule(
285 simpleSchedule()
286 .withIntervalInSeconds(1)
287 .repeatForever()
288 )
289 .withPriority(2)
290 .build();
291
292 this.scheduler.scheduleJob(job, trigger);
293 this.scheduler.start();
294 } catch (SchedulerException ex) {
295 this.logger.severe(String.format("703d6d7f::error in class %s exception %s", ex.getClass(), ex.getMessage()));
296 }
297 }
298
299 /**
300 * Starts a background job which updates the spelling corrector
301 */
302 private void startSpellingJob() {
303 try {
304 var job = newJob(PopulateSpellingCorrectorJob.class)
305 .withIdentity("spellingjob")
306 .build();
307
308 var trigger = newTrigger()
309 .withIdentity("spellingjob")
310 .withSchedule(
311 simpleSchedule()
312 .withIntervalInSeconds(3600)
313 .repeatForever()
314 )
315 .withPriority(1)
316 .build();
317
318 this.scheduler.scheduleJob(job, trigger);
319 this.scheduler.start();
320 } catch (SchedulerException ex) {
321 this.logger.severe(String.format("6e131da2::error in class %s exception %s", ex.getClass(), ex.getMessage()));
322 }
323 }
324
325 /**
326 * This job runs in the background connecting to the searchcode.com database
327 * pulling out files and adding them to the queue to be indexed
328 */
329 public void startReIndexer() {
330 var job = newJob(ReindexerJob.class)
331 .withIdentity("reindexer")
332 .build();
333
334 var trigger = newTrigger()
335 .withIdentity("reindexer")
336 .withSchedule(simpleSchedule()
337 .withIntervalInSeconds(this.INDEXTIME)
338 .repeatForever()
339 )
340 .withPriority(15)
341 .build();
342
343 try {
344 this.scheduler.scheduleJob(job, trigger);
345 this.scheduler.start();
346 } catch (SchedulerException ex) {
347 this.logger.severe(String.format("5e7b51d8::error in class %s exception %s", ex.getClass(), ex.getMessage()));
348 }
349 }
350
351 /**
352 * Starts a background process used to highlight code
353 */
354 public void startHighlighter() {
355 try {
356 if (SystemUtils.IS_OS_LINUX) {
357 new ProcessExecutor().command(HIGHLIGHTER_BINARY_LOCATION + "/searchcode-server-highlighter-x86_64-unknown-linux").destroyOnExit().start().getFuture();
358 } else if (SystemUtils.IS_OS_WINDOWS) {
359 new ProcessExecutor().command(HIGHLIGHTER_BINARY_LOCATION + "/searchcode-server-highlighter-x86_64-pc-windows.exe").destroyOnExit().start().getFuture();
360 } else if (SystemUtils.IS_OS_MAC) {
361 new ProcessExecutor().command(HIGHLIGHTER_BINARY_LOCATION + "/searchcode-server-highlighter-x86_64-apple-darwin").destroyOnExit().start().getFuture();
362 }
363 } catch (IOException ex) {
364 this.logger.severe(String.format("947e8a85::error in class %s exception %s", ex.getClass(), ex.getMessage()));
365 }
366 }
367
368 /**
369 * Sets up the indexer job which runs in the background forever
370 * indexing files that are added to the index queue
371 */
372 private void startIndexerJob() {
373 var job = newJob(IndexDocumentsJob.class)
374 .withIdentity("indexerjob")
375 .build();
376
377 var trigger = newTrigger()
378 .withIdentity("indexerjob")
379 .withSchedule(
380 simpleSchedule()
381 .withIntervalInSeconds(this.INDEXTIME)
382 .repeatForever()
383 )
384 .withPriority(15)
385 .build();
386
387 try {
388 this.scheduler.scheduleJob(job, trigger);
389 this.scheduler.start();
390 } catch (SchedulerException ex) {
391 this.logger.severe(String.format("8c3cd302::error in class %s exception %s", ex.getClass(), ex.getMessage()));
392 }
393 }
394
395 private void startRepositoryJobs() {
396 // Create a pool of crawlers which read from the queue
397 for (int i = 0; i < this.NUMBERGITPROCESSORS; i++) {
398 this.startIndexGitRepoJobs(Values.EMPTYSTRING + i);
399 }
400
401 if (SVNENABLED) {
402 for (int i = 0; i < this.NUMBERSVNPROCESSORS; i++) {
403 this.startIndexSvnRepoJobs(Values.EMPTYSTRING + i);
404 }
405 }
406
407 for (int i = 0; i < this.NUMBERFILEPROCESSORS; i++) {
408 this.startIndexFileRepoJobs(Values.EMPTYSTRING + i);
409 }
410 }
411
412 private void shutdownScheduler() {
413 try {
414 this.scheduler.shutdown();
415 } catch (SchedulerException ex) {
416 this.logger.severe(String.format("12cce757::error in class %s exception %s", ex.getClass(), ex.getMessage()));
417 }
418 }
419
420 private boolean attemptMoveToTrash(String repoLocation, String indexLocation) {
421 boolean successful;
422 this.logger.severe(String.format("e71a5492::searchcode was unable to remove files or folders in the index %s or repository %s they have been moved to trash and must be removed manually", indexLocation, repoLocation));
423 successful = true;
424
425 try {
426 if (new File(repoLocation).exists()) {
427 this.moveDirectoryToTrash(repoLocation);
428 }
429 } catch (IOException ex) {
430 successful = false;
431 this.logger.severe(String.format("dfd26713::error in class %s exception %s it is unlikely that searchcode can recover from this remove all please remove the folder %s manually and restart searchcode", ex.getClass(), ex.getMessage(), repoLocation));
432 }
433
434 try {
435 if (new File(repoLocation).exists()) {
436 this.moveDirectoryToTrash(indexLocation);
437 }
438 } catch (IOException ex) {
439 successful = false;
440 this.logger.severe(String.format("fa274f76::error in class %s exception %s it is unlikely that searchcode can recover from this remove all please remove the folder %s manually and restart searchcode", ex.getClass(), ex.getMessage(), indexLocation));
441 }
442
443 return successful;
444 }
445
446 public void moveDirectoryToTrash(String troublesome) throws IOException {
447 Date date = new Date();
448 DateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss");
449 String newLocation = this.TRASHLOCATION + "/" + dateFormat.format(date);
450 FileUtils.moveDirectory(new File(troublesome), new File(newLocation));
451 }
452
453 public boolean forceEnqueue() {
454 // TODO refactor this dependency, because we have circular dependencies
455 if (Singleton.getIndexService().shouldPause(IIndexService.JobType.REPO_ADDER)) {
456 return false;
457 }
458
459 var repoResultList = this.helpers.filterRunningAndDeletedRepoJobs(this.repo.getAllRepo());
460 this.logger.info(String.format("ea4fc311::adding %d repositories to be indexed", repoResultList.size()));
461 repoResultList.forEach(this::enqueueRepository);
462
463 return true;
464 }
465
466 public int forceEnqueueWithCount() {
467 // Get all of the repositories and enqueue them
468 var repoResultList = this.helpers.filterRunningAndDeletedRepoJobs(this.repo.getAllRepo());
469 this.logger.info(String.format("de4d4b59::adding %d repositories to be indexed", repoResultList.size()));
470 repoResultList.forEach(this::enqueueRepository);
471
472 return repoResultList.size();
473 }
474
475 public boolean forceEnqueue(RepoResult repoResult) {
476 // TODO refactor this dependency, because we have circular dependencies
477 if (Singleton.getIndexService().shouldPause(IIndexService.JobType.REPO_ADDER)) {
478 return false;
479 }
480
481 if (this.dataservice.getPersistentDelete().contains(repoResult.getName()) ||
482 Singleton.getRunningIndexRepoJobs().containsKey(repoResult.getName())) {
483 return false;
484 }
485
486 this.enqueueRepository(repoResult);
487
488 return true;
489 }
490
491 private void enqueueRepository(RepoResult rr) {
492 var repoGitQueue = Singleton.getUniqueGitRepoQueue();
493 var repoSvnQueue = Singleton.getUniqueSvnRepoQueue();
494 var repoFileQueue = Singleton.getUniqueFileRepoQueue();
495
496 this.logger.info(String.format("e30a1dca::adding %s to %s queue", rr.getName(), rr.getScm()));
497
498 switch (rr.getScm().toLowerCase()) {
499 case "git":
500 repoGitQueue.add(rr);
501 break;
502 case "svn":
503 repoSvnQueue.add(rr);
504 break;
505 case "file":
506 repoFileQueue.add(rr);
507 break;
508 default:
509 this.logger.severe(String.format("e9cc3dd6::unknown scm type for %s type %s queue, this should be removed from the list of repositories", rr.getName(), rr.getScm()));
510 break;
511 }
512 }
513}