/src/main/scala/com/cloudwick/sync/jobs/fetcher/Start.scala

https://gitlab.com/thugside/sync · Scala · 132 lines · 110 code · 16 blank · 6 comment · 2 complexity · 075d6659f87ed2d0e5d66f045e39b208 MD5 · raw file

  1. package com.cloudwick.sync.jobs.fetcher
  2. import java.util.Date
  3. import akka.actor.{Props, ActorSystem}
  4. import akka.pattern.ask
  5. import akka.util.Timeout
  6. import com.mongodb.MongoTimeoutException
  7. import com.mongodb.casbah.Imports._
  8. import com.mongodb.casbah.commons.conversions.scala.RegisterJodaTimeConversionHelpers
  9. import com.typesafe.config.ConfigFactory
  10. import com.typesafe.scalalogging.slf4j.Logger
  11. import org.slf4j.LoggerFactory
  12. import scala.concurrent.ExecutionContext.Implicits.global
  13. import scala.concurrent.duration._
  14. /**
  15. * Start the jobs fetcher
  16. * @author ashrith
  17. */
  18. object Start extends App {
  19. val logger = Logger(LoggerFactory getLogger this.getClass)
  20. if (args.length != 3) {
  21. logger.error(s"Usage: ${this.getClass.getName} [dice|indeed] [hourly|daily|deep] [hadoop|spark|cassandra]")
  22. System.exit(1)
  23. }
  24. val apiType = args(0)
  25. val fetchType = args(1)
  26. val searchTerm = args(2)
  27. val config = ConfigFactory.load()
  28. val system = ActorSystem("JobProcessing")
  29. implicit val timeout = Timeout(10 minutes)
  30. def conn: MongoClient = {
  31. MongoClient(
  32. config.getString("sync.mongo.host"),
  33. config.getInt("sync.mongo.port")
  34. )
  35. }
  36. def checkConnection = {
  37. try {
  38. conn(config.getString("sync.mongo.db")).command("serverStatus")
  39. } catch {
  40. case ex: MongoTimeoutException =>
  41. logger.error("Failed connecting to MongoDB. Reason: " + ex.getMessage, ex)
  42. System.exit(1)
  43. }
  44. }
  45. def processDice(fType: String, sTerm: String) = {
  46. checkConnection
  47. val actor = system.actorOf(
  48. Props(classOf[Master],
  49. conn(config.getString("sync.mongo.db")),
  50. config.getString("sync.api.dice.url"),
  51. sTerm,
  52. config.getInt(s"sync.api.dice.fetch.$fType.age"),
  53. config.getInt(s"sync.api.dice.fetch.$fType.depth"),
  54. config.getInt(s"sync.api.dice.fetch.$fType.sort"),
  55. config.getString("sync.api.dice.grep")),
  56. name=s"${sTerm}jobs")
  57. /** called when you have shut down the ActorSystem, and the callbacks will be executed after
  58. * all actors have been stopped. */
  59. system.registerOnTermination {
  60. conn.close()
  61. }
  62. val future = actor ? Messages.Start
  63. val fetcherCollection = conn(config.getString("sync.mongo.db"))("fetchers")
  64. future.map { result =>
  65. result match {
  66. case (jobsProcessed: Int, jobsFiltered: Int, jobsInserted: Int) =>
  67. logger.info("Total number of jobs processed: " + jobsProcessed)
  68. fetcherCollection.insert(
  69. MongoDBObject(
  70. "job_status" -> "completed",
  71. "progress" -> 100,
  72. "jobs_filtered" -> jobsFiltered,
  73. "jobs_inserted" -> jobsInserted,
  74. "message" -> "completed by fetcher batch",
  75. "jobs_processed" -> jobsProcessed,
  76. "init_time" -> new Date()
  77. )
  78. )
  79. }
  80. system.shutdown()
  81. }
  82. }
  83. def processIndeed(fType: String) = {
  84. logger.warn("WARN: Not yet implemented")
  85. System.exit(1)
  86. }
  87. apiType match {
  88. case "dice" =>
  89. fetchType match {
  90. case "hourly" =>
  91. processDice("hourly", searchTerm)
  92. case "daily" =>
  93. processDice("daily", searchTerm)
  94. case "deep" =>
  95. processDice("deep", searchTerm)
  96. case _ =>
  97. logger.error("Unexpected argument")
  98. System.exit(1)
  99. }
  100. case "indeed" =>
  101. fetchType match {
  102. case "hourly" =>
  103. processIndeed("hourly")
  104. case "daily" =>
  105. processIndeed("daily")
  106. case "deep" =>
  107. processIndeed("deep")
  108. case _ =>
  109. logger.error("Unexpected argument")
  110. System.exit(1)
  111. }
  112. case _ =>
  113. logger.error("Error: Unexpected argument")
  114. System.exit(1)
  115. }
  116. }