PageRenderTime 56ms CodeModel.GetById 24ms RepoModel.GetById 0ms app.codeStats 0ms

/src/test/scala/mesosphere/marathon/integration/setup/ProcessKeeper.scala

https://gitlab.com/wilane/marathon
Scala | 298 lines | 248 code | 39 blank | 11 comment | 7 complexity | 5249e5687c35d996675c117f6f091fc8 MD5 | raw file
  1. package mesosphere.marathon.integration.setup
  2. import java.io.File
  3. import java.util.concurrent.{ Executors, TimeUnit }
  4. import com.google.common.util.concurrent.{ AbstractIdleService, Service }
  5. import com.google.inject.Guice
  6. import mesosphere.chaos.http.{ HttpConf, HttpModule, HttpService }
  7. import mesosphere.chaos.metrics.MetricsModule
  8. import org.apache.commons.io.FileUtils
  9. import org.rogach.scallop.ScallopConf
  10. import org.slf4j.LoggerFactory
  11. import scala.collection.immutable.ListMap
  12. import scala.concurrent.duration._
  13. import scala.concurrent.{ Await, ExecutionContext, Future, Promise }
  14. import scala.sys.ShutdownHookThread
  15. import scala.sys.process._
  16. import scala.util.control.NonFatal
  17. import scala.util.{ Failure, Success, Try }
  18. /**
  19. * Book Keeper for processes and services.
  20. * During integration tests, several services and processes have to be launched.
  21. * The ProcessKeeper knows about them and can handle their lifecycle.
  22. */
  23. object ProcessKeeper {
  24. private[this] val log = LoggerFactory.getLogger(getClass.getName)
  25. private[this] var processes = ListMap.empty[String, Process]
  26. private[this] var services = List.empty[Service]
  27. private[this] val ENV_MESOS_WORK_DIR: String = "MESOS_WORK_DIR"
  28. def startHttpService(port: Int, assetPath: String) = {
  29. startService {
  30. log.info(s"Start Http Service on port $port")
  31. val conf = new ScallopConf(Array("--http_port", port.toString, "--assets_path", assetPath)) with HttpConf {
  32. verify()
  33. }
  34. val injector = Guice.createInjector(new MetricsModule, new HttpModule(conf), new HttpServiceTestModule)
  35. injector.getInstance(classOf[HttpService])
  36. }
  37. }
  38. def startZooKeeper(port: Int, workDir: String, wipeWorkDir: Boolean = true, superCreds: Option[String] = None) {
  39. val systemArgs: List[String] = "-Dzookeeper.jmx.log4j.disable=true" :: Nil
  40. val sd: List[String] = superCreds match {
  41. case None => Nil
  42. case Some(userPassword) =>
  43. val digest = org.apache.zookeeper.server.auth.DigestAuthenticationProvider.generateDigest(userPassword)
  44. s"-Dzookeeper.DigestAuthenticationProvider.superDigest=$digest" :: Nil
  45. }
  46. val app = "org.apache.zookeeper.server.ZooKeeperServerMain" :: port.toString :: workDir :: Nil
  47. val workDirFile = new File(workDir)
  48. if (wipeWorkDir) {
  49. FileUtils.deleteDirectory(workDirFile)
  50. FileUtils.forceMkdir(workDirFile)
  51. }
  52. startJavaProcess("zookeeper", heapInMegs = 256, systemArgs ++ sd ++ app, new File("."),
  53. sys.env, _.contains("binding to port"))
  54. }
  55. def startMesosLocal(): Process = {
  56. val mesosWorkDirForMesos: String = "/tmp/marathon-itest-mesos"
  57. val mesosWorkDirFile: File = new File(mesosWorkDirForMesos)
  58. FileUtils.deleteDirectory(mesosWorkDirFile)
  59. FileUtils.forceMkdir(mesosWorkDirFile)
  60. val credentialsPath = write(mesosWorkDirFile, fileName = "credentials", content = "principal1 secret1")
  61. val aclsPath = write(mesosWorkDirFile, fileName = "acls.json", content =
  62. """
  63. |{
  64. | "run_tasks": [{
  65. | "principals": { "type": "ANY" },
  66. | "users": { "type": "ANY" }
  67. | }],
  68. | "register_frameworks": [{
  69. | "principals": { "type": "ANY" },
  70. | "roles": { "type": "ANY" }
  71. | }],
  72. | "reserve_resources": [{
  73. | "roles": { "type": "ANY" },
  74. | "principals": { "type": "ANY" },
  75. | "resources": { "type": "ANY" }
  76. | }],
  77. | "create_volumes": [{
  78. | "roles": { "type": "ANY" },
  79. | "principals": { "type": "ANY" },
  80. | "volume_types": { "type": "ANY" }
  81. | }]
  82. |}
  83. """.stripMargin)
  84. log.info(s">>> credentialsPath = $credentialsPath")
  85. val mesosEnv = Seq(
  86. ENV_MESOS_WORK_DIR -> mesosWorkDirForMesos,
  87. "MESOS_LAUNCHER" -> "posix",
  88. "MESOS_CONTAINERIZERS" -> "docker,mesos",
  89. "MESOS_ROLES" -> "public,foo",
  90. "MESOS_ACLS" -> s"file://$aclsPath",
  91. "MESOS_CREDENTIALS" -> s"file://$credentialsPath")
  92. startProcess(
  93. "mesos",
  94. Process(Seq("mesos-local", "--ip=127.0.0.1"), cwd = None, mesosEnv: _*),
  95. upWhen = _.toLowerCase.contains("registered with master"))
  96. }
  97. def startMarathon(cwd: File, env: Map[String, String], arguments: List[String],
  98. mainClass: String = "mesosphere.marathon.Main",
  99. startupLine: String = "Started ServerConnector",
  100. processName: String = "marathon"): Process = {
  101. val debugArgs = List(
  102. "-Dakka.loglevel=DEBUG",
  103. "-Dakka.actor.debug.receive=true",
  104. "-Dakka.actor.debug.autoreceive=true",
  105. "-Dakka.actor.debug.lifecycle=true"
  106. )
  107. val marathonWorkDir: String = s"/tmp/marathon-itest-$processName"
  108. val marathonWorkDirFile: File = new File(marathonWorkDir)
  109. FileUtils.deleteDirectory(marathonWorkDirFile)
  110. FileUtils.forceMkdir(marathonWorkDirFile)
  111. val secretPath = write(marathonWorkDirFile, fileName = "marathon-secret", content = "secret1")
  112. val authSettings = List(
  113. "--mesos_authentication_principal", "principal1",
  114. "--mesos_role", "foo",
  115. "--mesos_authentication_secret_file", s"$secretPath"
  116. )
  117. val argsWithMain = mainClass :: arguments ++ authSettings
  118. startJavaProcess(
  119. processName, heapInMegs = 512, /* debugArgs ++ */ argsWithMain, cwd,
  120. env + (ENV_MESOS_WORK_DIR -> marathonWorkDir),
  121. upWhen = _.contains(startupLine))
  122. }
  123. private[this] def write(dir: File, fileName: String, content: String): String = {
  124. val file = File.createTempFile(fileName, "", dir)
  125. file.deleteOnExit()
  126. FileUtils.write(file, content)
  127. file.setReadable(true)
  128. file.getAbsolutePath
  129. }
  130. def startJavaProcess(name: String, heapInMegs: Int, arguments: List[String],
  131. cwd: File = new File("."), env: Map[String, String] = Map.empty, upWhen: String => Boolean): Process = {
  132. val javaExecutable = sys.props.get("java.home").fold("java")(_ + "/bin/java")
  133. val classPath = sys.props.getOrElse("java.class.path", "target/classes")
  134. val memSettings = s"-Xmx${heapInMegs}m"
  135. // Omit the classpath in order to avoid cluttering the tests output
  136. log.info(s"Start java process $name with command: ${(javaExecutable :: memSettings :: arguments).mkString(" ")}")
  137. val command: List[String] = javaExecutable :: memSettings :: "-classpath" :: classPath :: arguments
  138. val builder = Process(command, cwd, env.toList: _*)
  139. val process = startProcess(name, builder, upWhen)
  140. log.info(s"Java process $name up and running!")
  141. process
  142. }
  143. def startProcess(name: String, processBuilder: ProcessBuilder, upWhen: String => Boolean, timeout: Duration = 30.seconds): Process = {
  144. require(!processes.contains(name), s"Process with $name already started")
  145. sealed trait ProcessState
  146. case object ProcessIsUp extends ProcessState
  147. case object ProcessExited extends ProcessState
  148. val up = Promise[ProcessIsUp.type]()
  149. val logger = new ProcessLogger {
  150. def checkUp(out: String) = {
  151. log.info(s"$name: $out")
  152. if (!up.isCompleted && upWhen(out)) up.trySuccess(ProcessIsUp)
  153. }
  154. override def buffer[T](f: => T): T = f
  155. override def out(s: => String) = checkUp(s)
  156. override def err(s: => String) = checkUp(s)
  157. }
  158. val process = processBuilder.run(logger)
  159. val processExitCode: Future[ProcessExited.type] = Future {
  160. val exitCode = scala.concurrent.blocking {
  161. process.exitValue()
  162. }
  163. log.info(s"Process $name finished with exit code $exitCode")
  164. // Sometimes this finishes before the other future finishes parsing the output
  165. // and we incorrectly report ProcessExited instead of ProcessIsUp as the result of upOrExited.
  166. Await.result(up.future, 1.second)
  167. ProcessExited
  168. }(ExecutionContext.fromExecutor(Executors.newCachedThreadPool()))
  169. val upOrExited = Future.firstCompletedOf(Seq(up.future, processExitCode))(ExecutionContext.global)
  170. Try(Await.result(upOrExited, timeout)) match {
  171. case Success(result) =>
  172. result match {
  173. case ProcessExited =>
  174. throw new IllegalStateException(s"Process $name exited before coming up. Give up. $processBuilder")
  175. case ProcessIsUp =>
  176. processes += name -> process
  177. log.info(s"Process $name is up and running. ${processes.size} processes in total.")
  178. }
  179. case Failure(_) =>
  180. process.destroy()
  181. throw new IllegalStateException(
  182. s"Process $name did not come up within time bounds ($timeout). Give up. $processBuilder")
  183. }
  184. process
  185. }
  186. def onStopServices(block: => Unit): Unit = {
  187. services ::= new AbstractIdleService {
  188. override def shutDown(): Unit = {
  189. block
  190. }
  191. override def startUp(): Unit = {}
  192. }
  193. }
  194. val PIDRE = """^\s*(\d+)\s+(\S*)$""".r
  195. def stopJavaProcesses(wantedMainClass: String): Unit = {
  196. val pids = "jps -l".!!.split("\n").collect {
  197. case PIDRE(pid, mainClass) if mainClass.contains(wantedMainClass) => pid
  198. }
  199. if (pids.nonEmpty) {
  200. val killCommand = s"kill -9 ${pids.mkString(" ")}"
  201. log.warn(s"Left over processes, executing: $killCommand")
  202. val ret = killCommand.!
  203. if (ret != 0) {
  204. log.error(s"kill returned $ret")
  205. }
  206. }
  207. }
  208. def stopProcess(name: String): Unit = {
  209. import mesosphere.util.ThreadPoolContext.ioContext
  210. log.info(s"Stop Process $name")
  211. val process = processes(name)
  212. def killProcess: Int = {
  213. // Unfortunately, there seem to be race conditions in Process.exitValue.
  214. // Thus this ugly workaround.
  215. Await.result(Future {
  216. scala.concurrent.blocking {
  217. Try(process.destroy())
  218. process.exitValue()
  219. }
  220. }, 5.seconds)
  221. }
  222. //retry on fail
  223. Try(killProcess) recover { case _ => killProcess } match {
  224. case Success(value) => processes -= name
  225. case Failure(NonFatal(e)) => log.error("giving up waiting for processes to finish", e)
  226. }
  227. log.info(s"Stop Process $name: Done")
  228. }
  229. def stopAllProcesses(): Unit = {
  230. processes.keys.toSeq.reverse.foreach(stopProcess)
  231. processes = ListMap.empty
  232. }
  233. def startService(service: Service): Unit = {
  234. services ::= service
  235. service.startAsync().awaitRunning()
  236. }
  237. def stopAllServices(): Unit = {
  238. services.foreach(_.stopAsync())
  239. services.par.foreach { service =>
  240. try { service.awaitTerminated(5, TimeUnit.SECONDS) }
  241. catch {
  242. case NonFatal(ex) => log.error(s"Could not stop service $service", ex)
  243. }
  244. }
  245. services = Nil
  246. }
  247. def shutdown(): Unit = {
  248. log.info(s"Cleaning up Processes $processes and Services $services")
  249. stopAllProcesses()
  250. stopAllServices()
  251. log.info(s"Cleaning up Processes $processes and Services $services: Done")
  252. }
  253. def exitValue(processName: String): Int = {
  254. processes(processName).exitValue()
  255. }
  256. val shutDownHook: ShutdownHookThread = sys.addShutdownHook {
  257. shutdown()
  258. }
  259. }