/spark-mongodb-examples/src/main/scala/com/stratio/datasource/mongodb/examples/ExampleUtils.scala

https://github.com/Stratio/Spark-MongoDB · Scala · 84 lines · 56 code · 13 blank · 15 comment · 2 complexity · 679db6f543b81e1ccec27f1f320b944a MD5 · raw file

  1. /*
  2. * Copyright (C) 2015 Stratio (http://stratio.com)
  3. *
  4. * Licensed under the Apache License, Version 2.0 (the "License");
  5. * you may not use this file except in compliance with the License.
  6. * You may obtain a copy of the License at
  7. *
  8. * http://www.apache.org/licenses/LICENSE-2.0
  9. *
  10. * Unless required by applicable law or agreed to in writing, software
  11. * distributed under the License is distributed on an "AS IS" BASIS,
  12. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. * See the License for the specific language governing permissions and
  14. * limitations under the License.
  15. */
  16. package com.stratio.datasource.mongodb.examples
  17. import com.mongodb.QueryBuilder
  18. import com.mongodb.casbah.MongoClient
  19. import com.mongodb.casbah.commons.{MongoDBList, MongoDBObject}
  20. import com.stratio.datasource.mongodb.examples.DataFrameAPIExample._
  21. import org.apache.spark.sql.SQLContext
  22. import org.apache.spark.{SparkConf, SparkContext}
  23. trait MongoDefaultConstants {
  24. val Database = "highschool"
  25. val Collection = "students"
  26. val MongoHost = "127.0.0.1"
  27. val MongoPort = 27017
  28. val MongoProvider = "com.stratio.datasource.mongodb"
  29. }
  30. object MongoExampleFunctions {
  31. def withSQLContext(block: SQLContext => Unit) = {
  32. val sparkConf = new SparkConf().
  33. setAppName("MongoDFExample").
  34. setMaster("local[4]")
  35. val sc = new SparkContext(sparkConf)
  36. try {
  37. val sqlContext = new SQLContext(sc)
  38. block(sqlContext)
  39. } finally {
  40. sc.stop()
  41. }
  42. }
  43. def prepareEnvironment(): MongoClient = {
  44. val mongoClient = MongoClient(MongoHost, MongoPort)
  45. populateTable(mongoClient)
  46. mongoClient
  47. }
  48. def cleanEnvironment(mongoClient: MongoClient) = {
  49. cleanData(mongoClient)
  50. mongoClient.close()
  51. }
  52. private def populateTable(client: MongoClient): Unit = {
  53. val collection = client(Database)(Collection)
  54. for (a <- 1 to 10) {
  55. collection.insert {
  56. MongoDBObject("id" -> a.toString,
  57. "age" -> (10 + a),
  58. "description" -> s"description $a",
  59. "enrolled" -> (a % 2 == 0),
  60. "name" -> s"Name $a"
  61. )
  62. }
  63. }
  64. collection.update(QueryBuilder.start("age").greaterThan(14).get, MongoDBObject(("$set", MongoDBObject(("optionalField", true)))), multi = true)
  65. collection.update(QueryBuilder.start("age").is(14).get, MongoDBObject(("$set", MongoDBObject(("fieldWithSubDoc", MongoDBObject(("subDoc", MongoDBList("foo", "bar"))))))))
  66. }
  67. private def cleanData(client: MongoClient): Unit = {
  68. val collection = client(Database)(Collection)
  69. collection.dropCollection()
  70. }
  71. }