/src/clj/backtype/storm/node.clj

  1(ns backtype.storm.node
  2  (:import
  3   [java.util Map List]
  4   [org.jvyaml YAML]
  5   [java.io FileReader File])
  6  (:require
  7   [backtype.storm.crate.storm :as storm]
  8   [backtype.storm.crate.leiningen :as leiningen]
  9   [backtype.storm.crate.zeromq :as zeromq]
 10   [backtype.storm.crate.ganglia :as ganglia]
 11
 12   [pallet.crate.git :as git]
 13   [pallet.crate.maven :as maven]
 14   [pallet.crate.java :as java]
 15   [pallet.crate.automated-admin-user :as automated-admin-user]
 16   [backtype.storm.crate.zookeeper :as zookeeper]
 17   [pallet.crate.ssh-key :as ssh-key]
 18   [pallet.action.service :as action-service]
 19
 20   [pallet.resource.directory :as directory]
 21   [pallet.resource.service :as service]
 22   [pallet.resource.remote-file :as remote-file]
 23   [pallet.resource.exec-script :as exec-script])
 24  (:use
 25   [backtype.storm config]
 26   [backtype.storm.branch :only [branch>]]
 27   [pallet compute core resource phase]
 28   [pallet [utils :only [make-user]]]
 29   [org.jclouds.compute2 :only [nodes-in-group]]
 30   [clojure.walk]))
 31
 32;; CONSTANTS
 33
 34(def clusters-conf
 35  (read-yaml-config "clusters.yaml"))
 36
 37(def storm-yaml-path
 38  (.getPath (ClassLoader/getSystemResource "storm.yaml"))
 39  )
 40
 41(def storm-conf (read-storm-config))
 42
 43(defn nimbus-name [name]
 44  (str "nimbus-" name))
 45
 46(defn configure-ssh-client [request & {:keys [host-key-checking]}]
 47  (let [yes-or-no #(if % "yes" "no")]
 48    (-> request
 49        (remote-file/remote-file
 50         "/root/.ssh/config"
 51         :content (str "StrictHostKeyChecking=" (yes-or-no host-key-checking))
 52         :mode 600))))
 53
 54(def *USER* nil)
 55
 56(defn base-server-spec []
 57  (server-spec
 58   :phases {:bootstrap (fn [req] (automated-admin-user/automated-admin-user
 59                                  req
 60                                  (:username *USER*)
 61                                  (:public-key-path *USER*)))
 62            :configure (phase-fn
 63                        (java/java :openjdk))}))
 64
 65(defn zookeeper-server-spec []
 66     (server-spec
 67      :extends (base-server-spec)
 68      :phases {:configure (phase-fn
 69                           (zookeeper/install :version "3.3.6")
 70                           (zookeeper/configure
 71                            :clientPort (storm-conf "storm.zookeeper.port")
 72                            :maxClientCnxns 0)
 73                           (zookeeper/init))
 74                           }))
 75
 76(defn storm-base-server-spec [name]
 77     (server-spec
 78      :extends (base-server-spec)
 79      :phases {:post-configure (phase-fn
 80                                (storm/write-storm-yaml
 81                                 name
 82                                 storm-yaml-path))
 83               :configure (phase-fn
 84                           (configure-ssh-client :host-key-checking false))
 85               :exec (phase-fn
 86                      (storm/exec-daemon)
 87                      (ganglia/ganglia-finish))}))
 88
 89(defn supervisor-server-spec [name branch commit]
 90     (server-spec
 91      :extends (storm-base-server-spec name)
 92      :phases {:configure (phase-fn
 93                           (ganglia/ganglia-node (nimbus-name name))
 94                           (storm/install-supervisor
 95                            branch commit
 96                            "/mnt/storm"))
 97               :post-configure (phase-fn
 98                                (ganglia/ganglia-finish)
 99                                (storm/write-storm-exec
100                                 "supervisor"))}))
101
102(defn maybe-install-drpc [req branch]
103  (if (or (not branch) (= branch "master") (branch> branch "0.5.3"))
104    (storm/install-drpc req)
105    req
106    ))
107
108(defn maybe-exec-drpc [req branch]
109  (if (or (not branch) (= branch "master") (branch> branch "0.5.3"))
110    (storm/exec-drpc req)
111    req
112    ))
113
114(defn nimbus-server-spec [name branch commit]
115     (server-spec
116      :extends (storm-base-server-spec name)
117      :phases {:configure (phase-fn
118                           (ganglia/ganglia-master (nimbus-name name))
119                           (storm/install-nimbus
120                            branch commit
121                            "/mnt/storm")
122                           (storm/install-ui)
123                           (maybe-install-drpc branch))
124               :post-configure (phase-fn
125                                (ganglia/ganglia-finish)
126                                (storm/write-storm-exec
127                                 "nimbus")
128                                 )
129               :exec (phase-fn
130                        (storm/exec-ui)
131                        (maybe-exec-drpc branch))}))
132
133(defn node-spec-from-config [group-name inbound-ports]
134  (letfn [(assoc-with-conf-key [image image-key conf-key & {:keys [f] :or {f identity}}]
135            (if-let [val (clusters-conf (str group-name "." conf-key))]
136              (assoc image image-key (f val))
137              image))]
138       (node-spec
139        :image (-> {:inbound-ports (concat inbound-ports [22])
140                    ;; :security-groups ["backend"]
141                    }
142                   (assoc-with-conf-key :image-id "image")
143                   (assoc-with-conf-key :hardware-id "hardware")
144                   (assoc-with-conf-key :spot-price "spot.price" :f float)
145                   ))))
146
147(defn zookeeper
148  ([name server-spec]
149     (group-spec
150      (str "zookeeper-" name)
151      :node-spec (node-spec-from-config "zookeeper"
152                                        [(storm-conf "storm.zookeeper.port")])
153      :extends server-spec))
154  ([name]
155     (zookeeper name (zookeeper-server-spec))
156    ))
157
158(defn nimbus* [name server-spec]
159  (group-spec
160    (nimbus-name name)
161    :node-spec (node-spec-from-config "nimbus"
162                                      [(storm-conf "nimbus.thrift.port")])
163    :extends server-spec))
164
165(defn nimbus [name branch commit]
166  (nimbus* name (nimbus-server-spec name branch commit)))
167
168(defn supervisor* [name server-spec]
169  (group-spec
170    (str "supervisor-" name)
171    :node-spec (node-spec-from-config "supervisor"
172                                    (storm-conf "supervisor.slots.ports"))
173    :extends server-spec))
174
175(defn supervisor [name branch commit]
176  (supervisor* name (supervisor-server-spec name branch commit)))
177
178