/src/clj/backtype/storm/provision.clj

  1(ns backtype.storm.provision
  2  (:import [java.io File])
  3  (:use [clojure.contrib.command-line]
  4        [pallet.compute :exclude [admin-user]]
  5        [backtype.storm security]
  6        [pallet.core]
  7        [org.jclouds.compute2 :only [nodes-in-group]]
  8        [backtype.storm.util :only [with-var-roots]]
  9        [clojure.tools.logging]
 10        )
 11  (:require [pallet.configure])
 12  (:require [pallet.compute.jclouds])
 13  (:require [backtype.storm.node :as node])
 14  (:require [backtype.storm.crate.storm :as storm])
 15  (:require [backtype.storm.deploy-util :as util]))
 16
 17(log-capture! "java.logging")
 18
 19;; memoize this
 20(defn my-region []
 21  (-> (pallet.configure/pallet-config) :services :default :jclouds.regions)
 22  )
 23
 24(defn jclouds-group [& group-pieces]
 25  (str "jclouds#" (apply str group-pieces)))
 26
 27(defn- print-ips-for-tag! [aws tag-str]
 28  (let [running-node (filter running? (map (partial pallet.compute.jclouds/jclouds-node->node aws) (nodes-in-group aws tag-str)))]
 29    (info (str "TAG:     " tag-str))
 30    (info (str "PUBLIC:  " (clojure.string/join ", " (map primary-ip running-node))))
 31    (info (str "PRIVATE: " (clojure.string/join ", " (map private-ip running-node))))))
 32
 33(defn print-all-ips! [aws name]
 34  (let [all-tags [(str "zookeeper-" name) (str "nimbus-" name) (str "supervisor-" name)]]
 35       (doseq [tag all-tags]
 36         (print-ips-for-tag! aws tag))))
 37
 38(defn converge! [name branch commit aws sn zn nn]
 39  (converge {(node/nimbus name branch commit) nn
 40             (node/supervisor name branch commit) sn
 41             (node/zookeeper name) zn
 42             }
 43            :compute aws))
 44
 45(defn sync-storm-conf-dir [aws name]
 46  (let [conf-dir (str (System/getProperty "user.home") "/.storm")
 47        storm-yaml (storm/mk-storm-yaml name node/storm-yaml-path aws)
 48        supervisor-yaml (storm/mk-supervisor-yaml aws name)]
 49    (.mkdirs (File. conf-dir))
 50    (spit (str conf-dir "/storm.yaml") storm-yaml)
 51    (spit (str conf-dir "/supervisor.yaml") supervisor-yaml)))
 52
 53(defn attach! [aws name]
 54  (info "Attaching to Available Cluster...")
 55  (sync-storm-conf-dir aws name)
 56  (authorizeme aws (jclouds-group "nimbus-" name) 80 (my-region))
 57  (authorizeme aws (jclouds-group "nimbus-" name) (node/storm-conf "nimbus.thrift.port") (my-region))
 58  (authorizeme aws (jclouds-group "nimbus-" name) (node/storm-conf "ui.port") (my-region))
 59  (authorizeme aws (jclouds-group "nimbus-" name) (node/storm-conf "drpc.port") (my-region))
 60  (info "Attaching Complete."))
 61
 62(defn start-with-nodes! [aws name nimbus supervisor zookeeper]
 63  (let [nimbus (node/nimbus* name nimbus)
 64        supervisor (node/supervisor* name supervisor)
 65        zookeeper (node/zookeeper name zookeeper)
 66        sn (int (node/clusters-conf "supervisor.count" 1))
 67        zn (int (node/clusters-conf "zookeeper.count" 1))]
 68    (info (format "Provisioning nodes [nn=1, sn=%d, zn=%d]" sn zn))
 69    (converge {nimbus 1
 70              supervisor sn
 71              zookeeper zn
 72              }
 73            :compute aws
 74            )
 75    (debug "Finished converge")
 76
 77    (authorize-group aws (my-region) (jclouds-group "nimbus-" name) (jclouds-group "supervisor-" name))
 78    (authorize-group aws (my-region) (jclouds-group "supervisor-" name) (jclouds-group "nimbus-" name))
 79    (debug "Finished authorizing groups")
 80
 81    (lift nimbus :compute aws :phase [:post-configure :exec])
 82    (lift supervisor :compute aws :phase [:post-configure :exec])
 83    (debug "Finished post-configure and exec phases")
 84
 85    (attach! aws name)
 86    (info "Provisioning Complete.")
 87    (print-all-ips! aws name)))
 88
 89(defn start! [aws name branch commit]
 90  (println "Starting cluster with storm branch " branch (if (empty? commit) "" (str " and commit " commit)))
 91  (start-with-nodes! aws name (node/nimbus-server-spec name branch commit) (node/supervisor-server-spec name branch commit) (node/zookeeper-server-spec))
 92  )
 93
 94(defn upgrade-with-nodes! [aws name nimbus supervisor zookeeper]
 95  (let [nimbus (node/nimbus* name nimbus)
 96        supervisor (node/supervisor* name supervisor)
 97        zookeeper (node/zookeeper name zookeeper)]
 98;    (authorize-group aws (my-region) (jclouds-group "nimbus-" name) (jclouds-group "supervisor-" name))
 99;    (authorize-group aws (my-region) (jclouds-group "supervisor-" name) (jclouds-group "nimbus-" name))
100
101;    (lift zookeeper :compute aws :phase [:configure])
102;    (lift nimbus :compute aws :phase [:configure :post-configure :exec])
103    (lift supervisor :compute aws :phase [:configure :post-configure :exec])
104    (println "Upgrade Complete.")))
105
106
107(defn upgrade! [aws name branch commit]
108  (println "Upgrading cluster with storm branch " branch (if (empty? commit) "" (str " and commit " commit)))
109  (upgrade-with-nodes! aws name (node/nimbus-server-spec name branch commit) (node/supervisor-server-spec name branch commit) (node/zookeeper-server-spec))
110  )
111
112(defn stop! [aws name]
113  (println "Shutting Down nodes...")
114  (converge! name nil nil aws 0 0 0)
115  (println "Shutdown Finished."))
116
117(defn mk-aws []
118  (let [storm-conf (-> (storm/storm-config "default")
119                       (update-in [:environment :user] util/resolve-keypaths))]
120    (compute-service-from-map storm-conf)))
121
122(defn -main [& args]
123  (let [aws (mk-aws)
124        user (-> (storm/storm-config "default")
125                 :environment
126                 :user
127                 (util/resolve-keypaths))
128        ]
129    (System/setProperty "jna.nosys" "true")
130    (with-var-roots [node/*USER* user]
131      (with-command-line args
132        "Provisioning tool for Storm Clusters"
133        [[start? "Start Cluster?"]
134         [stop? "Shutdown Cluster?"]
135         [attach? "Attach to Cluster?"]
136         [upgrade? "Upgrade existing cluster"]
137         [ips? "Print Cluster IP Addresses?"]
138         [name "Cluster name" "dev"]
139         [branch "Branch" "master"]; default branch is master.
140         [commit "Commit SHA1" nil]]; default is not to pass a commit
141
142        (cond
143         stop? (stop! aws name)
144         start? (start! aws name branch commit)
145         upgrade? (upgrade! aws name branch commit)
146         attach? (attach! aws name)
147         ips? (print-all-ips! aws name)
148         :else (println "Must pass --start, --stop , upgrade, --attach or --ips")))))
149  (shutdown-agents)
150  (println "Done.")
151  (System/exit 0))
152
153;; DEBUGGING
154(comment
155(use 'backtype.storm.provision)
156(ns backtype.storm.provision)
157(def aws (mk-aws))
158(lift (node/supervisor "test" nil) :compute aws :phase [:post-configure] )
159(sync-storm-conf-dir aws)
160(print-all-ips! aws)
161)