/map-reduce/src/map_reduce/plugin/rtiming.clj

https://github.com/siclait/6.824-cljlabs-2020 · Clojure · 64 lines · 56 code · 8 blank · 0 comment · 2 complexity · 6aa0567a11d53ea91e7adff82f3f17b4 MD5 · raw file

  1. (ns map-reduce.plugin.rtiming
  2. (:require [clojure.java.io :as io]
  3. [clojure.java.shell :as shell]
  4. [clojure.string :as string]
  5. [map-reduce.plugin :as plugin])
  6. (:import java.lang.management.ManagementFactory))
  7. (def output-directory "mr-tmp")
  8. (defn get-pid
  9. []
  10. (-> (ManagementFactory/getRuntimeMXBean)
  11. .getName
  12. (string/split #"@")
  13. first))
  14. (defn n-parallel
  15. [phase]
  16. (let [pid (get-pid)
  17. filename (io/file output-directory (format "mr-worker-%s-%s" phase pid))
  18. _ (spit filename "x")
  19. names (->> (io/file output-directory)
  20. file-seq
  21. (map str)
  22. (filter #(re-find (-> "mr-worker-%s"
  23. (format phase)
  24. re-pattern)
  25. %)))
  26. running-list (into [] (for [name names]
  27. (let [[_ x-pid] (re-find (-> "mr-worker-%s-(\\d+)"
  28. (format phase)
  29. re-pattern)
  30. name)]
  31. (if (zero? (-> (shell/sh "kill" "-0" x-pid)
  32. :exit))
  33. 1
  34. 0))))]
  35. (Thread/sleep 1000)
  36. (io/delete-file filename)
  37. (reduce + running-list)))
  38. (defn mapf
  39. [_ _]
  40. [{:key "a" :value "1"}
  41. {:key "b" :value "1"}
  42. {:key "c" :value "1"}
  43. {:key "d" :value "1"}
  44. {:key "e" :value "1"}
  45. {:key "f" :value "1"}
  46. {:key "g" :value "1"}
  47. {:key "h" :value "1"}
  48. {:key "i" :value "1"}
  49. {:key "j" :value "1"}])
  50. (defn reducef
  51. [_ _]
  52. (let [n (n-parallel "reduce")]
  53. (str n)))
  54. (defmethod plugin/load-plugin :rtiming [_]
  55. {:mapf mapf
  56. :reducef reducef})