PageRenderTime 74ms CodeModel.GetById 27ms RepoModel.GetById 1ms app.codeStats 0ms

/third_party/launchpad.net/mgo/cluster.go

https://code.google.com/
Go | 481 lines | 363 code | 53 blank | 65 comment | 86 complexity | 9add91a423661f89d863197127073f7c MD5 | raw file
Possible License(s): Apache-2.0, BSD-3-Clause
  1. // mgo - MongoDB driver for Go
  2. //
  3. // Copyright (c) 2010-2012 - Gustavo Niemeyer <gustavo@niemeyer.net>
  4. //
  5. // All rights reserved.
  6. //
  7. // Redistribution and use in source and binary forms, with or without
  8. // modification, are permitted provided that the following conditions are met:
  9. //
  10. // 1. Redistributions of source code must retain the above copyright notice, this
  11. // list of conditions and the following disclaimer.
  12. // 2. Redistributions in binary form must reproduce the above copyright notice,
  13. // this list of conditions and the following disclaimer in the documentation
  14. // and/or other materials provided with the distribution.
  15. //
  16. // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
  17. // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18. // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19. // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
  20. // ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  21. // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  22. // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  23. // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  24. // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  25. // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26. package mgo
  27. import (
  28. "errors"
  29. "math/rand"
  30. "sync"
  31. "time"
  32. )
  33. // ---------------------------------------------------------------------------
  34. // Mongo cluster encapsulation.
  35. //
  36. // A cluster enables the communication with one or more servers participating
  37. // in a mongo cluster. This works with individual servers, a replica set,
  38. // a replica pair, one or multiple mongos routers, etc.
  39. type mongoCluster struct {
  40. sync.RWMutex
  41. serverSynced sync.Cond
  42. userSeeds []string
  43. dynaSeeds []string
  44. servers mongoServers
  45. masters mongoServers
  46. slaves mongoServers
  47. references int
  48. syncing bool
  49. direct bool
  50. cachedIndex map[string]bool
  51. sync chan bool
  52. }
  53. func newCluster(userSeeds []string, direct bool) *mongoCluster {
  54. cluster := &mongoCluster{userSeeds: userSeeds, references: 1, direct: direct}
  55. cluster.serverSynced.L = cluster.RWMutex.RLocker()
  56. cluster.sync = make(chan bool, 1)
  57. go cluster.syncServersLoop()
  58. return cluster
  59. }
  60. // Acquire increases the reference count for the cluster.
  61. func (cluster *mongoCluster) Acquire() {
  62. cluster.Lock()
  63. cluster.references++
  64. debugf("Cluster %p acquired (refs=%d)", cluster, cluster.references)
  65. cluster.Unlock()
  66. }
  67. // Release decreases the reference count for the cluster. Once
  68. // it reaches zero, all servers will be closed.
  69. func (cluster *mongoCluster) Release() {
  70. cluster.Lock()
  71. if cluster.references == 0 {
  72. panic("cluster.Release() with references == 0")
  73. }
  74. cluster.references--
  75. debugf("Cluster %p released (refs=%d)", cluster, cluster.references)
  76. if cluster.references == 0 {
  77. for _, server := range cluster.servers.Slice() {
  78. server.Close()
  79. }
  80. // Wake up the sync loop so it can die.
  81. cluster.syncServers()
  82. }
  83. cluster.Unlock()
  84. }
  85. func (cluster *mongoCluster) LiveServers() (servers []string) {
  86. cluster.RLock()
  87. for _, serv := range cluster.servers.Slice() {
  88. servers = append(servers, serv.Addr)
  89. }
  90. cluster.RUnlock()
  91. return servers
  92. }
  93. func (cluster *mongoCluster) removeServer(server *mongoServer) {
  94. cluster.Lock()
  95. cluster.masters.Remove(server)
  96. cluster.slaves.Remove(server)
  97. other := cluster.servers.Remove(server)
  98. cluster.Unlock()
  99. if other != nil {
  100. other.Close()
  101. log("Removed server ", server.Addr, " from cluster.")
  102. }
  103. server.Close()
  104. }
  105. type isMasterResult struct {
  106. IsMaster bool
  107. Secondary bool
  108. Primary string
  109. Hosts []string
  110. Passives []string
  111. }
  112. func (cluster *mongoCluster) syncServer(server *mongoServer) (hosts []string, err error) {
  113. addr := server.Addr
  114. log("SYNC Processing ", addr, "...")
  115. defer func() {
  116. if err != nil {
  117. cluster.removeServer(server)
  118. }
  119. }()
  120. socket, err := server.AcquireSocket()
  121. if err != nil {
  122. log("SYNC Failed to get socket to ", addr, ": ", err.Error())
  123. return
  124. }
  125. // Monotonic will let us talk to a slave and still hold the socket.
  126. session := newSession(Monotonic, cluster, socket, 10 * time.Second)
  127. defer session.Close()
  128. // session holds the socket now.
  129. socket.Release()
  130. result := isMasterResult{}
  131. err = session.Run("ismaster", &result)
  132. if err != nil {
  133. log("SYNC Command 'ismaster' to ", addr, " failed: ", err.Error())
  134. return
  135. }
  136. debugf("SYNC Result of 'ismaster' from %s: %#v", addr, result)
  137. if result.IsMaster {
  138. log("SYNC ", addr, " is a master.")
  139. // Made an incorrect assumption above, so fix stats.
  140. stats.conn(-1, false)
  141. server.SetMaster(true)
  142. stats.conn(+1, true)
  143. } else if result.Secondary {
  144. log("SYNC ", addr, " is a slave.")
  145. } else {
  146. log("SYNC ", addr, " is neither a master nor a slave.")
  147. // Made an incorrect assumption above, so fix stats.
  148. stats.conn(-1, false)
  149. return nil, errors.New(addr + " is not a master nor slave")
  150. }
  151. hosts = make([]string, 0, 1+len(result.Hosts)+len(result.Passives))
  152. if result.Primary != "" {
  153. // First in the list to speed up master discovery.
  154. hosts = append(hosts, result.Primary)
  155. }
  156. hosts = append(hosts, result.Hosts...)
  157. hosts = append(hosts, result.Passives...)
  158. debugf("SYNC %s knows about the following peers: %#v", addr, hosts)
  159. return hosts, nil
  160. }
  161. func (cluster *mongoCluster) mergeServer(server *mongoServer) {
  162. cluster.Lock()
  163. previous := cluster.servers.Search(server)
  164. isMaster := server.IsMaster()
  165. if previous == nil {
  166. cluster.servers.Add(server)
  167. if isMaster {
  168. log("SYNC Adding ", server.Addr, " to cluster as a master.")
  169. cluster.masters.Add(server)
  170. } else {
  171. log("SYNC Adding ", server.Addr, " to cluster as a slave.")
  172. cluster.slaves.Add(server)
  173. }
  174. } else {
  175. if isMaster != previous.IsMaster() {
  176. if isMaster {
  177. log("SYNC Server ", server.Addr, " is now a master.")
  178. cluster.slaves.Remove(previous)
  179. cluster.masters.Add(previous)
  180. } else {
  181. log("SYNC Server ", server.Addr, " is now a slave.")
  182. cluster.masters.Remove(previous)
  183. cluster.slaves.Add(previous)
  184. }
  185. }
  186. previous.Merge(server)
  187. }
  188. debugf("SYNC Broadcasting availability of server %s", server.Addr)
  189. cluster.serverSynced.Broadcast()
  190. cluster.Unlock()
  191. }
  192. func (cluster *mongoCluster) getKnownAddrs() []string {
  193. cluster.RLock()
  194. max := len(cluster.userSeeds) + len(cluster.dynaSeeds) + cluster.servers.Len()
  195. seen := make(map[string]bool, max)
  196. known := make([]string, 0, max)
  197. add := func(addr string) {
  198. if _, found := seen[addr]; !found {
  199. seen[addr] = true
  200. known = append(known, addr)
  201. }
  202. }
  203. for _, addr := range cluster.userSeeds {
  204. add(addr)
  205. }
  206. for _, addr := range cluster.dynaSeeds {
  207. add(addr)
  208. }
  209. for _, serv := range cluster.servers.Slice() {
  210. add(serv.Addr)
  211. }
  212. cluster.RUnlock()
  213. return known
  214. }
  215. // syncServers injects a value into the cluster.sync channel to force
  216. // an iteration of the syncServersLoop function.
  217. func (cluster *mongoCluster) syncServers() {
  218. select {
  219. case cluster.sync <- true:
  220. default:
  221. }
  222. }
  223. // How long to wait for a checkup of the cluster topology if nothing
  224. // else kicks a synchronization first.
  225. const syncServersDelay = 5 * time.Minute
  226. // syncServersLoop loops while the cluster is alive to keep its idea of
  227. // the server topology up-to-date. It must be called just once from
  228. // newCluster. The loop iterates once syncServersDelay has passed, or
  229. // if somebody injects a value into the cluster.sync channel to force a
  230. // synchronization. A loop iteration will contact all servers in
  231. // parallel, ask them about known peers and their own role within the
  232. // cluster, and then attempt to do the same with all the peers
  233. // retrieved.
  234. func (cluster *mongoCluster) syncServersLoop() {
  235. for {
  236. debugf("SYNC Cluster %p is starting a sync loop iteration.", cluster)
  237. cluster.Lock()
  238. if cluster.references == 0 {
  239. cluster.Unlock()
  240. break
  241. }
  242. cluster.references++ // Keep alive while syncing.
  243. direct := cluster.direct
  244. cluster.Unlock()
  245. cluster.syncServersIteration(direct)
  246. // We just synchronized, so consume any outstanding requests.
  247. select {
  248. case <-cluster.sync:
  249. default:
  250. }
  251. cluster.Release()
  252. // Hold off before allowing another sync. No point in
  253. // burning CPU looking for down servers.
  254. time.Sleep(5e8)
  255. cluster.Lock()
  256. if cluster.references == 0 {
  257. cluster.Unlock()
  258. break
  259. }
  260. // Poke all waiters so they have a chance to timeout or
  261. // restart syncing if they wish to.
  262. cluster.serverSynced.Broadcast()
  263. // Check if we have to restart immediately either way.
  264. restart := !direct && cluster.masters.Empty() || cluster.servers.Empty()
  265. cluster.Unlock()
  266. if restart {
  267. log("SYNC No masters found. Will synchronize again.")
  268. continue
  269. }
  270. debugf("SYNC Cluster %p waiting for next requested or scheduled sync.", cluster)
  271. // Hold off until somebody explicitly requests a synchronization
  272. // or it's time to check for a cluster topology change again.
  273. select {
  274. case <-cluster.sync:
  275. case <-time.After(syncServersDelay):
  276. }
  277. }
  278. debugf("SYNC Cluster %p is stopping its sync loop.", cluster)
  279. }
  280. func (cluster *mongoCluster) syncServersIteration(direct bool) {
  281. log("SYNC Starting full topology synchronization...")
  282. var wg sync.WaitGroup
  283. var m sync.Mutex
  284. mergePending := make(map[string]*mongoServer)
  285. mergeRequested := make(map[string]bool)
  286. seen := make(map[string]bool)
  287. goodSync := false
  288. var spawnSync func(addr string, byMaster bool)
  289. spawnSync = func(addr string, byMaster bool) {
  290. wg.Add(1)
  291. go func() {
  292. defer wg.Done()
  293. server, err := newServer(addr, cluster.sync)
  294. if err != nil {
  295. log("SYNC Failed to start sync of ", addr, ": ", err.Error())
  296. return
  297. }
  298. m.Lock()
  299. if byMaster {
  300. if s, found := mergePending[server.ResolvedAddr]; found {
  301. delete(mergePending, server.ResolvedAddr)
  302. m.Unlock()
  303. cluster.mergeServer(s)
  304. return
  305. }
  306. mergeRequested[server.ResolvedAddr] = true
  307. }
  308. if seen[server.ResolvedAddr] {
  309. m.Unlock()
  310. return
  311. }
  312. seen[server.ResolvedAddr] = true
  313. m.Unlock()
  314. hosts, err := cluster.syncServer(server)
  315. if err == nil {
  316. isMaster := server.IsMaster()
  317. if !direct {
  318. for _, addr := range hosts {
  319. spawnSync(addr, isMaster)
  320. }
  321. }
  322. m.Lock()
  323. merge := direct || isMaster
  324. if mergeRequested[server.ResolvedAddr] {
  325. merge = true
  326. } else if !merge {
  327. mergePending[server.ResolvedAddr] = server
  328. }
  329. if merge {
  330. goodSync = true
  331. }
  332. m.Unlock()
  333. if merge {
  334. cluster.mergeServer(server)
  335. }
  336. }
  337. }()
  338. }
  339. for _, addr := range cluster.getKnownAddrs() {
  340. spawnSync(addr, false)
  341. }
  342. wg.Wait()
  343. for _, server := range mergePending {
  344. if goodSync {
  345. cluster.removeServer(server)
  346. } else {
  347. server.Close()
  348. }
  349. }
  350. cluster.Lock()
  351. log("SYNC Synchronization completed: ", cluster.masters.Len(),
  352. " master(s) and, ", cluster.slaves.Len(), " slave(s) alive.")
  353. // Update dynamic seeds, but only if we have any good servers. Otherwise,
  354. // leave them alone for better chances of a successful sync in the future.
  355. if goodSync {
  356. dynaSeeds := make([]string, cluster.servers.Len())
  357. for i, server := range cluster.servers.Slice() {
  358. dynaSeeds[i] = server.Addr
  359. }
  360. cluster.dynaSeeds = dynaSeeds
  361. debugf("SYNC New dynamic seeds: %#v\n", dynaSeeds)
  362. }
  363. cluster.Unlock()
  364. }
  365. // AcquireSocket returns a socket to a server in the cluster. If slaveOk is
  366. // true, it will attempt to return a socket to a slave server. If it is
  367. // false, the socket will necessarily be to a master server.
  368. func (cluster *mongoCluster) AcquireSocket(slaveOk bool, syncTimeout time.Duration) (s *mongoSocket, err error) {
  369. started := time.Now()
  370. for {
  371. cluster.RLock()
  372. for {
  373. debugf("Cluster has %d known masters and %d known slaves.", cluster.masters.Len(), cluster.slaves.Len())
  374. if !cluster.masters.Empty() || slaveOk && !cluster.slaves.Empty() {
  375. break
  376. }
  377. if syncTimeout != 0 && started.Before(time.Now().Add(-syncTimeout)) {
  378. cluster.RUnlock()
  379. return nil, errors.New("no reachable servers")
  380. }
  381. log("Waiting for servers to synchronize...")
  382. cluster.syncServers()
  383. cluster.serverSynced.Wait()
  384. }
  385. var server *mongoServer
  386. if !slaveOk || cluster.slaves.Empty() {
  387. i := rand.Intn(cluster.masters.Len())
  388. server = cluster.masters.Get(i)
  389. } else {
  390. i := rand.Intn(cluster.slaves.Len())
  391. server = cluster.slaves.Get(i)
  392. }
  393. cluster.RUnlock()
  394. s, err = server.AcquireSocket()
  395. if err != nil {
  396. cluster.removeServer(server)
  397. cluster.syncServers()
  398. continue
  399. }
  400. return s, nil
  401. }
  402. panic("unreached")
  403. }
  404. func (cluster *mongoCluster) CacheIndex(cacheKey string, exists bool) {
  405. cluster.Lock()
  406. if cluster.cachedIndex == nil {
  407. cluster.cachedIndex = make(map[string]bool)
  408. }
  409. if exists {
  410. cluster.cachedIndex[cacheKey] = true
  411. } else {
  412. delete(cluster.cachedIndex, cacheKey)
  413. }
  414. cluster.Unlock()
  415. }
  416. func (cluster *mongoCluster) HasCachedIndex(cacheKey string) (result bool) {
  417. cluster.RLock()
  418. if cluster.cachedIndex != nil {
  419. result = cluster.cachedIndex[cacheKey]
  420. }
  421. cluster.RUnlock()
  422. return
  423. }
  424. func (cluster *mongoCluster) ResetIndexCache() {
  425. cluster.Lock()
  426. cluster.cachedIndex = make(map[string]bool)
  427. cluster.Unlock()
  428. }