PageRenderTime 240ms CodeModel.GetById 27ms RepoModel.GetById 0ms app.codeStats 1ms

/golang/src/gopkg.in/olivere/elastic.v3/bulk_processor_test.go

https://gitlab.com/vantruong/GolangRestAPI
Go | 421 lines | 347 code | 52 blank | 22 comment | 121 complexity | dd334b58aefcf4870033aee08fc29105 MD5 | raw file
  1. // Copyright 2012-2016 Oliver Eilhard. All rights reserved.
  2. // Use of this source code is governed by a MIT-license.
  3. // See http://olivere.mit-license.org/license.txt for details.
  4. package elastic
  5. import (
  6. "fmt"
  7. "math/rand"
  8. "sync/atomic"
  9. "testing"
  10. "time"
  11. )
  12. func TestBulkProcessorDefaults(t *testing.T) {
  13. client := setupTestClientAndCreateIndex(t)
  14. p := client.BulkProcessor()
  15. if p == nil {
  16. t.Fatalf("expected BulkProcessorService; got: %v", p)
  17. }
  18. if got, want := p.name, ""; got != want {
  19. t.Errorf("expected %q; got: %q", want, got)
  20. }
  21. if got, want := p.numWorkers, 1; got != want {
  22. t.Errorf("expected %d; got: %d", want, got)
  23. }
  24. if got, want := p.bulkActions, 1000; got != want {
  25. t.Errorf("expected %d; got: %d", want, got)
  26. }
  27. if got, want := p.bulkSize, 5*1024*1024; got != want {
  28. t.Errorf("expected %d; got: %d", want, got)
  29. }
  30. if got, want := p.flushInterval, time.Duration(0); got != want {
  31. t.Errorf("expected %v; got: %v", want, got)
  32. }
  33. if got, want := p.wantStats, false; got != want {
  34. t.Errorf("expected %v; got: %v", want, got)
  35. }
  36. }
  37. func TestBulkProcessorCommitOnBulkActions(t *testing.T) {
  38. //client := setupTestClientAndCreateIndexAndLog(t, SetTraceLog(log.New(os.Stdout, "", 0)))
  39. client := setupTestClientAndCreateIndex(t)
  40. testBulkProcessor(t,
  41. 10000,
  42. client.BulkProcessor().
  43. Name("Actions-1").
  44. Workers(1).
  45. BulkActions(100).
  46. BulkSize(-1),
  47. )
  48. testBulkProcessor(t,
  49. 10000,
  50. client.BulkProcessor().
  51. Name("Actions-2").
  52. Workers(2).
  53. BulkActions(100).
  54. BulkSize(-1),
  55. )
  56. }
  57. func TestBulkProcessorCommitOnBulkSize(t *testing.T) {
  58. //client := setupTestClientAndCreateIndexAndLog(t, SetTraceLog(log.New(os.Stdout, "", 0)))
  59. client := setupTestClientAndCreateIndex(t)
  60. testBulkProcessor(t,
  61. 10000,
  62. client.BulkProcessor().
  63. Name("Size-1").
  64. Workers(1).
  65. BulkActions(-1).
  66. BulkSize(64*1024),
  67. )
  68. testBulkProcessor(t,
  69. 10000,
  70. client.BulkProcessor().
  71. Name("Size-2").
  72. Workers(2).
  73. BulkActions(-1).
  74. BulkSize(64*1024),
  75. )
  76. }
  77. func TestBulkProcessorBasedOnFlushInterval(t *testing.T) {
  78. //client := setupTestClientAndCreateIndexAndLog(t, SetTraceLog(log.New(os.Stdout, "", 0)))
  79. client := setupTestClientAndCreateIndex(t)
  80. var beforeRequests int64
  81. var befores int64
  82. var afters int64
  83. var failures int64
  84. var afterRequests int64
  85. beforeFn := func(executionId int64, requests []BulkableRequest) {
  86. atomic.AddInt64(&beforeRequests, int64(len(requests)))
  87. atomic.AddInt64(&befores, 1)
  88. }
  89. afterFn := func(executionId int64, requests []BulkableRequest, response *BulkResponse, err error) {
  90. atomic.AddInt64(&afters, 1)
  91. if err != nil {
  92. atomic.AddInt64(&failures, 1)
  93. }
  94. atomic.AddInt64(&afterRequests, int64(len(requests)))
  95. }
  96. svc := client.BulkProcessor().
  97. Name("FlushInterval-1").
  98. Workers(2).
  99. BulkActions(-1).
  100. BulkSize(-1).
  101. FlushInterval(1 * time.Second).
  102. Before(beforeFn).
  103. After(afterFn)
  104. p, err := svc.Do()
  105. if err != nil {
  106. t.Fatal(err)
  107. }
  108. const numDocs = 1000 // low-enough number that flush should be invoked
  109. for i := 1; i <= numDocs; i++ {
  110. tweet := tweet{User: "olivere", Message: fmt.Sprintf("%d. %s", i, randomString(rand.Intn(64)))}
  111. request := NewBulkIndexRequest().Index(testIndexName).Type("tweet").Id(fmt.Sprintf("%d", i)).Doc(tweet)
  112. p.Add(request)
  113. }
  114. // Should flush at least once
  115. time.Sleep(2 * time.Second)
  116. err = p.Close()
  117. if err != nil {
  118. t.Fatal(err)
  119. }
  120. if p.stats.Flushed == 0 {
  121. t.Errorf("expected at least 1 flush; got: %d", p.stats.Flushed)
  122. }
  123. if got, want := beforeRequests, int64(numDocs); got != want {
  124. t.Errorf("expected %d requests to before callback; got: %d", want, got)
  125. }
  126. if got, want := afterRequests, int64(numDocs); got != want {
  127. t.Errorf("expected %d requests to after callback; got: %d", want, got)
  128. }
  129. if befores == 0 {
  130. t.Error("expected at least 1 call to before callback")
  131. }
  132. if afters == 0 {
  133. t.Error("expected at least 1 call to after callback")
  134. }
  135. if failures != 0 {
  136. t.Errorf("expected 0 calls to failure callback; got: %d", failures)
  137. }
  138. // Check number of documents that were bulk indexed
  139. _, err = p.c.Flush(testIndexName).Do()
  140. if err != nil {
  141. t.Fatal(err)
  142. }
  143. count, err := p.c.Count(testIndexName).Do()
  144. if err != nil {
  145. t.Fatal(err)
  146. }
  147. if count != int64(numDocs) {
  148. t.Fatalf("expected %d documents; got: %d", numDocs, count)
  149. }
  150. }
  151. func TestBulkProcessorClose(t *testing.T) {
  152. //client := setupTestClientAndCreateIndexAndLog(t, SetTraceLog(log.New(os.Stdout, "", 0)))
  153. client := setupTestClientAndCreateIndex(t)
  154. var beforeRequests int64
  155. var befores int64
  156. var afters int64
  157. var failures int64
  158. var afterRequests int64
  159. beforeFn := func(executionId int64, requests []BulkableRequest) {
  160. atomic.AddInt64(&beforeRequests, int64(len(requests)))
  161. atomic.AddInt64(&befores, 1)
  162. }
  163. afterFn := func(executionId int64, requests []BulkableRequest, response *BulkResponse, err error) {
  164. atomic.AddInt64(&afters, 1)
  165. if err != nil {
  166. atomic.AddInt64(&failures, 1)
  167. }
  168. atomic.AddInt64(&afterRequests, int64(len(requests)))
  169. }
  170. p, err := client.BulkProcessor().
  171. Name("FlushInterval-1").
  172. Workers(2).
  173. BulkActions(-1).
  174. BulkSize(-1).
  175. FlushInterval(30 * time.Second). // 30 seconds to flush
  176. Before(beforeFn).After(afterFn).
  177. Do()
  178. if err != nil {
  179. t.Fatal(err)
  180. }
  181. const numDocs = 1000 // low-enough number that flush should be invoked
  182. for i := 1; i <= numDocs; i++ {
  183. tweet := tweet{User: "olivere", Message: fmt.Sprintf("%d. %s", i, randomString(rand.Intn(64)))}
  184. request := NewBulkIndexRequest().Index(testIndexName).Type("tweet").Id(fmt.Sprintf("%d", i)).Doc(tweet)
  185. p.Add(request)
  186. }
  187. // Should not flush because 30s > 1s
  188. time.Sleep(1 * time.Second)
  189. // Close should flush
  190. err = p.Close()
  191. if err != nil {
  192. t.Fatal(err)
  193. }
  194. if p.stats.Flushed != 0 {
  195. t.Errorf("expected no flush; got: %d", p.stats.Flushed)
  196. }
  197. if got, want := beforeRequests, int64(numDocs); got != want {
  198. t.Errorf("expected %d requests to before callback; got: %d", want, got)
  199. }
  200. if got, want := afterRequests, int64(numDocs); got != want {
  201. t.Errorf("expected %d requests to after callback; got: %d", want, got)
  202. }
  203. if befores == 0 {
  204. t.Error("expected at least 1 call to before callback")
  205. }
  206. if afters == 0 {
  207. t.Error("expected at least 1 call to after callback")
  208. }
  209. if failures != 0 {
  210. t.Errorf("expected 0 calls to failure callback; got: %d", failures)
  211. }
  212. // Check number of documents that were bulk indexed
  213. _, err = p.c.Flush(testIndexName).Do()
  214. if err != nil {
  215. t.Fatal(err)
  216. }
  217. count, err := p.c.Count(testIndexName).Do()
  218. if err != nil {
  219. t.Fatal(err)
  220. }
  221. if count != int64(numDocs) {
  222. t.Fatalf("expected %d documents; got: %d", numDocs, count)
  223. }
  224. }
  225. func TestBulkProcessorFlush(t *testing.T) {
  226. //client := setupTestClientAndCreateIndexAndLog(t, SetTraceLog(log.New(os.Stdout, "", 0)))
  227. client := setupTestClientAndCreateIndex(t)
  228. p, err := client.BulkProcessor().
  229. Name("ManualFlush").
  230. Workers(10).
  231. BulkActions(-1).
  232. BulkSize(-1).
  233. FlushInterval(30 * time.Second). // 30 seconds to flush
  234. Stats(true).
  235. Do()
  236. if err != nil {
  237. t.Fatal(err)
  238. }
  239. const numDocs = 100
  240. for i := 1; i <= numDocs; i++ {
  241. tweet := tweet{User: "olivere", Message: fmt.Sprintf("%d. %s", i, randomString(rand.Intn(64)))}
  242. request := NewBulkIndexRequest().Index(testIndexName).Type("tweet").Id(fmt.Sprintf("%d", i)).Doc(tweet)
  243. p.Add(request)
  244. }
  245. // Should not flush because 30s > 1s
  246. time.Sleep(1 * time.Second)
  247. // No flush yet
  248. stats := p.Stats()
  249. if stats.Flushed != 0 {
  250. t.Errorf("expected no flush; got: %d", p.stats.Flushed)
  251. }
  252. // Manual flush
  253. err = p.Flush()
  254. if err != nil {
  255. t.Fatal(err)
  256. }
  257. time.Sleep(1 * time.Second)
  258. // Now flushed
  259. stats = p.Stats()
  260. if got, want := p.stats.Flushed, int64(1); got != want {
  261. t.Errorf("expected %d flush; got: %d", want, got)
  262. }
  263. // Close should not start another flush
  264. err = p.Close()
  265. if err != nil {
  266. t.Fatal(err)
  267. }
  268. // Still 1 flush
  269. stats = p.Stats()
  270. if got, want := p.stats.Flushed, int64(1); got != want {
  271. t.Errorf("expected %d flush; got: %d", want, got)
  272. }
  273. // Check number of documents that were bulk indexed
  274. _, err = p.c.Flush(testIndexName).Do()
  275. if err != nil {
  276. t.Fatal(err)
  277. }
  278. count, err := p.c.Count(testIndexName).Do()
  279. if err != nil {
  280. t.Fatal(err)
  281. }
  282. if count != int64(numDocs) {
  283. t.Fatalf("expected %d documents; got: %d", numDocs, count)
  284. }
  285. }
  286. // -- Helper --
  287. func testBulkProcessor(t *testing.T, numDocs int, svc *BulkProcessorService) {
  288. var beforeRequests int64
  289. var befores int64
  290. var afters int64
  291. var failures int64
  292. var afterRequests int64
  293. beforeFn := func(executionId int64, requests []BulkableRequest) {
  294. atomic.AddInt64(&beforeRequests, int64(len(requests)))
  295. atomic.AddInt64(&befores, 1)
  296. }
  297. afterFn := func(executionId int64, requests []BulkableRequest, response *BulkResponse, err error) {
  298. atomic.AddInt64(&afters, 1)
  299. if err != nil {
  300. atomic.AddInt64(&failures, 1)
  301. }
  302. atomic.AddInt64(&afterRequests, int64(len(requests)))
  303. }
  304. p, err := svc.Before(beforeFn).After(afterFn).Stats(true).Do()
  305. if err != nil {
  306. t.Fatal(err)
  307. }
  308. for i := 1; i <= numDocs; i++ {
  309. tweet := tweet{User: "olivere", Message: fmt.Sprintf("%07d. %s", i, randomString(1+rand.Intn(63)))}
  310. request := NewBulkIndexRequest().Index(testIndexName).Type("tweet").Id(fmt.Sprintf("%d", i)).Doc(tweet)
  311. p.Add(request)
  312. }
  313. err = p.Close()
  314. if err != nil {
  315. t.Fatal(err)
  316. }
  317. stats := p.Stats()
  318. if stats.Flushed != 0 {
  319. t.Errorf("expected no flush; got: %d", stats.Flushed)
  320. }
  321. if stats.Committed <= 0 {
  322. t.Errorf("expected committed > %d; got: %d", 0, stats.Committed)
  323. }
  324. if got, want := stats.Indexed, int64(numDocs); got != want {
  325. t.Errorf("expected indexed = %d; got: %d", want, got)
  326. }
  327. if got, want := stats.Created, int64(0); got != want {
  328. t.Errorf("expected created = %d; got: %d", want, got)
  329. }
  330. if got, want := stats.Updated, int64(0); got != want {
  331. t.Errorf("expected updated = %d; got: %d", want, got)
  332. }
  333. if got, want := stats.Deleted, int64(0); got != want {
  334. t.Errorf("expected deleted = %d; got: %d", want, got)
  335. }
  336. if got, want := stats.Succeeded, int64(numDocs); got != want {
  337. t.Errorf("expected succeeded = %d; got: %d", want, got)
  338. }
  339. if got, want := stats.Failed, int64(0); got != want {
  340. t.Errorf("expected failed = %d; got: %d", want, got)
  341. }
  342. if got, want := beforeRequests, int64(numDocs); got != want {
  343. t.Errorf("expected %d requests to before callback; got: %d", want, got)
  344. }
  345. if got, want := afterRequests, int64(numDocs); got != want {
  346. t.Errorf("expected %d requests to after callback; got: %d", want, got)
  347. }
  348. if befores == 0 {
  349. t.Error("expected at least 1 call to before callback")
  350. }
  351. if afters == 0 {
  352. t.Error("expected at least 1 call to after callback")
  353. }
  354. if failures != 0 {
  355. t.Errorf("expected 0 calls to failure callback; got: %d", failures)
  356. }
  357. // Check number of documents that were bulk indexed
  358. _, err = p.c.Flush(testIndexName).Do()
  359. if err != nil {
  360. t.Fatal(err)
  361. }
  362. count, err := p.c.Count(testIndexName).Do()
  363. if err != nil {
  364. t.Fatal(err)
  365. }
  366. if count != int64(numDocs) {
  367. t.Fatalf("expected %d documents; got: %d", numDocs, count)
  368. }
  369. }