PageRenderTime 61ms CodeModel.GetById 28ms RepoModel.GetById 0ms app.codeStats 0ms

/test/plugin/test_in_http.rb

https://gitlab.com/vectorci/fluentd
Ruby | 442 lines | 339 code | 98 blank | 5 comment | 3 complexity | 789ddd488122ff0d8cbc990a0a2352f0 MD5 | raw file
  1. require_relative '../helper'
  2. require 'fluent/test'
  3. require 'fluent/plugin/in_http'
  4. require 'net/http'
  5. class HttpInputTest < Test::Unit::TestCase
  6. class << self
  7. def startup
  8. socket_manager_path = ServerEngine::SocketManager::Server.generate_path
  9. @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
  10. ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
  11. end
  12. def shutdown
  13. @server.close
  14. end
  15. end
  16. def setup
  17. Fluent::Test.setup
  18. end
  19. PORT = unused_port
  20. CONFIG = %[
  21. port #{PORT}
  22. bind "127.0.0.1"
  23. body_size_limit 10m
  24. keepalive_timeout 5
  25. respond_with_empty_img true
  26. ]
  27. def create_driver(conf=CONFIG)
  28. Fluent::Test::InputTestDriver.new(Fluent::HttpInput).configure(conf, true)
  29. end
  30. def test_configure
  31. d = create_driver
  32. assert_equal PORT, d.instance.port
  33. assert_equal '127.0.0.1', d.instance.bind
  34. assert_equal 10*1024*1024, d.instance.body_size_limit
  35. assert_equal 5, d.instance.keepalive_timeout
  36. assert_equal false, d.instance.add_http_headers
  37. end
  38. def test_time
  39. d = create_driver
  40. time = Fluent::EventTime.new(Time.parse("2011-01-02 13:14:15 UTC").to_i)
  41. Fluent::Engine.now = time
  42. d.expect_emit "tag1", time, {"a"=>1}
  43. d.expect_emit "tag2", time, {"a"=>2}
  44. d.run do
  45. d.expected_emits.each {|tag,_time,record|
  46. res = post("/#{tag}", {"json"=>record.to_json})
  47. assert_equal "200", res.code
  48. }
  49. end
  50. end
  51. def test_time_as_float
  52. d = create_driver
  53. float_time = Time.parse("2011-01-02 13:14:15.123 UTC").to_f
  54. time = Fluent::EventTime.from_time(Time.at(float_time))
  55. d.expect_emit "tag1", time, {"a"=>1}
  56. d.run do
  57. d.expected_emits.each {|tag,_time,record|
  58. res = post("/#{tag}", {"json"=>record.to_json, "time"=>float_time.to_s})
  59. assert_equal "200", res.code
  60. }
  61. end
  62. end
  63. def test_json
  64. d = create_driver
  65. time = Fluent::EventTime.new(Time.parse("2011-01-02 13:14:15 UTC").to_i)
  66. d.expect_emit "tag1", time, {"a"=>1}
  67. d.expect_emit "tag2", time, {"a"=>2}
  68. d.run do
  69. d.expected_emits.each {|tag,_time,record|
  70. res = post("/#{tag}", {"json"=>record.to_json, "time"=>_time.to_s})
  71. assert_equal "200", res.code
  72. }
  73. end
  74. d.emit_streams.each { |tag, es|
  75. assert !include_http_header?(es.first[1])
  76. }
  77. end
  78. def test_multi_json
  79. d = create_driver
  80. time = Fluent::EventTime.new(Time.parse("2011-01-02 13:14:15 UTC").to_i)
  81. events = [{"a"=>1},{"a"=>2}]
  82. tag = "tag1"
  83. events.each { |ev|
  84. d.expect_emit tag, time, ev
  85. }
  86. d.run do
  87. res = post("/#{tag}", {"json"=>events.to_json, "time"=>time.to_s})
  88. assert_equal "200", res.code
  89. end
  90. end
  91. def test_json_with_add_remote_addr
  92. d = create_driver(CONFIG + "add_remote_addr true")
  93. time = Time.parse("2011-01-02 13:14:15 UTC").to_i
  94. d.expect_emit "tag1", time, {"REMOTE_ADDR"=>"127.0.0.1", "a"=>1}
  95. d.expect_emit "tag2", time, {"REMOTE_ADDR"=>"127.0.0.1", "a"=>2}
  96. d.run do
  97. d.expected_emits.each {|tag,_time,record|
  98. res = post("/#{tag}", {"json"=>record.to_json, "time"=>_time.to_s})
  99. assert_equal "200", res.code
  100. }
  101. end
  102. end
  103. def test_multi_json_with_add_remote_addr
  104. d = create_driver(CONFIG + "add_remote_addr true")
  105. time = Time.parse("2011-01-02 13:14:15 UTC").to_i
  106. events = [{"a"=>1},{"a"=>2}]
  107. tag = "tag1"
  108. d.expect_emit "tag1", time, {"REMOTE_ADDR"=>"127.0.0.1", "a"=>1}
  109. d.expect_emit "tag1", time, {"REMOTE_ADDR"=>"127.0.0.1", "a"=>2}
  110. d.run do
  111. res = post("/#{tag}", {"json"=>events.to_json, "time"=>time.to_s})
  112. assert_equal "200", res.code
  113. end
  114. end
  115. def test_json_with_add_remote_addr_given_x_forwarded_for
  116. d = create_driver(CONFIG + "add_remote_addr true")
  117. time = Time.parse("2011-01-02 13:14:15 UTC").to_i
  118. d.expect_emit "tag1", time, {"REMOTE_ADDR"=>"129.78.138.66", "a"=>1}
  119. d.expect_emit "tag2", time, {"REMOTE_ADDR"=>"129.78.138.66", "a"=>1}
  120. d.run do
  121. d.expected_emits.each {|tag,_time,record|
  122. res = post("/#{tag}", {"json"=>record.to_json, "time"=>_time.to_s}, {"X-Forwarded-For"=>"129.78.138.66, 127.0.0.1"})
  123. assert_equal "200", res.code
  124. }
  125. end
  126. end
  127. def test_multi_json_with_add_remote_addr_given_x_forwarded_for
  128. d = create_driver(CONFIG + "add_remote_addr true")
  129. time = Time.parse("2011-01-02 13:14:15 UTC").to_i
  130. events = [{"a"=>1},{"a"=>2}]
  131. tag = "tag1"
  132. d.expect_emit "tag1", time, {"REMOTE_ADDR"=>"129.78.138.66", "a"=>1}
  133. d.expect_emit "tag1", time, {"REMOTE_ADDR"=>"129.78.138.66", "a"=>2}
  134. d.run do
  135. res = post("/#{tag}", {"json"=>events.to_json, "time"=>time.to_s}, {"X-Forwarded-For"=>"129.78.138.66, 127.0.0.1"})
  136. assert_equal "200", res.code
  137. end
  138. end
  139. def test_multi_json_with_add_http_headers
  140. d = create_driver(CONFIG + "add_http_headers true")
  141. time = Time.parse("2011-01-02 13:14:15 UTC").to_i
  142. events = [{"a"=>1},{"a"=>2}]
  143. tag = "tag1"
  144. d.run do
  145. res = post("/#{tag}", {"json"=>events.to_json, "time"=>time.to_s})
  146. assert_equal "200", res.code
  147. end
  148. d.emit_streams.each { |_tag, es|
  149. assert include_http_header?(es.first[1])
  150. }
  151. end
  152. def test_json_with_add_http_headers
  153. d = create_driver(CONFIG + "add_http_headers true")
  154. time = Fluent::EventTime.new(Time.parse("2011-01-02 13:14:15 UTC").to_i)
  155. records = [["tag1", time, {"a"=>1}], ["tag2", time, {"a"=>2}]]
  156. d.run do
  157. records.each {|tag,_time,record|
  158. res = post("/#{tag}", {"json"=>record.to_json, "time"=>_time.to_s})
  159. assert_equal "200", res.code
  160. }
  161. end
  162. d.emit_streams.each { |tag, es|
  163. assert include_http_header?(es.first[1])
  164. }
  165. end
  166. def test_application_json
  167. d = create_driver
  168. time = Fluent::EventTime.new(Time.parse("2011-01-02 13:14:15 UTC").to_i)
  169. d.expect_emit "tag1", time, {"a"=>1}
  170. d.expect_emit "tag2", time, {"a"=>2}
  171. d.run do
  172. d.expected_emits.each {|tag,_time,record|
  173. res = post("/#{tag}?time=#{_time.to_s}", record.to_json, {"Content-Type"=>"application/json; charset=utf-8"})
  174. assert_equal "200", res.code
  175. }
  176. end
  177. end
  178. def test_msgpack
  179. d = create_driver
  180. time = Fluent::EventTime.new(Time.parse("2011-01-02 13:14:15 UTC").to_i)
  181. d.expect_emit "tag1", time, {"a"=>1}
  182. d.expect_emit "tag2", time, {"a"=>2}
  183. d.run do
  184. d.expected_emits.each {|tag,_time,record|
  185. res = post("/#{tag}", {"msgpack"=>record.to_msgpack, "time"=>_time.to_s})
  186. assert_equal "200", res.code
  187. }
  188. end
  189. end
  190. def test_multi_msgpack
  191. d = create_driver
  192. time = Fluent::EventTime.new(Time.parse("2011-01-02 13:14:15 UTC").to_i)
  193. events = [{"a"=>1},{"a"=>2}]
  194. tag = "tag1"
  195. events.each { |ev|
  196. d.expect_emit tag, time, ev
  197. }
  198. d.run do
  199. res = post("/#{tag}", {"msgpack"=>events.to_msgpack, "time"=>time.to_s})
  200. assert_equal "200", res.code
  201. end
  202. end
  203. def test_with_regexp
  204. d = create_driver(CONFIG + %[
  205. format /^(?<field_1>\\d+):(?<field_2>\\w+)$/
  206. types field_1:integer
  207. ])
  208. time = Fluent::EventTime.new(Time.parse("2011-01-02 13:14:15 UTC").to_i)
  209. d.expect_emit "tag1", time, {"field_1" => 1, "field_2" => 'str'}
  210. d.expect_emit "tag2", time, {"field_1" => 2, "field_2" => 'str'}
  211. d.run do
  212. d.expected_emits.each { |tag, _time, record|
  213. body = record.map { |k, v|
  214. v.to_s
  215. }.join(':')
  216. res = post("/#{tag}?time=#{_time.to_s}", body, {'Content-Type' => 'application/octet-stream'})
  217. assert_equal "200", res.code
  218. }
  219. end
  220. end
  221. def test_with_csv
  222. require 'csv'
  223. d = create_driver(CONFIG + %[
  224. format csv
  225. keys foo,bar
  226. ])
  227. time = Fluent::EventTime.new(Time.parse("2011-01-02 13:14:15 UTC").to_i)
  228. d.expect_emit "tag1", time, {"foo" => "1", "bar" => 'st"r'}
  229. d.expect_emit "tag2", time, {"foo" => "2", "bar" => 'str'}
  230. d.run do
  231. d.expected_emits.each { |tag, _time, record|
  232. body = record.map { |k, v| v }.to_csv
  233. res = post("/#{tag}?time=#{_time.to_s}", body, {'Content-Type' => 'text/comma-separated-values'})
  234. assert_equal "200", res.code
  235. }
  236. end
  237. end
  238. def test_resonse_with_empty_img
  239. d = create_driver(CONFIG + "respond_with_empty_img true")
  240. assert_equal true, d.instance.respond_with_empty_img
  241. time = Fluent::EventTime.new(Time.parse("2011-01-02 13:14:15 UTC").to_i)
  242. d.expect_emit "tag1", time, {"a"=>1}
  243. d.expect_emit "tag2", time, {"a"=>2}
  244. d.run do
  245. d.expected_emits.each {|tag,_time,record|
  246. res = post("/#{tag}", {"json"=>record.to_json, "time"=>_time.to_s})
  247. assert_equal "200", res.code
  248. # Ruby returns ASCII-8 encoded string for GIF.
  249. assert_equal Fluent::HttpInput::EMPTY_GIF_IMAGE, res.body.force_encoding("UTF-8")
  250. }
  251. end
  252. end
  253. def test_cors_allowed
  254. d = create_driver(CONFIG + "cors_allow_origins [\"http://foo.com\"]")
  255. assert_equal ["http://foo.com"], d.instance.cors_allow_origins
  256. time = Fluent::EventTime.new(Time.parse("2011-01-02 13:14:15 UTC").to_i)
  257. d.expect_emit "tag1", time, {"a"=>1}
  258. d.expect_emit "tag2", time, {"a"=>1}
  259. d.run do
  260. d.expected_emits.each {|tag,_time,record|
  261. res = post("/#{tag}", {"json"=>record.to_json, "time"=>_time.to_s}, {"Origin"=>"http://foo.com"})
  262. assert_equal "200", res.code
  263. assert_equal "http://foo.com", res["Access-Control-Allow-Origin"]
  264. }
  265. end
  266. end
  267. def test_cors_disallowed
  268. d = create_driver(CONFIG + "cors_allow_origins [\"http://foo.com\"]")
  269. assert_equal ["http://foo.com"], d.instance.cors_allow_origins
  270. time = Fluent::EventTime.new(Time.parse("2011-01-02 13:14:15 UTC").to_i)
  271. d.expected_emits_length = 0
  272. d.run do
  273. res = post("/tag1", {"json"=>{"a"=>1}.to_json, "time"=>time.to_s}, {"Origin"=>"http://bar.com"})
  274. assert_equal "403", res.code
  275. res = post("/tag2", {"json"=>{"a"=>1}.to_json, "time"=>time.to_s}, {"Origin"=>"http://bar.com"})
  276. assert_equal "403", res.code
  277. end
  278. end
  279. $test_in_http_connection_object_ids = []
  280. $test_in_http_content_types = []
  281. $test_in_http_content_types_flag = false
  282. module ContentTypeHook
  283. def initialize(*args)
  284. @io_handler = nil
  285. super
  286. end
  287. def on_headers_complete(headers)
  288. super
  289. if $test_in_http_content_types_flag
  290. $test_in_http_content_types << self.content_type
  291. end
  292. end
  293. def on_message_begin
  294. super
  295. if $test_in_http_content_types_flag
  296. $test_in_http_connection_object_ids << @io_handler.object_id
  297. end
  298. end
  299. end
  300. class Fluent::HttpInput::Handler
  301. prepend ContentTypeHook
  302. end
  303. def test_if_content_type_is_initialized_properly
  304. # This test is to check if Fluent::HttpInput::Handler's @content_type is initialized properly.
  305. # Especially when in Keep-Alive and the second request has no 'Content-Type'.
  306. begin
  307. d = create_driver
  308. $test_in_http_content_types_flag = true
  309. d.run do
  310. # Send two requests the second one has no Content-Type in Keep-Alive
  311. Net::HTTP.start("127.0.0.1", PORT) do |http|
  312. req = Net::HTTP::Post.new("/foodb/bartbl", {"connection" => "keepalive", "Content-Type" => "application/json"})
  313. res = http.request(req)
  314. req = Net::HTTP::Get.new("/foodb/bartbl", {"connection" => "keepalive"})
  315. res = http.request(req)
  316. end
  317. end
  318. ensure
  319. $test_in_http_content_types_flag = false
  320. end
  321. assert_equal(['application/json', ''], $test_in_http_content_types)
  322. # Asserting keepalive
  323. assert_equal $test_in_http_connection_object_ids[0], $test_in_http_connection_object_ids[1]
  324. end
  325. def post(path, params, header = {})
  326. http = Net::HTTP.new("127.0.0.1", PORT)
  327. req = Net::HTTP::Post.new(path, header)
  328. if params.is_a?(String)
  329. unless header.has_key?('Content-Type')
  330. header['Content-Type'] = 'application/octet-stream'
  331. end
  332. req.body = params
  333. else
  334. unless header.has_key?('Content-Type')
  335. header['Content-Type'] = 'application/x-www-form-urlencoded'
  336. end
  337. req.set_form_data(params)
  338. end
  339. http.request(req)
  340. end
  341. def include_http_header?(record)
  342. record.keys.find { |header| header.start_with?('HTTP_') }
  343. end
  344. end