PageRenderTime 571ms CodeModel.GetById 14ms RepoModel.GetById 0ms app.codeStats 0ms

/pipeline/config.go

https://gitlab.com/karouf/heka
Go | 846 lines | 596 code | 89 blank | 161 comment | 116 complexity | 2bec0e12fad8aa6079f668d7bc0c50e3 MD5 | raw file
  1. /***** BEGIN LICENSE BLOCK *****
  2. # This Source Code Form is subject to the terms of the Mozilla Public
  3. # License, v. 2.0. If a copy of the MPL was not distributed with this file,
  4. # You can obtain one at http://mozilla.org/MPL/2.0/.
  5. #
  6. # The Initial Developer of the Original Code is the Mozilla Foundation.
  7. # Portions created by the Initial Developer are Copyright (C) 2012-2015
  8. # the Initial Developer. All Rights Reserved.
  9. #
  10. # Contributor(s):
  11. # Rob Miller (rmiller@mozilla.com)
  12. # Mike Trinkala (trink@mozilla.com)
  13. # Justin Judd (justin@justinjudd.org)
  14. #
  15. # ***** END LICENSE BLOCK *****/
  16. package pipeline
  17. import (
  18. "bufio"
  19. "bytes"
  20. "errors"
  21. "fmt"
  22. "io"
  23. "io/ioutil"
  24. "log"
  25. "os"
  26. "reflect"
  27. "regexp"
  28. "sync"
  29. "time"
  30. "github.com/bbangert/toml"
  31. "github.com/pborman/uuid"
  32. )
  33. const (
  34. HEKA_DAEMON = "hekad"
  35. invalidEnvChars = "\n\r\t "
  36. )
  37. var (
  38. invalidEnvPrefix = []byte("%ENV[")
  39. AvailablePlugins = make(map[string]func() interface{})
  40. ErrMissingCloseDelim = errors.New("Missing closing delimiter")
  41. ErrInvalidChars = errors.New("Invalid characters in environmental variable")
  42. LogInfo = log.New(os.Stdout, "", log.LstdFlags)
  43. LogError = log.New(os.Stderr, "", log.LstdFlags)
  44. )
  45. // Adds a plugin to the set of usable Heka plugins that can be referenced from
  46. // a Heka config file.
  47. func RegisterPlugin(name string, factory func() interface{}) {
  48. AvailablePlugins[name] = factory
  49. }
  50. // Generic plugin configuration type that will be used for plugins that don't
  51. // provide the `HasConfigStruct` interface.
  52. type PluginConfig map[string]toml.Primitive
  53. // API made available to all plugins providing Heka-wide utility functions.
  54. type PluginHelper interface {
  55. // Returns an `OutputRunner` for an output plugin registered using the
  56. // specified name, or ok == false if no output by that name is registered.
  57. Output(name string) (oRunner OutputRunner, ok bool)
  58. // Returns the running `FilterRunner` for a filter plugin registered using
  59. // the specified name, or ok == false if no filter by that name is
  60. // registered.
  61. Filter(name string) (fRunner FilterRunner, ok bool)
  62. // Instantiates and returns an `Encoder` plugin of the specified name, or
  63. // ok == false if no encoder by that name is registered.
  64. Encoder(base_name, full_name string) (encoder Encoder, ok bool)
  65. // Returns the currently running Heka instance's unique PipelineConfig
  66. // object.
  67. PipelineConfig() *PipelineConfig
  68. // Instantiates, starts, and returns a DecoderRunner wrapped around a newly
  69. // created Decoder of the specified name.
  70. DecoderRunner(base_name, full_name string) (dRunner DecoderRunner, ok bool)
  71. // Stops and unregisters the provided DecoderRunner.
  72. StopDecoderRunner(dRunner DecoderRunner) (ok bool)
  73. // Expects a loop count value from an existing message (or zero if there's
  74. // no relevant existing message), returns an initialized `PipelinePack`
  75. // pointer that can be populated w/ message data and inserted into the
  76. // Heka pipeline. Returns `nil` if the loop count value provided is
  77. // greater than the maximum allowed by the Heka instance.
  78. PipelinePack(msgLoopCount uint) (*PipelinePack, error)
  79. // Returns an input plugin of the given name that provides the
  80. // StatAccumulator interface, or an error value if such a plugin
  81. // can't be found.
  82. StatAccumulator(name string) (statAccum StatAccumulator, err error)
  83. // Returns the configured Hostname for the Heka process. This can come
  84. // either from the runtime or from the Heka config.
  85. Hostname() string
  86. }
  87. // Indicates a plug-in has a specific-to-itself config struct that should be
  88. // passed in to its Init method.
  89. type HasConfigStruct interface {
  90. // Returns a default-value-populated configuration structure into which
  91. // the plugin's TOML configuration will be deserialized.
  92. ConfigStruct() interface{}
  93. }
  94. // Master config object encapsulating the entire heka/pipeline configuration.
  95. type PipelineConfig struct {
  96. // Heka global values.
  97. Globals *GlobalConfigStruct
  98. // PluginMakers for every registered plugin, by category.
  99. makers map[string]map[string]PluginMaker
  100. // Direct access to makers["Decoder"] since it's needed by MultiDecoder
  101. // outside of the pipeline package.
  102. DecoderMakers map[string]PluginMaker
  103. // Mutex protecting the makers map.
  104. makersLock sync.RWMutex
  105. // All running InputRunners, by name.
  106. InputRunners map[string]InputRunner
  107. // All running FilterRunners, by name.
  108. FilterRunners map[string]FilterRunner
  109. // All running OutputRunners, by name.
  110. OutputRunners map[string]OutputRunner
  111. // Heka message router instance.
  112. router *messageRouter
  113. // PipelinePack supply for Input plugins.
  114. inputRecycleChan chan *PipelinePack
  115. // PipelinePack supply for Filter plugins (separate pool prevents
  116. // deadlocks).
  117. injectRecycleChan chan *PipelinePack
  118. // Stores log messages generated by plugin config errors.
  119. LogMsgs []string
  120. // Lock protecting access to the set of running filters so dynamic filters
  121. // can be safely added and removed while Heka is running.
  122. filtersLock sync.RWMutex
  123. // Is freed when all FilterRunners have stopped.
  124. filtersWg sync.WaitGroup
  125. // Is freed when all DecoderRunners have stopped.
  126. decodersWg sync.WaitGroup
  127. // Slice providing access to all running DecoderRunners.
  128. allDecoders []DecoderRunner
  129. // Mutex protecting allDecoders.
  130. allDecodersLock sync.RWMutex
  131. // Slice providing access to all Decoders called synchronously by InputRunner
  132. allSyncDecoders []ReportingDecoder
  133. // Mutex protecting allSyncDecoders
  134. allSyncDecodersLock sync.RWMutex
  135. // Slice providing access to all Splitters
  136. allSplitters []SplitterRunner
  137. // Mutex protecting AllSplitters
  138. allSplittersLock sync.RWMutex
  139. // Slice providing access to all instantiated Encoders.
  140. allEncoders map[string]Encoder
  141. // Mutex protecting allEncoders.
  142. allEncodersLock sync.RWMutex
  143. // Name of host on which Heka is running.
  144. hostname string
  145. // Heka process id.
  146. pid int32
  147. // Lock protecting access to the set of running inputs so they
  148. // can be safely added while Heka is running.
  149. inputsLock sync.RWMutex
  150. // Is freed when all Input runners have stopped.
  151. inputsWg sync.WaitGroup
  152. // Lock protecting access to running outputs so they can be removed
  153. // safely.
  154. outputsLock sync.RWMutex
  155. // Internal reporting channel.
  156. reportRecycleChan chan *PipelinePack
  157. // The next few values are used only during the initial configuration
  158. // loading process.
  159. // Track default plugin registration.
  160. defaultConfigs map[string]bool
  161. // Loaded PluginMakers sorted by category.
  162. makersByCategory map[string][]PluginMaker
  163. // Number of config loading errors.
  164. errcnt uint
  165. }
  166. // Creates and initializes a PipelineConfig object. `nil` value for `globals`
  167. // argument means we should use the default global config values.
  168. func NewPipelineConfig(globals *GlobalConfigStruct) (config *PipelineConfig) {
  169. config = new(PipelineConfig)
  170. if globals == nil {
  171. globals = DefaultGlobals()
  172. }
  173. config.Globals = globals
  174. config.makers = make(map[string]map[string]PluginMaker)
  175. config.makers["Input"] = make(map[string]PluginMaker)
  176. config.makers["Decoder"] = make(map[string]PluginMaker)
  177. config.makers["Filter"] = make(map[string]PluginMaker)
  178. config.makers["Encoder"] = make(map[string]PluginMaker)
  179. config.makers["Output"] = make(map[string]PluginMaker)
  180. config.makers["Splitter"] = make(map[string]PluginMaker)
  181. config.DecoderMakers = config.makers["Decoder"]
  182. config.InputRunners = make(map[string]InputRunner)
  183. config.FilterRunners = make(map[string]FilterRunner)
  184. config.OutputRunners = make(map[string]OutputRunner)
  185. config.allEncoders = make(map[string]Encoder)
  186. config.router = NewMessageRouter(globals.PluginChanSize, globals.abortChan)
  187. config.inputRecycleChan = make(chan *PipelinePack, globals.PoolSize)
  188. config.injectRecycleChan = make(chan *PipelinePack, globals.PoolSize)
  189. config.LogMsgs = make([]string, 0, 4)
  190. config.allDecoders = make([]DecoderRunner, 0, 10)
  191. config.allSyncDecoders = make([]ReportingDecoder, 0, 10)
  192. config.allSplitters = make([]SplitterRunner, 0, 10)
  193. config.hostname = globals.Hostname
  194. config.pid = int32(os.Getpid())
  195. config.reportRecycleChan = make(chan *PipelinePack, 1)
  196. return config
  197. }
  198. // Callers should pass in the msgLoopCount value from any relevant Message
  199. // objects they are holding. Returns a PipelinePack for injection into Heka
  200. // pipeline, or nil if the msgLoopCount is above the configured maximum.
  201. func (self *PipelineConfig) PipelinePack(msgLoopCount uint) (*PipelinePack, error) {
  202. if msgLoopCount++; msgLoopCount > self.Globals.MaxMsgLoops {
  203. return nil, fmt.Errorf("exceeded MaxMsgLoops = %d", self.Globals.MaxMsgLoops)
  204. }
  205. var pack *PipelinePack
  206. select {
  207. case pack = <-self.injectRecycleChan:
  208. case <-self.Globals.abortChan:
  209. return nil, AbortError
  210. }
  211. pack.Message.SetTimestamp(time.Now().UnixNano())
  212. pack.Message.SetUuid(uuid.NewRandom())
  213. pack.Message.SetHostname(self.hostname)
  214. pack.Message.SetPid(self.pid)
  215. pack.RefCount = 1
  216. pack.MsgLoopCount = msgLoopCount
  217. return pack, nil
  218. }
  219. // Returns the router.
  220. func (self *PipelineConfig) Router() MessageRouter {
  221. return self.router
  222. }
  223. // Returns the inputRecycleChannel.
  224. func (self *PipelineConfig) InputRecycleChan() chan *PipelinePack {
  225. return self.inputRecycleChan
  226. }
  227. // Returns the injectRecycleChannel.
  228. func (self *PipelineConfig) InjectRecycleChan() chan *PipelinePack {
  229. return self.injectRecycleChan
  230. }
  231. // Returns the hostname.
  232. func (self *PipelineConfig) Hostname() string {
  233. return self.hostname
  234. }
  235. // Returns OutputRunner registered under the specified name, or nil (and ok ==
  236. // false) if no such name is registered.
  237. func (self *PipelineConfig) Output(name string) (oRunner OutputRunner, ok bool) {
  238. oRunner, ok = self.OutputRunners[name]
  239. return
  240. }
  241. // Returns the underlying config object via the Helper interface.
  242. func (self *PipelineConfig) PipelineConfig() *PipelineConfig {
  243. return self
  244. }
  245. // Instantiates and returns a Decoder of the specified name. Note that any
  246. // time this method is used to fetch an unwrapped Decoder instance, it is up
  247. // to the caller to check for and possibly satisfy the WantsDecoderRunner and
  248. // WantsDecoderRunnerShutdown interfaces.
  249. func (self *PipelineConfig) Decoder(name string) (decoder Decoder, ok bool) {
  250. var maker PluginMaker
  251. self.makersLock.RLock()
  252. defer self.makersLock.RUnlock()
  253. if maker, ok = self.DecoderMakers[name]; !ok {
  254. return
  255. }
  256. plugin, _, err := maker.Make()
  257. if err != nil {
  258. return nil, false
  259. }
  260. decoder = plugin.(Decoder)
  261. return
  262. }
  263. // Instantiates, starts, and returns a DecoderRunner wrapped around a newly
  264. // created Decoder of the specified name.
  265. func (self *PipelineConfig) DecoderRunner(baseName, fullName string) (
  266. dRunner DecoderRunner, ok bool) {
  267. self.makersLock.RLock()
  268. var maker PluginMaker
  269. if maker, ok = self.DecoderMakers[baseName]; !ok {
  270. self.makersLock.RUnlock()
  271. return
  272. }
  273. runner, err := maker.MakeRunner(fullName)
  274. self.makersLock.RUnlock()
  275. if err != nil {
  276. return nil, false
  277. }
  278. dRunner = runner.(DecoderRunner)
  279. self.allDecodersLock.Lock()
  280. self.allDecoders = append(self.allDecoders, dRunner)
  281. self.allDecodersLock.Unlock()
  282. self.decodersWg.Add(1)
  283. dRunner.Start(self, &self.decodersWg)
  284. return
  285. }
  286. // Stops and unregisters the provided DecoderRunner.
  287. func (self *PipelineConfig) StopDecoderRunner(dRunner DecoderRunner) (ok bool) {
  288. self.allDecodersLock.Lock()
  289. defer self.allDecodersLock.Unlock()
  290. for i, r := range self.allDecoders {
  291. if r == dRunner {
  292. close(dRunner.InChan())
  293. self.allDecoders = append(self.allDecoders[:i], self.allDecoders[i+1:]...)
  294. ok = true
  295. break
  296. }
  297. }
  298. return
  299. }
  300. // Instantiates and returns an Encoder of the specified name.
  301. func (self *PipelineConfig) Encoder(baseName, fullName string) (Encoder, bool) {
  302. self.makersLock.RLock()
  303. maker, ok := self.makers["Encoder"][baseName]
  304. if !ok {
  305. self.makersLock.RUnlock()
  306. return nil, false
  307. }
  308. plugin, _, err := maker.Make()
  309. self.makersLock.RUnlock()
  310. if err != nil {
  311. msg := fmt.Sprintf("Error creating encoder '%s': %s", fullName, err.Error())
  312. self.log(msg)
  313. return nil, false
  314. }
  315. encoder := plugin.(Encoder)
  316. if wantsName, ok := encoder.(WantsName); ok {
  317. wantsName.SetName(fullName)
  318. }
  319. self.allEncodersLock.Lock()
  320. self.allEncoders[fullName] = encoder
  321. self.allEncodersLock.Unlock()
  322. return encoder, true
  323. }
  324. // Returns a FilterRunner with the given name, or nil and ok == false if no
  325. // such name is registered.
  326. func (self *PipelineConfig) Filter(name string) (fRunner FilterRunner, ok bool) {
  327. self.filtersLock.RLock()
  328. defer self.filtersLock.RUnlock()
  329. fRunner, ok = self.FilterRunners[name]
  330. return
  331. }
  332. // Returns the specified StatAccumulator input plugin, or an error if it can't
  333. // be found.
  334. func (self *PipelineConfig) StatAccumulator(name string) (statAccum StatAccumulator,
  335. err error) {
  336. self.inputsLock.RLock()
  337. defer self.inputsLock.RUnlock()
  338. iRunner, ok := self.InputRunners[name]
  339. if !ok {
  340. err = fmt.Errorf("No Input named '%s", name)
  341. return
  342. }
  343. input := iRunner.Input()
  344. if statAccum, ok = input.(StatAccumulator); !ok {
  345. err = fmt.Errorf("Input '%s' is not a StatAccumulator", name)
  346. }
  347. return
  348. }
  349. // Starts the provided FilterRunner and adds it to the set of running Filters.
  350. func (self *PipelineConfig) AddFilterRunner(fRunner FilterRunner) error {
  351. self.filtersLock.Lock()
  352. defer self.filtersLock.Unlock()
  353. self.FilterRunners[fRunner.Name()] = fRunner
  354. self.filtersWg.Add(1)
  355. if err := fRunner.Start(self, &self.filtersWg); err != nil {
  356. self.filtersWg.Done()
  357. return fmt.Errorf("AddFilterRunner '%s' failed to start: %s",
  358. fRunner.Name(), err)
  359. } else {
  360. self.router.AddFilterMatcher() <- fRunner.MatchRunner()
  361. }
  362. return nil
  363. }
  364. // Removes the specified FilterRunner from the configuration and the
  365. // MessageRouter which signals the filter to shutdown by closing the input
  366. // channel. Returns true if the filter was removed.
  367. func (self *PipelineConfig) RemoveFilterRunner(name string) bool {
  368. if self.Globals.IsShuttingDown() {
  369. return false
  370. }
  371. self.filtersLock.Lock()
  372. defer self.filtersLock.Unlock()
  373. if fRunner, ok := self.FilterRunners[name]; ok {
  374. self.router.RemoveFilterMatcher() <- fRunner.MatchRunner()
  375. delete(self.FilterRunners, name)
  376. return true
  377. }
  378. return false
  379. }
  380. // AddInputRunner Starts the provided InputRunner and adds it to the set of
  381. // running Inputs.
  382. func (self *PipelineConfig) AddInputRunner(iRunner InputRunner) error {
  383. self.inputsLock.Lock()
  384. defer self.inputsLock.Unlock()
  385. self.InputRunners[iRunner.Name()] = iRunner
  386. self.inputsWg.Add(1)
  387. if err := iRunner.Start(self, &self.inputsWg); err != nil {
  388. self.inputsWg.Done()
  389. return fmt.Errorf("AddInputRunner '%s' failed to start: %s", iRunner.Name(), err)
  390. }
  391. return nil
  392. }
  393. // RemoveInputRunner unregisters the provided InputRunner, and stops it.
  394. func (self *PipelineConfig) RemoveInputRunner(iRunner InputRunner) {
  395. name := iRunner.Name()
  396. self.makersLock.Lock()
  397. inputMakers := self.makers["Input"]
  398. if _, ok := inputMakers[name]; ok {
  399. delete(inputMakers, name)
  400. }
  401. self.makersLock.Unlock()
  402. self.inputsLock.Lock()
  403. delete(self.InputRunners, name)
  404. self.inputsLock.Unlock()
  405. iRunner.Input().Stop()
  406. }
  407. // RemoveOutputRunner unregisters the provided OutputRunner from heka, and
  408. // removes it's message matcher from the heka router.
  409. func (self *PipelineConfig) RemoveOutputRunner(oRunner OutputRunner) {
  410. name := oRunner.Name()
  411. self.makersLock.Lock()
  412. outputMakers := self.makers["Output"]
  413. if _, ok := outputMakers[name]; ok {
  414. self.router.RemoveOutputMatcher() <- oRunner.MatchRunner()
  415. delete(outputMakers, name)
  416. }
  417. self.makersLock.Unlock()
  418. self.outputsLock.Lock()
  419. delete(self.OutputRunners, name)
  420. self.outputsLock.Unlock()
  421. }
  422. type ConfigFile PluginConfig
  423. var unknownOptionRegex = regexp.MustCompile("^Configuration contains key \\[(?P<key>\\S+)\\]")
  424. // getAttr uses reflection to extract an attribute value from an arbitrary
  425. // struct type that may or may not actually have the attribute, returning a
  426. // provided default if the provided object is not a struct or if the attribute
  427. // doesn't exist.
  428. func getAttr(ob interface{}, attr string, default_ interface{}) (ret interface{}) {
  429. ret = default_
  430. obVal := reflect.ValueOf(ob)
  431. obVal = reflect.Indirect(obVal) // Dereference if it's a pointer.
  432. if obVal.Kind().String() != "struct" {
  433. // `FieldByName` will panic if we're not a struct.
  434. return ret
  435. }
  436. attrVal := obVal.FieldByName(attr)
  437. if !attrVal.IsValid() {
  438. return ret
  439. }
  440. return attrVal.Interface()
  441. }
  442. // getDefaultBool expects the name of a boolean setting and will extract and
  443. // return the struct's default value for the setting, as a boolean pointer.
  444. func getDefaultBool(ob interface{}, name string) (*bool, error) {
  445. defaultValue := getAttr(ob, name, false)
  446. switch defaultValue := defaultValue.(type) {
  447. case bool:
  448. return &defaultValue, nil
  449. case *bool:
  450. if defaultValue == nil {
  451. b := false
  452. defaultValue = &b
  453. }
  454. return defaultValue, nil
  455. }
  456. // If you hit this then a non-boolean config setting is conflicting
  457. // with one of Heka's defined boolean settings.
  458. return nil, fmt.Errorf("%s config setting must be boolean", name)
  459. }
  460. // Used internally to log and record plugin config loading errors.
  461. func (self *PipelineConfig) log(msg string) {
  462. self.LogMsgs = append(self.LogMsgs, msg)
  463. LogError.Println(msg)
  464. }
  465. var PluginTypeRegex = regexp.MustCompile("(Decoder|Encoder|Filter|Input|Output|Splitter)$")
  466. func getPluginCategory(pluginType string) string {
  467. pluginCats := PluginTypeRegex.FindStringSubmatch(pluginType)
  468. if len(pluginCats) < 2 {
  469. return ""
  470. }
  471. return pluginCats[1]
  472. }
  473. type CommonConfig struct {
  474. Typ string `toml:"type"`
  475. }
  476. type CommonInputConfig struct {
  477. Ticker uint `toml:"ticker_interval"`
  478. Decoder string
  479. Splitter string
  480. SyncDecode *bool `toml:"synchronous_decode"`
  481. SendDecodeFailures *bool `toml:"send_decode_failures"`
  482. LogDecodeFailures *bool `toml:"log_decode_failures"`
  483. CanExit *bool `toml:"can_exit"`
  484. Retries RetryOptions
  485. }
  486. type CommonFOConfig struct {
  487. Ticker uint `toml:"ticker_interval"`
  488. Matcher string `toml:"message_matcher"`
  489. Signer string `toml:"message_signer"`
  490. CanExit *bool `toml:"can_exit"`
  491. Retries RetryOptions
  492. Encoder string // Output only.
  493. UseFraming *bool `toml:"use_framing"` // Output only.
  494. UseBuffering *bool `toml:"use_buffering"`
  495. Buffering *QueueBufferConfig `toml:"buffering"`
  496. }
  497. type CommonSplitterConfig struct {
  498. KeepTruncated *bool `toml:"keep_truncated"`
  499. UseMsgBytes *bool `toml:"use_message_bytes"`
  500. BufferSize uint `toml:"min_buffer_size"`
  501. IncompleteFinal *bool `toml:"deliver_incomplete_final"`
  502. }
  503. // Default configurations.
  504. func makeDefaultConfigs() map[string]bool {
  505. return map[string]bool{
  506. "ProtobufDecoder": false,
  507. "ProtobufEncoder": false,
  508. "TokenSplitter": false,
  509. "PatternGroupingSplitter": false,
  510. "HekaFramingSplitter": false,
  511. "NullSplitter": false,
  512. }
  513. }
  514. func (self *PipelineConfig) RegisterDefault(name string) error {
  515. var config ConfigFile
  516. confStr := fmt.Sprintf("[%s]", name)
  517. toml.Decode(confStr, &config)
  518. LogInfo.Printf("Pre-loading: %s\n", confStr)
  519. maker, err := NewPluginMaker(name, self, config[name])
  520. if err != nil {
  521. // This really shouldn't happen.
  522. return err
  523. }
  524. LogInfo.Printf("Loading: [%s]\n", maker.Name())
  525. if _, err = maker.PrepConfig(); err != nil {
  526. return err
  527. }
  528. category := maker.Category()
  529. self.makersLock.Lock()
  530. self.makers[category][name] = maker
  531. self.makersLock.Unlock()
  532. // If we ever add a default input, filter, or output we'd need to call
  533. // maker.MakeRunner() here and store the runner on the PipelineConfig.
  534. return nil
  535. }
  536. // PreloadFromConfigFile loads all plugin configuration from a TOML
  537. // configuration file, generates a PluginMaker for each loaded section, and
  538. // stores the created PluginMakers in the makersByCategory map. The
  539. // PipelineConfig should be already initialized via the Init function before
  540. // this method is called. PreloadFromConfigFile is not reentrant, so it should
  541. // only be called serially, not from multiple concurrent goroutines.
  542. func (self *PipelineConfig) PreloadFromConfigFile(filename string) error {
  543. var (
  544. configFile ConfigFile
  545. err error
  546. )
  547. contents, err := ReplaceEnvsFile(filename)
  548. if err != nil {
  549. return err
  550. }
  551. if _, err = toml.Decode(contents, &configFile); err != nil {
  552. return fmt.Errorf("Error decoding config file: %s", err)
  553. }
  554. if self.makersByCategory == nil {
  555. self.makersByCategory = make(map[string][]PluginMaker)
  556. }
  557. if self.defaultConfigs == nil {
  558. self.defaultConfigs = makeDefaultConfigs()
  559. }
  560. // Load all the plugin makers and file them by category.
  561. for name, conf := range configFile {
  562. if name == HEKA_DAEMON {
  563. continue
  564. }
  565. if _, ok := self.defaultConfigs[name]; ok {
  566. self.defaultConfigs[name] = true
  567. }
  568. LogInfo.Printf("Pre-loading: [%s]\n", name)
  569. maker, err := NewPluginMaker(name, self, conf)
  570. if err != nil {
  571. self.log(err.Error())
  572. self.errcnt++
  573. continue
  574. }
  575. if maker.Type() == "MultiDecoder" {
  576. // Special case MultiDecoders so we can make sure they get
  577. // registered *after* all possible subdecoders.
  578. self.makersByCategory["MultiDecoder"] = append(
  579. self.makersByCategory["MultiDecoder"], maker)
  580. } else {
  581. category := maker.Category()
  582. self.makersByCategory[category] = append(
  583. self.makersByCategory[category], maker)
  584. }
  585. }
  586. return nil
  587. }
  588. // LoadConfig any not yet preloaded default plugins, then it finishes loading
  589. // and initializing all of the plugin config that has been prepped from calls
  590. // to PreloadFromConfigFile. This method should be called only once, after
  591. // PreloadFromConfigFile has been called as many times as needed.
  592. func (self *PipelineConfig) LoadConfig() error {
  593. // Make sure our default plugins are registered.
  594. for name, registered := range self.defaultConfigs {
  595. if registered {
  596. continue
  597. }
  598. if err := self.RegisterDefault(name); err != nil {
  599. self.log(err.Error())
  600. self.errcnt++
  601. }
  602. }
  603. makersByCategory := self.makersByCategory
  604. if len(makersByCategory) == 0 {
  605. return errors.New("Empty configuration, exiting.")
  606. }
  607. var err error
  608. multiDecoders := make([]multiDecoderNode, len(makersByCategory["MultiDecoder"]))
  609. multiMakers := make(map[string]PluginMaker)
  610. for i, maker := range makersByCategory["MultiDecoder"] {
  611. multiMakers[maker.Name()] = maker
  612. tomlSection := maker.(*pluginMaker).tomlSection
  613. multiDecoders[i] = newMultiDecoderNode(maker.Name(), subsFromSection(tomlSection))
  614. }
  615. multiDecoders, err = orderDependencies(multiDecoders)
  616. if err != nil {
  617. return err
  618. }
  619. for i, d := range multiDecoders {
  620. makersByCategory["MultiDecoder"][i] = multiMakers[d.name]
  621. }
  622. // Append MultiDecoders to the end of the Decoders list.
  623. makersByCategory["Decoder"] = append(makersByCategory["Decoder"],
  624. makersByCategory["MultiDecoder"]...)
  625. // Force decoders and encoders to be loaded before the other plugin
  626. // types are initialized so we know they'll be there for inputs and
  627. // outputs to use during initialization.
  628. order := []string{"Decoder", "Encoder", "Splitter", "Input", "Filter", "Output"}
  629. for _, category := range order {
  630. for _, maker := range makersByCategory[category] {
  631. LogInfo.Printf("Loading: [%s]\n", maker.Name())
  632. if _, err = maker.PrepConfig(); err != nil {
  633. self.log(err.Error())
  634. self.errcnt++
  635. }
  636. self.makers[category][maker.Name()] = maker
  637. if category == "Encoder" {
  638. continue
  639. }
  640. runner, err := maker.MakeRunner("")
  641. if err != nil {
  642. // Might be a duplicate error.
  643. seen := false
  644. for _, prevErr := range self.LogMsgs {
  645. if err.Error() == prevErr {
  646. seen = true
  647. break
  648. }
  649. }
  650. if !seen {
  651. msg := fmt.Sprintf("Error making runner for %s: %s", maker.Name(),
  652. err.Error())
  653. self.log(msg)
  654. self.errcnt++
  655. }
  656. continue
  657. }
  658. switch category {
  659. case "Input":
  660. self.InputRunners[maker.Name()] = runner.(InputRunner)
  661. case "Filter":
  662. self.FilterRunners[maker.Name()] = runner.(FilterRunner)
  663. case "Output":
  664. self.OutputRunners[maker.Name()] = runner.(OutputRunner)
  665. }
  666. }
  667. }
  668. if self.errcnt != 0 {
  669. return fmt.Errorf("%d errors loading plugins", self.errcnt)
  670. }
  671. return nil
  672. }
  673. func subsFromSection(section toml.Primitive) []string {
  674. secMap := section.(map[string]interface{})
  675. var subs []string
  676. if _, ok := secMap["subs"]; ok {
  677. subsUntyped, _ := secMap["subs"].([]interface{})
  678. subs = make([]string, len(subsUntyped))
  679. for i, subUntyped := range subsUntyped {
  680. subs[i], _ = subUntyped.(string)
  681. }
  682. }
  683. return subs
  684. }
  685. func ReplaceEnvsFile(path string) (string, error) {
  686. file, err := os.Open(path)
  687. if err != nil {
  688. return "", err
  689. }
  690. r, err := EnvSub(file)
  691. if err != nil {
  692. return "", err
  693. }
  694. contents, err := ioutil.ReadAll(r)
  695. if err != nil {
  696. return "", err
  697. }
  698. return string(contents), nil
  699. }
  700. func EnvSub(r io.Reader) (io.Reader, error) {
  701. bufIn := bufio.NewReader(r)
  702. bufOut := new(bytes.Buffer)
  703. for {
  704. chunk, err := bufIn.ReadBytes(byte('%'))
  705. if err != nil {
  706. if err == io.EOF {
  707. // We're done.
  708. bufOut.Write(chunk)
  709. break
  710. }
  711. return nil, err
  712. }
  713. bufOut.Write(chunk[:len(chunk)-1])
  714. tmp := make([]byte, 4)
  715. tmp, err = bufIn.Peek(4)
  716. if err != nil {
  717. if err == io.EOF {
  718. // End of file, write the last few bytes out and exit.
  719. bufOut.WriteRune('%')
  720. bufOut.Write(tmp)
  721. break
  722. }
  723. return nil, err
  724. }
  725. if string(tmp) == "ENV[" {
  726. // Found opening delimiter, advance the read cursor and look for
  727. // closing delimiter.
  728. tmp, err = bufIn.ReadBytes(byte('['))
  729. if err != nil {
  730. // This shouldn't happen, since the Peek succeeded.
  731. return nil, err
  732. }
  733. chunk, err = bufIn.ReadBytes(byte(']'))
  734. if err != nil {
  735. if err == io.EOF {
  736. // No closing delimiter, return an error
  737. return nil, ErrMissingCloseDelim
  738. }
  739. return nil, err
  740. }
  741. // `chunk` is now holding var name + closing delimiter.
  742. // var name contains invalid characters, return an error
  743. if bytes.IndexAny(chunk, invalidEnvChars) != -1 ||
  744. bytes.Index(chunk, invalidEnvPrefix) != -1 {
  745. return nil, ErrInvalidChars
  746. }
  747. varName := string(chunk[:len(chunk)-1])
  748. varVal := os.Getenv(varName)
  749. bufOut.WriteString(varVal)
  750. } else {
  751. // Just a random '%', not an opening delimiter, write it out and
  752. // keep going.
  753. bufOut.WriteRune('%')
  754. }
  755. }
  756. return bufOut, nil
  757. }