PageRenderTime 120ms CodeModel.GetById 33ms RepoModel.GetById 0ms app.codeStats 0ms

/analytics.go

https://gitlab.com/jjae2124/tyk
Go | 238 lines | 183 code | 43 blank | 12 comment | 35 complexity | 5b93d6fffd660e1c2a4efe770ce2df70 MD5 | raw file
  1. package main
  2. import (
  3. "github.com/oschwald/maxminddb-golang"
  4. "gopkg.in/vmihailenco/msgpack.v2"
  5. "net"
  6. "regexp"
  7. "time"
  8. )
  9. // AnalyticsRecord encodes the details of a request
  10. type AnalyticsRecord struct {
  11. Method string
  12. Path string
  13. RawPath string
  14. ContentLength int64
  15. UserAgent string
  16. Day int
  17. Month time.Month
  18. Year int
  19. Hour int
  20. ResponseCode int
  21. APIKey string
  22. TimeStamp time.Time
  23. APIVersion string
  24. APIName string
  25. APIID string
  26. OrgID string
  27. OauthID string
  28. RequestTime int64
  29. RawRequest string
  30. RawResponse string
  31. IPAddress string
  32. Geo GeoData
  33. Tags []string
  34. Alias string
  35. ExpireAt time.Time `bson:"expireAt" json:"expireAt"`
  36. }
  37. type GeoData struct {
  38. Country struct {
  39. ISOCode string `maxminddb:"iso_code"`
  40. } `maxminddb:"country"`
  41. City struct {
  42. GeoNameID uint `maxminddb:"geoname_id"`
  43. Names map[string]string `maxminddb:"names"`
  44. } `maxminddb:"city"`
  45. Location struct {
  46. Latitude float64 `maxminddb:"latitude"`
  47. Longitude float64 `maxminddb:"longitude"`
  48. TimeZone string `maxminddb:"time_zone"`
  49. } `maxminddb:"location"`
  50. }
  51. const (
  52. ANALYTICS_KEYNAME string = "tyk-system-analytics"
  53. )
  54. func (a *AnalyticsRecord) GetGeo(ipStr string) {
  55. if !config.AnalyticsConfig.EnableGeoIP {
  56. return
  57. }
  58. // Not great, tightly coupled
  59. if analytics.GeoIPDB == nil {
  60. return
  61. }
  62. ip := net.ParseIP(ipStr)
  63. var record GeoData // Or any appropriate struct
  64. err := analytics.GeoIPDB.Lookup(ip, &record)
  65. if err != nil {
  66. log.Error("GeoIP Failure (not recorded): ", err)
  67. return
  68. }
  69. log.Debug("ISO Code: ", record.Country.ISOCode)
  70. log.Debug("City: ", record.City.Names["en"])
  71. log.Debug("Lat: ", record.Location.Latitude)
  72. log.Debug("Lon: ", record.Location.Longitude)
  73. log.Debug("TZ: ", record.Location.TimeZone)
  74. a.Geo = record
  75. }
  76. type NormaliseURLPatterns struct {
  77. UUIDs *regexp.Regexp
  78. IDs *regexp.Regexp
  79. Custom []*regexp.Regexp
  80. }
  81. func InitNormalisationPatterns() NormaliseURLPatterns {
  82. thesePatterns := NormaliseURLPatterns{}
  83. uuidPat, pat1Err := regexp.Compile("[0-9a-fA-F]{8}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{12}")
  84. if pat1Err != nil {
  85. log.Error("failed to compile custom pattern: ", pat1Err)
  86. }
  87. numPat, pat2Err := regexp.Compile(`\/(\d+)`)
  88. if pat2Err != nil {
  89. log.Error("failed to compile custom pattern: ", pat2Err)
  90. }
  91. custPats := []*regexp.Regexp{}
  92. for _, pattern := range config.AnalyticsConfig.NormaliseUrls.Custom {
  93. thisPat, patErr := regexp.Compile(pattern)
  94. if patErr != nil {
  95. log.Error("failed to compile custom pattern: ", patErr)
  96. } else {
  97. custPats = append(custPats, thisPat)
  98. }
  99. }
  100. thesePatterns.UUIDs = uuidPat
  101. thesePatterns.IDs = numPat
  102. thesePatterns.Custom = custPats
  103. return thesePatterns
  104. }
  105. func (a *AnalyticsRecord) NormalisePath() {
  106. if config.AnalyticsConfig.NormaliseUrls.Enabled {
  107. if config.AnalyticsConfig.NormaliseUrls.NormaliseUUIDs {
  108. a.Path = config.AnalyticsConfig.NormaliseUrls.compiledPatternSet.UUIDs.ReplaceAllString(a.Path, "{uuid}")
  109. }
  110. if config.AnalyticsConfig.NormaliseUrls.NormaliseNumbers {
  111. a.Path = config.AnalyticsConfig.NormaliseUrls.compiledPatternSet.IDs.ReplaceAllString(a.Path, "/{id}")
  112. }
  113. if len(config.AnalyticsConfig.NormaliseUrls.compiledPatternSet.Custom) > 0 {
  114. for _, r := range config.AnalyticsConfig.NormaliseUrls.compiledPatternSet.Custom {
  115. a.Path = r.ReplaceAllString(a.Path, "{var}")
  116. }
  117. }
  118. }
  119. }
  120. func (a *AnalyticsRecord) SetExpiry(expiresInSeconds int64) {
  121. var expiry time.Duration
  122. expiry = time.Duration(expiresInSeconds) * time.Second
  123. if expiresInSeconds == 0 {
  124. // Expiry is set to 100 years
  125. expiry = (24 * time.Hour) * (365 * 100)
  126. }
  127. t := time.Now()
  128. t2 := t.Add(expiry)
  129. a.ExpireAt = t2
  130. }
  131. // AnalyticsError is an error for when writing to the storage engine fails
  132. type AnalyticsError struct{}
  133. func (e AnalyticsError) Error() string {
  134. return "Recording request failed!"
  135. }
  136. // AnalyticsHandler is an interface to record analytics data to a writer.
  137. type AnalyticsHandler interface {
  138. Init() error
  139. RecordHit(AnalyticsRecord) error
  140. }
  141. // RedisAnalyticsHandler implements AnalyticsHandler and will record analytics
  142. // data to a redis back end as defined in the Config object
  143. type RedisAnalyticsHandler struct {
  144. Store *RedisClusterStorageManager
  145. Clean Purger
  146. GeoIPDB *maxminddb.Reader
  147. }
  148. func (r *RedisAnalyticsHandler) Init() {
  149. if config.AnalyticsConfig.EnableGeoIP {
  150. go r.reloadDB()
  151. }
  152. analytics.Store.Connect()
  153. }
  154. func (r *RedisAnalyticsHandler) reloadDB() {
  155. thisDb, err := maxminddb.Open(config.AnalyticsConfig.GeoIPDBLocation)
  156. if err != nil {
  157. log.Error("Failed to init GeoIP Database: ", err)
  158. } else {
  159. oldDB := r.GeoIPDB
  160. r.GeoIPDB = thisDb
  161. if oldDB != nil {
  162. oldDB.Close()
  163. }
  164. }
  165. time.Sleep(time.Hour * 1)
  166. }
  167. // RecordHit will store an AnalyticsRecord in Redis
  168. func (r RedisAnalyticsHandler) RecordHit(thisRecord AnalyticsRecord) error {
  169. // If we are obfuscating API Keys, store the hashed representation (config check handled in hashing function)
  170. thisRecord.APIKey = publicHash(thisRecord.APIKey)
  171. if config.SlaveOptions.UseRPC {
  172. // Extend tag list to include this data so wecan segment by node if necessary
  173. thisRecord.Tags = append(thisRecord.Tags, "tyk-hybrid-rpc")
  174. }
  175. if config.DBAppConfOptions.NodeIsSegmented {
  176. // Extend tag list to include this data so wecan segment by node if necessary
  177. thisRecord.Tags = append(thisRecord.Tags, config.DBAppConfOptions.Tags...)
  178. }
  179. // Lets add some metadata
  180. if thisRecord.APIKey != "" {
  181. thisRecord.Tags = append(thisRecord.Tags, "key-"+thisRecord.APIKey)
  182. }
  183. if thisRecord.OrgID != "" {
  184. thisRecord.Tags = append(thisRecord.Tags, "org-"+thisRecord.OrgID)
  185. }
  186. thisRecord.Tags = append(thisRecord.Tags, "api-"+thisRecord.APIID)
  187. encoded, err := msgpack.Marshal(thisRecord)
  188. if err != nil {
  189. log.Error("Error encoding analytics data:")
  190. log.Error(err)
  191. return AnalyticsError{}
  192. }
  193. r.Store.AppendToSet(ANALYTICS_KEYNAME, string(encoded))
  194. return nil
  195. }