PageRenderTime 54ms CodeModel.GetById 21ms RepoModel.GetById 0ms app.codeStats 0ms

/services/search/search.go

https://gitlab.com/tamasd/ab
Go | 268 lines | 201 code | 54 blank | 13 comment | 36 complexity | 4db679af166999ad337fefdb5ec5f7f3 MD5 | raw file
  1. // Copyright 2015 Tamás Demeter-Haludka
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package search
  15. import (
  16. "errors"
  17. "fmt"
  18. "net/http"
  19. "regexp"
  20. "sort"
  21. "strings"
  22. "gitlab.com/tamasd/ab"
  23. "gitlab.com/tamasd/ab/util"
  24. )
  25. var ErrorDelegateNotFound = errors.New("delegate not found")
  26. type SearchResult struct {
  27. ab.Entity `json:"entity"`
  28. Type string `json:"type"`
  29. }
  30. type IndexData struct {
  31. Keyword string
  32. Relevance float64
  33. Owner string
  34. }
  35. type rawIndexData struct {
  36. UUID string
  37. Type string
  38. }
  39. type SearchServiceCacheDelegate interface {
  40. Set(search string, results []SearchResult)
  41. Get(search string) []SearchResult
  42. }
  43. type SearchServiceDelegate interface {
  44. IndexEntity(ab.Entity) []IndexData
  45. LoadEntities([]string) []ab.Entity
  46. }
  47. var _ ab.Service = &SearchService{}
  48. type SearchService struct {
  49. db ab.DB
  50. delegates map[string]SearchServiceDelegate
  51. cache SearchServiceCacheDelegate
  52. }
  53. func NewSearchService(db ab.DB, cache SearchServiceCacheDelegate) *SearchService {
  54. return &SearchService{
  55. db: db,
  56. cache: cache,
  57. delegates: make(map[string]SearchServiceDelegate),
  58. }
  59. }
  60. var searchSplitRegex = regexp.MustCompile(`[^\.\w]+`)
  61. func (s *SearchService) normalizeKeywords(keywords []string) string {
  62. sort.Strings(keywords)
  63. return strings.Join(keywords, " ")
  64. }
  65. func (s *SearchService) cacheLookup(keywords []string) []SearchResult {
  66. if len(keywords) == 0 || s.cache == nil {
  67. return []SearchResult{}
  68. }
  69. key := s.normalizeKeywords(keywords)
  70. return s.cache.Get(key)
  71. }
  72. func (s *SearchService) cacheSave(keywords []string, results []SearchResult) {
  73. if len(keywords) == 0 || len(results) == 0 || s.cache == nil {
  74. return
  75. }
  76. key := s.normalizeKeywords(keywords)
  77. s.cache.Set(key, results)
  78. }
  79. func (s *SearchService) Search(search string, owners []string) ([]SearchResult, error) {
  80. search = strings.TrimSpace(search)
  81. search = strings.ToLower(search)
  82. if len(search) == 0 {
  83. return []SearchResult{}, nil
  84. }
  85. keywords := searchSplitRegex.Split(search, -1)
  86. if len(keywords) == 0 {
  87. return []SearchResult{}, nil
  88. }
  89. if res := s.cacheLookup(keywords); len(res) > 0 {
  90. return res, nil
  91. }
  92. placeholders := util.GeneratePlaceholders(1, uint(len(keywords))+1)
  93. ownerCheck := ""
  94. if len(owners) > 0 {
  95. ownerPlaceholders := util.GeneratePlaceholders(uint(len(keywords))+1, uint(len(keywords)+len(owners))+1)
  96. ownerCheck = `AND owner IN (` + ownerPlaceholders + `)`
  97. }
  98. rows, err := s.db.Query(`
  99. WITH
  100. uuids AS (SELECT uuid, SUM(relevance) rel FROM search_metadata WHERE keyword IN (`+placeholders+`) GROUP BY uuid),
  101. types AS (SELECT DISTINCT uuid, type, owner FROM search_metadata)
  102. SELECT t.uuid, t.type FROM uuids u NATURAL JOIN types t WHERE u.rel > 0 `+ownerCheck+` ORDER BY u.rel DESC
  103. `, append(util.StringSliceToInterfaceSlice(keywords), util.StringSliceToInterfaceSlice(owners)...)...)
  104. if err != nil {
  105. return []SearchResult{}, err
  106. }
  107. uuids := make(map[string][]string)
  108. matches := []rawIndexData{}
  109. defer rows.Close()
  110. for rows.Next() {
  111. d := rawIndexData{}
  112. err = rows.Scan(&d.UUID, &d.Type)
  113. if err != nil {
  114. return []SearchResult{}, err
  115. }
  116. matches = append(matches, d)
  117. uuids[d.Type] = append(uuids[d.Type], d.UUID)
  118. }
  119. if err := rows.Err(); err != nil {
  120. return []SearchResult{}, err
  121. }
  122. entities := map[string]ab.Entity{}
  123. for t, u := range uuids {
  124. delegate := s.delegates[t]
  125. if delegate == nil {
  126. return []SearchResult{}, ErrorDelegateNotFound
  127. }
  128. for _, entity := range delegate.LoadEntities(u) {
  129. entities[entity.GetID()] = entity
  130. }
  131. }
  132. results := []SearchResult{}
  133. for _, match := range matches {
  134. if _, ok := entities[match.UUID]; !ok {
  135. continue
  136. }
  137. results = append(results, SearchResult{
  138. Entity: entities[match.UUID],
  139. Type: match.Type,
  140. })
  141. }
  142. s.cacheSave(keywords, results)
  143. return results, nil
  144. }
  145. func (s *SearchService) IndexEntity(entityType string, entity ab.Entity) error {
  146. s.db.Exec("DELETE FROM search_metadata WHERE uuid = $1", entity.GetID())
  147. delegate, ok := s.delegates[entityType]
  148. if !ok {
  149. return ErrorDelegateNotFound
  150. }
  151. data := delegate.IndexEntity(entity)
  152. if len(data) == 0 {
  153. return nil
  154. }
  155. placeholders := []string{}
  156. values := []interface{}{}
  157. uuid := entity.GetID()
  158. for i, d := range data {
  159. if d.Owner == "" {
  160. d.Owner = "00000000-0000-0000-0000-000000000000"
  161. }
  162. placeholders = append(placeholders, fmt.Sprintf("($%d, $%d, $%d, $%d, $%d)", i*5+1, i*5+2, i*5+3, i*5+4, i*5+5))
  163. values = append(values, uuid, entityType, d.Keyword, d.Relevance, d.Owner)
  164. }
  165. _, err := s.db.Exec("INSERT INTO search_metadata(uuid, type, keyword, relevance, owner) VALUES "+strings.Join(placeholders, ", ")+" ON CONFLICT DO NOTHING;", values...)
  166. return err
  167. }
  168. func (s *SearchService) RemoveEntity(uuid string) error {
  169. _, err := s.db.Exec("DELETE FROM search_metadata WHERE uuid = $1", uuid)
  170. return err
  171. }
  172. func (s *SearchService) PurgeIndex() error {
  173. _, err := s.db.Exec("DELETE FROM search_metadata")
  174. return err
  175. }
  176. func (s *SearchService) AddDelegate(delegateType string, delegate SearchServiceDelegate) *SearchService {
  177. s.delegates[delegateType] = delegate
  178. return s
  179. }
  180. func (s *SearchService) Register(srv *ab.Server) error {
  181. srv.Post("/api/search", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  182. d := SearchPostData{}
  183. ab.MustDecode(r, &d)
  184. results, err := s.Search(d.Search, d.Owners)
  185. ab.MaybeFail(http.StatusInternalServerError, err)
  186. ab.Render(r).JSON(results)
  187. }))
  188. return nil
  189. }
  190. func (s *SearchService) SchemaInstalled(db ab.DB) bool {
  191. return ab.TableExists(db, "search_metadata")
  192. }
  193. func (s *SearchService) SchemaSQL() string {
  194. return `
  195. CREATE TABLE search_metadata (
  196. uuid uuid NOT NULL,
  197. type character varying NOT NULL,
  198. owner uuid,
  199. keyword character varying NOT NULL,
  200. relevance double precision NOT NULL,
  201. CONSTRAINT search_metadata_pkey PRIMARY KEY (uuid, keyword, owner),
  202. CONSTRAINT search_metadata_keyword_check CHECK (keyword::text <> ''::text),
  203. CONSTRAINT search_metadata_relevance_check CHECK (relevance <= 1::double precision AND relevance >= 0::double precision)
  204. );
  205. CREATE INDEX search_metadata_keyword_idx
  206. ON search_metadata
  207. USING hash (keyword);
  208. `
  209. }
  210. type SearchPostData struct {
  211. Search string `json:"search"`
  212. Owners []string `json:"owners"`
  213. }