PageRenderTime 37ms CodeModel.GetById 13ms app.highlight 20ms RepoModel.GetById 1ms app.codeStats 0ms

/ucengine/src/backends/search/solr/uce_event_solr_search.erl

Relevant Search: With Applications for Solr and Elasticsearch

For more in depth reading about search, ranking and generally everything you could ever want to know about how lucene, elasticsearch or solr work under the hood I highly suggest this book. Easily one of the most interesting technical books I have read in a long time. If you are tasked with solving search relevance problems even if not in Solr or Elasticsearch it should be your first reference. Amazon Affiliate Link
http://github.com/AF83/ucengine
Erlang | 258 lines | 210 code | 30 blank | 18 comment | 4 complexity | f77d262232cad58be351a6fa51a3c798 MD5 | raw file
  1%%
  2%%  U.C.Engine - Unified Collaboration Engine
  3%%  Copyright (C) 2011 af83
  4%%
  5%%  This program is free software: you can redistribute it and/or modify
  6%%  it under the terms of the GNU Affero General Public License as published by
  7%%  the Free Software Foundation, either version 3 of the License, or
  8%%  (at your option) any later version.
  9%%
 10%%  This program is distributed in the hope that it will be useful,
 11%%  but WITHOUT ANY WARRANTY; without even the implied warranty of
 12%%  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13%%  GNU Affero General Public License for more details.
 14%%
 15%%  You should have received a copy of the GNU Affero General Public License
 16%%  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 17%%
 18-module(uce_event_solr_search).
 19
 20-export([add/2, commit/0, list/11, delete/2]).
 21
 22-include("uce.hrl").
 23
 24-define(DEFAULT_HOST, "http://localhost:8983/solr").
 25-define(SOLR_UPDATE, "/update").
 26-define(SOLR_SELECT, "/select?").
 27
 28-define(META_PREFIX, "metadata_").
 29
 30add(Domain, Event) ->
 31    [Host] = utils:get(config:get(solr), [host], [?DEFAULT_HOST]),
 32    ibrowse:send_req(Host ++ ?SOLR_UPDATE, [], post, to_solrxml(Domain, Event)),
 33    {ok, created}.
 34
 35commit() ->
 36    [Host] = utils:get(config:get(solr), [host], [?DEFAULT_HOST]),
 37    Commit = lists:flatten(xmerl:export_simple_element({commit, []}, xmerl_xml)),
 38    ibrowse:send_req(Host ++ ?SOLR_UPDATE, [], post, Commit),
 39    {ok, commited}.
 40
 41%% Encode event in solrxml format which be used to add solr index
 42to_solrxml(Domain, #uce_event{id=Id,
 43                              datetime=Datetime,
 44                              location=Location,
 45                              from=From,
 46                              to=To,
 47                              type=Type,
 48                              metadata=Metadata}) ->
 49
 50    LocationElement = [{field, [{name,"location"}], [Location]}],
 51
 52    MetadataFlattenElement =
 53        [{field, [{name, "metadata"}], [lists:flatten([Value ++ " " || {_, Value} <- Metadata])]}],
 54
 55    MetadataElement =
 56        lists:map(fun({Key, Value}) ->
 57                          {field, [{name, "metadata_" ++ Key}], [Value]}
 58                  end,
 59                  Metadata),
 60
 61    DocElements = [{field, [{name,"id"}], [Id]},
 62                   {field, [{name,"domain"}], [Domain]},
 63                   {field, [{name,"datetime"}], [integer_to_list(Datetime)]},
 64                   {field, [{name,"type"}], [Type]},
 65                   {field, [{name,"to"}], [To]},
 66                   {field, [{name,"from"}], [From]}] ++
 67        LocationElement ++
 68        MetadataFlattenElement ++
 69        MetadataElement,
 70
 71    Add = {add, [], [{doc, [], DocElements}]},
 72    lists:flatten(xmerl:export_simple_element(Add, xmerl_xml)).
 73
 74params_to_query([{Key, Value}|Tail]) ->
 75    Key ++ ":" ++ Value ++ case Tail of
 76                               [] ->
 77                                   [];
 78                               _ ->
 79                                   " AND " ++ params_to_query(Tail)
 80                           end;
 81params_to_query([Value|Tail])
 82  when is_list(Value) ->
 83    Value ++ case Tail of
 84                 [] ->
 85                     [];
 86                 _ ->
 87                     " AND " ++ params_to_query(Tail)
 88             end.
 89
 90list(Domain, Location, Search, From, Types, DateStart, DateEnd, Parent, Start, Rows, Order) ->
 91    [Host] = utils:get(config:get(solr), [host], [?DEFAULT_HOST]),
 92
 93    DomainSelector = [{"domain", Domain}],
 94
 95    LocationSelector =
 96        if
 97            Location /= "" ->
 98                [{"location", Location}];
 99            true ->
100                []
101        end,
102
103    FromSelector =
104        if
105            From /= "" ->
106                [{"from", From}];
107            true ->
108                []
109        end,
110
111    ParentSelector =
112        if
113            Parent /= "" ->
114                [{"parent", Parent}];
115            true ->
116                []
117        end,
118
119    SearchSelector =
120        if
121            Search /= [] ->
122                [{"metadata", string:join(Search, "+")}];
123            true ->
124                []
125        end,
126
127    TypesSelector =
128        if
129            Types == [] ->
130                [];
131            true ->
132                [" (" ++ string:join(["type:" ++ Type || Type <- Types], " OR ") ++ ") "]
133        end,
134
135    TimeRange =
136        if
137            DateStart /= 0, DateEnd /= infinity ->
138                "[" ++ integer_to_list(DateStart) ++ " TO " ++ integer_to_list(DateEnd) ++ "]";
139            DateStart /= 0 ->
140                "[" ++ integer_to_list(DateStart) ++ " TO *]";
141            DateEnd /= infinity ->
142                "[* TO " ++ integer_to_list(DateEnd) ++ "]";
143            true ->
144                []
145        end,
146
147    TimeSelector =
148        if
149            DateStart /= 0; DateEnd /= infinity ->
150                 [{"facet", "on"}, {"facet.field", "datetime"}, {"fq", "datetime:" ++ TimeRange}];
151            true ->
152                []
153        end,
154
155    Query = [{"q", params_to_query(LocationSelector ++
156                                       DomainSelector ++
157                                       ParentSelector ++
158                                       FromSelector ++
159                                       TypesSelector ++
160                                       SearchSelector)}],
161
162    Params = Query ++ TimeSelector ++ [{"wt", "json"},
163                                       {"start", integer_to_list(Start)},
164                                       {"rows", integer_to_list(Rows)},
165                                       {"sort", lists:concat(["datetime ", Order])}],
166    EncodedParams = lists:map(fun({Elem, Value}) ->
167                                      lists:concat([yaws_api:url_encode(Elem),
168                                                    "=",
169                                                    yaws_api:url_encode(Value)])
170                              end,
171                              Params),
172
173    case ibrowse:send_req(Host ++ ?SOLR_SELECT ++ string:join(EncodedParams, "&"), [], get) of
174        {ok, _, _, Body} ->
175            {NumTotal, Events} = json_to_events(Body),
176            {ok, NumTotal, Events};
177        {error, _} ->
178            throw({error, bad_parameters})
179    end.
180
181json_to_events(Body) ->
182    {struct, JSON} = mochijson:decode(Body),
183    {"response", {struct, ResponseJSON}} = lists:keyfind("response", 1, JSON),
184    {"numFound", NumTotal} = lists:keyfind("numFound", 1, ResponseJSON),
185    {"docs", {array, DocsJSON}} = lists:keyfind("docs", 1, ResponseJSON),
186    {NumTotal, make_list_json_events(DocsJSON)}.
187
188make_list_json_events([]) ->
189    [];
190make_list_json_events([{struct, Elems}|Tail]) ->
191    case utils:get(Elems,
192                   ["id", "domain", "datetime", "location","from", "to", "type", "parent"],
193                   [none,
194                    none,
195                    none,
196                    {array, [""]},
197                    none,
198                    {array, ["all"]},
199                    none,
200                    {array, [""]}]) of
201
202        [none, _, _, _, _, _, _, _, _] ->
203            {error, bad_record};
204        [_, none, _, _, _, _, _, _, _] ->
205            {error, bad_record};
206        [_, _, none, _, _, _, _, _, _] ->
207            {error, bad_record};
208        [_, _, _, _, _, none, _, _, _] ->
209            {error, bad_record};
210        [_, _, _, _, _, _, _, none, _] ->
211            {error, bad_record};
212
213        [{array, [Id]},
214         {array, [_Domain]},
215         Datetime,
216         {array, [Location]},
217         {array, [From]},
218         {array, [To]},
219         {array, [Type]},
220         {array, [Parent]}] ->
221            FlatMetadata =
222                lists:filter(fun({Name, _}) ->
223                                     if
224                                         length(Name) < length(?META_PREFIX) ->
225                                             false;
226                                         true ->
227                                             SubName = string:substr(Name, 1, length(?META_PREFIX)),
228                                             if
229                                                 SubName == ?META_PREFIX ->
230                                                     true;
231                                                 true ->
232                                                     false
233                                             end
234                                     end
235                             end,
236                             Elems),
237            Metadata = lists:map(fun({Name, {array, [Value]}}) ->
238                                         {string:substr(Name, length(?META_PREFIX) + 1, length(Name)),
239                                          Value}
240                                 end,
241                                 FlatMetadata),
242            [#uce_event{id=Id,
243                        datetime=case is_list(Datetime) of
244                                    true -> list_to_integer(Datetime);
245                                    _ -> Datetime
246                                 end,
247                        location=Location,
248                        from=From,
249                        to=To,
250                        type=Type,
251                        parent=Parent,
252                        metadata=Metadata}] ++ make_list_json_events(Tail)
253    end.
254
255delete(_Domain, Id) ->
256    [Host] = utils:get(config:get(solr), [host], [?DEFAULT_HOST]),
257    ibrowse:send_req(Host ++ ?SOLR_UPDATE, [], post, "<delete><query>"++ Id ++"</query></delete>"),
258    {ok, deleted}.