PageRenderTime 112ms CodeModel.GetById 22ms app.highlight 81ms RepoModel.GetById 2ms app.codeStats 0ms

/src/support/z_pivot_rsc.erl

http://github.com/zotonic/zotonic
Erlang | 909 lines | 660 code | 121 blank | 128 comment | 7 complexity | 77079ea753c00fada2e6e15ffaa27009 MD5 | raw file
  1%% @author Marc Worrell <marc@worrell.nl>
  2%% @copyright 2009-2015 Marc Worrell
  3%% @doc Pivoting server for the rsc table. Takes care of full text indices. Polls the pivot queue for any changed resources.
  4
  5%% Copyright 2009-2015 Marc Worrell
  6%%
  7%% Licensed under the Apache License, Version 2.0 (the "License");
  8%% you may not use this file except in compliance with the License.
  9%% You may obtain a copy of the License at
 10%% 
 11%%     http://www.apache.org/licenses/LICENSE-2.0
 12%% 
 13%% Unless required by applicable law or agreed to in writing, software
 14%% distributed under the License is distributed on an "AS IS" BASIS,
 15%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 16%% See the License for the specific language governing permissions and
 17%% limitations under the License.
 18
 19-module(z_pivot_rsc).
 20-author("Marc Worrell <marc@worrell.nl").
 21-behaviour(gen_server).
 22
 23%% gen_server exports
 24-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).
 25-export([start_link/1]).
 26
 27%% interface functions
 28-export([
 29    poll/1,
 30    pivot/2,
 31    pivot_delay/1,
 32    pivot_resource_update/4,
 33    queue_all/1,
 34    insert_queue/2,
 35         
 36    get_pivot_title/1,
 37    get_pivot_title/2,
 38
 39    insert_task/3,
 40    insert_task/4,
 41    insert_task/5,
 42    insert_task_after/6,
 43    get_task/1,
 44    get_task/2,
 45    get_task/3,
 46    get_task/4,
 47    delete_task/3,
 48    delete_task/4,
 49    
 50    pivot_resource/2,
 51    stemmer_language/1,
 52    stemmer_language_config/1,
 53    cleanup_tsv_text/1,
 54    pg_lang/1,
 55    pg_lang_extra/1,
 56    % get_pivot_data/2,
 57
 58    define_custom_pivot/3,
 59    lookup_custom_pivot/4
 60]).
 61
 62-include("zotonic.hrl").
 63
 64% Interval (in seconds) to check if there are any items to be pivoted.
 65-define(PIVOT_POLL_INTERVAL_FAST, 2).
 66-define(PIVOT_POLL_INTERVAL_SLOW, 20).
 67
 68% Number of queued ids taken from the queue at one go
 69-define(POLL_BATCH, 50).
 70
 71%% Minimum day, inserted for date start search ranges
 72-define(EPOCH_START, {{-4000,1,1},{0,0,0}}).
 73
 74
 75-record(state, {site, is_pivot_delay = false}).
 76
 77
 78%% @doc Poll the pivot queue for the database in the context
 79%% @spec poll(Context) -> void()
 80poll(Context) ->
 81    gen_server:cast(Context#context.pivot_server, poll).
 82
 83
 84%% @doc An immediate pivot request for a resource
 85-spec pivot(integer(), #context{}) -> ok.
 86pivot(Id, Context) ->
 87    gen_server:cast(Context#context.pivot_server, {pivot, Id}).
 88
 89%% @doc Delay the next pivot, useful when performing big updates
 90-spec pivot_delay(#context{}) -> ok.
 91pivot_delay(Context) ->
 92    gen_server:cast(Context#context.pivot_server, pivot_delay).
 93
 94
 95%% @doc Return a modified property list with fields that need immediate pivoting on an update.
 96pivot_resource_update(Id, UpdateProps, RawProps, Context) ->
 97    Props = lists:foldl(fun(Key, All) ->
 98                                case proplists:is_defined(Key, UpdateProps) of
 99                                    false ->
100                                        [{Key, proplists:get_value(Key, RawProps)}|All];
101                                    true ->
102                                        All
103                                end
104                        end, UpdateProps, [date_start, date_end, title]),
105
106    {DateStart, DateEnd} = pivot_date(Props),
107    PivotTitle = truncate(get_pivot_title(Props), 100),
108    Props1 = [
109        {pivot_date_start, DateStart},
110        {pivot_date_end, DateEnd},
111        {pivot_date_start_month_day, month_day(DateStart)},
112        {pivot_date_end_month_day, month_day(DateEnd)},
113        {pivot_title, PivotTitle}
114        | Props
115    ],
116    z_notifier:foldr(#pivot_update{id=Id, raw_props=RawProps}, Props1, Context).
117
118month_day(undefined) -> undefined;
119month_day(?EPOCH_START) -> undefined;
120month_day(?ST_JUTTEMIS) -> undefined;
121month_day({{_Y,M,D}, _}) -> M*100+D.
122
123
124%% @doc Rebuild the search index by queueing all resources for pivot.
125queue_all(Context) ->
126    erlang:spawn(fun() ->
127                    queue_all(0, Context)
128                 end).
129
130    queue_all(FromId, Context) ->
131        case z_db:q("select id from rsc where id > $1 order by id limit 1000", [FromId], Context) of
132            [] ->
133                done;
134            Ids ->
135                F = fun(Ctx) ->
136                        [ insert_queue(Id, Ctx) || {Id} <- Ids ]
137                    end,
138                z_db:transaction(F, Context),
139                {LastId} = lists:last(Ids),
140                queue_all(LastId, Context)
141        end.
142                
143%% @doc Insert a rsc_id in the pivot queue
144insert_queue(Id, Context) ->
145    insert_queue(Id, calendar:universal_time(), Context).
146
147%% @doc Insert a rsc_id in the pivot queue for a certain date
148-spec insert_queue(integer(), calendar:date(), #context{}) -> ok | {error, eexist}.
149insert_queue(Id, Date, Context) when is_integer(Id), is_tuple(Date) ->
150    z_db:transaction(
151        fun(Ctx) ->
152            case z_db:q("update rsc_pivot_queue 
153                         set serial = serial + 1,
154                             due = $2
155                         where rsc_id = $1", [Id, Date], Ctx) of
156                1 ->
157                    ok;
158                0 ->
159                    try
160                        z_db:q("insert into rsc_pivot_queue (rsc_id, due, is_update) values ($1, $2, true)",
161                               [Id, Date],
162                               Ctx),
163                        ok
164                    catch
165                        throw:{error, {error,error,<<"23503">>, _, _}} ->
166                            {error, eexist}
167                    end
168            end
169        end,
170        Context).
171
172%% @doc Insert a slow running pivot task. For example syncing category numbers after an category update.
173insert_task(Module, Function, Context) ->
174    insert_task(Module, Function, undefined, [], Context).
175
176%% @doc Insert a slow running pivot task. Use the UniqueKey to prevent double queued tasks.
177insert_task(Module, Function, UniqueKey, Context) ->
178    insert_task(Module, Function, UniqueKey, [], Context).
179    
180%% @doc Insert a slow running pivot task with unique key and arguments.
181insert_task(Module, Function, undefined, Args, Context) ->
182    insert_task(Module, Function, binary_to_list(z_ids:id()), Args, Context);
183insert_task(Module, Function, UniqueKey, Args, Context) ->
184    insert_task_after(undefined, Module, Function, UniqueKey, Args, Context).
185
186%% @doc Insert a slow running pivot task with unique key and arguments that should start after Seconds seconds.
187insert_task_after(SecondsOrDate, Module, Function, UniqueKey, Args, Context) ->
188    z_db:transaction(fun(Ctx) -> insert_transaction(SecondsOrDate, Module, Function, UniqueKey, Args, Ctx) end, Context).
189
190    insert_transaction(SecondsOrDate, Module, Function, UniqueKey, Args, Context) ->
191        Due = to_utc_date(SecondsOrDate),
192        UniqueKeyBin = z_convert:to_binary(UniqueKey), 
193        Fields = [
194            {module, Module},
195            {function, Function},
196            {key, UniqueKeyBin},
197            {args, Args},
198            {due, Due}
199        ],
200        case z_db:q1("select id 
201                      from pivot_task_queue 
202                      where module = $1 and function = $2 and key = $3", 
203                     [Module, Function, UniqueKeyBin], 
204                     Context) 
205        of
206            undefined -> 
207                z_db:insert(pivot_task_queue, Fields, Context);
208            Id when is_integer(Id) -> 
209                case Due of
210                    undefined -> nop;
211                    _ -> z_db:update(pivot_task_queue, Id, Fields, Context)
212                end,
213                {ok, Id}
214        end.
215
216
217get_task(Context) ->
218    z_db:assoc("
219            select *
220            from pivot_task_queue",
221            Context).
222
223get_task(Module, Context) ->
224    z_db:assoc("
225            select *
226            from pivot_task_queue
227            where module = $1", 
228            [Module], 
229            Context).
230
231get_task(Module, Function, Context) ->
232    z_db:assoc("
233            select *
234            from pivot_task_queue
235            where module = $1 and function = $2", 
236            [Module, Function], 
237            Context).
238
239get_task(Module, Function, UniqueKey, Context) ->
240    UniqueKeyBin = z_convert:to_binary(UniqueKey), 
241    z_db:assoc_row("
242            select *
243            from pivot_task_queue
244            where module = $1 and function = $2 and key = $3", 
245            [Module, Function, UniqueKeyBin], 
246            Context).
247
248to_utc_date(undefined) ->
249    undefined;
250to_utc_date(N) when is_integer(N) ->
251    calendar:gregorian_seconds_to_datetime(
252        calendar:datetime_to_gregorian_seconds(calendar:universal_time()) + N);
253to_utc_date({Y,M,D} = YMD) when is_integer(Y), is_integer(M), is_integer(D) ->
254    {YMD,{0,0,0}};
255to_utc_date({{Y,M,D},{H,I,S}} = Date) when is_integer(Y), is_integer(M), is_integer(D), is_integer(H), is_integer(I), is_integer(S) ->
256    Date.
257
258
259delete_task(Module, Function, Context) ->
260    z_db:q("delete from pivot_task_queue where module = $1 and function = $2", 
261           [Module, Function], 
262           Context).
263
264delete_task(Module, Function, UniqueKey, Context) ->
265    UniqueKeyBin = z_convert:to_binary(UniqueKey), 
266    z_db:q("delete from pivot_task_queue where module = $1 and function = $2 and key = $3",
267           [Module, Function, UniqueKeyBin], 
268           Context).
269
270
271%%====================================================================
272%% API
273%%====================================================================
274%% @spec start_link(SiteProps) -> {ok,Pid} | ignore | {error,Error}
275%% @doc Starts the server
276start_link(SiteProps) ->
277    {site, Site} = proplists:lookup(site, SiteProps),
278    Name = z_utils:name_for_site(?MODULE, Site),
279    gen_server:start_link({local, Name}, ?MODULE, Site, []).
280
281
282%%====================================================================
283%% gen_server callbacks
284%%====================================================================
285
286%% @spec init(Args) -> {ok, State} |
287%%                     {ok, State, Timeout} |
288%%                     ignore               |
289%%                     {stop, Reason}
290%% @doc Initiates the server.
291init(Site) ->
292    lager:md([
293        {site, Site},
294        {module, ?MODULE}
295      ]),
296    timer:send_after(?PIVOT_POLL_INTERVAL_SLOW*1000, poll),
297    {ok, #state{site=Site, is_pivot_delay=false}}.
298
299
300%% @spec handle_call(Request, From, State) -> {reply, Reply, State} |
301%%                                      {reply, Reply, State, Timeout} |
302%%                                      {noreply, State} |
303%%                                      {noreply, State, Timeout} |
304%%                                      {stop, Reason, Reply, State} |
305%%                                      {stop, Reason, State}
306%% @doc Trap unknown calls
307handle_call(Message, _From, State) ->
308    {stop, {unknown_call, Message}, State}.
309
310
311%% @spec handle_cast(Msg, State) -> {noreply, State} |
312%%                                  {noreply, State, Timeout} |
313%%                                  {stop, Reason, State}
314%% @doc Poll the queue for the default host
315handle_cast(poll, State) ->
316    do_poll(z_context:new(State#state.site)),
317    {noreply, State};
318
319
320%% @doc Poll the queue for a particular database
321handle_cast({pivot, Id}, State) ->
322    do_pivot(Id, z_context:new(State#state.site)),
323    {noreply, State};
324
325%% @doc Delay the next pivot, useful when performing big updates
326handle_cast(pivot_delay, State) ->
327    {noreply, State#state{is_pivot_delay=true}};
328
329%% @doc Trap unknown casts
330handle_cast(Message, State) ->
331    {stop, {unknown_cast, Message}, State}.
332
333%% @spec handle_info(Info, State) -> {noreply, State} |
334%%                                       {noreply, State, Timeout} |
335%%                                       {stop, Reason, State}
336%% @doc Handling all non call/cast messages
337handle_info(poll, #state{is_pivot_delay=true} = State) ->
338    timer:send_after(?PIVOT_POLL_INTERVAL_SLOW*1000, poll),
339    {noreply, State#state{is_pivot_delay=false}};
340handle_info(poll, State) ->
341    case do_poll(z_context:new(State#state.site)) of
342        true ->  timer:send_after(?PIVOT_POLL_INTERVAL_FAST*1000, poll);
343        false -> timer:send_after(?PIVOT_POLL_INTERVAL_SLOW*1000, poll)
344    end,
345    {noreply, State};
346
347handle_info(_Info, State) ->
348    {noreply, State}.
349
350%% @spec terminate(Reason, State) -> void()
351%% @doc This function is called by a gen_server when it is about to
352%% terminate. It should be the opposite of Module:init/1 and do any necessary
353%% cleaning up. When it returns, the gen_server terminates with Reason.
354%% The return value is ignored.
355terminate(_Reason, _State) ->
356    ok.
357
358%% @spec code_change(OldVsn, State, Extra) -> {ok, NewState}
359%% @doc Convert process state when code is changed
360code_change(_OldVsn, State, _Extra) ->
361    {ok, State}.
362
363
364%%====================================================================
365%% support functions
366%%====================================================================
367
368%% @doc Poll a database for any queued updates.
369do_poll(Context) ->
370    DidTask = do_poll_task(Context),
371    do_poll_queue(Context) or DidTask.
372
373do_poll_task(Context) ->
374    case poll_task(Context) of
375        {TaskId, Module, Function, _Key, Args} ->
376            try
377                case erlang:apply(Module, Function, z_convert:to_list(Args) ++ [Context]) of
378                    {delay, Delay} ->
379                        Due = if
380                                is_integer(Delay) ->
381                                    calendar:gregorian_seconds_to_datetime(
382                                        calendar:datetime_to_gregorian_seconds(calendar:universal_time()) + Delay);
383                                is_tuple(Delay) ->
384                                    Delay
385                              end,
386                        z_db:q("update pivot_task_queue set due = $1 where id = $2", [Due, TaskId], Context);
387                    {delay, Delay, NewArgs} ->
388                        Due = if
389                                is_integer(Delay) ->
390                                    calendar:gregorian_seconds_to_datetime(
391                                        calendar:datetime_to_gregorian_seconds(calendar:universal_time()) + Delay);
392                                is_tuple(Delay) ->
393                                    Delay
394                              end,
395                        Fields = [
396                            {due, Due},
397                            {args, NewArgs}
398                        ],
399                        z_db:update(pivot_task_queue, TaskId, Fields, Context);
400                    _OK ->
401                        z_db:q("delete from pivot_task_queue where id = $1", [TaskId], Context)
402                end
403            catch
404                error:undef -> 
405                    ?zWarning(io_lib:format("Undefined task, aborting: ~p:~p(~p) ~p~n", 
406                                [Module, Function, Args, erlang:get_stacktrace()]),
407                                Context),
408                    z_db:q("delete from pivot_task_queue where id = $1", [TaskId], Context);
409                Error:Reason -> 
410                    ?zWarning(io_lib:format("Task failed(~p:~p): ~p:~p(~p) ~p~n", 
411                                [Error, Reason, Module, Function, Args, erlang:get_stacktrace()]), 
412                                Context)
413            end,
414            true;
415        empty ->
416            false
417    end.
418
419do_poll_queue(Context) ->
420    case fetch_queue(Context) of
421        [] ->
422            false;
423        Qs ->
424            F = fun(Ctx) ->
425                        [ {Id, catch pivot_resource(Id, Ctx)} || {Id,_Serial} <- Qs]
426                end,
427            case z_db:transaction(F, Context) of
428                {rollback, PivotError} -> 
429                    lager:error("[~p] Pivot error: ~p: ~p~n", 
430                                [z_context:site(Context), PivotError, Qs]);
431                L when is_list(L) -> 
432                    lists:map(fun({Id, _Serial}) ->
433                                    IsA = m_rsc:is_a(Id, Context),
434                                    z_notifier:notify(#rsc_pivot_done{id=Id, is_a=IsA}, Context),
435                                    % Flush the resource, as some synthesized attributes might depend on the pivoted fields.
436                                    % @todo Only do this if some fields are changed
437                                    m_rsc_update:flush(Id, Context)
438                              end, Qs),
439                    lists:map(fun({_Id, ok}) -> ok; 
440                                 ({Id,Error}) -> log_error(Id, Error, Context) end, 
441                              L),
442                    delete_queue(Qs, Context)
443            end,
444            true
445    end.
446
447log_error(Id, Error, Context) ->
448    ?zWarning(io_lib:format("Pivot error ~p: ~p", [Id, Error]), Context).
449
450%% @doc Fetch the next task uit de task queue, if any.
451poll_task(Context) ->
452    case z_db:q_row("select id, module, function, key, props 
453                     from pivot_task_queue 
454                     where due is null
455                        or due < current_timestamp
456                     order by due asc 
457                     limit 1", Context) 
458    of
459        {Id,Module,Function,Key,Props} ->
460            Args = case Props of
461                [{args,Args0}] -> Args0;
462                _ -> []
463            end,
464            %% @todo We delete the task right now, this needs to be changed to a deletion after the task has been running.
465            {Id, z_convert:to_atom(Module), z_convert:to_atom(Function), Key, Args};
466        undefined ->
467            empty
468    end.
469    
470
471%% @doc Pivot a specific id, delete its queue record if present
472do_pivot(Id, Context) ->
473    Serial = fetch_queue_id(Id, Context),
474    pivot_resource(Id, Context),
475    delete_queue(Id, Serial, Context).
476
477
478%% @doc Fetch the next batch of ids from the queue. Remembers the serials, as a new
479%% pivot request might come in while we are pivoting.
480%% @spec fetch_queue(Context) -> [{Id,Serial}]
481fetch_queue(Context) ->
482    z_db:q("select rsc_id, serial from rsc_pivot_queue where due < current_timestamp - '10 second'::interval order by is_update, due limit $1", [?POLL_BATCH], Context).
483
484%% @doc Fetch the serial of id's queue record
485fetch_queue_id(Id, Context) ->
486    z_db:q1("select serial from rsc_pivot_queue where rsc_id = $1", [Id], Context).
487
488%% @doc Delete the previously queued ids iff the queue entry has not been updated in the meanwhile
489delete_queue(Qs, Context) ->
490    F = fun(Ctx) ->
491        [ z_db:q("delete from rsc_pivot_queue where rsc_id = $1 and serial = $2", [Id,Serial], Ctx) || {Id,Serial} <- Qs ]
492    end,
493    z_db:transaction(F, Context).
494
495%% @doc Delete a specific id/serial combination
496delete_queue(_Id, undefined, _Context) ->
497    ok;
498delete_queue(Id, Serial, Context) ->
499    z_db:q("delete from rsc_pivot_queue where rsc_id = $1 and serial = $2", [Id,Serial], Context).
500
501
502-spec pivot_resource(integer(), #context{}) -> ok | {error, eexist}.
503pivot_resource(Id, Context0) ->
504    Lang = stemmer_language_config(Context0),
505    Context = z_context:set_language(Lang, 
506                 z_context:set_tz(<<"UTC">>, 
507                    z_acl:sudo(Context0))),
508    case m_rsc:exists(Id, Context) of
509        true ->
510            RscProps = get_pivot_rsc(Id, Context),
511            Vars = #{
512                id => Id,
513                props => RscProps,
514                z_language => Lang
515            },
516            {ok, Template} = z_template_compiler_runtime:map_template({cat, <<"pivot/pivot.tpl">>}, Vars, Context),
517            TextA = render_block(a, Template, Vars, Context),
518            TextB = render_block(b, Template, Vars, Context),
519            TextC = render_block(c, Template, Vars, Context),
520            TextD = render_block(d, Template, Vars, Context),
521            TsvIds = render_block(related_ids, Template, Vars, Context),
522            Title = render_block(title, Template, Vars, Context),
523            Street = render_block(address_street, Template, Vars, Context),
524            City = render_block(address_city, Template, Vars, Context),
525            Postcode = render_block(address_postcode, Template, Vars, Context),
526            State = render_block(address_state, Template, Vars, Context),
527            Country = render_block(address_country, Template, Vars, Context),
528            NameFirst = render_block(name_first, Template, Vars, Context),
529            NameSurname = render_block(name_surname, Template, Vars, Context),
530            Gender = render_block(gender, Template, Vars, Context),
531            DateStart = to_datetime(render_block(date_start, Template, Vars, Context)),
532            DateEnd = to_datetime(render_block(date_end, Template, Vars, Context)),
533            DateStartMonthDay = to_integer(render_block(date_start_month_day, Template, Vars, Context)),
534            DateEndMonthDay = to_integer(render_block(date_end_month_day, Template, Vars, Context)),
535            LocationLat = to_float(render_block(location_lat, Template, Vars, Context)),
536            LocationLng = to_float(render_block(location_lng, Template, Vars, Context)),
537
538            % Make psql tsv texts from the A..D blocks
539            StemmerLanguage = stemmer_language(Context),
540            {SqlA, ArgsA} = to_tsv(TextA, $A, [], StemmerLanguage),
541            {SqlB, ArgsB} = to_tsv(TextB, $B, ArgsA, StemmerLanguage),
542            {SqlC, ArgsC} = to_tsv(TextC, $C, ArgsB, StemmerLanguage),
543            {SqlD, ArgsD} = to_tsv(TextD, $D, ArgsC, StemmerLanguage),
544
545            % Make the text and object-ids vectors for the pivot
546            TsvSql = [SqlA, " || ", SqlB, " || ", SqlC, " || ", SqlD],
547            Tsv  = z_db:q1(iolist_to_binary(["select ", TsvSql]), ArgsD, Context),
548            Rtsv = z_db:q1("select to_tsvector($1)", [TsvIds], Context),
549
550            KVs = [
551                {pivot_tsv, Tsv},
552                {pivot_rtsv, Rtsv},
553                {pivot_street, truncate(Street, 120)},
554                {pivot_city, truncate(City, 100)},
555                {pivot_postcode, truncate(Postcode, 30)},
556                {pivot_state, truncate(State, 50)},
557                {pivot_country, truncate(Country, 80)},
558                {pivot_first_name, truncate(NameFirst, 100)},
559                {pivot_surname, truncate(NameSurname, 100)},
560                {pivot_gender, truncate(Gender, 1)},
561                {pivot_date_start, DateStart},
562                {pivot_date_end, DateEnd},
563                {pivot_date_start_month_day, DateStartMonthDay},
564                {pivot_date_end_month_day, DateEndMonthDay},
565                {pivot_title, truncate(Title, 100)},
566                {pivot_location_lat, LocationLat},
567                {pivot_location_lng, LocationLng}
568            ],
569            KVs1= z_notifier:foldr(#pivot_fields{id=Id, rsc=RscProps}, KVs, Context),
570            update_changed(Id, KVs1, RscProps, Context),
571            pivot_resource_custom(Id, Context),
572
573            case to_datetime(render_block(date_repivot, Template, Vars, Context)) of
574                undefined -> ok;
575                DateRepivot -> insert_queue(Id, DateRepivot, Context)
576            end,
577            ok;
578        false ->
579            {error, eexist}
580    end.
581
582render_block(Block, Template, Vars, Context) ->
583    {Output, _} = z_template:render_block_to_iolist(Block, Template, Vars, Context),
584    iolist_to_binary(Output). 
585
586%% @doc Check which pivot fields are changed, update only those
587update_changed(Id, KVs, RscProps, Context) ->
588    case lists:filter(
589                fun
590                    ({K,V}) when is_list(V) ->
591                        proplists:get_value(K, RscProps) =/= iolist_to_binary(V);
592                    ({K,undefined}) ->
593                        proplists:get_value(K, RscProps) =/= undefined;
594                    ({K,V}) when is_atom(V) -> 
595                        proplists:get_value(K, RscProps) =/= z_convert:to_binary(V);
596                    ({K,V}) ->
597                        proplists:get_value(K, RscProps) =/= V
598                end, 
599                KVs)
600    of
601        [] ->
602            % No fields changed, nothing to do
603            nop;
604        KVsChanged ->
605            % Make Sql update statement for the changed fields
606            {Sql, Args} = lists:foldl(fun({K,V}, {Sq,As}) ->
607                                         {[Sq,
608                                            case As of [] -> []; _ -> $, end,
609                                            z_convert:to_list(K),
610                                            " = $", integer_to_list(length(As)+1)
611                                          ],
612                                          [V|As]}
613                                      end,
614                                      {"update rsc set ",[]},
615                                      KVsChanged),
616
617            z_db:q1(iolist_to_binary([Sql, " where id = $", integer_to_list(length(Args)+1)]),
618                    lists:reverse([Id|Args]),
619                    Context)
620    end.
621
622
623pivot_resource_custom(Id, Context) ->
624    CustomPivots = z_notifier:map(#custom_pivot{id=Id}, Context),
625    lists:foreach(
626            fun
627                (undefined) -> ok;
628                (none) -> ok;
629                ({error, _} = Error) ->
630                    lager:error("[~p] Error return from custom pivot of ~p, error: ~p",
631                                [z_context:site(Context), Id, Error]);
632                (Res) ->
633                    update_custom_pivot(Id, Res, Context)
634            end,
635            CustomPivots),
636    ok.
637
638
639
640to_datetime(Text) ->
641    case z_string:trim(Text) of
642        <<>> -> undefined;
643        Text1 -> check_datetime(z_datetime:to_datetime(Text1))
644    end.
645
646check_datetime({{Y,M,D},{H,I,S}} = Date)
647    when is_integer(Y), is_integer(M), is_integer(D),
648         is_integer(H), is_integer(I), is_integer(S) ->
649    Date;
650check_datetime({Y,M,D} = Date)
651    when is_integer(Y), is_integer(M), is_integer(D) ->
652    {Date, {0,0,0}};
653check_datetime(_) ->
654    undefined.
655
656
657%% Make the setweight(to_tsvector()) parts of the update statement
658to_tsv(Text, Level, Args, StemmingLanguage) when is_binary(Text) ->
659    case cleanup_tsv_text(z_html:unescape(z_html:strip(Text))) of
660        <<>> ->
661            {"tsvector('')", Args};
662        TsvText ->
663            N = length(Args) + 1,
664            Args1 = Args ++ [TsvText],
665        {["setweight(to_tsvector('pg_catalog.",StemmingLanguage,"', $",integer_to_list(N),"), '",Level,"')"], Args1}
666    end.
667
668to_float(undefined) ->
669    undefined;
670to_float(Text) ->
671    case z_string:trim(Text) of
672        <<>> -> undefined;
673        Text1 -> z_convert:to_float(Text1)
674    end.
675
676to_integer(undefined) ->
677    undefined;
678to_integer(Text) ->
679    case z_string:trim(Text) of
680        <<>> -> undefined;
681        Text1 -> z_convert:to_integer(Text1)
682    end.
683
684cleanup_tsv_text(Text) when is_binary(Text) ->
685    Text1 = binary:replace(Text,  <<"-">>, <<" ">>, [global]),
686    Text2 = binary:replace(Text1, <<"/">>, <<" ">>, [global]),
687    z_string:trim(Text2).
688
689truncate(undefined, _Len) -> undefined;
690truncate(S, Len) -> iolist_to_binary(
691                        z_string:trim(
692                            z_string:to_lower(
693                                truncate_1(S, Len, Len)))).
694    
695truncate_1(_S, 0, _Bytes) ->
696    "";
697truncate_1(S, Utf8Len, Bytes) ->
698    case z_string:truncate(S, Utf8Len, "") of
699        T when length(T) > Bytes -> truncate_1(T, Utf8Len-1, Bytes);
700        L -> L
701    end.
702    
703
704%% @doc Fetch the date range from the record
705pivot_date(R) ->
706    DateStart = z_datetime:undefined_if_invalid_date(proplists:get_value(date_start, R)),
707    DateEnd   = z_datetime:undefined_if_invalid_date(proplists:get_value(date_end, R)),
708    pivot_date1(DateStart, DateEnd).
709
710    pivot_date1(S, E) when not is_tuple(S) andalso not is_tuple(E) ->
711        {undefined, undefined};
712    pivot_date1(S, E) when not is_tuple(S) andalso is_tuple(E) ->
713        { ?EPOCH_START, E};
714    pivot_date1(S, E) when is_tuple(S) andalso not is_tuple(E) ->
715        {S, ?ST_JUTTEMIS};
716    pivot_date1(S, E) when is_tuple(S) andalso is_tuple(E) ->
717        {S, E}.
718
719
720%% @doc Fetch the first title from the record for sorting.
721get_pivot_title(Id, Context) ->
722    z_string:to_lower(get_pivot_title([{title, m_rsc:p(Id, title, Context)}])).
723
724get_pivot_title(Props) ->
725    case proplists:get_value(title, Props) of
726        {trans, []} ->
727            "";
728        {trans, [{_, Text}|_]} ->
729            z_string:to_lower(Text);
730        T -> 
731            z_string:to_lower(T)
732    end.
733
734
735%% @doc Return the raw resource data for the pivoter
736get_pivot_rsc(Id, Context) ->
737    FullRecord = z_db:assoc_props_row("select * from rsc where id = $1", [Id], Context),
738    z_notifier:foldl(pivot_rsc_data, FullRecord, Context).
739          
740
741%% @doc Translate a language to a language string as used by
742%% postgresql. This language list is the intersection of the default
743%% catalogs of postgres with the languages supported by
744%% mod_translation.
745pg_lang(dk) -> "danish";
746pg_lang(nl) -> "dutch";
747pg_lang(en) -> "english";
748pg_lang(fi) -> "finnish";
749pg_lang(fr) -> "french";
750pg_lang(de) -> "german";
751pg_lang(hu) -> "hungarian";
752pg_lang(it) -> "italian";
753pg_lang(no) -> "norwegian";
754pg_lang(ro) -> "romanian";
755pg_lang(ru) -> "russian";
756pg_lang(es) -> "spanish";
757pg_lang(se) -> "swedish";
758pg_lang(tr) -> "turkish";
759pg_lang(<<"dk">>) -> "danish";
760pg_lang(<<"nl">>) -> "dutch";
761pg_lang(<<"en">>) -> "english";
762pg_lang(<<"fi">>) -> "finnish";
763pg_lang(<<"fr">>) -> "french";
764pg_lang(<<"de">>) -> "german";
765pg_lang(<<"hu">>) -> "hungarian";
766pg_lang(<<"it">>) -> "italian";
767pg_lang(<<"no">>) -> "norwegian";
768pg_lang(<<"ro">>) -> "romanian";
769pg_lang(<<"ru">>) -> "russian";
770pg_lang(<<"es">>) -> "spanish";
771pg_lang(<<"se">>) -> "swedish";
772pg_lang(<<"tr">>) -> "turkish";
773pg_lang(_) -> "english".
774
775%% Map extra languages, these are from the i18n.language_stemmer configuration and not
776%% per default installed in PostgreSQL
777pg_lang_extra(Iso) ->
778    case iso639:lc2lang(z_convert:to_list(Iso)) of
779        <<"">> ->
780            pg_lang(Iso);
781        Lang ->
782            lists:takewhile(fun
783                                (C) when C >= $a, C =< $z -> true;
784                                (_) -> false
785                            end,
786                            z_convert:to_list(z_string:to_lower(Lang)))
787    end.
788
789% Default stemmers in a Ubuntu psql install:
790%
791%  pg_catalog | danish_stem     | snowball stemmer for danish language
792%  pg_catalog | dutch_stem      | snowball stemmer for dutch language
793%  pg_catalog | english_stem    | snowball stemmer for english language
794%  pg_catalog | finnish_stem    | snowball stemmer for finnish language
795%  pg_catalog | french_stem     | snowball stemmer for french language
796%  pg_catalog | german_stem     | snowball stemmer for german language
797%  pg_catalog | hungarian_stem  | snowball stemmer for hungarian language
798%  pg_catalog | italian_stem    | snowball stemmer for italian language
799%  pg_catalog | norwegian_stem  | snowball stemmer for norwegian language
800%  pg_catalog | portuguese_stem | snowball stemmer for portuguese language
801%  pg_catalog | romanian_stem   | snowball stemmer for romanian language
802%  pg_catalog | russian_stem    | snowball stemmer for russian language
803%  pg_catalog | simple          | simple dictionary: just lower case and check for stopword
804%  pg_catalog | spanish_stem    | snowball stemmer for spanish language
805%  pg_catalog | swedish_stem    | snowball stemmer for swedish language
806%  pg_catalog | turkish_stem    | snowball stemmer for turkish language
807
808%% @doc Return the language used for stemming the full text index.
809%%      We use a single stemming to prevent having seperate indexes per language.
810-spec stemmer_language(#context{}) -> string().
811stemmer_language(Context) ->
812    StemmingLanguage = m_config:get_value(i18n, language_stemmer, Context),
813    case z_utils:is_empty(StemmingLanguage) of
814        true -> pg_lang(z_trans:default_language(Context));
815        false -> pg_lang_extra(StemmingLanguage)
816    end.
817
818-spec stemmer_language_config(#context{}) -> atom().
819stemmer_language_config(Context) ->
820    StemmingLanguage = m_config:get_value(i18n, language_stemmer, Context),
821    case z_utils:is_empty(StemmingLanguage) of
822        true ->
823            z_trans:default_language(Context);
824        false -> 
825            case z_trans:to_language_atom(StemmingLanguage) of
826                {ok, LangAtom} -> LangAtom;
827                {error, not_a_language} -> z_trans:default_language(Context)
828            end
829    end.
830
831%% @spec define_custom_pivot(Module, columns(), Context) -> ok
832%% @doc Let a module define a custom pivot
833%% columns() -> [column()]
834%% column()  -> {ColumName::atom(), ColSpec::string()} | {atom(), string(), options::list()}
835define_custom_pivot(Module, Columns, Context) ->
836    TableName = "pivot_" ++ z_convert:to_list(Module),
837    case z_db:table_exists(TableName, Context) of
838        true ->
839            ok;
840        false ->
841            ok = z_db:transaction(
842                    fun(Ctx) ->
843                        Fields = custom_columns(Columns),
844                        Sql = "CREATE TABLE " ++ TableName ++ "(" ++
845                              "id int NOT NULL," ++ Fields ++ " primary key(id))",
846
847                        [] = z_db:q(lists:flatten(Sql), Ctx),
848
849                        [] = z_db:q("ALTER TABLE " ++ TableName ++ 
850                                    " ADD CONSTRAINT fk_" ++ TableName ++ "_id " ++
851                                    " FOREIGN KEY (id) REFERENCES rsc(id) ON UPDATE CASCADE ON DELETE CASCADE", Ctx),
852                        
853                        Indexable = lists:filter(fun({_,_}) -> true;
854                                                    ({_,_,Opts}) -> not lists:member(noindex, Opts)
855                                                 end,
856                                                 Columns),
857                        Idx = [ 
858                                begin
859                                    K = element(1,Col),
860                                    "CREATE INDEX " ++ z_convert:to_list(K) ++ "_key ON " 
861                                    ++ TableName ++ "(" ++ z_convert:to_list(K) ++ ")"
862                                end
863                                || Col <- Indexable
864                            ],
865                        lists:foreach(
866                            fun(Sql1) ->
867                                [] = z_db:q(Sql1, Ctx)
868                            end,
869                            Idx),
870                        ok
871                    end,
872                    Context),
873            z_db:flush(Context),
874            ok
875    end.
876
877
878custom_columns(Cols) ->
879    custom_columns(Cols, []).
880
881custom_columns([], Acc) ->
882    lists:reverse(Acc);
883custom_columns([{Name, Spec}|Rest], Acc) ->
884    custom_columns(Rest, [ [z_convert:to_list(Name), " ", Spec, ","] |  Acc]);
885custom_columns([{Name, Spec, _Opts}|Rest], Acc) ->
886    custom_columns(Rest, [ [z_convert:to_list(Name), " ", Spec, ","] |  Acc]).
887
888
889update_custom_pivot(Id, {Module, Columns}, Context) ->
890    TableName = "pivot_" ++ z_convert:to_list(Module),
891    case z_db:select(TableName, Id, Context) of
892        {ok, []} ->
893            {ok, _} = z_db:insert(TableName, [{id, Id}|Columns], Context);
894        {ok, _}  ->
895            {ok, _} = z_db:update(TableName, Id, Columns, Context)
896    end.
897
898
899%% @doc Lookup a custom pivot; give back the Id based on a column. Will always return the first Id found.
900%% @spec lookup_custom_pivot(Module, Column, Value, Context) -> Id | undefined
901lookup_custom_pivot(Module, Column, Value, Context) ->
902    TableName = "pivot_" ++ z_convert:to_list(Module),
903    Column1 = z_convert:to_list(Column),
904    Query = "SELECT id FROM " ++ TableName ++ " WHERE " ++ Column1 ++ " = $1",
905    case z_db:q(Query, [Value], Context) of
906        [] -> undefined;
907        [{Id}|_] -> Id
908    end.
909