/src/support/z_pivot_rsc.erl
Erlang | 909 lines | 660 code | 121 blank | 128 comment | 7 complexity | 77079ea753c00fada2e6e15ffaa27009 MD5 | raw file
1%% @author Marc Worrell <marc@worrell.nl> 2%% @copyright 2009-2015 Marc Worrell 3%% @doc Pivoting server for the rsc table. Takes care of full text indices. Polls the pivot queue for any changed resources. 4 5%% Copyright 2009-2015 Marc Worrell 6%% 7%% Licensed under the Apache License, Version 2.0 (the "License"); 8%% you may not use this file except in compliance with the License. 9%% You may obtain a copy of the License at 10%% 11%% http://www.apache.org/licenses/LICENSE-2.0 12%% 13%% Unless required by applicable law or agreed to in writing, software 14%% distributed under the License is distributed on an "AS IS" BASIS, 15%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16%% See the License for the specific language governing permissions and 17%% limitations under the License. 18 19-module(z_pivot_rsc). 20-author("Marc Worrell <marc@worrell.nl"). 21-behaviour(gen_server). 22 23%% gen_server exports 24-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). 25-export([start_link/1]). 26 27%% interface functions 28-export([ 29 poll/1, 30 pivot/2, 31 pivot_delay/1, 32 pivot_resource_update/4, 33 queue_all/1, 34 insert_queue/2, 35 36 get_pivot_title/1, 37 get_pivot_title/2, 38 39 insert_task/3, 40 insert_task/4, 41 insert_task/5, 42 insert_task_after/6, 43 get_task/1, 44 get_task/2, 45 get_task/3, 46 get_task/4, 47 delete_task/3, 48 delete_task/4, 49 50 pivot_resource/2, 51 stemmer_language/1, 52 stemmer_language_config/1, 53 cleanup_tsv_text/1, 54 pg_lang/1, 55 pg_lang_extra/1, 56 % get_pivot_data/2, 57 58 define_custom_pivot/3, 59 lookup_custom_pivot/4 60]). 61 62-include("zotonic.hrl"). 63 64% Interval (in seconds) to check if there are any items to be pivoted. 65-define(PIVOT_POLL_INTERVAL_FAST, 2). 66-define(PIVOT_POLL_INTERVAL_SLOW, 20). 67 68% Number of queued ids taken from the queue at one go 69-define(POLL_BATCH, 50). 70 71%% Minimum day, inserted for date start search ranges 72-define(EPOCH_START, {{-4000,1,1},{0,0,0}}). 73 74 75-record(state, {site, is_pivot_delay = false}). 76 77 78%% @doc Poll the pivot queue for the database in the context 79%% @spec poll(Context) -> void() 80poll(Context) -> 81 gen_server:cast(Context#context.pivot_server, poll). 82 83 84%% @doc An immediate pivot request for a resource 85-spec pivot(integer(), #context{}) -> ok. 86pivot(Id, Context) -> 87 gen_server:cast(Context#context.pivot_server, {pivot, Id}). 88 89%% @doc Delay the next pivot, useful when performing big updates 90-spec pivot_delay(#context{}) -> ok. 91pivot_delay(Context) -> 92 gen_server:cast(Context#context.pivot_server, pivot_delay). 93 94 95%% @doc Return a modified property list with fields that need immediate pivoting on an update. 96pivot_resource_update(Id, UpdateProps, RawProps, Context) -> 97 Props = lists:foldl(fun(Key, All) -> 98 case proplists:is_defined(Key, UpdateProps) of 99 false -> 100 [{Key, proplists:get_value(Key, RawProps)}|All]; 101 true -> 102 All 103 end 104 end, UpdateProps, [date_start, date_end, title]), 105 106 {DateStart, DateEnd} = pivot_date(Props), 107 PivotTitle = truncate(get_pivot_title(Props), 100), 108 Props1 = [ 109 {pivot_date_start, DateStart}, 110 {pivot_date_end, DateEnd}, 111 {pivot_date_start_month_day, month_day(DateStart)}, 112 {pivot_date_end_month_day, month_day(DateEnd)}, 113 {pivot_title, PivotTitle} 114 | Props 115 ], 116 z_notifier:foldr(#pivot_update{id=Id, raw_props=RawProps}, Props1, Context). 117 118month_day(undefined) -> undefined; 119month_day(?EPOCH_START) -> undefined; 120month_day(?ST_JUTTEMIS) -> undefined; 121month_day({{_Y,M,D}, _}) -> M*100+D. 122 123 124%% @doc Rebuild the search index by queueing all resources for pivot. 125queue_all(Context) -> 126 erlang:spawn(fun() -> 127 queue_all(0, Context) 128 end). 129 130 queue_all(FromId, Context) -> 131 case z_db:q("select id from rsc where id > $1 order by id limit 1000", [FromId], Context) of 132 [] -> 133 done; 134 Ids -> 135 F = fun(Ctx) -> 136 [ insert_queue(Id, Ctx) || {Id} <- Ids ] 137 end, 138 z_db:transaction(F, Context), 139 {LastId} = lists:last(Ids), 140 queue_all(LastId, Context) 141 end. 142 143%% @doc Insert a rsc_id in the pivot queue 144insert_queue(Id, Context) -> 145 insert_queue(Id, calendar:universal_time(), Context). 146 147%% @doc Insert a rsc_id in the pivot queue for a certain date 148-spec insert_queue(integer(), calendar:date(), #context{}) -> ok | {error, eexist}. 149insert_queue(Id, Date, Context) when is_integer(Id), is_tuple(Date) -> 150 z_db:transaction( 151 fun(Ctx) -> 152 case z_db:q("update rsc_pivot_queue 153 set serial = serial + 1, 154 due = $2 155 where rsc_id = $1", [Id, Date], Ctx) of 156 1 -> 157 ok; 158 0 -> 159 try 160 z_db:q("insert into rsc_pivot_queue (rsc_id, due, is_update) values ($1, $2, true)", 161 [Id, Date], 162 Ctx), 163 ok 164 catch 165 throw:{error, {error,error,<<"23503">>, _, _}} -> 166 {error, eexist} 167 end 168 end 169 end, 170 Context). 171 172%% @doc Insert a slow running pivot task. For example syncing category numbers after an category update. 173insert_task(Module, Function, Context) -> 174 insert_task(Module, Function, undefined, [], Context). 175 176%% @doc Insert a slow running pivot task. Use the UniqueKey to prevent double queued tasks. 177insert_task(Module, Function, UniqueKey, Context) -> 178 insert_task(Module, Function, UniqueKey, [], Context). 179 180%% @doc Insert a slow running pivot task with unique key and arguments. 181insert_task(Module, Function, undefined, Args, Context) -> 182 insert_task(Module, Function, binary_to_list(z_ids:id()), Args, Context); 183insert_task(Module, Function, UniqueKey, Args, Context) -> 184 insert_task_after(undefined, Module, Function, UniqueKey, Args, Context). 185 186%% @doc Insert a slow running pivot task with unique key and arguments that should start after Seconds seconds. 187insert_task_after(SecondsOrDate, Module, Function, UniqueKey, Args, Context) -> 188 z_db:transaction(fun(Ctx) -> insert_transaction(SecondsOrDate, Module, Function, UniqueKey, Args, Ctx) end, Context). 189 190 insert_transaction(SecondsOrDate, Module, Function, UniqueKey, Args, Context) -> 191 Due = to_utc_date(SecondsOrDate), 192 UniqueKeyBin = z_convert:to_binary(UniqueKey), 193 Fields = [ 194 {module, Module}, 195 {function, Function}, 196 {key, UniqueKeyBin}, 197 {args, Args}, 198 {due, Due} 199 ], 200 case z_db:q1("select id 201 from pivot_task_queue 202 where module = $1 and function = $2 and key = $3", 203 [Module, Function, UniqueKeyBin], 204 Context) 205 of 206 undefined -> 207 z_db:insert(pivot_task_queue, Fields, Context); 208 Id when is_integer(Id) -> 209 case Due of 210 undefined -> nop; 211 _ -> z_db:update(pivot_task_queue, Id, Fields, Context) 212 end, 213 {ok, Id} 214 end. 215 216 217get_task(Context) -> 218 z_db:assoc(" 219 select * 220 from pivot_task_queue", 221 Context). 222 223get_task(Module, Context) -> 224 z_db:assoc(" 225 select * 226 from pivot_task_queue 227 where module = $1", 228 [Module], 229 Context). 230 231get_task(Module, Function, Context) -> 232 z_db:assoc(" 233 select * 234 from pivot_task_queue 235 where module = $1 and function = $2", 236 [Module, Function], 237 Context). 238 239get_task(Module, Function, UniqueKey, Context) -> 240 UniqueKeyBin = z_convert:to_binary(UniqueKey), 241 z_db:assoc_row(" 242 select * 243 from pivot_task_queue 244 where module = $1 and function = $2 and key = $3", 245 [Module, Function, UniqueKeyBin], 246 Context). 247 248to_utc_date(undefined) -> 249 undefined; 250to_utc_date(N) when is_integer(N) -> 251 calendar:gregorian_seconds_to_datetime( 252 calendar:datetime_to_gregorian_seconds(calendar:universal_time()) + N); 253to_utc_date({Y,M,D} = YMD) when is_integer(Y), is_integer(M), is_integer(D) -> 254 {YMD,{0,0,0}}; 255to_utc_date({{Y,M,D},{H,I,S}} = Date) when is_integer(Y), is_integer(M), is_integer(D), is_integer(H), is_integer(I), is_integer(S) -> 256 Date. 257 258 259delete_task(Module, Function, Context) -> 260 z_db:q("delete from pivot_task_queue where module = $1 and function = $2", 261 [Module, Function], 262 Context). 263 264delete_task(Module, Function, UniqueKey, Context) -> 265 UniqueKeyBin = z_convert:to_binary(UniqueKey), 266 z_db:q("delete from pivot_task_queue where module = $1 and function = $2 and key = $3", 267 [Module, Function, UniqueKeyBin], 268 Context). 269 270 271%%==================================================================== 272%% API 273%%==================================================================== 274%% @spec start_link(SiteProps) -> {ok,Pid} | ignore | {error,Error} 275%% @doc Starts the server 276start_link(SiteProps) -> 277 {site, Site} = proplists:lookup(site, SiteProps), 278 Name = z_utils:name_for_site(?MODULE, Site), 279 gen_server:start_link({local, Name}, ?MODULE, Site, []). 280 281 282%%==================================================================== 283%% gen_server callbacks 284%%==================================================================== 285 286%% @spec init(Args) -> {ok, State} | 287%% {ok, State, Timeout} | 288%% ignore | 289%% {stop, Reason} 290%% @doc Initiates the server. 291init(Site) -> 292 lager:md([ 293 {site, Site}, 294 {module, ?MODULE} 295 ]), 296 timer:send_after(?PIVOT_POLL_INTERVAL_SLOW*1000, poll), 297 {ok, #state{site=Site, is_pivot_delay=false}}. 298 299 300%% @spec handle_call(Request, From, State) -> {reply, Reply, State} | 301%% {reply, Reply, State, Timeout} | 302%% {noreply, State} | 303%% {noreply, State, Timeout} | 304%% {stop, Reason, Reply, State} | 305%% {stop, Reason, State} 306%% @doc Trap unknown calls 307handle_call(Message, _From, State) -> 308 {stop, {unknown_call, Message}, State}. 309 310 311%% @spec handle_cast(Msg, State) -> {noreply, State} | 312%% {noreply, State, Timeout} | 313%% {stop, Reason, State} 314%% @doc Poll the queue for the default host 315handle_cast(poll, State) -> 316 do_poll(z_context:new(State#state.site)), 317 {noreply, State}; 318 319 320%% @doc Poll the queue for a particular database 321handle_cast({pivot, Id}, State) -> 322 do_pivot(Id, z_context:new(State#state.site)), 323 {noreply, State}; 324 325%% @doc Delay the next pivot, useful when performing big updates 326handle_cast(pivot_delay, State) -> 327 {noreply, State#state{is_pivot_delay=true}}; 328 329%% @doc Trap unknown casts 330handle_cast(Message, State) -> 331 {stop, {unknown_cast, Message}, State}. 332 333%% @spec handle_info(Info, State) -> {noreply, State} | 334%% {noreply, State, Timeout} | 335%% {stop, Reason, State} 336%% @doc Handling all non call/cast messages 337handle_info(poll, #state{is_pivot_delay=true} = State) -> 338 timer:send_after(?PIVOT_POLL_INTERVAL_SLOW*1000, poll), 339 {noreply, State#state{is_pivot_delay=false}}; 340handle_info(poll, State) -> 341 case do_poll(z_context:new(State#state.site)) of 342 true -> timer:send_after(?PIVOT_POLL_INTERVAL_FAST*1000, poll); 343 false -> timer:send_after(?PIVOT_POLL_INTERVAL_SLOW*1000, poll) 344 end, 345 {noreply, State}; 346 347handle_info(_Info, State) -> 348 {noreply, State}. 349 350%% @spec terminate(Reason, State) -> void() 351%% @doc This function is called by a gen_server when it is about to 352%% terminate. It should be the opposite of Module:init/1 and do any necessary 353%% cleaning up. When it returns, the gen_server terminates with Reason. 354%% The return value is ignored. 355terminate(_Reason, _State) -> 356 ok. 357 358%% @spec code_change(OldVsn, State, Extra) -> {ok, NewState} 359%% @doc Convert process state when code is changed 360code_change(_OldVsn, State, _Extra) -> 361 {ok, State}. 362 363 364%%==================================================================== 365%% support functions 366%%==================================================================== 367 368%% @doc Poll a database for any queued updates. 369do_poll(Context) -> 370 DidTask = do_poll_task(Context), 371 do_poll_queue(Context) or DidTask. 372 373do_poll_task(Context) -> 374 case poll_task(Context) of 375 {TaskId, Module, Function, _Key, Args} -> 376 try 377 case erlang:apply(Module, Function, z_convert:to_list(Args) ++ [Context]) of 378 {delay, Delay} -> 379 Due = if 380 is_integer(Delay) -> 381 calendar:gregorian_seconds_to_datetime( 382 calendar:datetime_to_gregorian_seconds(calendar:universal_time()) + Delay); 383 is_tuple(Delay) -> 384 Delay 385 end, 386 z_db:q("update pivot_task_queue set due = $1 where id = $2", [Due, TaskId], Context); 387 {delay, Delay, NewArgs} -> 388 Due = if 389 is_integer(Delay) -> 390 calendar:gregorian_seconds_to_datetime( 391 calendar:datetime_to_gregorian_seconds(calendar:universal_time()) + Delay); 392 is_tuple(Delay) -> 393 Delay 394 end, 395 Fields = [ 396 {due, Due}, 397 {args, NewArgs} 398 ], 399 z_db:update(pivot_task_queue, TaskId, Fields, Context); 400 _OK -> 401 z_db:q("delete from pivot_task_queue where id = $1", [TaskId], Context) 402 end 403 catch 404 error:undef -> 405 ?zWarning(io_lib:format("Undefined task, aborting: ~p:~p(~p) ~p~n", 406 [Module, Function, Args, erlang:get_stacktrace()]), 407 Context), 408 z_db:q("delete from pivot_task_queue where id = $1", [TaskId], Context); 409 Error:Reason -> 410 ?zWarning(io_lib:format("Task failed(~p:~p): ~p:~p(~p) ~p~n", 411 [Error, Reason, Module, Function, Args, erlang:get_stacktrace()]), 412 Context) 413 end, 414 true; 415 empty -> 416 false 417 end. 418 419do_poll_queue(Context) -> 420 case fetch_queue(Context) of 421 [] -> 422 false; 423 Qs -> 424 F = fun(Ctx) -> 425 [ {Id, catch pivot_resource(Id, Ctx)} || {Id,_Serial} <- Qs] 426 end, 427 case z_db:transaction(F, Context) of 428 {rollback, PivotError} -> 429 lager:error("[~p] Pivot error: ~p: ~p~n", 430 [z_context:site(Context), PivotError, Qs]); 431 L when is_list(L) -> 432 lists:map(fun({Id, _Serial}) -> 433 IsA = m_rsc:is_a(Id, Context), 434 z_notifier:notify(#rsc_pivot_done{id=Id, is_a=IsA}, Context), 435 % Flush the resource, as some synthesized attributes might depend on the pivoted fields. 436 % @todo Only do this if some fields are changed 437 m_rsc_update:flush(Id, Context) 438 end, Qs), 439 lists:map(fun({_Id, ok}) -> ok; 440 ({Id,Error}) -> log_error(Id, Error, Context) end, 441 L), 442 delete_queue(Qs, Context) 443 end, 444 true 445 end. 446 447log_error(Id, Error, Context) -> 448 ?zWarning(io_lib:format("Pivot error ~p: ~p", [Id, Error]), Context). 449 450%% @doc Fetch the next task uit de task queue, if any. 451poll_task(Context) -> 452 case z_db:q_row("select id, module, function, key, props 453 from pivot_task_queue 454 where due is null 455 or due < current_timestamp 456 order by due asc 457 limit 1", Context) 458 of 459 {Id,Module,Function,Key,Props} -> 460 Args = case Props of 461 [{args,Args0}] -> Args0; 462 _ -> [] 463 end, 464 %% @todo We delete the task right now, this needs to be changed to a deletion after the task has been running. 465 {Id, z_convert:to_atom(Module), z_convert:to_atom(Function), Key, Args}; 466 undefined -> 467 empty 468 end. 469 470 471%% @doc Pivot a specific id, delete its queue record if present 472do_pivot(Id, Context) -> 473 Serial = fetch_queue_id(Id, Context), 474 pivot_resource(Id, Context), 475 delete_queue(Id, Serial, Context). 476 477 478%% @doc Fetch the next batch of ids from the queue. Remembers the serials, as a new 479%% pivot request might come in while we are pivoting. 480%% @spec fetch_queue(Context) -> [{Id,Serial}] 481fetch_queue(Context) -> 482 z_db:q("select rsc_id, serial from rsc_pivot_queue where due < current_timestamp - '10 second'::interval order by is_update, due limit $1", [?POLL_BATCH], Context). 483 484%% @doc Fetch the serial of id's queue record 485fetch_queue_id(Id, Context) -> 486 z_db:q1("select serial from rsc_pivot_queue where rsc_id = $1", [Id], Context). 487 488%% @doc Delete the previously queued ids iff the queue entry has not been updated in the meanwhile 489delete_queue(Qs, Context) -> 490 F = fun(Ctx) -> 491 [ z_db:q("delete from rsc_pivot_queue where rsc_id = $1 and serial = $2", [Id,Serial], Ctx) || {Id,Serial} <- Qs ] 492 end, 493 z_db:transaction(F, Context). 494 495%% @doc Delete a specific id/serial combination 496delete_queue(_Id, undefined, _Context) -> 497 ok; 498delete_queue(Id, Serial, Context) -> 499 z_db:q("delete from rsc_pivot_queue where rsc_id = $1 and serial = $2", [Id,Serial], Context). 500 501 502-spec pivot_resource(integer(), #context{}) -> ok | {error, eexist}. 503pivot_resource(Id, Context0) -> 504 Lang = stemmer_language_config(Context0), 505 Context = z_context:set_language(Lang, 506 z_context:set_tz(<<"UTC">>, 507 z_acl:sudo(Context0))), 508 case m_rsc:exists(Id, Context) of 509 true -> 510 RscProps = get_pivot_rsc(Id, Context), 511 Vars = #{ 512 id => Id, 513 props => RscProps, 514 z_language => Lang 515 }, 516 {ok, Template} = z_template_compiler_runtime:map_template({cat, <<"pivot/pivot.tpl">>}, Vars, Context), 517 TextA = render_block(a, Template, Vars, Context), 518 TextB = render_block(b, Template, Vars, Context), 519 TextC = render_block(c, Template, Vars, Context), 520 TextD = render_block(d, Template, Vars, Context), 521 TsvIds = render_block(related_ids, Template, Vars, Context), 522 Title = render_block(title, Template, Vars, Context), 523 Street = render_block(address_street, Template, Vars, Context), 524 City = render_block(address_city, Template, Vars, Context), 525 Postcode = render_block(address_postcode, Template, Vars, Context), 526 State = render_block(address_state, Template, Vars, Context), 527 Country = render_block(address_country, Template, Vars, Context), 528 NameFirst = render_block(name_first, Template, Vars, Context), 529 NameSurname = render_block(name_surname, Template, Vars, Context), 530 Gender = render_block(gender, Template, Vars, Context), 531 DateStart = to_datetime(render_block(date_start, Template, Vars, Context)), 532 DateEnd = to_datetime(render_block(date_end, Template, Vars, Context)), 533 DateStartMonthDay = to_integer(render_block(date_start_month_day, Template, Vars, Context)), 534 DateEndMonthDay = to_integer(render_block(date_end_month_day, Template, Vars, Context)), 535 LocationLat = to_float(render_block(location_lat, Template, Vars, Context)), 536 LocationLng = to_float(render_block(location_lng, Template, Vars, Context)), 537 538 % Make psql tsv texts from the A..D blocks 539 StemmerLanguage = stemmer_language(Context), 540 {SqlA, ArgsA} = to_tsv(TextA, $A, [], StemmerLanguage), 541 {SqlB, ArgsB} = to_tsv(TextB, $B, ArgsA, StemmerLanguage), 542 {SqlC, ArgsC} = to_tsv(TextC, $C, ArgsB, StemmerLanguage), 543 {SqlD, ArgsD} = to_tsv(TextD, $D, ArgsC, StemmerLanguage), 544 545 % Make the text and object-ids vectors for the pivot 546 TsvSql = [SqlA, " || ", SqlB, " || ", SqlC, " || ", SqlD], 547 Tsv = z_db:q1(iolist_to_binary(["select ", TsvSql]), ArgsD, Context), 548 Rtsv = z_db:q1("select to_tsvector($1)", [TsvIds], Context), 549 550 KVs = [ 551 {pivot_tsv, Tsv}, 552 {pivot_rtsv, Rtsv}, 553 {pivot_street, truncate(Street, 120)}, 554 {pivot_city, truncate(City, 100)}, 555 {pivot_postcode, truncate(Postcode, 30)}, 556 {pivot_state, truncate(State, 50)}, 557 {pivot_country, truncate(Country, 80)}, 558 {pivot_first_name, truncate(NameFirst, 100)}, 559 {pivot_surname, truncate(NameSurname, 100)}, 560 {pivot_gender, truncate(Gender, 1)}, 561 {pivot_date_start, DateStart}, 562 {pivot_date_end, DateEnd}, 563 {pivot_date_start_month_day, DateStartMonthDay}, 564 {pivot_date_end_month_day, DateEndMonthDay}, 565 {pivot_title, truncate(Title, 100)}, 566 {pivot_location_lat, LocationLat}, 567 {pivot_location_lng, LocationLng} 568 ], 569 KVs1= z_notifier:foldr(#pivot_fields{id=Id, rsc=RscProps}, KVs, Context), 570 update_changed(Id, KVs1, RscProps, Context), 571 pivot_resource_custom(Id, Context), 572 573 case to_datetime(render_block(date_repivot, Template, Vars, Context)) of 574 undefined -> ok; 575 DateRepivot -> insert_queue(Id, DateRepivot, Context) 576 end, 577 ok; 578 false -> 579 {error, eexist} 580 end. 581 582render_block(Block, Template, Vars, Context) -> 583 {Output, _} = z_template:render_block_to_iolist(Block, Template, Vars, Context), 584 iolist_to_binary(Output). 585 586%% @doc Check which pivot fields are changed, update only those 587update_changed(Id, KVs, RscProps, Context) -> 588 case lists:filter( 589 fun 590 ({K,V}) when is_list(V) -> 591 proplists:get_value(K, RscProps) =/= iolist_to_binary(V); 592 ({K,undefined}) -> 593 proplists:get_value(K, RscProps) =/= undefined; 594 ({K,V}) when is_atom(V) -> 595 proplists:get_value(K, RscProps) =/= z_convert:to_binary(V); 596 ({K,V}) -> 597 proplists:get_value(K, RscProps) =/= V 598 end, 599 KVs) 600 of 601 [] -> 602 % No fields changed, nothing to do 603 nop; 604 KVsChanged -> 605 % Make Sql update statement for the changed fields 606 {Sql, Args} = lists:foldl(fun({K,V}, {Sq,As}) -> 607 {[Sq, 608 case As of [] -> []; _ -> $, end, 609 z_convert:to_list(K), 610 " = $", integer_to_list(length(As)+1) 611 ], 612 [V|As]} 613 end, 614 {"update rsc set ",[]}, 615 KVsChanged), 616 617 z_db:q1(iolist_to_binary([Sql, " where id = $", integer_to_list(length(Args)+1)]), 618 lists:reverse([Id|Args]), 619 Context) 620 end. 621 622 623pivot_resource_custom(Id, Context) -> 624 CustomPivots = z_notifier:map(#custom_pivot{id=Id}, Context), 625 lists:foreach( 626 fun 627 (undefined) -> ok; 628 (none) -> ok; 629 ({error, _} = Error) -> 630 lager:error("[~p] Error return from custom pivot of ~p, error: ~p", 631 [z_context:site(Context), Id, Error]); 632 (Res) -> 633 update_custom_pivot(Id, Res, Context) 634 end, 635 CustomPivots), 636 ok. 637 638 639 640to_datetime(Text) -> 641 case z_string:trim(Text) of 642 <<>> -> undefined; 643 Text1 -> check_datetime(z_datetime:to_datetime(Text1)) 644 end. 645 646check_datetime({{Y,M,D},{H,I,S}} = Date) 647 when is_integer(Y), is_integer(M), is_integer(D), 648 is_integer(H), is_integer(I), is_integer(S) -> 649 Date; 650check_datetime({Y,M,D} = Date) 651 when is_integer(Y), is_integer(M), is_integer(D) -> 652 {Date, {0,0,0}}; 653check_datetime(_) -> 654 undefined. 655 656 657%% Make the setweight(to_tsvector()) parts of the update statement 658to_tsv(Text, Level, Args, StemmingLanguage) when is_binary(Text) -> 659 case cleanup_tsv_text(z_html:unescape(z_html:strip(Text))) of 660 <<>> -> 661 {"tsvector('')", Args}; 662 TsvText -> 663 N = length(Args) + 1, 664 Args1 = Args ++ [TsvText], 665 {["setweight(to_tsvector('pg_catalog.",StemmingLanguage,"', $",integer_to_list(N),"), '",Level,"')"], Args1} 666 end. 667 668to_float(undefined) -> 669 undefined; 670to_float(Text) -> 671 case z_string:trim(Text) of 672 <<>> -> undefined; 673 Text1 -> z_convert:to_float(Text1) 674 end. 675 676to_integer(undefined) -> 677 undefined; 678to_integer(Text) -> 679 case z_string:trim(Text) of 680 <<>> -> undefined; 681 Text1 -> z_convert:to_integer(Text1) 682 end. 683 684cleanup_tsv_text(Text) when is_binary(Text) -> 685 Text1 = binary:replace(Text, <<"-">>, <<" ">>, [global]), 686 Text2 = binary:replace(Text1, <<"/">>, <<" ">>, [global]), 687 z_string:trim(Text2). 688 689truncate(undefined, _Len) -> undefined; 690truncate(S, Len) -> iolist_to_binary( 691 z_string:trim( 692 z_string:to_lower( 693 truncate_1(S, Len, Len)))). 694 695truncate_1(_S, 0, _Bytes) -> 696 ""; 697truncate_1(S, Utf8Len, Bytes) -> 698 case z_string:truncate(S, Utf8Len, "") of 699 T when length(T) > Bytes -> truncate_1(T, Utf8Len-1, Bytes); 700 L -> L 701 end. 702 703 704%% @doc Fetch the date range from the record 705pivot_date(R) -> 706 DateStart = z_datetime:undefined_if_invalid_date(proplists:get_value(date_start, R)), 707 DateEnd = z_datetime:undefined_if_invalid_date(proplists:get_value(date_end, R)), 708 pivot_date1(DateStart, DateEnd). 709 710 pivot_date1(S, E) when not is_tuple(S) andalso not is_tuple(E) -> 711 {undefined, undefined}; 712 pivot_date1(S, E) when not is_tuple(S) andalso is_tuple(E) -> 713 { ?EPOCH_START, E}; 714 pivot_date1(S, E) when is_tuple(S) andalso not is_tuple(E) -> 715 {S, ?ST_JUTTEMIS}; 716 pivot_date1(S, E) when is_tuple(S) andalso is_tuple(E) -> 717 {S, E}. 718 719 720%% @doc Fetch the first title from the record for sorting. 721get_pivot_title(Id, Context) -> 722 z_string:to_lower(get_pivot_title([{title, m_rsc:p(Id, title, Context)}])). 723 724get_pivot_title(Props) -> 725 case proplists:get_value(title, Props) of 726 {trans, []} -> 727 ""; 728 {trans, [{_, Text}|_]} -> 729 z_string:to_lower(Text); 730 T -> 731 z_string:to_lower(T) 732 end. 733 734 735%% @doc Return the raw resource data for the pivoter 736get_pivot_rsc(Id, Context) -> 737 FullRecord = z_db:assoc_props_row("select * from rsc where id = $1", [Id], Context), 738 z_notifier:foldl(pivot_rsc_data, FullRecord, Context). 739 740 741%% @doc Translate a language to a language string as used by 742%% postgresql. This language list is the intersection of the default 743%% catalogs of postgres with the languages supported by 744%% mod_translation. 745pg_lang(dk) -> "danish"; 746pg_lang(nl) -> "dutch"; 747pg_lang(en) -> "english"; 748pg_lang(fi) -> "finnish"; 749pg_lang(fr) -> "french"; 750pg_lang(de) -> "german"; 751pg_lang(hu) -> "hungarian"; 752pg_lang(it) -> "italian"; 753pg_lang(no) -> "norwegian"; 754pg_lang(ro) -> "romanian"; 755pg_lang(ru) -> "russian"; 756pg_lang(es) -> "spanish"; 757pg_lang(se) -> "swedish"; 758pg_lang(tr) -> "turkish"; 759pg_lang(<<"dk">>) -> "danish"; 760pg_lang(<<"nl">>) -> "dutch"; 761pg_lang(<<"en">>) -> "english"; 762pg_lang(<<"fi">>) -> "finnish"; 763pg_lang(<<"fr">>) -> "french"; 764pg_lang(<<"de">>) -> "german"; 765pg_lang(<<"hu">>) -> "hungarian"; 766pg_lang(<<"it">>) -> "italian"; 767pg_lang(<<"no">>) -> "norwegian"; 768pg_lang(<<"ro">>) -> "romanian"; 769pg_lang(<<"ru">>) -> "russian"; 770pg_lang(<<"es">>) -> "spanish"; 771pg_lang(<<"se">>) -> "swedish"; 772pg_lang(<<"tr">>) -> "turkish"; 773pg_lang(_) -> "english". 774 775%% Map extra languages, these are from the i18n.language_stemmer configuration and not 776%% per default installed in PostgreSQL 777pg_lang_extra(Iso) -> 778 case iso639:lc2lang(z_convert:to_list(Iso)) of 779 <<"">> -> 780 pg_lang(Iso); 781 Lang -> 782 lists:takewhile(fun 783 (C) when C >= $a, C =< $z -> true; 784 (_) -> false 785 end, 786 z_convert:to_list(z_string:to_lower(Lang))) 787 end. 788 789% Default stemmers in a Ubuntu psql install: 790% 791% pg_catalog | danish_stem | snowball stemmer for danish language 792% pg_catalog | dutch_stem | snowball stemmer for dutch language 793% pg_catalog | english_stem | snowball stemmer for english language 794% pg_catalog | finnish_stem | snowball stemmer for finnish language 795% pg_catalog | french_stem | snowball stemmer for french language 796% pg_catalog | german_stem | snowball stemmer for german language 797% pg_catalog | hungarian_stem | snowball stemmer for hungarian language 798% pg_catalog | italian_stem | snowball stemmer for italian language 799% pg_catalog | norwegian_stem | snowball stemmer for norwegian language 800% pg_catalog | portuguese_stem | snowball stemmer for portuguese language 801% pg_catalog | romanian_stem | snowball stemmer for romanian language 802% pg_catalog | russian_stem | snowball stemmer for russian language 803% pg_catalog | simple | simple dictionary: just lower case and check for stopword 804% pg_catalog | spanish_stem | snowball stemmer for spanish language 805% pg_catalog | swedish_stem | snowball stemmer for swedish language 806% pg_catalog | turkish_stem | snowball stemmer for turkish language 807 808%% @doc Return the language used for stemming the full text index. 809%% We use a single stemming to prevent having seperate indexes per language. 810-spec stemmer_language(#context{}) -> string(). 811stemmer_language(Context) -> 812 StemmingLanguage = m_config:get_value(i18n, language_stemmer, Context), 813 case z_utils:is_empty(StemmingLanguage) of 814 true -> pg_lang(z_trans:default_language(Context)); 815 false -> pg_lang_extra(StemmingLanguage) 816 end. 817 818-spec stemmer_language_config(#context{}) -> atom(). 819stemmer_language_config(Context) -> 820 StemmingLanguage = m_config:get_value(i18n, language_stemmer, Context), 821 case z_utils:is_empty(StemmingLanguage) of 822 true -> 823 z_trans:default_language(Context); 824 false -> 825 case z_trans:to_language_atom(StemmingLanguage) of 826 {ok, LangAtom} -> LangAtom; 827 {error, not_a_language} -> z_trans:default_language(Context) 828 end 829 end. 830 831%% @spec define_custom_pivot(Module, columns(), Context) -> ok 832%% @doc Let a module define a custom pivot 833%% columns() -> [column()] 834%% column() -> {ColumName::atom(), ColSpec::string()} | {atom(), string(), options::list()} 835define_custom_pivot(Module, Columns, Context) -> 836 TableName = "pivot_" ++ z_convert:to_list(Module), 837 case z_db:table_exists(TableName, Context) of 838 true -> 839 ok; 840 false -> 841 ok = z_db:transaction( 842 fun(Ctx) -> 843 Fields = custom_columns(Columns), 844 Sql = "CREATE TABLE " ++ TableName ++ "(" ++ 845 "id int NOT NULL," ++ Fields ++ " primary key(id))", 846 847 [] = z_db:q(lists:flatten(Sql), Ctx), 848 849 [] = z_db:q("ALTER TABLE " ++ TableName ++ 850 " ADD CONSTRAINT fk_" ++ TableName ++ "_id " ++ 851 " FOREIGN KEY (id) REFERENCES rsc(id) ON UPDATE CASCADE ON DELETE CASCADE", Ctx), 852 853 Indexable = lists:filter(fun({_,_}) -> true; 854 ({_,_,Opts}) -> not lists:member(noindex, Opts) 855 end, 856 Columns), 857 Idx = [ 858 begin 859 K = element(1,Col), 860 "CREATE INDEX " ++ z_convert:to_list(K) ++ "_key ON " 861 ++ TableName ++ "(" ++ z_convert:to_list(K) ++ ")" 862 end 863 || Col <- Indexable 864 ], 865 lists:foreach( 866 fun(Sql1) -> 867 [] = z_db:q(Sql1, Ctx) 868 end, 869 Idx), 870 ok 871 end, 872 Context), 873 z_db:flush(Context), 874 ok 875 end. 876 877 878custom_columns(Cols) -> 879 custom_columns(Cols, []). 880 881custom_columns([], Acc) -> 882 lists:reverse(Acc); 883custom_columns([{Name, Spec}|Rest], Acc) -> 884 custom_columns(Rest, [ [z_convert:to_list(Name), " ", Spec, ","] | Acc]); 885custom_columns([{Name, Spec, _Opts}|Rest], Acc) -> 886 custom_columns(Rest, [ [z_convert:to_list(Name), " ", Spec, ","] | Acc]). 887 888 889update_custom_pivot(Id, {Module, Columns}, Context) -> 890 TableName = "pivot_" ++ z_convert:to_list(Module), 891 case z_db:select(TableName, Id, Context) of 892 {ok, []} -> 893 {ok, _} = z_db:insert(TableName, [{id, Id}|Columns], Context); 894 {ok, _} -> 895 {ok, _} = z_db:update(TableName, Id, Columns, Context) 896 end. 897 898 899%% @doc Lookup a custom pivot; give back the Id based on a column. Will always return the first Id found. 900%% @spec lookup_custom_pivot(Module, Column, Value, Context) -> Id | undefined 901lookup_custom_pivot(Module, Column, Value, Context) -> 902 TableName = "pivot_" ++ z_convert:to_list(Module), 903 Column1 = z_convert:to_list(Column), 904 Query = "SELECT id FROM " ++ TableName ++ " WHERE " ++ Column1 ++ " = $1", 905 case z_db:q(Query, [Value], Context) of 906 [] -> undefined; 907 [{Id}|_] -> Id 908 end. 909