/modules/mod_translation/support/translation_scan.erl

  1% @author Marc Worrell <marc@worrell.nl>
  2%% @copyright 2010,2011 Marc Worrell
  3%% Date: 2010-05-19
  4%% @doc Parse templates / erlang files in modules, extract all translations.
  5
  6%% Copyright 2010,2011 Marc Worrell
  7%%
  8%% Licensed under the Apache License, Version 2.0 (the "License");
  9%% you may not use this file except in compliance with the License.
 10%% You may obtain a copy of the License at
 11%% 
 12%%     http://www.apache.org/licenses/LICENSE-2.0
 13%% 
 14%% Unless required by applicable law or agreed to in writing, software
 15%% distributed under the License is distributed on an "AS IS" BASIS,
 16%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 17%% See the License for the specific language governing permissions and
 18%% limitations under the License.
 19
 20-module(translation_scan).
 21-author("Marc Worrell <marc@worrell.nl>").
 22
 23-export([scan/1, scan_file/2]).
 24
 25-include("zotonic.hrl").
 26
 27scan(Context) ->
 28    ModTemplates = [{Path, Files} || {_M, {Path, Files}} <- z_module_indexer:all(template, Context)],
 29    ModErlang = [{Path, Files} || {_M, {Path, Files}} <- z_module_indexer:all(erlang, Context)],
 30    Combined = lists:foldl(
 31                 fun({Path, Files}, Acc) ->
 32                         [{Path, Files ++ proplists:get_value(Path, ModTemplates, [])}
 33                          |proplists:delete(Path, Acc)]
 34                 end, [], ModErlang),
 35    [ scan_module(Mod) || Mod <- Combined ].
 36
 37scan_module({Path, Templates}) ->
 38    {Path, dedupl(lists:flatten([ scan_file(filename:extension(File), File) || {_BaseName,File} <- Templates ]))}.
 39
 40
 41dedupl(Trans) ->
 42    Dict = dict:new(),
 43    List = dict:to_list(insert(Trans, Dict)),
 44    [ Tr || {_Key,Tr} <- List ].
 45    
 46    insert([], Dict) ->
 47        Dict;
 48    insert([{Text, Args, Loc}|Trans], Dict) ->
 49        case dict:find(Text, Dict) of
 50            {ok, {_Text, Args0, Loc0}} ->
 51                Dict1 = dict:store(Text, {Text, merge_args(Args,Args0), [Loc|Loc0]}, Dict),
 52                insert(Trans, Dict1);
 53            error ->
 54                Dict1 = dict:store(Text, {Text, Args, [Loc]}, Dict),
 55                insert(Trans, Dict1)
 56        end.
 57
 58    merge_args([], Args) -> Args;
 59    merge_args(Args, []) -> Args;
 60    merge_args([{Lang,Text}|Rest], Args) ->
 61        case proplists:get_value(Lang, Args) of
 62            undefined -> merge_args(Rest, [{Lang,Text}|Args]);
 63            _ -> merge_args(Rest, Args)
 64        end.
 65
 66
 67%% @doc Parse the Erlang module. Extract all translation tags.
 68scan_file(".erl", File) ->
 69    {ok, Path} = zotonic_app:get_path(),
 70    case epp:open(File, [filename:join(Path, "include")]) of
 71        {ok, Epp} ->
 72            parse_erl(File, Epp);
 73        {error, Reason} ->
 74            ?ERROR("POT generation, erlang error in ~p: ~p~n", [File, Reason]),
 75            []
 76    end;
 77
 78%% @doc Parse the template in the file. Extract all translation tags.
 79scan_file(".tpl", File) ->
 80    case parse(File) of
 81        {ok, ParseTree} ->
 82            extract(ParseTree, [], File);
 83        {error, Reason} ->
 84            ?ERROR("POT generation, template error in ~p: ~p~n", [File, Reason]),
 85            []
 86    end.
 87
 88
 89parse(File) when is_list(File) ->  
 90    case catch file:read_file(File) of
 91        {ok, Data} ->
 92            case parse(Data) of
 93                {ok, Val} ->
 94                    {ok, Val};
 95                Err ->
 96                    Err
 97            end;
 98        Error ->
 99            {error, io_lib:format("reading ~p failed (~p)", [File, Error])}  
100    end;
101parse(Data) when is_binary(Data) ->
102    case erlydtl_scanner:scan(binary_to_list(Data)) of
103        {ok, Tokens} ->
104            erlydtl_parser:parse(Tokens);
105        Err ->
106            Err
107    end.
108    
109
110%% @doc Extract all translation tags from the parse tree.
111extract(ParseTree, Acc, F) when is_list(ParseTree) ->
112    lists:foldl(fun(Tree,A) -> extract(Tree, A, F) end, Acc, ParseTree);
113extract({trans, {trans_text, {_File, Line,_Col}, Text}}, Acc, F) ->
114    [{z_string:trim(Text), [], {F,Line}}|Acc];
115extract({trans_literal, {_File, Line,_Col}, Text}, Acc, F) ->
116    [{Text, [], {F,Line}}|Acc];
117extract({trans_ext, {string_literal, {_File, Line,_Col}, Text}, Args}, Acc, F) ->
118    [{Text, trans_ext_args(Args,[]), {F,Line}}|Acc];
119extract({text, _, _}, Acc, _F) -> Acc;
120extract({string_literal, _, _}, Acc, _F) -> Acc;
121extract({number_literal, _, _}, Acc, _F) -> Acc;
122extract({comment, _}, Acc, _F) -> Acc;
123extract({auto_id, _}, Acc, _F) -> Acc;
124extract({variable, _}, Acc, _F) -> Acc;
125extract(T, Acc, F) when is_tuple(T) ->
126    extract(tl(tuple_to_list(T)), Acc, F);
127extract(N, Acc, _F) when is_integer(N); is_atom(N) ->
128    Acc.
129
130    trans_ext_args([], Acc) ->
131        Acc;
132    trans_ext_args([{{identifier,_,Lang}, {string_literal, _, Text}}|Args], Acc) ->
133        trans_ext_args(Args, [{list_to_atom(Lang), Text}|Acc]).
134
135
136%% Scan binary for erlang ?__(..., Context) syntax with either a binary or a string as first arg.
137parse_erl(File, Epp) ->
138    parse_erl_form(epp:parse_erl_form(Epp), File, Epp, []).
139
140parse_erl_form({eof, _}, _File, Epp, Acc) ->
141    epp:close(Epp),
142    Acc;
143parse_erl_form({ok, Other}, File, Epp, Acc) ->
144    parse_erl_form(epp:parse_erl_form(Epp), File, Epp, 
145                   parse_erl_form_part(Other, File, Acc));
146parse_erl_form({error, _}, File, Epp, Acc) ->
147    parse_erl_form(epp:parse_erl_form(Epp), File, Epp, Acc).
148
149
150parse_erl_form_part({function, _, _, _, Expressions}, File, Acc) ->
151    lists:foldl(fun(Part,A) -> parse_erl_form_part(Part, File, A) end, Acc, Expressions);
152parse_erl_form_part({tuple, _, Expressions}, File, Acc) ->
153    lists:foldl(fun(Part,A) -> parse_erl_form_part(Part, File, A) end, Acc, Expressions);
154parse_erl_form_part({clause, _, _, _, Expressions}, File, Acc) ->
155    lists:foldl(fun(Part,A) -> parse_erl_form_part(Part, File, A) end, Acc, Expressions);
156parse_erl_form_part({match, _, X, Y}, File, Acc) ->
157    parse_erl_form_part(X, File, []) ++ parse_erl_form_part(Y, File, []) ++ Acc;
158parse_erl_form_part({cons, _, X, Y}, File, Acc) ->
159    parse_erl_form_part(X, File, []) ++ parse_erl_form_part(Y, File, []) ++ Acc;
160parse_erl_form_part({'case', _, Expr, Exprs}, File, Acc) ->
161    parse_erl_form_part(Expr, File, []) ++ 
162        lists:foldl(fun(Part,A) -> parse_erl_form_part(Part, File, A) end, Acc, Exprs);
163
164parse_erl_form_part({call, _, {remote, _, {atom, _, z_trans}, {atom, _, trans}},
165                     [{string, Line, S}, _]}, File, Acc) ->
166    [{S, [], {File,Line}}|Acc];
167parse_erl_form_part({call, _, {remote, _, {atom, _, z_trans}, {atom, _, trans}},
168                     [{bin, _, [{bin_element, _, {string, Line, S}, _, _}|_]}|_]}, File, Acc) ->
169    [{S, [], {File,Line}}|Acc];
170
171parse_erl_form_part({call, _, _, Expressions}, File, Acc) ->
172    lists:foldl(fun(Part,A) -> parse_erl_form_part(Part, File, A) end, Acc, Expressions);
173parse_erl_form_part(_Part, _File, Acc) ->
174    Acc. %% ignore