/src/support/z_dropbox.erl

  1%% @author Marc Worrell <marc@worrell.nl>
  2%% @copyright 2009 Marc Worrell
  3%%
  4%% @doc Simple dropbox handler, monitors a directory and signals new files.
  5%% @todo Make this into a module
  6%%
  7%% Flow:
  8%% 1. An user uploads/moves a file to the dropbox
  9%% 2. Dropbox handler sees the file, moves it so a safe place, and notifies the file handler of it existance.
 10
 11%% Copyright 2009 Marc Worrell
 12%%
 13%% Licensed under the Apache License, Version 2.0 (the "License");
 14%% you may not use this file except in compliance with the License.
 15%% You may obtain a copy of the License at
 16%% 
 17%%     http://www.apache.org/licenses/LICENSE-2.0
 18%% 
 19%% Unless required by applicable law or agreed to in writing, software
 20%% distributed under the License is distributed on an "AS IS" BASIS,
 21%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 22%% See the License for the specific language governing permissions and
 23%% limitations under the License.
 24
 25-module(z_dropbox).
 26-author("Marc Worrell <marc@worrell.nl>").
 27-behaviour(gen_server).
 28
 29%% gen_server exports
 30-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).
 31-export([start_link/1]).
 32
 33%% interface functions
 34-export([
 35    scan/1
 36]).
 37
 38%% internal
 39-export([]).
 40
 41-include_lib("zotonic.hrl").
 42
 43-record(state, {dropbox_dir, processing_dir, unhandled_dir, min_age, max_age, host, context}).
 44
 45%%====================================================================
 46%% API
 47%%====================================================================
 48%% @spec start_link(SiteArgs) -> {ok,Pid} | ignore | {error,Error}
 49%% @doc Starts the dropbox server
 50start_link(SiteProps) ->
 51    {host, Host} = proplists:lookup(host, SiteProps),
 52    Name = z_utils:name_for_host(?MODULE, Host),
 53    gen_server:start_link({local, Name}, ?MODULE, SiteProps, []).
 54
 55
 56%% @spec scan(context()) -> void()
 57%% @doc Perform a scan of the dropbox, periodically called by a timer.
 58scan(Context) ->
 59    gen_server:cast(Context#context.dropbox_server, scan).
 60
 61%%====================================================================
 62%% gen_server callbacks
 63%%====================================================================
 64
 65%% @spec init(SiteProps) -> {ok, State} |
 66%%                     {ok, State, Timeout} |
 67%%                     ignore               |
 68%%                     {stop, Reason}
 69%% @doc Initiates the server.  Options are: dropbox_dir, processing_dir, unhandled_dir, interval, max_age and min_age
 70init(SiteProps) ->
 71    Host     = proplists:get_value(host, SiteProps),
 72    Context  = z_context:new(Host),
 73	DefaultDropBoxDir = z_path:files_subdir_ensure("dropbox", Context),
 74	DefaultProcessingDir = z_path:files_subdir_ensure("processing", Context),
 75	DefaultUnhandledDir = z_path:files_subdir_ensure("unhandled", Context),
 76    DropBox  = string:strip(proplists:get_value(dropbox_dir,            SiteProps, DefaultDropBoxDir),    right, $/), 
 77    ProcDir  = string:strip(proplists:get_value(dropbox_processing_dir, SiteProps, DefaultProcessingDir), right, $/), 
 78    UnDir    = string:strip(proplists:get_value(dropbox_unhandled_dir,  SiteProps, DefaultUnhandledDir),  right, $/), 
 79    Interval = proplists:get_value(dropbox_interval, SiteProps, 10000),
 80    MinAge   = proplists:get_value(dropbox_min_age, SiteProps, 10),
 81    MaxAge   = proplists:get_value(dropbox_max_age, SiteProps, 3600),
 82    State    = #state{dropbox_dir=DropBox, processing_dir=ProcDir, unhandled_dir=UnDir, min_age=MinAge, max_age=MaxAge, host=Host, context=Context},
 83    timer:apply_interval(Interval, ?MODULE, scan, [Context]),
 84    {ok, State}.
 85
 86
 87%% @spec handle_call(Request, From, State) -> {reply, Reply, State} |
 88%%                                      {reply, Reply, State, Timeout} |
 89%%                                      {noreply, State} |
 90%%                                      {noreply, State, Timeout} |
 91%%                                      {stop, Reason, Reply, State} |
 92%%                                      {stop, Reason, State}
 93%% @doc Trap unknown calls
 94handle_call(Message, _From, State) ->
 95    {stop, {unknown_call, Message}, State}.
 96
 97
 98%% @spec handle_cast(Msg, State) -> {noreply, State} |
 99%%                                  {noreply, State, Timeout} |
100%%                                  {stop, Reason, State}
101%% @doc Scan the dropbox, broadcast found files.
102handle_cast(scan, State) ->
103    do_scan(State),
104    z_utils:flush_message({'$gen_cast', scan}),
105    {noreply, State};
106    
107%% @doc Trap unknown casts
108handle_cast(Message, State) ->
109    {stop, {unknown_cast, Message}, State}.
110
111
112%% @spec handle_info(Info, State) -> {noreply, State} |
113%%                                       {noreply, State, Timeout} |
114%%                                       {stop, Reason, State}
115%% @doc Handling all non call/cast messages
116handle_info(_Info, State) ->
117    {noreply, State}.
118
119
120%% @spec terminate(Reason, State) -> void()
121%% @doc This function is called by a gen_server when it is about to
122%% terminate. It should be the opposite of Module:init/1 and do any necessary
123%% cleaning up. When it returns, the gen_server terminates with Reason.
124%% The return value is ignored.
125terminate(_Reason, _State) ->
126    ok.
127
128%% @spec code_change(OldVsn, State, Extra) -> {ok, NewState}
129%% @doc Convert process state when code is changed
130
131code_change(_OldVsn, State, _Extra) ->
132    {ok, State}.
133
134
135%%====================================================================
136%% support functions
137%%====================================================================
138
139%% @spec do_scan(State) -> void()
140%% @doc Perform a scan of the dropbox, broadcast all to be processed files.
141do_scan(State) ->
142    #state{processing_dir=ProcDir, dropbox_dir=DropDir, unhandled_dir=UnhandledDir, min_age=MinAge, max_age=MaxAge} = State,
143     
144    % Move all old files in the processing directory to the unhandled directory
145    ProcFiles = scan_directory(ProcDir),
146    {ToProcess,ToRemove} = lists:foldl(fun(F, Acc) -> max_age_split(F, MaxAge, Acc) end,
147                                       {[],[]},
148                                       ProcFiles),
149    lists:foreach(fun(F) -> move_file(ProcDir, F, true, UnhandledDir) end, ToRemove),
150
151    % Move all new dropbox files to the processing directory
152    AllDropFiles  = scan_directory(DropDir),
153    SafeDropFiles = lists:foldl(fun(F, Acc)-> min_age_check(F, MinAge, Acc) end,
154                                [],
155                                AllDropFiles), 
156    Moved      = lists:map(fun(F) -> move_file(DropDir, F, false, ProcDir) end, SafeDropFiles),
157    ToProcess1 = lists:foldl(   fun
158                                    ({ok, File}, Acc) -> [File|Acc];
159                                    ({error, _Reason}, Acc) -> Acc
160                                end,
161                                ToProcess,
162                                Moved),
163    lists:foreach(fun(F) -> z_notifier:first({dropbox_file, F}, State#state.context) end, ToProcess1).
164
165
166%% @doc Scan a directory, return list of files not changed in the last 10 seconds.
167scan_directory(Dir) ->
168    filelib:fold_files(Dir, "", true, fun(F,Acc) -> append_file(F, Acc) end, []).
169
170
171%% @todo Check if this is a file we are interested in, should not be part of a .svn or other directory
172append_file([$.|_Rest], Acc) ->
173    Acc;
174append_file(File, Acc) ->
175    case string:str(File, "/.") of
176        0 -> [File|Acc];
177        _ -> Acc
178    end.
179
180
181min_age_check(File, MinAge, Acc) ->
182    Mod     = filelib:last_modified(File),
183    ModSecs = calendar:datetime_to_gregorian_seconds(Mod),
184    Now     = calendar:local_time(),
185    NowSecs = calendar:datetime_to_gregorian_seconds(Now),
186    case NowSecs - ModSecs > MinAge of
187        true -> [File|Acc];
188        false -> Acc
189    end.
190
191max_age_split(File, MaxAge, {AccNew, AccOld}) ->
192    Mod     = filelib:last_modified(File),
193    ModSecs = calendar:datetime_to_gregorian_seconds(Mod),
194    Now     = calendar:local_time(),
195    NowSecs = calendar:datetime_to_gregorian_seconds(Now),
196    case NowSecs - ModSecs > MaxAge of
197        true ->  {AccNew,        [File|AccOld]};
198        false -> {[File|AccNew], AccOld}
199    end.
200
201
202%% @spec move_file(BaseDir, File, DeleteTarget, ToDir) -> {ok, NewFile} | {error, Reason}
203%% @doc Move a file relative to one directory to another directory
204move_file(BaseDir, File, DeleteTarget, ToDir) ->
205    Rel    = rel_file(BaseDir, File),
206    Target = filename:join(ToDir,Rel),
207    case filelib:is_dir(Target) of
208        true -> file:del_dir(Target);
209        false -> ok
210    end,
211    case DeleteTarget of
212        true -> file:delete(Target);
213        false -> ok
214    end,
215    case filelib:is_regular(Target) of
216        false ->
217            case filelib:ensure_dir(Target) of
218                ok ->
219                    case file:rename(File,Target) of
220                        ok -> {ok, Target};
221                        Error -> Error
222                    end;
223                Error ->
224                    Error
225            end;
226        true ->
227            {error, eexist}
228    end.
229
230%% @doc Return the relative path of the file to a BaseDir
231rel_file(BaseDir, File) ->
232    case lists:prefix(BaseDir, File) of
233        true -> lists:nthtail(length(BaseDir)+1, File);
234        false -> filename:basename(File)
235    end.