/src/support/z_dropbox.erl

https://code.google.com/p/zotonic/ · Erlang · 235 lines · 122 code · 38 blank · 75 comment · 0 complexity · 39c366e90128de3ceb863fb47a29d7b2 MD5 · raw file

  1. %% @author Marc Worrell <marc@worrell.nl>
  2. %% @copyright 2009 Marc Worrell
  3. %%
  4. %% @doc Simple dropbox handler, monitors a directory and signals new files.
  5. %% @todo Make this into a module
  6. %%
  7. %% Flow:
  8. %% 1. An user uploads/moves a file to the dropbox
  9. %% 2. Dropbox handler sees the file, moves it so a safe place, and notifies the file handler of it existance.
  10. %% Copyright 2009 Marc Worrell
  11. %%
  12. %% Licensed under the Apache License, Version 2.0 (the "License");
  13. %% you may not use this file except in compliance with the License.
  14. %% You may obtain a copy of the License at
  15. %%
  16. %% http://www.apache.org/licenses/LICENSE-2.0
  17. %%
  18. %% Unless required by applicable law or agreed to in writing, software
  19. %% distributed under the License is distributed on an "AS IS" BASIS,
  20. %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  21. %% See the License for the specific language governing permissions and
  22. %% limitations under the License.
  23. -module(z_dropbox).
  24. -author("Marc Worrell <marc@worrell.nl>").
  25. -behaviour(gen_server).
  26. %% gen_server exports
  27. -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).
  28. -export([start_link/1]).
  29. %% interface functions
  30. -export([
  31. scan/1
  32. ]).
  33. %% internal
  34. -export([]).
  35. -include_lib("zotonic.hrl").
  36. -record(state, {dropbox_dir, processing_dir, unhandled_dir, min_age, max_age, host, context}).
  37. %%====================================================================
  38. %% API
  39. %%====================================================================
  40. %% @spec start_link(SiteArgs) -> {ok,Pid} | ignore | {error,Error}
  41. %% @doc Starts the dropbox server
  42. start_link(SiteProps) ->
  43. {host, Host} = proplists:lookup(host, SiteProps),
  44. Name = z_utils:name_for_host(?MODULE, Host),
  45. gen_server:start_link({local, Name}, ?MODULE, SiteProps, []).
  46. %% @spec scan(context()) -> void()
  47. %% @doc Perform a scan of the dropbox, periodically called by a timer.
  48. scan(Context) ->
  49. gen_server:cast(Context#context.dropbox_server, scan).
  50. %%====================================================================
  51. %% gen_server callbacks
  52. %%====================================================================
  53. %% @spec init(SiteProps) -> {ok, State} |
  54. %% {ok, State, Timeout} |
  55. %% ignore |
  56. %% {stop, Reason}
  57. %% @doc Initiates the server. Options are: dropbox_dir, processing_dir, unhandled_dir, interval, max_age and min_age
  58. init(SiteProps) ->
  59. Host = proplists:get_value(host, SiteProps),
  60. Context = z_context:new(Host),
  61. DefaultDropBoxDir = z_path:files_subdir_ensure("dropbox", Context),
  62. DefaultProcessingDir = z_path:files_subdir_ensure("processing", Context),
  63. DefaultUnhandledDir = z_path:files_subdir_ensure("unhandled", Context),
  64. DropBox = string:strip(proplists:get_value(dropbox_dir, SiteProps, DefaultDropBoxDir), right, $/),
  65. ProcDir = string:strip(proplists:get_value(dropbox_processing_dir, SiteProps, DefaultProcessingDir), right, $/),
  66. UnDir = string:strip(proplists:get_value(dropbox_unhandled_dir, SiteProps, DefaultUnhandledDir), right, $/),
  67. Interval = proplists:get_value(dropbox_interval, SiteProps, 10000),
  68. MinAge = proplists:get_value(dropbox_min_age, SiteProps, 10),
  69. MaxAge = proplists:get_value(dropbox_max_age, SiteProps, 3600),
  70. State = #state{dropbox_dir=DropBox, processing_dir=ProcDir, unhandled_dir=UnDir, min_age=MinAge, max_age=MaxAge, host=Host, context=Context},
  71. timer:apply_interval(Interval, ?MODULE, scan, [Context]),
  72. {ok, State}.
  73. %% @spec handle_call(Request, From, State) -> {reply, Reply, State} |
  74. %% {reply, Reply, State, Timeout} |
  75. %% {noreply, State} |
  76. %% {noreply, State, Timeout} |
  77. %% {stop, Reason, Reply, State} |
  78. %% {stop, Reason, State}
  79. %% @doc Trap unknown calls
  80. handle_call(Message, _From, State) ->
  81. {stop, {unknown_call, Message}, State}.
  82. %% @spec handle_cast(Msg, State) -> {noreply, State} |
  83. %% {noreply, State, Timeout} |
  84. %% {stop, Reason, State}
  85. %% @doc Scan the dropbox, broadcast found files.
  86. handle_cast(scan, State) ->
  87. do_scan(State),
  88. z_utils:flush_message({'$gen_cast', scan}),
  89. {noreply, State};
  90. %% @doc Trap unknown casts
  91. handle_cast(Message, State) ->
  92. {stop, {unknown_cast, Message}, State}.
  93. %% @spec handle_info(Info, State) -> {noreply, State} |
  94. %% {noreply, State, Timeout} |
  95. %% {stop, Reason, State}
  96. %% @doc Handling all non call/cast messages
  97. handle_info(_Info, State) ->
  98. {noreply, State}.
  99. %% @spec terminate(Reason, State) -> void()
  100. %% @doc This function is called by a gen_server when it is about to
  101. %% terminate. It should be the opposite of Module:init/1 and do any necessary
  102. %% cleaning up. When it returns, the gen_server terminates with Reason.
  103. %% The return value is ignored.
  104. terminate(_Reason, _State) ->
  105. ok.
  106. %% @spec code_change(OldVsn, State, Extra) -> {ok, NewState}
  107. %% @doc Convert process state when code is changed
  108. code_change(_OldVsn, State, _Extra) ->
  109. {ok, State}.
  110. %%====================================================================
  111. %% support functions
  112. %%====================================================================
  113. %% @spec do_scan(State) -> void()
  114. %% @doc Perform a scan of the dropbox, broadcast all to be processed files.
  115. do_scan(State) ->
  116. #state{processing_dir=ProcDir, dropbox_dir=DropDir, unhandled_dir=UnhandledDir, min_age=MinAge, max_age=MaxAge} = State,
  117. % Move all old files in the processing directory to the unhandled directory
  118. ProcFiles = scan_directory(ProcDir),
  119. {ToProcess,ToRemove} = lists:foldl(fun(F, Acc) -> max_age_split(F, MaxAge, Acc) end,
  120. {[],[]},
  121. ProcFiles),
  122. lists:foreach(fun(F) -> move_file(ProcDir, F, true, UnhandledDir) end, ToRemove),
  123. % Move all new dropbox files to the processing directory
  124. AllDropFiles = scan_directory(DropDir),
  125. SafeDropFiles = lists:foldl(fun(F, Acc)-> min_age_check(F, MinAge, Acc) end,
  126. [],
  127. AllDropFiles),
  128. Moved = lists:map(fun(F) -> move_file(DropDir, F, false, ProcDir) end, SafeDropFiles),
  129. ToProcess1 = lists:foldl( fun
  130. ({ok, File}, Acc) -> [File|Acc];
  131. ({error, _Reason}, Acc) -> Acc
  132. end,
  133. ToProcess,
  134. Moved),
  135. lists:foreach(fun(F) -> z_notifier:first({dropbox_file, F}, State#state.context) end, ToProcess1).
  136. %% @doc Scan a directory, return list of files not changed in the last 10 seconds.
  137. scan_directory(Dir) ->
  138. filelib:fold_files(Dir, "", true, fun(F,Acc) -> append_file(F, Acc) end, []).
  139. %% @todo Check if this is a file we are interested in, should not be part of a .svn or other directory
  140. append_file([$.|_Rest], Acc) ->
  141. Acc;
  142. append_file(File, Acc) ->
  143. case string:str(File, "/.") of
  144. 0 -> [File|Acc];
  145. _ -> Acc
  146. end.
  147. min_age_check(File, MinAge, Acc) ->
  148. Mod = filelib:last_modified(File),
  149. ModSecs = calendar:datetime_to_gregorian_seconds(Mod),
  150. Now = calendar:local_time(),
  151. NowSecs = calendar:datetime_to_gregorian_seconds(Now),
  152. case NowSecs - ModSecs > MinAge of
  153. true -> [File|Acc];
  154. false -> Acc
  155. end.
  156. max_age_split(File, MaxAge, {AccNew, AccOld}) ->
  157. Mod = filelib:last_modified(File),
  158. ModSecs = calendar:datetime_to_gregorian_seconds(Mod),
  159. Now = calendar:local_time(),
  160. NowSecs = calendar:datetime_to_gregorian_seconds(Now),
  161. case NowSecs - ModSecs > MaxAge of
  162. true -> {AccNew, [File|AccOld]};
  163. false -> {[File|AccNew], AccOld}
  164. end.
  165. %% @spec move_file(BaseDir, File, DeleteTarget, ToDir) -> {ok, NewFile} | {error, Reason}
  166. %% @doc Move a file relative to one directory to another directory
  167. move_file(BaseDir, File, DeleteTarget, ToDir) ->
  168. Rel = rel_file(BaseDir, File),
  169. Target = filename:join(ToDir,Rel),
  170. case filelib:is_dir(Target) of
  171. true -> file:del_dir(Target);
  172. false -> ok
  173. end,
  174. case DeleteTarget of
  175. true -> file:delete(Target);
  176. false -> ok
  177. end,
  178. case filelib:is_regular(Target) of
  179. false ->
  180. case filelib:ensure_dir(Target) of
  181. ok ->
  182. case file:rename(File,Target) of
  183. ok -> {ok, Target};
  184. Error -> Error
  185. end;
  186. Error ->
  187. Error
  188. end;
  189. true ->
  190. {error, eexist}
  191. end.
  192. %% @doc Return the relative path of the file to a BaseDir
  193. rel_file(BaseDir, File) ->
  194. case lists:prefix(BaseDir, File) of
  195. true -> lists:nthtail(length(BaseDir)+1, File);
  196. false -> filename:basename(File)
  197. end.