/src/smoosh/src/smoosh_priority_queue.erl

https://github.com/apache/couchdb · Erlang · 177 lines · 138 code · 28 blank · 11 comment · 0 complexity · 2603b376d690ba019f326522cd23e77c MD5 · raw file

  1. % Licensed under the Apache License, Version 2.0 (the "License"); you may not
  2. % use this file except in compliance with the License. You may obtain a copy of
  3. % the License at
  4. %
  5. % http://www.apache.org/licenses/LICENSE-2.0
  6. %
  7. % Unless required by applicable law or agreed to in writing, software
  8. % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  9. % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  10. % License for the specific language governing permissions and limitations under
  11. % the License.
  12. -module(smoosh_priority_queue).
  13. -export([new/1, recover/1]).
  14. -export([last_updated/2, is_key/2, in/4, in/5, out/1, size/1, info/1]).
  15. -export([flush/1]).
  16. -export([from_list/2, to_list/1]).
  17. -export([is_empty/1]).
  18. -export([file_name/1, write_to_file/1]).
  19. -define(VSN, 1).
  20. -record(priority_queue, {
  21. name,
  22. map,
  23. tree
  24. }).
  25. new(Name) ->
  26. #priority_queue{name = Name, map = maps:new(), tree = gb_trees:empty()}.
  27. recover(#priority_queue{name = Name, map = Map0} = Q) ->
  28. case do_recover(file_name(Q)) of
  29. {ok, Terms} ->
  30. Map = maps:merge(Map0, Terms),
  31. Tree = maps:fold(
  32. fun(Key, {TreeKey, Value}, TreeAcc) ->
  33. gb_trees:enter(TreeKey, {Key, Value}, TreeAcc)
  34. end,
  35. gb_trees:empty(),
  36. Map
  37. ),
  38. #priority_queue{name = Name, map = Map, tree = Tree};
  39. error ->
  40. Q
  41. end.
  42. write_to_file(#priority_queue{map = Map} = Q) ->
  43. smoosh_utils:write_to_file(Map, file_name(Q), ?VSN).
  44. flush(#priority_queue{name = Name} = Q) ->
  45. Q#priority_queue{name = Name, map = maps:new(), tree = gb_trees:empty()}.
  46. last_updated(Key, #priority_queue{map = Map}) ->
  47. case maps:find(Key, Map) of
  48. {ok, {_Priority, {LastUpdatedMTime, _MInt}}} ->
  49. LastUpdatedMTime;
  50. error ->
  51. false
  52. end.
  53. is_key(Key, #priority_queue{map = Map}) ->
  54. maps:is_key(Key, Map).
  55. in(Key, Value, Priority, Q) ->
  56. in(Key, Value, Priority, infinity, Q).
  57. in(Key, Value, Priority, Capacity, #priority_queue{name = Name, map = Map, tree = Tree}) ->
  58. Tree1 =
  59. case maps:find(Key, Map) of
  60. {ok, TreeKey} ->
  61. gb_trees:delete_any(TreeKey, Tree);
  62. error ->
  63. Tree
  64. end,
  65. Now = {erlang:monotonic_time(), erlang:unique_integer([monotonic])},
  66. TreeKey1 = {Priority, Now},
  67. Tree2 = gb_trees:enter(TreeKey1, {Key, Value}, Tree1),
  68. Map1 = maps:put(Key, TreeKey1, Map),
  69. truncate(Capacity, #priority_queue{name = Name, map = Map1, tree = Tree2}).
  70. out(#priority_queue{name = Name, map = Map, tree = Tree}) ->
  71. case gb_trees:is_empty(Tree) of
  72. true ->
  73. false;
  74. false ->
  75. {_, {Key, Value}, Tree1} = gb_trees:take_largest(Tree),
  76. Map1 = maps:remove(Key, Map),
  77. Q = #priority_queue{name = Name, map = Map1, tree = Tree1},
  78. {Key, Value, Q}
  79. end.
  80. size(#priority_queue{tree = Tree}) ->
  81. gb_trees:size(Tree).
  82. info(#priority_queue{tree = Tree} = Q) ->
  83. [
  84. {size, ?MODULE:size(Q)}
  85. | case gb_trees:is_empty(Tree) of
  86. true ->
  87. [];
  88. false ->
  89. {Min, _, _} = gb_trees:take_smallest(Tree),
  90. {Max, _, _} = gb_trees:take_largest(Tree),
  91. [{min, Min}, {max, Max}]
  92. end
  93. ].
  94. from_list(Orddict, #priority_queue{name = Name}) ->
  95. Map = maps:from_list(Orddict),
  96. Tree = gb_trees:from_orddict(Orddict),
  97. #priority_queue{name = Name, map = Map, tree = Tree}.
  98. to_list(#priority_queue{tree = Tree}) ->
  99. gb_trees:to_list(Tree).
  100. is_empty(#priority_queue{tree = Tree}) ->
  101. gb_trees:is_empty(Tree).
  102. file_name(#priority_queue{name = Name}) ->
  103. filename:join(config:get("smoosh", "state_dir", "."), Name ++ ".waiting").
  104. truncate(infinity, Q) ->
  105. Q;
  106. truncate(Capacity, Q) when Capacity > 0 ->
  107. truncate(Capacity, ?MODULE:size(Q), Q).
  108. truncate(Capacity, Size, Q) when Size =< Capacity ->
  109. Q;
  110. truncate(Capacity, Size, #priority_queue{name = Name, map = Map, tree = Tree}) when Size > 0 ->
  111. {_, {Key, _}, Tree1} = gb_trees:take_smallest(Tree),
  112. Q1 = #priority_queue{name = Name, map = maps:remove(Key, Map), tree = Tree1},
  113. truncate(Capacity, ?MODULE:size(Q1), Q1).
  114. do_recover(FilePath) ->
  115. case file:read_file(FilePath) of
  116. {ok, Content} ->
  117. <<Vsn, Binary/binary>> = Content,
  118. try parse_queue(Vsn, ?VSN, Binary) of
  119. Bin ->
  120. Level = smoosh_utils:log_level("compaction_log_level", "debug"),
  121. couch_log:Level(
  122. "~p Successfully restored state file ~s", [?MODULE, FilePath]
  123. ),
  124. {ok, Bin}
  125. catch
  126. error:Reason ->
  127. couch_log:error(
  128. "~p Invalid queue file (~p). Deleting ~s", [?MODULE, Reason, FilePath]
  129. ),
  130. file:delete(FilePath),
  131. error
  132. end;
  133. {error, enoent} ->
  134. Level = smoosh_utils:log_level("compaction_log_level", "debug"),
  135. couch_log:Level(
  136. "~p (~p) Queue file ~s does not exist. Not restoring.", [?MODULE, enoent, FilePath]
  137. ),
  138. error;
  139. {error, Reason} ->
  140. couch_log:error(
  141. "~p Cannot read the queue file (~p). Deleting ~s", [?MODULE, Reason, FilePath]
  142. ),
  143. file:delete(FilePath),
  144. error
  145. end.
  146. parse_queue(1, ?VSN, Binary) ->
  147. erlang:binary_to_term(Binary, [safe]);
  148. parse_queue(Vsn, ?VSN, _) ->
  149. error({unsupported_version, Vsn}).