/lib/stdlib/test/erl_scan_SUITE.erl

https://github.com/bsmr-erlang/otp · Erlang · 1442 lines · 1166 code · 191 blank · 85 comment · 30 complexity · 56e5f28ad59df1cb626c9edb0e5f9c30 MD5 · raw file 

    

  1. %%
    2. 

  2. %% %CopyrightBegin%
    3. 

  3. %%
    4. 

  4. %% Copyright Ericsson AB 1998-2017. All Rights Reserved.
    5. 

  5. %%
    6. 

  6. %% Licensed under the Apache License, Version 2.0 (the "License");
    7. 

  7. %% you may not use this file except in compliance with the License.
    8. 

  8. %% You may obtain a copy of the License at
    9. 

  9. %%
    10. 

  10. %%     http://www.apache.org/licenses/LICENSE-2.0
    11. 

  11. %%
    12. 

  12. %% Unless required by applicable law or agreed to in writing, software
    13. 

  13. %% distributed under the License is distributed on an "AS IS" BASIS,
    14. 

  14. %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    15. 

  15. %% See the License for the specific language governing permissions and
    16. 

  16. %% limitations under the License.
    17. 

  17. %%
    18. 

  18. %% %CopyrightEnd%
    19. 

  19. 

  20. -module(erl_scan_SUITE).
    21. 

  21. -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1,
    22. 

  22. 	 init_per_testcase/2, end_per_testcase/2,
    23. 

  23. 	 init_per_group/2,end_per_group/2]).
    24. 

  24. 

  25. -export([error_1/1, error_2/1, iso88591/1, otp_7810/1, otp_10302/1,
    26. 

  26. 	 otp_10990/1, otp_10992/1, otp_11807/1]).
    27. 

  27. 

  28. -import(lists, [nth/2,flatten/1]).
    29. 

  29. -import(io_lib, [print/1]).
    30. 

  30. 

  31. %%
    32. 

  32. %% Define to run outside of test server
    33. 

  33. %%
    34. 

  34. %%-define(STANDALONE,1).
    35. 

  35. 

  36. -ifdef(STANDALONE).
    37. 

  37. -compile(export_all).
    38. 

  38. -define(line, put(line, ?LINE), ).
    39. 

  39. -define(config(A,B),config(A,B)).
    40. 

  40. -define(t, test_server).
    41. 

  41. %% config(priv_dir, _) ->
    42. 

  42. %%     ".";
    43. 

  43. %% config(data_dir, _) ->
    44. 

  44. %%     ".".
    45. 

  45. -else.
    46. 

  46. -include_lib("common_test/include/ct.hrl").
    47. 

  47. -endif.
    48. 

  48. 

  49. init_per_testcase(_Case, Config) ->
    50. 

  50.     Config.
    51. 

  51. 

  52. end_per_testcase(_Case, _Config) ->
    53. 

  53.     ok.
    54. 

  54. 

  55. suite() ->
    56. 

  56.     [{ct_hooks,[ts_install_cth]},
    57. 

  57.      {timetrap,{minutes,20}}].
    58. 

  58. 

  59. all() -> 
    60. 

  60.     [{group, error}, iso88591, otp_7810, otp_10302, otp_10990, otp_10992,
    61. 

  61.      otp_11807].
    62. 

  62. 

  63. groups() -> 
    64. 

  64.     [{error, [], [error_1, error_2]}].
    65. 

  65. 

  66. init_per_suite(Config) ->
    67. 

  67.     Config.
    68. 

  68. 

  69. end_per_suite(_Config) ->
    70. 

  70.     ok.
    71. 

  71. 

  72. init_per_group(_GroupName, Config) ->
    73. 

  73.     Config.
    74. 

  74. 

  75. end_per_group(_GroupName, Config) ->
    76. 

  76.     Config.
    77. 

  77. 

  78. 

  79. 

  80. %% (OTP-2347)
    81. 

  81. error_1(Config) when is_list(Config) ->
    82. 

  82.     {error, _, _} = erl_scan:string("'a"),
    83. 

  83.     ok.
    84. 

  84. 

  85. %% Checks that format_error works on the error cases.
    86. 

  86. error_2(Config) when is_list(Config) ->
    87. 

  87.     lists:foreach(fun check/1, error_cases()),
    88. 

  88.     ok.
    89. 

  89. 

  90. error_cases() ->
    91. 

  91.     ["'a",
    92. 

  92.      "\"a",
    93. 

  93.      "'\\",
    94. 

  94.      "\"\\",
    95. 

  95.      "$",
    96. 

  96.      "$\\",
    97. 

  97.      "2.3e",
    98. 

  98.      "2.3e-",
    99. 

  99.      "91#9"
    100. 

  100.     ].
    101. 

  101. 

  102. assert_type(N, integer) when is_integer(N) ->
    103. 

  103.     ok;
    104. 

  104. assert_type(N, atom) when is_atom(N) ->
    105. 

  105.     ok.
    106. 

  106. 

  107. check(String) ->
    108. 

  108.     Error = erl_scan:string(String),
    109. 

  109.     check_error(Error, erl_scan).
    110. 

  110. 

  111. %%% (This should be useful for all format_error functions.)
    112. 

  112. check_error({error, Info, EndLine}, Module0) ->
    113. 

  113.     {ErrorLine, Module, Desc} = Info,
    114. 

  114.     true = (Module == Module0),
    115. 

  115.     assert_type(EndLine, integer),
    116. 

  116.     assert_type(ErrorLine, integer),
    117. 

  117.     true = (ErrorLine =< EndLine),
    118. 

  118.     String = lists:flatten(Module0:format_error(Desc)),
    119. 

  119.     true = io_lib:printable_list(String).
    120. 

  120. 

  121. %% Tests the support for ISO-8859-1 i.e Latin-1.
    122. 

  122. iso88591(Config) when is_list(Config) ->
    123. 

  123.     ok =
    124. 

  124. 	case catch begin
    125. 

  125. 		       %% Some atom and variable names
    126. 

  126. 		       V1s = [$Á,$á,$é,$ë],
    127. 

  127. 		       V2s = [$N,$ä,$r],
    128. 

  128. 		       A1s = [$h,$ä,$r],
    129. 

  129. 		       A2s = [$ö,$r,$e],
    130. 

  130. 		       %% Test parsing atom and variable characters.
    131. 

  131. 		       {ok,Ts1,_} = erl_scan_string(V1s ++ " " ++ V2s ++
    132. 

  132. 							"\327" ++
    133. 

  133. 							A1s ++ " " ++ A2s),
    134. 

  134. 		       V1s = atom_to_list(element(3, nth(1, Ts1))),
    135. 

  135. 		       V2s = atom_to_list(element(3, nth(2, Ts1))),
    136. 

  136. 		       A1s = atom_to_list(element(3, nth(4, Ts1))),
    137. 

  137. 		       A2s = atom_to_list(element(3, nth(5, Ts1))),
    138. 

  138. 		       %% Test printing atoms
    139. 

  139. 		       A1s = flatten(print(element(3, nth(4, Ts1)))),
    140. 

  140. 		       A2s = flatten(print(element(3, nth(5, Ts1)))),
    141. 

  141. 		       %% Test parsing and printing strings.
    142. 

  142. 		       S1 = V1s ++ "\327" ++ A1s ++ "\250" ++ A2s,
    143. 

  143. 		       S1s = "\"" ++ S1 ++ "\"",
    144. 

  144. 		       {ok,Ts2,_} = erl_scan_string(S1s),
    145. 

  145. 		       S1 = element(3, nth(1, Ts2)),
    146. 

  146. 		       S1s = flatten(print(element(3, nth(1, Ts2)))),
    147. 

  147. 		       ok				%It all worked
    148. 

  148. 		   end of
    149. 

  149. 	    {'EXIT',R} ->				%Something went wrong!
    150. 

  150. 		{error,R};
    151. 

  151. 	    ok -> ok				%Aok
    152. 

  152. 	end.
    153. 

  153. 

  154. %% OTP-7810. White spaces, comments, and more...
    155. 

  155. otp_7810(Config) when is_list(Config) ->
    156. 

  156.     ok = reserved_words(),
    157. 

  157.     ok = atoms(),
    158. 

  158.     ok = punctuations(),
    159. 

  159.     ok = comments(),
    160. 

  160.     ok = errors(),
    161. 

  161.     ok = integers(),
    162. 

  162.     ok = base_integers(),
    163. 

  163.     ok = floats(),
    164. 

  164.     ok = dots(),
    165. 

  165.     ok = chars(),
    166. 

  166.     ok = variables(),
    167. 

  167.     ok = eof(),
    168. 

  168.     ok = illegal(),
    169. 

  169.     ok = crashes(),
    170. 

  170. 

  171.     ok = options(),
    172. 

  172.     ok = token_info(),
    173. 

  173.     ok = column_errors(),
    174. 

  174.     ok = white_spaces(),
    175. 

  175. 

  176.     ok = unicode(),
    177. 

  177. 

  178.     ok = more_chars(),
    179. 

  179.     ok = more_options(),
    180. 

  180.     ok = anno_info(),
    181. 

  181. 

  182.     ok.
    183. 

  183. 

  184. reserved_words() ->
    185. 

  185.     L = ['after', 'begin', 'case', 'try', 'cond', 'catch',
    186. 

  186.          'andalso', 'orelse', 'end', 'fun', 'if', 'let', 'of',
    187. 

  187.          'receive', 'when', 'bnot', 'not', 'div',
    188. 

  188.          'rem', 'band', 'and', 'bor', 'bxor', 'bsl', 'bsr',
    189. 

  189.          'or', 'xor'],
    190. 

  190.     [begin
    191. 

  191.          {RW, true} = {RW, erl_scan:reserved_word(RW)},
    192. 

  192.          S = atom_to_list(RW),
    193. 

  193.          Ts = [{RW,{1,1}}],
    194. 

  194.          test_string(S, Ts)
    195. 

  195.      end || RW <- L],
    196. 

  196.     ok.
    197. 

  197. 

  198. 

  199. atoms() ->
    200. 

  200.     test_string("a
    201. 

  201.                  b", [{atom,{1,1},a},{atom,{2,18},b}]),
    202. 

  202.     test_string("'a b'", [{atom,{1,1},'a b'}]),
    203. 

  203. 		test_string("a", [{atom,{1,1},a}]),
    204. 

  204. 		test_string("a@2", [{atom,{1,1},a@2}]),
    205. 

  205. 		test_string([39,65,200,39], [{atom,{1,1},'AÈ'}]),
    206. 

  206. 		test_string("ärlig östen", [{atom,{1,1},ärlig},{atom,{1,7},östen}]),
    207. 

  207. 		{ok,[{atom,_,'$a'}],{1,6}} =
    208. 

  208. 		    erl_scan_string("'$\\a'", {1,1}),
    209. 

  209. 		test("'$\\a'"),
    210. 

  210. 		ok.
    211. 

  211. 

  212. punctuations() ->
    213. 

  213.     L = ["<<", "<-", "<=", "<", ">>", ">=", ">", "->", "--",
    214. 

  214.          "-", "++", "+", "=:=", "=/=", "=<", "=>", "==", "=", "/=",
    215. 

  215.          "/", "||", "|", ":=", "::", ":"],
    216. 

  216.     %% One token at a time:
    217. 

  217.     [begin
    218. 

  218.          W = list_to_atom(S),
    219. 

  219.          Ts = [{W,{1,1}}],
    220. 

  220.          test_string(S, Ts)
    221. 

  221.      end || S <- L],
    222. 

  222.     Three = ["/=:=", "<=:=", "==:=", ">=:="], % three tokens...
    223. 

  223.     No = Three ++ L,
    224. 

  224.     SL0 = [{S1++S2,{-length(S1),S1,S2}} ||
    225. 

  225.               S1 <- L,
    226. 

  226.               S2 <- L,
    227. 

  227.               not lists:member(S1++S2, No)],
    228. 

  228.     SL = family_list(SL0),
    229. 

  229.     %% Two tokens. When there are several answers, the one with
    230. 

  230.     %% the longest first token is chosen:
    231. 

  231.     %% [the special case "=<<" is among the tested ones]
    232. 

  232.     [begin
    233. 

  233.          W1 = list_to_atom(S1),
    234. 

  234.          W2 = list_to_atom(S2),
    235. 

  235.          Ts = [{W1,{1,1}},{W2,{1,-L2+1}}],
    236. 

  236.          test_string(S, Ts)
    237. 

  237.      end || {S,[{L2,S1,S2}|_]}  <- SL],
    238. 

  238. 

  239.     PTs1 = [{'!',{1,1}},{'(',{1,2}},{')',{1,3}},{',',{1,4}},{';',{1,5}},
    240. 

  240.             {'=',{1,6}},{'[',{1,7}},{']',{1,8}},{'{',{1,9}},{'|',{1,10}},
    241. 

  241.             {'}',{1,11}}],
    242. 

  242.     test_string("!(),;=[]{|}", PTs1),
    243. 

  243. 

  244.     PTs2 = [{'#',{1,1}},{'&',{1,2}},{'*',{1,3}},{'+',{1,4}},{'/',{1,5}},
    245. 

  245.             {':',{1,6}},{'<',{1,7}},{'>',{1,8}},{'?',{1,9}},{'@',{1,10}},
    246. 

  246.             {'\\',{1,11}},{'^',{1,12}},{'`',{1,13}},{'~',{1,14}}],
    247. 

  247.     test_string("#&*+/:<>?@\\^`~", PTs2),
    248. 

  248. 

  249.     test_string(".. ", [{'..',{1,1}}]),
    250. 

  250.     test_string("1 .. 2",
    251. 

  251.                 [{integer,{1,1},1},{'..',{1,3}},{integer,{1,6},2}]),
    252. 

  252.     test_string("...", [{'...',{1,1}}]),
    253. 

  253.     ok.
    254. 

  254. 

  255. comments() ->
    256. 

  256.     test("a %%\n b"),
    257. 

  257.     {ok,[],1} = erl_scan_string("%"),
    258. 

  258.     test("a %%\n b"),
    259. 

  259.     {ok,[{atom,{1,1},a},{atom,{2,2},b}],{2,3}} =
    260. 

  260.         erl_scan_string("a %%\n b", {1,1}),
    261. 

  261.     {ok,[{atom,{1,1},a},{comment,{1,3},"%%"},{atom,{2,2},b}],{2,3}} =
    262. 

  262.         erl_scan_string("a %%\n b",{1,1}, [return_comments]),
    263. 

  263.     {ok,[{atom,{1,1},a},
    264. 

  264.          {white_space,{1,2}," "},
    265. 

  265.          {white_space,{1,5},"\n "},
    266. 

  266.          {atom,{2,2},b}],
    267. 

  267.      {2,3}} =
    268. 

  268.         erl_scan_string("a %%\n b",{1,1},[return_white_spaces]),
    269. 

  269.     {ok,[{atom,{1,1},a},
    270. 

  270.          {white_space,{1,2}," "},
    271. 

  271.          {comment,{1,3},"%%"},
    272. 

  272.          {white_space,{1,5},"\n "},
    273. 

  273.          {atom,{2,2},b}],
    274. 

  274.      {2,3}} = erl_scan_string("a %%\n b",{1,1},[return]),
    275. 

  275.     ok.
    276. 

  276. 

  277. errors() ->
    278. 

  278.     {error,{1,erl_scan,{string,$',"qa"}},1} = erl_scan:string("'qa"), %'
    279. 

  279.     {error,{{1,1},erl_scan,{string,$',"qa"}},{1,4}} = %'
    280. 

  280.         erl_scan:string("'qa", {1,1}, []), %'
    281. 

  281.     {error,{1,erl_scan,{string,$","str"}},1} = %"
    282. 

  282.         erl_scan:string("\"str"), %"
    283. 

  283.     {error,{{1,1},erl_scan,{string,$","str"}},{1,5}} = %"
    284. 

  284.         erl_scan:string("\"str", {1,1}, []), %"
    285. 

  285.     {error,{1,erl_scan,char},1} = erl_scan:string("$"),
    286. 

  286.     {error,{{1,1},erl_scan,char},{1,2}} = erl_scan:string("$", {1,1}, []),
    287. 

  287.     test_string([34,65,200,34], [{string,{1,1},"AÈ"}]),
    288. 

  288.     test_string("\\", [{'\\',{1,1}}]),
    289. 

  289.     {'EXIT',_} =
    290. 

  290.         (catch {foo, erl_scan:string('$\\a', {1,1})}), % type error
    291. 

  291.     {'EXIT',_} =
    292. 

  292.         (catch {foo, erl_scan:tokens([], '$\\a', {1,1})}), % type error
    293. 

  293. 

  294.     "{a,tuple}" = erl_scan:format_error({a,tuple}),
    295. 

  295.     ok.
    296. 

  296. 

  297. integers() ->
    298. 

  298.     [begin
    299. 

  299.          I = list_to_integer(S),
    300. 

  300.          Ts = [{integer,{1,1},I}],
    301. 

  301.          test_string(S, Ts)
    302. 

  302.      end || S <- [[N] || N <- lists:seq($0, $9)] ++ ["2323","000"] ],
    303. 

  303.     ok.
    304. 

  304. 

  305. base_integers() ->
    306. 

  306.     [begin
    307. 

  307.          B = list_to_integer(BS),
    308. 

  308.          I = erlang:list_to_integer(S, B),
    309. 

  309.          Ts = [{integer,{1,1},I}],
    310. 

  310.          test_string(BS++"#"++S, Ts)
    311. 

  311.      end || {BS,S} <- [{"2","11"}, {"5","23234"}, {"12","05a"},
    312. 

  312.                        {"16","abcdef"}, {"16","ABCDEF"}] ],
    313. 

  313. 

  314.     {error,{1,erl_scan,{base,1}},1} = erl_scan:string("1#000"),
    315. 

  315.     {error,{{1,1},erl_scan,{base,1}},{1,2}} =
    316. 

  316.         erl_scan:string("1#000", {1,1}, []),
    317. 

  317. 

  318.     test_string("12#bc", [{integer,{1,1},11},{atom,{1,5},c}]),
    319. 

  319. 

  320.     [begin
    321. 

  321.          Str = BS ++ "#" ++ S,
    322. 

  322.          {error,{1,erl_scan,{illegal,integer}},1} =
    323. 

  323.              erl_scan:string(Str)
    324. 

  324.      end || {BS,S} <- [{"3","3"},{"15","f"}, {"12","c"}] ],
    325. 

  325. 

  326.     {ok,[{integer,1,239},{'@',1}],1} = erl_scan_string("16#ef@"),
    327. 

  327.     {ok,[{integer,{1,1},239},{'@',{1,6}}],{1,7}} =
    328. 

  328.         erl_scan_string("16#ef@", {1,1}, []),
    329. 

  329.     {ok,[{integer,{1,1},14},{atom,{1,5},g@}],{1,7}} =
    330. 

  330.         erl_scan_string("16#eg@", {1,1}, []),
    331. 

  331. 

  332.     ok.
    333. 

  333. 

  334. floats() ->
    335. 

  335.     [begin
    336. 

  336.          F = list_to_float(FS),
    337. 

  337.          Ts = [{float,{1,1},F}],
    338. 

  338.          test_string(FS, Ts)
    339. 

  339.      end || FS <- ["1.0","001.17","3.31200","1.0e0","1.0E17",
    340. 

  340.                    "34.21E-18", "17.0E+14"]],
    341. 

  341.     test_string("1.e2", [{integer,{1,1},1},{'.',{1,2}},{atom,{1,3},e2}]),
    342. 

  342. 

  343.     {error,{1,erl_scan,{illegal,float}},1} =
    344. 

  344.         erl_scan:string("1.0e400"),
    345. 

  345.     {error,{{1,1},erl_scan,{illegal,float}},{1,8}} =
    346. 

  346.         erl_scan:string("1.0e400", {1,1}, []),
    347. 

  347.     [begin
    348. 

  348.          {error,{1,erl_scan,{illegal,float}},1} = erl_scan:string(S),
    349. 

  349.          {error,{{1,1},erl_scan,{illegal,float}},{1,_}} =
    350. 

  350.              erl_scan:string(S, {1,1}, [])
    351. 

  351.      end || S <- ["1.14Ea"]],
    352. 

  352. 

  353.     ok.
    354. 

  354. 

  355. dots() ->
    356. 

  356.     Dot = [{".",    {ok,[{dot,1}],1}, {ok,[{dot,{1,1}}],{1,2}}},
    357. 

  357.            {". ",   {ok,[{dot,1}],1}, {ok,[{dot,{1,1}}],{1,3}}},
    358. 

  358.            {".\n",  {ok,[{dot,1}],2}, {ok,[{dot,{1,1}}],{2,1}}},
    359. 

  359.            {".%",   {ok,[{dot,1}],1}, {ok,[{dot,{1,1}}],{1,3}}},
    360. 

  360.            {".\210",{ok,[{dot,1}],1}, {ok,[{dot,{1,1}}],{1,3}}},
    361. 

  361.            {".% öh",{ok,[{dot,1}],1}, {ok,[{dot,{1,1}}],{1,6}}},
    362. 

  362.            {".%\n", {ok,[{dot,1}],2}, {ok,[{dot,{1,1}}],{2,1}}},
    363. 

  363.            {".$",   {error,{1,erl_scan,char},1},
    364. 

  364. 	    {error,{{1,2},erl_scan,char},{1,3}}},
    365. 

  365.            {".$\\", {error,{1,erl_scan,char},1},
    366. 

  366.                     {error,{{1,2},erl_scan,char},{1,4}}},
    367. 

  367.            {".a",   {ok,[{'.',1},{atom,1,a}],1},
    368. 

  368. 	    {ok,[{'.',{1,1}},{atom,{1,2},a}],{1,3}}}
    369. 

  369.           ],
    370. 

  370.     [begin
    371. 

  371.          R = erl_scan_string(S),
    372. 

  372.          R2 = erl_scan_string(S, {1,1}, [])
    373. 

  373.      end || {S, R, R2} <- Dot],
    374. 

  374. 

  375.     {ok,[{dot,_}=T1],{1,2}} = erl_scan:string(".", {1,1}, text),
    376. 

  376.     [1, 1, "."] = token_info(T1),
    377. 

  377.     {ok,[{dot,_}=T2],{1,3}} = erl_scan:string(".%", {1,1}, text),
    378. 

  378.     [1, 1, "."] = token_info(T2),
    379. 

  379.     {ok,[{dot,_}=T3],{1,6}} =
    380. 

  380.         erl_scan:string(".% öh", {1,1}, text),
    381. 

  381.     [1, 1, "."] = token_info(T3),
    382. 

  382.     {error,{{1,2},erl_scan,char},{1,3}} = erl_scan:string(".$", {1,1}),
    383. 

  383.     {error,{{1,2},erl_scan,char},{1,4}} = erl_scan:string(".$\\", {1,1}),
    384. 

  384. 

  385.     test_string(". ", [{dot,{1,1}}]),
    386. 

  386.     test_string(".  ", [{dot,{1,1}}]),
    387. 

  387.     test_string(".\n", [{dot,{1,1}}]),
    388. 

  388.     test_string(".\n\n", [{dot,{1,1}}]),
    389. 

  389.     test_string(".\n\r", [{dot,{1,1}}]),
    390. 

  390.     test_string(".\n\n\n", [{dot,{1,1}}]),
    391. 

  391.     test_string(".\210", [{dot,{1,1}}]),
    392. 

  392.     test_string(".%\n", [{dot,{1,1}}]),
    393. 

  393.     test_string(".a", [{'.',{1,1}},{atom,{1,2},a}]),
    394. 

  394. 

  395.     test_string("%. \n. ", [{dot,{2,1}}]),
    396. 

  396.     {more,C} = erl_scan:tokens([], "%. ",{1,1}, return),
    397. 

  397.     {done,{ok,[{comment,{1,1},"%. "},
    398. 

  398.                {white_space,{1,4},"\n"},
    399. 

  399.                {dot,{2,1}}],
    400. 

  400.            {2,3}}, ""} =
    401. 

  401.         erl_scan_tokens(C, "\n. ", {1,1}, return), % any loc, any options
    402. 

  402. 

  403.     [test_string(S, R) ||
    404. 

  404.         {S, R} <- [{".$\n",   [{'.',{1,1}},{char,{1,2},$\n}]},
    405. 

  405.                    {"$\\\n",  [{char,{1,1},$\n}]},
    406. 

  406.                    {"'\\\n'", [{atom,{1,1},'\n'}]},
    407. 

  407.                    {"$\n",    [{char,{1,1},$\n}]}] ],
    408. 

  408.     ok.
    409. 

  409. 

  410. chars() ->
    411. 

  411.     [begin
    412. 

  412.          L = lists:flatten(io_lib:format("$\\~.8b", [C])),
    413. 

  413.          Ts = [{char,{1,1},C}],
    414. 

  414.          test_string(L, Ts)
    415. 

  415.      end || C <- lists:seq(0, 255)],
    416. 

  416. 

  417.     %% Leading zeroes...
    418. 

  418.     [begin
    419. 

  419.          L = lists:flatten(io_lib:format("$\\~3.8.0b", [C])),
    420. 

  420.          Ts = [{char,{1,1},C}],
    421. 

  421.          test_string(L, Ts)
    422. 

  422.      end || C <- lists:seq(0, 255)],
    423. 

  423. 

  424.     %% $\^\n now increments the line...
    425. 

  425.     [begin
    426. 

  426.          L = "$\\^" ++ [C],
    427. 

  427.          Ts = [{char,{1,1},C band 2#11111}],
    428. 

  428.          test_string(L, Ts)
    429. 

  429.      end || C <- lists:seq(0, 255)],
    430. 

  430. 

  431.     [begin
    432. 

  432.          L = "$\\" ++ [C],
    433. 

  433.          Ts = [{char,{1,1},V}],
    434. 

  434.          test_string(L, Ts)
    435. 

  435.      end || {C,V} <- [{$n,$\n}, {$r,$\r}, {$t,$\t}, {$v,$\v},
    436. 

  436.                       {$b,$\b}, {$f,$\f}, {$e,$\e}, {$s,$\s},
    437. 

  437.                       {$d,$\d}]],
    438. 

  438. 

  439.     EC = [$\n,$\r,$\t,$\v,$\b,$\f,$\e,$\s,$\d],
    440. 

  440.     Ds = lists:seq($0, $9),
    441. 

  441.     X = [$^,$n,$r,$t,$v,$b,$f,$e,$s,$d],
    442. 

  442.     New = [${,$x],
    443. 

  443.     No = EC ++ Ds ++ X ++ New,
    444. 

  444.     [begin
    445. 

  445.          L = "$\\" ++ [C],
    446. 

  446.          Ts = [{char,{1,1},C}],
    447. 

  447.          test_string(L, Ts)
    448. 

  448.      end || C <- lists:seq(0, 255) -- No],
    449. 

  449. 

  450.     [begin
    451. 

  451.          L = "'$\\" ++ [C] ++ "'",
    452. 

  452.          Ts = [{atom,{1,1},list_to_atom("$"++[C])}],
    453. 

  453.          test_string(L, Ts)
    454. 

  454.      end || C <- lists:seq(0, 255) -- No],
    455. 

  455. 

  456.     test_string("\"\\013a\\\n\"", [{string,{1,1},"\va\n"}]),
    457. 

  457. 

  458.     test_string("'\n'", [{atom,{1,1},'\n'}]),
    459. 

  459.     test_string("\"\n\a\"", [{string,{1,1},"\na"}]),
    460. 

  460. 

  461.     %% No escape
    462. 

  462.     [begin
    463. 

  463.          L = "$" ++ [C],
    464. 

  464.          Ts = [{char,{1,1},C}],
    465. 

  465.          test_string(L, Ts)
    466. 

  466.      end || C <- lists:seq(0, 255) -- (No ++ [$\\])],
    467. 

  467.     test_string("$\n", [{char,{1,1},$\n}]),
    468. 

  468. 

  469.     {error,{{1,1},erl_scan,char},{1,4}} =
    470. 

  470.         erl_scan:string("$\\^",{1,1}),
    471. 

  471.     test_string("$\\\n", [{char,{1,1},$\n}]),
    472. 

  472.     %% Robert's scanner returns line 1:
    473. 

  473.     test_string("$\\\n", [{char,{1,1},$\n}]),
    474. 

  474.     test_string("$\n\n", [{char,{1,1},$\n}]),
    475. 

  475.     test("$\n\n"),
    476. 

  476.     ok.
    477. 

  477. 

  478. 

  479. variables() ->
    480. 

  480.     test_string("     \237_Aouåeiyäö", [{var,{1,7},'_Aouåeiyäö'}]),
    481. 

  481.     test_string("A_b_c@", [{var,{1,1},'A_b_c@'}]),
    482. 

  482.     test_string("V@2", [{var,{1,1},'V@2'}]),
    483. 

  483.     test_string("ABDÀ", [{var,{1,1},'ABDÀ'}]),
    484. 

  484.     test_string("Ärlig Östen", [{var,{1,1},'Ärlig'},{var,{1,7},'Östen'}]),
    485. 

  485.     ok.
    486. 

  486. 

  487. eof() ->
    488. 

  488.     {done,{eof,1},eof} = erl_scan:tokens([], eof, 1),
    489. 

  489.     {more, C1} = erl_scan:tokens([],"    \n", 1),
    490. 

  490.     {done,{eof,2},eof} = erl_scan:tokens(C1, eof, 1),
    491. 

  491.     {more, C2} = erl_scan:tokens([], "abra", 1),
    492. 

  492.     %% An error before R13A.
    493. 

  493.     %% {done,Err={error,{1,erl_scan,scan},1},eof} =
    494. 

  494.     {done,{ok,[{atom,1,abra}],1},eof} =
    495. 

  495.         erl_scan_tokens(C2, eof, 1),
    496. 

  496. 

  497.     %% With column.
    498. 

  498.     {more, C3} = erl_scan:tokens([],"    \n",{1,1}),
    499. 

  499.     {done,{eof,{2,1}},eof} = erl_scan:tokens(C3, eof, 1),
    500. 

  500.     {more, C4} = erl_scan:tokens([], "abra", {1,1}),
    501. 

  501.     %% An error before R13A.
    502. 

  502.     %% {done,{error,{{1,1},erl_scan,scan},{1,5}},eof} =
    503. 

  503.     {done,{ok,[{atom,_,abra}],{1,5}},eof} =
    504. 

  504.         erl_scan_tokens(C4, eof, 1),
    505. 

  505. 

  506.     %% Robert's scanner returns "" as LeftoverChars;
    507. 

  507.     %% the R12B scanner returns eof as LeftoverChars: (eof is correct)
    508. 

  508.     {more, C5} = erl_scan:tokens([], "a", 1),
    509. 

  509.     %% An error before R13A.
    510. 

  510.     %% {done,{error,{1,erl_scan,scan},1},eof} =
    511. 

  511.     {done,{ok,[{atom,1,a}],1},eof} =
    512. 

  512.         erl_scan_tokens(C5,eof,1),
    513. 

  513. 

  514.     %% With column.
    515. 

  515.     {more, C6} = erl_scan:tokens([], "a", {1,1}),
    516. 

  516.     %% An error before R13A.
    517. 

  517.     %% {done,{error,{1,erl_scan,scan},1},eof} =
    518. 

  518.     {done,{ok,[{atom,{1,1},a}],{1,2}},eof} =
    519. 

  519.         erl_scan_tokens(C6,eof,1),
    520. 

  520. 

  521.     %% A dot followed by eof is special:
    522. 

  522.     {more, C} = erl_scan:tokens([], "a.", 1),
    523. 

  523.     {done,{ok,[{atom,1,a},{dot,1}],1},eof} = erl_scan_tokens(C,eof,1),
    524. 

  524.     {ok,[{atom,1,foo},{dot,1}],1} = erl_scan_string("foo."),
    525. 

  525. 

  526.     %% With column.
    527. 

  527.     {more, CCol} = erl_scan:tokens([], "a.", {1,1}),
    528. 

  528.     {done,{ok,[{atom,{1,1},a},{dot,{1,2}}],{1,3}},eof} =
    529. 

  529.         erl_scan_tokens(CCol,eof,1),
    530. 

  530.     {ok,[{atom,{1,1},foo},{dot,{1,4}}],{1,5}} =
    531. 

  531.         erl_scan_string("foo.", {1,1}, []),
    532. 

  532. 

  533.     ok.
    534. 

  534. 

  535. illegal() ->
    536. 

  536.     Atom = lists:duplicate(1000, $a),
    537. 

  537.     {error,{1,erl_scan,{illegal,atom}},1} = erl_scan:string(Atom),
    538. 

  538.     {done,{error,{1,erl_scan,{illegal,atom}},1},". "} =
    539. 

  539.         erl_scan:tokens([], Atom++". ", 1),
    540. 

  540.     QAtom = "'" ++ Atom ++ "'",
    541. 

  541.     {error,{1,erl_scan,{illegal,atom}},1} = erl_scan:string(QAtom),
    542. 

  542.     {done,{error,{1,erl_scan,{illegal,atom}},1},". "} =
    543. 

  543.         erl_scan:tokens([], QAtom++". ", 1),
    544. 

  544.     Var = lists:duplicate(1000, $A),
    545. 

  545.     {error,{1,erl_scan,{illegal,var}},1} = erl_scan:string(Var),
    546. 

  546.     {done,{error,{1,erl_scan,{illegal,var}},1},". "} =
    547. 

  547.         erl_scan:tokens([], Var++". ", 1),
    548. 

  548.     Float = "1" ++ lists:duplicate(400, $0) ++ ".0",
    549. 

  549.     {error,{1,erl_scan,{illegal,float}},1} = erl_scan:string(Float),
    550. 

  550.     {done,{error,{1,erl_scan,{illegal,float}},1},". "} =
    551. 

  551.         erl_scan:tokens([], Float++". ", 1),
    552. 

  552.     String = "\"43\\x{aaaaaa}34\"",
    553. 

  553.     {error,{1,erl_scan,{illegal,character}},1} = erl_scan:string(String),
    554. 

  554.     {done,{error,{1,erl_scan,{illegal,character}},1},"34\". "} =
    555. 

  555.         %% Would be nice if `34\"' were skipped...
    556. 

  556.         %% Maybe, but then the LeftOverChars would not be the characters
    557. 

  557.         %% immediately following the end location of the error.
    558. 

  558.         erl_scan:tokens([], String++". ", 1),
    559. 

    560. 

  560.     {error,{{1,1},erl_scan,{illegal,atom}},{1,1001}} =
    561. 

  561.         erl_scan:string(Atom, {1,1}),
    562. 

  562.     {done,{error,{{1,5},erl_scan,{illegal,atom}},{1,1005}},". "} =
    563. 

  563.         erl_scan:tokens([], "foo "++Atom++". ", {1,1}),
    564. 

  564.     {error,{{1,1},erl_scan,{illegal,atom}},{1,1003}} =
    565. 

  565.         erl_scan:string(QAtom, {1,1}),
    566. 

  566.     {done,{error,{{1,5},erl_scan,{illegal,atom}},{1,1007}},". "} =
    567. 

  567.         erl_scan:tokens([], "foo "++QAtom++". ", {1,1}),
    568. 

  568.     {error,{{1,1},erl_scan,{illegal,var}},{1,1001}} =
    569. 

  569.         erl_scan:string(Var, {1,1}),
    570. 

  570.     {done,{error,{{1,5},erl_scan,{illegal,var}},{1,1005}},". "} =
    571. 

  571.         erl_scan:tokens([], "foo "++Var++". ", {1,1}),
    572. 

  572.     {error,{{1,1},erl_scan,{illegal,float}},{1,404}} =
    573. 

  573.         erl_scan:string(Float, {1,1}),
    574. 

  574.     {done,{error,{{1,5},erl_scan,{illegal,float}},{1,408}},". "} =
    575. 

  575.         erl_scan:tokens([], "foo "++Float++". ", {1,1}),
    576. 

  576.     {error,{{1,4},erl_scan,{illegal,character}},{1,14}} =
    577. 

  577.         erl_scan:string(String, {1,1}),
    578. 

  578.     {done,{error,{{1,4},erl_scan,{illegal,character}},{1,14}},"34\". "} =
    579. 

  579.         erl_scan:tokens([], String++". ", {1,1}),
    580. 

  580.     ok.
    581. 

    582. 

  582. crashes() ->
    583. 

  583.     {'EXIT',_} = (catch {foo, erl_scan:string([-1])}), % type error
    584. 

  584.     {'EXIT',_} = (catch {foo, erl_scan:string("$"++[-1])}),
    585. 

  585.     {'EXIT',_} = (catch {foo, erl_scan:string("$\\"++[-1])}),
    586. 

  586.     {'EXIT',_} = (catch {foo, erl_scan:string("$\\^"++[-1])}),
    587. 

  587.     {'EXIT',_} = (catch {foo, erl_scan:string([$",-1,$"],{1,1})}),
    588. 

  588.     {'EXIT',_} = (catch {foo, erl_scan:string("\"\\v"++[-1,$"])}), %$"
    589. 

  589.     {'EXIT',_} = (catch {foo, erl_scan:string([$",-1,$"])}),
    590. 

  590.     {'EXIT',_} = (catch {foo, erl_scan:string("% foo"++[-1])}),
    591. 

  591.     {'EXIT',_} =
    592. 

  592.          (catch {foo, erl_scan:string("% foo"++[-1],{1,1})}),
    593. 

    594. 

  594.     {'EXIT',_} = (catch {foo, erl_scan:string([a])}), % type error
    595. 

  595.     {'EXIT',_} = (catch {foo, erl_scan:string("$"++[a])}),
    596. 

  596.     {'EXIT',_} = (catch {foo, erl_scan:string("$\\"++[a])}),
    597. 

  597.     {'EXIT',_} = (catch {foo, erl_scan:string("$\\^"++[a])}),
    598. 

  598.     {'EXIT',_} = (catch {foo, erl_scan:string([$",a,$"],{1,1})}),
    599. 

  599.     {'EXIT',_} = (catch {foo, erl_scan:string("\"\\v"++[a,$"])}), %$"
    600. 

  600.     {'EXIT',_} = (catch {foo, erl_scan:string([$",a,$"])}),
    601. 

  601.     {'EXIT',_} = (catch {foo, erl_scan:string("% foo"++[a])}),
    602. 

  602.     {'EXIT',_} =
    603. 

  603.          (catch {foo, erl_scan:string("% foo"++[a],{1,1})}),
    604. 

    605. 

  605.     {'EXIT',_} = (catch {foo, erl_scan:string([3.0])}), % type error
    606. 

    607. 

  607.     ok.
    608. 

    609. 

  609. options() ->
    610. 

  610.     %% line and column are not options, but tested here
    611. 

  611.     {ok,[{atom,1,foo},{white_space,1," "},{comment,1,"% bar"}], 1} =
    612. 

  612.         erl_scan_string("foo % bar", 1, return),
    613. 

  613.     {ok,[{atom,1,foo},{white_space,1," "}],1} =
    614. 

  614.         erl_scan_string("foo % bar", 1, return_white_spaces),
    615. 

  615.     {ok,[{atom,1,foo},{comment,1,"% bar"}],1} =
    616. 

  616.         erl_scan_string("foo % bar", 1, return_comments),
    617. 

  617.     {ok,[{atom,17,foo}],17} =
    618. 

  618.         erl_scan_string("foo % bar", 17),
    619. 

  619.     {'EXIT',{function_clause,_}} =
    620. 

  620.         (catch {foo,
    621. 

  621.                 erl_scan:string("foo % bar", {a,1}, [])}), % type error
    622. 

  622.     {ok,[{atom,_,foo}],{17,18}} =
    623. 

  623.         erl_scan_string("foo % bar", {17,9}, []),
    624. 

  624.     {'EXIT',{function_clause,_}} =
    625. 

  625.         (catch {foo,
    626. 

  626.                 erl_scan:string("foo % bar", {1,0}, [])}), % type error
    627. 

  627.     {ok,[{foo,1}],1} =
    628. 

  628.         erl_scan_string("foo % bar",1, [{reserved_word_fun,
    629. 

  629.                                          fun(W) -> W =:= foo end}]),
    630. 

  630.     {'EXIT',{badarg,_}} =
    631. 

  631.         (catch {foo,
    632. 

  632.                 erl_scan:string("foo % bar",1, % type error
    633. 

  633.                                 [{reserved_word_fun,
    634. 

  634.                                   fun(W,_) -> W =:= foo end}])}),
    635. 

  635.     ok.
    636. 

    637. 

  637. more_options() ->
    638. 

  638.     {ok,[{atom,_,foo}=T1],{19,20}} =
    639. 

  639.         erl_scan:string("foo", {19,17},[]),
    640. 

  640.     {19,17} = erl_scan:location(T1),
    641. 

  641.     {done,{ok,[{atom,_,foo}=T2,{dot,_}],{19,22}},[]} =
    642. 

  642.         erl_scan:tokens([], "foo. ", {19,17}, [bad_opt]), % type error
    643. 

  643.     {19,17} = erl_scan:location(T2),
    644. 

  644.     {ok,[{atom,_,foo}=T3],{19,20}} =
    645. 

  645.         erl_scan:string("foo", {19,17},[text]),
    646. 

  646.     {19,17} = erl_scan:location(T3),
    647. 

  647.     "foo" = erl_scan:text(T3),
    648. 

    649. 

  649.     {ok,[{atom,_,foo}=T4],1} = erl_scan:string("foo", 1, [text]),
    650. 

  650.     1 = erl_scan:line(T4),
    651. 

  651.     1 = erl_scan:location(T4),
    652. 

  652.     "foo" = erl_scan:text(T4),
    653. 

    654. 

  654.     ok.
    655. 

    656. 

  656. token_info() ->
    657. 

  657.     {ok,[T1],_} = erl_scan:string("foo", {1,18}, [text]),
    658. 

  658.     {'EXIT',{badarg,_}} =
    659. 

  659.         (catch {foo, erl_scan:category(foo)}), % type error
    660. 

  660.     {'EXIT',{badarg,_}} =
    661. 

  661.         (catch {foo, erl_scan:symbol(foo)}), % type error
    662. 

  662.     atom = erl_scan:category(T1),
    663. 

  663.     foo = erl_scan:symbol(T1),
    664. 

    665. 

  665.     {ok,[T2],_} = erl_scan:string("foo", 1, []),
    666. 

  666.     1 = erl_scan:line(T2),
    667. 

  667.     undefined = erl_scan:column(T2),
    668. 

  668.     undefined = erl_scan:text(T2),
    669. 

  669.     1 = erl_scan:location(T2),
    670. 

    671. 

  671.     {ok,[T3],_} = erl_scan:string("=", 1, []),
    672. 

  672.     '=' = erl_scan:category(T3),
    673. 

  673.     '=' = erl_scan:symbol(T3),
    674. 

  674.     ok.
    675. 

    676. 

  676. anno_info() ->
    677. 

  677.     {'EXIT',_} =
    678. 

  678.         (catch {foo,erl_scan:line(foo)}), % type error
    679. 

  679.     {ok,[{atom,_,foo}=T0],_} = erl_scan:string("foo", 19, [text]),
    680. 

  680.     19 = erl_scan:location(T0),
    681. 

  681.     19 = erl_scan:end_location(T0),
    682. 

    683. 

  683.     {ok,[{atom,_,foo}=T3],_} = erl_scan:string("foo", {1,3}, [text]),
    684. 

  684.     1 = erl_scan:line(T3),
    685. 

  685.     3 = erl_scan:column(T3),
    686. 

  686.     {1,3} = erl_scan:location(T3),
    687. 

  687.     {1,6} = erl_scan:end_location(T3),
    688. 

  688.     "foo" = erl_scan:text(T3),
    689. 

    690. 

  690.     {ok,[{atom,_,foo}=T4],_} = erl_scan:string("foo", 2, [text]),
    691. 

  691.     2 = erl_scan:line(T4),
    692. 

  692.     undefined = erl_scan:column(T4),
    693. 

  693.     2 = erl_scan:location(T4),
    694. 

  694.     "foo" = erl_scan:text(T4),
    695. 

    696. 

  696.     {ok,[{atom,_,foo}=T5],_} = erl_scan:string("foo", {1,3}, []),
    697. 

  697.     1 = erl_scan:line(T5),
    698. 

  698.     3 = erl_scan:column(T5),
    699. 

  699.     {1,3} = erl_scan:location(T5),
    700. 

  700.     undefined = erl_scan:text(T5),
    701. 

    702. 

  702.     ok.
    703. 

    704. 

  704. column_errors() ->
    705. 

  705.     {error,{{1,1},erl_scan,{string,$',""}},{1,3}} = % $'
    706. 

  706.         erl_scan:string("'\\",{1,1}),
    707. 

  707.     {error,{{1,1},erl_scan,{string,$",""}},{1,3}} = % $"
    708. 

  708.         erl_scan:string("\"\\",{1,1}),
    709. 

    710. 

  710.     {error,{{1,1},erl_scan,{string,$',""}},{1,2}} =  % $'
    711. 

  711.         erl_scan:string("'",{1,1}),
    712. 

  712.     {error,{{1,1},erl_scan,{string,$",""}},{1,2}} =  % $"
    713. 

  713.         erl_scan:string("\"",{1,1}),
    714. 

    715. 

  715.     {error,{{1,1},erl_scan,char},{1,2}} =
    716. 

  716.         erl_scan:string("$",{1,1}),
    717. 

    718. 

  718.     {error,{{1,2},erl_scan,{string,$',"1234567890123456"}},{1,20}} = %'
    719. 

  719.         erl_scan:string(" '12345678901234567", {1,1}),
    720. 

  720.     {error,{{1,2},erl_scan,{string,$',"123456789012345 "}}, {1,20}} = %'
    721. 

  721.         erl_scan:string(" '123456789012345\\s", {1,1}),
    722. 

  722.     {error,{{1,2},erl_scan,{string,$","1234567890123456"}},{1,20}} = %"
    723. 

  723.         erl_scan:string(" \"12345678901234567", {1,1}),
    724. 

  724.     {error,{{1,2},erl_scan,{string,$","123456789012345 "}}, {1,20}} = %"
    725. 

  725.         erl_scan:string(" \"123456789012345\\s", {1,1}),
    726. 

  726.     {error,{{1,2},erl_scan,{string,$',"1234567890123456"}},{2,1}} = %'
    727. 

  727.         erl_scan:string(" '12345678901234567\n", {1,1}),
    728. 

  728.     ok.
    729. 

    730. 

  730. white_spaces() ->
    731. 

  731.     {ok,[{white_space,_,"\r"},
    732. 

  732.                {white_space,_,"   "},
    733. 

  733.                {atom,_,a},
    734. 

  734.                {white_space,_,"\n"}],
    735. 

  735.            _} = erl_scan_string("\r   a\n", {1,1}, return),
    736. 

  736.     test("\r   a\n"),
    737. 

  737.     L = "{\"a\nb\", \"a\\nb\",\nabc\r,def}.\n\n",
    738. 

  738.     {ok,[{'{',_},
    739. 

  739.                {string,_,"a\nb"},
    740. 

  740.                {',',_},
    741. 

  741.                {white_space,_," "},
    742. 

  742.                {string,_,"a\nb"},
    743. 

  743.                {',',_},
    744. 

  744.                {white_space,_,"\n"},
    745. 

  745.                {atom,_,abc},
    746. 

  746.                {white_space,_,"\r"},
    747. 

  747.                {',',_},
    748. 

  748.                {atom,_,def},
    749. 

  749.                {'}',_},
    750. 

  750.                {dot,_},
    751. 

  751.                {white_space,_,"\n"}],
    752. 

  752.            _} = erl_scan_string(L, {1,1}, return),
    753. 

  753.     test(L),
    754. 

  754.     test("\"\n\"\n"),
    755. 

  755.     test("\n\r\n"),
    756. 

  756.     test("\n\r"),
    757. 

  757.     test("\r\n"),
    758. 

  758.     test("\n\f"),
    759. 

  759.     [test(lists:duplicate(N, $\t)) || N <- lists:seq(1, 20)],
    760. 

  760.     [test([$\n|lists:duplicate(N, $\t)]) || N <- lists:seq(1, 20)],
    761. 

  761.     [test(lists:duplicate(N, $\s)) || N <- lists:seq(1, 20)],
    762. 

  762.     [test([$\n|lists:duplicate(N, $\s)]) || N <- lists:seq(1, 20)],
    763. 

  763.     test("\v\f\n\v "),
    764. 

  764.     test("\n\e\n\b\f\n\da\n"),
    765. 

  765.     ok.
    766. 

    767. 

  767. unicode() ->
    768. 

  768.     {ok,[{char,1,83},{integer,1,45}],1} =
    769. 

  769.         erl_scan_string("$\\12345"), % not unicode
    770. 

    771. 

  771.     {error,{1,erl_scan,{illegal,character}},1} =
    772. 

  772.         erl_scan:string([1089]),
    773. 

  773.     {error,{{1,1},erl_scan,{illegal,character}},{1,2}} =
    774. 

  774.         erl_scan:string([1089], {1,1}),
    775. 

  775.     {error,{{1,3},erl_scan,{illegal,character}},{1,4}} =
    776. 

  776.         erl_scan:string("'a" ++ [999999999] ++ "c'", {1,1}),
    777. 

    778. 

  778.     test("\"a"++[1089]++"b\""),
    779. 

  779.     {ok,[{char,1,1}],1} =
    780. 

  780.         erl_scan_string([$$,$\\,$^,1089], 1),
    781. 

    782. 

  782.     {error,{1,erl_scan,Error},1} =
    783. 

  783.         erl_scan:string("\"qa\x{aaa}", 1),
    784. 

  784.     "unterminated string starting with \"qa"++[2730]++"\"" =
    785. 

  785.         erl_scan:format_error(Error),
    786. 

  786.     {error,{{1,1},erl_scan,_},{1,11}} =
    787. 

  787.         erl_scan:string("\"qa\\x{aaa}",{1,1}),
    788. 

  788.     {error,{{1,1},erl_scan,_},{1,11}} =
    789. 

  789.         erl_scan:string("'qa\\x{aaa}",{1,1}),
    790. 

    791. 

  791.     {ok,[{char,1,1089}],1} =
    792. 

  792.         erl_scan_string([$$,1089], 1),
    793. 

  793.     {ok,[{char,1,1089}],1} =
    794. 

  794.         erl_scan_string([$$,$\\,1089], 1),
    795. 

    796. 

  796.     Qs = "$\\x{aaa}",
    797. 

  797.     {ok,[{char,1,$\x{aaa}}],1} =
    798. 

  798.         erl_scan_string(Qs, 1),
    799. 

  799.     {ok,[Q2],{1,9}} =
    800. 

  800.         erl_scan:string("$\\x{aaa}", {1,1}, [text]),
    801. 

  801.     [{category,char},{column,1},{line,1},{symbol,16#aaa},{text,Qs}] =
    802. 

  802.         token_info_long(Q2),
    803. 

    804. 

  804.     U1 = "\"\\x{aaa}\"",
    805. 

  805.     {ok,[{string,_,[2730]}=T1],{1,10}} = erl_scan:string(U1, {1,1}, [text]),
    806. 

  806.     {1,1} = erl_scan:location(T1),
    807. 

  807.     "\"\\x{aaa}\"" = erl_scan:text(T1),
    808. 

  808.     {ok,[{string,1,[2730]}],1} = erl_scan_string(U1, 1),
    809. 

    810. 

  810.     U2 = "\"\\x41\\x{fff}\\x42\"",
    811. 

  811.     {ok,[{string,1,[$\x41,$\x{fff},$\x42]}],1} = erl_scan_string(U2, 1),
    812. 

    813. 

  813.     U3 = "\"a\n\\x{fff}\n\"",
    814. 

  814.     {ok,[{string,1,[$a,$\n,$\x{fff},$\n]}],3} = erl_scan_string(U3, 1),
    815. 

    816. 

  816.     U4 = "\"\\^\n\\x{aaa}\\^\n\"",
    817. 

  817.     {ok,[{string,1,[$\n,$\x{aaa},$\n]}],3} = erl_scan_string(U4, 1),
    818. 

    819. 

  819.     %% Keep these tests:
    820. 

  820.     test(Qs),
    821. 

  821.     test(U1),
    822. 

  822.     test(U2),
    823. 

  823.     test(U3),
    824. 

  824.     test(U4),
    825. 

    826. 

  826.     Str1 = "\"ab" ++ [1089] ++ "cd\"",
    827. 

  827.     {ok,[{string,1,[$a,$b,1089,$c,$d]}],1} = erl_scan_string(Str1, 1),
    828. 

  828.     {ok,[{string,{1,1},[$a,$b,1089,$c,$d]}],{1,8}} =
    829. 

  829.         erl_scan_string(Str1, {1,1}),
    830. 

  830.     test(Str1),
    831. 

  831.     Comment = "%% "++[1089],
    832. 

  832.     {ok,[{comment,1,[$%,$%,$\s,1089]}],1} =
    833. 

  833.         erl_scan_string(Comment, 1, [return]),
    834. 

  834.     {ok,[{comment,{1,1},[$%,$%,$\s,1089]}],{1,5}} =
    835. 

  835.         erl_scan_string(Comment, {1,1}, [return]),
    836. 

  836.     ok.
    837. 

    838. 

  838. more_chars() ->
    839. 

  839.     %% Due to unicode, the syntax has been incompatibly augmented:
    840. 

  840.     %% $\x{...}, $\xHH
    841. 

    842. 

  842.     %% All kinds of tests...
    843. 

  843.     {ok,[{char,_,123}],{1,4}} =
    844. 

  844.         erl_scan_string("$\\{",{1,1}),
    845. 

  845.     {more, C1} = erl_scan:tokens([], "$\\{", {1,1}),
    846. 

  846.     {done,{ok,[{char,_,123}],{1,4}},eof} =
    847. 

  847.         erl_scan_tokens(C1, eof, 1),
    848. 

  848.     {ok,[{char,1,123},{atom,1,a},{'}',1}],1} =
    849. 

  849.         erl_scan_string("$\\{a}"),
    850. 

    851. 

  851.     {error,{{1,1},erl_scan,char},{1,4}} =
    852. 

  852.         erl_scan:string("$\\x", {1,1}),
    853. 

  853.     {error,{{1,1},erl_scan,char},{1,5}} =
    854. 

  854.         erl_scan:string("$\\x{",{1,1}),
    855. 

  855.     {more, C3} = erl_scan:tokens([], "$\\x", {1,1}),
    856. 

  856.     {done,{error,{{1,1},erl_scan,char},{1,4}},eof} =
    857. 

  857.         erl_scan:tokens(C3, eof, 1),
    858. 

  858.     {error,{{1,1},erl_scan,char},{1,5}} =
    859. 

  859.         erl_scan:string("$\\x{",{1,1}),
    860. 

  860.     {more, C2} = erl_scan:tokens([], "$\\x{", {1,1}),
    861. 

  861.     {done,{error,{{1,1},erl_scan,char},{1,5}},eof} =
    862. 

  862.         erl_scan:tokens(C2, eof, 1),
    863. 

  863.     {error,{1,erl_scan,{illegal,character}},1} =
    864. 

  864.         erl_scan:string("$\\x{g}"),
    865. 

  865.     {error,{{1,1},erl_scan,{illegal,character}},{1,5}} =
    866. 

  866.         erl_scan:string("$\\x{g}", {1,1}),
    867. 

  867.     {error,{{1,1},erl_scan,{illegal,character}},{1,6}} =
    868. 

  868.         erl_scan:string("$\\x{}",{1,1}),
    869. 

    870. 

  870.     test("\"\\{0}\""),
    871. 

  871.     test("\"\\x{0}\""),
    872. 

  872.     test("\'\\{0}\'"),
    873. 

  873.     test("\'\\x{0}\'"),
    874. 

    875. 

  875.     {error,{{2,3},erl_scan,{illegal,character}},{2,6}} =
    876. 

  876.         erl_scan:string("\"ab \n $\\x{g}\"",{1,1}),
    877. 

  877.     {error,{{2,3},erl_scan,{illegal,character}},{2,6}} =
    878. 

  878.         erl_scan:string("\'ab \n $\\x{g}\'",{1,1}),
    879. 

    880. 

  880.     test("$\\{34}"),
    881. 

  881.     test("$\\x{34}"),
    882. 

  882.     test("$\\{377}"),
    883. 

  883.     test("$\\x{FF}"),
    884. 

  884.     test("$\\{400}"),
    885. 

  885.     test("$\\x{100}"),
    886. 

  886.     test("$\\x{10FFFF}"),
    887. 

  887.     test("$\\x{10ffff}"),
    888. 

  888.     test("\"$\n \\{1}\""),
    889. 

  889.     {error,{1,erl_scan,{illegal,character}},1} =
    890. 

  890.         erl_scan:string("$\\x{110000}"),
    891. 

  891.     {error,{{1,1},erl_scan,{illegal,character}},{1,12}} =
    892. 

  892.         erl_scan:string("$\\x{110000}", {1,1}),
    893. 

    894. 

  894.     {error,{{1,1},erl_scan,{illegal,character}},{1,4}} =
    895. 

  895.         erl_scan:string("$\\xfg", {1,1}),
    896. 

    897. 

  897.     test("$\\xffg"),
    898. 

    899. 

  899.     {error,{{1,1},erl_scan,{illegal,character}},{1,4}} =
    900. 

  900.         erl_scan:string("$\\xg", {1,1}),
    901. 

  901.     ok.
    902. 

    903. 

  903. %% OTP-10302. Unicode characters scanner/parser.
    904. 

  904. otp_10302(Config) when is_list(Config) ->
    905. 

  905.     %% From unicode():
    906. 

  906.     {ok,[{atom,1,'aсb'}],1} =
    907. 

  907.         erl_scan_string("'a"++[1089]++"b'", 1),
    908. 

  908.     {ok,[{atom,{1,1},'qaપ'}],{1,12}} =
    909. 

  909.         erl_scan_string("'qa\\x{aaa}'",{1,1}),
    910. 

    911. 

  911.     {ok,[{char,1,1089}],1} = erl_scan_string([$$,1089], 1),
    912. 

  912.     {ok,[{char,1,1089}],1} = erl_scan_string([$$,$\\,1089],1),
    913. 

    914. 

  914.     Qs = "$\\x{aaa}",
    915. 

  915.     {ok,[{char,1,2730}],1} = erl_scan_string(Qs, 1),
    916. 

  916.     {ok,[Q2],{1,9}} = erl_scan:string(Qs,{1,1},[text]),
    917. 

  917.     [{category,char},{column,1},{line,1},{symbol,16#aaa},{text,Qs}] =
    918. 

  918.         token_info_long(Q2),
    919. 

    920. 

  920.     U1 = "\"\\x{aaa}\"",
    921. 

  921.     {ok,[T1],{1,10}} = erl_scan:string(U1, {1,1}, [text]),
    922. 

  922.     [{category,string},{column,1},{line,1},{symbol,[16#aaa]},{text,U1}] =
    923. 

  923.         token_info_long(T1),
    924. 

    925. 

  925.     U2 = "\"\\x41\\x{fff}\\x42\"",
    926. 

  926.     {ok,[{string,1,[65,4095,66]}],1} = erl_scan_string(U2, 1),
    927. 

    928. 

  928.     U3 = "\"a\n\\x{fff}\n\"",
    929. 

  929.     {ok,[{string,1,[97,10,4095,10]}],3} = erl_scan_string(U3, 1),
    930. 

    931. 

  931.     U4 = "\"\\^\n\\x{aaa}\\^\n\"",
    932. 

  932.     {ok,[{string,1,[10,2730,10]}],3} = erl_scan_string(U4, 1,[]),
    933. 

    934. 

  934.     Str1 = "\"ab" ++ [1089] ++ "cd\"",
    935. 

  935.     {ok,[{string,1,[97,98,1089,99,100]}],1} =
    936. 

  936.         erl_scan_string(Str1,1),
    937. 

  937.     {ok,[{string,{1,1},[97,98,1089,99,100]}],{1,8}} =
    938. 

  938.         erl_scan_string(Str1, {1,1}),
    939. 

    940. 

  940.     OK1 = 16#D800-1,
    941. 

  941.     OK2 = 16#DFFF+1,
    942. 

  942.     OK3 = 16#FFFE-1,
    943. 

  943.     OK4 = 16#FFFF+1,
    944. 

  944.     OKL = [OK1,OK2,OK3,OK4],
    945. 

    946. 

  946.     Illegal1 = 16#D800,
    947. 

  947.     Illegal2 = 16#DFFF,
    948. 

  948.     Illegal3 = 16#FFFE,
    949. 

  949.     Illegal4 = 16#FFFF,
    950. 

  950.     IllegalL = [Illegal1,Illegal2,Illegal3,Illegal4],
    951. 

    952. 

  952.     [{ok,[{comment,1,[$%,$%,$\s,OK]}],1} =
    953. 

  953.          erl_scan_string("%% "++[OK], 1, [return]) ||
    954. 

  954.         OK <- OKL],
    955. 

  955.     {ok,[{comment,_,[$%,$%,$\s,OK1]}],{1,5}} =
    956. 

  956.         erl_scan_string("%% "++[OK1], {1,1}, [return]),
    957. 

  957.     [{error,{1,erl_scan,{illegal,character}},1} =
    958. 

  958.          erl_scan:string("%% "++[Illegal], 1, [return]) ||
    959. 

  959.         Illegal <- IllegalL],
    960. 

  960.     {error,{{1,1},erl_scan,{illegal,character}},{1,5}} =
    961. 

  961.         erl_scan:string("%% "++[Illegal1], {1,1}, [return]),
    962. 

    963. 

  963.     [{ok,[],1} = erl_scan_string("%% "++[OK], 1, []) ||
    964. 

  964.         OK <- OKL],
    965. 

  965.     {ok,[],{1,5}} = erl_scan_string("%% "++[OK1], {1,1}, []),
    966. 

  966.     [{error,{1,erl_scan,{illegal,character}},1} =
    967. 

  967.          erl_scan:string("%% "++[Illegal], 1, []) ||
    968. 

  968.         Illegal <- IllegalL],
    969. 

  969.     {error,{{1,1},erl_scan,{illegal,character}},{1,5}} =
    970. 

  970.         erl_scan:string("%% "++[Illegal1], {1,1}, []),
    971. 

    972. 

  972.     [{ok,[{string,{1,1},[OK]}],{1,4}} =
    973. 

  973.         erl_scan_string("\""++[OK]++"\"",{1,1}) ||
    974. 

  974.         OK <- OKL],
    975. 

  975.     [{error,{{1,2},erl_scan,{illegal,character}},{1,3}} =
    976. 

  976.          erl_scan:string("\""++[OK]++"\"",{1,1}) ||
    977. 

  977.         OK <- IllegalL],
    978. 

    979. 

  979.     [{error,{{1,1},erl_scan,{illegal,character}},{1,2}} =
    980. 

  980.         erl_scan:string([Illegal],{1,1}) ||
    981. 

  981.         Illegal <- IllegalL],
    982. 

    983. 

  983.     {ok,[{char,{1,1},OK1}],{1,3}} =
    984. 

  984.         erl_scan_string([$$,OK1],{1,1}),
    985. 

  985.     {error,{{1,1},erl_scan,{illegal,character}},{1,2}} =
    986. 

  986.         erl_scan:string([$$,Illegal1],{1,1}),
    987. 

    988. 

  988.     {ok,[{char,{1,1},OK1}],{1,4}} =
    989. 

  989.         erl_scan_string([$$,$\\,OK1],{1,1}),
    990. 

  990.     {error,{{1,1},erl_scan,{illegal,character}},{1,4}} =
    991. 

  991.         erl_scan:string([$$,$\\,Illegal1],{1,1}),
    992. 

    993. 

  993.     {ok,[{string,{1,1},[55295]}],{1,5}} =
    994. 

  994.         erl_scan_string("\"\\"++[OK1]++"\"",{1,1}),
    995. 

  995.     {error,{{1,2},erl_scan,{illegal,character}},{1,4}} =
    996. 

  996.         erl_scan:string("\"\\"++[Illegal1]++"\"",{1,1}),
    997. 

    998. 

  998.     {ok,[{char,{1,1},OK1}],{1,10}} =
    999. 

  999.         erl_scan_string("$\\x{D7FF}",{1,1}),
    1000. 

  1000.     {error,{{1,1},erl_scan,{illegal,character}},{1,10}} =
    1001. 

  1001.         erl_scan:string("$\\x{D800}",{1,1}),
    1002. 

    1003. 

  1003.     %% Not erl_scan, but erl_parse.
    1004. 

  1004.     {integer,0,1} = erl_parse_abstract(1),
    1005. 

  1005.     Float = 3.14, {float,0,Float} = erl_parse_abstract(Float),
    1006. 

  1006.     {nil,0} = erl_parse_abstract([]),
    1007. 

  1007.     {bin,0,
    1008. 

  1008.      [{bin_element,0,{integer,0,1},default,default},
    1009. 

  1009.       {bin_element,0,{integer,0,2},default,default}]} =
    1010. 

  1010.         erl_parse_abstract(<<1,2>>),
    1011. 

  1011.     {cons,0,{tuple,0,[{atom,0,a}]},{atom,0,b}} =
    1012. 

  1012.         erl_parse_abstract([{a} | b]),
    1013. 

  1013.     {string,0,"str"} = erl_parse_abstract("str"),
    1014. 

  1014.     {cons,0,
    1015. 

  1015.      {integer,0,$a},
    1016. 

  1016.      {cons,0,{integer,0,55296},{string,0,"c"}}} =
    1017. 

  1017.         erl_parse_abstract("a"++[55296]++"c"),
    1018. 

    1019. 

  1019.     Line = 17,
    1020. 

  1020.     {integer,Line,1} = erl_parse_abstract(1, Line),
    1021. 

  1021.     Float = 3.14, {float,Line,Float} = erl_parse_abstract(Float, Line),
    1022. 

  1022.     {nil,Line} = erl_parse_abstract([], Line),
    1023. 

  1023.     {bin,Line,
    1024. 

  1024.      [{bin_element,Line,{integer,Line,1},default,default},
    1025. 

  1025.       {bin_element,Line,{integer,Line,2},default,default}]} =
    1026. 

  1026.         erl_parse_abstract(<<1,2>>, Line),
    1027. 

  1027.     {cons,Line,{tuple,Line,[{atom,Line,a}]},{atom,Line,b}} =
    1028. 

  1028.         erl_parse_abstract([{a} | b], Line),
    1029. 

  1029.     {string,Line,"str"} = erl_parse_abstract("str", Line),
    1030. 

  1030.     {cons,Line,
    1031. 

  1031.      {integer,Line,$a},
    1032. 

  1032.      {cons,Line,{integer,Line,55296},{string,Line,"c"}}} =
    1033. 

  1033.         erl_parse_abstract("a"++[55296]++"c", Line),
    1034. 

    1035. 

  1035.     Opts1 = [{line,17}],
    1036. 

  1036.     {integer,Line,1} = erl_parse_abstract(1, Opts1),
    1037. 

  1037.     Float = 3.14, {float,Line,Float} = erl_parse_abstract(Float, Opts1),
    1038. 

  1038.     {nil,Line} = erl_parse_abstract([], Opts1),
    1039. 

  1039.     {bin,Line,
    1040. 

  1040.      [{bin_element,Line,{integer,Line,1},default,default},
    1041. 

  1041.       {bin_element,Line,{integer,Line,2},default,default}]} =
    1042. 

  1042.         erl_parse_abstract(<<1,2>>, Opts1),
    1043. 

  1043.     {cons,Line,{tuple,Line,[{atom,Line,a}]},{atom,Line,b}} =
    1044. 

  1044.         erl_parse_abstract([{a} | b], Opts1),
    1045. 

  1045.     {string,Line,"str"} = erl_parse_abstract("str", Opts1),
    1046. 

  1046.     {cons,Line,
    1047. 

  1047.      {integer,Line,$a},
    1048. 

  1048.      {cons,Line,{integer,Line,55296},{string,Line,"c"}}} =
    1049. 

  1049.         erl_parse_abstract("a"++[55296]++"c", Opts1),
    1050. 

    1051. 

  1051.     [begin
    1052. 

  1052.          {integer,Line,1} = erl_parse_abstract(1, Opts2),
    1053. 

  1053.          Float = 3.14, {float,Line,Float} = erl_parse_abstract(Float, Opts2),
    1054. 

  1054.          {nil,Line} = erl_parse_abstract([], Opts2),
    1055. 

  1055.          {bin,Line,
    1056. 

  1056.           [{bin_element,Line,{integer,Line,1},default,default},
    1057. 

  1057.            {bin_element,Line,{integer,Line,2},default,default}]} =
    1058. 

  1058.              erl_parse_abstract(<<1,2>>, Opts2),
    1059. 

  1059.          {cons,Line,{tuple,Line,[{atom,Line,a}]},{atom,Line,b}} =
    1060. 

  1060.              erl_parse_abstract([{a} | b], Opts2),
    1061. 

  1061.          {string,Line,"str"} = erl_parse_abstract("str", Opts2),
    1062. 

  1062.          {string,Line,[97,1024,99]} =
    1063. 

  1063.              erl_parse_abstract("a"++[1024]++"c", Opts2)
    1064. 

  1064.      end || Opts2 <- [[{encoding,unicode},{line,Line}],
    1065. 

  1065.                       [{encoding,utf8},{line,Line}]]],
    1066. 

    1067. 

  1067.     {cons,0,
    1068. 

  1068.      {integer,0,97},
    1069. 

  1069.      {cons,0,{integer,0,1024},{string,0,"c"}}} =
    1070. 

  1070.         erl_parse_abstract("a"++[1024]++"c", [{encoding,latin1}]),
    1071. 

  1071.     ok.
    1072. 

    1073. 

  1073. %% OTP-10990. Floating point number in input string.
    1074. 

  1074. otp_10990(Config) when is_list(Config) ->
    1075. 

  1075.     {'EXIT',_} = (catch {foo, erl_scan:string([$",42.0,$"],1)}),
    1076. 

  1076.     ok.
    1077. 

    1078. 

  1078. %% OTP-10992. List of floats to abstract format.
    1079. 

  1079. otp_10992(Config) when is_list(Config) ->
    1080. 

  1080.     {cons,0,{float,0,42.0},{nil,0}} =
    1081. 

  1081.         erl_parse_abstract([42.0], [{encoding,unicode}]),
    1082. 

  1082.     {cons,0,{float,0,42.0},{nil,0}} =
    1083. 

  1083.         erl_parse_abstract([42.0], [{encoding,utf8}]),
    1084. 

  1084.     {cons,0,{integer,0,65},{cons,0,{float,0,42.0},{nil,0}}} =
    1085. 

  1085.         erl_parse_abstract([$A,42.0], [{encoding,unicode}]),
    1086. 

  1086.     {cons,0,{integer,0,65},{cons,0,{float,0,42.0},{nil,0}}} =
    1087. 

  1087.         erl_parse_abstract([$A,42.0], [{encoding,utf8}]),
    1088. 

  1088.     ok.
    1089. 

    1090. 

  1090. %% OTP-11807. Generalize erl_parse:abstract/2.
    1091. 

  1091. otp_11807(Config) when is_list(Config) ->
    1092. 

  1092.     {cons,0,{integer,0,97},{cons,0,{integer,0,98},{nil,0}}} =
    1093. 

  1093.         erl_parse_abstract("ab", [{encoding,none}]),
    1094. 

  1094.     {cons,0,{integer,0,-1},{nil,0}} =
    1095. 

  1095.         erl_parse_abstract([-1], [{encoding,latin1}]),
    1096. 

  1096.     ASCII = fun(I) -> I >= 0 andalso I < 128 end,
    1097. 

  1097.     {string,0,"xyz"} = erl_parse_abstract("xyz", [{encoding,ASCII}]),
    1098. 

  1098.     {cons,0,{integer,0,228},{nil,0}} =
    1099. 

  1099.         erl_parse_abstract([228], [{encoding,ASCII}]),
    1100. 

  1100.     {cons,0,{integer,0,97},{atom,0,a}} =
    1101. 

  1101.         erl_parse_abstract("a"++a, [{encoding,latin1}]),
    1102. 

  1102.     {'EXIT', {{badarg,bad},_}} = % minor backward incompatibility
    1103. 

  1103.          (catch erl_parse:abstract("string", [{encoding,bad}])),
    1104. 

  1104.    ok.
    1105. 

    1106. 

  1106. test_string(String, ExpectedWithCol) ->
    1107. 

  1107.     {ok, ExpectedWithCol, _EndWithCol} = erl_scan_string(String, {1, 1}, []),
    1108. 

  1108.     Expected = [ begin
    1109. 

  1109.                      {L,_C} = element(2, T),
    1110. 

  1110.                      setelement(2, T, L)
    1111. 

  1111.                  end
    1112. 

  1112.                     || T <- ExpectedWithCol ],
    1113. 

  1113.     {ok, Expected, _End} = erl_scan_string(String),
    1114. 

  1114.     test(String).
    1115. 

    1116. 

  1116. erl_scan_string(String) ->
    1117. 

  1117.     erl_scan_string(String, 1, []).
    1118. 

    1119. 

  1119. erl_scan_string(String, StartLocation) ->
    1120. 

  1120.     erl_scan_string(String, StartLocation, []).
    1121. 

    1122. 

  1122. erl_scan_string(String, StartLocation, Options) ->
    1123. 

  1123.     case erl_scan:string(String, StartLocation, Options) of
    1124. 

  1124.         {ok, Tokens, EndLocation} ->
    1125. 

  1125.             {ok, unopaque_tokens(Tokens), EndLocation};
    1126. 

  1126.         Else ->
    1127. 

  1127.             Else
    1128. 

  1128.     end.
    1129. 

    1130. 

  1130. erl_scan_tokens(C, S, L) ->
    1131. 

  1131.     erl_scan_tokens(C, S, L, []).
    1132. 

    1133. 

  1133. erl_scan_tokens(C, S, L, O) ->
    1134. 

  1134.     case erl_scan:tokens(C, S, L, O) of
    1135. 

  1135.         {done, {ok, Ts, End}, R} ->
    1136. 

  1136.             {done, {ok, unopaque_tokens(Ts), End}, R};
    1137. 

  1137.         Else ->
    1138. 

  1138.             Else
    1139. 

  1139.     end.
    1140. 

    1141. 

  1141. unopaque_tokens([]) ->
    1142. 

  1142.     [];
    1143. 

  1143. unopaque_tokens([Token|Tokens]) ->
    1144. 

  1144.     Attrs = element(2, Token),
    1145. 

  1145.     Term = erl_anno:to_term(Attrs),
    1146. 

  1146.     T = setelement(2, Token, Term),
    1147. 

  1147.     [T | unopaque_tokens(Tokens)].
    1148. 

    1149. 

  1149. erl_parse_abstract(Term) ->
    1150. 

  1150.     erl_parse_abstract(Term, []).
    1151. 

    1152. 

  1152. erl_parse_abstract(Term, Options) ->
    1153. 

  1153.     Abstr = erl_parse:abstract(Term, Options),
    1154. 

  1154.     unopaque_abstract(Abstr).
    1155. 

    1156. 

  1156. unopaque_abstract(Abstr) ->
    1157. 

  1157.     erl_parse:anno_to_term(Abstr).
    1158. 

    1159. 

  1159. %% test_string(String, Expected, StartLocation, Options) ->
    1160. 

  1160. %%     {ok, Expected, _End} = erl_scan:string(String, StartLocation, Options),
    1161. 

  1161. %%     test(String).
    1162. 

    1163. 

  1163. %% There are no checks of the tags...
    1164. 

  1164. test(String) ->
    1165. 

  1165.     %% io:format("Testing `~ts'~n", [String]),
    1166. 

  1166.     [{Tokens, End},
    1167. 

  1167.      {Wtokens, Wend},
    1168. 

  1168.      {Ctokens, Cend},
    1169. 

  1169.      {CWtokens, CWend},
    1170. 

  1170.      {CWtokens2, _}] =
    1171. 

  1171.         [scan_string_with_column(String, X) ||
    1172. 

  1172.             X <- [[],
    1173. 

  1173.                   [return_white_spaces],
    1174. 

  1174.                   [return_comments],
    1175. 

  1175.                   [return],
    1176. 

  1176.                   [return]]], % for white space compaction test
    1177. 

  1177. 

  1178.     {end1,End,Wend} = {end1,Wend,End},
    1179. 

  1179.     {end2,Wend,Cend} = {end2,Cend,Wend},
    1180. 

  1180.     {end3,Cend,CWend} = {end3,CWend,Cend},
    1181. 

  1181. 

  1182.     %% Test that the tokens that are common to two token lists are identical.
    1183. 

  1183.     {none,Tokens} = {none, filter_tokens(CWtokens, [white_space,comment])},
    1184. 

  1184.     {comments,Ctokens} =
    1185. 

  1185.         {comments,filter_tokens(CWtokens, [white_space])},
    1186. 

  1186.     {white_spaces,Wtokens} =
    1187. 

  1187.         {white_spaces,filter_tokens(CWtokens, [comment])},
    1188. 

  1188. 

  1189.     %% Use token attributes to extract parts from the original string,
    1190. 

  1190.     %% and check that the parts are identical to the token strings.
    1191. 

  1191.     {Line,Column} = test_decorated_tokens(String, CWtokens),
    1192. 

  1192.     {deco,{Line,Column},End} = {deco,End,{Line,Column}},
    1193. 

  1193. 

  1194.     %% Almost the same again: concat texts to get the original:
    1195. 

  1195.     Text = get_text(CWtokens),
    1196. 

  1196.     {text,Text,String} = {text,String,Text},
    1197. 

  1197. 

  1198.     %% Test that white spaces occupy less heap than the worst case.
    1199. 

  1199.     ok = test_white_space_compaction(CWtokens, CWtokens2),
    1200. 

  1200. 

  1201.     %% Test that white newlines are always first in text:
    1202. 

  1202.     WhiteTokens = select_tokens(CWtokens, [white_space]),
    1203. 

  1203.     ok = newlines_first(WhiteTokens),
    1204. 

  1204. 

  1205.     %% Line attribute only:
    1206. 

  1206.     [Simple,Wsimple,Csimple,WCsimple] = Simples =
    1207. 

  1207.         [element(2, erl_scan:string(String, 1, Opts)) ||
    1208. 

  1208.             Opts <- [[],
    1209. 

  1209.                      [return_white_spaces],
    1210. 

  1210.                      [return_comments],
    1211. 

  1211.                      [return]]],
    1212. 

  1212.     {consistent,true} = {consistent,consistent_attributes(Simples)},
    1213. 

  1213.     {simple_wc,WCsimple} = {simple_wc,simplify(CWtokens)},
    1214. 

  1214.     {simple,Simple} = {simple,filter_tokens(WCsimple, [white_space,comment])},
    1215. 

  1215.     {simple_c,Csimple} = {simple_c,filter_tokens(WCsimple, [white_space])},
    1216. 

  1216.     {simple_w,Wsimple} = {simple_w,filter_tokens(WCsimple, [comment])},
    1217. 

  1217. 

  1218.     %% Line attribute only, with text:
    1219. 

  1219.     [SimpleTxt,WsimpleTxt,CsimpleTxt,WCsimpleTxt] = SimplesTxt =
    1220. 

  1220.         [element(2, erl_scan:string(String, 1, [text|Opts])) ||
    1221. 

  1221.             Opts <- [[],
    1222. 

  1222.                      [return_white_spaces],
    1223. 

  1223.                      [return_comments],
    1224. 

  1224.                      [return]]],
    1225. 

  1225.     TextTxt = get_text(WCsimpleTxt),
    1226. 

  1226.     {text_txt,TextTxt,String} = {text_txt,String,TextTxt},
    1227. 

  1227.     {consistent_txt,true} =
    1228. 

  1228.         {consistent_txt,consistent_attributes(SimplesTxt)},
    1229. 

  1229.     {simple_txt,SimpleTxt} =
    1230. 

  1230.         {simple_txt,filter_tokens(WCsimpleTxt, [white_space,comment])},
    1231. 

  1231.     {simple_c_txt,CsimpleTxt} =
    1232. 

  1232.         {simple_c_txt,filter_tokens(WCsimpleTxt, [white_space])},
    1233. 

  1233.     {simple_w_txt,WsimpleTxt} =
    1234. 

  1234.         {simple_w_txt,filter_tokens(WCsimpleTxt, [comment])},
    1235. 

  1235. 

  1236.     ok.
    1237. 

  1237. 

  1238. test_white_space_compaction(Tokens, Tokens2) when Tokens =:= Tokens2 ->
    1239. 

  1239.     [WS, WS2] = [select_tokens(Ts, [white_space]) || Ts <- [Tokens, Tokens2]],
    1240. 

  1240.     test_wsc(WS, WS2).
    1241. 

  1241. 

  1242. test_wsc([], []) ->
    1243. 

  1243.     ok;
    1244. 

  1244. test_wsc([Token|Tokens], [Token2|Tokens2]) ->
    1245. 

  1245.     [Text, Text2] = [Text ||
    1246. 

  1246.                         Text <- [erl_scan:text(T) || T <- [Token, Token2]]],
    1247. 

  1247.     Sz = erts_debug:size(Text),
    1248. 

  1248.     Sz2 = erts_debug:size({Text, Text2}),
    1249. 

  1249.     IsCompacted = Sz2 < 2*Sz+erts_debug:size({a,a}),
    1250. 

  1250.     ToBeCompacted = is_compacted(Text),
    1251. 

  1251.     if
    1252. 

  1252.         IsCompacted =:= ToBeCompacted ->
    1253. 

  1253.             test_wsc(Tokens, Tokens2);
    1254. 

  1254.         true ->
    1255. 

  1255.             {compaction_error, Token}
    1256. 

  1256.     end.
    1257. 

  1257. 

  1258. is_compacted("\r") ->
    1259. 

  1259.     true;
    1260. 

  1260. is_compacted("\n\r") ->
    1261. 

  1261.     true;
    1262. 

  1262. is_compacted("\n\f") ->
    1263. 

  1263.     true;
    1264. 

  1264. is_compacted([$\n|String]) ->
    1265. 

  1265.       all_spaces(String)
    1266. 

  1266.     orelse
    1267. 

  1267.       all_tabs(String);
    1268. 

  1268. is_compacted(String) ->
    1269. 

  1269.       all_spaces(String)
    1270. 

  1270.     orelse
    1271. 

  1271.       all_tabs(String).
    1272. 

  1272. 

  1273. all_spaces(L) ->
    1274. 

  1274.     all_same(L, $\s).
    1275. 

  1275. 

  1276. all_tabs(L) ->
    1277. 

  1277.     all_same(L, $\t).
    1278. 

  1278. 

  1279. all_same(L, Char) ->
    1280. 

  1280.     lists:all(fun(C) -> C =:= Char end, L).
    1281. 

  1281. 

  1282. newlines_first([]) ->
    1283. 

  1283.     ok;
    1284. 

  1284. newlines_first([Token|Tokens]) ->
    1285. 

  1285.     Text = erl_scan:text(Token),
    1286. 

  1286.     Nnls = length([C || C <- Text, C =:= $\n]),
    1287. 

  1287.     OK = case Text of
    1288. 

  1288.              [$\n|_] ->
    1289. 

  1289.                  Nnls =:= 1;
    1290. 

  1290.              _ ->
    1291. 

  1291.                  Nnls =:= 0
    1292. 

  1292.          end,
    1293. 

  1293.     if
    1294. 

  1294.         OK -> newlines_first(Tokens);
    1295. 

  1295.         true -> OK
    1296. 

  1296.     end.
    1297. 

  1297. 

  1298. filter_tokens(Tokens, Tags) ->
    1299. 

  1299.     lists:filter(fun(T) -> not lists:member(element(1, T), Tags) end, Tokens).
    1300. 

  1300. 

  1301. select_tokens(Tokens, Tags) ->
    1302. 

  1302.     lists:filter(fun(T) -> lists:member(element(1, T), Tags) end, Tokens).
    1303. 

  1303. 

  1304. simplify([Token|Tokens]) ->
    1305. 

  1305.     Line = erl_scan:line(Token),
    1306. 

  1306.     [setelement(2, Token, erl_anno:new(Line)) | simplify(Tokens)];
    1307. 

  1307. simplify([]) ->
    1308. 

  1308.     [].
    1309. 

  1309. 

  1310. get_text(Tokens) ->
    1311. 

  1311.     lists:flatten(
    1312. 

  1312.       [T ||
    1313. 

  1313.           Token <- Tokens,
    1314. 

  1314.           (T = erl_scan:text(Token)) =/= []]).
    1315. 

  1315. 

  1316. test_decorated_tokens(String, Tokens) ->
    1317. 

  1317.     ToksAttrs = token_attrs(Tokens),
    1318. 

  1318.     test_strings(ToksAttrs, String, 1, 1).
    1319. 

  1319. 

  1320. token_attrs(Tokens) ->
    1321. 

  1321.     [{L,C,length(T),T} ||
    1322. 

  1322.         Token <- Tokens,
    1323. 

  1323.         ([C,L,T] = token_info(Token)) =/= []].
    1324. 

  1324. 

  1325. token_info(T) ->
    1326. 

  1326.     Column = erl_scan:column(T),
    1327. 

  1327.     Line = erl_scan:line(T),
    1328. 

  1328.     Text = erl_scan:text(T),
    1329. 

  1329.     [Column, Line, Text].
    1330. 

  1330. 

  1331. token_info_long(T) ->
    1332. 

  1332.     Column = erl_scan:column(T),
    1333. 

  1333.     Line = erl_scan:line(T),
    1334. 

  1334.     Text = erl_scan:text(T),
    1335. 

  1335.     Category = erl_scan:category(T),
    1336. 

  1336.     Symbol = erl_scan:symbol(T),
    1337. 

  1337.     [{category,Category},{column,Column},{line,Line},
    1338. 

  1338.      {symbol,Symbol},{text,Text}].
    1339. 

  1339. 

  1340. test_strings([], _S, Line, Column) ->
    1341. 

  1341.     {Line,Column};
    1342. 

  1342. test_strings([{L,C,Len,T}=Attr|Attrs], String0, Line0, Column0) ->
    1343. 

  1343.     {String1, Column1} = skip_newlines(String0, L, Line0, Column0),
    1344. 

  1344.     String = skip_chars(String1, C-Column1),
    1345. 

  1345.     {Str,Rest} = lists:split(Len, String),
    1346. 

  1346.     if
    1347. 

  1347.         Str =:= T ->
    1348. 

  1348.             {Line,Column} = string_newlines(T, L, C),
    1349. 

  1349.             test_strings(Attrs, Rest, Line, Column);
    1350. 

  1350.         true ->
    1351. 

  1351.             {token_error, Attr, Str}
    1352. 

  1352.     end.
    1353. 

  1353. 

  1354. skip_newlines(String, Line, Line, Column) ->
    1355. 

  1355.     {String, Column};
    1356. 

  1356. skip_newlines([$\n|String], L, Line, _Column) ->
    1357. 

  1357.     skip_newlines(String, L, Line+1, 1);
    1358. 

  1358. skip_newlines([_|String], L, Line, Column) ->
    1359. 

  1359.     skip_newlines(String, L, Line, Column+1).
    1360. 

  1360. 

  1361. skip_chars(String, 0) ->
    1362. 

  1362.     String;
    1363. 

  1363. skip_chars([_|String], N) ->
    1364. 

  1364.     skip_chars(String, N-1).
    1365. 

  1365. 

  1366. string_newlines([$\n|String], Line, _Column) ->
    1367. 

  1367.     string_newlines(String, Line+1, 1);
    1368. 

  1368. string_newlines([], Line, Column) ->
    1369. 

  1369.     {Line, Column};
    1370. 

  1370. string_newlines([_|String], Line, Column) ->
    1371. 

  1371.     string_newlines(String, Line, Column+1).
    1372. 

  1372. 

  1373. scan_string_with_column(String, Options0) ->
    1374. 

  1374.     Options = [text | Options0],
    1375. 

  1375.     StartLoc = {1, 1},
    1376. 

  1376.     {ok, Ts1, End1} = erl_scan:string(String, StartLoc, Options),
    1377. 

  1377.     TString = String ++ ". ",
    1378. 

  1378.     {ok,Ts2,End2} = scan_tokens(TString, Options, [], StartLoc),
    1379. 

  1379.     {ok, Ts3, End3} =
    1380. 

  1380.         scan_tokens_1({more, []}, TString, Options, [], StartLoc),
    1381. 

  1381.     {end_2,End2,End3} = {end_2,End3,End2},
    1382. 

  1382.     {EndLine1,EndColumn1} = End1,
    1383. 

  1383.     End2 = {EndLine1,EndColumn1+2},
    1384. 

  1384.     {ts_1,Ts2,Ts3} = {ts_1,Ts3,Ts2},
    1385. 

  1385.     Ts2 = Ts1 ++ [lists:last(Ts2)],
    1386. 

  1386. 

  1387.     %% Attributes are keylists, but have no text.
    1388. 

  1388.     {ok, Ts7, End7} = erl_scan:string(String, {1,1}, Options),
    1389. 

  1389.     {ok, Ts8, End8} = scan_tokens(TString, Options, [], {1,1}),
    1390. 

  1390.     {end1, End1} = {end1, End7},
    1391. 

  1391.     {end2, End2} = {end2, End8},
    1392. 

  1392.     Ts8 = Ts7 ++ [lists:last(Ts8)],
    1393. 

  1393.     {cons,true} = {cons,consistent_attributes([Ts1,Ts2,Ts3,Ts7,Ts8])},
    1394. 

  1394. 

  1395.     {Ts1, End1}.
    1396. 

  1396. 

  1397. scan_tokens(String, Options, Rs, Location) ->
    1398. 

  1398.     case erl_scan:tokens([], String, Location, Options) of
    1399. 

  1399.         {done, {ok,Ts,End}, ""} ->
    1400. 

  1400.             {ok, lists:append(lists:reverse([Ts|Rs])), End};
    1401. 

  1401.         {done, {ok,Ts,End}, Rest} ->
    1402. 

  1402.             scan_tokens(Rest, Options, [Ts|Rs], End)
    1403. 

  1403.     end.
    1404. 

  1404. 

  1405. scan_tokens_1({done, {ok,Ts,End}, ""}, "", _Options, Rs, _Location) ->
    1406. 

  1406.     {ok,lists:append(lists:reverse([Ts|Rs])),End};
    1407. 

  1407. scan_tokens_1({done, {ok,Ts,End}, Rest}, Cs, Options, Rs, _Location) ->
    1408. 

  1408.     scan_tokens_1({more,[]}, Rest++Cs, Options, [Ts|Rs], End);
    1409. 

  1409. scan_tokens_1({more, Cont}, [C | Cs], Options, Rs, Loc) ->
    1410. 

  1410.     R = erl_scan:tokens(Cont, [C], Loc, Options),
    1411. 

  1411.     scan_tokens_1(R, Cs, Options, Rs, Loc).
    1412. 

  1412. 

  1413. consistent_attributes([]) ->
    1414. 

  1414.     true;
    1415. 

  1415. consistent_attributes([Ts | TsL]) ->
    1416. 

  1416.     L = [T || T <- Ts, is_integer(element(2, T))],
    1417. 

  1417.     case L of
    1418. 

  1418.         [] ->
    1419. 

  1419.             TagsL = [[Tag || {Tag,_} <- defined(token_info_long(T))] ||
    1420. 

  1420.                         T <- Ts],
    1421. 

  1421.             case lists:usort(TagsL) of
    1422. 

  1422.                 [_] ->
    1423. 

  1423.                     consistent_attributes(TsL);
    1424. 

  1424.                 [] when Ts =:= [] ->
    1425. 

  1425.                     consistent_attributes(TsL);
    1426. 

  1426.                 _ ->
    1427. 

  1427.                     Ts
    1428. 

  1428.             end;
    1429. 

  1429.         Ts ->
    1430. 

  1430.             consistent_attributes(TsL);
    1431. 

  1431.         _ ->
    1432. 

  1432.             Ts
    1433. 

  1433.     end.
    1434. 

  1434. 

  1435. defined(L) ->
    1436. 

  1436.     [{T,V} || {T,V} <- L, V =/= undefined].
    1437. 

  1437. 

  1438. family_list(L) ->
    1439. 

  1439.     sofs:to_external(family(L)).
    1440. 

  1440. 

  1441. family(L) ->
    1442. 

  1442.     sofs:relation_to_family(sofs:relation(L)).
    1443.