/src/test/java/com/alibaba/json/bvt/LexerTest.java

https://github.com/alibaba/fastjson · Java · 312 lines · 245 code · 50 blank · 17 comment · 5 complexity · 1bf8b6327506323939564f481e60be88 MD5 · raw file

  1. /*
  2. * Copyright 1999-2017 Alibaba Group.
  3. *
  4. * Licensed under the Apache License, Version 2.0 (the "License");
  5. * you may not use this file except in compliance with the License.
  6. * You may obtain a copy of the License at
  7. *
  8. * http://www.apache.org/licenses/LICENSE-2.0
  9. *
  10. * Unless required by applicable law or agreed to in writing, software
  11. * distributed under the License is distributed on an "AS IS" BASIS,
  12. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. * See the License for the specific language governing permissions and
  14. * limitations under the License.
  15. */
  16. package com.alibaba.json.bvt;
  17. import java.math.BigDecimal;
  18. import java.math.BigInteger;
  19. import org.junit.Assert;
  20. import junit.framework.TestCase;
  21. import com.alibaba.fastjson.JSON;
  22. import com.alibaba.fastjson.JSONArray;
  23. import com.alibaba.fastjson.parser.JSONScanner;
  24. import com.alibaba.fastjson.parser.JSONToken;
  25. public class LexerTest extends TestCase {
  26. public void test_float() throws Exception {
  27. String text = "123456789.0123";
  28. JSONScanner lexer = new JSONScanner(text);
  29. lexer.nextToken();
  30. BigDecimal decimalValue = lexer.decimalValue();
  31. Assert.assertEquals(new BigDecimal("123456789.0123"), decimalValue);
  32. }
  33. public void test_string() throws Exception {
  34. {
  35. JSONScanner lexer = new JSONScanner("\"中国\"");
  36. lexer.nextToken();
  37. Assert.assertEquals("中国", lexer.stringVal());
  38. }
  39. {
  40. JSONScanner lexer = new JSONScanner("\"中国\t\"");
  41. lexer.nextToken();
  42. Assert.assertEquals("中国\t", lexer.stringVal());
  43. }
  44. {
  45. JSONScanner lexer = new JSONScanner("\"中国\tV5\"");
  46. lexer.nextToken();
  47. Assert.assertEquals("中国\tV5", lexer.stringVal());
  48. }
  49. StringBuilder buf = new StringBuilder();
  50. buf.append('"');
  51. buf.append("\\\\\\/\\b\\f\\n\\r\\t\\u" + Integer.toHexString('中'));
  52. buf.append('"');
  53. buf.append('\u2001');
  54. String text = buf.toString();
  55. JSONScanner lexer = new JSONScanner(text.toCharArray(), text.length() - 1);
  56. lexer.nextToken();
  57. Assert.assertEquals(0, lexer.pos());
  58. String stringVal = lexer.stringVal();
  59. Assert.assertEquals("\"\\\\/\\b\\f\\n\\r\\t中\"", JSON.toJSONString(stringVal));
  60. }
  61. public void test_string2() throws Exception {
  62. StringBuilder buf = new StringBuilder();
  63. buf.append('"');
  64. for (int i = 0; i < 200; ++i) {
  65. buf.append("\\\\\\/\\b\\f\\n\\r\\t\\u" + Integer.toHexString('中'));
  66. }
  67. buf.append('"');
  68. String text = buf.toString();
  69. JSONScanner lexer = new JSONScanner(text.toCharArray(), text.length());
  70. lexer.nextToken();
  71. Assert.assertEquals(0, lexer.pos());
  72. lexer.stringVal();
  73. }
  74. public void test_string3() throws Exception {
  75. StringBuilder buf = new StringBuilder();
  76. buf.append('"');
  77. for (int i = 0; i < 200; ++i) {
  78. buf.append("abcdefghijklmn012345689ABCDEFG");
  79. }
  80. buf.append('"');
  81. String text = buf.toString();
  82. JSONScanner lexer = new JSONScanner(text.toCharArray(), text.length());
  83. lexer.nextToken();
  84. Assert.assertEquals(0, lexer.pos());
  85. lexer.stringVal();
  86. }
  87. public void test_string4() throws Exception {
  88. StringBuilder buf = new StringBuilder();
  89. buf.append('"');
  90. for (int i = 0; i < 200; ++i) {
  91. buf.append("\\tabcdefghijklmn012345689ABCDEFG");
  92. }
  93. buf.append('"');
  94. String text = buf.toString();
  95. JSONScanner lexer = new JSONScanner(text.toCharArray(), text.length());
  96. lexer.nextToken();
  97. Assert.assertEquals(0, lexer.pos());
  98. lexer.stringVal();
  99. // Assert.assertEquals("\"\\\\\\/\\b\\f\\n\\r\\t中\"",
  100. // JSON.toJSONString(stringVal));
  101. }
  102. public void test_empty() throws Exception {
  103. JSONScanner lexer = new JSONScanner("".toCharArray(), 0);
  104. lexer.nextToken();
  105. Assert.assertEquals(JSONToken.EOF, lexer.token());
  106. }
  107. public void test_isWhitespace() throws Exception {
  108. new JSONScanner("".toCharArray(), 0);
  109. Assert.assertTrue(JSONScanner.isWhitespace(' '));
  110. Assert.assertTrue(JSONScanner.isWhitespace('\b'));
  111. Assert.assertTrue(JSONScanner.isWhitespace('\f'));
  112. Assert.assertTrue(JSONScanner.isWhitespace('\n'));
  113. Assert.assertTrue(JSONScanner.isWhitespace('\r'));
  114. Assert.assertTrue(JSONScanner.isWhitespace('\t'));
  115. Assert.assertFalse(JSONScanner.isWhitespace('k'));
  116. }
  117. public void test_error() throws Exception {
  118. JSONScanner lexer = new JSONScanner("k");
  119. lexer.nextToken();
  120. Assert.assertEquals(JSONToken.ERROR, lexer.token());
  121. }
  122. public void test_error1() throws Exception {
  123. Exception error = null;
  124. try {
  125. JSONScanner lexer = new JSONScanner("\"\\k\"");
  126. lexer.nextToken();
  127. } catch (Exception ex) {
  128. error = ex;
  129. }
  130. Assert.assertNotNull(error);
  131. }
  132. public void f_test_ident() throws Exception {
  133. {
  134. JSONScanner lexer = new JSONScanner("ttue");
  135. lexer.nextToken();
  136. Assert.assertEquals(JSONToken.IDENTIFIER, lexer.token());
  137. }
  138. {
  139. JSONScanner lexer = new JSONScanner("tree");
  140. lexer.nextToken();
  141. Assert.assertEquals(JSONToken.IDENTIFIER, lexer.token());
  142. }
  143. {
  144. JSONScanner lexer = new JSONScanner("truu");
  145. lexer.nextToken();
  146. Assert.assertEquals(JSONToken.IDENTIFIER, lexer.token());
  147. }
  148. {
  149. JSONScanner lexer = new JSONScanner("fflse");
  150. lexer.nextToken();
  151. Assert.assertEquals(JSONToken.IDENTIFIER, lexer.token());
  152. }
  153. {
  154. JSONScanner lexer = new JSONScanner("nalse");
  155. lexer.nextToken();
  156. Assert.assertEquals(JSONToken.IDENTIFIER, lexer.token());
  157. }
  158. {
  159. JSONScanner lexer = new JSONScanner("faase");
  160. lexer.nextToken();
  161. Assert.assertEquals(JSONToken.IDENTIFIER, lexer.token());
  162. }
  163. {
  164. JSONScanner lexer = new JSONScanner("falle");
  165. lexer.nextToken();
  166. Assert.assertEquals(JSONToken.IDENTIFIER, lexer.token());
  167. }
  168. {
  169. JSONScanner lexer = new JSONScanner("falss");
  170. lexer.nextToken();
  171. Assert.assertEquals(JSONToken.IDENTIFIER, lexer.token());
  172. }
  173. {
  174. JSONScanner lexer = new JSONScanner("nnll");
  175. lexer.nextToken();
  176. Assert.assertEquals(JSONToken.IDENTIFIER, lexer.token());
  177. }
  178. {
  179. JSONScanner lexer = new JSONScanner("nuul");
  180. lexer.nextToken();
  181. Assert.assertEquals(JSONToken.IDENTIFIER, lexer.token());
  182. }
  183. {
  184. JSONScanner lexer = new JSONScanner("nulk");
  185. lexer.nextToken();
  186. Assert.assertEquals(JSONToken.IDENTIFIER, lexer.token());
  187. }
  188. {
  189. StringBuilder buf = new StringBuilder();
  190. buf.append('n');
  191. for (char ch = 'A'; ch <= 'Z'; ++ch) {
  192. buf.append(ch);
  193. }
  194. for (char ch = 'a'; ch <= 'z'; ++ch) {
  195. buf.append(ch);
  196. }
  197. JSONScanner lexer = new JSONScanner(buf.toString());
  198. lexer.nextToken();
  199. Assert.assertEquals(JSONToken.IDENTIFIER, lexer.token());
  200. }
  201. }
  202. public void test_number() throws Exception {
  203. String text = "[0,1,-1,2E3,2E+3,2E-3,2e3,2e+3,2e-3]";
  204. JSONArray array = JSON.parseArray(text);
  205. Assert.assertEquals(0, array.get(0));
  206. Assert.assertEquals(1, array.get(1));
  207. Assert.assertEquals(-1, array.get(2));
  208. Assert.assertEquals(new BigDecimal("2E3"), array.get(3));
  209. Assert.assertEquals(new BigDecimal("2E3"), array.get(4));
  210. Assert.assertEquals(new BigDecimal("2E-3"), array.get(5));
  211. Assert.assertEquals(new BigDecimal("2E3"), array.get(6));
  212. Assert.assertEquals(new BigDecimal("2E3"), array.get(7));
  213. Assert.assertEquals(new BigDecimal("2E-3"), array.get(8));
  214. for (long i = Long.MIN_VALUE; i <= Long.MIN_VALUE + 1000 * 10; ++i) {
  215. Assert.assertEquals(i, JSON.parse(Long.toString(i)));
  216. }
  217. for (long i = Long.MAX_VALUE - 1000 * 10; i <= Long.MAX_VALUE && i > 0; ++i) {
  218. Assert.assertEquals(i, JSON.parse(Long.toString(i)));
  219. }
  220. }
  221. public void test_big_integer_1() throws Exception {
  222. String text = Long.MAX_VALUE + "1234";
  223. JSONScanner lexer = new JSONScanner(text);
  224. lexer.nextToken();
  225. Assert.assertEquals(new BigInteger(text), lexer.integerValue());
  226. }
  227. public void test_big_integer_2() throws Exception {
  228. String text = Long.MIN_VALUE + "1234";
  229. JSONScanner lexer = new JSONScanner(text);
  230. lexer.nextToken();
  231. Assert.assertEquals(new BigInteger(text), lexer.integerValue());
  232. }
  233. public void test_big_integer_3() throws Exception {
  234. String text = "9223372036854775809";
  235. JSONScanner lexer = new JSONScanner(text);
  236. lexer.nextToken();
  237. Assert.assertEquals(new BigInteger(text), lexer.integerValue());
  238. }
  239. public void test_error2() {
  240. Exception error = null;
  241. try {
  242. JSONScanner lexer = new JSONScanner("--");
  243. lexer.nextToken();
  244. lexer.integerValue();
  245. } catch (Exception ex) {
  246. error = ex;
  247. }
  248. Assert.assertNotNull(error);
  249. }
  250. public void test_error3() {
  251. Exception error = null;
  252. try {
  253. JSONScanner lexer = new JSONScanner("");
  254. lexer.nextToken();
  255. lexer.nextToken();
  256. lexer.integerValue();
  257. } catch (Exception ex) {
  258. error = ex;
  259. }
  260. Assert.assertNotNull(error);
  261. }
  262. }