/Modules/cjkcodecs/cjkcodecs.h

http://unladen-swallow.googlecode.com/ · C Header · 398 lines · 336 code · 54 blank · 8 comment · 72 complexity · a808a7b45879ea3f41d5e1eccd72e166 MD5 · raw file

  1. /*
  2. * cjkcodecs.h: common header for cjkcodecs
  3. *
  4. * Written by Hye-Shik Chang <perky@FreeBSD.org>
  5. */
  6. #ifndef _CJKCODECS_H_
  7. #define _CJKCODECS_H_
  8. #define PY_SSIZE_T_CLEAN
  9. #include "Python.h"
  10. #include "multibytecodec.h"
  11. /* a unicode "undefined" codepoint */
  12. #define UNIINV 0xFFFE
  13. /* internal-use DBCS codepoints which aren't used by any charsets */
  14. #define NOCHAR 0xFFFF
  15. #define MULTIC 0xFFFE
  16. #define DBCINV 0xFFFD
  17. /* shorter macros to save source size of mapping tables */
  18. #define U UNIINV
  19. #define N NOCHAR
  20. #define M MULTIC
  21. #define D DBCINV
  22. struct dbcs_index {
  23. const ucs2_t *map;
  24. unsigned char bottom, top;
  25. };
  26. typedef struct dbcs_index decode_map;
  27. struct widedbcs_index {
  28. const ucs4_t *map;
  29. unsigned char bottom, top;
  30. };
  31. typedef struct widedbcs_index widedecode_map;
  32. struct unim_index {
  33. const DBCHAR *map;
  34. unsigned char bottom, top;
  35. };
  36. typedef struct unim_index encode_map;
  37. struct unim_index_bytebased {
  38. const unsigned char *map;
  39. unsigned char bottom, top;
  40. };
  41. struct dbcs_map {
  42. const char *charset;
  43. const struct unim_index *encmap;
  44. const struct dbcs_index *decmap;
  45. };
  46. struct pair_encodemap {
  47. ucs4_t uniseq;
  48. DBCHAR code;
  49. };
  50. static const MultibyteCodec *codec_list;
  51. static const struct dbcs_map *mapping_list;
  52. #define CODEC_INIT(encoding) \
  53. static int encoding##_codec_init(const void *config)
  54. #define ENCODER_INIT(encoding) \
  55. static int encoding##_encode_init( \
  56. MultibyteCodec_State *state, const void *config)
  57. #define ENCODER(encoding) \
  58. static Py_ssize_t encoding##_encode( \
  59. MultibyteCodec_State *state, const void *config, \
  60. const Py_UNICODE **inbuf, Py_ssize_t inleft, \
  61. unsigned char **outbuf, Py_ssize_t outleft, int flags)
  62. #define ENCODER_RESET(encoding) \
  63. static Py_ssize_t encoding##_encode_reset( \
  64. MultibyteCodec_State *state, const void *config, \
  65. unsigned char **outbuf, Py_ssize_t outleft)
  66. #define DECODER_INIT(encoding) \
  67. static int encoding##_decode_init( \
  68. MultibyteCodec_State *state, const void *config)
  69. #define DECODER(encoding) \
  70. static Py_ssize_t encoding##_decode( \
  71. MultibyteCodec_State *state, const void *config, \
  72. const unsigned char **inbuf, Py_ssize_t inleft, \
  73. Py_UNICODE **outbuf, Py_ssize_t outleft)
  74. #define DECODER_RESET(encoding) \
  75. static Py_ssize_t encoding##_decode_reset( \
  76. MultibyteCodec_State *state, const void *config)
  77. #if Py_UNICODE_SIZE == 4
  78. #define UCS4INVALID(code) \
  79. if ((code) > 0xFFFF) \
  80. return 1;
  81. #else
  82. #define UCS4INVALID(code) \
  83. if (0) ;
  84. #endif
  85. #define NEXT_IN(i) \
  86. (*inbuf) += (i); \
  87. (inleft) -= (i);
  88. #define NEXT_OUT(o) \
  89. (*outbuf) += (o); \
  90. (outleft) -= (o);
  91. #define NEXT(i, o) \
  92. NEXT_IN(i) NEXT_OUT(o)
  93. #define REQUIRE_INBUF(n) \
  94. if (inleft < (n)) \
  95. return MBERR_TOOFEW;
  96. #define REQUIRE_OUTBUF(n) \
  97. if (outleft < (n)) \
  98. return MBERR_TOOSMALL;
  99. #define IN1 ((*inbuf)[0])
  100. #define IN2 ((*inbuf)[1])
  101. #define IN3 ((*inbuf)[2])
  102. #define IN4 ((*inbuf)[3])
  103. #define OUT1(c) ((*outbuf)[0]) = (c);
  104. #define OUT2(c) ((*outbuf)[1]) = (c);
  105. #define OUT3(c) ((*outbuf)[2]) = (c);
  106. #define OUT4(c) ((*outbuf)[3]) = (c);
  107. #define WRITE1(c1) \
  108. REQUIRE_OUTBUF(1) \
  109. (*outbuf)[0] = (c1);
  110. #define WRITE2(c1, c2) \
  111. REQUIRE_OUTBUF(2) \
  112. (*outbuf)[0] = (c1); \
  113. (*outbuf)[1] = (c2);
  114. #define WRITE3(c1, c2, c3) \
  115. REQUIRE_OUTBUF(3) \
  116. (*outbuf)[0] = (c1); \
  117. (*outbuf)[1] = (c2); \
  118. (*outbuf)[2] = (c3);
  119. #define WRITE4(c1, c2, c3, c4) \
  120. REQUIRE_OUTBUF(4) \
  121. (*outbuf)[0] = (c1); \
  122. (*outbuf)[1] = (c2); \
  123. (*outbuf)[2] = (c3); \
  124. (*outbuf)[3] = (c4);
  125. #if Py_UNICODE_SIZE == 2
  126. # define WRITEUCS4(c) \
  127. REQUIRE_OUTBUF(2) \
  128. (*outbuf)[0] = 0xd800 + (((c) - 0x10000) >> 10); \
  129. (*outbuf)[1] = 0xdc00 + (((c) - 0x10000) & 0x3ff); \
  130. NEXT_OUT(2)
  131. #else
  132. # define WRITEUCS4(c) \
  133. REQUIRE_OUTBUF(1) \
  134. **outbuf = (Py_UNICODE)(c); \
  135. NEXT_OUT(1)
  136. #endif
  137. #define _TRYMAP_ENC(m, assi, val) \
  138. ((m)->map != NULL && (val) >= (m)->bottom && \
  139. (val)<= (m)->top && ((assi) = (m)->map[(val) - \
  140. (m)->bottom]) != NOCHAR)
  141. #define TRYMAP_ENC_COND(charset, assi, uni) \
  142. _TRYMAP_ENC(&charset##_encmap[(uni) >> 8], assi, (uni) & 0xff)
  143. #define TRYMAP_ENC(charset, assi, uni) \
  144. if TRYMAP_ENC_COND(charset, assi, uni)
  145. #define _TRYMAP_DEC(m, assi, val) \
  146. ((m)->map != NULL && (val) >= (m)->bottom && \
  147. (val)<= (m)->top && ((assi) = (m)->map[(val) - \
  148. (m)->bottom]) != UNIINV)
  149. #define TRYMAP_DEC(charset, assi, c1, c2) \
  150. if _TRYMAP_DEC(&charset##_decmap[c1], assi, c2)
  151. #define _TRYMAP_ENC_MPLANE(m, assplane, asshi, asslo, val) \
  152. ((m)->map != NULL && (val) >= (m)->bottom && \
  153. (val)<= (m)->top && \
  154. ((assplane) = (m)->map[((val) - (m)->bottom)*3]) != 0 && \
  155. (((asshi) = (m)->map[((val) - (m)->bottom)*3 + 1]), 1) && \
  156. (((asslo) = (m)->map[((val) - (m)->bottom)*3 + 2]), 1))
  157. #define TRYMAP_ENC_MPLANE(charset, assplane, asshi, asslo, uni) \
  158. if _TRYMAP_ENC_MPLANE(&charset##_encmap[(uni) >> 8], \
  159. assplane, asshi, asslo, (uni) & 0xff)
  160. #define TRYMAP_DEC_MPLANE(charset, assi, plane, c1, c2) \
  161. if _TRYMAP_DEC(&charset##_decmap[plane][c1], assi, c2)
  162. #if Py_UNICODE_SIZE == 2
  163. #define DECODE_SURROGATE(c) \
  164. if (c >> 10 == 0xd800 >> 10) { /* high surrogate */ \
  165. REQUIRE_INBUF(2) \
  166. if (IN2 >> 10 == 0xdc00 >> 10) { /* low surrogate */ \
  167. c = 0x10000 + ((ucs4_t)(c - 0xd800) << 10) + \
  168. ((ucs4_t)(IN2) - 0xdc00); \
  169. } \
  170. }
  171. #define GET_INSIZE(c) ((c) > 0xffff ? 2 : 1)
  172. #else
  173. #define DECODE_SURROGATE(c) {;}
  174. #define GET_INSIZE(c) 1
  175. #endif
  176. #define BEGIN_MAPPINGS_LIST static const struct dbcs_map _mapping_list[] = {
  177. #define MAPPING_ENCONLY(enc) {#enc, (void*)enc##_encmap, NULL},
  178. #define MAPPING_DECONLY(enc) {#enc, NULL, (void*)enc##_decmap},
  179. #define MAPPING_ENCDEC(enc) {#enc, (void*)enc##_encmap, (void*)enc##_decmap},
  180. #define END_MAPPINGS_LIST \
  181. {"", NULL, NULL} }; \
  182. static const struct dbcs_map *mapping_list = \
  183. (const struct dbcs_map *)_mapping_list;
  184. #define BEGIN_CODECS_LIST static const MultibyteCodec _codec_list[] = {
  185. #define _STATEFUL_METHODS(enc) \
  186. enc##_encode, \
  187. enc##_encode_init, \
  188. enc##_encode_reset, \
  189. enc##_decode, \
  190. enc##_decode_init, \
  191. enc##_decode_reset,
  192. #define _STATELESS_METHODS(enc) \
  193. enc##_encode, NULL, NULL, \
  194. enc##_decode, NULL, NULL,
  195. #define CODEC_STATEFUL(enc) { \
  196. #enc, NULL, NULL, \
  197. _STATEFUL_METHODS(enc) \
  198. },
  199. #define CODEC_STATELESS(enc) { \
  200. #enc, NULL, NULL, \
  201. _STATELESS_METHODS(enc) \
  202. },
  203. #define CODEC_STATELESS_WINIT(enc) { \
  204. #enc, NULL, \
  205. enc##_codec_init, \
  206. _STATELESS_METHODS(enc) \
  207. },
  208. #define END_CODECS_LIST \
  209. {"", NULL,} }; \
  210. static const MultibyteCodec *codec_list = \
  211. (const MultibyteCodec *)_codec_list;
  212. static PyObject *
  213. getmultibytecodec(void)
  214. {
  215. static PyObject *cofunc = NULL;
  216. if (cofunc == NULL) {
  217. PyObject *mod = PyImport_ImportModuleNoBlock("_multibytecodec");
  218. if (mod == NULL)
  219. return NULL;
  220. cofunc = PyObject_GetAttrString(mod, "__create_codec");
  221. Py_DECREF(mod);
  222. }
  223. return cofunc;
  224. }
  225. static PyObject *
  226. getcodec(PyObject *self, PyObject *encoding)
  227. {
  228. PyObject *codecobj, *r, *cofunc;
  229. const MultibyteCodec *codec;
  230. const char *enc;
  231. if (!PyString_Check(encoding)) {
  232. PyErr_SetString(PyExc_TypeError,
  233. "encoding name must be a string.");
  234. return NULL;
  235. }
  236. cofunc = getmultibytecodec();
  237. if (cofunc == NULL)
  238. return NULL;
  239. enc = PyString_AS_STRING(encoding);
  240. for (codec = codec_list; codec->encoding[0]; codec++)
  241. if (strcmp(codec->encoding, enc) == 0)
  242. break;
  243. if (codec->encoding[0] == '\0') {
  244. PyErr_SetString(PyExc_LookupError,
  245. "no such codec is supported.");
  246. return NULL;
  247. }
  248. codecobj = PyCObject_FromVoidPtr((void *)codec, NULL);
  249. if (codecobj == NULL)
  250. return NULL;
  251. r = PyObject_CallFunctionObjArgs(cofunc, codecobj, NULL);
  252. Py_DECREF(codecobj);
  253. return r;
  254. }
  255. static struct PyMethodDef __methods[] = {
  256. {"getcodec", (PyCFunction)getcodec, METH_O, ""},
  257. {NULL, NULL},
  258. };
  259. static int
  260. register_maps(PyObject *module)
  261. {
  262. const struct dbcs_map *h;
  263. for (h = mapping_list; h->charset[0] != '\0'; h++) {
  264. char mhname[256] = "__map_";
  265. int r;
  266. strcpy(mhname + sizeof("__map_") - 1, h->charset);
  267. r = PyModule_AddObject(module, mhname,
  268. PyCObject_FromVoidPtr((void *)h, NULL));
  269. if (r == -1)
  270. return -1;
  271. }
  272. return 0;
  273. }
  274. #ifdef USING_BINARY_PAIR_SEARCH
  275. static DBCHAR
  276. find_pairencmap(ucs2_t body, ucs2_t modifier,
  277. const struct pair_encodemap *haystack, int haystacksize)
  278. {
  279. int pos, min, max;
  280. ucs4_t value = body << 16 | modifier;
  281. min = 0;
  282. max = haystacksize;
  283. for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1)
  284. if (value < haystack[pos].uniseq) {
  285. if (max == pos) break;
  286. else max = pos;
  287. }
  288. else if (value > haystack[pos].uniseq) {
  289. if (min == pos) break;
  290. else min = pos;
  291. }
  292. else
  293. break;
  294. if (value == haystack[pos].uniseq)
  295. return haystack[pos].code;
  296. else
  297. return DBCINV;
  298. }
  299. #endif
  300. #ifdef USING_IMPORTED_MAPS
  301. #define IMPORT_MAP(locale, charset, encmap, decmap) \
  302. importmap("_codecs_" #locale, "__map_" #charset, \
  303. (const void**)encmap, (const void**)decmap)
  304. static int
  305. importmap(const char *modname, const char *symbol,
  306. const void **encmap, const void **decmap)
  307. {
  308. PyObject *o, *mod;
  309. mod = PyImport_ImportModule((char *)modname);
  310. if (mod == NULL)
  311. return -1;
  312. o = PyObject_GetAttrString(mod, (char*)symbol);
  313. if (o == NULL)
  314. goto errorexit;
  315. else if (!PyCObject_Check(o)) {
  316. PyErr_SetString(PyExc_ValueError,
  317. "map data must be a CObject.");
  318. goto errorexit;
  319. }
  320. else {
  321. struct dbcs_map *map;
  322. map = PyCObject_AsVoidPtr(o);
  323. if (encmap != NULL)
  324. *encmap = map->encmap;
  325. if (decmap != NULL)
  326. *decmap = map->decmap;
  327. Py_DECREF(o);
  328. }
  329. Py_DECREF(mod);
  330. return 0;
  331. errorexit:
  332. Py_DECREF(mod);
  333. return -1;
  334. }
  335. #endif
  336. #define I_AM_A_MODULE_FOR(loc) \
  337. void \
  338. init_codecs_##loc(void) \
  339. { \
  340. PyObject *m = Py_InitModule("_codecs_" #loc, __methods);\
  341. if (m != NULL) \
  342. (void)register_maps(m); \
  343. }
  344. #endif