/Modules/cjkcodecs/cjkcodecs.h

  1/*
  2 * cjkcodecs.h: common header for cjkcodecs
  3 *
  4 * Written by Hye-Shik Chang <perky@FreeBSD.org>
  5 */
  6
  7#ifndef _CJKCODECS_H_
  8#define _CJKCODECS_H_
  9
 10#define PY_SSIZE_T_CLEAN
 11#include "Python.h"
 12#include "multibytecodec.h"
 13
 14
 15/* a unicode "undefined" codepoint */
 16#define UNIINV	0xFFFE
 17
 18/* internal-use DBCS codepoints which aren't used by any charsets */
 19#define NOCHAR	0xFFFF
 20#define MULTIC	0xFFFE
 21#define DBCINV	0xFFFD
 22
 23/* shorter macros to save source size of mapping tables */
 24#define U UNIINV
 25#define N NOCHAR
 26#define M MULTIC
 27#define D DBCINV
 28
 29struct dbcs_index {
 30	const ucs2_t *map;
 31	unsigned char bottom, top;
 32};
 33typedef struct dbcs_index decode_map;
 34
 35struct widedbcs_index {
 36	const ucs4_t *map;
 37	unsigned char bottom, top;
 38};
 39typedef struct widedbcs_index widedecode_map;
 40
 41struct unim_index {
 42	const DBCHAR *map;
 43	unsigned char bottom, top;
 44};
 45typedef struct unim_index encode_map;
 46
 47struct unim_index_bytebased {
 48	const unsigned char *map;
 49	unsigned char bottom, top;
 50};
 51
 52struct dbcs_map {
 53	const char *charset;
 54	const struct unim_index *encmap;
 55	const struct dbcs_index *decmap;
 56};
 57
 58struct pair_encodemap {
 59	ucs4_t uniseq;
 60	DBCHAR code;
 61};
 62
 63static const MultibyteCodec *codec_list;
 64static const struct dbcs_map *mapping_list;
 65
 66#define CODEC_INIT(encoding)						\
 67	static int encoding##_codec_init(const void *config)
 68
 69#define ENCODER_INIT(encoding)						\
 70	static int encoding##_encode_init(				\
 71		MultibyteCodec_State *state, const void *config)
 72#define ENCODER(encoding)						\
 73	static Py_ssize_t encoding##_encode(				\
 74		MultibyteCodec_State *state, const void *config,	\
 75		const Py_UNICODE **inbuf, Py_ssize_t inleft,		\
 76		unsigned char **outbuf, Py_ssize_t outleft, int flags)
 77#define ENCODER_RESET(encoding)						\
 78	static Py_ssize_t encoding##_encode_reset(			\
 79		MultibyteCodec_State *state, const void *config,	\
 80		unsigned char **outbuf, Py_ssize_t outleft)
 81
 82#define DECODER_INIT(encoding)						\
 83	static int encoding##_decode_init(				\
 84		MultibyteCodec_State *state, const void *config)
 85#define DECODER(encoding)						\
 86	static Py_ssize_t encoding##_decode(				\
 87		MultibyteCodec_State *state, const void *config,	\
 88		const unsigned char **inbuf, Py_ssize_t inleft,		\
 89		Py_UNICODE **outbuf, Py_ssize_t outleft)
 90#define DECODER_RESET(encoding)						\
 91	static Py_ssize_t encoding##_decode_reset(			\
 92		MultibyteCodec_State *state, const void *config)
 93
 94#if Py_UNICODE_SIZE == 4
 95#define UCS4INVALID(code)	\
 96	if ((code) > 0xFFFF)	\
 97	return 1;
 98#else
 99#define UCS4INVALID(code)	\
100	if (0) ;
101#endif
102
103#define NEXT_IN(i)				\
104	(*inbuf) += (i);			\
105	(inleft) -= (i);
106#define NEXT_OUT(o)				\
107	(*outbuf) += (o);			\
108	(outleft) -= (o);
109#define NEXT(i, o)				\
110	NEXT_IN(i) NEXT_OUT(o)
111
112#define REQUIRE_INBUF(n)			\
113	if (inleft < (n))			\
114		return MBERR_TOOFEW;
115#define REQUIRE_OUTBUF(n)			\
116	if (outleft < (n))			\
117		return MBERR_TOOSMALL;
118
119#define IN1 ((*inbuf)[0])
120#define IN2 ((*inbuf)[1])
121#define IN3 ((*inbuf)[2])
122#define IN4 ((*inbuf)[3])
123
124#define OUT1(c) ((*outbuf)[0]) = (c);
125#define OUT2(c) ((*outbuf)[1]) = (c);
126#define OUT3(c) ((*outbuf)[2]) = (c);
127#define OUT4(c) ((*outbuf)[3]) = (c);
128
129#define WRITE1(c1)		\
130	REQUIRE_OUTBUF(1)	\
131	(*outbuf)[0] = (c1);
132#define WRITE2(c1, c2)		\
133	REQUIRE_OUTBUF(2)	\
134	(*outbuf)[0] = (c1);	\
135	(*outbuf)[1] = (c2);
136#define WRITE3(c1, c2, c3)	\
137	REQUIRE_OUTBUF(3)	\
138	(*outbuf)[0] = (c1);	\
139	(*outbuf)[1] = (c2);	\
140	(*outbuf)[2] = (c3);
141#define WRITE4(c1, c2, c3, c4)	\
142	REQUIRE_OUTBUF(4)	\
143	(*outbuf)[0] = (c1);	\
144	(*outbuf)[1] = (c2);	\
145	(*outbuf)[2] = (c3);	\
146	(*outbuf)[3] = (c4);
147
148#if Py_UNICODE_SIZE == 2
149# define WRITEUCS4(c)						\
150	REQUIRE_OUTBUF(2)					\
151	(*outbuf)[0] = 0xd800 + (((c) - 0x10000) >> 10);	\
152	(*outbuf)[1] = 0xdc00 + (((c) - 0x10000) & 0x3ff);	\
153	NEXT_OUT(2)
154#else
155# define WRITEUCS4(c)						\
156	REQUIRE_OUTBUF(1)					\
157	**outbuf = (Py_UNICODE)(c);				\
158	NEXT_OUT(1)
159#endif
160
161#define _TRYMAP_ENC(m, assi, val)				\
162	((m)->map != NULL && (val) >= (m)->bottom &&		\
163	    (val)<= (m)->top && ((assi) = (m)->map[(val) -	\
164	    (m)->bottom]) != NOCHAR)
165#define TRYMAP_ENC_COND(charset, assi, uni)			\
166	_TRYMAP_ENC(&charset##_encmap[(uni) >> 8], assi, (uni) & 0xff)
167#define TRYMAP_ENC(charset, assi, uni)				\
168	if TRYMAP_ENC_COND(charset, assi, uni)
169
170#define _TRYMAP_DEC(m, assi, val)				\
171	((m)->map != NULL && (val) >= (m)->bottom &&		\
172	    (val)<= (m)->top && ((assi) = (m)->map[(val) -	\
173	    (m)->bottom]) != UNIINV)
174#define TRYMAP_DEC(charset, assi, c1, c2)			\
175	if _TRYMAP_DEC(&charset##_decmap[c1], assi, c2)
176
177#define _TRYMAP_ENC_MPLANE(m, assplane, asshi, asslo, val)	\
178	((m)->map != NULL && (val) >= (m)->bottom &&		\
179	    (val)<= (m)->top &&					\
180	    ((assplane) = (m)->map[((val) - (m)->bottom)*3]) != 0 && \
181	    (((asshi) = (m)->map[((val) - (m)->bottom)*3 + 1]), 1) && \
182	    (((asslo) = (m)->map[((val) - (m)->bottom)*3 + 2]), 1))
183#define TRYMAP_ENC_MPLANE(charset, assplane, asshi, asslo, uni)	\
184	if _TRYMAP_ENC_MPLANE(&charset##_encmap[(uni) >> 8], \
185			   assplane, asshi, asslo, (uni) & 0xff)
186#define TRYMAP_DEC_MPLANE(charset, assi, plane, c1, c2)		\
187	if _TRYMAP_DEC(&charset##_decmap[plane][c1], assi, c2)
188
189#if Py_UNICODE_SIZE == 2
190#define DECODE_SURROGATE(c)					\
191	if (c >> 10 == 0xd800 >> 10) { /* high surrogate */	\
192		REQUIRE_INBUF(2)				\
193		if (IN2 >> 10 == 0xdc00 >> 10) { /* low surrogate */ \
194		    c = 0x10000 + ((ucs4_t)(c - 0xd800) << 10) + \
195			((ucs4_t)(IN2) - 0xdc00);		\
196		}						\
197	}
198#define GET_INSIZE(c)	((c) > 0xffff ? 2 : 1)
199#else
200#define DECODE_SURROGATE(c) {;}
201#define GET_INSIZE(c)	1
202#endif
203
204#define BEGIN_MAPPINGS_LIST static const struct dbcs_map _mapping_list[] = {
205#define MAPPING_ENCONLY(enc) {#enc, (void*)enc##_encmap, NULL},
206#define MAPPING_DECONLY(enc) {#enc, NULL, (void*)enc##_decmap},
207#define MAPPING_ENCDEC(enc) {#enc, (void*)enc##_encmap, (void*)enc##_decmap},
208#define END_MAPPINGS_LIST				\
209	{"", NULL, NULL} };				\
210	static const struct dbcs_map *mapping_list =	\
211		(const struct dbcs_map *)_mapping_list;
212
213#define BEGIN_CODECS_LIST static const MultibyteCodec _codec_list[] = {
214#define _STATEFUL_METHODS(enc)		\
215	enc##_encode,			\
216	enc##_encode_init,		\
217	enc##_encode_reset,		\
218	enc##_decode,			\
219	enc##_decode_init,		\
220	enc##_decode_reset,
221#define _STATELESS_METHODS(enc)		\
222	enc##_encode, NULL, NULL,	\
223	enc##_decode, NULL, NULL,
224#define CODEC_STATEFUL(enc) {		\
225	#enc, NULL, NULL,		\
226	_STATEFUL_METHODS(enc)		\
227},
228#define CODEC_STATELESS(enc) {		\
229	#enc, NULL, NULL,		\
230	_STATELESS_METHODS(enc)		\
231},
232#define CODEC_STATELESS_WINIT(enc) {	\
233	#enc, NULL,			\
234	enc##_codec_init,		\
235	_STATELESS_METHODS(enc)		\
236},
237#define END_CODECS_LIST					\
238	{"", NULL,} };					\
239	static const MultibyteCodec *codec_list =	\
240		(const MultibyteCodec *)_codec_list;
241
242static PyObject *
243getmultibytecodec(void)
244{
245	static PyObject *cofunc = NULL;
246
247	if (cofunc == NULL) {
248		PyObject *mod = PyImport_ImportModuleNoBlock("_multibytecodec");
249		if (mod == NULL)
250			return NULL;
251		cofunc = PyObject_GetAttrString(mod, "__create_codec");
252		Py_DECREF(mod);
253	}
254	return cofunc;
255}
256
257static PyObject *
258getcodec(PyObject *self, PyObject *encoding)
259{
260	PyObject *codecobj, *r, *cofunc;
261	const MultibyteCodec *codec;
262	const char *enc;
263
264	if (!PyString_Check(encoding)) {
265		PyErr_SetString(PyExc_TypeError,
266				"encoding name must be a string.");
267		return NULL;
268	}
269
270	cofunc = getmultibytecodec();
271	if (cofunc == NULL)
272		return NULL;
273
274	enc = PyString_AS_STRING(encoding);
275	for (codec = codec_list; codec->encoding[0]; codec++)
276		if (strcmp(codec->encoding, enc) == 0)
277			break;
278
279	if (codec->encoding[0] == '\0') {
280		PyErr_SetString(PyExc_LookupError,
281				"no such codec is supported.");
282		return NULL;
283	}
284
285	codecobj = PyCObject_FromVoidPtr((void *)codec, NULL);
286	if (codecobj == NULL)
287		return NULL;
288
289	r = PyObject_CallFunctionObjArgs(cofunc, codecobj, NULL);
290	Py_DECREF(codecobj);
291
292	return r;
293}
294
295static struct PyMethodDef __methods[] = {
296	{"getcodec", (PyCFunction)getcodec, METH_O, ""},
297	{NULL, NULL},
298};
299
300static int
301register_maps(PyObject *module)
302{
303	const struct dbcs_map *h;
304
305	for (h = mapping_list; h->charset[0] != '\0'; h++) {
306		char mhname[256] = "__map_";
307		int r;
308		strcpy(mhname + sizeof("__map_") - 1, h->charset);
309		r = PyModule_AddObject(module, mhname,
310				PyCObject_FromVoidPtr((void *)h, NULL));
311		if (r == -1)
312			return -1;
313	}
314	return 0;
315}
316
317#ifdef USING_BINARY_PAIR_SEARCH
318static DBCHAR
319find_pairencmap(ucs2_t body, ucs2_t modifier,
320		const struct pair_encodemap *haystack, int haystacksize)
321{
322	int pos, min, max;
323	ucs4_t value = body << 16 | modifier;
324
325	min = 0;
326	max = haystacksize;
327
328	for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1)
329		if (value < haystack[pos].uniseq) {
330			if (max == pos) break;
331			else max = pos;
332		}
333		else if (value > haystack[pos].uniseq) {
334			if (min == pos) break;
335			else min = pos;
336		}
337		else
338			break;
339
340		if (value == haystack[pos].uniseq)
341			return haystack[pos].code;
342		else
343			return DBCINV;
344}
345#endif
346
347#ifdef USING_IMPORTED_MAPS
348#define IMPORT_MAP(locale, charset, encmap, decmap) \
349	importmap("_codecs_" #locale, "__map_" #charset, \
350		  (const void**)encmap, (const void**)decmap)
351
352static int
353importmap(const char *modname, const char *symbol,
354	  const void **encmap, const void **decmap)
355{
356	PyObject *o, *mod;
357
358	mod = PyImport_ImportModule((char *)modname);
359	if (mod == NULL)
360		return -1;
361
362	o = PyObject_GetAttrString(mod, (char*)symbol);
363	if (o == NULL)
364		goto errorexit;
365	else if (!PyCObject_Check(o)) {
366		PyErr_SetString(PyExc_ValueError,
367				"map data must be a CObject.");
368		goto errorexit;
369	}
370	else {
371		struct dbcs_map *map;
372		map = PyCObject_AsVoidPtr(o);
373		if (encmap != NULL)
374			*encmap = map->encmap;
375		if (decmap != NULL)
376			*decmap = map->decmap;
377		Py_DECREF(o);
378	}
379
380	Py_DECREF(mod);
381	return 0;
382
383errorexit:
384	Py_DECREF(mod);
385	return -1;
386}
387#endif
388
389#define I_AM_A_MODULE_FOR(loc)						\
390	void								\
391	init_codecs_##loc(void)						\
392	{								\
393		PyObject *m = Py_InitModule("_codecs_" #loc, __methods);\
394		if (m != NULL)						\
395			(void)register_maps(m);				\
396	}
397
398#endif