/Parser/asdl_c.py

http://unladen-swallow.googlecode.com/ · Python · 1210 lines · 1035 code · 124 blank · 51 comment · 140 complexity · 599a148b59405c45e28c56ccad0d03b4 MD5 · raw file

  1. #! /usr/bin/env python
  2. """Generate C code from an ASDL description."""
  3. # TO DO
  4. # handle fields that have a type but no name
  5. import os, sys
  6. import asdl
  7. TABSIZE = 8
  8. MAX_COL = 80
  9. def get_c_type(name):
  10. """Return a string for the C name of the type.
  11. This function special cases the default types provided by asdl:
  12. identifier, string, int, bool.
  13. """
  14. # XXX ack! need to figure out where Id is useful and where string
  15. if isinstance(name, asdl.Id):
  16. name = name.value
  17. if name in asdl.builtin_types:
  18. return name
  19. else:
  20. return "%s_ty" % name
  21. def reflow_lines(s, depth):
  22. """Reflow the line s indented depth tabs.
  23. Return a sequence of lines where no line extends beyond MAX_COL
  24. when properly indented. The first line is properly indented based
  25. exclusively on depth * TABSIZE. All following lines -- these are
  26. the reflowed lines generated by this function -- start at the same
  27. column as the first character beyond the opening { in the first
  28. line.
  29. """
  30. size = MAX_COL - depth * TABSIZE
  31. if len(s) < size:
  32. return [s]
  33. lines = []
  34. cur = s
  35. padding = ""
  36. while len(cur) > size:
  37. i = cur.rfind(' ', 0, size)
  38. # XXX this should be fixed for real
  39. if i == -1 and 'GeneratorExp' in cur:
  40. i = size + 3
  41. assert i != -1, "Impossible line %d to reflow: %r" % (size, s)
  42. lines.append(padding + cur[:i])
  43. if len(lines) == 1:
  44. # find new size based on brace
  45. j = cur.find('{', 0, i)
  46. if j >= 0:
  47. j += 2 # account for the brace and the space after it
  48. size -= j
  49. padding = " " * j
  50. else:
  51. j = cur.find('(', 0, i)
  52. if j >= 0:
  53. j += 1 # account for the paren (no space after it)
  54. size -= j
  55. padding = " " * j
  56. cur = cur[i+1:]
  57. else:
  58. lines.append(padding + cur)
  59. return lines
  60. def is_simple(sum):
  61. """Return True if a sum is a simple.
  62. A sum is simple if its types have no fields, e.g.
  63. unaryop = Invert | Not | UAdd | USub
  64. """
  65. for t in sum.types:
  66. if t.fields:
  67. return False
  68. return True
  69. class EmitVisitor(asdl.VisitorBase):
  70. """Visit that emits lines"""
  71. def __init__(self, file):
  72. self.file = file
  73. super(EmitVisitor, self).__init__()
  74. def emit(self, s, depth, reflow=1):
  75. # XXX reflow long lines?
  76. if reflow:
  77. lines = reflow_lines(s, depth)
  78. else:
  79. lines = [s]
  80. for line in lines:
  81. line = (" " * TABSIZE * depth) + line + "\n"
  82. self.file.write(line)
  83. class TypeDefVisitor(EmitVisitor):
  84. def visitModule(self, mod):
  85. for dfn in mod.dfns:
  86. self.visit(dfn)
  87. def visitType(self, type, depth=0):
  88. self.visit(type.value, type.name, depth)
  89. def visitSum(self, sum, name, depth):
  90. if is_simple(sum):
  91. self.simple_sum(sum, name, depth)
  92. else:
  93. self.sum_with_constructors(sum, name, depth)
  94. def simple_sum(self, sum, name, depth):
  95. enum = []
  96. for i in range(len(sum.types)):
  97. type = sum.types[i]
  98. enum.append("%s=%d" % (type.name, i + 1))
  99. enums = ", ".join(enum)
  100. ctype = get_c_type(name)
  101. s = "typedef enum _%s { %s } %s;" % (name, enums, ctype)
  102. self.emit(s, depth)
  103. self.emit("", depth)
  104. def sum_with_constructors(self, sum, name, depth):
  105. ctype = get_c_type(name)
  106. s = "typedef struct _%(name)s *%(ctype)s;" % locals()
  107. self.emit(s, depth)
  108. self.emit("", depth)
  109. def visitProduct(self, product, name, depth):
  110. ctype = get_c_type(name)
  111. s = "typedef struct _%(name)s *%(ctype)s;" % locals()
  112. self.emit(s, depth)
  113. self.emit("", depth)
  114. class StructVisitor(EmitVisitor):
  115. """Visitor to generate typdefs for AST."""
  116. def visitModule(self, mod):
  117. for dfn in mod.dfns:
  118. self.visit(dfn)
  119. def visitType(self, type, depth=0):
  120. self.visit(type.value, type.name, depth)
  121. def visitSum(self, sum, name, depth):
  122. if not is_simple(sum):
  123. self.sum_with_constructors(sum, name, depth)
  124. def sum_with_constructors(self, sum, name, depth):
  125. def emit(s, depth=depth):
  126. self.emit(s % sys._getframe(1).f_locals, depth)
  127. enum = []
  128. for i in range(len(sum.types)):
  129. type = sum.types[i]
  130. enum.append("%s_kind=%d" % (type.name, i + 1))
  131. emit("enum _%(name)s_kind {" + ", ".join(enum) + "};")
  132. emit("struct _%(name)s {")
  133. emit("enum _%(name)s_kind kind;", depth + 1)
  134. emit("union {", depth + 1)
  135. for t in sum.types:
  136. self.visit(t, depth + 2)
  137. emit("} v;", depth + 1)
  138. for field in sum.attributes:
  139. # rudimentary attribute handling
  140. type = str(field.type)
  141. assert type in asdl.builtin_types, type
  142. emit("%s %s;" % (type, field.name), depth + 1);
  143. emit("};")
  144. emit("")
  145. def visitConstructor(self, cons, depth):
  146. if cons.fields:
  147. self.emit("struct {", depth)
  148. for f in cons.fields:
  149. self.visit(f, depth + 1)
  150. self.emit("} %s;" % cons.name, depth)
  151. self.emit("", depth)
  152. else:
  153. # XXX not sure what I want here, nothing is probably fine
  154. pass
  155. def visitField(self, field, depth):
  156. # XXX need to lookup field.type, because it might be something
  157. # like a builtin...
  158. ctype = get_c_type(field.type)
  159. name = field.name
  160. if field.seq:
  161. if field.type.value in ('cmpop',):
  162. self.emit("asdl_int_seq *%(name)s;" % locals(), depth)
  163. else:
  164. self.emit("asdl_seq *%(name)s;" % locals(), depth)
  165. else:
  166. self.emit("%(ctype)s %(name)s;" % locals(), depth)
  167. def visitProduct(self, product, name, depth):
  168. self.emit("struct _%(name)s {" % locals(), depth)
  169. for f in product.fields:
  170. self.visit(f, depth + 1)
  171. self.emit("};", depth)
  172. self.emit("", depth)
  173. class PrototypeVisitor(EmitVisitor):
  174. """Generate function prototypes for the .h file"""
  175. def visitModule(self, mod):
  176. for dfn in mod.dfns:
  177. self.visit(dfn)
  178. def visitType(self, type):
  179. self.visit(type.value, type.name)
  180. def visitSum(self, sum, name):
  181. if is_simple(sum):
  182. pass # XXX
  183. else:
  184. for t in sum.types:
  185. self.visit(t, name, sum.attributes)
  186. def get_args(self, fields):
  187. """Return list of C argument into, one for each field.
  188. Argument info is 3-tuple of a C type, variable name, and flag
  189. that is true if type can be NULL.
  190. """
  191. args = []
  192. unnamed = {}
  193. for f in fields:
  194. if f.name is None:
  195. name = f.type
  196. c = unnamed[name] = unnamed.get(name, 0) + 1
  197. if c > 1:
  198. name = "name%d" % (c - 1)
  199. else:
  200. name = f.name
  201. # XXX should extend get_c_type() to handle this
  202. if f.seq:
  203. if f.type.value in ('cmpop',):
  204. ctype = "asdl_int_seq *"
  205. else:
  206. ctype = "asdl_seq *"
  207. else:
  208. ctype = get_c_type(f.type)
  209. args.append((ctype, name, f.opt or f.seq))
  210. return args
  211. def visitConstructor(self, cons, type, attrs):
  212. args = self.get_args(cons.fields)
  213. attrs = self.get_args(attrs)
  214. ctype = get_c_type(type)
  215. self.emit_function(cons.name, ctype, args, attrs)
  216. def emit_function(self, name, ctype, args, attrs, union=1):
  217. args = args + attrs
  218. if args:
  219. argstr = ", ".join(["%s %s" % (atype, aname)
  220. for atype, aname, opt in args])
  221. argstr += ", PyArena *arena"
  222. else:
  223. argstr = "PyArena *arena"
  224. margs = "a0"
  225. for i in range(1, len(args)+1):
  226. margs += ", a%d" % i
  227. self.emit("#define %s(%s) _Py_%s(%s)" % (name, margs, name, margs), 0,
  228. reflow = 0)
  229. self.emit("%s _Py_%s(%s);" % (ctype, name, argstr), 0)
  230. def visitProduct(self, prod, name):
  231. self.emit_function(name, get_c_type(name),
  232. self.get_args(prod.fields), [], union=0)
  233. class FunctionVisitor(PrototypeVisitor):
  234. """Visitor to generate constructor functions for AST."""
  235. def emit_function(self, name, ctype, args, attrs, union=1):
  236. def emit(s, depth=0, reflow=1):
  237. self.emit(s, depth, reflow)
  238. argstr = ", ".join(["%s %s" % (atype, aname)
  239. for atype, aname, opt in args + attrs])
  240. if argstr:
  241. argstr += ", PyArena *arena"
  242. else:
  243. argstr = "PyArena *arena"
  244. self.emit("%s" % ctype, 0)
  245. emit("%s(%s)" % (name, argstr))
  246. emit("{")
  247. emit("%s p;" % ctype, 1)
  248. for argtype, argname, opt in args:
  249. # XXX hack alert: false is allowed for a bool
  250. if not opt and not (argtype == "bool" or argtype == "int"):
  251. emit("if (!%s) {" % argname, 1)
  252. emit("PyErr_SetString(PyExc_ValueError,", 2)
  253. msg = "field %s is required for %s" % (argname, name)
  254. emit(' "%s");' % msg,
  255. 2, reflow=0)
  256. emit('return NULL;', 2)
  257. emit('}', 1)
  258. emit("p = (%s)PyArena_Malloc(arena, sizeof(*p));" % ctype, 1);
  259. emit("if (!p)", 1)
  260. emit("return NULL;", 2)
  261. if union:
  262. self.emit_body_union(name, args, attrs)
  263. else:
  264. self.emit_body_struct(name, args, attrs)
  265. emit("return p;", 1)
  266. emit("}")
  267. emit("")
  268. def emit_body_union(self, name, args, attrs):
  269. def emit(s, depth=0, reflow=1):
  270. self.emit(s, depth, reflow)
  271. emit("p->kind = %s_kind;" % name, 1)
  272. for argtype, argname, opt in args:
  273. emit("p->v.%s.%s = %s;" % (name, argname, argname), 1)
  274. for argtype, argname, opt in attrs:
  275. emit("p->%s = %s;" % (argname, argname), 1)
  276. def emit_body_struct(self, name, args, attrs):
  277. def emit(s, depth=0, reflow=1):
  278. self.emit(s, depth, reflow)
  279. for argtype, argname, opt in args:
  280. emit("p->%s = %s;" % (argname, argname), 1)
  281. assert not attrs
  282. class PickleVisitor(EmitVisitor):
  283. def visitModule(self, mod):
  284. for dfn in mod.dfns:
  285. self.visit(dfn)
  286. def visitType(self, type):
  287. self.visit(type.value, type.name)
  288. def visitSum(self, sum, name):
  289. pass
  290. def visitProduct(self, sum, name):
  291. pass
  292. def visitConstructor(self, cons, name):
  293. pass
  294. def visitField(self, sum):
  295. pass
  296. class Obj2ModPrototypeVisitor(PickleVisitor):
  297. def visitProduct(self, prod, name):
  298. code = "static int obj2ast_%s(PyObject* obj, %s* out, PyArena* arena);"
  299. self.emit(code % (name, get_c_type(name)), 0)
  300. visitSum = visitProduct
  301. class Obj2ModVisitor(PickleVisitor):
  302. def funcHeader(self, name):
  303. ctype = get_c_type(name)
  304. self.emit("int", 0)
  305. self.emit("obj2ast_%s(PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0)
  306. self.emit("{", 0)
  307. self.emit("PyObject* tmp = NULL;", 1)
  308. self.emit("", 0)
  309. def sumTrailer(self, name):
  310. self.emit("", 0)
  311. self.emit("tmp = PyObject_Repr(obj);", 1)
  312. # there's really nothing more we can do if this fails ...
  313. self.emit("if (tmp == NULL) goto failed;", 1)
  314. error = "expected some sort of %s, but got %%.400s" % name
  315. format = "PyErr_Format(PyExc_TypeError, \"%s\", PyString_AS_STRING(tmp));"
  316. self.emit(format % error, 1, reflow=False)
  317. self.emit("failed:", 0)
  318. self.emit("Py_XDECREF(tmp);", 1)
  319. self.emit("return 1;", 1)
  320. self.emit("}", 0)
  321. self.emit("", 0)
  322. def simpleSum(self, sum, name):
  323. self.funcHeader(name)
  324. for t in sum.types:
  325. self.emit("if (PyObject_IsInstance(obj, (PyObject*)%s_type)) {" % t.name, 1)
  326. self.emit("*out = %s;" % t.name, 2)
  327. self.emit("return 0;", 2)
  328. self.emit("}", 1)
  329. self.sumTrailer(name)
  330. def buildArgs(self, fields):
  331. return ", ".join(fields + ["arena"])
  332. def complexSum(self, sum, name):
  333. self.funcHeader(name)
  334. for a in sum.attributes:
  335. self.visitAttributeDeclaration(a, name, sum=sum)
  336. self.emit("", 0)
  337. # XXX: should we only do this for 'expr'?
  338. self.emit("if (obj == Py_None) {", 1)
  339. self.emit("*out = NULL;", 2)
  340. self.emit("return 0;", 2)
  341. self.emit("}", 1)
  342. for a in sum.attributes:
  343. self.visitField(a, name, sum=sum, depth=1)
  344. for t in sum.types:
  345. self.emit("if (PyObject_IsInstance(obj, (PyObject*)%s_type)) {" % t.name, 1)
  346. for f in t.fields:
  347. self.visitFieldDeclaration(f, t.name, sum=sum, depth=2)
  348. self.emit("", 0)
  349. for f in t.fields:
  350. self.visitField(f, t.name, sum=sum, depth=2)
  351. args = [f.name.value for f in t.fields] + [a.name.value for a in sum.attributes]
  352. self.emit("*out = %s(%s);" % (t.name, self.buildArgs(args)), 2)
  353. self.emit("if (*out == NULL) goto failed;", 2)
  354. self.emit("return 0;", 2)
  355. self.emit("}", 1)
  356. self.sumTrailer(name)
  357. def visitAttributeDeclaration(self, a, name, sum=sum):
  358. ctype = get_c_type(a.type)
  359. self.emit("%s %s;" % (ctype, a.name), 1)
  360. def visitSum(self, sum, name):
  361. if is_simple(sum):
  362. self.simpleSum(sum, name)
  363. else:
  364. self.complexSum(sum, name)
  365. def visitProduct(self, prod, name):
  366. ctype = get_c_type(name)
  367. self.emit("int", 0)
  368. self.emit("obj2ast_%s(PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0)
  369. self.emit("{", 0)
  370. self.emit("PyObject* tmp = NULL;", 1)
  371. for f in prod.fields:
  372. self.visitFieldDeclaration(f, name, prod=prod, depth=1)
  373. self.emit("", 0)
  374. for f in prod.fields:
  375. self.visitField(f, name, prod=prod, depth=1)
  376. args = [f.name.value for f in prod.fields]
  377. self.emit("*out = %s(%s);" % (name, self.buildArgs(args)), 1)
  378. self.emit("return 0;", 1)
  379. self.emit("failed:", 0)
  380. self.emit("Py_XDECREF(tmp);", 1)
  381. self.emit("return 1;", 1)
  382. self.emit("}", 0)
  383. self.emit("", 0)
  384. def visitFieldDeclaration(self, field, name, sum=None, prod=None, depth=0):
  385. ctype = get_c_type(field.type)
  386. if field.seq:
  387. if self.isSimpleType(field):
  388. self.emit("asdl_int_seq* %s;" % field.name, depth)
  389. else:
  390. self.emit("asdl_seq* %s;" % field.name, depth)
  391. else:
  392. ctype = get_c_type(field.type)
  393. self.emit("%s %s;" % (ctype, field.name), depth)
  394. def isSimpleSum(self, field):
  395. # XXX can the members of this list be determined automatically?
  396. return field.type.value in ('expr_context', 'boolop', 'operator',
  397. 'unaryop', 'cmpop')
  398. def isNumeric(self, field):
  399. return get_c_type(field.type) in ("int", "bool")
  400. def isSimpleType(self, field):
  401. return self.isSimpleSum(field) or self.isNumeric(field)
  402. def visitField(self, field, name, sum=None, prod=None, depth=0):
  403. ctype = get_c_type(field.type)
  404. self.emit("if (PyObject_HasAttrString(obj, \"%s\")) {" % field.name, depth)
  405. self.emit("int res;", depth+1)
  406. if field.seq:
  407. self.emit("Py_ssize_t len;", depth+1)
  408. self.emit("Py_ssize_t i;", depth+1)
  409. self.emit("tmp = PyObject_GetAttrString(obj, \"%s\");" % field.name, depth+1)
  410. self.emit("if (tmp == NULL) goto failed;", depth+1)
  411. if field.seq:
  412. self.emit("if (!PyList_Check(tmp)) {", depth+1)
  413. self.emit("PyErr_Format(PyExc_TypeError, \"%s field \\\"%s\\\" must "
  414. "be a list, not a %%.200s\", tmp->ob_type->tp_name);" %
  415. (name, field.name),
  416. depth+2, reflow=False)
  417. self.emit("goto failed;", depth+2)
  418. self.emit("}", depth+1)
  419. self.emit("len = PyList_GET_SIZE(tmp);", depth+1)
  420. if self.isSimpleType(field):
  421. self.emit("%s = asdl_int_seq_new(len, arena);" % field.name, depth+1)
  422. else:
  423. self.emit("%s = asdl_seq_new(len, arena);" % field.name, depth+1)
  424. self.emit("if (%s == NULL) goto failed;" % field.name, depth+1)
  425. self.emit("for (i = 0; i < len; i++) {", depth+1)
  426. self.emit("%s value;" % ctype, depth+2)
  427. self.emit("res = obj2ast_%s(PyList_GET_ITEM(tmp, i), &value, arena);" %
  428. field.type, depth+2, reflow=False)
  429. self.emit("if (res != 0) goto failed;", depth+2)
  430. self.emit("asdl_seq_SET(%s, i, value);" % field.name, depth+2)
  431. self.emit("}", depth+1)
  432. else:
  433. self.emit("res = obj2ast_%s(tmp, &%s, arena);" %
  434. (field.type, field.name), depth+1)
  435. self.emit("if (res != 0) goto failed;", depth+1)
  436. self.emit("Py_XDECREF(tmp);", depth+1)
  437. self.emit("tmp = NULL;", depth+1)
  438. self.emit("} else {", depth)
  439. if not field.opt:
  440. message = "required field \\\"%s\\\" missing from %s" % (field.name, name)
  441. format = "PyErr_SetString(PyExc_TypeError, \"%s\");"
  442. self.emit(format % message, depth+1, reflow=False)
  443. self.emit("return 1;", depth+1)
  444. else:
  445. if self.isNumeric(field):
  446. self.emit("%s = 0;" % field.name, depth+1)
  447. elif not self.isSimpleType(field):
  448. self.emit("%s = NULL;" % field.name, depth+1)
  449. else:
  450. raise TypeError("could not determine the default value for %s" % field.name)
  451. self.emit("}", depth)
  452. class MarshalPrototypeVisitor(PickleVisitor):
  453. def prototype(self, sum, name):
  454. ctype = get_c_type(name)
  455. self.emit("static int marshal_write_%s(PyObject **, int *, %s);"
  456. % (name, ctype), 0)
  457. visitProduct = visitSum = prototype
  458. class PyTypesDeclareVisitor(PickleVisitor):
  459. def visitProduct(self, prod, name):
  460. self.emit("static PyTypeObject *%s_type;" % name, 0)
  461. self.emit("static PyObject* ast2obj_%s(void*);" % name, 0)
  462. if prod.fields:
  463. self.emit("static char *%s_fields[]={" % name,0)
  464. for f in prod.fields:
  465. self.emit('"%s",' % f.name, 1)
  466. self.emit("};", 0)
  467. def visitSum(self, sum, name):
  468. self.emit("static PyTypeObject *%s_type;" % name, 0)
  469. if sum.attributes:
  470. self.emit("static char *%s_attributes[] = {" % name, 0)
  471. for a in sum.attributes:
  472. self.emit('"%s",' % a.name, 1)
  473. self.emit("};", 0)
  474. ptype = "void*"
  475. if is_simple(sum):
  476. ptype = get_c_type(name)
  477. tnames = []
  478. for t in sum.types:
  479. tnames.append(str(t.name)+"_singleton")
  480. tnames = ", *".join(tnames)
  481. self.emit("static PyObject *%s;" % tnames, 0)
  482. self.emit("static PyObject* ast2obj_%s(%s);" % (name, ptype), 0)
  483. for t in sum.types:
  484. self.visitConstructor(t, name)
  485. def visitConstructor(self, cons, name):
  486. self.emit("static PyTypeObject *%s_type;" % cons.name, 0)
  487. if cons.fields:
  488. self.emit("static char *%s_fields[]={" % cons.name, 0)
  489. for t in cons.fields:
  490. self.emit('"%s",' % t.name, 1)
  491. self.emit("};",0)
  492. class PyTypesVisitor(PickleVisitor):
  493. def visitModule(self, mod):
  494. self.emit("""
  495. static int
  496. ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
  497. {
  498. Py_ssize_t i, numfields = 0;
  499. int res = -1;
  500. PyObject *key, *value, *fields;
  501. fields = PyObject_GetAttrString((PyObject*)Py_TYPE(self), "_fields");
  502. if (!fields)
  503. PyErr_Clear();
  504. if (fields) {
  505. numfields = PySequence_Size(fields);
  506. if (numfields == -1)
  507. goto cleanup;
  508. }
  509. res = 0; /* if no error occurs, this stays 0 to the end */
  510. if (PyTuple_GET_SIZE(args) > 0) {
  511. if (numfields != PyTuple_GET_SIZE(args)) {
  512. PyErr_Format(PyExc_TypeError, "%.400s constructor takes %s"
  513. "%zd positional argument%s",
  514. Py_TYPE(self)->tp_name,
  515. numfields == 0 ? "" : "either 0 or ",
  516. numfields, numfields == 1 ? "" : "s");
  517. res = -1;
  518. goto cleanup;
  519. }
  520. for (i = 0; i < PyTuple_GET_SIZE(args); i++) {
  521. /* cannot be reached when fields is NULL */
  522. PyObject *name = PySequence_GetItem(fields, i);
  523. if (!name) {
  524. res = -1;
  525. goto cleanup;
  526. }
  527. res = PyObject_SetAttr(self, name, PyTuple_GET_ITEM(args, i));
  528. Py_DECREF(name);
  529. if (res < 0)
  530. goto cleanup;
  531. }
  532. }
  533. if (kw) {
  534. i = 0; /* needed by PyDict_Next */
  535. while (PyDict_Next(kw, &i, &key, &value)) {
  536. res = PyObject_SetAttr(self, key, value);
  537. if (res < 0)
  538. goto cleanup;
  539. }
  540. }
  541. cleanup:
  542. Py_XDECREF(fields);
  543. return res;
  544. }
  545. /* Pickling support */
  546. static PyObject *
  547. ast_type_reduce(PyObject *self, PyObject *unused)
  548. {
  549. PyObject *res;
  550. PyObject *dict = PyObject_GetAttrString(self, "__dict__");
  551. if (dict == NULL) {
  552. if (PyErr_ExceptionMatches(PyExc_AttributeError))
  553. PyErr_Clear();
  554. else
  555. return NULL;
  556. }
  557. if (dict) {
  558. res = Py_BuildValue("O()O", Py_TYPE(self), dict);
  559. Py_DECREF(dict);
  560. return res;
  561. }
  562. return Py_BuildValue("O()", Py_TYPE(self));
  563. }
  564. static PyMethodDef ast_type_methods[] = {
  565. {"__reduce__", ast_type_reduce, METH_NOARGS, NULL},
  566. {NULL}
  567. };
  568. static PyTypeObject AST_type = {
  569. PyVarObject_HEAD_INIT(&PyType_Type, 0)
  570. "_ast.AST",
  571. sizeof(PyObject),
  572. 0,
  573. 0, /* tp_dealloc */
  574. 0, /* tp_print */
  575. 0, /* tp_getattr */
  576. 0, /* tp_setattr */
  577. 0, /* tp_compare */
  578. 0, /* tp_repr */
  579. 0, /* tp_as_number */
  580. 0, /* tp_as_sequence */
  581. 0, /* tp_as_mapping */
  582. 0, /* tp_hash */
  583. 0, /* tp_call */
  584. 0, /* tp_str */
  585. PyObject_GenericGetAttr, /* tp_getattro */
  586. PyObject_GenericSetAttr, /* tp_setattro */
  587. 0, /* tp_as_buffer */
  588. Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
  589. 0, /* tp_doc */
  590. 0, /* tp_traverse */
  591. 0, /* tp_clear */
  592. 0, /* tp_richcompare */
  593. 0, /* tp_weaklistoffset */
  594. 0, /* tp_iter */
  595. 0, /* tp_iternext */
  596. ast_type_methods, /* tp_methods */
  597. 0, /* tp_members */
  598. 0, /* tp_getset */
  599. 0, /* tp_base */
  600. 0, /* tp_dict */
  601. 0, /* tp_descr_get */
  602. 0, /* tp_descr_set */
  603. 0, /* tp_dictoffset */
  604. (initproc)ast_type_init, /* tp_init */
  605. PyType_GenericAlloc, /* tp_alloc */
  606. PyType_GenericNew, /* tp_new */
  607. PyObject_Del, /* tp_free */
  608. };
  609. static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int num_fields)
  610. {
  611. PyObject *fnames, *result;
  612. int i;
  613. fnames = PyTuple_New(num_fields);
  614. if (!fnames) return NULL;
  615. for (i = 0; i < num_fields; i++) {
  616. PyObject *field = PyString_FromString(fields[i]);
  617. if (!field) {
  618. Py_DECREF(fnames);
  619. return NULL;
  620. }
  621. PyTuple_SET_ITEM(fnames, i, field);
  622. }
  623. result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){sOss}",
  624. type, base, "_fields", fnames, "__module__", "_ast");
  625. Py_DECREF(fnames);
  626. return (PyTypeObject*)result;
  627. }
  628. static int add_attributes(PyTypeObject* type, char**attrs, int num_fields)
  629. {
  630. int i, result;
  631. PyObject *s, *l = PyTuple_New(num_fields);
  632. if (!l) return 0;
  633. for(i = 0; i < num_fields; i++) {
  634. s = PyString_FromString(attrs[i]);
  635. if (!s) {
  636. Py_DECREF(l);
  637. return 0;
  638. }
  639. PyTuple_SET_ITEM(l, i, s);
  640. }
  641. result = PyObject_SetAttrString((PyObject*)type, "_attributes", l) >= 0;
  642. Py_DECREF(l);
  643. return result;
  644. }
  645. /* Conversion AST -> Python */
  646. static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*))
  647. {
  648. int i, n = asdl_seq_LEN(seq);
  649. PyObject *result = PyList_New(n);
  650. PyObject *value;
  651. if (!result)
  652. return NULL;
  653. for (i = 0; i < n; i++) {
  654. value = func(asdl_seq_GET(seq, i));
  655. if (!value) {
  656. Py_DECREF(result);
  657. return NULL;
  658. }
  659. PyList_SET_ITEM(result, i, value);
  660. }
  661. return result;
  662. }
  663. static PyObject* ast2obj_object(void *o)
  664. {
  665. if (!o)
  666. o = Py_None;
  667. Py_INCREF((PyObject*)o);
  668. return (PyObject*)o;
  669. }
  670. #define ast2obj_identifier ast2obj_object
  671. #define ast2obj_string ast2obj_object
  672. static PyObject* ast2obj_bool(bool b)
  673. {
  674. return PyBool_FromLong(b);
  675. }
  676. static PyObject* ast2obj_int(long b)
  677. {
  678. return PyInt_FromLong(b);
  679. }
  680. /* Conversion Python -> AST */
  681. static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena)
  682. {
  683. if (obj == Py_None)
  684. obj = NULL;
  685. if (obj)
  686. PyArena_AddPyObject(arena, obj);
  687. Py_XINCREF(obj);
  688. *out = obj;
  689. return 0;
  690. }
  691. #define obj2ast_identifier obj2ast_object
  692. #define obj2ast_string obj2ast_object
  693. static int obj2ast_int(PyObject* obj, int* out, PyArena* arena)
  694. {
  695. int i;
  696. if (!PyInt_Check(obj) && !PyLong_Check(obj)) {
  697. PyObject *s = PyObject_Repr(obj);
  698. if (s == NULL) return 1;
  699. PyErr_Format(PyExc_ValueError, "invalid integer value: %.400s",
  700. PyString_AS_STRING(s));
  701. Py_DECREF(s);
  702. return 1;
  703. }
  704. i = (int)PyLong_AsLong(obj);
  705. if (i == -1 && PyErr_Occurred())
  706. return 1;
  707. *out = i;
  708. return 0;
  709. }
  710. static int obj2ast_bool(PyObject* obj, bool* out, PyArena* arena)
  711. {
  712. if (!PyBool_Check(obj)) {
  713. PyObject *s = PyObject_Repr(obj);
  714. if (s == NULL) return 1;
  715. PyErr_Format(PyExc_ValueError, "invalid boolean value: %.400s",
  716. PyString_AS_STRING(s));
  717. Py_DECREF(s);
  718. return 1;
  719. }
  720. *out = (obj == Py_True);
  721. return 0;
  722. }
  723. static int add_ast_fields(void)
  724. {
  725. PyObject *empty_tuple, *d;
  726. if (PyType_Ready(&AST_type) < 0)
  727. return -1;
  728. d = AST_type.tp_dict;
  729. empty_tuple = PyTuple_New(0);
  730. if (!empty_tuple ||
  731. PyDict_SetItemString(d, "_fields", empty_tuple) < 0 ||
  732. PyDict_SetItemString(d, "_attributes", empty_tuple) < 0) {
  733. Py_XDECREF(empty_tuple);
  734. return -1;
  735. }
  736. Py_DECREF(empty_tuple);
  737. return 0;
  738. }
  739. """, 0, reflow=False)
  740. self.emit("static int init_types(void)",0)
  741. self.emit("{", 0)
  742. self.emit("static int initialized;", 1)
  743. self.emit("if (initialized) return 1;", 1)
  744. self.emit("if (add_ast_fields() < 0) return 0;", 1)
  745. for dfn in mod.dfns:
  746. self.visit(dfn)
  747. self.emit("initialized = 1;", 1)
  748. self.emit("return 1;", 1);
  749. self.emit("}", 0)
  750. def visitProduct(self, prod, name):
  751. if prod.fields:
  752. fields = name.value+"_fields"
  753. else:
  754. fields = "NULL"
  755. self.emit('%s_type = make_type("%s", &AST_type, %s, %d);' %
  756. (name, name, fields, len(prod.fields)), 1)
  757. self.emit("if (!%s_type) return 0;" % name, 1)
  758. def visitSum(self, sum, name):
  759. self.emit('%s_type = make_type("%s", &AST_type, NULL, 0);' %
  760. (name, name), 1)
  761. self.emit("if (!%s_type) return 0;" % name, 1)
  762. if sum.attributes:
  763. self.emit("if (!add_attributes(%s_type, %s_attributes, %d)) return 0;" %
  764. (name, name, len(sum.attributes)), 1)
  765. else:
  766. self.emit("if (!add_attributes(%s_type, NULL, 0)) return 0;" % name, 1)
  767. simple = is_simple(sum)
  768. for t in sum.types:
  769. self.visitConstructor(t, name, simple)
  770. def visitConstructor(self, cons, name, simple):
  771. if cons.fields:
  772. fields = cons.name.value+"_fields"
  773. else:
  774. fields = "NULL"
  775. self.emit('%s_type = make_type("%s", %s_type, %s, %d);' %
  776. (cons.name, cons.name, name, fields, len(cons.fields)), 1)
  777. self.emit("if (!%s_type) return 0;" % cons.name, 1)
  778. if simple:
  779. self.emit("%s_singleton = PyType_GenericNew(%s_type, NULL, NULL);" %
  780. (cons.name, cons.name), 1)
  781. self.emit("if (!%s_singleton) return 0;" % cons.name, 1)
  782. def parse_version(mod):
  783. return mod.version.value[12:-3]
  784. class ASTModuleVisitor(PickleVisitor):
  785. def visitModule(self, mod):
  786. self.emit("PyMODINIT_FUNC", 0)
  787. self.emit("init_ast(void)", 0)
  788. self.emit("{", 0)
  789. self.emit("PyObject *m, *d;", 1)
  790. self.emit("if (!init_types()) return;", 1)
  791. self.emit('m = Py_InitModule3("_ast", NULL, NULL);', 1)
  792. self.emit("if (!m) return;", 1)
  793. self.emit("d = PyModule_GetDict(m);", 1)
  794. self.emit('if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return;', 1)
  795. self.emit('if (PyModule_AddIntConstant(m, "PyCF_ONLY_AST", PyCF_ONLY_AST) < 0)', 1)
  796. self.emit("return;", 2)
  797. # Value of version: "$Revision: 67146 $"
  798. self.emit('if (PyModule_AddStringConstant(m, "__version__", "%s") < 0)'
  799. % parse_version(mod), 1)
  800. self.emit("return;", 2)
  801. for dfn in mod.dfns:
  802. self.visit(dfn)
  803. self.emit("}", 0)
  804. def visitProduct(self, prod, name):
  805. self.addObj(name)
  806. def visitSum(self, sum, name):
  807. self.addObj(name)
  808. for t in sum.types:
  809. self.visitConstructor(t, name)
  810. def visitConstructor(self, cons, name):
  811. self.addObj(cons.name)
  812. def addObj(self, name):
  813. self.emit('if (PyDict_SetItemString(d, "%s", (PyObject*)%s_type) < 0) return;' % (name, name), 1)
  814. _SPECIALIZED_SEQUENCES = ('stmt', 'expr')
  815. def find_sequence(fields, doing_specialization):
  816. """Return True if any field uses a sequence."""
  817. for f in fields:
  818. if f.seq:
  819. if not doing_specialization:
  820. return True
  821. if str(f.type) not in _SPECIALIZED_SEQUENCES:
  822. return True
  823. return False
  824. def has_sequence(types, doing_specialization):
  825. for t in types:
  826. if find_sequence(t.fields, doing_specialization):
  827. return True
  828. return False
  829. class StaticVisitor(PickleVisitor):
  830. CODE = '''Very simple, always emit this static code. Overide CODE'''
  831. def visit(self, object):
  832. self.emit(self.CODE, 0, reflow=False)
  833. class ObjVisitor(PickleVisitor):
  834. def func_begin(self, name):
  835. ctype = get_c_type(name)
  836. self.emit("PyObject*", 0)
  837. self.emit("ast2obj_%s(void* _o)" % (name), 0)
  838. self.emit("{", 0)
  839. self.emit("%s o = (%s)_o;" % (ctype, ctype), 1)
  840. self.emit("PyObject *result = NULL, *value = NULL;", 1)
  841. self.emit('if (!o) {', 1)
  842. self.emit("Py_INCREF(Py_None);", 2)
  843. self.emit('return Py_None;', 2)
  844. self.emit("}", 1)
  845. self.emit('', 0)
  846. def func_end(self):
  847. self.emit("return result;", 1)
  848. self.emit("failed:", 0)
  849. self.emit("Py_XDECREF(value);", 1)
  850. self.emit("Py_XDECREF(result);", 1)
  851. self.emit("return NULL;", 1)
  852. self.emit("}", 0)
  853. self.emit("", 0)
  854. def visitSum(self, sum, name):
  855. if is_simple(sum):
  856. self.simpleSum(sum, name)
  857. return
  858. self.func_begin(name)
  859. self.emit("switch (o->kind) {", 1)
  860. for i in range(len(sum.types)):
  861. t = sum.types[i]
  862. self.visitConstructor(t, i + 1, name)
  863. self.emit("}", 1)
  864. for a in sum.attributes:
  865. self.emit("value = ast2obj_%s(o->%s);" % (a.type, a.name), 1)
  866. self.emit("if (!value) goto failed;", 1)
  867. self.emit('if (PyObject_SetAttrString(result, "%s", value) < 0)' % a.name, 1)
  868. self.emit('goto failed;', 2)
  869. self.emit('Py_DECREF(value);', 1)
  870. self.func_end()
  871. def simpleSum(self, sum, name):
  872. self.emit("PyObject* ast2obj_%s(%s_ty o)" % (name, name), 0)
  873. self.emit("{", 0)
  874. self.emit("switch(o) {", 1)
  875. for t in sum.types:
  876. self.emit("case %s:" % t.name, 2)
  877. self.emit("Py_INCREF(%s_singleton);" % t.name, 3)
  878. self.emit("return %s_singleton;" % t.name, 3)
  879. self.emit("default:" % name, 2)
  880. self.emit('/* should never happen, but just in case ... */', 3)
  881. code = "PyErr_Format(PyExc_SystemError, \"unknown %s found\");" % name
  882. self.emit(code, 3, reflow=False)
  883. self.emit("return NULL;", 3)
  884. self.emit("}", 1)
  885. self.emit("}", 0)
  886. def visitProduct(self, prod, name):
  887. self.func_begin(name)
  888. self.emit("result = PyType_GenericNew(%s_type, NULL, NULL);" % name, 1);
  889. self.emit("if (!result) return NULL;", 1)
  890. for field in prod.fields:
  891. self.visitField(field, name, 1, True)
  892. self.func_end()
  893. def visitConstructor(self, cons, enum, name):
  894. self.emit("case %s_kind:" % cons.name, 1)
  895. self.emit("result = PyType_GenericNew(%s_type, NULL, NULL);" % cons.name, 2);
  896. self.emit("if (!result) goto failed;", 2)
  897. for f in cons.fields:
  898. self.visitField(f, cons.name, 2, False)
  899. self.emit("break;", 2)
  900. def visitField(self, field, name, depth, product):
  901. def emit(s, d):
  902. self.emit(s, depth + d)
  903. if product:
  904. value = "o->%s" % field.name
  905. else:
  906. value = "o->v.%s.%s" % (name, field.name)
  907. self.set(field, value, depth)
  908. emit("if (!value) goto failed;", 0)
  909. emit('if (PyObject_SetAttrString(result, "%s", value) == -1)' % field.name, 0)
  910. emit("goto failed;", 1)
  911. emit("Py_DECREF(value);", 0)
  912. def emitSeq(self, field, value, depth, emit):
  913. emit("seq = %s;" % value, 0)
  914. emit("n = asdl_seq_LEN(seq);", 0)
  915. emit("value = PyList_New(n);", 0)
  916. emit("if (!value) goto failed;", 0)
  917. emit("for (i = 0; i < n; i++) {", 0)
  918. self.set("value", field, "asdl_seq_GET(seq, i)", depth + 1)
  919. emit("if (!value1) goto failed;", 1)
  920. emit("PyList_SET_ITEM(value, i, value1);", 1)
  921. emit("value1 = NULL;", 1)
  922. emit("}", 0)
  923. def set(self, field, value, depth):
  924. if field.seq:
  925. # XXX should really check for is_simple, but that requires a symbol table
  926. if field.type.value == "cmpop":
  927. # While the sequence elements are stored as void*,
  928. # ast2obj_cmpop expects an enum
  929. self.emit("{", depth)
  930. self.emit("int i, n = asdl_seq_LEN(%s);" % value, depth+1)
  931. self.emit("value = PyList_New(n);", depth+1)
  932. self.emit("if (!value) goto failed;", depth+1)
  933. self.emit("for(i = 0; i < n; i++)", depth+1)
  934. # This cannot fail, so no need for error handling
  935. self.emit("PyList_SET_ITEM(value, i, ast2obj_cmpop((cmpop_ty)asdl_seq_GET(%s, i)));" % value,
  936. depth+2, reflow=False)
  937. self.emit("}", depth)
  938. else:
  939. self.emit("value = ast2obj_list(%s, ast2obj_%s);" % (value, field.type), depth)
  940. else:
  941. ctype = get_c_type(field.type)
  942. self.emit("value = ast2obj_%s(%s);" % (field.type, value), depth, reflow=False)
  943. class PartingShots(StaticVisitor):
  944. CODE = """
  945. PyObject* PyAST_mod2obj(mod_ty t)
  946. {
  947. init_types();
  948. return ast2obj_mod(t);
  949. }
  950. /* mode is 0 for "exec", 1 for "eval" and 2 for "single" input */
  951. mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
  952. {
  953. mod_ty res;
  954. PyObject *req_type[] = {(PyObject*)Module_type, (PyObject*)Expression_type,
  955. (PyObject*)Interactive_type};
  956. char *req_name[] = {"Module", "Expression", "Interactive"};
  957. assert(0 <= mode && mode <= 2);
  958. init_types();
  959. if (!PyObject_IsInstance(ast, req_type[mode])) {
  960. PyErr_Format(PyExc_TypeError, "expected %s node, got %.400s",
  961. req_name[mode], Py_TYPE(ast)->tp_name);
  962. return NULL;
  963. }
  964. if (obj2ast_mod(ast, &res, arena) != 0)
  965. return NULL;
  966. else
  967. return res;
  968. }
  969. int PyAST_Check(PyObject* obj)
  970. {
  971. init_types();
  972. return PyObject_IsInstance(obj, (PyObject*)&AST_type);
  973. }
  974. """
  975. class ChainOfVisitors:
  976. def __init__(self, *visitors):
  977. self.visitors = visitors
  978. def visit(self, object):
  979. for v in self.visitors:
  980. v.visit(object)
  981. v.emit("", 0)
  982. common_msg = "/* File automatically generated by %s. */\n\n"
  983. c_file_msg = """
  984. /*
  985. __version__ %s.
  986. This module must be committed separately after each AST grammar change;
  987. The __version__ number is set to the revision number of the commit
  988. containing the grammar change.
  989. */
  990. """
  991. def main(srcfile):
  992. argv0 = sys.argv[0]
  993. components = argv0.split(os.sep)
  994. argv0 = os.sep.join(components[-2:])
  995. auto_gen_msg = common_msg % argv0
  996. mod = asdl.parse(srcfile)
  997. if not asdl.check(mod):
  998. sys.exit(1)
  999. if INC_DIR:
  1000. p = "%s/%s-ast.h" % (INC_DIR, mod.name)
  1001. f = open(p, "wb")
  1002. f.write(auto_gen_msg)
  1003. f.write('#include "asdl.h"\n\n')
  1004. f.write('#ifdef __cplusplus\n'
  1005. 'extern "C" {\n'
  1006. '#endif\n\n')
  1007. c = ChainOfVisitors(TypeDefVisitor(f),
  1008. StructVisitor(f),
  1009. PrototypeVisitor(f),
  1010. )
  1011. c.visit(mod)
  1012. f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
  1013. f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
  1014. f.write("int PyAST_Check(PyObject* obj);\n")
  1015. f.write('\n#ifdef __cplusplus\n'
  1016. '}\n' # To end the extern "C"
  1017. '#endif\n')
  1018. f.close()
  1019. if SRC_DIR:
  1020. p = os.path.join(SRC_DIR, str(mod.name) + "-ast.c")
  1021. f = open(p, "wb")
  1022. f.write(auto_gen_msg)
  1023. f.write(c_file_msg % parse_version(mod))
  1024. f.write('#include "Python.h"\n')
  1025. f.write('#include "%s-ast.h"\n' % mod.name)
  1026. f.write('\n')
  1027. f.write("static PyTypeObject AST_type;\n")
  1028. v = ChainOfVisitors(
  1029. PyTypesDeclareVisitor(f),
  1030. PyTypesVisitor(f),
  1031. Obj2ModPrototypeVisitor(f),
  1032. FunctionVisitor(f),
  1033. ObjVisitor(f),
  1034. Obj2ModVisitor(f),
  1035. ASTModuleVisitor(f),
  1036. PartingShots(f),
  1037. )
  1038. v.visit(mod)
  1039. f.close()
  1040. if __name__ == "__main__":
  1041. import sys
  1042. import getopt
  1043. INC_DIR = ''
  1044. SRC_DIR = ''
  1045. opts, args = getopt.getopt(sys.argv[1:], "h:c:")
  1046. if len(opts) != 1:
  1047. print "Must specify exactly one output file"
  1048. sys.exit(1)
  1049. for o, v in opts:
  1050. if o == '-h':
  1051. INC_DIR = v
  1052. if o == '-c':
  1053. SRC_DIR = v
  1054. if len(args) != 1:
  1055. print "Must specify single input file"
  1056. sys.exit(1)
  1057. main(args[0])