/Python/system/spark.py

#  Copyright (c) 1998-2000 John Aycock

#  

#  Permission is hereby granted, free of charge, to any person obtaining

#  a copy of this software and associated documentation files (the

#  "Software"), to deal in the Software without restriction, including

#  without limitation the rights to use, copy, modify, merge, publish,

#  distribute, sublicense, and/or sell copies of the Software, and to

#  permit persons to whom the Software is furnished to do so, subject to

#  the following conditions:

#  

#  The above copyright notice and this permission notice shall be

#  included in all copies or substantial portions of the Software.

#  

#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,

#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF

#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.

#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY

#  CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,

#  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE

#  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.



__version__ = 'SPARK-0.6.1'



import re

import sys

import string



def _namelist(instance):

	namelist, namedict, classlist = [], {}, [instance.__class__]

	for c in classlist:

		for b in c.__bases__:

			classlist.append(b)

		for name in dir(c):

			if not namedict.has_key(name):

				namelist.append(name)

				namedict[name] = 1

	return namelist



class GenericScanner:

	def __init__(self):

		pattern = self.reflect()

		self.re = re.compile(pattern, re.VERBOSE)



		self.index2func = {}

		for name, number in self.re.groupindex.items():

			self.index2func[number-1] = getattr(self, 't_' + name)



	def makeRE(self, name):

		doc = getattr(self, name).__doc__

		rv = '(?P<%s>%s)' % (name[2:], doc)

		return rv



	def reflect(self):

		rv = []

		for name in _namelist(self):

			if name[:2] == 't_' and name != 't_default':

				rv.append(self.makeRE(name))



		rv.append(self.makeRE('t_default'))

		return string.join(rv, '|')



	def error(self, s, pos):

		print "Lexical error at position %s" % pos

		raise SystemExit



	def tokenize(self, s):

		pos = 0

		n = len(s)

		while pos < n:

			m = self.re.match(s, pos)

			if m is None:

				self.error(s, pos)



			groups = m.groups()

			for i in range(len(groups)):

				if groups[i] and self.index2func.has_key(i):

					self.index2func[i](groups[i])

			pos = m.end()



	def t_default(self, s):

		r'( . | \n )+'

		pass



class GenericParser:

	def __init__(self, start):

		self.rules = {}

		self.rule2func = {}

		self.rule2name = {}

		self.collectRules()

		self.startRule = self.augment(start)

		self.ruleschanged = 1



	_START = 'START'

	_EOF = 'EOF'



	#

	#  A hook for GenericASTBuilder and GenericASTMatcher.

	#

	def preprocess(self, rule, func):	return rule, func



	def addRule(self, doc, func):

		rules = string.split(doc)



		index = []

		for i in range(len(rules)):

			if rules[i] == '::=':

				index.append(i-1)

		index.append(len(rules))



		for i in range(len(index)-1):

			lhs = rules[index[i]]

			rhs = rules[index[i]+2:index[i+1]]

			rule = (lhs, tuple(rhs))



			rule, fn = self.preprocess(rule, func)



			if self.rules.has_key(lhs):

				self.rules[lhs].append(rule)

			else:

				self.rules[lhs] = [ rule ]

			self.rule2func[rule] = fn

			self.rule2name[rule] = func.__name__[2:]

		self.ruleschanged = 1



	def collectRules(self):

		for name in _namelist(self):

			if name[:2] == 'p_':

				func = getattr(self, name)

				doc = func.__doc__

				self.addRule(doc, func)



	def augment(self, start):

		#

		#  Tempting though it is, this isn't made into a call

		#  to self.addRule() because the start rule shouldn't

		#  be subject to preprocessing.

		#

		startRule = (self._START, ( start, self._EOF ))

		self.rule2func[startRule] = lambda args: args[0]

		self.rules[self._START] = [ startRule ]

		self.rule2name[startRule] = ''

		return startRule



	def makeFIRST(self):

		union = {}

		self.first = {}

		

		for rulelist in self.rules.values():

			for lhs, rhs in rulelist:

				if not self.first.has_key(lhs):

					self.first[lhs] = {}



				if len(rhs) == 0:

					self.first[lhs][None] = 1

					continue



				sym = rhs[0]

				if not self.rules.has_key(sym):

					self.first[lhs][sym] = 1

				else:

					union[(sym, lhs)] = 1

		changes = 1

		while changes:

			changes = 0

			for src, dest in union.keys():

				destlen = len(self.first[dest])

				self.first[dest].update(self.first[src])

				if len(self.first[dest]) != destlen:

					changes = 1



	#

	#  An Earley parser, as per J. Earley, "An Efficient Context-Free

	#  Parsing Algorithm", CACM 13(2), pp. 94-102.  Also J. C. Earley,

	#  "An Efficient Context-Free Parsing Algorithm", Ph.D. thesis,

	#  Carnegie-Mellon University, August 1968, p. 27.

	#

	

	def typestring(self, token):

		return None



	def error(self, token):

		print "Syntax error at or near `%s' token" % token

		raise SystemExit



	def parse(self, tokens):

		tree = {}

		tokens.append(self._EOF)

		states = { 0: [ (self.startRule, 0, 0) ] }

		

		if self.ruleschanged:

			self.makeFIRST()



		for i in xrange(len(tokens)):

			states[i+1] = []



			if states[i] == []:

				break				

			self.buildState(tokens[i], states, i, tree)



		#_dump(tokens, states)



		if i < len(tokens)-1 or states[i+1] != [(self.startRule, 2, 0)]:

			del tokens[-1]

			self.error(tokens[i-1])

		rv = self.buildTree(tokens, tree, ((self.startRule, 2, 0), i+1))

		del tokens[-1]

		return rv



	def buildState(self, token, states, i, tree):

		needsCompletion = {}

		state = states[i]

		predicted = {}

		

		for item in state:

			rule, pos, parent = item

			lhs, rhs = rule



			#

			#  A -> a . (completer)

			#

			if pos == len(rhs):

				if len(rhs) == 0:

					needsCompletion[lhs] = (item, i)



				for pitem in states[parent]:

					if pitem is item:

						break



					prule, ppos, pparent = pitem

					plhs, prhs = prule



					if prhs[ppos:ppos+1] == (lhs,):

						new = (prule,

						       ppos+1,

						       pparent)

						if new not in state:

							state.append(new)

							tree[(new, i)] = [(item, i)]

						else:

							tree[(new, i)].append((item, i))

				continue



			nextSym = rhs[pos]



			#

			#  A -> a . B (predictor)

			#

			if self.rules.has_key(nextSym):

				#

				#  Work on completer step some more; for rules

				#  with empty RHS, the "parent state" is the

				#  current state we're adding Earley items to,

				#  so the Earley items the completer step needs

				#  may not all be present when it runs.

				#

				if needsCompletion.has_key(nextSym):

					new = (rule, pos+1, parent)

					olditem_i = needsCompletion[nextSym]

					if new not in state:

						state.append(new)

						tree[(new, i)] = [olditem_i]

					else:

						tree[(new, i)].append(olditem_i)



				#

				#  Has this been predicted already?

				#

				if predicted.has_key(nextSym):

					continue

				predicted[nextSym] = 1



				ttype = token is not self._EOF and \

					self.typestring(token) or \

					None

				if ttype is not None:

					#

					#  Even smarter predictor, when the

					#  token's type is known.  The code is

					#  grungy, but runs pretty fast.  Three

					#  cases are looked for: rules with

					#  empty RHS; first symbol on RHS is a

					#  terminal; first symbol on RHS is a

					#  nonterminal (and isn't nullable).

					#

					for prule in self.rules[nextSym]:

						new = (prule, 0, i)

						prhs = prule[1]

						if len(prhs) == 0:

							state.append(new)

							continue

						prhs0 = prhs[0]

						if not self.rules.has_key(prhs0):

							if prhs0 != ttype:

								continue

							else:

								state.append(new)

								continue

						first = self.first[prhs0]

						if not first.has_key(None) and \

						   not first.has_key(ttype):

							continue

						state.append(new)

					continue



				for prule in self.rules[nextSym]:

					#

					#  Smarter predictor, as per Grune &

					#  Jacobs' _Parsing Techniques_.  Not

					#  as good as FIRST sets though.

					#

					prhs = prule[1]

					if len(prhs) > 0 and \

					   not self.rules.has_key(prhs[0]) and \

					   token != prhs[0]:

						continue

					state.append((prule, 0, i))



			#

			#  A -> a . c (scanner)

			#

			elif token == nextSym:

				#assert new not in states[i+1]

				states[i+1].append((rule, pos+1, parent))



	def buildTree(self, tokens, tree, root):

		stack = []

		self.buildTree_r(stack, tokens, -1, tree, root)

		return stack[0]



	def buildTree_r(self, stack, tokens, tokpos, tree, root):

		(rule, pos, parent), state = root

		

		while pos > 0:

			want = ((rule, pos, parent), state)

			if not tree.has_key(want):

				#

				#  Since pos > 0, it didn't come from closure,

				#  and if it isn't in tree[], then there must

				#  be a terminal symbol to the left of the dot.

				#  (It must be from a "scanner" step.)

				#

				pos = pos - 1

				state = state - 1

				stack.insert(0, tokens[tokpos])

				tokpos = tokpos - 1

			else:

				#

				#  There's a NT to the left of the dot.

				#  Follow the tree pointer recursively (>1

				#  tree pointers from it indicates ambiguity).

				#  Since the item must have come about from a

				#  "completer" step, the state where the item

				#  came from must be the parent state of the

				#  item the tree pointer points to.

				#

				children = tree[want]

				if len(children) > 1:

					child = self.ambiguity(children)

				else:

					child = children[0]

				

				tokpos = self.buildTree_r(stack,

							  tokens, tokpos,

							  tree, child)

				pos = pos - 1

				(crule, cpos, cparent), cstate = child

				state = cparent

				

		lhs, rhs = rule

		result = self.rule2func[rule](stack[:len(rhs)])

		stack[:len(rhs)] = [result]

		return tokpos



	def ambiguity(self, children):

		#

		#  XXX - problem here and in collectRules() if the same

		#	 rule appears in >1 method.  But in that case the

		#	 user probably gets what they deserve :-)  Also

		#	 undefined results if rules causing the ambiguity

		#	 appear in the same method.

		#

		sortlist = []

		name2index = {}

		for i in range(len(children)):

			((rule, pos, parent), index) = children[i]

			lhs, rhs = rule

			name = self.rule2name[rule]

			sortlist.append((len(rhs), name))

			name2index[name] = i

		sortlist.sort()

		list = map(lambda (a,b): b, sortlist)

		return children[name2index[self.resolve(list)]]



	def resolve(self, list):

		#

		#  Resolve ambiguity in favor of the shortest RHS.

		#  Since we walk the tree from the top down, this

		#  should effectively resolve in favor of a "shift".

		#

		return list[0]



#

#  GenericASTBuilder automagically constructs a concrete/abstract syntax tree

#  for a given input.  The extra argument is a class (not an instance!)

#  which supports the "__setslice__" and "__len__" methods.

#

#  XXX - silently overrides any user code in methods.

#



class GenericASTBuilder(GenericParser):

	def __init__(self, AST, start):

		GenericParser.__init__(self, start)

		self.AST = AST



	def preprocess(self, rule, func):

		rebind = lambda lhs, self=self: \

				lambda args, lhs=lhs, self=self: \

					self.buildASTNode(args, lhs)

		lhs, rhs = rule

		return rule, rebind(lhs)



	def buildASTNode(self, args, lhs):

		children = []

		for arg in args:

			if isinstance(arg, self.AST):

				children.append(arg)

			else:

				children.append(self.terminal(arg))

		return self.nonterminal(lhs, children)



	def terminal(self, token):	return token



	def nonterminal(self, type, args):

		rv = self.AST(type)

		rv[:len(args)] = args

		return rv



#

#  GenericASTTraversal is a Visitor pattern according to Design Patterns.  For

#  each node it attempts to invoke the method n_<node type>, falling

#  back onto the default() method if the n_* can't be found.  The preorder

#  traversal also looks for an exit hook named n_<node type>_exit (no default

#  routine is called if it's not found).  To prematurely halt traversal

#  of a subtree, call the prune() method -- this only makes sense for a

#  preorder traversal.  Node type is determined via the typestring() method.

#



class GenericASTTraversalPruningException:

	pass



class GenericASTTraversal:

	def __init__(self, ast):

		self.ast = ast



	def typestring(self, node):

		return node.type



	def prune(self):

		raise GenericASTTraversalPruningException



	def preorder(self, node=None):

		if node is None:

			node = self.ast



		try:

			name = 'n_' + self.typestring(node)

			if hasattr(self, name):

				func = getattr(self, name)

				func(node)

			else:

				self.default(node)

		except GenericASTTraversalPruningException:

			return



		for kid in node:

			self.preorder(kid)



		name = name + '_exit'

		if hasattr(self, name):

			func = getattr(self, name)

			func(node)



	def postorder(self, node=None):

		if node is None:

			node = self.ast



		for kid in node:

			self.postorder(kid)



		name = 'n_' + self.typestring(node)

		if hasattr(self, name):

			func = getattr(self, name)

			func(node)

		else:

			self.default(node)





	def default(self, node):

		pass



#

#  GenericASTMatcher.  AST nodes must have "__getitem__" and "__cmp__"

#  implemented.

#

#  XXX - makes assumptions about how GenericParser walks the parse tree.

#



class GenericASTMatcher(GenericParser):

	def __init__(self, start, ast):

		GenericParser.__init__(self, start)

		self.ast = ast



	def preprocess(self, rule, func):

		rebind = lambda func, self=self: \

				lambda args, func=func, self=self: \

					self.foundMatch(args, func)

		lhs, rhs = rule

		rhslist = list(rhs)

		rhslist.reverse()



		return (lhs, tuple(rhslist)), rebind(func)



	def foundMatch(self, args, func):

		func(args[-1])

		return args[-1]



	def match_r(self, node):

		self.input.insert(0, node)

		children = 0



		for child in node:

			if children == 0:

				self.input.insert(0, '(')

			children = children + 1

			self.match_r(child)



		if children > 0:

			self.input.insert(0, ')')



	def match(self, ast=None):

		if ast is None:

			ast = self.ast

		self.input = []



		self.match_r(ast)

		self.parse(self.input)



	def resolve(self, list):

		#

		#  Resolve ambiguity in favor of the longest RHS.

		#

		return list[-1]



def _dump(tokens, states):

	for i in range(len(states)):

		print 'state', i

		for (lhs, rhs), pos, parent in states[i]:

			print '\t', lhs, '::=',

			print string.join(rhs[:pos]),

			print '.',

			print string.join(rhs[pos:]),

			print ',', parent

		if i < len(tokens):

			print

			print 'token', str(tokens[i])

			print