/src/pentest/grabber/crystal.py

#!/usr/bin/env python

"""

	Crystal Module for Grabber v0.1

	Copyright (C) 2006 - Romain Gaucher - http://rgaucher.info

"""

import sys,os,re, string, shutil

from xml.sax import *   # Need PyXML [http://pyxml.sourceforge.net/]

from grabber import getContent_POST, getContentDirectURL_POST

from grabber import getContent_GET , getContentDirectURL_GET

from grabber import single_urlencode, partially_in, unescape

from grabber import investigate, setDatabase

from spider  import flatten, htmlencode, dict_add

from spider  import database





vulnToDescritiveNames = {

	'xss' : 'Cross-Site Scripting',

	'sql' : 'SQL Injection',

	'bsql': 'Specific Blind Injection...',

	'include' : 'PHP Include Vulnerability'

}





"""

	Crystal Module Cooking Book

	---------------------------



	Make-ahead Tip: Prepare lots of coffee before starting...

	Preparation: 24 hours

	Ingredients:

		A: PHP-Sat

		B: Grabber Modules lambda

	Tools:

		A: Context editor

		B: Python 2.4

		C: Nice music (Opera is not needed but you should listen this)



	Directions:

		0) Read the configuration file (with boolean operator in patterns)

		1) Scan the PHP sources with PHP-Sat handler (which copy everything in the

		'/local/crystal/' directory).

		2) Make a kind of diff then:

			If the diff results, check for the patterns (given in the configuration file)

				Parse the PHP line under the end of the pattern

				Try to get a variable value

				<after>

					If no direct variable... backtrack sequentially or in the AST

				</after>

		3) Generate the XML report of "the crystal-static-analysis" module

		4) Build a database of:

			>>> transformed_into_URL(hypothetical flawed files) : {list of "flawed" params}

		5) Run the classical tools

"""



# Crystal Configuration variables

crystalFiles      = None

crystalUrl        = None

crystalExtension  = None



crystalAnalyzerBin= None

crystalAnalyzerInputParam = None

crystalAnalyzerOutputParam = None



crystalCheckStart = None

crystalCheckEnd   = None

# example: {'xss' : ['pattern_1 __AND__ pattern_3','pattern_2'], 'sql' : ['pattern_3'], 'bsql' : ['pattern_3']}

crystalPatterns   = {}

# example: {'xss' : [{'var-position' : reg1}], 'sql' : [{'var-position' : reg2}]}

crystalRegExpPatterns = {}

crystalStorage    = []

crystalDatabase   = {}

crystalFinalStorage = {}





def normalize_whitespace(text):

	return ' '.join(text.split())



def clear_whitespace(text):

	return text.replace(' ','')



# Handle the XML file with a SAX Parser

class CrystalConfHandler(ContentHandler):

	def __init__(self):

		self.inAnalyzer  = False

		self.inPatterns  = False

		self.inPattern   = False

		self.isRegExp    = False

		self.curretVarPos= None

		self.currentKeys = []

		self.string     = ""

	def startElement(self, name, attrs):

		global crystalAnalyzerInputParam, crystalAnalyzerOutputParam, crystalPatterns, crystalCheckStart, crystalCheckEnd

		self.string = ""

		self.currentKeys = []

		if name == 'analyzer':

			self.inAnalyzer = True

		elif name == 'path' and self.inAnalyzer:

			# store the attributes input and output

			if 'input' in attrs.keys() and 'output' in attrs.keys():

				crystalAnalyzerInputParam  = attrs.getValue('input')

				crystalAnalyzerOutputParam = attrs.getValue('output')

			else:

				raise KeyError("CrystalXMLConf: <path> needs 'input' and 'output' attributes")

		elif name == 'patterns' and self.inAnalyzer:

			self.inPatterns = True

			if 'start' in attrs.keys() and 'end' in attrs.keys():

				crystalCheckStart = attrs.getValue('start')

				crystalCheckEnd   = attrs.getValue('end')

			else:

				raise KeyError("CrystalXMLConf: <patterns> needs 'start' and 'end' attributes")

			if 'name' in attrs.keys():

				if attrs.getValue('name').lower() == 'regexp':

					self.isRegExp = True



		elif self.inPatterns and name == 'pattern':

			self.inPattern = True

			if 'module' in attrs.keys():

				modules = attrs.getValue('module')

				modules.replace(' ','')

				self.currentKeys = modules.split(',')

				if self.isRegExp:

					if 'varposition' in attrs.keys():

						curretVarPos = attrs.getValue('varposition')

			else:

				raise KeyError("CrystalXMLConf: <pattern > needs 'varposition' attribute")



	def characters(self, ch):

		self.string = self.string + ch



	def endElement(self, name):

		global crystalFiles, crystalUrl, crystalExtension, crystalAnalyzerBin, crystalPatterns, crystalRegExpPatterns

		if name == 'files':

			crystalFiles = normalize_whitespace(self.string)

		elif name == 'url':

			crystalUrl = normalize_whitespace(self.string)

		elif name == 'extension' and self.inAnalyzer:

			crystalExtension = normalize_whitespace(self.string)

		elif name == 'path' and self.inAnalyzer:

			crystalAnalyzerBin = normalize_whitespace(self.string)

		elif not self.isRegExp and name == 'pattern' and self.inPattern:

			tempList = self.string.split('__OR__')

			for a in self.currentKeys:

				if a not in crystalPatterns:

					crystalPatterns[a] = []

				l = crystalPatterns[a]

				for t in tempList:

					l.append(normalize_whitespace(t))

		elif self.isRegExp and name == 'pattern' and self.inPattern:

			"""

			tempList = self.string.split('__OR__')

			for a in self.currentKeys:

				if a not in crystalPatterns:

					crystalRegExpPatterns[a] = []

				l = crystalRegExpPatterns[a]

				# build the compiled regexp

				plop = normalize_whitespace(l)

				plop = re.compile(plop, re.I)

				l.append({currentVarPos : plop})

			"""

		elif name == "patterns" and self.inPatterns:

			self.inPatterns = False

			if self.isRegExp:

				self.isRegExp = False

		elif name == "analyzer" and self.inAnalyzer:

			self.inAnalyzer = False



def copySubTree(src, dst, regFilter):

	global crystalStorage

	names = os.listdir(src)

	try:

		os.mkdir(dst)

	except OSError:

		a = 0

	try:

		os.mkdir(dst.replace('crystal/current', 'crystal/analyzed'))

	except OSError:

		a = 0

	for name in names:

		srcname = os.path.join(src, name)

		dstname = os.path.join(dst, name)

		try:

			if os.path.islink(srcname):

				linkto = os.readlink(srcname)

				os.symlink(linkto, dstname)

			elif os.path.isdir(srcname):

				copySubTree(srcname, dstname, regFilter)

			elif regFilter.match(srcname):

				shutil.copy2(srcname, dstname)

				crystalStorage.append(dstname)

		except (IOError, os.error), why:

			continue



def execCmd(program, args):

	p = os.popen(program + " " + args)

	p.close()





def generateListOfFiles():

	"""

		Create a ghost in ./local/crystal/current and /local/crystal/analyzed

		And run the SwA tool

	"""

	regScripts = re.compile(r'(.*).' + crystalExtension + '$', re.I)

	copySubTree(crystalFiles, 'local/crystal/current', regScripts)

	print "Running the static analysis tool..."

	for file in crystalStorage:

		fileIn  = os.path.abspath(os.path.join('./', file))

		fileOut = os.path.abspath(os.path.join('./', file.replace('current', 'analyzed')))

		cmdLine = crystalAnalyzerInputParam + " " + fileIn + " " + crystalAnalyzerOutputParam + " " + fileOut

		# execCmd(crystalAnalyzerBin, cmdLine)

		print crystalAnalyzerBin,cmdLine

		os.system(crystalAnalyzerBin +" "+ cmdLine)



def stripNoneASCII(output):

	# should be somepthing to do that.. :/

	newOutput = ""

	for s in output:

		try:

			s = s.encode()

			newOutput += s

		except UnicodeDecodeError:

			continue

	return newOutput



def isPatternInFile(fileName):

	global crystalDatabase

	file = None

	try:

		file = open(fileName, 'r')

	except IOError:

		print "Crystal: Cannot open the file [%s]" % fileName

		return False

	inZone, inLined = False, False

	detectPattern = False

	lineNumber = 0

	shortName  = fileName[fileName.rfind('analyzed') + 9 : ]

	vulnName = ""

	for l in file.readlines():

		lineNumber += 1

		l = l.replace('\n','')

		try:

			"""

				Check for the regular expression patterns

			if len(crystalRegExpPatterns) > 0:

				for modules in crystalRegExpPatterns:

					for regexp in crystalRegExpPatterns[modules]:

						if regexp.match()

			"""

			if len(vulnName) > 0 and (detectPattern and not inZone or inLined):

				# creating the nice structure

				# { 'index.php' : {'xss' : {'12', 'echo $_GET["plop"]'}}}

				if shortName not in crystalDatabase:

					crystalDatabase[shortName] = {}

				if vulnName not in crystalDatabase[shortName]:

					crystalDatabase[shortName][vulnName] = {}

				if str(lineNumber) not in crystalDatabase[shortName][vulnName]:

					crystalDatabase[shortName][vulnName][str(lineNumber)] = l

				detectPattern = False

				inLined = False

				vulnName = ""

	

			if l.count(crystalCheckStart) > 0 and not inZone:

				b1 = l.find(crystalCheckStart)

				inZone = True

				# same line for start and end ?

				if l.count(crystalCheckEnd) > 0:

					b2 = l.find(crystalCheckStart)

					if b1 < b2:

						inZone = False

						position = l.lower().find(pattern.lower())

						if b1 < position and position < b2:

							detectPattern = True

							inLined = True

			elif inZone:

				# is there any pattern around the corner ?

				for modules in crystalPatterns:

					for p in crystalPatterns[modules]:

						p = p.lower()

						l = l.lower()

						# The folowing code is stupid!

						# I have to change the algorithm for the __AND__ parsing...

						if '__AND__' in p:

							listPatterns = p.split('__AND__')

							isIn = True

							for patton in listPatterns:

								if patton not in l:

									isIn = isIn and False

							if isIn:

								detectPattern = True

								vulnName = modules

							a=0

						else:

							# test if the simple pattern is in the line

							if p in l:

								detectPattern = True

								vulnName = modules

				if l.count(crystalCheckEnd) > 0:

					inZone = False

		except UnicodeDecodeError:

			continue

	return True



def buildDatabase():

	"""

		Read the analzed files (indirectly with the crystalStorage.replace('current','analyzed') )

		And look for the patterns

	"""

	listOut = []

	for file in crystalStorage:

		fileOut = os.path.abspath(os.path.join('./', file.replace('current', 'analyzed')))

		if not isPatternInFile(fileOut):

			print "Error with the file [%s]" % file



def createStructure():

	"""

		Create the structure in the ./local directory

	"""

	try:

		os.mkdir("local/crystal/")

	except OSError,e :

		a=0

	try:

		os.mkdir("local/crystal/current")

	except OSError,e :

		a=0

	try:

		os.mkdir("local/crystal/analyzed")

	except OSError,e :

		a=0



"""

def realLineNumberReverse(fileName, codeStr):

	print fileName, codeStr

	try:

		fN = os.path.abspath(os.path.join('./local/crystal/current/', fileName))

		file = open(fN, 'r')

		lineNumber = 0

		for a in file.readlines():

			lineNumber += 1

			if codeStr in a:

				print a

				file.close()

				return lineNumber

		file.close()

	except IOError,e:

		print e

		return 0

	return 0

"""



def generateReport_1():

	"""

		Create a first report like:



		* Developer report:

			# using XSLT...

			<site>

				<file name="index.php">

					<vulnerability line="9">xss</vulnerability>

					<vulnerability line="25">sql</vulnerability>

				</file>

				...

			</site>

		

		* Security report:

			<site>

				<vulnerability name="xss">

					<file name="index.php" line="9" />

					...

				</vulnerabilty>

				<vulnerability name="sql">

					<file name="index.php" line="25" />

				</vulnerabilty>

			</site>

	"""

	plop = open('results/crystal_SecurityReport_Grabber.xml','w')

	plop.write("<crystal>\n")

	plop.write("<site>\n")

	plop.write("<!-- The line numbers are from the files in the 'analyzed' directory -->\n")

	for file in crystalDatabase:

		plop.write("\t<file name='%s'>\n" % file)

		for vuln in crystalDatabase[file]:

			for line in crystalDatabase[file][vuln]:

				# lineNumber = realLineNumberReverse(file,crystalDatabase[file][vuln][line])

				localVuln = vuln

				if localVuln in vulnToDescritiveNames:

					localVuln = vulnToDescritiveNames[localVuln]

				plop.write("\t\t<vulnerability name='%s' line='%s' >%s</vulnerability>\n" % (localVuln, line, htmlencode(crystalDatabase[file][vuln][line])))

		plop.write("\t</file>\n")

	plop.write("</site>\n")

	plop.write("</crystal>\n")

	plop.close()





def buildUrlKey(file):

	fileName = file.replace('\\','/') # on windows...

	keyUrl = crystalUrl

	if keyUrl[len(keyUrl)-1] != '/' and fileName[0] != '/':

		keyUrl += '/'

	keyUrl += fileName

	return keyUrl





reParamPOST = re.compile(r'(.*)\$_POST\[(.+)\](.*)',re.I)

reParamGET  = re.compile(r'(.*)\$_GET\[(.+)\](.*)' ,re.I)



def getSimpleParamFromCode_GET(code):

	"""

		Using the regular expression above, try to get some parameters name

	"""

	params = [] # we can have multiple params...

	code = code.replace("'",'');

	code = code.replace('"','');

	if code.lower().count('get') > 0:

		# try to match the $_GET

		if reParamGET.match(code):

			out = reParamGET.search(code)

			params.append(out.group(2))

			params.append(getSimpleParamFromCode_GET(out.group(3)))

			params = flatten(params)

	return params





def getSimpleParamFromCode_POST(code):

	"""

		Using the regular expression above, try to get some parameters name

	"""

	params = [] # we can have multiple params...

	code = code.replace("'",'');

	code = code.replace('"','');

	if code.lower().count('post') > 0:

		# try to match the $_GET

		if reParamPOST.match(code):

			out = reParamPOST.search(code)

			params.append(out.group(2))

			params.append(getSimpleParamFromCode_POST(out.group(3)))

			params = flatten(params)

	return params





def createClassicalDatabase(vulnsType, localCrystalDB):

	"""

		From the crystalDatabase, generate the same database as in Spider

		This is generated for calling the differents modules



		ClassicalDB = { url : { 'GET' : { param : value } } }

	"""

	classicalDB = {}

	for file in localCrystalDB:

		# build the URL

		keyUrl = buildUrlKey(file)

		if keyUrl not in classicalDB:

			classicalDB[keyUrl] = {'GET' : {}, 'POST' : {}}

		for vuln in localCrystalDB[file]:

			# only get the kind of vulnerability we want

			if vuln != vulnsType:

				continue

			for line in localCrystalDB[file][vuln]:

				code = localCrystalDB[file][vuln][line]

				# try to extract some data...

				params_GET  = getSimpleParamFromCode_GET (code)

				params_POST = getSimpleParamFromCode_POST(code)

				if len(params_GET) > 0:

					for p in params_GET:

						lG = classicalDB[keyUrl]['GET']

						if p not in classicalDB[keyUrl]['GET']:

							lG = dict_add(lG,{p:''})

						classicalDB[keyUrl]['GET'] = lG

				if len(params_POST) > 0:

					for p in params_POST:

						lP = classicalDB[keyUrl]['POST']

						if p not in classicalDB[keyUrl]['POST']:

							lP = dict_add(lP,{p:''})

						classicalDB[keyUrl]['POST'] = lP

	return classicalDB





def retrieveVulnList():

	vulnList = []

	for file in crystalDatabase:

		for vuln in crystalDatabase[file]:

			if vuln not in vulnList:

				vulnList.append(vuln)

	return vulnList





def process(urlGlobal, localDB, attack_list):

	"""

		Crystal Module entry point

	"""

	print "Crystal Module Start"

	try:

		f = open("crystal.conf.xml", 'r')

		f.close()

	except IOError:

		print "The crystal module needs the 'crystal.conf.xml' configuration file."

		sys.exit(1)

	parser = make_parser()

	crystal_handler = CrystalConfHandler()

	# Tell the parser to use our handler

	parser.setContentHandler(crystal_handler)

	try:

		parser.parse("crystal.conf.xml")

	except KeyError, e:

		print e

		sys.exit(1)



	#---------- White box testing

	

	createStructure()

	generateListOfFiles()



	buildDatabase()

	print "Build first report: List of vulneratilities and places in the code"

	generateReport_1()

	#---------- Start the Black Box testing



	# need to create a classical database like, so losing information

	# but for a type of vulnerability

	listVulns = retrieveVulnList()



	for vulns in listVulns:

		localDatabase = createClassicalDatabase(vulns, crystalDatabase)

		setDatabase(localDatabase)

		print "inProcess Crystal DB = ", localDatabase

		# print vulns, database

		# Call the Black Box Module

		print "Scan for ", vulns

		investigate(crystalUrl, vulns)



	print "Crystal Module Stop"