/Core/Dependencies/Havok/Tools/Serialize/reflectionDatabase/gccxmlGenerateDB.py

# 
# Confidential Information of Telekinesys Research Limited (t/a Havok). Not for disclosure or distribution without Havok's 
# prior written consent. This software contains code, techniques and know-how which is confidential and proprietary to Havok. 
# Level 2 and Level 3 source code contains trade secrets of Havok. Havok Software (C) Copyright 1999-2011 Telekinesys Research Limited t/a Havok. All Rights Reserved. Use of this software is subject to the terms of an end user license agreement. 
#

#!/usr/bin/env python

# Generate a project database using gccxml. This should not be called directly.
# generateDB is the external interface. This examines the project database corresponding
# to the specified project, creating or updating it if necessary and returning the
# resulting (up-to-date) database object
#
# This manages all of the interface to gccxml. headerToDomXML parses the resulting data
# structures using the pygccxml bindings

from __future__ import with_statement
import reflectionDatabase
import headerToDomXML
import string
import sys
import os
import re
import util
import cPickle

## GCCXML settings file, customise gccxml behaviour in this file
import gccxmlSettings

try:
    for p in gccxmlSettings.HK_PYGCCXML_PATHADD:
        sys.path.append(p)
except (NameError, AttributeError):
    pass

BASE_DIR = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir)

def handleErrorIfThisIsRelease(msg):
    if 'HAVOK_SDKS_DIR' not in os.environ:
        import checkPrerequisites
        print msg
        checkPrerequisites.checkPrerequisites()
        sys.exit(-1)

# If we don't have the HAVOK_SDKS_DIR it must be in the system path
try:
    sys.path.append(os.path.join(os.environ['HAVOK_SDKS_DIR'], "win32", "gccXML", "pygccxml-1.0.0"))
except KeyError:
    pass

try:
    from pygccxml import parser as pygccxmlParser, declarations as pygccxmlDecls
except:
    handleErrorIfThisIsRelease('Error locating PyGCCXML, launching prerequisites checker...\n')
    raise RuntimeError, """Unable to load pygccxml module. Ensure that it is visible to
the default python installation. See Common Havok Components > Serialization for more information"""

## Find the gccxml executable
def findGCCXML():
    gccxmlExe = 'gccxml'
    exeExtention = {'nt': '.exe',
                     'posix': ''}
    gccxmlExe += exeExtention[os.name]

    gccxmlFile = None
    try:
        gccxmlFile = gccxmlSettings.HK_GCCXML_PATH
    except (NameError, AttributeError):
        pass

    if not gccxmlFile:
        #check for the GCCXML_ROOT env var
        gccxml_root_env_var = 'GCCXML_ROOT'
        if gccxml_root_env_var in os.environ:
            gccxml_root = os.path.join(os.environ[gccxml_root_env_var].replace('"',''), '..', '..')
            possibleGccxmlExe = os.path.join(gccxml_root, 'bin', gccxmlExe)
            gccxmlFile = possibleGccxmlExe

    if not gccxmlFile and os.name == "nt": #try HAVOK_SDKS_DIR if GCCXML_ROOT didnt work out, and we're on windows
        havok_sdks_dir_env_var = 'HAVOK_SDKS_DIR'
        if havok_sdks_dir_env_var in os.environ:
            gccxml_root = os.path.join(os.environ[havok_sdks_dir_env_var], "win32", "gccXML")
            gccxmlFile = os.path.join(gccxml_root, "gccxml.exe")

    if not gccxmlFile:
        #Search the PATH
        for path in os.environ['PATH'].split(os.path.pathsep):
            possible_gccxmlExe = os.path.join(path, gccxmlExe)
            if os.path.exists(possible_gccxmlExe):
                gccxmlFile = possible_gccxmlExe
    if gccxmlFile and os.path.exists(gccxmlFile):
        return gccxmlFile
    raise RuntimeError, """Unable to find gccxml.exe -- check Tools/Serialization/reflectionDatabase/gccxmlSettings.py
See Common Havok Components > Serialization for more information"""

env_re = re.compile(r'\$\(([\w\d_]+)\)')
## Resolve any environment variables in a given string
## Any substring of the form $(NAME) is replaced with the
## environment variable NAME
def resolveEnv(path):
    ret = env_re.search(path)
    while ret:
        try:
            path = path.replace('$(' + ret.groups()[0] + ')', os.environ[ret.groups()[0]])
        except KeyError:
            path = path.replace('$(' + ret.groups()[0] + ')', "")
        ret = env_re.search(path)
    return path     

def generateCachedDecls(ns):
    localDicts = {"": {}}
    globalDict = localDicts[""]    
    fnameCache = {}
    for d in ns.decls():
        try:
            globalDict[d.name].append(d)
        except KeyError:
            globalDict[d.name] = [d]
        if d.location:
            try:
                fname = fnameCache[d.location.file_name]
            except KeyError:
                fname = os.path.normpath(os.path.abspath(d.location.file_name)).lower()
                fnameCache[d.location.file_name] = fname

            try:
                localDicts[fname].append(d)
            except KeyError:
                localDicts[fname] = [d]
    return localDicts

## Generate the configuration for gccxml to parse a given project.
## When run on an internal Havok build this reads "global_settings.build"
## and the project's "settings.build" file. It saves the required settings
## (include directories and files) into a cached settings file called
## "reflectionSettings.cache". Build settings are then extracted from
## these caches, as unlike the "settings.build" files they're shipped in
## Havok's SDK packages.
def generateConfig(project_dir, DBObject, compilerOverride = None):
    standardCFlags = r"-include " + os.path.abspath(os.path.join( os.path.dirname(__file__), "tweakdefines.h")) + " -include " + os.path.abspath(os.path.join( BASE_DIR, "Source", "Common", "Base", "hkBase.h"))
    standardIncludePaths = [os.path.dirname(__file__), os.path.join( BASE_DIR, "Source"), BASE_DIR]

    DBObject.project_dir = project_dir

    settings_build = os.path.join(project_dir, "settings.build")
    requiredSettings = ['addIncludePath', 'setPchFile', 'setPrefix'] # The only settings.build settings the reflection system uses.    
    cachedSettingsFile = os.path.join(project_dir, "reflectionSettings.cache")

    # Read the settings.build file (if it exists) and create a cached settings file.
    if os.path.exists(settings_build) and os.path.isfile(settings_build):

        sys.path.append(os.path.join(BASE_DIR, "Build", "ReleaseSystem"))
        import BuildSystem.Modules

        with open(settings_build, "r") as project:
            settingsGlobals = {}

            globalSettingsFile = os.path.join(BASE_DIR, "Build", "ReleaseSystem", "global_settings.build")
            if os.path.exists(globalSettingsFile):
                for line in open(globalSettingsFile).readlines():
                    line=line.strip()
                    if line!='':
                        (key, value) = map(string.strip, line.split('='))
                        value = resolveEnv(value)
                        settingsGlobals[key] = value

            settingsDict = BuildSystem.Modules.evalSettings(settings_build, "win32", "msvc", "9", "ai+animation+behavior+cloth+destruction+physics", "", False, "win32", settingsGlobals)
            prunedSettingsDict = {}
            for x in settingsDict:
                if x in requiredSettings:
                    prunedSettingsDict[x] = settingsDict[x]
            import pprint
            #open( cachedSettingsFile, 'w' ).write( pprint.pformat(prunedSettingsDict) )
            util.writeIfDifferent(pprint.pformat(prunedSettingsDict), cachedSettingsFile)

    # The cached file will be present in internal Havok modules or modules in Havok sdk packages.
    if os.path.exists( cachedSettingsFile ):
        # We need to explicitly remove all '\r' characters otherwise eval fails on linux.
        DBObject.settings = eval(open(cachedSettingsFile).read().replace('\r\n','\n' ))

    # Update standardCFlags and standardIncludePaths using the information in the cached settings file.
    try:
        for path in DBObject.settings['addIncludePath']:
            path = resolveEnv(path)
            relativeDirName = os.path.join( BASE_DIR, path)
            dirName = path
            if os.path.exists(relativeDirName) and os.path.isdir(relativeDirName):
                standardIncludePaths.append(relativeDirName)
            if os.path.exists(dirName) and os.path.isdir(dirName):
                standardIncludePaths.append(dirName)
    except (KeyError, TypeError):
        pass
    try:
        for f in DBObject.settings['setPchFile']:
            standardCFlags += r" -include %s" % f
    except (KeyError, TypeError):
        pass

    # Find gccxml executable
    try:
        gccxml_path = findGCCXML()
    except:
        handleErrorIfThisIsRelease('Error locating gccxml, launching prerequisites checker...\n')
        raise

    # Compiler should be set but set it if it isn't
    if compilerOverride:
        HK_GCCXML_COMPILER = compilerOverride
    else:
        try:
            HK_GCCXML_COMPILER = gccxmlSettings.HK_GCCXML_COMPILER
        except (NameError, AttributeError):
            HK_GCCXML_COMPILER="cl"

    try:
        parseConfig = pygccxmlParser.config_t(gccxml_path=gccxml_path,
                                          include_paths=standardIncludePaths,
                                          working_directory=os.path.dirname(__file__),
                                          cflags=(standardCFlags),
                                          compiler=HK_GCCXML_COMPILER)
    except RuntimeError:
        handleErrorIfThisIsRelease('Error locating gccxml, launching prerequisites checker...\n')
        sys.stderr.write("Error running GCCXML  -- check the system is setup correctly.\nGCCXML must be present in the ThirdParty/sdks directory\n")
        raise

    return parseConfig

def getProjectDBFile(project_dir):
    return os.path.join(project_dir, "reflections.db")

# Platform is treated differently if there is a reflection declaration
# With NO reflection declaration, allow only of platforms is 'ALL'
# WITH reflection declaration, allow if platform is not 'NONE' or 'SPU'
def allowPlatform(platformString, hasReflection):
    if not platformString:
        return True
    platforms = set(platformString.split(' '))
    if 'NONE' in platforms:
        return False
    if 'ALL' in platforms:
        return True
    if 'SPU' in platforms:
        return False
    return hasReflection

def allowProduct(product, tkbms=None):
    if tkbms:
        if not product or product == 'ALL':
            return True
        tkbmsList = tkbms.split("+")
        productList = product.split("+")
        # If any of the tags in product are not in tkbms, fail
        return not any([tag not in tkbmsList for tag in productList])
    else:
        return product != 'NONE'

excludeFiles = ["tests.h", "alltests.h", "hkpps2solverinternal.h", "hkpsolverinternal.h", "hkppspgccsolverinternal.h", "hkpfpusolverinternal.h",
                "hkpps2jacobianbuilder.h", "hkpps3jacobianbuilder.h", "hkppspgccjacobianbuilder.h", "hkpfpujacobianbuilder.h", "hkpjacobianbuilder.h",
                "hkppspsolverexport.h", "hkpfpusolverexport.h", "hkcompatversions.h", "hkhavokversionclasses.h", "hkmemoryclasses.h",
                "hkgptriangulatordebugger.h", "hkfinishloadedobjectflag.h", "hkworldstresstesttestfunctions.h", "bdferror.h", "hkbaseext.h",
                "hkpseudopad.h", "hkmxvector4.h", "hkvector4mxvector4.h", "hkpcollisionqueryutil.h"]

excludeFileEndings = ["demovariants.h", "spu.h"]
excludeFileStarts = ["wii", "hkspu", "hkps3", "hkbsd"]

def allowFileName(filename, dirname, hasReflection):
    filename = filename.lower()
    dirname = dirname.lower()
    # Exclude various platform-specific files that aren't useful to the
    # reflection and tracking system
    if filename in excludeFiles:
        return False
    for e in excludeFileEndings:
        if filename.endswith(e):
            return False
    for e in excludeFileStarts:
        if filename.startswith(e):
            return False
    if os.path.basename(dirname) in ["classlist", "classes"]:
        return False
    if re.search(r'\b((demo[\\/]graphics[\\/]common[\\/]font|demo[\\/]graphics[\\/]common[\\/]texture)|base[\\/]system[\\/]stopwatch|base[\\/]system[\\/]io[\\/](directory|filesystem|reader)|demo[\\/]demos[\\/]resources|spu|wii)\b', dirname):
        return False
    if re.search(r'\bcontenttools\b', dirname) and not hasReflection:
        return False
    if re.search(r'\bbase[\\/]system[\\/]io[\\/]writer\b', dirname) and filename != 'hkstreamwriter.h':
        return False
    if re.search(r'\bdemo[\\/]demos\b', dirname) and filename == 'main.h':
        return False
    # If it's not in source or demos, or it is in contenttools, we are only interested in reflected files
    # This is from memoryTracker.py -- it should match the condition there
    if not re.search(r'\b(source|demos)\b', dirname) or re.search(r'\bcontenttools\b', dirname):
        return hasReflection

    return True

def filterProjectFiles(filename, build_tkbms=None):
    if not filename.endswith(".h"):
        return None

    (dirname, justfilename) = os.path.split(filename)
    
    fileText = open(filename).read()
    tkbms = util.extract_tkbms(fileText)
    hasReflection = util.hasReflectionDeclaration(fileText)
    if not allowProduct(tkbms.get("product", ""), build_tkbms) or not allowPlatform(tkbms.get("platform",""), hasReflection) or not allowFileName(justfilename, dirname, hasReflection):
        return None
    
##    if not util.hasReflectionDeclaration(fileText):
##        return None
    filehash = reflectionDatabase.hashFileContents(fileText)
    return (filename, filehash)

## Find all of the files with reflection declarations from a given starting point.
def findProjectFiles(project_dir, tkbms = None):
    fileList = set()
    for dirname, subdirs, files in os.walk(project_dir):
        headers = filter(None, [filterProjectFiles(fname, tkbms) for fname in [reflectionDatabase.standardFileName(os.path.join(dirname,f)) for f in files]])
        fileList.update(headers)

    return sorted(fileList)

def createDB(project_dir, options, DBObject, filesToUpdate):
    reflections_db = getProjectDBFile(project_dir)
    try:
        if not DBObject:
            DBObject = reflectionDatabase.createReflectionDatabase()

        if options and options.compiler_msvc8:
            compilerOverride = "msvc8"
        elif options and options.compiler_msvc9:
            compilerOverride = "msvc9"
        elif options and options.compiler_gcc:
            compilerOverride = "gcc"
        else:
            compilerOverride = None

        projectConfig = generateConfig(project_dir, DBObject, compilerOverride)

        # If there are no reflected files, just return an empty object
        if filesToUpdate and len(zip(*filesToUpdate)):
            projectParser = headerToDomXML.hkXMLParser()
            
            try:
                decls = pygccxmlParser.parse(zip(*filesToUpdate)[0], compilation_mode = pygccxmlParser.COMPILATION_MODE.ALL_AT_ONCE, config = projectConfig)
            except (RuntimeError, IOError, OSError, pygccxmlParser.source_reader.gccxml_runtime_error_t), error:
                print "%s: error creating reflection file : %s" % (project_dir, error)
                handleErrorIfThisIsRelease('Error in pygccxml, launching prerequisites checker...\n')
                raise

            ns = pygccxmlDecls.get_global_namespace(decls)
            dicts = generateCachedDecls(ns)

            for (headerfile, filehash) in filesToUpdate:
                try:
                    projectClass = projectParser.parseFileGeneric(headerfile, ns, dicts)
                    projectClass.origfilename = reflectionDatabase.standardFileName(headerfile)
                    DBObject.Documents[projectClass.origfilename] = projectClass
                    DBObject.contentsHash[projectClass.origfilename] = filehash
                    DBObject.new = True
                except (RuntimeError, IOError, OSError), error:
                    print "%s: error creating reflection file : %s" % (headerfile, error)
                    raise

        util.writeDatabase(DBObject, reflections_db)

    except:
        if os.path.exists(reflections_db):
            os.remove(reflections_db)
        raise
    return DBObject


def checkProjectDependencies(project_dir, reflections_db, DBObject, havokProducts=None):
    try:
        projectFileList = findProjectFiles(project_dir, havokProducts) # (filename, filehash) tuples

        if not DBObject:
            return projectFileList

        if not len(zip(projectFileList)):
            return []
        
        # If any of the files in the DB no longer exist or are no longer in the database, remove them
        for removedFile in [x.origfilename for x in DBObject.getDocuments() if x.origfilename not in zip(*projectFileList)[0]]:
            del DBObject.Documents[removedFile]
            del DBObject.contentsHash[removedFile]
            DBObject.new = True

        # Check the file hash for each included file. Note this is based on the file contents NOT the mtime
        return [(fname, fhash) for (fname, fhash) in projectFileList if DBObject.contentsHash.get(fname, "") != fhash]
    except OSError:
        # If any file does not exist or is not readable, fail the dependency
        return projectFileList

def generateDB(project_dir, options):
    """Create if necessary and return a project information database"""

    project_dir = reflectionDatabase.standardFileName(project_dir)
    if not os.path.exists(os.path.join(project_dir, "settings.build")) and not os.path.exists(os.path.join(project_dir, "reflectionSettings.cache")):
        # This is a customer build
        setattr(options, "customer_build", True)

    reflections_db = getProjectDBFile(project_dir)
    try:
        DBObject = util.readDatabase(reflections_db)
    except cPickle.PickleError:
        DBObject = None
    # Else the exception will be passed on 

    # Check that the objects look the same
    localDB = reflectionDatabase.createReflectionDatabase()
    if not DBObject or sorted(localDB.__dict__.keys()) != sorted(DBObject.__dict__.keys()) or localDB.version != DBObject.version:
        DBObject = None

    filesToUpdate = checkProjectDependencies(project_dir, reflections_db, DBObject, (options and options.havok_products))

    if (not DBObject) or (options and options.force_rebuild):
        filesToUpdate = findProjectFiles(project_dir, (options and options.havok_products))
    
    if (not DBObject) or (options and options.force_rebuild) or len(filesToUpdate):
        return createDB(project_dir, options, DBObject, filesToUpdate)
    return DBObject

# 
# Havok SDK - NO SOURCE PC DOWNLOAD, BUILD(#20110914) 
#  
# Confidential Information of Havok.  (C) Copyright 1999-2011 
# Telekinesys Research Limited t/a Havok. All Rights Reserved. The Havok 
# Logo, and the Havok buzzsaw logo are trademarks of Havok.  Title, ownership 
# rights, and intellectual property rights in the Havok software remain in 
# Havok and/or its suppliers. 
#  
# Use of this software for evaluation purposes is subject to and indicates 
# acceptance of the End User licence Agreement for this product. A copy of 
# the license is included with this software and is also available at www.havok.com/tryhavok. 
#  
#