/config/BuildSystem/sourceDatabase.py
Python | 374 lines | 326 code | 11 blank | 37 comment | 40 complexity | 7e6808bf1349c7bbf25b43d8fe235cbc MD5 | raw file
- #!/usr/bin/env python
- '''A source code database
- SourceDB is a database of file information used to determine whether files
- should be rebuilt by the build system. All files names are stored relative
- to a given root, which is intended as the root of a Project.
- Relative or absolute pathnames may be used as keys, but absolute pathnames
- must fall under the database root. The value format is a tuple of the following:
- Checksum: The md5 checksum of the file
- Mod Time: The time the file was last modified
- Timestamp: The time theentry was last modified
- Dependencies: A tuple of files upon which this entry depends
- This script also provides some default actions:
- - insert <database file> <filename>
- Inserts this file from the database, or updates its entry if it
- already exists.
- - remove <database file> <filename>
- Removes this file from the database. The filename may also be a
- regular expression.
- '''
- import logger
- import errno
- import os
- import re
- import time
- import cPickle
- try:
- from hashlib import md5 as new_md5
- except ImportError:
- from md5 import new as new_md5
- class SourceDB (dict, logger.Logger):
- '''A SourceDB is a dictionary of file data used during the build process.'''
- includeRE = re.compile(r'^#include (<|")(?P<includeFile>.+)\1')
- isLoading = 0
- def __init__(self, root, filename = None):
- dict.__init__(self)
- logger.Logger.__init__(self)
- self.root = root
- self.filename = filename
- if self.filename is None:
- self.filename = os.path.join(str(root), 'bsSource.db')
- self.isDirty = 0
- return
- def __str__(self):
- output = ''
- for source in self:
- (checksum, mtime, timestamp, dependencies) = self[source]
- output += source+'\n'
- output += ' Checksum: '+str(checksum)+'\n'
- output += ' Mod Time: '+str(mtime)+'\n'
- output += ' Timestamp: '+str(timestamp)+'\n'
- output += ' Deps: '+str(dependencies)+'\n'
- return output
- def __setstate__(self, d):
- logger.Logger.__setstate__(self, d)
- # We have to prevent recursive calls to this when the pickled database is loaded in load()
- # This is to ensure that fresh copies of the database are obtained after unpickling
- if not SourceDB.isLoading:
- SourceDB.isLoading = 1
- self.load()
- SourceDB.isLoading = 0
- return
- def getRelativePath(self, path):
- '''Returns a relative source file path using the root'''
- if os.path.isabs(path):
- root = str(self.root)
- if not path.startswith(root+os.sep):
- raise ValueError('Absolute path '+path+' conflicts with root '+root)
- else:
- path = path[len(root)+1:]
- return path
- def checkValue(self, value):
- '''Validate the value, raising ValueError for problems'''
- if not isinstance(value, tuple):
- raise ValueError('Source database values must be tuples, '+str(type(value))+' given')
- if not len(value) == 4:
- raise ValueError('Source database values must have 4 items, '+str(len(value))+' given')
- (checksum, mtime, timestamp, dependencies) = value
- if not isinstance(checksum, str):
- raise ValueError('Invalid checksum for source database, '+str(type(checksum))+' given')
- if not isinstance(mtime, int):
- raise ValueError('Invalid modification time for source database, '+str(type(mtime))+' given')
- elif mtime < 0:
- raise ValueError('Negative modification time for source database, '+str(mtime))
- if not isinstance(timestamp, float):
- raise ValueError('Invalid timestamp for source database, '+str(type(timestamp))+' given')
- elif timestamp < 0:
- raise ValueError('Negative timestamp for source database, '+str(timestamp))
- if not isinstance(dependencies, tuple):
- raise ValueError('Invalid dependencies for source database, '+str(type(dependencies))+' given')
- return value
- def __getitem__(self, key):
- '''Converts the key to a relative source file path using the root'''
- return dict.__getitem__(self, self.getRelativePath(key))
- def __setitem__(self, key, value):
- '''Converts the key to a relative source file path using the root, and checks the validity of the value'''
- self.isDirty = 1
- return dict.__setitem__(self, self.getRelativePath(key), self.checkValue(value))
- def __delitem__(self, key):
- '''Converts the key to a relative source file path using the root'''
- self.isDirty = 1
- return dict.__delitem__(self, self.getRelativePath(key))
- def __contains__(self, key):
- '''Converts the key to a relative source file path using the root'''
- return dict.__contains__(self, self.getRelativePath(key))
- def has_key(self, key):
- '''This method just calls self.__contains__(key)'''
- return self.__contains__(key)
- def items(self):
- '''Converts each key to a relative source file path using the root'''
- return [(self.getRelativePath(item[0]), item[1]) for item in dict.items(self)]
- def keys(self):
- '''Converts each key to a relative source file path using the root'''
- return map(self.getRelativePath, dict.keys(self))
- def update(self, d):
- '''Update the dictionary with the contents of d'''
- self.isDirty = 1
- for k in d:
- self[k] = d[k]
- return
- def getChecksum(source, chunkSize = 1024*1024):
- '''Return the md5 checksum for a given file, which may also be specified by its filename
- - The chunkSize argument specifies the size of blocks read from the file'''
- if isinstance(source, file):
- f = source
- else:
- f = file(source)
- m = new_md5()
- size = chunkSize
- buf = f.read(size)
- while buf:
- m.update(buf)
- buf = f.read(size)
- f.close()
- return m.hexdigest()
- getChecksum = staticmethod(getChecksum)
- def getModificationTime(source):
- t = os.path.getmtime(source)
- if isinstance(t, float):
- t = int(t)
- return t
- getModificationTime = staticmethod(getModificationTime)
- def updateSource(self, source, noChecksum = 0):
- self.isDirty = 1
- dependencies = ()
- try:
- (checksum, mtime, timestamp, dependencies) = self[source]
- except KeyError:
- pass
- self.logPrint('Updating '+source+' in source database', 3, 'sourceDB')
- if noChecksum:
- checksum = ''
- else:
- checksum = SourceDB.getChecksum(source)
- self[source] = (checksum, SourceDB.getModificationTime(source), time.time(), dependencies)
- return
- def clearSource(self, source):
- '''This removes source information, but preserved dependencies'''
- if source in self:
- self.isDirty = 1
- self.logPrint('Clearing '+source+' from source database', 3, 'sourceDB')
- (checksum, mtime, timestamp, dependencies) = self[source]
- self[source] = ('', 0, time.time(), dependencies)
- return
- def getDependencies(self, source):
- try:
- (checksum, mtime, timestamp, dependencies) = self[source]
- except KeyError:
- dependencies = ()
- return dependencies
- def addDependency(self, source, dependency):
- self.isDirty = 1
- dependencies = ()
- try:
- (checksum, mtime, timestamp, dependencies) = self[source]
- except KeyError:
- checksum = ''
- mtime = 0
- if not dependency in dependencies:
- self.logPrint('Adding dependency '+dependency+' to source '+source+' in source database', 3, 'sourceDB')
- dependencies = dependencies+(dependency,)
- self[source] = (checksum, mtime, time.time(), dependencies)
- return
- def calculateDependencies(self):
- self.logPrint('Recalculating dependencies', 1, 'sourceDB')
- for source in self:
- self.logPrint('Calculating '+source, 3, 'sourceDB')
- (checksum, mtime, timestamp, dependencies) = self[source]
- newDep = []
- try:
- file = file(source)
- except IOError, e:
- if e.errno == errno.ENOENT:
- del self[source]
- else:
- raise e
- comps = source.split('/')
- for line in file.xreadlines():
- m = self.includeRE.match(line)
- if m:
- filename = m.group('includeFile')
- matchNum = 0
- matchName = filename
- self.logPrint(' Includes '+filename, 3, 'sourceDB')
- for s in self:
- if s.find(filename) >= 0:
- self.logPrint(' Checking '+s, 3, 'sourceDB')
- c = s.split('/')
- for i in range(len(c)):
- if not comps[i] == c[i]: break
- if i > matchNum:
- self.logPrint(' Choosing '+s+'('+str(i)+')', 3, 'sourceDB')
- matchName = s
- matchNum = i
- newDep.append(matchName)
- # Grep for #include, then put these files in a tuple, we can be recursive later in a fixpoint algorithm
-