/conary/build/packagepolicy.py

Large files files are truncated, but you can click here to view the full file

#
# Copyright (c) rPath, Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#


"""
Module used after C{%(destdir)s} has been finalized to create the
initial packaging.  Also contains error reporting.
"""
import codecs
import imp
import itertools
import os
import re
import site
import sre_constants
import stat
import subprocess
import sys

from conary import files, trove
from conary.build import buildpackage, filter, policy, recipe, tags, use
from conary.build import smartform
from conary.deps import deps
from conary.lib import elf, magic, util, pydeps, fixedglob, graph

from conary.build.action import TARGET_LINUX
from conary.build.action import TARGET_WINDOWS

try:
    from xml.etree import ElementTree
except ImportError:
    try:
        from elementtree import ElementTree
    except ImportError:
        ElementTree = None


# Helper class
class _DatabaseDepCache(object):
    __slots__ = ['db', 'cache']
    def __init__(self, db):
        self.db = db
        self.cache = {}

    def getProvides(self, depSetList):
        ret = {}
        missing = []
        for depSet in depSetList:
            if depSet in self.cache:
                ret[depSet] = self.cache[depSet]
            else:
                missing.append(depSet)
        newresults = self.db.getTrovesWithProvides(missing)
        ret.update(newresults)
        self.cache.update(newresults)
        return ret


class _filterSpec(policy.Policy):
    """
    Pure virtual base class from which C{ComponentSpec} and C{PackageSpec}
    are derived.
    """
    bucket = policy.PACKAGE_CREATION
    processUnmodified = False
    supported_targets = (TARGET_LINUX, TARGET_WINDOWS)
    def __init__(self, *args, **keywords):
        self.extraFilters = []
        policy.Policy.__init__(self, *args, **keywords)

    def updateArgs(self, *args, **keywords):
        """
        Call derived classes (C{ComponentSpec} or C{PackageSpec}) as::
            ThisClass('<name>', 'filterexp1', 'filterexp2')
        where C{filterexp} is either a regular expression or a
        tuple of C{(regexp[, setmodes[, unsetmodes]])}
        """
        if args:
            theName = args[0]
            for filterexp in args[1:]:
                self.extraFilters.append((theName, filterexp))
        policy.Policy.updateArgs(self, **keywords)


class _addInfo(policy.Policy):
    """
    Pure virtual class for policies that add information such as tags,
    requirements, and provision, to files.
    """
    bucket = policy.PACKAGE_CREATION
    processUnmodified = False
    requires = (
        ('PackageSpec', policy.REQUIRED_PRIOR),
    )
    keywords = {
        'included': {},
        'excluded': {}
    }
    supported_targets = (TARGET_LINUX, TARGET_WINDOWS)

    def updateArgs(self, *args, **keywords):
        """
        Call as::
            C{I{ClassName}(I{info}, I{filterexp})}
        or::
            C{I{ClassName}(I{info}, exceptions=I{filterexp})}
        where C{I{filterexp}} is either a regular expression or a
        tuple of C{(regexp[, setmodes[, unsetmodes]])}
        """
        if args:
            args = list(args)
            info = args.pop(0)
            if args:
                if not self.included:
                    self.included = {}
                if info not in self.included:
                    self.included[info] = []
                self.included[info].extend(args)
            elif 'exceptions' in keywords:
                # not the usual exception handling, this is an exception
                if not self.excluded:
                    self.excluded = {}
                if info not in self.excluded:
                    self.excluded[info] = []
                self.excluded[info].append(keywords.pop('exceptions'))
            else:
                raise TypeError, 'no paths provided'
        policy.Policy.updateArgs(self, **keywords)

    def doProcess(self, recipe):
        # for filters
        self.rootdir = self.rootdir % recipe.macros

        # instantiate filters
        d = {}
        for info in self.included:
            newinfo = info % recipe.macros
            l = []
            for item in self.included[info]:
                l.append(filter.Filter(item, recipe.macros))
            d[newinfo] = l
        self.included = d

        d = {}
        for info in self.excluded:
            newinfo = info % recipe.macros
            l = []
            for item in self.excluded[info]:
                l.append(filter.Filter(item, recipe.macros))
            d[newinfo] = l
        self.excluded = d

        policy.Policy.doProcess(self, recipe)

    def doFile(self, path):
        fullpath = self.recipe.macros.destdir+path
        if not util.isregular(fullpath) and not os.path.islink(fullpath):
            return
        self.runInfo(path)

    def runInfo(self, path):
        'pure virtual'
        pass


class Config(policy.Policy):
    """
    NAME
    ====
    B{C{r.Config()}} - Mark files as configuration files

    SYNOPSIS
    ========
    C{r.Config([I{filterexp}] || [I{exceptions=filterexp}])}

    DESCRIPTION
    ===========
    The C{r.Config} policy marks all files below C{%(sysconfdir)s}
    (that is, C{/etc}) and C{%(taghandlerdir)s} (that is,
    C{/usr/libexec/conary/tags/}), and any other files explicitly
    mentioned, as configuration files.

        - To mark files as exceptions, use
          C{r.Config(exceptions='I{filterexp}')}.
        - To mark explicit inclusions as configuration files, use:
          C{r.Config('I{filterexp}')}

    A file marked as a Config file cannot also be marked as a
    Transient file or an InitialContents file.  Conary enforces this
    requirement.

    EXAMPLES
    ========
    C{r.Config(exceptions='%(sysconfdir)s/X11/xkb/xkbcomp')}

    The file C{/etc/X11/xkb/xkbcomp} is marked as an exception, since it is
    not actually a configuration file even though it is within the C{/etc}
    (C{%(sysconfdir)s}) directory hierarchy and would be marked as a
    configuration file by default.

    C{r.Config('%(mmdir)s/Mailman/mm_cfg.py')}

    Marks the file C{%(mmdir)s/Mailman/mm_cfg.py} as a configuration file;
    it would not be automatically marked as a configuration file otherwise.
    """
    bucket = policy.PACKAGE_CREATION
    processUnmodified = True
    requires = (
        # for :config component, ComponentSpec must run after Config
        # Otherwise, this policy would follow PackageSpec and just set isConfig
        # on each config file
        ('ComponentSpec', policy.REQUIRED_SUBSEQUENT),
    )
    invariantinclusions = [ '%(sysconfdir)s/', '%(taghandlerdir)s/']
    invariantexceptions = [ '%(userinfodir)s/', '%(groupinfodir)s' ]

    def doFile(self, filename):
        m = self.recipe.magic[filename]
        if m and m.name == "ELF":
            # an ELF file cannot be a config file, some programs put
            # ELF files under /etc (X, for example), and tag handlers
            # can be ELF or shell scripts; we just want tag handlers
            # to be config files if they are shell scripts.
            # Just in case it was not intentional, warn...
            if self.macros.sysconfdir in filename:
                self.info('ELF file %s found in config directory', filename)
            return
        fullpath = self.macros.destdir + filename
        if os.path.isfile(fullpath) and util.isregular(fullpath):
            if self._fileIsBinary(filename, fullpath):
                self.error("binary file '%s' is marked as config" % \
                        filename)
            self._markConfig(filename, fullpath)

    def _fileIsBinary(self, path, fn, maxsize=None, decodeFailIsError=True):
        limit = os.stat(fn)[stat.ST_SIZE]
        if maxsize is not None and limit > maxsize:
            self.warn('%s: file size %d longer than max %d',
                path, limit, maxsize)
            return True

        # we'll consider file to be binary file if we don't find any
        # good reason to mark it as text, or if we find a good reason
        # to mark it as binary
        foundFF = False
        foundNL = False
        f = open(fn, 'r')
        try:
            while f.tell() < limit:
                buf = f.read(65536)
                if chr(0) in buf:
                    self.warn('%s: file contains NULL byte', path)
                    return True
                if '\xff\xff' in buf:
                    self.warn('%s: file contains 0xFFFF sequence', path)
                    return True
                if '\xff' in buf:
                    foundFF = True
                if '\n' in buf:
                    foundNL = True
        finally:
            f.close()

        if foundFF and not foundNL:
            self.error('%s: found 0xFF without newline', path)

        utf8 = codecs.open(fn, 'r', 'utf-8')
        win1252 = codecs.open(fn, 'r', 'windows-1252')
        try:
            try:
                while utf8.tell() < limit:
                    utf8.read(65536)
            except UnicodeDecodeError, e:
                # Still want to print a warning if it is not unicode;
                # Note that Code Page 1252 is considered a legacy
                # encoding on Windows
                self.warn('%s: %s', path, str(e))
                try:
                    while win1252.tell() < limit:
                        win1252.read(65536)
                except UnicodeDecodeError, e:
                    self.warn('%s: %s', path, str(e))
                    return decodeFailIsError
        finally:
            utf8.close()
            win1252.close()

        return False

    def _addTrailingNewline(self, filename, fullpath):
        # FIXME: This exists only for stability; there is no longer
        # any need to add trailing newlines to config files.  This
        # also violates the rule that no files are modified after
        # destdir modification has been completed.
        self.warn("adding trailing newline to config file '%s'" % \
                filename)
        mode = os.lstat(fullpath)[stat.ST_MODE]
        oldmode = None
        if mode & 0600 != 0600:
            # need to be able to read and write the file to fix it
            oldmode = mode
            os.chmod(fullpath, mode|0600)

        f = open(fullpath, 'a')
        f.seek(0, 2)
        f.write('\n')
        f.close()
        if oldmode is not None:
            os.chmod(fullpath, oldmode)

    def _markConfig(self, filename, fullpath):
        self.info(filename)
        f = file(fullpath)
        f.seek(0, 2)
        if f.tell():
            # file has contents
            f.seek(-1, 2)
            lastchar = f.read(1)
            f.close()
            if lastchar != '\n':
                self._addTrailingNewline(filename, fullpath)

        f.close()
        self.recipe.ComponentSpec(_config=filename)


class ComponentSpec(_filterSpec):
    """
    NAME
    ====
    B{C{r.ComponentSpec()}} - Determines which component each file is in

    SYNOPSIS
    ========
    C{r.ComponentSpec([I{componentname}, I{filterexp}] || [I{packagename}:I{componentname}, I{filterexp}])}

    DESCRIPTION
    ===========
    The C{r.ComponentSpec} policy includes the filter expressions that specify
    the default assignment of files to components.  The expressions are
    considered in the order in which they are evaluated in the recipe, and the
    first match wins.  After all the recipe-provided expressions are
    evaluated, the default expressions are evaluated.  If no expression
    matches, then the file is assigned to the C{catchall} component.
    Note that in the C{I{packagename}:I{componentname}} form, the C{:}
    must be literal, it cannot be part of a macro.

    KEYWORDS
    ========
    B{catchall} : Specify the  component name which gets all otherwise
    unassigned files. Default: C{runtime}

    EXAMPLES
    ========
    C{r.ComponentSpec('manual', '%(contentdir)s/manual/')}

    Uses C{r.ComponentSpec} to specify that all files below the
    C{%(contentdir)s/manual/} directory are part of the C{:manual} component.

    C{r.ComponentSpec('foo:bar', '%(sharedir)s/foo/')}

    Uses C{r.ComponentSpec} to specify that all files below the
    C{%(sharedir)s/foo/} directory are part of the C{:bar} component
    of the C{foo} package, avoiding the need to invoke both the
    C{ComponentSpec} and C{PackageSpec} policies.

    C{r.ComponentSpec(catchall='data')}

    Uses C{r.ComponentSpec} to specify that all files not otherwise specified
    go into the C{:data} component instead of the default {:runtime}
    component.
    """
    requires = (
        ('Config', policy.REQUIRED_PRIOR),
        ('PackageSpec', policy.REQUIRED_SUBSEQUENT),
    )
    keywords = { 'catchall': 'runtime' }

    def __init__(self, *args, **keywords):
        """
        @keyword catchall: The component name which gets all otherwise
        unassigned files.  Default: C{runtime}
        """
        _filterSpec.__init__(self, *args, **keywords)
        self.configFilters = []
        self.derivedFilters = []

    def updateArgs(self, *args, **keywords):
        if '_config' in keywords:
            configPath=keywords.pop('_config')
            self.recipe.PackageSpec(_config=configPath)

        if args:
            name = args[0]
            if ':' in name:
                package, name = name.split(':')
                args = list(itertools.chain([name], args[1:]))
                if package:
                    # we've got a package as well as a component, pass it on
                    pkgargs = list(itertools.chain((package,), args[1:]))
                    self.recipe.PackageSpec(*pkgargs)

        _filterSpec.updateArgs(self, *args, **keywords)

    def doProcess(self, recipe):
        compFilters = []
        self.macros = recipe.macros
        self.rootdir = self.rootdir % recipe.macros

        self.loadFilterDirs()

        # The extras need to come before base in order to override decisions
        # in the base subfilters; invariants come first for those very few
        # specs that absolutely should not be overridden in recipes.
        for filteritem in itertools.chain(self.invariantFilters,
                                          self.extraFilters,
                                          self.derivedFilters,
                                          self.configFilters,
                                          self.baseFilters):
            if not isinstance(filteritem, (filter.Filter, filter.PathSet)):
                name = filteritem[0] % self.macros
                assert(name != 'source')
                args, kwargs = self.filterExpArgs(filteritem[1:], name=name)
                filteritem = filter.Filter(*args, **kwargs)

            compFilters.append(filteritem)

        # by default, everything that hasn't matched a filter pattern yet
        # goes in the catchall component ('runtime' by default)
        compFilters.append(filter.Filter('.*', self.macros, name=self.catchall))

        # pass these down to PackageSpec for building the package
        recipe.PackageSpec(compFilters=compFilters)


    def loadFilterDirs(self):
        invariantFilterMap = {}
        baseFilterMap = {}
        self.invariantFilters = []
        self.baseFilters = []

        # Load all component python files
        for componentDir in self.recipe.cfg.componentDirs:
            for filterType, map in (('invariant', invariantFilterMap),
                                    ('base', baseFilterMap)):
                oneDir = os.sep.join((componentDir, filterType))
                if not os.path.isdir(oneDir):
                    continue
                for filename in os.listdir(oneDir):
                    fullpath = os.sep.join((oneDir, filename))
                    if (not filename.endswith('.py') or
                        not util.isregular(fullpath)):
                        continue
                    self.loadFilter(filterType, map, filename, fullpath)

        # populate the lists with dependency-sorted information
        for filterType, map, filterList in (
            ('invariant', invariantFilterMap, self.invariantFilters),
            ('base', baseFilterMap, self.baseFilters)):
            dg = graph.DirectedGraph()
            for filterName in map.keys():
                dg.addNode(filterName)
                filter, follows, precedes  = map[filterName]

                def warnMissing(missing):
                    self.error('%s depends on missing %s', filterName, missing)

                for prior in follows:
                    if not prior in map:
                        warnMissing(prior)
                    dg.addEdge(prior, filterName)
                for subsequent in precedes:
                    if not subsequent in map:
                        warnMissing(subsequent)
                    dg.addEdge(filterName, subsequent)

            # test for dependency loops
            depLoops = [x for x in dg.getStronglyConnectedComponents()
                        if len(x) > 1]
            if depLoops:
                self.error('dependency loop(s) in component filters: %s',
                           ' '.join(sorted(':'.join(x)
                                           for x in sorted(list(depLoops)))))
                return

            # Create a stably-sorted list of config filters where
            # the filter is not empty.  (An empty filter with both
            # follows and precedes specified can be used to induce
            # ordering between otherwise unrelated components.)
            #for name in dg.getTotalOrdering(nodeSort=lambda a, b: cmp(a,b)):
            for name in dg.getTotalOrdering():
                filters = map[name][0]
                if not filters:
                    continue

                componentName = filters[0]
                for filterExp in filters[1]:
                    filterList.append((componentName, filterExp))


    def loadFilter(self, filterType, map, filename, fullpath):
        # do not load shared libraries
        desc = [x for x in imp.get_suffixes() if x[0] == '.py'][0]
        f = file(fullpath)
        modname = filename[:-3]
        m = imp.load_module(modname, f, fullpath, desc)
        f.close()

        if not 'filters' in m.__dict__:
            self.warn('%s missing "filters"; not a valid component'
                      ' specification file', fullpath)
            return
        filters = m.__dict__['filters']

        if filters and len(filters) > 1 and type(filters[1]) not in (list,
                                                                     tuple):
            self.error('invalid expression in %s: filters specification'
                       " must be ('name', ('expression', ...))", fullpath)

        follows = ()
        if 'follows' in m.__dict__:
            follows = m.__dict__['follows']

        precedes = ()
        if 'precedes' in m.__dict__:
            precedes = m.__dict__['precedes']

        map[modname] = (filters, follows, precedes)



class PackageSpec(_filterSpec):
    """
    NAME
    ====
    B{C{r.PackageSpec()}} - Determines which package each file is in

    SYNOPSIS
    ========
    C{r.PackageSpec(I{packagename}, I{filterexp})}

    DESCRIPTION
    ===========
    The C{r.PackageSpec()} policy determines which package each file
    is in. (Use C{r.ComponentSpec()} to specify the component without
    specifying the package, or to specify C{I{package}:I{component}}
    in one invocation.)

    EXAMPLES
    ========
    C{r.PackageSpec('openssh-server', '%(sysconfdir)s/pam.d/sshd')}

    Specifies that the file C{%(sysconfdir)s/pam.d/sshd} is in the package
    C{openssh-server} rather than the default (which in this case would have
    been C{openssh} because this example was provided by C{openssh.recipe}).
    """
    requires = (
        ('ComponentSpec', policy.REQUIRED_PRIOR),
    )
    keywords = { 'compFilters': None }

    def __init__(self, *args, **keywords):
        """
        @keyword compFilters: reserved for C{ComponentSpec} to pass information
        needed by C{PackageSpec}.
        """
        _filterSpec.__init__(self, *args, **keywords)
        self.configFiles = []
        self.derivedFilters = []

    def updateArgs(self, *args, **keywords):
        if '_config' in keywords:
            self.configFiles.append(keywords.pop('_config'))
        # keep a list of packages filtered for in PackageSpec in the recipe
        if args:
            newTrove = args[0] % self.recipe.macros

            self.recipe.packages[newTrove] = True
        _filterSpec.updateArgs(self, *args, **keywords)

    def preProcess(self):
        self.pkgFilters = []
        recipe = self.recipe
        self.destdir = recipe.macros.destdir
        if self.exceptions:
            self.warn('PackageSpec does not honor exceptions')
            self.exceptions = None
        if self.inclusions:
            # would have an effect only with exceptions listed, so no warning...
            self.inclusions = None

        # userinfo and groupinfo are invariant filters, so they must come first
        for infoType in ('user', 'group'):
            infoDir = '%%(%sinfodir)s' % infoType % self.macros
            realDir = util.joinPaths(self.destdir, infoDir)
            if not os.path.isdir(realDir):
                continue
            for infoPkgName in os.listdir(realDir):
                pkgPath = util.joinPaths(infoDir, infoPkgName)
                self.pkgFilters.append( \
                        filter.Filter(pkgPath, self.macros,
                                      name = 'info-%s' % infoPkgName))
        # extras need to come before derived so that derived packages
        # can change the package to which a file is assigned
        for filteritem in itertools.chain(self.extraFilters,
                                          self.derivedFilters):
            if not isinstance(filteritem, (filter.Filter, filter.PathSet)):
                name = filteritem[0] % self.macros
                if not trove.troveNameIsValid(name):
                    self.error('%s is not a valid package name', name)

                args, kwargs = self.filterExpArgs(filteritem[1:], name=name)
                self.pkgFilters.append(filter.Filter(*args, **kwargs))
            else:
                self.pkgFilters.append(filteritem)

        # by default, everything that hasn't matched a pattern in the
        # main package filter goes in the package named recipe.name
        self.pkgFilters.append(filter.Filter('.*', self.macros, name=recipe.name))

        # OK, all the filters exist, build an autopackage object that
        # knows about them
        recipe.autopkg = buildpackage.AutoBuildPackage(
            self.pkgFilters, self.compFilters, recipe)
        self.autopkg = recipe.autopkg

    def do(self):
        # Walk capsule contents ignored by doFile
        for filePath, _, componentName in self.recipe._iterCapsulePaths():
            realPath = self.destdir + filePath
            if util.exists(realPath):
                # Files that do not exist on the filesystem (devices)
                # are handled separately
                self.autopkg.addFile(filePath, realPath, componentName)
        # Walk normal files
        _filterSpec.do(self)

    def doFile(self, path):
        # all policy classes after this require that the initial tree is built
        if not self.recipe._getCapsulePathsForFile(path):
            realPath = self.destdir + path
            self.autopkg.addFile(path, realPath)

    def postProcess(self):
        # flag all config files
        for confname in self.configFiles:
            self.recipe.autopkg.pathMap[confname].flags.isConfig(True)




class InitialContents(policy.Policy):
    """
    NAME
    ====
    B{C{r.InitialContents()}} - Mark only explicit inclusions as initial
    contents files

    SYNOPSIS
    ========
    C{InitialContents([I{filterexp}])}

    DESCRIPTION
    ===========
    By default, C{r.InitialContents()} does not apply to any files.
    It is used to specify all files that Conary needs to mark as
    providing only initial contents.  When Conary installs or
    updates one of these files, it will never replace existing
    contents; it uses the provided contents only if the file does
    not yet exist at the time Conary is creating it.

    A file marked as an InitialContents file cannot also be marked
    as a Transient file or a Config file.  Conary enforces this
    requirement.

    EXAMPLES
    ========
    C{r.InitialContents('%(sysconfdir)s/conary/.*gpg')}

    The files C{%(sysconfdir)s/conary/.*gpg} are being marked as initial
    contents files.  Conary will use those contents when creating the files
    the first time, but will never overwrite existing contents in those files.
    """
    requires = (
        ('PackageSpec', policy.REQUIRED_PRIOR),
        ('Config', policy.REQUIRED_PRIOR),
    )
    bucket = policy.PACKAGE_CREATION
    processUnmodified = True

    invariantexceptions = [ '%(userinfodir)s/', '%(groupinfodir)s' ]
    invariantinclusions = ['%(localstatedir)s/run/',
                           '%(localstatedir)s/log/',
                           '%(cachedir)s/']

    def postInit(self, *args, **kwargs):
        self.recipe.Config(exceptions = self.invariantinclusions,
                allowUnusedFilters = True)

    def updateArgs(self, *args, **keywords):
        policy.Policy.updateArgs(self, *args, **keywords)
        self.recipe.Config(exceptions=args, allowUnusedFilters = True)

    def doFile(self, filename):
        fullpath = self.macros.destdir + filename
        recipe = self.recipe
        if os.path.isfile(fullpath) and util.isregular(fullpath):
            self.info(filename)
            f = recipe.autopkg.pathMap[filename]
            f.flags.isInitialContents(True)
            if f.flags.isConfig():
                self.error(
                    '%s is marked as both a configuration file and'
                    ' an initial contents file', filename)


class Transient(policy.Policy):
    """
    NAME
    ====
    B{C{r.Transient()}} - Mark files that have transient contents

    SYNOPSIS
    ========
    C{r.Transient([I{filterexp}])}

    DESCRIPTION
    ===========
    The C{r.Transient()} policy marks files as containing transient
    contents. It automatically marks the two most common uses of transient
    contents: python and emacs byte-compiled files
    (C{.pyc}, C{.pyo}, and C{.elc} files).

    Files containing transient contents are almost the opposite of
    configuration files: their contents should be overwritten by
    the new contents without question at update time, even if the
    contents in the filesystem have changed.  (Conary raises an
    error if file contents have changed in the filesystem for normal
    files.)

    A file marked as a Transient file cannot also be marked as an
    InitialContents file or a Config file.  Conary enforces this
    requirement.

    EXAMPLES
    ========
    C{r.Transient('%(libdir)s/firefox/extensions/')}

    Marks all the files in the directory C{%(libdir)s/firefox/extensions/} as
    having transient contents.
    """
    bucket = policy.PACKAGE_CREATION
    filetree = policy.PACKAGE
    processUnmodified = True
    requires = (
        ('PackageSpec', policy.REQUIRED_PRIOR),
        ('Config', policy.REQUIRED_PRIOR),
        ('InitialContents', policy.REQUIRED_PRIOR),
    )

    invariantinclusions = [
        r'..*\.py(c|o)$',
        r'..*\.elc$',
        r'%(userinfodir)s/',
        r'%(groupinfodir)s'
    ]

    def doFile(self, filename):
        fullpath = self.macros.destdir + filename
        if os.path.isfile(fullpath) and util.isregular(fullpath):
            recipe = self.recipe
            f = recipe.autopkg.pathMap[filename]
            f.flags.isTransient(True)
            if f.flags.isConfig() or f.flags.isInitialContents():
                self.error(
                    '%s is marked as both a transient file and'
                    ' a configuration or initial contents file', filename)


class TagDescription(policy.Policy):
    """
    NAME
    ====
    B{C{r.TagDescription()}} - Marks tag description files

    SYNOPSIS
    ========
    C{r.TagDescription([I{filterexp}])}

    DESCRIPTION
    ===========
    The C{r.TagDescription} class marks tag description files as
    such so that conary handles them correctly. Every file in
    C{%(tagdescriptiondir)s/} is marked as a tag description file by default.

    No file outside of C{%(tagdescriptiondir)s/} will be considered by this
    policy.

    EXAMPLES
    ========
    This policy is not called explicitly.
    """
    bucket = policy.PACKAGE_CREATION
    processUnmodified = False
    requires = (
        ('PackageSpec', policy.REQUIRED_PRIOR),
    )

    invariantsubtrees = [ '%(tagdescriptiondir)s/' ]

    def doFile(self, path):
        if self.recipe._getCapsulePathsForFile(path):
            return
        fullpath = self.macros.destdir + path
        if os.path.isfile(fullpath) and util.isregular(fullpath):
            self.info('conary tag file: %s', path)
            self.recipe.autopkg.pathMap[path].tags.set("tagdescription")


class TagHandler(policy.Policy):
    """
    NAME
    ====
    B{C{r.TagHandler()}} - Mark tag handler files

    SYNOPSIS
    ========
    C{r.TagHandler([I{filterexp}])}

    DESCRIPTION
    ===========
    All files in C{%(taghandlerdir)s/} are marked as a tag
    handler files.

    EXAMPLES
    ========
    This policy is not called explicitly.
    """
    bucket = policy.PACKAGE_CREATION
    processUnmodified = False
    requires = (
        ('PackageSpec', policy.REQUIRED_PRIOR),
    )
    invariantsubtrees = [ '%(taghandlerdir)s/' ]

    def doFile(self, path):
        if self.recipe._getCapsulePathsForFile(path):
            return
        fullpath = self.macros.destdir + path
        if os.path.isfile(fullpath) and util.isregular(fullpath):
            self.info('conary tag handler: %s', path)
            self.recipe.autopkg.pathMap[path].tags.set("taghandler")


class TagSpec(_addInfo):
    """
    NAME
    ====
    B{C{r.TagSpec()}} - Apply tags defined by tag descriptions

    SYNOPSIS
    ========
    C{r.TagSpec([I{tagname}, I{filterexp}] || [I{tagname}, I{exceptions=filterexp}])}

    DESCRIPTION
    ===========
    The C{r.TagSpec()} policy automatically applies tags defined by tag
    descriptions in both the current system and C{%(destdir)s} to all
    files in C{%(destdir)s}.

    To apply tags manually (removing a dependency on the tag description
    file existing when the packages is cooked), use the syntax:
    C{r.TagSpec(I{tagname}, I{filterexp})}.
    To set an exception to this policy, use:
    C{r.TagSpec(I{tagname}, I{exceptions=filterexp})}.

    EXAMPLES
    ========
    C{r.TagSpec('initscript', '%(initdir)s/')}

    Applies the C{initscript} tag to all files in the directory
    C{%(initdir)s/}.
    """
    requires = (
        ('PackageSpec', policy.REQUIRED_PRIOR),
    )
    def doProcess(self, recipe):
        self.tagList = []
        self.buildReqsComputedForTags = set()
        self.suggestBuildRequires = set()
        # read the system and %(destdir)s tag databases
        for directory in (recipe.macros.destdir+'/etc/conary/tags/',
                          '/etc/conary/tags/'):
            if os.path.isdir(directory):
                for filename in os.listdir(directory):
                    path = util.joinPaths(directory, filename)
                    self.tagList.append(tags.TagFile(path, recipe.macros, True))
        self.fullReqs = self.recipe._getTransitiveBuildRequiresNames()
        _addInfo.doProcess(self, recipe)

    def markTag(self, name, tag, path, tagFile=None):
        # commonly, a tagdescription will nominate a file to be
        # tagged, but it will also be set explicitly in the recipe,
        # and therefore markTag will be called twice.
        if (len(tag.split()) > 1 or
            not tag.replace('-', '').replace('_', '').isalnum()):
            # handlers for multiple tags require strict tag names:
            # no whitespace, only alphanumeric plus - and _ characters
            self.error('illegal tag name %s for file %s' %(tag, path))
            return
        tags = self.recipe.autopkg.pathMap[path].tags
        if tag not in tags:
            self.info('%s: %s', name, path)
            tags.set(tag)
            if tagFile and tag not in self.buildReqsComputedForTags:
                self.buildReqsComputedForTags.add(tag)
                db = self._getDb()
                for trove in db.iterTrovesByPath(tagFile.tagFile):
                    troveName = trove.getName()
                    if troveName not in self.fullReqs:
                        # XXX should be error, change after bootstrap
                        self.warn("%s assigned by %s to file %s, so add '%s'"
                                   ' to buildRequires or call r.TagSpec()'
                                   %(tag, tagFile.tagFile, path, troveName))
                        self.suggestBuildRequires.add(troveName)

    def runInfo(self, path):
        if self.recipe._getCapsulePathsForFile(path):
            # capsules do not participate in the tag protocol
            return
        excludedTags = {}
        for tag in self.included:
            for filt in self.included[tag]:
                if filt.match(path):
                    isExcluded = False
                    if tag in self.excluded:
                        for filt in self.excluded[tag]:
                            if filt.match(path):
                                s = excludedTags.setdefault(tag, set())
                                s.add(path)
                                isExcluded = True
                                break
                    if not isExcluded:
                        self.markTag(tag, tag, path)

        for tag in self.tagList:
            if tag.match(path):
                if tag.name:
                    name = tag.name
                else:
                    name = tag.tag
                isExcluded = False
                if tag.tag in self.excluded:
                    for filt in self.excluded[tag.tag]:
                        # exception handling is per-tag, so handled specially
                        if filt.match(path):
                            s = excludedTags.setdefault(name, set())
                            s.add(path)
                            isExcluded = True
                            break
                if not isExcluded:
                    self.markTag(name, tag.tag, path, tag)
        if excludedTags:
            for tag in excludedTags:
                self.info('ignoring tag match for %s: %s',
                          tag, ', '.join(sorted(excludedTags[tag])))

    def postProcess(self):
        if self.suggestBuildRequires:
            self.info('possibly add to buildRequires: %s',
                      str(sorted(list(self.suggestBuildRequires))))
            self.recipe.reportMissingBuildRequires(self.suggestBuildRequires)


class Properties(policy.Policy):
    """
    NAME
    ====
    B{C{r.Properties()}} - Read property definition files

    SYNOPSIS
    ========
    C{r.Properties(I{exceptions=filterexp} || [I{contents=xml},
                   I{package=pkg:component}])}

    DESCRIPTION
    ===========
    The C{r.Properties()} policy automatically parses iconfig property
    definition files, making the properties available for configuration
    management with iconfig.

    To add configuration properties manually, use the syntax:
    C{r.Properties(I{contents=ipropcontents}, I{package=pkg:component}}
    Where contents is the xml string that would normally be stored in the iprop
    file and package is the component where to attach the config metadata.
    (NOTE: This component must exist)
    """
    supported_targets = (TARGET_LINUX, TARGET_WINDOWS)
    bucket = policy.PACKAGE_CREATION
    processUnmodified = True
    invariantinclusions = [ r'%(prefix)s/lib/iconfig/properties/.*\.iprop' ]
    requires = (
        # We need to know what component files have been assigned to
        ('PackageSpec', policy.REQUIRED_PRIOR),
    )

    def __init__(self, *args, **kwargs):
        policy.Policy.__init__(self, *args, **kwargs)

        self.contents = []

    def updateArgs(self, *args, **kwargs):
        if 'contents' in kwargs:
            contents = kwargs.pop('contents')
            pkg = kwargs.pop('package', None)

            self.contents.append((pkg, contents))

        policy.Policy.updateArgs(self, *args, **kwargs)

    def doFile(self, path):
        fullpath = self.recipe.macros.destdir + path
        if not os.path.isfile(fullpath) or not util.isregular(fullpath):
            return

        componentMap = self.recipe.autopkg.componentMap
        if path not in componentMap:
            return
        main, comp = componentMap[path].getName().split(':')

        xml = open(fullpath).read()
        self._parsePropertyData(xml, main, comp)

    def postProcess(self):
        for pkg, content in self.contents:
            pkg = pkg % self.macros
            pkgName, compName = pkg.split(':')
            self._parsePropertyData(content, pkgName, compName)

    def _parsePropertyData(self, xml, pkgName, compName):
        xmldata = smartform.SmartFormFieldParser(xml)

        self.recipe._addProperty(trove._PROPERTY_TYPE_SMARTFORM,
            pkgName, compName, xmldata.name, xml, xmldata.default)


class MakeDevices(policy.Policy):
    """
    NAME
    ====
    B{C{r.MakeDevices()}} - Make device nodes

    SYNOPSIS
    ========
    C{MakeDevices([I{path},] [I{type},] [I{major},] [I{minor},] [I{owner},] [I{groups},] [I{mode}])}

    DESCRIPTION
    ===========
    The C{r.MakeDevices()} policy creates device nodes.  Conary's
    policy of non-root builds requires that these nodes exist only in the
    package, and not in the filesystem, as only root may actually create
    device nodes.


    EXAMPLES
    ========
    C{r.MakeDevices(I{'/dev/tty', 'c', 5, 0, 'root', 'root', mode=0666, package=':dev'})}

    Creates the device node C{/dev/tty}, as type 'c' (character, as opposed to
    type 'b', or block) with a major number of '5', minor number of '0',
    owner, and group are both the root user, and permissions are 0666.
    """
    bucket = policy.PACKAGE_CREATION
    processUnmodified = True
    requires = (
        ('PackageSpec', policy.REQUIRED_PRIOR),
        ('Ownership', policy.REQUIRED_SUBSEQUENT),
    )

    def __init__(self, *args, **keywords):
        self.devices = []
        policy.Policy.__init__(self, *args, **keywords)

    def updateArgs(self, *args, **keywords):
        """
        MakeDevices(path, devtype, major, minor, owner, group, mode=0400)
        """
        if args:
            args = list(args)
            l = len(args)
            if not ((l > 5) and (l < 9)):
                self.recipe.error('MakeDevices: incorrect arguments: %r %r'
                    %(args, keywords))
            mode = keywords.pop('mode', None)
            package = keywords.pop('package', None)
            if l > 6 and mode is None:
                mode = args[6]
            if mode is None:
                mode = 0400
            if l > 7 and package is None:
                package = args[7]
            self.devices.append(
                (args[0:6], {'perms': mode, 'package': package}))
        policy.Policy.updateArgs(self, **keywords)

    def do(self):
        for device, kwargs in self.devices:
            r = self.recipe
            filename = device[0]
            owner = device[4]
            group = device[5]
            r.Ownership(owner, group, filename)
            device[0] = device[0] % r.macros
            r.autopkg.addDevice(*device, **kwargs)


class setModes(policy.Policy):
    """
    Do not call from recipes; this is used internally by C{r.SetModes},
    C{r.ParseManifest}, and unpacking derived packages.  This policy
    modified modes relative to the mode on the file in the filesystem.
    It adds setuid/setgid bits not otherwise set/honored on files on the
    filesystem, and sets user r/w/x bits if they were altered for the
    purposes of accessing the files during packaging.  Otherwise,
    it honors the bits found on the filesystem.  It does not modify
    bits in capsules.
    """
    bucket = policy.PACKAGE_CREATION
    processUnmodified = True
    requires = (
        ('PackageSpec', policy.REQUIRED_PRIOR),
        ('WarnWriteable', policy.REQUIRED_SUBSEQUENT),
        ('ExcludeDirectories', policy.CONDITIONAL_SUBSEQUENT),
    )
    def __init__(self, *args, **keywords):
        self.sidbits = {}
        self.userbits = {}
        policy.Policy.__init__(self, *args, **keywords)

    def updateArgs(self, *args, **keywords):
        """
        setModes(path(s), [sidbits=int], [userbits=int])
        """
        sidbits = keywords.pop('sidbits', None)
        userbits = keywords.pop('userbits', None)
        for path in args:
            if sidbits is not None:
                self.sidbits[path] = sidbits
            if userbits is not None:
                self.userbits[path] = userbits
                self.recipe.WarnWriteable(
                    exceptions=re.escape(path).replace('%', '%%'),
                    allowUnusedFilters = True)
        policy.Policy.updateArgs(self, **keywords)

    def doFile(self, path):
        # Don't set modes on capsule files
        if self.recipe._getCapsulePathsForFile(path):
            return
        # Skip files that aren't part of the package
        if path not in self.recipe.autopkg.pathMap:
            return
        newmode = oldmode = self.recipe.autopkg.pathMap[path].inode.perms()
        if path in self.userbits:
            newmode = (newmode & 077077) | self.userbits[path]
        if path in self.sidbits and self.sidbits[path]:
            newmode |= self.sidbits[path]
            self.info('suid/sgid: %s mode 0%o', path, newmode & 07777)
        if newmode != oldmode:
            self.recipe.autopkg.pathMap[path].inode.perms.set(newmode)


class LinkType(policy.Policy):
    """
    NAME
    ====
    B{C{r.LinkType()}} - Ensures only regular, non-configuration files are hardlinked

    SYNOPSIS
    ========
    C{r.LinkType([I{filterexp}])}

    DESCRIPTION
    ===========
    The C{r.LinkType()} policy ensures that only regular, non-configuration
    files are hardlinked.


    EXAMPLES
    ========
    This policy is not called explicitly.
    """
    bucket = policy.PACKAGE_CREATION
    processUnmodified = True
    requires = (
        ('Config', policy.REQUIRED_PRIOR),
        ('PackageSpec', policy.REQUIRED_PRIOR),
    )
    def do(self):
        for component in self.recipe.autopkg.getComponents():
            for path in sorted(component.hardlinkMap.keys()):
                if self.recipe.autopkg.pathMap[path].flags.isConfig():
                    self.error("Config file %s has illegal hard links", path)
            for path in component.badhardlinks:
                self.error("Special file %s has illegal hard links", path)


class LinkCount(policy.Policy):
    """
    NAME
    ====
    B{C{r.LinkCount()}} - Restricts hardlinks across directories.

    SYNOPSIS
    ========
    C{LinkCount([I{filterexp}] | [I{exceptions=filterexp}])}

    DESCRIPTION
    ===========
    The C{r.LinkCount()} policy restricts hardlinks across directories.

    It is generally an error to have hardlinks across directories, except when
    the packager knows that there is no reasonable chance that they will be on
    separate filesystems.

    In cases where the packager is certain hardlinks will not cross
    filesystems, a list of regular expressions specifying files
    which are excepted from this rule may be passed to C{r.LinkCount}.

    EXAMPLES
    ========
    C{r.LinkCount(exceptions='/usr/share/zoneinfo/')}

    Uses C{r.LinkCount} to except zoneinfo files, located in
    C{/usr/share/zoneinfo/}, from the policy against cross-directory
    hardlinks.
    """
    bucket = policy.PACKAGE_CREATION
    processUnmodified = False
    requires = (
        ('PackageSpec', policy.REQUIRED_PRIOR),
    )
    def __init__(self, *args, **keywords):
        policy.Policy.__init__(self, *args, **keywords)
        self.excepts = set()

    def updateArgs(self, *args, **keywords):
        allowUnusedFilters = keywords.pop('allowUnusedFilters', False) or \
                self.allowUnusedFilters
        exceptions = keywords.pop('exceptions', None)
        if exceptions:
            if type(exceptions) is str:
                self.excepts.add(exceptions)
                if not allowUnusedFilters:
                    self.unusedFilters['exceptions'].add(exceptions)
            elif type(exceptions) in (tuple, list):
                self.excepts.update(exceptions)
                if not allowUnusedFilters:
                    self.unusedFilters['exceptions'].update(exceptions)
        # FIXME: we may want to have another keyword argument
        # that passes information down to the buildpackage
        # that causes link groups to be broken for some
        # directories but not others.  We need to research
        # first whether this is useful; it may not be.

    def do(self):
        if self.recipe.getType() == recipe.RECIPE_TYPE_CAPSULE:
            return
        filters = [(x, filter.Filter(x, self.macros)) for x in self.excepts]
        for component in self.recipe.autopkg.getComponents():
            for inode in component.linkGroups:
                # ensure all in same directory, except for directories
                # matching regexps that have been passed in

                allPaths = [x for x in component.linkGroups[inode]]
                for path in allPaths[:]:
                    for regexp, f in filters:
                        if f.match(path):
                            self.unusedFilters['exceptions'].discard(regexp)
                            allPaths.remove(path)
                dirSet = set(os.path.dirname(x) + '/' for x in allPaths)

                if len(dirSet) > 1:
                    self.error('files %s are hard links across directories %s',
                               ', '.join(sorted(component.linkGroups[inode])),
                               ', '.join(sorted(list(dirSet))))
                    self.error('If these directories cannot reasonably be'
                               ' on different filesystems, disable this'
                               ' warning by calling'
                               " r.LinkCount(exceptions=('%s')) or"
                               " equivalent"
                               % "', '".join(sorted(list(dirSet))))


class ExcludeDirectories(policy.Policy):
    """
    NAME
    ====
    B{C{r.ExcludeDirectories()}} - Exclude directories from package

    SYNOPSIS
    ========
    C{r.ExcludeDirectories([I{filterexp}] | [I{exceptions=filterexp}])}

    DESCRIPTION
    ===========
    The C{r.ExcludeDirectories} policy causes directories to be
    excluded from the package by default.  Use
    C{r.ExcludeDirectories(exceptions=I{filterexp})} to set exceptions to this
    policy, which will cause directories matching the regular expression
    C{filterexp} to be included in the package.  Remember that Conary
    packages cannot share files, including directories, so only one
    package installed on a system at any one time can own the same
    directory.

    There are only three reasons to explicitly package a directory: the
    directory needs permissions other than 0755, it needs non-root owner
    or group, or it must exist even if it is empty.

    Therefore, it should generally not be necessary to invoke this policy
    directly.  If your directory requires permissions other than 0755, simply
    use C{r.SetMode} to specify the permissions, and the directory will be
    automatically included.  Similarly, if you wish to include an empty
    directory with owner or group information, call C{r.Ownership} on that
    empty directory,

    Because C{r.Ownership} can reasonably be called on an entire
    subdirectory tree and indiscriminately applied to files and
    directories alike, non-empty directories with owner or group
    set will be excluded from packaging unless an exception is
    explicitly provided.

    If you call C{r.Ownership} with a filter that applies to an
    empty directory, but you do not want to package that directory,
    you will have to remove the directory with C{r.Remove}.

    Packages do not need to explicitly include directories to ensure
    existence of a target to place a file in. Conary will appropriately
    create the directory, and delete it later if the directory becomes empty.

    EXAMPLES
    ========
    C{r.ExcludeDirectories(exceptions='/tftpboot')}

    Sets the directory C{/tftboot} as an exception to the
    C{r.ExcludeDirectories} policy, so that the C{/tftpboot}
    directory will be included in the package.
    """
    bucket = policy.PACKAGE_CREATION
    processUnmodified = True
    requires = (
        ('PackageSpec', policy.REQUIRED_PRIOR),
        ('Ownership', policy.REQUIRED_PRIOR),
        ('MakeDevices', policy.CONDITIONAL_PRIOR),
    )
    invariantinclusions = [ ('.*', stat.S_IFDIR) ]
    supported_targets …