/components/openstack/nova/files/solariszones/driver.py

Large files files are truncated, but you can click here to view the full file

# Copyright 2011 Justin Santa Barbara
# All Rights Reserved.
#
# Copyright (c) 2013, 2016, Oracle and/or its affiliates. All rights reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Driver for Solaris Zones (nee Containers):
"""

import base64
import glob
import os
import platform
import shutil
import tempfile
import uuid

from collections import defaultdict
from openstack_common import get_ovsdb_info
import rad.bindings.com.oracle.solaris.rad.archivemgr_1 as archivemgr
import rad.bindings.com.oracle.solaris.rad.kstat_2 as kstat
import rad.bindings.com.oracle.solaris.rad.zonemgr_1 as zonemgr
import rad.client
import rad.connect
from solaris_install.target.size import Size

from cinderclient import exceptions as cinder_exception
from cinderclient.v1 import client as v1_client
from eventlet import greenthread
from keystoneclient import exceptions as keystone_exception
from lxml import etree
from oslo_concurrency import lockutils, processutils
from oslo_config import cfg
from oslo_log import log as logging
from oslo_serialization import jsonutils
from oslo_utils import excutils
from oslo_utils import fileutils
from oslo_utils import strutils
from oslo_utils import versionutils
from passlib.hash import sha256_crypt

from nova.api.metadata import password
from nova.compute import arch
from nova.compute import hv_type
from nova.compute import power_state
from nova.compute import task_states
from nova.compute import vm_mode
from nova.compute import vm_states
from nova import conductor
import nova.conf
from nova.console import type as ctype
from nova import context as nova_context
from nova import crypto
from nova import exception
from nova.i18n import _, _LE, _LI
from nova.image import API as glance_api
from nova.image import glance
from nova.network.neutronv2 import api as neutronv2_api
from nova import objects
from nova.objects import flavor as flavor_obj
from nova.objects import migrate_data as migrate_data_obj
from nova import utils
from nova.virt import driver
from nova.virt import event as virtevent
from nova.virt import hardware
from nova.virt import images
from nova.virt.solariszones import sysconfig
from nova.volume.cinder import API
from nova.volume.cinder import cinderclient
from nova.volume.cinder import translate_volume_exception
from nova.volume.cinder import _untranslate_volume_summary_view

solariszones_opts = [
    cfg.StrOpt('boot_volume_type',
               default=None,
               help='Cinder volume type to use for boot volumes'),
    cfg.StrOpt('boot_volume_az',
               default=None,
               help='Cinder availability zone to use for boot volumes'),
    cfg.StrOpt('glancecache_dirname',
               default='/var/share/nova/images',
               help='Default path to Glance cache for Solaris Zones.'),
    cfg.StrOpt('live_migration_cipher',
               help='Cipher to use for encryption of memory traffic during '
                    'live migration. If not specified, a common encryption '
                    'algorithm will be negotiated. Options include: none or '
                    'the name of a supported OpenSSL cipher algorithm.'),
    cfg.StrOpt('solariszones_snapshots_directory',
               default='$instances_path/snapshots',
               help='Location to store snapshots before uploading them to the '
                    'Glance image service.'),
    cfg.StrOpt('zones_suspend_path',
               default='/var/share/zones/SYSsuspend',
               help='Default path for suspend images for Solaris Zones.'),
    cfg.BoolOpt('solariszones_boot_options',
                default=True,
                help='Allow kernel boot options to be set in instance '
                     'metadata.'),
]

CONF = nova.conf.CONF
CONF.register_opts(solariszones_opts, 'solariszones')
LOG = logging.getLogger(__name__)

# These should match the strings returned by the zone_state_str()
# function in the (private) libzonecfg library. These values are in turn
# returned in the 'state' string of the Solaris Zones' RAD interface by
# the zonemgr(3RAD) provider.
ZONE_STATE_CONFIGURED = 'configured'
ZONE_STATE_INCOMPLETE = 'incomplete'
ZONE_STATE_UNAVAILABLE = 'unavailable'
ZONE_STATE_INSTALLED = 'installed'
ZONE_STATE_READY = 'ready'
ZONE_STATE_RUNNING = 'running'
ZONE_STATE_SHUTTING_DOWN = 'shutting_down'
ZONE_STATE_DOWN = 'down'
ZONE_STATE_MOUNTED = 'mounted'

# Mapping between zone state and Nova power_state.
SOLARISZONES_POWER_STATE = {
    ZONE_STATE_CONFIGURED:      power_state.NOSTATE,
    ZONE_STATE_INCOMPLETE:      power_state.NOSTATE,
    ZONE_STATE_UNAVAILABLE:     power_state.NOSTATE,
    ZONE_STATE_INSTALLED:       power_state.SHUTDOWN,
    ZONE_STATE_READY:           power_state.RUNNING,
    ZONE_STATE_RUNNING:         power_state.RUNNING,
    ZONE_STATE_SHUTTING_DOWN:   power_state.RUNNING,
    ZONE_STATE_DOWN:            power_state.RUNNING,
    ZONE_STATE_MOUNTED:         power_state.NOSTATE
}

# Solaris Zones brands as defined in brands(5).
ZONE_BRAND_LABELED = 'labeled'
ZONE_BRAND_SOLARIS = 'solaris'
ZONE_BRAND_SOLARIS_KZ = 'solaris-kz'
ZONE_BRAND_SOLARIS10 = 'solaris10'

# Mapping between supported zone brands and the name of the corresponding
# brand template.
ZONE_BRAND_TEMPLATE = {
    ZONE_BRAND_SOLARIS:         'SYSdefault',
    ZONE_BRAND_SOLARIS_KZ:      'SYSsolaris-kz',
}

MAX_CONSOLE_BYTES = 102400

VNC_CONSOLE_BASE_FMRI = 'svc:/application/openstack/nova/zone-vnc-console'
# Required in order to create a zone VNC console SMF service instance
VNC_SERVER_PATH = '/usr/bin/vncserver'
XTERM_PATH = '/usr/bin/xterm'

ROOTZPOOL_RESOURCE = 'rootzpool'

# The underlying Solaris Zones framework does not expose a specific
# version number, instead relying on feature tests to identify what is
# and what is not supported. A HYPERVISOR_VERSION is defined here for
# Nova's use but it generally should not be changed unless there is a
# incompatible change such as concerning kernel zone live migration.
HYPERVISOR_VERSION = '5.11'

shared_storage = ['iscsi', 'fibre_channel']

KSTAT_TYPE = {
    'NVVT_STR': 'string',
    'NVVT_STRS': 'strings',
    'NVVT_INT': 'integer',
    'NVVT_INTS': 'integers',
    'NVVT_KSTAT': 'kstat',
}


def lookup_resource(zone, resource):
    """Lookup specified resource from specified Solaris Zone."""
    try:
        val = zone.getResources(zonemgr.Resource(resource))
    except rad.client.ObjectError:
        return None
    except Exception:
        raise
    return val[0] if val else None


def lookup_resource_property(zone, resource, prop, filter=None):
    """Lookup specified property from specified Solaris Zone resource."""
    try:
        val = zone.getResourceProperties(zonemgr.Resource(resource, filter),
                                         [prop])
    except rad.client.ObjectError:
        return None
    except Exception:
        raise
    return val[0].value if val else None


def lookup_resource_property_value(zone, resource, prop, value):
    """Lookup specified property with value from specified Solaris Zone
    resource. Returns resource object if matching value is found, else None
    """
    try:
        resources = zone.getResources(zonemgr.Resource(resource))
        for resource in resources:
            for propertee in resource.properties:
                if propertee.name == prop and propertee.value == value:
                    return resource
        else:
            return None
    except rad.client.ObjectError:
        return None
    except Exception:
        raise


def zonemgr_strerror(ex):
    """Format the payload from a zonemgr(3RAD) rad.client.ObjectError
    exception into a sensible error string that can be logged. Newlines
    are converted to a colon-space string to create a single line.

    If the exception was something other than rad.client.ObjectError,
    just return it as a string.
    """
    if not isinstance(ex, rad.client.ObjectError):
        return str(ex)
    payload = ex.get_payload()
    if payload.code == zonemgr.ErrorCode.NONE:
        return str(ex)
    error = [str(payload.code)]
    if payload.str is not None and payload.str != '':
        error.append(payload.str)
    if payload.stderr is not None and payload.stderr != '':
        stderr = payload.stderr.rstrip()
        error.append(stderr.replace('\n', ': '))
    result = ': '.join(error)
    return result


class MemoryAlignmentIncorrect(exception.FlavorMemoryTooSmall):
    msg_fmt = _("Requested flavor, %(flavor)s, memory size %(memsize)s does "
                "not align on %(align)s boundary.")


class SolarisVolumeAPI(API):
    """ Extending the volume api to support additional cinder sub-commands
    """
    @translate_volume_exception
    def create(self, context, size, name, description, snapshot=None,
               image_id=None, volume_type=None, metadata=None,
               availability_zone=None, source_volume=None):
        """Clone the source volume by calling the cinderclient version of
        create with a source_volid argument

        :param context: the context for the clone
        :param size: size of the new volume, must be the same as the source
            volume
        :param name: display_name of the new volume
        :param description: display_description of the new volume
        :param snapshot: Snapshot object
        :param image_id: image_id to create the volume from
        :param volume_type: type of volume
        :param metadata: Additional metadata for the volume
        :param availability_zone: zone:host where the volume is to be created
        :param source_volume: Volume object

        Returns a volume object
        """
        client = cinderclient(context)

        if snapshot is not None:
            snapshot_id = snapshot['id']
        else:
            snapshot_id = None

        if source_volume is not None:
            source_volid = source_volume['id']
        else:
            source_volid = None

        kwargs = dict(snapshot_id=snapshot_id,
                      volume_type=volume_type,
                      user_id=context.user_id,
                      project_id=context.project_id,
                      availability_zone=availability_zone,
                      metadata=metadata,
                      imageRef=image_id,
                      source_volid=source_volid)

        if isinstance(client, v1_client.Client):
            kwargs['display_name'] = name
            kwargs['display_description'] = description
        else:
            kwargs['name'] = name
            kwargs['description'] = description

        try:
            item = cinderclient(context).volumes.create(size, **kwargs)
            return _untranslate_volume_summary_view(context, item)
        except cinder_exception.OverLimit:
            raise exception.OverQuota(overs='volumes')
        except (cinder_exception.BadRequest,
                keystone_exception.BadRequest) as reason:
            raise exception.InvalidInput(reason=reason)

    @translate_volume_exception
    def update(self, context, volume_id, fields):
        """Update the fields of a volume for example used to rename a volume
        via a call to cinderclient

        :param context: the context for the update
        :param volume_id: the id of the volume to update
        :param fields: a dictionary of of the name/value pairs to update
        """
        cinderclient(context).volumes.update(volume_id, **fields)

    @translate_volume_exception
    def extend(self, context, volume, newsize):
        """Extend the size of a cinder volume by calling the cinderclient

        :param context: the context for the extend
        :param volume: the volume object to extend
        :param newsize: the new size of the volume in GB
        """
        cinderclient(context).volumes.extend(volume, newsize)


class ZoneConfig(object):
    """ZoneConfig - context manager for access zone configurations.
    Automatically opens the configuration for a zone and commits any changes
    before exiting
    """
    def __init__(self, zone):
        """zone is a zonemgr object representing either a kernel zone or
        non-global zone.
        """
        self.zone = zone
        self.editing = False

    def __enter__(self):
        """enables the editing of the zone."""
        try:
            self.zone.editConfig()
            self.editing = True
            return self
        except Exception as ex:
            reason = zonemgr_strerror(ex)
            LOG.exception(_("Unable to initialize editing of instance '%s' "
                            "via zonemgr(3RAD): %s")
                          % (self.zone.name, reason))
            raise

    def __exit__(self, exc_type, exc_val, exc_tb):
        """looks for any kind of exception before exiting.  If one is found,
        cancel any configuration changes and reraise the exception.  If not,
        commit the new configuration.
        """
        if exc_type is not None and self.editing:
            # We received some kind of exception.  Cancel the config and raise.
            self.zone.cancelConfig()
            raise
        else:
            # commit the config
            try:
                self.zone.commitConfig()
            except Exception as ex:
                reason = zonemgr_strerror(ex)
                LOG.exception(_("Unable to commit the new configuration for "
                                "instance '%s' via zonemgr(3RAD): %s")
                              % (self.zone.name, reason))

                # Last ditch effort to cleanup.
                self.zone.cancelConfig()
                raise

    def setprop(self, resource, prop, value):
        """sets a property for an existing resource OR creates a new resource
        with the given property(s).
        """
        current = lookup_resource_property(self.zone, resource, prop)
        if current is not None and current == value:
            # the value is already set
            return

        try:
            if current is None:
                self.zone.addResource(zonemgr.Resource(
                    resource, [zonemgr.Property(prop, value)]))
            else:
                self.zone.setResourceProperties(
                    zonemgr.Resource(resource),
                    [zonemgr.Property(prop, value)])
        except Exception as ex:
            reason = zonemgr_strerror(ex)
            LOG.exception(_("Unable to set '%s' property on '%s' resource for "
                            "instance '%s' via zonemgr(3RAD): %s")
                          % (prop, resource, self.zone.name, reason))
            raise

    def addresource(self, resource, props=None, ignore_exists=False):
        """creates a new resource with an optional property list, or set the
        property if the resource exists and ignore_exists is true.

        :param ignore_exists: If the resource exists, set the property for the
            resource.
        """
        if props is None:
            props = []

        try:
            self.zone.addResource(zonemgr.Resource(resource, props))
        except Exception as ex:
            if isinstance(ex, rad.client.ObjectError):
                code = ex.get_payload().code
                if (ignore_exists and
                        code == zonemgr.ErrorCode.RESOURCE_ALREADY_EXISTS):
                    self.zone.setResourceProperties(
                        zonemgr.Resource(resource, None), props)
                    return
            reason = zonemgr_strerror(ex)
            LOG.exception(_("Unable to create new resource '%s' for instance "
                            "'%s' via zonemgr(3RAD): %s")
                          % (resource, self.zone.name, reason))
            raise

    def removeresources(self, resource, props=None):
        """removes resources whose properties include the optional property
        list specified in props.
        """
        if props is None:
            props = []

        try:
            self.zone.removeResources(zonemgr.Resource(resource, props))
        except Exception as ex:
            reason = zonemgr_strerror(ex)
            LOG.exception(_("Unable to remove resource '%s' for instance '%s' "
                            "via zonemgr(3RAD): %s")
                          % (resource, self.zone.name, reason))
            raise

    def clear_resource_props(self, resource, props):
        """Clear property values of a given resource
        """
        try:
            self.zone.clearResourceProperties(zonemgr.Resource(resource, None),
                                              props)
        except rad.client.ObjectError as ex:
            reason = zonemgr_strerror(ex)
            LOG.exception(_("Unable to clear '%s' property on '%s' resource "
                            "for instance '%s' via zonemgr(3RAD): %s")
                          % (props, resource, self.zone.name, reason))
            raise


class SolarisZonesDriver(driver.ComputeDriver):
    """Solaris Zones Driver using the zonemgr(3RAD) and kstat(3RAD) providers.

    The interface to this class talks in terms of 'instances' (Amazon EC2 and
    internal Nova terminology), by which we mean 'running virtual machine'
    (XenAPI terminology) or domain (Xen or libvirt terminology).

    An instance has an ID, which is the identifier chosen by Nova to represent
    the instance further up the stack.  This is unfortunately also called a
    'name' elsewhere.  As far as this layer is concerned, 'instance ID' and
    'instance name' are synonyms.

    Note that the instance ID or name is not human-readable or
    customer-controlled -- it's an internal ID chosen by Nova.  At the
    nova.virt layer, instances do not have human-readable names at all -- such
    things are only known higher up the stack.

    Most virtualization platforms will also have their own identity schemes,
    to uniquely identify a VM or domain.  These IDs must stay internal to the
    platform-specific layer, and never escape the connection interface.  The
    platform-specific layer is responsible for keeping track of which instance
    ID maps to which platform-specific ID, and vice versa.

    Some methods here take an instance of nova.compute.service.Instance.  This
    is the data structure used by nova.compute to store details regarding an
    instance, and pass them into this layer.  This layer is responsible for
    translating that generic data structure into terms that are specific to the
    virtualization platform.

    """

    capabilities = {
        "has_imagecache": False,
        "supports_recreate": True,
        "supports_migrate_to_same_host": False
    }

    def __init__(self, virtapi):
        self.virtapi = virtapi
        self._archive_manager = None
        self._compute_event_callback = None
        self._conductor_api = conductor.API()
        self._fc_hbas = None
        self._fc_wwnns = None
        self._fc_wwpns = None
        self._host_stats = {}
        self._initiator = None
        self._install_engine = None
        self._kstat_control = None
        self._pagesize = os.sysconf('SC_PAGESIZE')
        self._rad_connection = None
        self._rootzpool_suffix = ROOTZPOOL_RESOURCE
        self._uname = os.uname()
        self._validated_archives = list()
        self._volume_api = SolarisVolumeAPI()
        self._zone_manager = None

    @property
    def rad_connection(self):
        if self._rad_connection is None:
            self._rad_connection = rad.connect.connect_unix()
        else:
            # taken from rad.connect.RadConnection.__repr__ to look for a
            # closed connection
            if self._rad_connection._closed is not None:
                # the RAD connection has been lost.  Reconnect to RAD
                self._rad_connection = rad.connect.connect_unix()

        return self._rad_connection

    @property
    def zone_manager(self):
        try:
            if (self._zone_manager is None or
                    self._zone_manager._conn._closed is not None):
                self._zone_manager = self.rad_connection.get_object(
                    zonemgr.ZoneManager())
        except Exception as ex:
            reason = _("Unable to obtain RAD object: %s") % ex
            raise exception.NovaException(reason)

        return self._zone_manager

    @property
    def kstat_control(self):
        try:
            if (self._kstat_control is None or
                    self._kstat_control._conn._closed is not None):
                self._kstat_control = self.rad_connection.get_object(
                    kstat.Control())
        except Exception as ex:
            reason = _("Unable to obtain RAD object: %s") % ex
            raise exception.NovaException(reason)

        return self._kstat_control

    @property
    def archive_manager(self):
        try:
            if (self._archive_manager is None or
                    self._archive_manager._conn._closed is not None):
                self._archive_manager = self.rad_connection.get_object(
                    archivemgr.ArchiveManager())
        except Exception as ex:
            reason = _("Unable to obtain RAD object: %s") % ex
            raise exception.NovaException(reason)

        return self._archive_manager

    def init_host(self, host):
        """Initialize anything that is necessary for the driver to function,
        including catching up with currently running VM's on the given host.
        """
        # TODO(Vek): Need to pass context in for access to auth_token
        pass

    def cleanup_host(self, host):
        """Clean up anything that is necessary for the driver gracefully stop,
        including ending remote sessions. This is optional.
        """
        pass

    def _get_fc_hbas(self):
        """Get Fibre Channel HBA information."""
        if self._fc_hbas:
            return self._fc_hbas

        out = None
        try:
            out, err = utils.execute('/usr/sbin/fcinfo', 'hba-port')
        except processutils.ProcessExecutionError:
            return []

        if out is None:
            raise RuntimeError(_("Cannot find any Fibre Channel HBAs"))

        hbas = []
        hba = {}
        for line in out.splitlines():
            line = line.strip()
            # Collect the following hba-port data:
            # 1: Port WWN
            # 2: State (online|offline)
            # 3: Node WWN
            if line.startswith("HBA Port WWN:"):
                # New HBA port entry
                hba = {}
                wwpn = line.split()[-1]
                hba['port_name'] = wwpn
                continue
            elif line.startswith("Port Mode:"):
                mode = line.split()[-1]
                # Skip Target mode ports
                if mode != 'Initiator':
                    break
            elif line.startswith("State:"):
                state = line.split()[-1]
                hba['port_state'] = state
                continue
            elif line.startswith("Node WWN:"):
                wwnn = line.split()[-1]
                hba['node_name'] = wwnn
                continue
            if len(hba) == 3:
                hbas.append(hba)
                hba = {}
        self._fc_hbas = hbas
        return self._fc_hbas

    def _get_fc_wwnns(self):
        """Get Fibre Channel WWNNs from the system, if any."""
        hbas = self._get_fc_hbas()

        wwnns = []
        for hba in hbas:
            if hba['port_state'] == 'online':
                wwnn = hba['node_name']
                wwnns.append(wwnn)
        return wwnns

    def _get_fc_wwpns(self):
        """Get Fibre Channel WWPNs from the system, if any."""
        hbas = self._get_fc_hbas()

        wwpns = []
        for hba in hbas:
            if hba['port_state'] == 'online':
                wwpn = hba['port_name']
                wwpns.append(wwpn)
        return wwpns

    def _get_iscsi_initiator(self):
        """ Return the iSCSI initiator node name IQN for this host """
        try:
            out, err = utils.execute('/usr/sbin/iscsiadm', 'list',
                                     'initiator-node')
            # Sample first line of command output:
            # Initiator node name: iqn.1986-03.com.sun:01:e00000000000.4f757217
            initiator_name_line = out.splitlines()[0]
            initiator_iqn = initiator_name_line.rsplit(' ', 1)[1]
            return initiator_iqn
        except processutils.ProcessExecutionError as ex:
            LOG.info(_("Failed to get the initiator-node info: %s") % (ex))
            return None

    def _get_zone_by_name(self, name):
        """Return a Solaris Zones object via RAD by name."""
        try:
            zone = self.rad_connection.get_object(
                zonemgr.Zone(), rad.client.ADRGlobPattern({'name': name}))
        except rad.client.NotFoundError:
            return None
        except Exception:
            raise
        return zone

    def _get_state(self, zone):
        """Return the running state, one of the power_state codes."""
        return SOLARISZONES_POWER_STATE[zone.state]

    def _pages_to_kb(self, pages):
        """Convert a number of pages of memory into a total size in KBytes."""
        return (pages * self._pagesize) / 1024

    def _get_max_mem(self, zone):
        """Return the maximum memory in KBytes allowed."""
        if zone.brand == ZONE_BRAND_SOLARIS:
            mem_resource = 'swap'
        else:
            mem_resource = 'physical'

        max_mem = lookup_resource_property(zone, 'capped-memory', mem_resource)
        if max_mem is not None:
            return strutils.string_to_bytes("%sB" % max_mem) / 1024

        # If physical property in capped-memory doesn't exist, this may
        # represent a non-global zone so just return the system's total
        # memory.
        return self._pages_to_kb(os.sysconf('SC_PHYS_PAGES'))

    def _get_mem(self, zone):
        """Return the memory in KBytes used by the domain."""

        # There isn't any way of determining this from the hypervisor
        # perspective in Solaris, so just return the _get_max_mem() value
        # for now.
        return self._get_max_mem(zone)

    def _get_num_cpu(self, zone):
        """Return the number of virtual CPUs for the domain.

        In the case of kernel zones, the number of virtual CPUs a zone
        ends up with depends on whether or not there were 'virtual-cpu'
        or 'dedicated-cpu' resources in the configuration or whether
        there was an assigned pool in the configuration. This algorithm
        attempts to emulate what the virtual platform code does to
        determine a number of virtual CPUs to use.
        """
        # If a 'virtual-cpu' resource exists, use the minimum number of
        # CPUs defined there.
        ncpus = lookup_resource_property(zone, 'virtual-cpu', 'ncpus')
        if ncpus is not None:
            min = ncpus.split('-', 1)[0]
            if min.isdigit():
                return int(min)

        # Otherwise if a 'dedicated-cpu' resource exists, use the maximum
        # number of CPUs defined there.
        ncpus = lookup_resource_property(zone, 'dedicated-cpu', 'ncpus')
        if ncpus is not None:
            max = ncpus.split('-', 1)[-1]
            if max.isdigit():
                return int(max)

        # Finally if neither resource exists but the zone was assigned a
        # pool in the configuration, the number of CPUs would be the size
        # of the processor set. Currently there's no way of easily
        # determining this so use the system's notion of the total number
        # of online CPUs.
        return os.sysconf('SC_NPROCESSORS_ONLN')

    def _kstat_data(self, uri):
        """Return Kstat snapshot data via RAD as a dictionary."""
        if not isinstance(uri, str):
            raise exception.NovaException("kstat URI must be string type: "
                                          "%s is %s" % (uri, type(uri)))

        if not uri.startswith("kstat:/"):
            uri = "kstat:/" + uri

        try:
            self.kstat_control.update()
            kstat_obj = self.rad_connection.get_object(
                kstat.Kstat(), rad.client.ADRGlobPattern({"uri": uri}))

        except Exception as reason:
            LOG.info(_("Unable to retrieve kstat object '%s' via kstat(3RAD): "
                       "%s") % (uri, reason))
            return None

        ks_data = {}
        for name, data in kstat_obj.getMap().items():
            ks_data[name] = getattr(data, KSTAT_TYPE[str(data.type)])

        return ks_data

    def _sum_kstat_statistic(self, kstat_data, statistic):
        total = 0
        for ks in kstat_data.values():
            data = ks.getMap()[statistic]
            value = getattr(data, KSTAT_TYPE[str(data.type)])
            try:
                total += value
            except TypeError:
                LOG.error(_("Unable to aggregate non-summable kstat %s;%s "
                            " of type %s") % (ks.getParent().uri, statistic,
                                              type(value)))
                return None

        return total

    def _get_kstat_statistic(self, ks, statistic):
        if not isinstance(ks, kstat.Kstat):
            reason = (_("Attempted to get a kstat from %s type.") % (type(ks)))
            raise TypeError(reason)

        try:
            data = ks.getMap()[statistic]
            value = getattr(data, KSTAT_TYPE[str(data.type)])
        except TypeError:
            value = None

        return value

    def _get_cpu_time(self, zone):
        """Return the CPU time used in nanoseconds."""
        if zone.id == -1:
            return 0

        # The retry value of 3 was determined by the "we shouldn't hit this
        # often, but if we do it should resolve quickly so try again"+1
        # algorithm.
        for _attempt in range(3):
            total = 0

            accum_uri = "kstat:/zones/cpu/sys_zone_accum/%d" % zone.id
            uri = "kstat:/zones/cpu/sys_zone_%d" % zone.id

            initial = self._kstat_data(accum_uri)
            cpus = self._kstat_data(uri)

            total += self._sum_kstat_statistic(cpus, 'cpu_nsec_kernel_cur')
            total += self._sum_kstat_statistic(cpus, 'cpu_nsec_user_cur')

            final = self._kstat_data(accum_uri)

            if initial['gen_num'] == final['gen_num']:
                total += initial['cpu_nsec_user'] + initial['cpu_nsec_kernel']
                return total

        LOG.error(_("Unable to get accurate cpu usage beacuse cpu list "
                    "keeps changing"))
        return 0

    def get_info(self, instance):
        """Get the current status of an instance, by name (not ID!)

        :param instance: nova.objects.instance.Instance object

        Returns a InstanceInfo object
        """
        # TODO(Vek): Need to pass context in for access to auth_token
        name = instance['name']
        zone = self._get_zone_by_name(name)
        if zone is None:
            raise exception.InstanceNotFound(instance_id=name)
        return hardware.InstanceInfo(state=self._get_state(zone),
                                     max_mem_kb=self._get_max_mem(zone),
                                     mem_kb=self._get_mem(zone),
                                     num_cpu=self._get_num_cpu(zone),
                                     cpu_time_ns=self._get_cpu_time(zone))

    def get_num_instances(self):
        """Return the total number of virtual machines.

        Return the number of virtual machines that the hypervisor knows
        about.

        .. note::

            This implementation works for all drivers, but it is
            not particularly efficient. Maintainers of the virt drivers are
            encouraged to override this method with something more
            efficient.
        """
        return len(self.list_instances())

    def instance_exists(self, instance):
        """Checks existence of an instance on the host.

        :param instance: The instance to lookup

        Returns True if an instance with the supplied ID exists on
        the host, False otherwise.

        .. note::

            This implementation works for all drivers, but it is
            not particularly efficient. Maintainers of the virt drivers are
            encouraged to override this method with something more
            efficient.
        """
        try:
            return instance.uuid in self.list_instance_uuids()
        except NotImplementedError:
            return instance.name in self.list_instances()

    def estimate_instance_overhead(self, instance_info):
        """Estimate the virtualization overhead required to build an instance
        of the given flavor.

        Defaults to zero, drivers should override if per-instance overhead
        calculations are desired.

        :param instance_info: Instance/flavor to calculate overhead for.
        :returns: Dict of estimated overhead values.
        """
        return {'memory_mb': 0}

    def _get_list_zone_object(self):
        """Return a list of all Solaris Zones objects via RAD."""
        return self.rad_connection.list_objects(zonemgr.Zone())

    def list_instances(self):
        """Return the names of all the instances known to the virtualization
        layer, as a list.
        """
        # TODO(Vek): Need to pass context in for access to auth_token
        instances_list = []
        for zone in self._get_list_zone_object():
            instances_list.append(self.rad_connection.get_object(zone).name)
        return instances_list

    def list_instance_uuids(self):
        """Return the UUIDS of all the instances known to the virtualization
        layer, as a list.
        """
        raise NotImplementedError()

    def _rebuild_block_devices(self, context, instance, bdms, recreate):
        root_ci = None
        rootmp = instance['root_device_name']
        for entry in bdms:
            if entry['connection_info'] is None:
                continue

            if entry['device_name'] == rootmp:
                root_ci = jsonutils.loads(entry['connection_info'])
                # Let's make sure this is a well formed connection_info, by
                # checking if it has a serial key that represents the
                # volume_id. If not check to see if the block device has a
                # volume_id, if so then assign this to the root_ci.serial.
                #
                # If we cannot repair the connection_info then simply do not
                # return a root_ci and let the caller decide if they want to
                # fail or not.
                if root_ci.get('serial') is None:
                    if entry.get('volume_id') is not None:
                        root_ci['serial'] = entry['volume_id']
                    else:
                        LOG.debug(_("Unable to determine the volume id for "
                                    "the connection info for the root device "
                                    "for instance '%s'") % instance['name'])
                        root_ci = None

                continue

            if not recreate:
                ci = jsonutils.loads(entry['connection_info'])
                self.detach_volume(ci, instance, entry['device_name'])

        if root_ci is None and recreate:
            msg = (_("Unable to find the root device for instance '%s'.")
                   % instance['name'])
            raise exception.NovaException(msg)

        return root_ci

    def _set_instance_metahostid(self, instance):
        """Attempt to get the hostid from the current configured zone and
        return the hostid.  Otherwise return None, and do not set the hostid in
        the instance
        """
        hostid = instance.system_metadata.get('hostid')
        if hostid is not None:
            return hostid

        zone = self._get_zone_by_name(instance['name'])
        if zone is None:
            return None

        hostid = lookup_resource_property(zone, 'global', 'hostid')
        if hostid:
            instance.system_metadata['hostid'] = hostid

        return hostid

    def rebuild(self, context, instance, image_meta, injected_files,
                admin_password, bdms, detach_block_devices,
                attach_block_devices, network_info=None,
                recreate=False, block_device_info=None,
                preserve_ephemeral=False):
        """Destroy and re-make this instance.

        A 'rebuild' effectively purges all existing data from the system and
        remakes the VM with given 'metadata' and 'personalities'.

        This base class method shuts down the VM, detaches all block devices,
        then spins up the new VM afterwards. It may be overridden by
        hypervisors that need to - e.g. for optimisations, or when the 'VM'
        is actually proxied and needs to be held across the shutdown + spin
        up steps.

        :param context: security context
        :param instance: nova.objects.instance.Instance
                         This function should use the data there to guide
                         the creation of the new instance.
        :param nova.objects.ImageMeta image_meta:
            The metadata of the image of the instance.
        :param injected_files: User files to inject into instance.
        :param admin_password: Administrator password to set in instance.
        :param bdms: block-device-mappings to use for rebuild
        :param detach_block_devices: function to detach block devices. See
            nova.compute.manager.ComputeManager:_rebuild_default_impl for
            usage.
        :param attach_block_devices: function to attach block devices. See
            nova.compute.manager.ComputeManager:_rebuild_default_impl for
            usage.
        :param network_info:
           :py:meth:`~nova.network.manager.NetworkManager.get_instance_nw_info`
        :param recreate: True if the instance is being recreated on a new
            hypervisor - all the cleanup of old state is skipped.
        :param block_device_info: Information about block devices to be
                                  attached to the instance.
        :param preserve_ephemeral: True if the default ephemeral storage
                                   partition must be preserved on rebuild
        """
        if recreate:
            instance.system_metadata['evac_from'] = instance['launched_on']
            instance.save()
            extra_specs = self._get_flavor(instance)['extra_specs'].copy()
            brand = extra_specs.get('zonecfg:brand', ZONE_BRAND_SOLARIS)
            if brand == ZONE_BRAND_SOLARIS:
                msg = (_("'%s' branded zones do not currently support "
                         "evacuation.") % brand)
                raise exception.NovaException(msg)
        else:
            self._power_off(instance, "HALT")

        instance.task_state = task_states.REBUILD_BLOCK_DEVICE_MAPPING
        instance.save(expected_task_state=[task_states.REBUILDING])
        root_ci = self._rebuild_block_devices(context, instance, bdms,
                                              recreate)

        if recreate:
            if root_ci is not None:
                driver_type = root_ci['driver_volume_type']
            else:
                driver_type = 'local'
            if driver_type not in shared_storage:
                msg = (_("Root device is not on shared storage for instance "
                         "'%s'.") % instance['name'])
                raise exception.NovaException(msg)

        if not recreate:
            self.destroy(context, instance, network_info, block_device_info)
            if root_ci is not None:
                self._volume_api.detach(context, root_ci['serial'])
                self._volume_api.delete(context, root_ci['serial'])

                # Go ahead and remove the root bdm from the bdms so that we do
                # not trip up spawn either checking against the use of c1d0 or
                # attempting to re-attach the root device.
                bdms.objects.remove(bdms.root_bdm())
                rootdevname = block_device_info.get('root_device_name')
                if rootdevname is not None:
                    bdi_bdms = block_device_info.get('block_device_mapping')
                    for entry in bdi_bdms:
                        if entry['mount_device'] == rootdevname:
                            bdi_bdms.remove(entry)
                            break

        instance.task_state = task_states.REBUILD_SPAWNING
        instance.save(
            expected_task_state=[task_states.REBUILD_BLOCK_DEVICE_MAPPING])

        # Instead of using a boolean for 'rebuilding' scratch data, use a
        # string because the object will translate it to a string anyways.
        if recreate:
            extra_specs = self._get_flavor(instance)['extra_specs'].copy()

            instance.system_metadata['rebuilding'] = 'false'
            self._create_config(context, instance, network_info, root_ci, None)
            del instance.system_metadata['evac_from']
            instance.save()
        else:
            instance.system_metadata['rebuilding'] = 'true'
            self.spawn(context, instance, image_meta, injected_files,
                       admin_password, network_info, block_device_info)

        del instance.system_metadata['rebuilding']
        name = instance['name']
        zone = self._get_zone_by_name(name)
        if zone is None:
            raise exception.InstanceNotFound(instance_id=name)

        if recreate:
            zone.attach(['-x', 'initialize-hostdata'])

            rootmp = instance['root_device_name']
            for entry in bdms:
                if (entry['connection_info'] is None or
                        rootmp == entry['device_name']):
                    continue

                connection_info = jsonutils.loads(entry['connection_info'])
                mount = entry['device_name']
                self.attach_volume(context, connection_info, instance, mount)

            self._power_on(instance, network_info)

        if admin_password is not None:
            # Because there is no way to make sure a zone is ready upon
            # returning from a boot request. We must give the zone a few
            # seconds to boot before attempting to set the admin password.
            greenthread.sleep(15)
            self.set_admin_password(instance, admin_password)

    def _get_flavor(self, instance):
        """Retrieve the flavor object as specified in the instance object"""
        return flavor_obj.Flavor.get_by_id(
            nova_context.get_admin_context(read_deleted='yes'),
            instance['instance_type_id'])

    def _fetch_image(self, context, instance):
        """Fetch an image using Glance given the instance's image_ref."""
        glancecache_dirname = CONF.solariszones.glancecache_dirname
        fileutils.ensure_tree(glancecache_dirname)
        iref = instance['image_ref']
        image = os.path.join(glancecache_dirname, iref)
        downloading = image + '.downloading'

        with lockutils.lock('glance-image-%s' % iref):
            if os.path.isfile(downloading):
                LOG.debug(_('Cleaning partial download of %s' % iref))
                os.unlink(image)
                os.unlink(downloading)

            elif os.path.exists(image):
                LOG.debug(_("Using existing, cached Glance image: id %s")
                          % iref)
                return image

            LOG.debug(_("Fetching new Glance image: id %s") % iref)
            try:
                # touch the empty .downloading file
                with open(downloading, 'w'):
                    pass
                images.fetch(context, iref, image, instance['user_id'],
                             instance['project_id'])
                os.unlink(downloading)
                return image
            except Exception as reason:
                LOG.exception(_("Unable to fetch Glance image: id %s: %s")
                              % (iref, reason))
                raise

    @lockutils.synchronized('validate_image')
    def _validate_image(self, context, image, instance):
        """Validate a glance image for compatibility with the instance."""
        # Skip if the image was already checked and confirmed as valid.
        if instance['image_ref'] in self._validated_archives:
            return

        try:
            ua = self.archive_manager.getArchive(image)
        except Exception as ex:
            if isinstance(ex, rad.client.ObjectError):
                reason = ex.get_payload().info
            else:
                reason = str(ex)
            raise exception.ImageUnacceptable(image_id=instance['image_ref'],
                                              reason=reason)

        # Validate the image at this point to ensure:
        # - contains one deployable system
        deployables = ua.getArchivedSystems()
        if len(deployables) != 1:
            reason = _("Image must contain only a single deployable system.")
            raise exception.ImageUnacceptable(image_id=instance['image_ref'],
                                              reason=reason)
        # - matching architecture
        deployable_arch = str(ua.isa)
        compute_arch = platform.processor()
        if deployable_arch.lower() != compute_arch:
            reason = (_("Unified Archive architecture '%s' is incompatible "
                      "with this compute host's architecture, '%s'.")
                      % (deployable_arch, compute_arch))

            # For some reason we have gotten the wrong architecture image,
            # which should have been filtered by the scheduler. One reason this
            # could happen is because the images architecture type is
            # incorrectly set. Check for this and report a better reason.
            glanceapi = glance_api()
            image_meta = glanceapi.get(context, instance['image_ref'])
            image_properties = image_meta.get('properties')
            if image_properties.get('architecture') is None:
                reason = reason + (_(" The 'architecture' property is not set "
                                     "on the Glance image."))

            raise exception.ImageUnacceptable(image_id=instance['image_ref'],
                                              reason=reason)
        # - single root pool only
        if not deployables[0].rootOnly:
            reason = _("Image contains more than one ZFS pool.")
            raise exception.ImageUnacceptable(image_id=instance['image_ref'],
                                              reason=reason)
        # - looks like it's OK
        self._validated_archives.append(instance['image_ref'])

    def _validate_flavor(self, instance):
        """Validate the flavor for compatibility with zone brands"""
        flavor = self._get_flavor(instance)
        extra_specs = flavor['extra_specs'].copy()
        brand = extra_specs.get('zonecfg:brand', ZONE_BRAND_SOLARIS)

        if brand == ZONE_BRAND_SOLARIS_KZ:
            # verify the memory is 256mb aligned
            test_size = Size('256MB')
            instance_size = Size('%sMB' % instance['memory_mb'])

            if instance_size.byte_value % test_size.byte_value:
                # non-zero result so it doesn't align
                raise MemoryAlignmentIncorrect(
                    flavor=flavor['name'],
                    memsize=str(instance['memory_mb']),
                    align='256')

    def _suri_from_volume_info(self, connection_info):
        """Returns a suri(5) formatted string based on connection_info.
        Currently supports local ZFS volume, NFS, Fibre Channel and iSCSI
        driver types.
        """
        driver_type = connection_info['driver_volume_type']
        if driver_type not in ['iscsi', 'fibre_channel', 'local', 'nfs']:
            raise exception.VolumeDriverNotFound(driver_type=driver_type)
        if driver_type == 'local':
            suri = 'dev:/dev/zvol/dsk/%s' % connection_info['volume_path']
        elif driver_type == 'iscsi':
            data = connection_info['data']
            # suri(5) format:
            #       iscsi://<host>[:<port>]/target.<IQN>,lun.<LUN>
            # luname-only URI format for the multipathing:
            #       iscsi://<host>[:<port>]/luname.naa.<ID>
            # Sample iSCSI connection data values:
            # target_portal: 192.168.1.244:3260
            # target_iqn: iqn.2010-10.org.openstack:volume-a89c.....
            # target_lun: 1
            suri = None
            if 'target_iqns' in data:
                target = data['target_iqns'][0]
                target_lun = data['target_luns'][0]
                try:
                    utils.execute('/usr/sbin/iscsiadm', 'list', 'target',
                                  '-vS', target)
                    out, err = utils.execute('/usr/sbin/suriadm', 'lookup-uri',
                                             '-t', 'iscsi',
                                             '-p', 'target=%s' % target,
                                             '-p', 'lun=%s' % target_lun)
                    for line in [l.strip() for l in out.splitlines()]:
                        if "luname.naa." in line:
                            LOG.debug(_("The found luname-only URI for the "
                         …