/client/virt/virt_utils.py
Python | 3718 lines | 3566 code | 57 blank | 95 comment | 44 complexity | 6029af17aad132999256e10d97d0bf41 MD5 | raw file
Possible License(s): LGPL-3.0, GPL-2.0
Large files files are truncated, but you can click here to view the full file
- """
- KVM test utility functions.
- @copyright: 2008-2009 Red Hat Inc.
- """
- import time, string, random, socket, os, signal, re, logging, commands, cPickle
- import fcntl, shelve, ConfigParser, threading, sys, UserDict, inspect, tarfile
- import struct, shutil, glob
- from autotest_lib.client.bin import utils, os_dep
- from autotest_lib.client.common_lib import error, logging_config
- from autotest_lib.client.common_lib import logging_manager, git
- import rss_client, aexpect
- import platform
- try:
- import koji
- KOJI_INSTALLED = True
- except ImportError:
- KOJI_INSTALLED = False
- ARCH = platform.machine()
- if ARCH == "ppc64":
- # From include/linux/sockios.h
- SIOCSIFHWADDR = 0x8924
- SIOCGIFHWADDR = 0x8927
- SIOCSIFFLAGS = 0x8914
- SIOCGIFINDEX = 0x8933
- SIOCBRADDIF = 0x89a2
- # From linux/include/linux/if_tun.h
- TUNSETIFF = 0x800454ca
- TUNGETIFF = 0x400454d2
- TUNGETFEATURES = 0x400454cf
- IFF_TAP = 0x2
- IFF_NO_PI = 0x1000
- IFF_VNET_HDR = 0x4000
- # From linux/include/linux/if.h
- IFF_UP = 0x1
- else:
- # From include/linux/sockios.h
- SIOCSIFHWADDR = 0x8924
- SIOCGIFHWADDR = 0x8927
- SIOCSIFFLAGS = 0x8914
- SIOCGIFINDEX = 0x8933
- SIOCBRADDIF = 0x89a2
- # From linux/include/linux/if_tun.h
- TUNSETIFF = 0x400454ca
- TUNGETIFF = 0x800454d2
- TUNGETFEATURES = 0x800454cf
- IFF_TAP = 0x0002
- IFF_NO_PI = 0x1000
- IFF_VNET_HDR = 0x4000
- # From linux/include/linux/if.h
- IFF_UP = 0x1
- def _lock_file(filename):
- f = open(filename, "w")
- fcntl.lockf(f, fcntl.LOCK_EX)
- return f
- def _unlock_file(f):
- fcntl.lockf(f, fcntl.LOCK_UN)
- f.close()
- def is_vm(obj):
- """
- Tests whether a given object is a VM object.
- @param obj: Python object.
- """
- return obj.__class__.__name__ == "VM"
- class NetError(Exception):
- pass
- class TAPModuleError(NetError):
- def __init__(self, devname, action="open", details=None):
- NetError.__init__(self, devname)
- self.devname = devname
- self.details = details
- def __str__(self):
- e_msg = "Can't %s %s" % (self.action, self.devname)
- if self.details is not None:
- e_msg += " : %s" % self.details
- return e_msg
- class TAPNotExistError(NetError):
- def __init__(self, ifname):
- NetError.__init__(self, ifname)
- self.ifname = ifname
- def __str__(self):
- return "Interface %s does not exist" % self.ifname
- class TAPCreationError(NetError):
- def __init__(self, ifname, details=None):
- NetError.__init__(self, ifname, details)
- self.ifname = ifname
- self.details = details
- def __str__(self):
- e_msg = "Cannot create TAP device %s" % self.ifname
- if self.details is not None:
- e_msg += ": %s" % self.details
- return e_msg
- class TAPBringUpError(NetError):
- def __init__(self, ifname):
- NetError.__init__(self, ifname)
- self.ifname = ifname
- def __str__(self):
- return "Cannot bring up TAP %s" % self.ifname
- class BRAddIfError(NetError):
- def __init__(self, ifname, brname, details):
- NetError.__init__(self, ifname, brname, details)
- self.ifname = ifname
- self.brname = brname
- self.details = details
- def __str__(self):
- return ("Can not add if %s to bridge %s: %s" %
- (self.ifname, self.brname, self.details))
- class HwAddrSetError(NetError):
- def __init__(self, ifname, mac):
- NetError.__init__(self, ifname, mac)
- self.ifname = ifname
- self.mac = mac
- def __str__(self):
- return "Can not set mac %s to interface %s" % (self.mac, self.ifname)
- class HwAddrGetError(NetError):
- def __init__(self, ifname):
- NetError.__init__(self, ifname)
- self.ifname = ifname
- def __str__(self):
- return "Can not get mac of interface %s" % self.ifname
- class Env(UserDict.IterableUserDict):
- """
- A dict-like object containing global objects used by tests.
- """
- def __init__(self, filename=None, version=0):
- """
- Create an empty Env object or load an existing one from a file.
- If the version recorded in the file is lower than version, or if some
- error occurs during unpickling, or if filename is not supplied,
- create an empty Env object.
- @param filename: Path to an env file.
- @param version: Required env version (int).
- """
- UserDict.IterableUserDict.__init__(self)
- empty = {"version": version}
- if filename:
- self._filename = filename
- try:
- if os.path.isfile(filename):
- f = open(filename, "r")
- env = cPickle.load(f)
- f.close()
- if env.get("version", 0) >= version:
- self.data = env
- else:
- logging.warn("Incompatible env file found. Not using it.")
- self.data = empty
- else:
- # No previous env file found, proceed...
- self.data = empty
- # Almost any exception can be raised during unpickling, so let's
- # catch them all
- except Exception, e:
- logging.warn(e)
- self.data = empty
- else:
- self.data = empty
- def save(self, filename=None):
- """
- Pickle the contents of the Env object into a file.
- @param filename: Filename to pickle the dict into. If not supplied,
- use the filename from which the dict was loaded.
- """
- filename = filename or self._filename
- f = open(filename, "w")
- cPickle.dump(self.data, f)
- f.close()
- def get_all_vms(self):
- """
- Return a list of all VM objects in this Env object.
- """
- return [o for o in self.values() if is_vm(o)]
- def get_vm(self, name):
- """
- Return a VM object by its name.
- @param name: VM name.
- """
- return self.get("vm__%s" % name)
- def register_vm(self, name, vm):
- """
- Register a VM in this Env object.
- @param name: VM name.
- @param vm: VM object.
- """
- self["vm__%s" % name] = vm
- def unregister_vm(self, name):
- """
- Remove a given VM.
- @param name: VM name.
- """
- del self["vm__%s" % name]
- def register_installer(self, installer):
- """
- Register a installer that was just run
- The installer will be available for other tests, so that
- information about the installed KVM modules and qemu-kvm can be used by
- them.
- """
- self['last_installer'] = installer
- def previous_installer(self):
- """
- Return the last installer that was registered
- """
- return self.get('last_installer')
- class Params(UserDict.IterableUserDict):
- """
- A dict-like object passed to every test.
- """
- def objects(self, key):
- """
- Return the names of objects defined using a given key.
- @param key: The name of the key whose value lists the objects
- (e.g. 'nics').
- """
- return self.get(key, "").split()
- def object_params(self, obj_name):
- """
- Return a dict-like object containing the parameters of an individual
- object.
- This method behaves as follows: the suffix '_' + obj_name is removed
- from all key names that have it. Other key names are left unchanged.
- The values of keys with the suffix overwrite the values of their
- suffixless versions.
- @param obj_name: The name of the object (objects are listed by the
- objects() method).
- """
- suffix = "_" + obj_name
- new_dict = self.copy()
- for key in self:
- if key.endswith(suffix):
- new_key = key.split(suffix)[0]
- new_dict[new_key] = self[key]
- return new_dict
- # Functions related to MAC/IP addresses
- def _open_mac_pool(lock_mode):
- lock_file = open("/tmp/mac_lock", "w+")
- fcntl.lockf(lock_file, lock_mode)
- pool = shelve.open("/tmp/address_pool")
- return pool, lock_file
- def _close_mac_pool(pool, lock_file):
- pool.close()
- fcntl.lockf(lock_file, fcntl.LOCK_UN)
- lock_file.close()
- def _generate_mac_address_prefix(mac_pool):
- """
- Generate a random MAC address prefix and add it to the MAC pool dictionary.
- If there's a MAC prefix there already, do not update the MAC pool and just
- return what's in there. By convention we will set KVM autotest MAC
- addresses to start with 0x9a.
- @param mac_pool: The MAC address pool object.
- @return: The MAC address prefix.
- """
- if "prefix" in mac_pool:
- prefix = mac_pool["prefix"]
- else:
- r = random.SystemRandom()
- prefix = "9a:%02x:%02x:%02x:" % (r.randint(0x00, 0xff),
- r.randint(0x00, 0xff),
- r.randint(0x00, 0xff))
- mac_pool["prefix"] = prefix
- return prefix
- def generate_mac_address(vm_instance, nic_index):
- """
- Randomly generate a MAC address and add it to the MAC address pool.
- Try to generate a MAC address based on a randomly generated MAC address
- prefix and add it to a persistent dictionary.
- key = VM instance + NIC index, value = MAC address
- e.g. {'20100310-165222-Wt7l:0': '9a:5d:94:6a:9b:f9'}
- @param vm_instance: The instance attribute of a VM.
- @param nic_index: The index of the NIC.
- @return: MAC address string.
- """
- mac_pool, lock_file = _open_mac_pool(fcntl.LOCK_EX)
- key = "%s:%s" % (vm_instance, nic_index)
- if key in mac_pool:
- mac = mac_pool[key]
- else:
- prefix = _generate_mac_address_prefix(mac_pool)
- r = random.SystemRandom()
- while key not in mac_pool:
- mac = prefix + "%02x:%02x" % (r.randint(0x00, 0xff),
- r.randint(0x00, 0xff))
- if mac in mac_pool.values():
- continue
- mac_pool[key] = mac
- _close_mac_pool(mac_pool, lock_file)
- return mac
- def free_mac_address(vm_instance, nic_index):
- """
- Remove a MAC address from the address pool.
- @param vm_instance: The instance attribute of a VM.
- @param nic_index: The index of the NIC.
- """
- mac_pool, lock_file = _open_mac_pool(fcntl.LOCK_EX)
- key = "%s:%s" % (vm_instance, nic_index)
- if key in mac_pool:
- del mac_pool[key]
- _close_mac_pool(mac_pool, lock_file)
- def set_mac_address(vm_instance, nic_index, mac):
- """
- Set a MAC address in the pool.
- @param vm_instance: The instance attribute of a VM.
- @param nic_index: The index of the NIC.
- """
- mac_pool, lock_file = _open_mac_pool(fcntl.LOCK_EX)
- mac_pool["%s:%s" % (vm_instance, nic_index)] = mac
- _close_mac_pool(mac_pool, lock_file)
- def get_mac_address(vm_instance, nic_index):
- """
- Return a MAC address from the pool.
- @param vm_instance: The instance attribute of a VM.
- @param nic_index: The index of the NIC.
- @return: MAC address string.
- """
- mac_pool, lock_file = _open_mac_pool(fcntl.LOCK_SH)
- mac = mac_pool.get("%s:%s" % (vm_instance, nic_index))
- _close_mac_pool(mac_pool, lock_file)
- return mac
- def verify_ip_address_ownership(ip, macs, timeout=10.0):
- """
- Use arping and the ARP cache to make sure a given IP address belongs to one
- of the given MAC addresses.
- @param ip: An IP address.
- @param macs: A list or tuple of MAC addresses.
- @return: True iff ip is assigned to a MAC address in macs.
- """
- # Compile a regex that matches the given IP address and any of the given
- # MAC addresses
- mac_regex = "|".join("(%s)" % mac for mac in macs)
- regex = re.compile(r"\b%s\b.*\b(%s)\b" % (ip, mac_regex), re.IGNORECASE)
- # Check the ARP cache
- o = commands.getoutput("%s -n" % find_command("arp"))
- if regex.search(o):
- return True
- # Get the name of the bridge device for arping
- o = commands.getoutput("%s route get %s" % (find_command("ip"), ip))
- dev = re.findall("dev\s+\S+", o, re.IGNORECASE)
- if not dev:
- return False
- dev = dev[0].split()[-1]
- # Send an ARP request
- o = commands.getoutput("%s -f -c 3 -I %s %s" %
- (find_command("arping"), dev, ip))
- return bool(regex.search(o))
- # Utility functions for dealing with external processes
- def find_command(cmd):
- for dir in ["/usr/local/sbin", "/usr/local/bin",
- "/usr/sbin", "/usr/bin", "/sbin", "/bin"]:
- file = os.path.join(dir, cmd)
- if os.path.exists(file):
- return file
- raise ValueError('Missing command: %s' % cmd)
- def pid_exists(pid):
- """
- Return True if a given PID exists.
- @param pid: Process ID number.
- """
- try:
- os.kill(pid, 0)
- return True
- except Exception:
- return False
- def safe_kill(pid, signal):
- """
- Attempt to send a signal to a given process that may or may not exist.
- @param signal: Signal number.
- """
- try:
- os.kill(pid, signal)
- return True
- except Exception:
- return False
- def kill_process_tree(pid, sig=signal.SIGKILL):
- """Signal a process and all of its children.
- If the process does not exist -- return.
- @param pid: The pid of the process to signal.
- @param sig: The signal to send to the processes.
- """
- if not safe_kill(pid, signal.SIGSTOP):
- return
- children = commands.getoutput("ps --ppid=%d -o pid=" % pid).split()
- for child in children:
- kill_process_tree(int(child), sig)
- safe_kill(pid, sig)
- safe_kill(pid, signal.SIGCONT)
- def check_kvm_source_dir(source_dir):
- """
- Inspects the kvm source directory and verifies its disposition. In some
- occasions build may be dependant on the source directory disposition.
- The reason why the return codes are numbers is that we might have more
- changes on the source directory layout, so it's not scalable to just use
- strings like 'old_repo', 'new_repo' and such.
- @param source_dir: Source code path that will be inspected.
- """
- os.chdir(source_dir)
- has_qemu_dir = os.path.isdir('qemu')
- has_kvm_dir = os.path.isdir('kvm')
- if has_qemu_dir:
- logging.debug("qemu directory detected, source dir layout 1")
- return 1
- if has_kvm_dir and not has_qemu_dir:
- logging.debug("kvm directory detected, source dir layout 2")
- return 2
- else:
- raise error.TestError("Unknown source dir layout, cannot proceed.")
- # Functions and classes used for logging into guests and transferring files
- class LoginError(Exception):
- def __init__(self, msg, output):
- Exception.__init__(self, msg, output)
- self.msg = msg
- self.output = output
- def __str__(self):
- return "%s (output: %r)" % (self.msg, self.output)
- class LoginAuthenticationError(LoginError):
- pass
- class LoginTimeoutError(LoginError):
- def __init__(self, output):
- LoginError.__init__(self, "Login timeout expired", output)
- class LoginProcessTerminatedError(LoginError):
- def __init__(self, status, output):
- LoginError.__init__(self, None, output)
- self.status = status
- def __str__(self):
- return ("Client process terminated (status: %s, output: %r)" %
- (self.status, self.output))
- class LoginBadClientError(LoginError):
- def __init__(self, client):
- LoginError.__init__(self, None, None)
- self.client = client
- def __str__(self):
- return "Unknown remote shell client: %r" % self.client
- class SCPError(Exception):
- def __init__(self, msg, output):
- Exception.__init__(self, msg, output)
- self.msg = msg
- self.output = output
- def __str__(self):
- return "%s (output: %r)" % (self.msg, self.output)
- class SCPAuthenticationError(SCPError):
- pass
- class SCPAuthenticationTimeoutError(SCPAuthenticationError):
- def __init__(self, output):
- SCPAuthenticationError.__init__(self, "Authentication timeout expired",
- output)
- class SCPTransferTimeoutError(SCPError):
- def __init__(self, output):
- SCPError.__init__(self, "Transfer timeout expired", output)
- class SCPTransferFailedError(SCPError):
- def __init__(self, status, output):
- SCPError.__init__(self, None, output)
- self.status = status
- def __str__(self):
- return ("SCP transfer failed (status: %s, output: %r)" %
- (self.status, self.output))
- def _remote_login(session, username, password, prompt, timeout=10, debug=False):
- """
- Log into a remote host (guest) using SSH or Telnet. Wait for questions
- and provide answers. If timeout expires while waiting for output from the
- child (e.g. a password prompt or a shell prompt) -- fail.
- @brief: Log into a remote host (guest) using SSH or Telnet.
- @param session: An Expect or ShellSession instance to operate on
- @param username: The username to send in reply to a login prompt
- @param password: The password to send in reply to a password prompt
- @param prompt: The shell prompt that indicates a successful login
- @param timeout: The maximal time duration (in seconds) to wait for each
- step of the login procedure (i.e. the "Are you sure" prompt, the
- password prompt, the shell prompt, etc)
- @raise LoginTimeoutError: If timeout expires
- @raise LoginAuthenticationError: If authentication fails
- @raise LoginProcessTerminatedError: If the client terminates during login
- @raise LoginError: If some other error occurs
- """
- password_prompt_count = 0
- login_prompt_count = 0
- while True:
- try:
- match, text = session.read_until_last_line_matches(
- [r"[Aa]re you sure", r"[Pp]assword:\s*$", r"[Ll]ogin:\s*$",
- r"[Cc]onnection.*closed", r"[Cc]onnection.*refused",
- r"[Pp]lease wait", r"[Ww]arning", prompt],
- timeout=timeout, internal_timeout=0.5)
- if match == 0: # "Are you sure you want to continue connecting"
- if debug:
- logging.debug("Got 'Are you sure...', sending 'yes'")
- session.sendline("yes")
- continue
- elif match == 1: # "password:"
- if password_prompt_count == 0:
- if debug:
- logging.debug("Got password prompt, sending '%s'", password)
- session.sendline(password)
- password_prompt_count += 1
- continue
- else:
- raise LoginAuthenticationError("Got password prompt twice",
- text)
- elif match == 2: # "login:"
- if login_prompt_count == 0 and password_prompt_count == 0:
- if debug:
- logging.debug("Got username prompt; sending '%s'", username)
- session.sendline(username)
- login_prompt_count += 1
- continue
- else:
- if login_prompt_count > 0:
- msg = "Got username prompt twice"
- else:
- msg = "Got username prompt after password prompt"
- raise LoginAuthenticationError(msg, text)
- elif match == 3: # "Connection closed"
- raise LoginError("Client said 'connection closed'", text)
- elif match == 4: # "Connection refused"
- raise LoginError("Client said 'connection refused'", text)
- elif match == 5: # "Please wait"
- if debug:
- logging.debug("Got 'Please wait'")
- timeout = 30
- continue
- elif match == 6: # "Warning added RSA"
- if debug:
- logging.debug("Got 'Warning added RSA to known host list")
- continue
- elif match == 7: # prompt
- if debug:
- logging.debug("Got shell prompt -- logged in")
- break
- except aexpect.ExpectTimeoutError, e:
- raise LoginTimeoutError(e.output)
- except aexpect.ExpectProcessTerminatedError, e:
- raise LoginProcessTerminatedError(e.status, e.output)
- def remote_login(client, host, port, username, password, prompt, linesep="\n",
- log_filename=None, timeout=10):
- """
- Log into a remote host (guest) using SSH/Telnet/Netcat.
- @param client: The client to use ('ssh', 'telnet' or 'nc')
- @param host: Hostname or IP address
- @param port: Port to connect to
- @param username: Username (if required)
- @param password: Password (if required)
- @param prompt: Shell prompt (regular expression)
- @param linesep: The line separator to use when sending lines
- (e.g. '\\n' or '\\r\\n')
- @param log_filename: If specified, log all output to this file
- @param timeout: The maximal time duration (in seconds) to wait for
- each step of the login procedure (i.e. the "Are you sure" prompt
- or the password prompt)
- @raise LoginBadClientError: If an unknown client is requested
- @raise: Whatever _remote_login() raises
- @return: A ShellSession object.
- """
- if client == "ssh":
- cmd = ("ssh -o UserKnownHostsFile=/dev/null "
- "-o PreferredAuthentications=password -p %s %s@%s" %
- (port, username, host))
- elif client == "telnet":
- cmd = "telnet -l %s %s %s" % (username, host, port)
- elif client == "nc":
- cmd = "nc %s %s" % (host, port)
- else:
- raise LoginBadClientError(client)
- logging.debug("Login command: '%s'", cmd)
- session = aexpect.ShellSession(cmd, linesep=linesep, prompt=prompt)
- try:
- _remote_login(session, username, password, prompt, timeout)
- except Exception:
- session.close()
- raise
- if log_filename:
- session.set_output_func(log_line)
- session.set_output_params((log_filename,))
- return session
- def wait_for_login(client, host, port, username, password, prompt, linesep="\n",
- log_filename=None, timeout=240, internal_timeout=10):
- """
- Make multiple attempts to log into a remote host (guest) until one succeeds
- or timeout expires.
- @param timeout: Total time duration to wait for a successful login
- @param internal_timeout: The maximal time duration (in seconds) to wait for
- each step of the login procedure (e.g. the "Are you sure" prompt
- or the password prompt)
- @see: remote_login()
- @raise: Whatever remote_login() raises
- @return: A ShellSession object.
- """
- logging.debug("Attempting to log into %s:%s using %s (timeout %ds)",
- host, port, client, timeout)
- end_time = time.time() + timeout
- while time.time() < end_time:
- try:
- return remote_login(client, host, port, username, password, prompt,
- linesep, log_filename, internal_timeout)
- except LoginError, e:
- logging.debug(e)
- time.sleep(2)
- # Timeout expired; try one more time but don't catch exceptions
- return remote_login(client, host, port, username, password, prompt,
- linesep, log_filename, internal_timeout)
- def _remote_scp(session, password_list, transfer_timeout=600, login_timeout=20):
- """
- Transfer file(s) to a remote host (guest) using SCP. Wait for questions
- and provide answers. If login_timeout expires while waiting for output
- from the child (e.g. a password prompt), fail. If transfer_timeout expires
- while waiting for the transfer to complete, fail.
- @brief: Transfer files using SCP, given a command line.
- @param session: An Expect or ShellSession instance to operate on
- @param password_list: Password list to send in reply to the password prompt
- @param transfer_timeout: The time duration (in seconds) to wait for the
- transfer to complete.
- @param login_timeout: The maximal time duration (in seconds) to wait for
- each step of the login procedure (i.e. the "Are you sure" prompt or
- the password prompt)
- @raise SCPAuthenticationError: If authentication fails
- @raise SCPTransferTimeoutError: If the transfer fails to complete in time
- @raise SCPTransferFailedError: If the process terminates with a nonzero
- exit code
- @raise SCPError: If some other error occurs
- """
- password_prompt_count = 0
- timeout = login_timeout
- authentication_done = False
- scp_type = len(password_list)
- while True:
- try:
- match, text = session.read_until_last_line_matches(
- [r"[Aa]re you sure", r"[Pp]assword:\s*$", r"lost connection"],
- timeout=timeout, internal_timeout=0.5)
- if match == 0: # "Are you sure you want to continue connecting"
- logging.debug("Got 'Are you sure...', sending 'yes'")
- session.sendline("yes")
- continue
- elif match == 1: # "password:"
- if password_prompt_count == 0:
- logging.debug("Got password prompt, sending '%s'" %
- password_list[password_prompt_count])
- session.sendline(password_list[password_prompt_count])
- password_prompt_count += 1
- timeout = transfer_timeout
- if scp_type == 1:
- authentication_done = True
- continue
- elif password_prompt_count == 1 and scp_type == 2:
- logging.debug("Got password prompt, sending '%s'" %
- password_list[password_prompt_count])
- session.sendline(password_list[password_prompt_count])
- password_prompt_count += 1
- timeout = transfer_timeout
- authentication_done = True
- continue
- else:
- raise SCPAuthenticationError("Got password prompt twice",
- text)
- elif match == 2: # "lost connection"
- raise SCPError("SCP client said 'lost connection'", text)
- except aexpect.ExpectTimeoutError, e:
- if authentication_done:
- raise SCPTransferTimeoutError(e.output)
- else:
- raise SCPAuthenticationTimeoutError(e.output)
- except aexpect.ExpectProcessTerminatedError, e:
- if e.status == 0:
- logging.debug("SCP process terminated with status 0")
- break
- else:
- raise SCPTransferFailedError(e.status, e.output)
- def remote_scp(command, password_list, log_filename=None, transfer_timeout=600,
- login_timeout=20):
- """
- Transfer file(s) to a remote host (guest) using SCP.
- @brief: Transfer files using SCP, given a command line.
- @param command: The command to execute
- (e.g. "scp -r foobar root@localhost:/tmp/").
- @param password_list: Password list to send in reply to a password prompt.
- @param log_filename: If specified, log all output to this file
- @param transfer_timeout: The time duration (in seconds) to wait for the
- transfer to complete.
- @param login_timeout: The maximal time duration (in seconds) to wait for
- each step of the login procedure (i.e. the "Are you sure" prompt
- or the password prompt)
- @raise: Whatever _remote_scp() raises
- """
- logging.debug("Trying to SCP with command '%s', timeout %ss",
- command, transfer_timeout)
- if log_filename:
- output_func = log_line
- output_params = (log_filename,)
- else:
- output_func = None
- output_params = ()
- session = aexpect.Expect(command,
- output_func=output_func,
- output_params=output_params)
- try:
- _remote_scp(session, password_list, transfer_timeout, login_timeout)
- finally:
- session.close()
- def scp_to_remote(host, port, username, password, local_path, remote_path,
- log_filename=None, timeout=600):
- """
- Copy files to a remote host (guest) through scp.
- @param host: Hostname or IP address
- @param username: Username (if required)
- @param password: Password (if required)
- @param local_path: Path on the local machine where we are copying from
- @param remote_path: Path on the remote machine where we are copying to
- @param log_filename: If specified, log all output to this file
- @param timeout: The time duration (in seconds) to wait for the transfer
- to complete.
- @raise: Whatever remote_scp() raises
- """
- command = ("scp -v -o UserKnownHostsFile=/dev/null "
- "-o PreferredAuthentications=password -r -P %s %s %s@%s:%s" %
- (port, local_path, username, host, remote_path))
- password_list = []
- password_list.append(password)
- return remote_scp(command, password_list, log_filename, timeout)
- def scp_from_remote(host, port, username, password, remote_path, local_path,
- log_filename=None, timeout=600):
- """
- Copy files from a remote host (guest).
- @param host: Hostname or IP address
- @param username: Username (if required)
- @param password: Password (if required)
- @param local_path: Path on the local machine where we are copying from
- @param remote_path: Path on the remote machine where we are copying to
- @param log_filename: If specified, log all output to this file
- @param timeout: The time duration (in seconds) to wait for the transfer
- to complete.
- @raise: Whatever remote_scp() raises
- """
- command = ("scp -v -o UserKnownHostsFile=/dev/null "
- "-o PreferredAuthentications=password -r -P %s %s@%s:%s %s" %
- (port, username, host, remote_path, local_path))
- password_list = []
- password_list.append(password)
- remote_scp(command, password_list, log_filename, timeout)
- def scp_between_remotes(src, dst, port, s_passwd, d_passwd, s_name, d_name,
- s_path, d_path, log_filename=None, timeout=600):
- """
- Copy files from a remote host (guest) to another remote host (guest).
- @param src/dst: Hostname or IP address of src and dst
- @param s_name/d_name: Username (if required)
- @param s_passwd/d_passwd: Password (if required)
- @param s_path/d_path: Path on the remote machine where we are copying
- from/to
- @param log_filename: If specified, log all output to this file
- @param timeout: The time duration (in seconds) to wait for the transfer
- to complete.
- @return: True on success and False on failure.
- """
- command = ("scp -v -o UserKnownHostsFile=/dev/null -o "
- "PreferredAuthentications=password -r -P %s %s@%s:%s %s@%s:%s" %
- (port, s_name, src, s_path, d_name, dst, d_path))
- password_list = []
- password_list.append(s_passwd)
- password_list.append(d_passwd)
- return remote_scp(command, password_list, log_filename, timeout)
- def copy_files_to(address, client, username, password, port, local_path,
- remote_path, log_filename=None, verbose=False, timeout=600):
- """
- Copy files to a remote host (guest) using the selected client.
- @param client: Type of transfer client
- @param username: Username (if required)
- @param password: Password (if requried)
- @param local_path: Path on the local machine where we are copying from
- @param remote_path: Path on the remote machine where we are copying to
- @param address: Address of remote host(guest)
- @param log_filename: If specified, log all output to this file (SCP only)
- @param verbose: If True, log some stats using logging.debug (RSS only)
- @param timeout: The time duration (in seconds) to wait for the transfer to
- complete.
- @raise: Whatever remote_scp() raises
- """
- if client == "scp":
- scp_to_remote(address, port, username, password, local_path,
- remote_path, log_filename, timeout)
- elif client == "rss":
- log_func = None
- if verbose:
- log_func = logging.debug
- c = rss_client.FileUploadClient(address, port, log_func)
- c.upload(local_path, remote_path, timeout)
- c.close()
- def copy_files_from(address, client, username, password, port, remote_path,
- local_path, log_filename=None, verbose=False, timeout=600):
- """
- Copy files from a remote host (guest) using the selected client.
- @param client: Type of transfer client
- @param username: Username (if required)
- @param password: Password (if requried)
- @param remote_path: Path on the remote machine where we are copying from
- @param local_path: Path on the local machine where we are copying to
- @param address: Address of remote host(guest)
- @param log_filename: If specified, log all output to this file (SCP only)
- @param verbose: If True, log some stats using logging.debug (RSS only)
- @param timeout: The time duration (in seconds) to wait for the transfer to
- complete.
- @raise: Whatever remote_scp() raises
- """
- if client == "scp":
- scp_from_remote(address, port, username, password, remote_path,
- local_path, log_filename, timeout)
- elif client == "rss":
- log_func = None
- if verbose:
- log_func = logging.debug
- c = rss_client.FileDownloadClient(address, port, log_func)
- c.download(remote_path, local_path, timeout)
- c.close()
- # The following are utility functions related to ports.
- def is_port_free(port, address):
- """
- Return True if the given port is available for use.
- @param port: Port number
- """
- try:
- s = socket.socket()
- #s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
- if address == "localhost":
- s.bind(("localhost", port))
- free = True
- else:
- s.connect((address, port))
- free = False
- except socket.error:
- if address == "localhost":
- free = False
- else:
- free = True
- s.close()
- return free
- def find_free_port(start_port, end_port, address="localhost"):
- """
- Return a host free port in the range [start_port, end_port].
- @param start_port: First port that will be checked.
- @param end_port: Port immediately after the last one that will be checked.
- """
- for i in range(start_port, end_port):
- if is_port_free(i, address):
- return i
- return None
- def find_free_ports(start_port, end_port, count, address="localhost"):
- """
- Return count of host free ports in the range [start_port, end_port].
- @count: Initial number of ports known to be free in the range.
- @param start_port: First port that will be checked.
- @param end_port: Port immediately after the last one that will be checked.
- """
- ports = []
- i = start_port
- while i < end_port and count > 0:
- if is_port_free(i, address):
- ports.append(i)
- count -= 1
- i += 1
- return ports
- # An easy way to log lines to files when the logging system can't be used
- _open_log_files = {}
- _log_file_dir = "/tmp"
- def log_line(filename, line):
- """
- Write a line to a file. '\n' is appended to the line.
- @param filename: Path of file to write to, either absolute or relative to
- the dir set by set_log_file_dir().
- @param line: Line to write.
- """
- global _open_log_files, _log_file_dir
- if filename not in _open_log_files:
- path = get_path(_log_file_dir, filename)
- try:
- os.makedirs(os.path.dirname(path))
- except OSError:
- pass
- _open_log_files[filename] = open(path, "w")
- timestr = time.strftime("%Y-%m-%d %H:%M:%S")
- _open_log_files[filename].write("%s: %s\n" % (timestr, line))
- _open_log_files[filename].flush()
- def set_log_file_dir(dir):
- """
- Set the base directory for log files created by log_line().
- @param dir: Directory for log files.
- """
- global _log_file_dir
- _log_file_dir = dir
- # The following are miscellaneous utility functions.
- def get_path(base_path, user_path):
- """
- Translate a user specified path to a real path.
- If user_path is relative, append it to base_path.
- If user_path is absolute, return it as is.
- @param base_path: The base path of relative user specified paths.
- @param user_path: The user specified path.
- """
- if os.path.isabs(user_path):
- return user_path
- else:
- return os.path.join(base_path, user_path)
- def generate_random_string(length):
- """
- Return a random string using alphanumeric characters.
- @length: length of the string that will be generated.
- """
- r = random.SystemRandom()
- str = ""
- chars = string.letters + string.digits
- while length > 0:
- str += r.choice(chars)
- length -= 1
- return str
- def generate_random_id():
- """
- Return a random string suitable for use as a qemu id.
- """
- return "id" + generate_random_string(6)
- def generate_tmp_file_name(file, ext=None, dir='/tmp/'):
- """
- Returns a temporary file name. The file is not created.
- """
- while True:
- file_name = (file + '-' + time.strftime("%Y%m%d-%H%M%S-") +
- generate_random_string(4))
- if ext:
- file_name += '.' + ext
- file_name = os.path.join(dir, file_name)
- if not os.path.exists(file_name):
- break
- return file_name
- def format_str_for_message(str):
- """
- Format str so that it can be appended to a message.
- If str consists of one line, prefix it with a space.
- If str consists of multiple lines, prefix it with a newline.
- @param str: string that will be formatted.
- """
- lines = str.splitlines()
- num_lines = len(lines)
- str = "\n".join(lines)
- if num_lines == 0:
- return ""
- elif num_lines == 1:
- return " " + str
- else:
- return "\n" + str
- def wait_for(func, timeout, first=0.0, step=1.0, text=None):
- """
- If func() evaluates to True before timeout expires, return the
- value of func(). Otherwise return None.
- @brief: Wait until func() evaluates to True.
- @param timeout: Timeout in seconds
- @param first: Time to sleep before first attempt
- @param steps: Time to sleep between attempts in seconds
- @param text: Text to print while waiting, for debug purposes
- """
- start_time = time.time()
- end_time = time.time() + timeout
- time.sleep(first)
- while time.time() < end_time:
- if text:
- logging.debug("%s (%f secs)", text, (time.time() - start_time))
- output = func()
- if output:
- return output
- time.sleep(step)
- return None
- def get_hash_from_file(hash_path, dvd_basename):
- """
- Get the a hash from a given DVD image from a hash file
- (Hash files are usually named MD5SUM or SHA1SUM and are located inside the
- download directories of the DVDs)
- @param hash_path: Local path to a hash file.
- @param cd_image: Basename of a CD image
- """
- hash_file = open(hash_path, 'r')
- for line in hash_file.readlines():
- if dvd_basename in line:
- return line.split()[0]
- def run_tests(parser, job):
- """
- Runs the sequence of KVM tests based on the list of dictionaries
- generated by the configuration system, handling dependencies.
- @param parser: Config parser object.
- @param job: Autotest job object.
- @return: True, if all tests ran passed, False if any of them failed.
- """
- for i, d in enumerate(parser.get_dicts()):
- logging.info("Test %4d: %s" % (i + 1, d["shortname"]))
- status_dict = {}
- failed = False
- for dict in parser.get_dicts():
- if dict.get("skip") == "yes":
- continue
- dependencies_satisfied = True
- for dep in dict.get("dep"):
- for test_name in status_dict.keys():
- if not dep in test_name:
- continue
- # So the only really non-fatal state is WARN,
- # All the others make it not safe to proceed with dependency
- # execution
- if status_dict[test_name] not in ['GOOD', 'WARN']:
- dependencies_satisfied = False
- break
- test_iterations = int(dict.get("iterations", 1))
- test_tag = dict.get("shortname")
- if dependencies_satisfied:
- # Setting up profilers during test execution.
- profilers = dict.get("profilers", "").split()
- for profiler in profilers:
- job.profilers.add(profiler)
- # We need only one execution, profiled, hence we're passing
- # the profile_only parameter to job.run_test().
- profile_only = bool(profilers) or None
- current_status = job.run_test_detail(dict.get("vm_type"),
- params=dict,
- tag=test_tag,
- iterations=test_iterations,
- profile_only=profile_only)
- for profiler in profilers:
- job.profilers.delete(profiler)
- else:
- # We will force the test to fail as TestNA during preprocessing
- dict['dependency_failed'] = 'yes'
- current_status = job.run_test_detail(dict.get("vm_type"),
- params=dict,
- tag=test_tag,
- iterations=test_iterations)
- if not current_status:
- failed = True
- status_dict[dict.get("name")] = current_status
- return not failed
- def display_attributes(instance):
- """
- Inspects a given class instance attributes and displays them, convenient
- for debugging.
- """
- logging.debug("Attributes set:")
- for member in inspect.getmembers(instance):
- name, value = member
- attribute = getattr(instance, name)
- if not (name.startswith("__") or callable(attribute) or not value):
- logging.debug(" %s: %s", name, value)
- def get_full_pci_id(pci_id):
- """
- Get full PCI ID of pci_id.
- @param pci_id: PCI ID of a device.
- """
- cmd = "lspci -D | awk '/%s/ {print $1}'" % pci_id
- status, full_id = commands.getstatusoutput(cmd)
- if status != 0:
- return None
- return full_id
- def get_vendor_from_pci_id(pci_id):
- """
- Check out the device vendor ID according to pci_id.
- @param pci_id: PCI ID of a device.
- """
- cmd = "lspci -n | awk '/%s/ {print $3}'" % pci_id
- return re.sub(":", " ", commands.getoutput(cmd))
- class Flag(str):
- """
- Class for easy merge cpuflags.
- """
- aliases = {}
- def __new__(cls, flag):
- if flag in Flag.aliases:
- flag = Flag.aliases[flag]
- return str.__new__(cls, flag)
- def __eq__(self, other):
- s = set(self.split("|"))
- o = set(other.split("|"))
- if s & o:
- return True
- else:
- return False
- def __hash__(self, *args, **kwargs):
- return 0
- kvm_map_flags_to_test = {
- Flag('avx') :set(['avx']),
- Flag('sse3') :set(['sse3']),
- Flag('ssse3') :set(['ssse3']),
- Flag('sse4.1|sse4_1|sse4.2|sse4_2'):set(['sse4']),
- Flag('aes') :set(['aes','pclmul']),
- Flag('pclmuldq') :set(['pclmul']),
- Flag('pclmulqdq') :set(['pclmul']),
- Flag('rdrand') :set(['rdrand']),
- Flag('sse4a') :set(['sse4a']),
- Flag('fma4') :set(['fma4']),
- Flag('xop') :set(['xop']),
- }
- kvm_map_flags_aliases = {
- 'sse4.1' :'sse4_1',
- 'sse4.2' :'sse4_2',
- 'pclmulqdq' :'pclmuldq',
- }
- def kvm_flags_to_stresstests(flags):
- """
- Covert [cpu flags] to [tests]
- @param cpuflags: list of cpuflags
- @return: Return tests like string.
- """
- tests = set([])
- for f in flags:
- tests |= kvm_map_flags_to_test[f]
- param = ""
- for f in tests:
- param += ","+f
- return param
- def get_cpu_flags():
- """
- Returns a list of the CPU flags
- """
- flags_re = re.compile(r'^flags\s*:(.*)')
- for line in open('/proc/cpuinfo').readlines():
- match = flags_re.match(line)
- if match:
- return match.groups()[0].split()
- return []
- def get_cpu_vendor(cpu_flags=[], verbose=True):
- """
- Returns the name of the CPU vendor, either intel, amd or unknown
- """
- if not cpu_flags:
- cpu_flags = get_cpu_flags()
- if 'vmx' in cpu_flags:
- vendor = 'intel'
- elif 'svm' in cpu_flags:
- vendor = 'amd'
- else:
- vendor = 'unknown'
- if verbose:
- logging.debug("Detected CPU vendor as '%s'", vendor)
- return vendor
- def get_archive_tarball_name(source_dir, tarball_name, compression):
- '''
- Get the name for a tarball file, based on source, name and compression
- '''
- if tarball_name is None:
- tarball_name = os.path.basename(source_dir)
- if not tarball_name.endswith('.tar'):
- tarball_name = '%s.tar' % tarball_name
- if compression and not tarball_name.endswith('.%s' % compression):
- tarball_name = '%s.%s' % (tarball_name, compression)
- return tarball_name
- def archive_as_tarball(source_dir, dest_dir, tarball_name=None,
- compression='bz2', verbose=True):
- '''
- Saves the given source directory to the given destination as a tarball
- If the name of the archive is omitted, it will be taken from the
- source_dir. If it is an absolute path, dest_dir will be ignored. But,
- if both the destination directory and tarball anem is given, and the
- latter is not an absolute path, they will be combined.
- For archiving directory '/tmp' in '/net/server/backup' as file
- 'tmp.tar.bz2', simply use:
- >>> virt_utils.archive_as_tarball('/tmp', '/net/server/backup')
- To save the file it with a different name, say 'host1-tmp.tar.bz2'
- and save it under '/net/server/backup', use:
- >>> virt_utils.archive_as_tarball('/tmp', '/net/server/backup',
- 'host1-tmp')
- To save with gzip compression instead (resulting in the file
- '/net/server/backup/host1-tmp.tar.gz'), use:
- >>> virt_utils.archive_as_tarball('/tmp', '/net/server/backup',
- 'host1-tmp', 'gz')
- '''
- tarball_name = get_archive_tarball_name(source_dir,
- tarball_name,
- compression)
- if not os.path.isabs(tarball_name):
- tarball_path = os.path.join(dest_dir, tarball_name)
- else:
- tarball_path = tarball_name
- if verbose:
- logging.debug('Archiving %s as %s' % (source_dir,
- tarball_path))
- os.chdir(os.path.dirname(source_dir))
- tarball = tarfile.TarFile(name=tarball_path, mode='w')
- tarball = tarball.open(name=tarball_path, mode='w:%s' % compression)
- tarball.add(os.path.basename(source_dir))
- tarball.close()
- class Thread(threading.Thread):
- """
- Run a function in a background thread.
- """
- def __init__(self, target, args=(), kwargs={}):
- """
- Initialize the instance.
- @param target: Function to run in the thread.
- @param args: Arguments to pass to target.
- @param kwargs: Keyword arguments to pass to target.
- """
- threading.Thread.__init__(self)
- self._target = target
- self._args = args
- self._kwargs = kwargs
- def run(self):
- """
- Run target (passed to the constructor). No point in calling this
- function directly. Call start() to make this function run in a new
- thread.
- """
- self._e = None
- self._retval = None
- try:
- try:
- self._retval = self._target(*self._args, **self._kwargs)
- except Exception:
- self._e = sys.exc_info()
- raise
- finally:
- # Avoid circular references (start() may be called only once so
- # it's OK to delete these)
- del self._target, self._args, self._kwargs
- def join(self, timeout=None, suppress_exception=False):
- """
- Join the thread. If target raised an exception, re-raise it.
- Otherwise, return the value returned by target.
- @param timeout: Timeout value to pass to threading.Thread.join().
- @param suppress_exception: If True, don't re-raise the exception.
- """
- threading.Thread.join(self, timeout)
- try:
- if self._e:
- if not suppress_exception:
- # Because…
Large files files are truncated, but you can click here to view the full file