/pytests/upgrade/newupgradebasetest.py

import re
import testconstants
import gc
import sys
import traceback
import Queue
from threading import Thread
from basetestcase import BaseTestCase
from mc_bin_client import MemcachedError
from memcached.helper.data_helper import VBucketAwareMemcached, MemcachedClientHelper
from membase.helper.bucket_helper import BucketOperationHelper
from membase.api.rest_client import RestConnection, RestHelper
from membase.helper.cluster_helper import ClusterOperationHelper
from remote.remote_util import RemoteMachineShellConnection, RemoteUtilHelper
from couchbase_helper.document import DesignDocument, View
from couchbase_helper.documentgenerator import BlobGenerator
from scripts.install import InstallerJob
from builds.build_query import BuildQuery
from couchbase_helper.tuq_generators import JsonGenerator
from pprint import pprint
from testconstants import CB_REPO
from testconstants import MV_LATESTBUILD_REPO
from testconstants import SHERLOCK_BUILD_REPO
from testconstants import COUCHBASE_VERSION_2
from testconstants import COUCHBASE_VERSION_3
from testconstants import COUCHBASE_VERSIONS
from testconstants import SHERLOCK_VERSION
from testconstants import CB_VERSION_NAME
from testconstants import COUCHBASE_FROM_VERSION_3
from testconstants import COUCHBASE_MP_VERSION
from testconstants import CE_EE_ON_SAME_FOLDER


class NewUpgradeBaseTest(BaseTestCase):
    def setUp(self):
        super(NewUpgradeBaseTest, self).setUp()
        self.released_versions = ["2.0.0-1976-rel", "2.0.1", "2.5.0", "2.5.1",
                                  "2.5.2", "3.0.0", "3.0.1",
                                  "3.0.1-1444", "3.0.2", "3.0.2-1603", "3.0.3",
                                  "3.1.0", "3.1.0-1776", "3.1.1", "3.1.1-1807",
                                  "3.1.2", "3.1.2-1815", "3.1.3", "3.1.3-1823",
                                  "4.0.0", "4.0.0-4051", "4.1.0", "4.1.0-5005",
                                  "4.5.0", "4.5.0-2601", "4.5.1", "4.5.1-2817",
                                  "4.6.0", "4.6.0-3573", '4.6.2', "4.6.2-3905"]
        self.use_hostnames = self.input.param("use_hostnames", False)
        self.product = self.input.param('product', 'couchbase-server')
        self.initial_version = self.input.param('initial_version', '2.5.1-1083')
        self.initial_vbuckets = self.input.param('initial_vbuckets', 1024)
        self.upgrade_versions = self.input.param('upgrade_version', '2.0.1-170-rel')
        self.upgrade_versions = self.upgrade_versions.split(";")
        self.skip_cleanup = self.input.param("skip_cleanup", False)
        self.init_nodes = self.input.param('init_nodes', True)

        self.is_downgrade = self.input.param('downgrade', False)
        if self.is_downgrade:
            self.initial_version, self.upgrade_versions = self.upgrade_versions[0], [self.initial_version]

        upgrade_path = self.input.param('upgrade_path', [])
        if upgrade_path:
            upgrade_path = upgrade_path.split(",")
        self.upgrade_versions = upgrade_path + self.upgrade_versions
        if self.input.param('released_upgrade_version', None) is not None:
            self.upgrade_versions = [self.input.param('released_upgrade_version', None)]

        self.initial_build_type = self.input.param('initial_build_type', None)
        self.stop_persistence = self.input.param('stop_persistence', False)
        self.std_vbucket_dist = self.input.param("std_vbucket_dist", None)
        self.rest_settings = self.input.membase_settings
        self.rest = None
        self.rest_helper = None
        self.is_ubuntu = False
        self.sleep_time = 15
        self.ddocs = []
        self.item_flag = self.input.param('item_flag', 0)
        self.expire_time = self.input.param('expire_time', 0)
        self.wait_expire = self.input.param('wait_expire', False)
        self.default_view_name = "upgrade-test-view"
        self.ddocs_num = self.input.param("ddocs_num", 1)
        self.view_num = self.input.param("view_per_ddoc", 2)
        self.is_dev_ddoc = self.input.param("is-dev-ddoc", False)
        self.during_ops = None
        if "during-ops" in self.input.test_params:
            self.during_ops = self.input.param("during-ops", None).split(",")
        if self.initial_version.startswith("1.6") or self.initial_version.startswith("1.7"):
            self.product = 'membase-server'
        else:
            self.product = 'couchbase-server'
        self.index_replicas = self.input.param("index_replicas", None)
        self.index_kv_store = self.input.param("kvstore", None)
        self.partitions_per_pindex = \
            self.input.param("max_partitions_pindex", 32)
        self.dataset = self.input.param("dataset", "emp")
        self.expiry = self.input.param("expiry", 0)
        self.create_ops_per = self.input.param("create_ops_per", 0)
        self.expiry_ops_per = self.input.param("expiry_ops_per", 0)
        self.delete_ops_per = self.input.param("delete_ops_per", 0)
        self.update_ops_per = self.input.param("update_ops_per", 0)
        self.docs_per_day = self.input.param("doc-per-day", 49)
        self.doc_ops = self.input.param("doc_ops", False)
        if self.doc_ops:
            self.ops_dist_map = self.calculate_data_change_distribution(
                create_per=self.create_ops_per , update_per=self.update_ops_per ,
                delete_per=self.delete_ops_per, expiry_per=self.expiry_ops_per,
                start=0, end=self.docs_per_day)
            self.log.info(self.ops_dist_map)
            self.docs_gen_map = self.generate_ops_docs(self.docs_per_day, 0)
            #self.full_docs_list_after_ops = self.generate_full_docs_list_after_ops(self.docs_gen_map)
        if self.max_verify is None:
            self.max_verify = min(self.num_items, 100000)
        shell = RemoteMachineShellConnection(self.master)
        type = shell.extract_remote_info().distribution_type
        shell.disconnect()
        if type.lower() == 'windows':
            self.is_linux = False
        else:
            self.is_linux = True
        if type.lower() == "ubuntu":
            self.is_ubuntu = True
        self.queue = Queue.Queue()
        self.upgrade_servers = []
        self.index_replicas = self.input.param("index_replicas", None)
        self.partitions_per_pindex = \
            self.input.param("max_partitions_pindex", 32)
        self.index_kv_store = self.input.param("kvstore", None)

    def tearDown(self):
        test_failed = (hasattr(self, '_resultForDoCleanups') and \
                       len(self._resultForDoCleanups.failures or \
                           self._resultForDoCleanups.errors)) or \
                                 (hasattr(self, '_exc_info') and \
                                  self._exc_info()[1] is not None)
        if test_failed and self.skip_cleanup:
                self.log.warn("CLEANUP WAS SKIPPED DUE TO FAILURES IN UPGRADE TEST")
                self.cluster.shutdown(force=True)
                self.log.info("Test Input params were:")
                pprint(self.input.test_params)

                if self.input.param('BUGS', False):
                    self.log.warn("Test failed. Possible reason is: {0}"
                                           .format(self.input.param('BUGS', False)))
        else:
            if not hasattr(self, 'rest'):
                return
            try:
                # cleanup only nodes that are in cluster
                # not all servers have been installed
                if self.rest is None:
                    self._new_master(self.master)
                nodes = self.rest.get_nodes()
                temp = []
                for server in self.servers:
                    if server.ip in [node.ip for node in nodes]:
                        temp.append(server)
                self.servers = temp
            except Exception, e:
                if e:
                    print "Exception ", e
                self.cluster.shutdown(force=True)
                self.fail(e)
            super(NewUpgradeBaseTest, self).tearDown()
            if self.upgrade_servers:
                self._install(self.upgrade_servers,version=self.initial_version)
        self.sleep(20, "sleep 20 seconds before run next test")

    def _install(self, servers):
        params = {}
        params['num_nodes'] = len(servers)
        params['product'] = self.product
        params['version'] = self.initial_version
        params['vbuckets'] = [self.initial_vbuckets]
        params['init_nodes'] = self.init_nodes
        if self.initial_build_type is not None:
            params['type'] = self.initial_build_type
        self.log.info("will install {0} on {1}".format(self.initial_version, [s.ip for s in servers]))
        InstallerJob().parallel_install(servers, params)
        if self.product in ["couchbase", "couchbase-server", "cb"]:
            success = True
            for server in servers:
                success &= RemoteMachineShellConnection(server).is_couchbase_installed()
                self.sleep(5, "sleep 5 seconds to let cb up completely")
                if not success:
                    sys.exit("some nodes were not install successfully!")
        if self.rest is None:
            self._new_master(self.master)
        if self.use_hostnames:
            for server in self.servers[:self.nodes_init]:
                hostname = RemoteUtilHelper.use_hostname_for_server_settings(server)
                server.hostname = hostname

    def operations(self, servers):
        self.quota = self._initialize_nodes(self.cluster, servers, self.disabled_consistent_view,
                                            self.rebalanceIndexWaitingDisabled, self.rebalanceIndexPausingDisabled,
                                            self.maxParallelIndexers, self.maxParallelReplicaIndexers, self.port)
        if self.port and self.port != '8091':
            self.rest = RestConnection(self.master)
            self.rest_helper = RestHelper(self.rest)
        self.sleep(7, "wait to make sure node is ready")
        if len(servers) > 1:
            self.cluster.rebalance([servers[0]], servers[1:], [],
                                   use_hostnames=self.use_hostnames)

        self.buckets = []
        gc.collect()
        if self.input.param('extra_verification', False):
            self.total_buckets += 2
            print self.total_buckets
        self.bucket_size = self._get_bucket_size(self.quota, self.total_buckets)
        self._bucket_creation()
        if self.stop_persistence:
            for server in servers:
                for bucket in self.buckets:
                    client = MemcachedClientHelper.direct_client(server, bucket)
                    client.stop_persistence()
            self.sleep(10)
        gen_load = BlobGenerator('upgrade', 'upgrade-', self.value_size, end=self.num_items)
        self._load_all_buckets(self.master, gen_load, "create", self.expire_time, flag=self.item_flag)
        if not self.stop_persistence:
            self._wait_for_stats_all_buckets(servers)
        else:
            for bucket in self.buckets:
                drain_rate = 0
                for server in servers:
                    client = MemcachedClientHelper.direct_client(server, bucket)
                    drain_rate += int(client.stats()["ep_queue_size"])
                self.sleep(3, "Pause to load all items")
                self.assertEqual(self.num_items * (self.num_replicas + 1), drain_rate,
                                 "Persistence is stopped, drain rate is incorrect %s. Expected %s" % (
                                    drain_rate, self.num_items * (self.num_replicas + 1)))
        self.change_settings()

    def _get_build(self, server, version, remote, is_amazon=False, info=None):
        if info is None:
            info = remote.extract_remote_info()
        build_repo = CB_REPO
        if version[:5] in COUCHBASE_VERSIONS:
            if version[:3] in CB_VERSION_NAME:
                build_repo = CB_REPO + CB_VERSION_NAME[version[:3]] + "/"
            elif version[:5] in COUCHBASE_MP_VERSION:
                build_repo = MV_LATESTBUILD_REPO
        builds, changes = BuildQuery().get_all_builds(version=version, timeout=self.wait_timeout * 5, \
                    deliverable_type=info.deliverable_type, architecture_type=info.architecture_type, \
                    edition_type="couchbase-server-enterprise", repo=build_repo, \
                    distribution_version=info.distribution_version.lower())
        self.log.info("finding build %s for machine %s" % (version, server))

        if re.match(r'[1-9].[0-9].[0-9]-[0-9]+$', version):
            version = version + "-rel"
        if version[:5] in self.released_versions:
            appropriate_build = BuildQuery().\
                find_couchbase_release_build('%s-enterprise' % (self.product),
                                           info.deliverable_type,
                                           info.architecture_type,
                                           version.strip(),
                                           is_amazon=is_amazon,
                                           os_version=info.distribution_version)
        else:
             appropriate_build = BuildQuery().\
                find_build(builds, '%s-enterprise' % (self.product), info.deliverable_type,
                                   info.architecture_type, version.strip())

        if appropriate_build is None:
            self.log.info("builds are: %s \n. Remote is %s, %s. Result is: %s" % (builds, remote.ip, remote.username, version))
            raise Exception("Build %s for machine %s is not found" % (version, server))
        return appropriate_build

    def _upgrade(self, upgrade_version, server, queue=None, skip_init=False, info=None):
        try:
            remote = RemoteMachineShellConnection(server)
            appropriate_build = self._get_build(server, upgrade_version, remote, info=info)
            self.assertTrue(appropriate_build.url, msg="unable to find build {0}".format(upgrade_version))
            self.assertTrue(remote.download_build(appropriate_build), "Build wasn't downloaded!")
            o, e = remote.couchbase_upgrade(appropriate_build, save_upgrade_config=False, forcefully=self.is_downgrade)
            self.log.info("upgrade {0} to version {1} is completed".format(server.ip, upgrade_version))
            """ remove this line when bug MB-11807 fixed """
            if self.is_ubuntu:
                remote.start_server()
            """ remove end here """
            remote.disconnect()
            self.sleep(10)
            if self.is_linux:
                self.wait_node_restarted(server, wait_time=testconstants.NS_SERVER_TIMEOUT * 4, wait_if_warmup=True)
            else:
                self.wait_node_restarted(server, wait_time=testconstants.NS_SERVER_TIMEOUT * 10, wait_if_warmup=True, check_service=True)
            if not skip_init:
                self.rest.init_cluster(self.rest_settings.rest_username, self.rest_settings.rest_password)
            self.sleep(self.sleep_time)
            return o, e
        except Exception, e:
            print traceback.extract_stack()
            if queue is not None:
                queue.put(False)
                if not self.is_linux:
                    remote = RemoteMachineShellConnection(server)
                    output, error = remote.execute_command("cmd /c schtasks /Query /FO LIST /TN removeme /V")
                    remote.log_command_output(output, error)
                    output, error = remote.execute_command("cmd /c schtasks /Query /FO LIST /TN installme /V")
                    remote.log_command_output(output, error)
                    output, error = remote.execute_command("cmd /c schtasks /Query /FO LIST /TN upgrademe /V")
                    remote.log_command_output(output, error)
                    remote.disconnect()
                raise e
        if queue is not None:
            queue.put(True)

    def _async_update(self, upgrade_version, servers, queue=None, skip_init=False):
        self.log.info("\n\n*** servers {0} will be upgraded to {1} version ***\n".
                      format([server.ip for server in servers], upgrade_version))
        q = queue or self.queue
        upgrade_threads = []
        for server in servers:
            upgrade_thread = Thread(target=self._upgrade,
                                    name="upgrade_thread" + server.ip,
                                    args=(upgrade_version, server, q, skip_init))
            upgrade_threads.append(upgrade_thread)
            upgrade_thread.start()
        return upgrade_threads

    def _new_master(self, server):
        self.master = server
        self.rest = RestConnection(self.master)
        self.rest_helper = RestHelper(self.rest)

    def verification(self, servers, check_items=True):
        if self.use_hostnames:
            for server in servers:
                node_info = RestConnection(server).get_nodes_self()
                new_hostname = node_info.hostname
                self.assertEqual("%s:%s" % (server.hostname, server.port), new_hostname,
                                 "Hostname is incorrect for server %s. Settings are %s" % (server.ip, new_hostname))
        if self.master.ip != self.rest.ip or \
           self.master.ip == self.rest.ip and str(self.master.port) != str(self.rest.port):
            if self.port:
                self.master.port = self.port
            self.rest = RestConnection(self.master)
            self.rest_helper = RestHelper(self.rest)
        if self.port and self.port != '8091':
            settings = self.rest.get_cluster_settings()
            if settings and 'port' in settings:
                self.assertTrue(self.port == str(settings['port']),
                                'Expected cluster port is %s, but is %s' % (self.port, settings['port']))
        for bucket in self.buckets:
            if not self.rest_helper.bucket_exists(bucket.name):
                raise Exception("bucket: %s not found" % bucket.name)
        self.verify_cluster_stats(servers, max_verify=self.max_verify, \
                                  timeout=self.wait_timeout * 20, check_items=check_items)

        if self.ddocs:
            self.verify_all_queries()
        if "update_notifications" in self.input.test_params:
            if self.rest.get_notifications() != self.input.param("update_notifications", True):
                self.fail("update notifications settings wasn't saved")
        if "autofailover_timeout" in self.input.test_params:
            if self.rest.get_autofailover_settings().timeout != self.input.param("autofailover_timeout", None):
                self.fail("autofailover settings wasn't saved")
        if "autofailover_alerts" in self.input.test_params:
            alerts = self.rest.get_alerts_settings()
            if alerts["recipients"] != ['couchbase@localhost']:
                self.fail("recipients value wasn't saved")
            if alerts["sender"] != 'root@localhost':
                self.fail("sender value wasn't saved")
            if alerts["emailServer"]["user"] != 'user':
                self.fail("email_username value wasn't saved")
            if alerts["emailServer"]["pass"] != '':
                self.fail("email_password should be empty for security")
        if "autocompaction" in self.input.test_params:
            cluster_status = self.rest.cluster_status()
            if cluster_status["autoCompactionSettings"]["viewFragmentationThreshold"]\
                             ["percentage"] != self.input.param("autocompaction", 50):
                    self.log.info("Cluster status: {0}".format(cluster_status))
                    self.fail("autocompaction settings weren't saved")

    def verify_all_queries(self, queue=None):
        query = {"connectionTimeout" : 60000}
        expected_rows = self.num_items
        if self.max_verify:
            expected_rows = self.max_verify
            query["limit"] = expected_rows
        if self.input.param("wait_expiration", False):
            expected_rows = 0
        self.log.info("Start verify queries...")
        for bucket in self.buckets:
            for ddoc in self.ddocs:
                prefix = ("", "dev_")[ddoc.views[0].dev_view]
                try:
                    self.perform_verify_queries(len(ddoc.views), prefix, ddoc.name,
                                                              query, bucket=bucket,
                                                   wait_time=self.wait_timeout * 5,
                                                       expected_rows=expected_rows,
                                                                     retry_time=10)
                except Exception, e:
                    print e
                    if queue is not None:
                        queue.put(False)
                if queue is not None:
                    queue.put(True)

    def change_settings(self):
        status = True
        if "update_notifications" in self.input.test_params:
            status &= self.rest.update_notifications(str(self.input.param("update_notifications", 'true')).lower())
        if "autofailover_timeout" in self.input.test_params:
            status &= self.rest.update_autofailover_settings(True, self.input.param("autofailover_timeout", None))
        if "autofailover_alerts" in self.input.test_params:
            status &= self.rest.set_alerts_settings('couchbase@localhost', 'root@localhost', 'user', 'pwd')
        if "autocompaction" in self.input.test_params:
            tmp, _, _ = self.rest.set_auto_compaction(viewFragmntThresholdPercentage=
                                     self.input.param("autocompaction", 50))
            status &= tmp
            if not status:
                self.fail("some settings were not set correctly!")

    def warm_up_node(self, warmup_nodes=None):
        if not warmup_nodes:
            warmup_nodes = [self.servers[:self.nodes_init][-1], ]
        for warmup_node in warmup_nodes:
            shell = RemoteMachineShellConnection(warmup_node)
            shell.stop_couchbase()
            shell.disconnect()
        self.sleep(20)
        for warmup_node in warmup_nodes:
            shell = RemoteMachineShellConnection(warmup_node)
            shell.start_couchbase()
            shell.disconnect()
        ClusterOperationHelper.wait_for_ns_servers_or_assert(warmup_nodes, self)

    def start_index(self):
        if self.ddocs:
            query = {"connectionTimeout" : 60000}
            for bucket in self.buckets:
                for ddoc in self.ddocs:
                    prefix = ("", "dev_")[ddoc.views[0].dev_view]
                    self.perform_verify_queries(len(ddoc.views), prefix, ddoc.name, query, bucket=bucket)

    def failover(self):
        rest = RestConnection(self.master)
        nodes = rest.node_statuses()
        nodes = [node for node in nodes
                if node.ip != self.master.ip or str(node.port) != self.master.port]
        self.failover_node = nodes[0]
        rest.fail_over(self.failover_node.id)

    def add_back_failover(self):
        rest = RestConnection(self.master)
        rest.add_back_node(self.failover_node.id)

    def create_ddocs_and_views(self, queue=None):
        self.default_view = View(self.default_view_name, None, None)
        for bucket in self.buckets:
            if int(self.ddocs_num) > 0:
                for i in xrange(int(self.ddocs_num)):
                    views = self.make_default_views(self.default_view_name,
                            self.view_num, self.is_dev_ddoc, different_map=True)
                    ddoc = DesignDocument(self.default_view_name + str(i), views)
                    self.ddocs.append(ddoc)
                    for view in views:
                        try:
                            self.cluster.create_view(self.master, ddoc.name, view,
                                                                   bucket=bucket)
                        except Exception, e:
                            print e
                            if queue is not None:
                                queue.put(False)
                        if queue is not None:
                            queue.put(True)
            else:
                self.fail("Check param ddocs_num value")

    def delete_data(self, servers, paths_to_delete):
        for server in servers:
            shell = RemoteMachineShellConnection(server)
            for path in paths_to_delete:
                output, error = shell.execute_command("rm -rf {0}".format(path))
                shell.log_command_output(output, error)
                # shell._ssh_client.open_sftp().rmdir(path)
            shell.disconnect()

    def check_seqno(self, seqno_expected, comparator='=='):
        for bucket in self.buckets:
            if bucket.type == 'memcached':
                continue
            ready = BucketOperationHelper.wait_for_memcached(self.master,
                                                          bucket.name)
            self.assertTrue(ready, "wait_for_memcached failed")
            client = VBucketAwareMemcached(RestConnection(self.master), bucket)
            valid_keys, deleted_keys = bucket.kvs[1].key_set()
            for valid_key in valid_keys:
                try:
                    _, flags, exp, seqno, cas = client.memcached(valid_key).getMeta(valid_key)
                except MemcachedError, e:
                    print e
                    client.reset(RestConnection(self.master))
                    _, flags, exp, seqno, cas = client.memcached(valid_key).getMeta(valid_key)
                self.assertTrue((comparator == '==' and seqno == seqno_expected) or
                                (comparator == '>=' and seqno >= seqno_expected),
                                msg="seqno {0} !{1} {2} for key:{3}".
                                format(seqno, comparator, seqno_expected, valid_key))
            client.done()

    def force_reinstall(self, servers):
        for server in servers:
            try:
                remote = RemoteMachineShellConnection(server)
                appropriate_build = self._get_build(server, self.initial_version, remote)
                self.assertTrue(appropriate_build.url, msg="unable to find build {0}".format(self.initial_version))
                remote.download_build(appropriate_build)
                remote.install_server(appropriate_build, force=True)
                self.log.info("upgrade {0} to version {1} is completed".format(server.ip, self.initial_version))
                remote.disconnect()
                self.sleep(10)
                if self.is_linux:
                    self.wait_node_restarted(server, wait_time=testconstants.NS_SERVER_TIMEOUT * 4, wait_if_warmup=True)
                else:
                    self.wait_node_restarted(server, wait_time=testconstants.NS_SERVER_TIMEOUT * 10, wait_if_warmup=True, check_service=True)
            except Exception, e:
                print traceback.extract_stack()
                if queue is not None:
                    queue.put(False)
                    if not self.is_linux:
                        remote = RemoteMachineShellConnection(server)
                        output, error = remote.execute_command("cmd /c schtasks /Query /FO LIST /TN installme /V")
                        remote.log_command_output(output, error)
                        remote.disconnect()
                    raise e

    def _verify_vbucket_nums_for_swap(self, old_vbs, new_vbs):
        out_servers = set(old_vbs) - set(new_vbs)
        in_servers = set(new_vbs) - set(old_vbs)
        self.assertEqual(len(out_servers), len(in_servers),
                        "Seems like it wasn't swap rebalance. Out %s, in %s" % (
                                                len(out_servers),len(in_servers)))
        for vb_type in ["active_vb", "replica_vb"]:
            self.log.info("Checking %s on nodes that remain in cluster..." % vb_type)
            for server, stats in old_vbs.iteritems():
                if server in new_vbs:
                    self.assertTrue(sorted(stats[vb_type]) == sorted(new_vbs[server][vb_type]),
                    "Server %s Seems like %s vbuckets were shuffled, old vbs is %s, new are %s" %(
                                    server.ip, vb_type, stats[vb_type], new_vbs[server][vb_type]))
            self.log.info("%s vbuckets were not suffled" % vb_type)
            self.log.info("Checking in-out nodes...")
            vbs_servs_out = vbs_servs_in = []
            for srv, stat in old_vbs.iteritems():
                if srv in out_servers:
                    vbs_servs_out.extend(stat[vb_type])
            for srv, stat in new_vbs.iteritems():
                if srv in in_servers:
                    vbs_servs_in.extend(stat[vb_type])
            self.assertTrue(sorted(vbs_servs_out) == sorted(vbs_servs_in),
                            "%s vbuckets seem to be suffled" % vb_type)

    def monitor_dcp_rebalance(self):
        """ released_upgrade_version """
        upgrade_version = ""
        if self.input.param('released_upgrade_version', None) is not None:
            upgrade_version = self.input.param('released_upgrade_version', None)[:5]
        else:
            upgrade_version = self.input.param('upgrade_version', '')[:5]
        if self.input.param('initial_version', '')[:5] in COUCHBASE_VERSION_2 and \
                                       (upgrade_version in COUCHBASE_VERSION_3 or \
                                        upgrade_version in SHERLOCK_VERSION):
            if int(self.initial_vbuckets) >= 512:
                if self.master.ip != self.rest.ip or \
                   self.master.ip == self.rest.ip and \
                   str(self.master.port) != str(self.rest.port):
                    if self.port:
                        self.master.port = self.port
                    self.rest = RestConnection(self.master)
                    self.rest_helper = RestHelper(self.rest)
                if self.rest._rebalance_progress_status() == 'running':
                    self.log.info("Start monitoring DCP upgrade from {0} to {1}"\
                           .format(self.input.param('initial_version', '')[:5], \
                                                                 upgrade_version))
                    status = self.rest.monitorRebalance()
                    if status:
                        self.log.info("Done DCP rebalance upgrade!")
                    else:
                        self.fail("Failed DCP rebalance upgrade")
                elif any ("DCP upgrade completed successfully.\n" \
                                    in d.values() for d in self.rest.get_logs(10)):
                    self.log.info("DCP upgrade is completed")
                else:
                    self.fail("DCP reabalance upgrade is not running")
            else:
                self.fail("Need vbuckets setting >= 256 for upgrade from 2.x.x to 3+")
        else:
            if self.master.ip != self.rest.ip:
                self.rest = RestConnection(self.master)
                self.rest_helper = RestHelper(self.rest)
            self.log.info("No need to do DCP rebalance upgrade")

    def _offline_upgrade(self, skip_init=False):
        try:
            self.log.info("offline_upgrade")
            stoped_nodes = self.servers[:self.nodes_init]
            for upgrade_version in self.upgrade_versions:
                self.sleep(self.sleep_time, "Pre-setup of old version is done. "
                        " Wait for upgrade to {0} version".format(upgrade_version))
                for server in stoped_nodes:
                    remote = RemoteMachineShellConnection(server)
                    remote.stop_server()
                    remote.disconnect()
                self.sleep(self.sleep_time)
                upgrade_threads = self._async_update(upgrade_version, stoped_nodes,
                                                     None, skip_init)
                for upgrade_thread in upgrade_threads:
                    upgrade_thread.join()
                success_upgrade = True
                while not self.queue.empty():
                    success_upgrade &= self.queue.get()
                if not success_upgrade:
                    self.fail("Upgrade failed!")
                self.dcp_rebalance_in_offline_upgrade_from_version2()
            """ set install cb version to upgrade version after done upgrade """
            self.initial_version = self.upgrade_versions[0]
        except Exception, ex:
            self.log.info(ex)
            raise

    def dcp_rebalance_in_offline_upgrade_from_version2(self):
        if self.input.param('initial_version', '')[:5] in COUCHBASE_VERSION_2 and \
           (self.input.param('upgrade_version', '')[:5] in COUCHBASE_VERSION_3 or \
            self.input.param('upgrade_version', '')[:5] in SHERLOCK_VERSION) and \
            self.input.param('num_stoped_nodes', self.nodes_init) >= self.nodes_init:
            otpNodes = []
            nodes = self.rest.node_statuses()
            for node in nodes:
                otpNodes.append(node.id)
            self.log.info("Start DCP rebalance after complete offline upgrade from {0} to {1}"\
                           .format(self.input.param('initial_version', '')[:5], \
                                   self.input.param('upgrade_version', '')[:5]))
            self.rest.rebalance(otpNodes, [])
            """ verify DCP upgrade in 3.0.0 version """
            self.monitor_dcp_rebalance()
        else:
            self.log.info("No need to do DCP rebalance upgrade")

    def generate_map_nodes_out_dist_upgrade(self, nodes_out_dist):
        self.nodes_out_dist = nodes_out_dist
        self.generate_map_nodes_out_dist()

    """ subdoc base test starts here """
    def generate_json_for_nesting(self):
        json = {
            "not_tested_integer_zero":0,
            "not_tested_integer_big":1038383839293939383938393,
            "not_tested_double_zero":0.0,
            "not_tested_integer":1,
            "not_tested_integer_negative":-1,
            "not_tested_double":1.1,
            "not_tested_double_negative":-1.1,
            "not_tested_float":2.99792458e8,
            "not_tested_float_negative":-2.99792458e8,
            "not_tested_array_numbers_integer" : [1,2,3,4,5],
            "not_tested_array_numbers_double" : [1.1,2.2,3.3,4.4,5.5],
            "not_tested_array_numbers_float" : [2.99792458e8,2.99792458e8,2.99792458e8],
            "not_tested_array_numbers_mix" : [0,2.99792458e8,1.1],
            "not_tested_array_array_mix" : [[2.99792458e8,2.99792458e8,2.99792458e8],[0,2.99792458e8,1.1],[],[0, 0, 0]],
            "not_tested_simple_string_lower_case":"abcdefghijklmnoprestuvxyz",
            "not_tested_simple_string_upper_case":"ABCDEFGHIJKLMNOPQRSTUVWXZYZ",
            "not_tested_simple_string_empty":"",
            "not_tested_simple_string_datetime":"2012-10-03 15:35:46.461491",
            "not_tested_simple_string_special_chars":"_-+!#@$%&*(){}\][;.,<>?/",
            "not_test_json" : { "not_to_bes_tested_string_field1": "not_to_bes_tested_string"}
        }
        return json

    def generate_simple_data_null(self):
        json = {
            "null":None,
            "null_array":[None, None]
        }
        return json

    def generate_simple_data_boolean(self):
        json = {
            "true":True,
            "false":False,
            "array":[True, False, True, False]
        }
        return json

    def generate_nested_json(self):
        json_data = self.generate_json_for_nesting()
        json = {
            "json_1": { "json_2": {"json_3":json_data}}
        }
        return json

    def generate_simple_data_numbers(self):
        json = {
            "integer_zero":0,
            "integer_big":1038383839293939383938393,
            "double_zero":0.0,
            "integer":1,
            "integer_negative":-1,
            "double":1.1,
            "double_negative":-1.1,
            "float":2.99792458e8,
            "float_negative":-2.99792458e8,
        }
        return json

    def subdoc_direct_client(self, server, bucket, timeout=30):
        # CREATE SDK CLIENT
        if self.use_sdk_client:
            try:
                from sdk_client import SDKClient
                scheme = "couchbase"
                host=self.master.ip
                if self.master.ip == "127.0.0.1":
                    scheme = "http"
                    host="{0}:{1}".format(self.master.ip,self.master.port)
                return SDKClient(scheme=scheme,hosts = [host], bucket = bucket.name)
            except Exception, ex:
                self.log.info("cannot load sdk client due to error {0}".format(str(ex)))
        # USE MC BIN CLIENT WHEN NOT USING SDK CLIENT
        return self.direct_mc_bin_client(server, bucket, timeout= timeout)
    """ subdoc base test ends here """

    def construct_plan_params(self):
        plan_params = {}
        plan_params['numReplicas'] = 0
        if self.index_replicas:
            plan_params['numReplicas'] = self.index_replicas
        if self.partitions_per_pindex:
            plan_params['maxPartitionsPerPIndex'] = self.partitions_per_pindex
        return plan_params

    def construct_cbft_query_json(self, query, fields=None, timeout=None):
        max_matches = TestInputSingleton.input.param("query_max_matches", 10000000)
        query_json = QUERY.JSON
        # query is a unicode dict
        query_json['query'] = query
        query_json['indexName'] = self.name
        if max_matches:
            query_json['size'] = int(max_matches)
        if timeout:
            query_json['timeout'] = int(timeout)
        if fields:
            query_json['fields'] = fields
        return query_json

    def generate_ops_docs(self, num_items, start=0):
        try:
            json_generator = JsonGenerator()
            if self.dataset == "simple":
                return self.generate_ops(num_items, start, json_generator.generate_docs_simple)
            if self.dataset == "sales":
                return self.generate_ops(num_items, start, json_generator.generate_docs_sales)
            if self.dataset == "employee" or self.dataset == "default":
                return self.generate_ops(num_items, start, json_generator.generate_docs_employee)
            if self.dataset == "sabre":
                return self.generate_ops(num_items, start, json_generator.generate_docs_sabre)
            if self.dataset == "bigdata":
                return self.generate_ops(num_items, start, json_generator.generate_docs_bigdata)
            if self.dataset == "array":
                return self.generate_ops(num_items, start, json_generator.generate_docs_array)
        except Exception, ex:
            self.log.info(ex)
            self.fail("There is no dataset %s, please enter a valid one" % self.dataset)

    def generate_ops(self, docs_per_day, start=0, method=None):
        gen_docs_map = {}
        for key in self.ops_dist_map.keys():
            isShuffle = False
            if key == "update":
                isShuffle = True
            if self.dataset != "bigdata":
                gen_docs_map[key] = method(docs_per_day=self.ops_dist_map[key]["end"],
                    start=self.ops_dist_map[key]["start"])
            else:
                gen_docs_map[key] = method(value_size=self.value_size,
                    end=self.ops_dist_map[key]["end"],
                    start=self.ops_dist_map[key]["start"])
        return gen_docs_map

    def _convert_server_map(self, servers):
        map = {}
        for server in servers:
            key  = self._gen_server_key(server)
            map[key] = server
        return map

    def _gen_server_key(self, server):
        return "{0}:{1}".format(server.ip, server.port)