PageRenderTime 2126ms CodeModel.GetById 36ms RepoModel.GetById 1ms app.codeStats 0ms

/test/e2e/storage/vsphere/vsphere_utils.go

https://bitbucket.org/Jake-Qu/kubernetes-mirror
Go | 822 lines | 659 code | 81 blank | 82 comment | 102 complexity | 0a9683c906a7ecf699512f3f08d30c30 MD5 | raw file
Possible License(s): MIT, MPL-2.0-no-copyleft-exception, 0BSD, CC0-1.0, BSD-2-Clause, Apache-2.0, BSD-3-Clause
  1. /*
  2. Copyright 2017 The Kubernetes Authors.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. package vsphere
  14. import (
  15. "context"
  16. "fmt"
  17. "math/rand"
  18. "path/filepath"
  19. "regexp"
  20. "strings"
  21. "time"
  22. "github.com/golang/glog"
  23. . "github.com/onsi/ginkgo"
  24. . "github.com/onsi/gomega"
  25. "github.com/vmware/govmomi/find"
  26. "github.com/vmware/govmomi/object"
  27. "github.com/vmware/govmomi/vim25/mo"
  28. vim25types "github.com/vmware/govmomi/vim25/types"
  29. vimtypes "github.com/vmware/govmomi/vim25/types"
  30. "k8s.io/api/core/v1"
  31. storage "k8s.io/api/storage/v1"
  32. "k8s.io/apimachinery/pkg/api/resource"
  33. metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  34. "k8s.io/apimachinery/pkg/util/uuid"
  35. "k8s.io/apimachinery/pkg/util/wait"
  36. clientset "k8s.io/client-go/kubernetes"
  37. "k8s.io/kubernetes/pkg/volume/util"
  38. "k8s.io/kubernetes/test/e2e/framework"
  39. "k8s.io/kubernetes/test/e2e/storage/utils"
  40. )
  41. const (
  42. volumesPerNode = 55
  43. storageclass1 = "sc-default"
  44. storageclass2 = "sc-vsan"
  45. storageclass3 = "sc-spbm"
  46. storageclass4 = "sc-user-specified-ds"
  47. DummyDiskName = "kube-dummyDisk.vmdk"
  48. ProviderPrefix = "vsphere://"
  49. )
  50. // volumeState represents the state of a volume.
  51. type volumeState int32
  52. const (
  53. volumeStateDetached volumeState = 1
  54. volumeStateAttached volumeState = 2
  55. )
  56. // Wait until vsphere volumes are detached from the list of nodes or time out after 5 minutes
  57. func waitForVSphereDisksToDetach(nodeVolumes map[string][]string) error {
  58. var (
  59. err error
  60. disksAttached = true
  61. detachTimeout = 5 * time.Minute
  62. detachPollTime = 10 * time.Second
  63. )
  64. err = wait.Poll(detachPollTime, detachTimeout, func() (bool, error) {
  65. attachedResult, err := disksAreAttached(nodeVolumes)
  66. if err != nil {
  67. return false, err
  68. }
  69. for nodeName, nodeVolumes := range attachedResult {
  70. for volumePath, attached := range nodeVolumes {
  71. if attached {
  72. framework.Logf("Waiting for volumes %q to detach from %q.", volumePath, string(nodeName))
  73. return false, nil
  74. }
  75. }
  76. }
  77. disksAttached = false
  78. framework.Logf("Volume are successfully detached from all the nodes: %+v", nodeVolumes)
  79. return true, nil
  80. })
  81. if err != nil {
  82. return err
  83. }
  84. if disksAttached {
  85. return fmt.Errorf("Gave up waiting for volumes to detach after %v", detachTimeout)
  86. }
  87. return nil
  88. }
  89. // Wait until vsphere vmdk moves to expected state on the given node, or time out after 6 minutes
  90. func waitForVSphereDiskStatus(volumePath string, nodeName string, expectedState volumeState) error {
  91. var (
  92. err error
  93. diskAttached bool
  94. currentState volumeState
  95. timeout = 6 * time.Minute
  96. pollTime = 10 * time.Second
  97. )
  98. var attachedState = map[bool]volumeState{
  99. true: volumeStateAttached,
  100. false: volumeStateDetached,
  101. }
  102. var attachedStateMsg = map[volumeState]string{
  103. volumeStateAttached: "attached to",
  104. volumeStateDetached: "detached from",
  105. }
  106. err = wait.Poll(pollTime, timeout, func() (bool, error) {
  107. diskAttached, err = diskIsAttached(volumePath, nodeName)
  108. if err != nil {
  109. return true, err
  110. }
  111. currentState = attachedState[diskAttached]
  112. if currentState == expectedState {
  113. framework.Logf("Volume %q has successfully %s %q", volumePath, attachedStateMsg[currentState], nodeName)
  114. return true, nil
  115. }
  116. framework.Logf("Waiting for Volume %q to be %s %q.", volumePath, attachedStateMsg[expectedState], nodeName)
  117. return false, nil
  118. })
  119. if err != nil {
  120. return err
  121. }
  122. if currentState != expectedState {
  123. err = fmt.Errorf("Gave up waiting for Volume %q to be %s %q after %v", volumePath, attachedStateMsg[expectedState], nodeName, timeout)
  124. }
  125. return err
  126. }
  127. // Wait until vsphere vmdk is attached from the given node or time out after 6 minutes
  128. func waitForVSphereDiskToAttach(volumePath string, nodeName string) error {
  129. return waitForVSphereDiskStatus(volumePath, nodeName, volumeStateAttached)
  130. }
  131. // Wait until vsphere vmdk is detached from the given node or time out after 6 minutes
  132. func waitForVSphereDiskToDetach(volumePath string, nodeName string) error {
  133. return waitForVSphereDiskStatus(volumePath, nodeName, volumeStateDetached)
  134. }
  135. // function to create vsphere volume spec with given VMDK volume path, Reclaim Policy and labels
  136. func getVSpherePersistentVolumeSpec(volumePath string, persistentVolumeReclaimPolicy v1.PersistentVolumeReclaimPolicy, labels map[string]string) *v1.PersistentVolume {
  137. var (
  138. pvConfig framework.PersistentVolumeConfig
  139. pv *v1.PersistentVolume
  140. claimRef *v1.ObjectReference
  141. )
  142. pvConfig = framework.PersistentVolumeConfig{
  143. NamePrefix: "vspherepv-",
  144. PVSource: v1.PersistentVolumeSource{
  145. VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
  146. VolumePath: volumePath,
  147. FSType: "ext4",
  148. },
  149. },
  150. Prebind: nil,
  151. }
  152. pv = &v1.PersistentVolume{
  153. ObjectMeta: metav1.ObjectMeta{
  154. GenerateName: pvConfig.NamePrefix,
  155. Annotations: map[string]string{
  156. util.VolumeGidAnnotationKey: "777",
  157. },
  158. },
  159. Spec: v1.PersistentVolumeSpec{
  160. PersistentVolumeReclaimPolicy: persistentVolumeReclaimPolicy,
  161. Capacity: v1.ResourceList{
  162. v1.ResourceName(v1.ResourceStorage): resource.MustParse("2Gi"),
  163. },
  164. PersistentVolumeSource: pvConfig.PVSource,
  165. AccessModes: []v1.PersistentVolumeAccessMode{
  166. v1.ReadWriteOnce,
  167. },
  168. ClaimRef: claimRef,
  169. },
  170. }
  171. if labels != nil {
  172. pv.Labels = labels
  173. }
  174. return pv
  175. }
  176. // function to get vsphere persistent volume spec with given selector labels.
  177. func getVSpherePersistentVolumeClaimSpec(namespace string, labels map[string]string) *v1.PersistentVolumeClaim {
  178. var (
  179. pvc *v1.PersistentVolumeClaim
  180. )
  181. pvc = &v1.PersistentVolumeClaim{
  182. ObjectMeta: metav1.ObjectMeta{
  183. GenerateName: "pvc-",
  184. Namespace: namespace,
  185. },
  186. Spec: v1.PersistentVolumeClaimSpec{
  187. AccessModes: []v1.PersistentVolumeAccessMode{
  188. v1.ReadWriteOnce,
  189. },
  190. Resources: v1.ResourceRequirements{
  191. Requests: v1.ResourceList{
  192. v1.ResourceName(v1.ResourceStorage): resource.MustParse("2Gi"),
  193. },
  194. },
  195. },
  196. }
  197. if labels != nil {
  198. pvc.Spec.Selector = &metav1.LabelSelector{MatchLabels: labels}
  199. }
  200. return pvc
  201. }
  202. // function to write content to the volume backed by given PVC
  203. func writeContentToVSpherePV(client clientset.Interface, pvc *v1.PersistentVolumeClaim, expectedContent string) {
  204. utils.RunInPodWithVolume(client, pvc.Namespace, pvc.Name, "echo "+expectedContent+" > /mnt/test/data")
  205. framework.Logf("Done with writing content to volume")
  206. }
  207. // function to verify content is matching on the volume backed for given PVC
  208. func verifyContentOfVSpherePV(client clientset.Interface, pvc *v1.PersistentVolumeClaim, expectedContent string) {
  209. utils.RunInPodWithVolume(client, pvc.Namespace, pvc.Name, "grep '"+expectedContent+"' /mnt/test/data")
  210. framework.Logf("Successfully verified content of the volume")
  211. }
  212. func getVSphereStorageClassSpec(name string, scParameters map[string]string) *storage.StorageClass {
  213. var sc *storage.StorageClass
  214. sc = &storage.StorageClass{
  215. TypeMeta: metav1.TypeMeta{
  216. Kind: "StorageClass",
  217. },
  218. ObjectMeta: metav1.ObjectMeta{
  219. Name: name,
  220. },
  221. Provisioner: "kubernetes.io/vsphere-volume",
  222. }
  223. if scParameters != nil {
  224. sc.Parameters = scParameters
  225. }
  226. return sc
  227. }
  228. func getVSphereClaimSpecWithStorageClass(ns string, diskSize string, storageclass *storage.StorageClass) *v1.PersistentVolumeClaim {
  229. claim := &v1.PersistentVolumeClaim{
  230. ObjectMeta: metav1.ObjectMeta{
  231. GenerateName: "pvc-",
  232. Namespace: ns,
  233. },
  234. Spec: v1.PersistentVolumeClaimSpec{
  235. AccessModes: []v1.PersistentVolumeAccessMode{
  236. v1.ReadWriteOnce,
  237. },
  238. Resources: v1.ResourceRequirements{
  239. Requests: v1.ResourceList{
  240. v1.ResourceName(v1.ResourceStorage): resource.MustParse(diskSize),
  241. },
  242. },
  243. StorageClassName: &(storageclass.Name),
  244. },
  245. }
  246. return claim
  247. }
  248. // func to get pod spec with given volume claim, node selector labels and command
  249. func getVSpherePodSpecWithClaim(claimName string, nodeSelectorKV map[string]string, command string) *v1.Pod {
  250. pod := &v1.Pod{
  251. TypeMeta: metav1.TypeMeta{
  252. Kind: "Pod",
  253. APIVersion: "v1",
  254. },
  255. ObjectMeta: metav1.ObjectMeta{
  256. GenerateName: "pod-pvc-",
  257. },
  258. Spec: v1.PodSpec{
  259. Containers: []v1.Container{
  260. {
  261. Name: "volume-tester",
  262. Image: "busybox",
  263. Command: []string{"/bin/sh"},
  264. Args: []string{"-c", command},
  265. VolumeMounts: []v1.VolumeMount{
  266. {
  267. Name: "my-volume",
  268. MountPath: "/mnt/test",
  269. },
  270. },
  271. },
  272. },
  273. RestartPolicy: v1.RestartPolicyNever,
  274. Volumes: []v1.Volume{
  275. {
  276. Name: "my-volume",
  277. VolumeSource: v1.VolumeSource{
  278. PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
  279. ClaimName: claimName,
  280. ReadOnly: false,
  281. },
  282. },
  283. },
  284. },
  285. },
  286. }
  287. if nodeSelectorKV != nil {
  288. pod.Spec.NodeSelector = nodeSelectorKV
  289. }
  290. return pod
  291. }
  292. // func to get pod spec with given volume paths, node selector lables and container commands
  293. func getVSpherePodSpecWithVolumePaths(volumePaths []string, keyValuelabel map[string]string, commands []string) *v1.Pod {
  294. var volumeMounts []v1.VolumeMount
  295. var volumes []v1.Volume
  296. for index, volumePath := range volumePaths {
  297. name := fmt.Sprintf("volume%v", index+1)
  298. volumeMounts = append(volumeMounts, v1.VolumeMount{Name: name, MountPath: "/mnt/" + name})
  299. vsphereVolume := new(v1.VsphereVirtualDiskVolumeSource)
  300. vsphereVolume.VolumePath = volumePath
  301. vsphereVolume.FSType = "ext4"
  302. volumes = append(volumes, v1.Volume{Name: name})
  303. volumes[index].VolumeSource.VsphereVolume = vsphereVolume
  304. }
  305. if commands == nil || len(commands) == 0 {
  306. commands = []string{
  307. "/bin/sh",
  308. "-c",
  309. "while true; do sleep 2; done",
  310. }
  311. }
  312. pod := &v1.Pod{
  313. TypeMeta: metav1.TypeMeta{
  314. Kind: "Pod",
  315. APIVersion: "v1",
  316. },
  317. ObjectMeta: metav1.ObjectMeta{
  318. GenerateName: "vsphere-e2e-",
  319. },
  320. Spec: v1.PodSpec{
  321. Containers: []v1.Container{
  322. {
  323. Name: "vsphere-e2e-container-" + string(uuid.NewUUID()),
  324. Image: "busybox",
  325. Command: commands,
  326. VolumeMounts: volumeMounts,
  327. },
  328. },
  329. RestartPolicy: v1.RestartPolicyNever,
  330. Volumes: volumes,
  331. },
  332. }
  333. if keyValuelabel != nil {
  334. pod.Spec.NodeSelector = keyValuelabel
  335. }
  336. return pod
  337. }
  338. func verifyFilesExistOnVSphereVolume(namespace string, podName string, filePaths ...string) {
  339. for _, filePath := range filePaths {
  340. _, err := framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", namespace), podName, "--", "/bin/ls", filePath)
  341. Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to verify file: %q on the pod: %q", filePath, podName))
  342. }
  343. }
  344. func createEmptyFilesOnVSphereVolume(namespace string, podName string, filePaths []string) {
  345. for _, filePath := range filePaths {
  346. err := framework.CreateEmptyFileOnPod(namespace, podName, filePath)
  347. Expect(err).NotTo(HaveOccurred())
  348. }
  349. }
  350. // verify volumes are attached to the node and are accessible in pod
  351. func verifyVSphereVolumesAccessible(c clientset.Interface, pod *v1.Pod, persistentvolumes []*v1.PersistentVolume) {
  352. nodeName := pod.Spec.NodeName
  353. namespace := pod.Namespace
  354. for index, pv := range persistentvolumes {
  355. // Verify disks are attached to the node
  356. isAttached, err := diskIsAttached(pv.Spec.VsphereVolume.VolumePath, nodeName)
  357. Expect(err).NotTo(HaveOccurred())
  358. Expect(isAttached).To(BeTrue(), fmt.Sprintf("disk %v is not attached with the node", pv.Spec.VsphereVolume.VolumePath))
  359. // Verify Volumes are accessible
  360. filepath := filepath.Join("/mnt/", fmt.Sprintf("volume%v", index+1), "/emptyFile.txt")
  361. _, err = framework.LookForStringInPodExec(namespace, pod.Name, []string{"/bin/touch", filepath}, "", time.Minute)
  362. Expect(err).NotTo(HaveOccurred())
  363. }
  364. }
  365. // Get vSphere Volume Path from PVC
  366. func getvSphereVolumePathFromClaim(client clientset.Interface, namespace string, claimName string) string {
  367. pvclaim, err := client.CoreV1().PersistentVolumeClaims(namespace).Get(claimName, metav1.GetOptions{})
  368. Expect(err).NotTo(HaveOccurred())
  369. pv, err := client.CoreV1().PersistentVolumes().Get(pvclaim.Spec.VolumeName, metav1.GetOptions{})
  370. Expect(err).NotTo(HaveOccurred())
  371. return pv.Spec.VsphereVolume.VolumePath
  372. }
  373. // Get canonical volume path for volume Path.
  374. // Example1: The canonical path for volume path - [vsanDatastore] kubevols/volume.vmdk will be [vsanDatastore] 25d8b159-948c-4b73-e499-02001ad1b044/volume.vmdk
  375. // Example2: The canonical path for volume path - [vsanDatastore] 25d8b159-948c-4b73-e499-02001ad1b044/volume.vmdk will be same as volume Path.
  376. func getCanonicalVolumePath(ctx context.Context, dc *object.Datacenter, volumePath string) (string, error) {
  377. var folderID string
  378. canonicalVolumePath := volumePath
  379. dsPathObj, err := getDatastorePathObjFromVMDiskPath(volumePath)
  380. if err != nil {
  381. return "", err
  382. }
  383. dsPath := strings.Split(strings.TrimSpace(dsPathObj.Path), "/")
  384. if len(dsPath) <= 1 {
  385. return canonicalVolumePath, nil
  386. }
  387. datastore := dsPathObj.Datastore
  388. dsFolder := dsPath[0]
  389. // Get the datastore folder ID if datastore or folder doesn't exist in datastoreFolderIDMap
  390. if !isValidUUID(dsFolder) {
  391. dummyDiskVolPath := "[" + datastore + "] " + dsFolder + "/" + DummyDiskName
  392. // Querying a non-existent dummy disk on the datastore folder.
  393. // It would fail and return an folder ID in the error message.
  394. _, err := getVirtualDiskPage83Data(ctx, dc, dummyDiskVolPath)
  395. if err != nil {
  396. re := regexp.MustCompile("File (.*?) was not found")
  397. match := re.FindStringSubmatch(err.Error())
  398. canonicalVolumePath = match[1]
  399. }
  400. }
  401. diskPath := getPathFromVMDiskPath(canonicalVolumePath)
  402. if diskPath == "" {
  403. return "", fmt.Errorf("Failed to parse canonicalVolumePath: %s in getcanonicalVolumePath method", canonicalVolumePath)
  404. }
  405. folderID = strings.Split(strings.TrimSpace(diskPath), "/")[0]
  406. canonicalVolumePath = strings.Replace(volumePath, dsFolder, folderID, 1)
  407. return canonicalVolumePath, nil
  408. }
  409. // getPathFromVMDiskPath retrieves the path from VM Disk Path.
  410. // Example: For vmDiskPath - [vsanDatastore] kubevols/volume.vmdk, the path is kubevols/volume.vmdk
  411. func getPathFromVMDiskPath(vmDiskPath string) string {
  412. datastorePathObj := new(object.DatastorePath)
  413. isSuccess := datastorePathObj.FromString(vmDiskPath)
  414. if !isSuccess {
  415. framework.Logf("Failed to parse vmDiskPath: %s", vmDiskPath)
  416. return ""
  417. }
  418. return datastorePathObj.Path
  419. }
  420. //getDatastorePathObjFromVMDiskPath gets the datastorePathObj from VM disk path.
  421. func getDatastorePathObjFromVMDiskPath(vmDiskPath string) (*object.DatastorePath, error) {
  422. datastorePathObj := new(object.DatastorePath)
  423. isSuccess := datastorePathObj.FromString(vmDiskPath)
  424. if !isSuccess {
  425. framework.Logf("Failed to parse volPath: %s", vmDiskPath)
  426. return nil, fmt.Errorf("Failed to parse volPath: %s", vmDiskPath)
  427. }
  428. return datastorePathObj, nil
  429. }
  430. // getVirtualDiskPage83Data gets the virtual disk UUID by diskPath
  431. func getVirtualDiskPage83Data(ctx context.Context, dc *object.Datacenter, diskPath string) (string, error) {
  432. if len(diskPath) > 0 && filepath.Ext(diskPath) != ".vmdk" {
  433. diskPath += ".vmdk"
  434. }
  435. vdm := object.NewVirtualDiskManager(dc.Client())
  436. // Returns uuid of vmdk virtual disk
  437. diskUUID, err := vdm.QueryVirtualDiskUuid(ctx, diskPath, dc)
  438. if err != nil {
  439. glog.Warningf("QueryVirtualDiskUuid failed for diskPath: %q. err: %+v", diskPath, err)
  440. return "", err
  441. }
  442. diskUUID = formatVirtualDiskUUID(diskUUID)
  443. return diskUUID, nil
  444. }
  445. // formatVirtualDiskUUID removes any spaces and hyphens in UUID
  446. // Example UUID input is 42375390-71f9-43a3-a770-56803bcd7baa and output after format is 4237539071f943a3a77056803bcd7baa
  447. func formatVirtualDiskUUID(uuid string) string {
  448. uuidwithNoSpace := strings.Replace(uuid, " ", "", -1)
  449. uuidWithNoHypens := strings.Replace(uuidwithNoSpace, "-", "", -1)
  450. return strings.ToLower(uuidWithNoHypens)
  451. }
  452. //isValidUUID checks if the string is a valid UUID.
  453. func isValidUUID(uuid string) bool {
  454. r := regexp.MustCompile("^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$")
  455. return r.MatchString(uuid)
  456. }
  457. // removeStorageClusterORFolderNameFromVDiskPath removes the cluster or folder path from the vDiskPath
  458. // for vDiskPath [DatastoreCluster/sharedVmfs-0] kubevols/e2e-vmdk-1234.vmdk, return value is [sharedVmfs-0] kubevols/e2e-vmdk-1234.vmdk
  459. // for vDiskPath [sharedVmfs-0] kubevols/e2e-vmdk-1234.vmdk, return value remains same [sharedVmfs-0] kubevols/e2e-vmdk-1234.vmdk
  460. func removeStorageClusterORFolderNameFromVDiskPath(vDiskPath string) string {
  461. datastore := regexp.MustCompile("\\[(.*?)\\]").FindStringSubmatch(vDiskPath)[1]
  462. if filepath.Base(datastore) != datastore {
  463. vDiskPath = strings.Replace(vDiskPath, datastore, filepath.Base(datastore), 1)
  464. }
  465. return vDiskPath
  466. }
  467. // getVirtualDeviceByPath gets the virtual device by path
  468. func getVirtualDeviceByPath(ctx context.Context, vm *object.VirtualMachine, diskPath string) (vim25types.BaseVirtualDevice, error) {
  469. vmDevices, err := vm.Device(ctx)
  470. if err != nil {
  471. framework.Logf("Failed to get the devices for VM: %q. err: %+v", vm.InventoryPath, err)
  472. return nil, err
  473. }
  474. // filter vm devices to retrieve device for the given vmdk file identified by disk path
  475. for _, device := range vmDevices {
  476. if vmDevices.TypeName(device) == "VirtualDisk" {
  477. virtualDevice := device.GetVirtualDevice()
  478. if backing, ok := virtualDevice.Backing.(*vim25types.VirtualDiskFlatVer2BackingInfo); ok {
  479. if matchVirtualDiskAndVolPath(backing.FileName, diskPath) {
  480. framework.Logf("Found VirtualDisk backing with filename %q for diskPath %q", backing.FileName, diskPath)
  481. return device, nil
  482. } else {
  483. framework.Logf("VirtualDisk backing filename %q does not match with diskPath %q", backing.FileName, diskPath)
  484. }
  485. }
  486. }
  487. }
  488. return nil, nil
  489. }
  490. func matchVirtualDiskAndVolPath(diskPath, volPath string) bool {
  491. fileExt := ".vmdk"
  492. diskPath = strings.TrimSuffix(diskPath, fileExt)
  493. volPath = strings.TrimSuffix(volPath, fileExt)
  494. return diskPath == volPath
  495. }
  496. // convertVolPathsToDevicePaths removes cluster or folder path from volPaths and convert to canonicalPath
  497. func convertVolPathsToDevicePaths(ctx context.Context, nodeVolumes map[string][]string) (map[string][]string, error) {
  498. vmVolumes := make(map[string][]string)
  499. for nodeName, volPaths := range nodeVolumes {
  500. nodeInfo := TestContext.NodeMapper.GetNodeInfo(nodeName)
  501. datacenter := nodeInfo.VSphere.GetDatacenterFromObjectReference(ctx, nodeInfo.DataCenterRef)
  502. for i, volPath := range volPaths {
  503. deviceVolPath, err := convertVolPathToDevicePath(ctx, datacenter, volPath)
  504. if err != nil {
  505. framework.Logf("Failed to convert vsphere volume path %s to device path for volume %s. err: %+v", volPath, deviceVolPath, err)
  506. return nil, err
  507. }
  508. volPaths[i] = deviceVolPath
  509. }
  510. vmVolumes[nodeName] = volPaths
  511. }
  512. return vmVolumes, nil
  513. }
  514. // convertVolPathToDevicePath takes volPath and returns canonical volume path
  515. func convertVolPathToDevicePath(ctx context.Context, dc *object.Datacenter, volPath string) (string, error) {
  516. volPath = removeStorageClusterORFolderNameFromVDiskPath(volPath)
  517. // Get the canonical volume path for volPath.
  518. canonicalVolumePath, err := getCanonicalVolumePath(ctx, dc, volPath)
  519. if err != nil {
  520. framework.Logf("Failed to get canonical vsphere volume path for volume: %s. err: %+v", volPath, err)
  521. return "", err
  522. }
  523. // Check if the volume path contains .vmdk extension. If not, add the extension and update the nodeVolumes Map
  524. if len(canonicalVolumePath) > 0 && filepath.Ext(canonicalVolumePath) != ".vmdk" {
  525. canonicalVolumePath += ".vmdk"
  526. }
  527. return canonicalVolumePath, nil
  528. }
  529. // get .vmx file path for a virtual machine
  530. func getVMXFilePath(vmObject *object.VirtualMachine) (vmxPath string) {
  531. ctx, cancel := context.WithCancel(context.Background())
  532. defer cancel()
  533. var nodeVM mo.VirtualMachine
  534. err := vmObject.Properties(ctx, vmObject.Reference(), []string{"config.files"}, &nodeVM)
  535. Expect(err).NotTo(HaveOccurred())
  536. Expect(nodeVM.Config).NotTo(BeNil())
  537. vmxPath = nodeVM.Config.Files.VmPathName
  538. framework.Logf("vmx file path is %s", vmxPath)
  539. return vmxPath
  540. }
  541. // verify ready node count. Try upto 3 minutes. Return true if count is expected count
  542. func verifyReadyNodeCount(client clientset.Interface, expectedNodes int) bool {
  543. numNodes := 0
  544. for i := 0; i < 36; i++ {
  545. nodeList := framework.GetReadySchedulableNodesOrDie(client)
  546. Expect(nodeList.Items).NotTo(BeEmpty(), "Unable to find ready and schedulable Node")
  547. numNodes = len(nodeList.Items)
  548. if numNodes == expectedNodes {
  549. break
  550. }
  551. time.Sleep(5 * time.Second)
  552. }
  553. return (numNodes == expectedNodes)
  554. }
  555. // poweroff nodeVM and confirm the poweroff state
  556. func poweroffNodeVM(nodeName string, vm *object.VirtualMachine) {
  557. ctx, cancel := context.WithCancel(context.Background())
  558. defer cancel()
  559. framework.Logf("Powering off node VM %s", nodeName)
  560. _, err := vm.PowerOff(ctx)
  561. Expect(err).NotTo(HaveOccurred())
  562. err = vm.WaitForPowerState(ctx, vimtypes.VirtualMachinePowerStatePoweredOff)
  563. Expect(err).NotTo(HaveOccurred(), "Unable to power off the node")
  564. }
  565. // poweron nodeVM and confirm the poweron state
  566. func poweronNodeVM(nodeName string, vm *object.VirtualMachine) {
  567. ctx, cancel := context.WithCancel(context.Background())
  568. defer cancel()
  569. framework.Logf("Powering on node VM %s", nodeName)
  570. vm.PowerOn(ctx)
  571. err := vm.WaitForPowerState(ctx, vimtypes.VirtualMachinePowerStatePoweredOn)
  572. Expect(err).NotTo(HaveOccurred(), "Unable to power on the node")
  573. }
  574. // unregister a nodeVM from VC
  575. func unregisterNodeVM(nodeName string, vm *object.VirtualMachine) {
  576. ctx, cancel := context.WithCancel(context.Background())
  577. defer cancel()
  578. poweroffNodeVM(nodeName, vm)
  579. framework.Logf("Unregistering node VM %s", nodeName)
  580. err := vm.Unregister(ctx)
  581. Expect(err).NotTo(HaveOccurred(), "Unable to unregister the node")
  582. }
  583. // register a nodeVM into a VC
  584. func registerNodeVM(nodeName, workingDir, vmxFilePath string, rpool *object.ResourcePool, host *object.HostSystem) {
  585. ctx, cancel := context.WithCancel(context.Background())
  586. defer cancel()
  587. framework.Logf("Registering node VM %s with vmx file path %s", nodeName, vmxFilePath)
  588. nodeInfo := TestContext.NodeMapper.GetNodeInfo(nodeName)
  589. finder := find.NewFinder(nodeInfo.VSphere.Client.Client, false)
  590. vmFolder, err := finder.FolderOrDefault(ctx, workingDir)
  591. Expect(err).NotTo(HaveOccurred())
  592. registerTask, err := vmFolder.RegisterVM(ctx, vmxFilePath, nodeName, false, rpool, host)
  593. Expect(err).NotTo(HaveOccurred())
  594. err = registerTask.Wait(ctx)
  595. Expect(err).NotTo(HaveOccurred())
  596. vmPath := filepath.Join(workingDir, nodeName)
  597. vm, err := finder.VirtualMachine(ctx, vmPath)
  598. Expect(err).NotTo(HaveOccurred())
  599. poweronNodeVM(nodeName, vm)
  600. }
  601. // disksAreAttached takes map of node and it's volumes and returns map of node, its volumes and attachment state
  602. func disksAreAttached(nodeVolumes map[string][]string) (nodeVolumesAttachMap map[string]map[string]bool, err error) {
  603. ctx, cancel := context.WithCancel(context.Background())
  604. defer cancel()
  605. disksAttached := make(map[string]map[string]bool)
  606. if len(nodeVolumes) == 0 {
  607. return disksAttached, nil
  608. }
  609. // Convert VolPaths into canonical form so that it can be compared with the VM device path.
  610. vmVolumes, err := convertVolPathsToDevicePaths(ctx, nodeVolumes)
  611. if err != nil {
  612. framework.Logf("Failed to convert volPaths to devicePaths: %+v. err: %+v", nodeVolumes, err)
  613. return nil, err
  614. }
  615. for vm, volumes := range vmVolumes {
  616. volumeAttachedMap := make(map[string]bool)
  617. for _, volume := range volumes {
  618. attached, err := diskIsAttached(volume, vm)
  619. if err != nil {
  620. return nil, err
  621. }
  622. volumeAttachedMap[volume] = attached
  623. }
  624. nodeVolumesAttachMap[vm] = volumeAttachedMap
  625. }
  626. return disksAttached, nil
  627. }
  628. // diskIsAttached returns if disk is attached to the VM using controllers supported by the plugin.
  629. func diskIsAttached(volPath string, nodeName string) (bool, error) {
  630. // Create context
  631. ctx, cancel := context.WithCancel(context.Background())
  632. defer cancel()
  633. nodeInfo := TestContext.NodeMapper.GetNodeInfo(nodeName)
  634. Connect(ctx, nodeInfo.VSphere)
  635. vm := object.NewVirtualMachine(nodeInfo.VSphere.Client.Client, nodeInfo.VirtualMachineRef)
  636. volPath = removeStorageClusterORFolderNameFromVDiskPath(volPath)
  637. device, err := getVirtualDeviceByPath(ctx, vm, volPath)
  638. if err != nil {
  639. framework.Logf("diskIsAttached failed to determine whether disk %q is still attached on node %q",
  640. volPath,
  641. nodeName)
  642. return false, err
  643. }
  644. if device == nil {
  645. return false, nil
  646. }
  647. framework.Logf("diskIsAttached found the disk %q attached on node %q", volPath, nodeName)
  648. return true, nil
  649. }
  650. // getUUIDFromProviderID strips ProviderPrefix - "vsphere://" from the providerID
  651. // this gives the VM UUID which can be used to find Node VM from vCenter
  652. func getUUIDFromProviderID(providerID string) string {
  653. return strings.TrimPrefix(providerID, ProviderPrefix)
  654. }
  655. // GetAllReadySchedulableNodeInfos returns NodeInfo objects for all nodes with Ready and schedulable state
  656. func GetReadySchedulableNodeInfos() []*NodeInfo {
  657. nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
  658. Expect(nodeList.Items).NotTo(BeEmpty(), "Unable to find ready and schedulable Node")
  659. var nodesInfo []*NodeInfo
  660. for _, node := range nodeList.Items {
  661. nodeInfo := TestContext.NodeMapper.GetNodeInfo(node.Name)
  662. if nodeInfo != nil {
  663. nodesInfo = append(nodesInfo, nodeInfo)
  664. }
  665. }
  666. return nodesInfo
  667. }
  668. // GetReadySchedulableRandomNodeInfo returns NodeInfo object for one of the Ready and Schedulable Node.
  669. // if multiple nodes are present with Ready and Scheduable state then one of the Node is selected randomly
  670. // and it's associated NodeInfo object is returned.
  671. func GetReadySchedulableRandomNodeInfo() *NodeInfo {
  672. nodesInfo := GetReadySchedulableNodeInfos()
  673. rand.Seed(time.Now().Unix())
  674. Expect(nodesInfo).NotTo(BeEmpty())
  675. return nodesInfo[rand.Int()%len(nodesInfo)]
  676. }
  677. // invokeVCenterServiceControl invokes the given command for the given service
  678. // via service-control on the given vCenter host over SSH.
  679. func invokeVCenterServiceControl(command, service, host string) error {
  680. sshCmd := fmt.Sprintf("service-control --%s %s", command, service)
  681. framework.Logf("Invoking command %v on vCenter host %v", sshCmd, host)
  682. result, err := framework.SSH(sshCmd, host, framework.TestContext.Provider)
  683. if err != nil || result.Code != 0 {
  684. framework.LogSSHResult(result)
  685. return fmt.Errorf("couldn't execute command: %s on vCenter host: %v", sshCmd, err)
  686. }
  687. return nil
  688. }
  689. // expectVolumeToBeAttached checks if the given Volume is attached to the given
  690. // Node, else fails.
  691. func expectVolumeToBeAttached(nodeName, volumePath string) {
  692. isAttached, err := diskIsAttached(volumePath, nodeName)
  693. Expect(err).NotTo(HaveOccurred())
  694. Expect(isAttached).To(BeTrue(), fmt.Sprintf("disk: %s is not attached with the node", volumePath))
  695. }
  696. // expectVolumesToBeAttached checks if the given Volumes are attached to the
  697. // corresponding set of Nodes, else fails.
  698. func expectVolumesToBeAttached(pods []*v1.Pod, volumePaths []string) {
  699. for i, pod := range pods {
  700. nodeName := pod.Spec.NodeName
  701. volumePath := volumePaths[i]
  702. By(fmt.Sprintf("Verifying that volume %v is attached to node %v", volumePath, nodeName))
  703. expectVolumeToBeAttached(nodeName, volumePath)
  704. }
  705. }
  706. // expectFilesToBeAccessible checks if the given files are accessible on the
  707. // corresponding set of Nodes, else fails.
  708. func expectFilesToBeAccessible(namespace string, pods []*v1.Pod, filePaths []string) {
  709. for i, pod := range pods {
  710. podName := pod.Name
  711. filePath := filePaths[i]
  712. By(fmt.Sprintf("Verifying that file %v is accessible on pod %v", filePath, podName))
  713. verifyFilesExistOnVSphereVolume(namespace, podName, filePath)
  714. }
  715. }
  716. // writeContentToPodFile writes the given content to the specified file.
  717. func writeContentToPodFile(namespace, podName, filePath, content string) error {
  718. _, err := framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", namespace), podName,
  719. "--", "/bin/sh", "-c", fmt.Sprintf("echo '%s' > %s", content, filePath))
  720. return err
  721. }
  722. // expectFileContentToMatch checks if a given file contains the specified
  723. // content, else fails.
  724. func expectFileContentToMatch(namespace, podName, filePath, content string) {
  725. _, err := framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", namespace), podName,
  726. "--", "/bin/sh", "-c", fmt.Sprintf("grep '%s' %s", content, filePath))
  727. Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to match content of file: %q on the pod: %q", filePath, podName))
  728. }
  729. // expectFileContentsToMatch checks if the given contents match the ones present
  730. // in corresponding files on respective Pods, else fails.
  731. func expectFileContentsToMatch(namespace string, pods []*v1.Pod, filePaths []string, contents []string) {
  732. for i, pod := range pods {
  733. podName := pod.Name
  734. filePath := filePaths[i]
  735. By(fmt.Sprintf("Matching file content for %v on pod %v", filePath, podName))
  736. expectFileContentToMatch(namespace, podName, filePath, contents[i])
  737. }
  738. }