PageRenderTime 451ms CodeModel.GetById 24ms RepoModel.GetById 1ms app.codeStats 0ms

/test/e2e_node/device_plugin.go

https://bitbucket.org/Jake-Qu/kubernetes-mirror
Go | 314 lines | 228 code | 57 blank | 29 comment | 23 complexity | d93c889b249d0a48b0a1bb0fb7bc8751 MD5 | raw file
Possible License(s): MIT, MPL-2.0-no-copyleft-exception, 0BSD, CC0-1.0, BSD-2-Clause, Apache-2.0, BSD-3-Clause
  1. /*
  2. Copyright 2017 The Kubernetes Authors.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. package e2e_node
  14. import (
  15. "fmt"
  16. "os"
  17. "path/filepath"
  18. "time"
  19. "regexp"
  20. "k8s.io/api/core/v1"
  21. "k8s.io/apimachinery/pkg/api/resource"
  22. "k8s.io/apimachinery/pkg/util/uuid"
  23. metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  24. "k8s.io/kubernetes/test/e2e/framework"
  25. pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
  26. dm "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager"
  27. . "github.com/onsi/ginkgo"
  28. . "github.com/onsi/gomega"
  29. )
  30. const (
  31. // fake resource name
  32. resourceName = "fake.com/resource"
  33. )
  34. // Serial because the test restarts Kubelet
  35. var _ = framework.KubeDescribe("Device Plugin [Feature:DevicePlugin][NodeFeature:DevicePlugin][Serial]", func() {
  36. f := framework.NewDefaultFramework("device-plugin-errors")
  37. Context("DevicePlugin", func() {
  38. It("Verifies the Kubelet device plugin functionality.", func() {
  39. By("Start stub device plugin")
  40. // fake devices for e2e test
  41. devs := []*pluginapi.Device{
  42. {ID: "Dev-1", Health: pluginapi.Healthy},
  43. {ID: "Dev-2", Health: pluginapi.Healthy},
  44. }
  45. socketPath := pluginapi.DevicePluginPath + "dp." + fmt.Sprintf("%d", time.Now().Unix())
  46. dp1 := dm.NewDevicePluginStub(devs, socketPath)
  47. dp1.SetAllocFunc(stubAllocFunc)
  48. err := dp1.Start()
  49. framework.ExpectNoError(err)
  50. By("Register resources")
  51. err = dp1.Register(pluginapi.KubeletSocket, resourceName, false)
  52. framework.ExpectNoError(err)
  53. By("Waiting for the resource exported by the stub device plugin to become available on the local node")
  54. devsLen := int64(len(devs))
  55. Eventually(func() bool {
  56. node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{})
  57. framework.ExpectNoError(err)
  58. return numberOfDevicesCapacity(node, resourceName) == devsLen &&
  59. numberOfDevicesAllocatable(node, resourceName) == devsLen
  60. }, 30*time.Second, framework.Poll).Should(BeTrue())
  61. By("Creating one pod on node with at least one fake-device")
  62. podRECMD := "devs=$(ls /tmp/ | egrep '^Dev-[0-9]+$') && echo stub devices: $devs"
  63. pod1 := f.PodClient().CreateSync(makeBusyboxPod(resourceName, podRECMD))
  64. deviceIDRE := "stub devices: (Dev-[0-9]+)"
  65. devId1 := parseLog(f, pod1.Name, pod1.Name, deviceIDRE)
  66. Expect(devId1).To(Not(Equal("")))
  67. pod1, err = f.PodClient().Get(pod1.Name, metav1.GetOptions{})
  68. framework.ExpectNoError(err)
  69. ensurePodContainerRestart(f, pod1.Name, pod1.Name)
  70. By("Confirming that device assignment persists even after container restart")
  71. devIdAfterRestart := parseLog(f, pod1.Name, pod1.Name, deviceIDRE)
  72. Expect(devIdAfterRestart).To(Equal(devId1))
  73. By("Restarting Kubelet")
  74. restartKubelet()
  75. ensurePodContainerRestart(f, pod1.Name, pod1.Name)
  76. By("Confirming that after a kubelet restart, fake-device assignement is kept")
  77. devIdRestart1 := parseLog(f, pod1.Name, pod1.Name, deviceIDRE)
  78. Expect(devIdRestart1).To(Equal(devId1))
  79. By("Wait for node is ready")
  80. framework.WaitForAllNodesSchedulable(f.ClientSet, framework.TestContext.NodeSchedulableTimeout)
  81. By("Re-Register resources after kubelet restart")
  82. dp1 = dm.NewDevicePluginStub(devs, socketPath)
  83. dp1.SetAllocFunc(stubAllocFunc)
  84. err = dp1.Start()
  85. framework.ExpectNoError(err)
  86. err = dp1.Register(pluginapi.KubeletSocket, resourceName, false)
  87. framework.ExpectNoError(err)
  88. By("Waiting for resource to become available on the local node after re-registration")
  89. Eventually(func() bool {
  90. node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{})
  91. framework.ExpectNoError(err)
  92. return numberOfDevicesCapacity(node, resourceName) == devsLen &&
  93. numberOfDevicesAllocatable(node, resourceName) == devsLen
  94. }, 30*time.Second, framework.Poll).Should(BeTrue())
  95. By("Creating another pod")
  96. pod2 := f.PodClient().CreateSync(makeBusyboxPod(resourceName, podRECMD))
  97. By("Checking that pod got a different fake device")
  98. devId2 := parseLog(f, pod2.Name, pod2.Name, deviceIDRE)
  99. Expect(devId1).To(Not(Equal(devId2)))
  100. By("Deleting device plugin.")
  101. err = dp1.Stop()
  102. framework.ExpectNoError(err)
  103. By("Waiting for stub device plugin to become unhealthy on the local node")
  104. Eventually(func() int64 {
  105. node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{})
  106. framework.ExpectNoError(err)
  107. return numberOfDevicesAllocatable(node, resourceName)
  108. }, 30*time.Second, framework.Poll).Should(Equal(int64(0)))
  109. By("Checking that scheduled pods can continue to run even after we delete device plugin.")
  110. ensurePodContainerRestart(f, pod1.Name, pod1.Name)
  111. devIdRestart1 = parseLog(f, pod1.Name, pod1.Name, deviceIDRE)
  112. Expect(devIdRestart1).To(Equal(devId1))
  113. ensurePodContainerRestart(f, pod2.Name, pod2.Name)
  114. devIdRestart2 := parseLog(f, pod2.Name, pod2.Name, deviceIDRE)
  115. Expect(devIdRestart2).To(Equal(devId2))
  116. By("Re-register resources")
  117. dp1 = dm.NewDevicePluginStub(devs, socketPath)
  118. dp1.SetAllocFunc(stubAllocFunc)
  119. err = dp1.Start()
  120. framework.ExpectNoError(err)
  121. err = dp1.Register(pluginapi.KubeletSocket, resourceName, false)
  122. framework.ExpectNoError(err)
  123. By("Waiting for the resource exported by the stub device plugin to become healthy on the local node")
  124. Eventually(func() int64 {
  125. node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{})
  126. framework.ExpectNoError(err)
  127. return numberOfDevicesAllocatable(node, resourceName)
  128. }, 30*time.Second, framework.Poll).Should(Equal(devsLen))
  129. By("Deleting device plugin again.")
  130. err = dp1.Stop()
  131. framework.ExpectNoError(err)
  132. By("Waiting for stub device plugin to become unavailable on the local node")
  133. Eventually(func() bool {
  134. node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{})
  135. framework.ExpectNoError(err)
  136. return numberOfDevicesCapacity(node, resourceName) <= 0
  137. }, 10*time.Minute, framework.Poll).Should(BeTrue())
  138. By("Restarting Kubelet second time.")
  139. restartKubelet()
  140. By("Checking that scheduled pods can continue to run even after we delete device plugin and restart Kubelet Eventually.")
  141. ensurePodContainerRestart(f, pod1.Name, pod1.Name)
  142. devIdRestart1 = parseLog(f, pod1.Name, pod1.Name, deviceIDRE)
  143. Expect(devIdRestart1).To(Equal(devId1))
  144. ensurePodContainerRestart(f, pod2.Name, pod2.Name)
  145. devIdRestart2 = parseLog(f, pod2.Name, pod2.Name, deviceIDRE)
  146. Expect(devIdRestart2).To(Equal(devId2))
  147. // Cleanup
  148. f.PodClient().DeleteSync(pod1.Name, &metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout)
  149. f.PodClient().DeleteSync(pod2.Name, &metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout)
  150. })
  151. })
  152. })
  153. // makeBusyboxPod returns a simple Pod spec with a busybox container
  154. // that requests resourceName and runs the specified command.
  155. func makeBusyboxPod(resourceName, cmd string) *v1.Pod {
  156. podName := "device-plugin-test-" + string(uuid.NewUUID())
  157. rl := v1.ResourceList{v1.ResourceName(resourceName): *resource.NewQuantity(1, resource.DecimalSI)}
  158. return &v1.Pod{
  159. ObjectMeta: metav1.ObjectMeta{Name: podName},
  160. Spec: v1.PodSpec{
  161. RestartPolicy: v1.RestartPolicyAlways,
  162. Containers: []v1.Container{{
  163. Image: busyboxImage,
  164. Name: podName,
  165. // Runs the specified command in the test pod.
  166. Command: []string{"sh", "-c", cmd},
  167. Resources: v1.ResourceRequirements{
  168. Limits: rl,
  169. Requests: rl,
  170. },
  171. }},
  172. },
  173. }
  174. }
  175. // ensurePodContainerRestart confirms that pod container has restarted at least once
  176. func ensurePodContainerRestart(f *framework.Framework, podName string, contName string) {
  177. var initialCount int32
  178. var currentCount int32
  179. p, err := f.PodClient().Get(podName, metav1.GetOptions{})
  180. if err != nil || len(p.Status.ContainerStatuses) < 1 {
  181. framework.Failf("ensurePodContainerRestart failed for pod %q: %v", podName, err)
  182. }
  183. initialCount = p.Status.ContainerStatuses[0].RestartCount
  184. Eventually(func() bool {
  185. p, err = f.PodClient().Get(podName, metav1.GetOptions{})
  186. if err != nil || len(p.Status.ContainerStatuses) < 1 {
  187. return false
  188. }
  189. currentCount = p.Status.ContainerStatuses[0].RestartCount
  190. framework.Logf("initial %v, current %v", initialCount, currentCount)
  191. return currentCount > initialCount
  192. }, 2*time.Minute, framework.Poll).Should(BeTrue())
  193. }
  194. // parseLog returns the matching string for the specified regular expression parsed from the container logs.
  195. func parseLog(f *framework.Framework, podName string, contName string, re string) string {
  196. logs, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, podName, contName)
  197. if err != nil {
  198. framework.Failf("GetPodLogs for pod %q failed: %v", podName, err)
  199. }
  200. framework.Logf("got pod logs: %v", logs)
  201. regex := regexp.MustCompile(re)
  202. matches := regex.FindStringSubmatch(logs)
  203. if len(matches) < 2 {
  204. return ""
  205. }
  206. return matches[1]
  207. }
  208. // numberOfDevicesCapacity returns the number of devices of resourceName advertised by a node capacity
  209. func numberOfDevicesCapacity(node *v1.Node, resourceName string) int64 {
  210. val, ok := node.Status.Capacity[v1.ResourceName(resourceName)]
  211. if !ok {
  212. return 0
  213. }
  214. return val.Value()
  215. }
  216. // numberOfDevicesAllocatable returns the number of devices of resourceName advertised by a node allocatable
  217. func numberOfDevicesAllocatable(node *v1.Node, resourceName string) int64 {
  218. val, ok := node.Status.Allocatable[v1.ResourceName(resourceName)]
  219. if !ok {
  220. return 0
  221. }
  222. return val.Value()
  223. }
  224. // stubAllocFunc will pass to stub device plugin
  225. func stubAllocFunc(r *pluginapi.AllocateRequest, devs map[string]pluginapi.Device) (*pluginapi.AllocateResponse, error) {
  226. var responses pluginapi.AllocateResponse
  227. for _, req := range r.ContainerRequests {
  228. response := &pluginapi.ContainerAllocateResponse{}
  229. for _, requestID := range req.DevicesIDs {
  230. dev, ok := devs[requestID]
  231. if !ok {
  232. return nil, fmt.Errorf("invalid allocation request with non-existing device %s", requestID)
  233. }
  234. if dev.Health != pluginapi.Healthy {
  235. return nil, fmt.Errorf("invalid allocation request with unhealthy device: %s", requestID)
  236. }
  237. // create fake device file
  238. fpath := filepath.Join("/tmp", dev.ID)
  239. // clean first
  240. os.RemoveAll(fpath)
  241. f, err := os.Create(fpath)
  242. if err != nil && !os.IsExist(err) {
  243. return nil, fmt.Errorf("failed to create fake device file: %s", err)
  244. }
  245. f.Close()
  246. response.Mounts = append(response.Mounts, &pluginapi.Mount{
  247. ContainerPath: fpath,
  248. HostPath: fpath,
  249. })
  250. }
  251. responses.ContainerResponses = append(responses.ContainerResponses, response)
  252. }
  253. return &responses, nil
  254. }