PageRenderTime 61ms CodeModel.GetById 26ms RepoModel.GetById 0ms app.codeStats 0ms

/test/e2e_node/runner/run_e2e.go

https://gitlab.com/github-cloud-corporation/kubernetes
Go | 566 lines | 469 code | 50 blank | 47 comment | 119 complexity | 86eedb4a8f2acfb9c186717c460458f4 MD5 | raw file
  1. /*
  2. Copyright 2016 The Kubernetes Authors.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. // To run the e2e tests against one or more hosts on gce:
  14. // $ go run run_e2e.go --logtostderr --v 2 --ssh-env gce --hosts <comma separated hosts>
  15. // To run the e2e tests against one or more images on gce and provision them:
  16. // $ go run run_e2e.go --logtostderr --v 2 --project <project> --zone <zone> --ssh-env gce --images <comma separated images>
  17. package main
  18. import (
  19. "flag"
  20. "fmt"
  21. "io/ioutil"
  22. "math/rand"
  23. "net/http"
  24. "os"
  25. "os/exec"
  26. "regexp"
  27. "sort"
  28. "strings"
  29. "sync"
  30. "time"
  31. "k8s.io/kubernetes/test/e2e_node"
  32. "github.com/ghodss/yaml"
  33. "github.com/golang/glog"
  34. "github.com/pborman/uuid"
  35. "golang.org/x/oauth2"
  36. "golang.org/x/oauth2/google"
  37. "google.golang.org/api/compute/v1"
  38. )
  39. var testArgs = flag.String("test_args", "", "Space-separated list of arguments to pass to Ginkgo test runner.")
  40. var instanceNamePrefix = flag.String("instance-name-prefix", "", "prefix for instance names")
  41. var zone = flag.String("zone", "", "gce zone the hosts live in")
  42. var project = flag.String("project", "", "gce project the hosts live in")
  43. var imageConfigFile = flag.String("image-config-file", "", "yaml file describing images to run")
  44. var imageProject = flag.String("image-project", "", "gce project the hosts live in")
  45. var images = flag.String("images", "", "images to test")
  46. var hosts = flag.String("hosts", "", "hosts to test")
  47. var cleanup = flag.Bool("cleanup", true, "If true remove files from remote hosts and delete temporary instances")
  48. var deleteInstances = flag.Bool("delete-instances", true, "If true, delete any instances created")
  49. var buildOnly = flag.Bool("build-only", false, "If true, build e2e_node_test.tar.gz and exit.")
  50. var setupNode = flag.Bool("setup-node", false, "When true, current user will be added to docker group on the test machine")
  51. var instanceMetadata = flag.String("instance-metadata", "", "key/value metadata for instances separated by '=' or '<', 'k=v' means the key is 'k' and the value is 'v'; 'k<p' means the key is 'k' and the value is extracted from the local path 'p', e.g. k1=v1,k2<p2")
  52. var gubernator = flag.Bool("gubernator", false, "If true, output Gubernator link to view logs")
  53. var computeService *compute.Service
  54. type Archive struct {
  55. sync.Once
  56. path string
  57. err error
  58. }
  59. var arc Archive
  60. type TestResult struct {
  61. output string
  62. err error
  63. host string
  64. exitOk bool
  65. }
  66. // ImageConfig specifies what images should be run and how for these tests.
  67. // It can be created via the `--images` and `--image-project` flags, or by
  68. // specifying the `--image-config-file` flag, pointing to a json or yaml file
  69. // of the form:
  70. //
  71. // images:
  72. // short-name:
  73. // image: gce-image-name
  74. // project: gce-image-project
  75. type ImageConfig struct {
  76. Images map[string]GCEImage `json:"images"`
  77. }
  78. type GCEImage struct {
  79. Image string `json:"image, omitempty"`
  80. Project string `json:"project"`
  81. Metadata string `json:"metadata"`
  82. ImageRegex string `json:"image_regex, omitempty"`
  83. // Defaults to using only the latest image. Acceptible values are [0, # of images that match the regex).
  84. // If the number of existing previous images is lesser than what is desired, the test will use that is available.
  85. PreviousImages int `json:"previous_images, omitempty"`
  86. }
  87. type internalImageConfig struct {
  88. images map[string]internalGCEImage
  89. }
  90. type internalGCEImage struct {
  91. image string
  92. project string
  93. metadata *compute.Metadata
  94. }
  95. func main() {
  96. flag.Parse()
  97. rand.Seed(time.Now().UTC().UnixNano())
  98. if *buildOnly {
  99. // Build the archive and exit
  100. e2e_node.CreateTestArchive()
  101. return
  102. }
  103. if *hosts == "" && *imageConfigFile == "" && *images == "" {
  104. glog.Fatalf("Must specify one of --image-config-file, --hosts, --images.")
  105. }
  106. var err error
  107. computeService, err = getComputeClient()
  108. if err != nil {
  109. glog.Fatalf("Unable to create gcloud compute service using defaults. Make sure you are authenticated. %v", err)
  110. }
  111. gceImages := &internalImageConfig{
  112. images: make(map[string]internalGCEImage),
  113. }
  114. if *imageConfigFile != "" {
  115. // parse images
  116. imageConfigData, err := ioutil.ReadFile(*imageConfigFile)
  117. if err != nil {
  118. glog.Fatalf("Could not read image config file provided: %v", err)
  119. }
  120. externalImageConfig := ImageConfig{Images: make(map[string]GCEImage)}
  121. err = yaml.Unmarshal(imageConfigData, &externalImageConfig)
  122. if err != nil {
  123. glog.Fatalf("Could not parse image config file: %v", err)
  124. }
  125. for _, imageConfig := range externalImageConfig.Images {
  126. var images []string
  127. if imageConfig.ImageRegex != "" && imageConfig.Image == "" {
  128. images, err = getGCEImages(imageConfig.ImageRegex, imageConfig.Project, imageConfig.PreviousImages)
  129. if err != nil {
  130. glog.Fatalf("Could not retrieve list of images based on image prefix %q: %v", imageConfig.ImageRegex, err)
  131. }
  132. } else {
  133. images = []string{imageConfig.Image}
  134. }
  135. for _, image := range images {
  136. gceImage := internalGCEImage{
  137. image: image,
  138. project: imageConfig.Project,
  139. metadata: getImageMetadata(imageConfig.Metadata),
  140. }
  141. gceImages.images[image] = gceImage
  142. }
  143. }
  144. }
  145. // Allow users to specify additional images via cli flags for local testing
  146. // convenience; merge in with config file
  147. if *images != "" {
  148. if *imageProject == "" {
  149. glog.Fatal("Must specify --image-project if you specify --images")
  150. }
  151. cliImages := strings.Split(*images, ",")
  152. for _, img := range cliImages {
  153. gceImage := internalGCEImage{
  154. image: img,
  155. project: *imageProject,
  156. metadata: getImageMetadata(*instanceMetadata),
  157. }
  158. gceImages.images[img] = gceImage
  159. }
  160. }
  161. if len(gceImages.images) != 0 && *zone == "" {
  162. glog.Fatal("Must specify --zone flag")
  163. }
  164. for shortName, image := range gceImages.images {
  165. if image.project == "" {
  166. glog.Fatalf("Invalid config for %v; must specify a project", shortName)
  167. }
  168. }
  169. if len(gceImages.images) != 0 {
  170. if *project == "" {
  171. glog.Fatal("Must specify --project flag to launch images into")
  172. }
  173. }
  174. if *instanceNamePrefix == "" {
  175. *instanceNamePrefix = "tmp-node-e2e-" + uuid.NewUUID().String()[:8]
  176. }
  177. // Setup coloring
  178. stat, _ := os.Stdout.Stat()
  179. useColor := (stat.Mode() & os.ModeCharDevice) != 0
  180. blue := ""
  181. noColour := ""
  182. if useColor {
  183. blue = "\033[0;34m"
  184. noColour = "\033[0m"
  185. }
  186. go arc.getArchive()
  187. defer arc.deleteArchive()
  188. results := make(chan *TestResult)
  189. running := 0
  190. for shortName := range gceImages.images {
  191. imageConfig := gceImages.images[shortName]
  192. fmt.Printf("Initializing e2e tests using image %s.\n", shortName)
  193. running++
  194. go func(image *internalGCEImage, junitFilePrefix string) {
  195. results <- testImage(image, junitFilePrefix)
  196. }(&imageConfig, shortName)
  197. }
  198. if *hosts != "" {
  199. for _, host := range strings.Split(*hosts, ",") {
  200. fmt.Printf("Initializing e2e tests using host %s.\n", host)
  201. running++
  202. go func(host string, junitFilePrefix string) {
  203. results <- testHost(host, *cleanup, junitFilePrefix, *setupNode)
  204. }(host, host)
  205. }
  206. }
  207. // Wait for all tests to complete and emit the results
  208. errCount := 0
  209. exitOk := true
  210. for i := 0; i < running; i++ {
  211. tr := <-results
  212. host := tr.host
  213. fmt.Printf("%s================================================================%s\n", blue, noColour)
  214. if tr.err != nil {
  215. errCount++
  216. fmt.Printf("Failure Finished Host %s Test Suite\n%s\n%v\n", host, tr.output, tr.err)
  217. } else {
  218. fmt.Printf("Success Finished Host %s Test Suite\n%s\n", host, tr.output)
  219. }
  220. exitOk = exitOk && tr.exitOk
  221. fmt.Printf("%s================================================================%s\n", blue, noColour)
  222. }
  223. // Set the exit code if there were failures
  224. if !exitOk {
  225. fmt.Printf("Failure: %d errors encountered.", errCount)
  226. callGubernator(*gubernator)
  227. os.Exit(1)
  228. }
  229. callGubernator(*gubernator)
  230. }
  231. func callGubernator(gubernator bool) {
  232. fmt.Println("Running gubernator.sh")
  233. if gubernator {
  234. output, err := exec.Command("./test/e2e_node/gubernator.sh", "y").Output()
  235. if err != nil {
  236. fmt.Println("gubernator.sh Failed")
  237. fmt.Println(err)
  238. return
  239. }
  240. fmt.Printf("%s", output)
  241. }
  242. return
  243. }
  244. func (a *Archive) getArchive() (string, error) {
  245. a.Do(func() { a.path, a.err = e2e_node.CreateTestArchive() })
  246. return a.path, a.err
  247. }
  248. func (a *Archive) deleteArchive() {
  249. path, err := a.getArchive()
  250. if err != nil {
  251. return
  252. }
  253. os.Remove(path)
  254. }
  255. func getImageMetadata(input string) *compute.Metadata {
  256. if input == "" {
  257. return nil
  258. }
  259. glog.V(3).Infof("parsing instance metadata: %q", input)
  260. raw := parseInstanceMetadata(input)
  261. glog.V(4).Infof("parsed instance metadata: %v", raw)
  262. metadataItems := []*compute.MetadataItems{}
  263. for k, v := range raw {
  264. val := v
  265. metadataItems = append(metadataItems, &compute.MetadataItems{
  266. Key: k,
  267. Value: &val,
  268. })
  269. }
  270. ret := compute.Metadata{Items: metadataItems}
  271. return &ret
  272. }
  273. // Run tests in archive against host
  274. func testHost(host string, deleteFiles bool, junitFilePrefix string, setupNode bool) *TestResult {
  275. instance, err := computeService.Instances.Get(*project, *zone, host).Do()
  276. if err != nil {
  277. return &TestResult{
  278. err: err,
  279. host: host,
  280. exitOk: false,
  281. }
  282. }
  283. if strings.ToUpper(instance.Status) != "RUNNING" {
  284. err = fmt.Errorf("instance %s not in state RUNNING, was %s.", host, instance.Status)
  285. return &TestResult{
  286. err: err,
  287. host: host,
  288. exitOk: false,
  289. }
  290. }
  291. externalIp := getExternalIp(instance)
  292. if len(externalIp) > 0 {
  293. e2e_node.AddHostnameIp(host, externalIp)
  294. }
  295. path, err := arc.getArchive()
  296. if err != nil {
  297. // Don't log fatal because we need to do any needed cleanup contained in "defer" statements
  298. return &TestResult{
  299. err: fmt.Errorf("unable to create test archive %v.", err),
  300. }
  301. }
  302. output, exitOk, err := e2e_node.RunRemote(path, host, deleteFiles, junitFilePrefix, setupNode, *testArgs)
  303. return &TestResult{
  304. output: output,
  305. err: err,
  306. host: host,
  307. exitOk: exitOk,
  308. }
  309. }
  310. type imageObj struct {
  311. creationTime time.Time
  312. name string
  313. }
  314. func (io imageObj) string() string {
  315. return fmt.Sprintf("%q created %q", io.name, io.creationTime.String())
  316. }
  317. type byCreationTime []imageObj
  318. func (a byCreationTime) Len() int { return len(a) }
  319. func (a byCreationTime) Less(i, j int) bool { return a[i].creationTime.After(a[j].creationTime) }
  320. func (a byCreationTime) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
  321. // Returns a list of image names based on regex and number of previous images requested.
  322. func getGCEImages(imageRegex, project string, previousImages int) ([]string, error) {
  323. ilc, err := computeService.Images.List(project).Do()
  324. if err != nil {
  325. return nil, fmt.Errorf("Failed to list images in project %q and zone %q", project, zone)
  326. }
  327. imageObjs := []imageObj{}
  328. imageRe := regexp.MustCompile(imageRegex)
  329. for _, instance := range ilc.Items {
  330. if imageRe.MatchString(instance.Name) {
  331. creationTime, err := time.Parse(time.RFC3339, instance.CreationTimestamp)
  332. if err != nil {
  333. return nil, fmt.Errorf("Failed to parse instance creation timestamp %q: %v", instance.CreationTimestamp, err)
  334. }
  335. io := imageObj{
  336. creationTime: creationTime,
  337. name: instance.Name,
  338. }
  339. glog.V(4).Infof("Found image %q based on regex %q in project %q", io.string(), imageRegex, project)
  340. imageObjs = append(imageObjs, io)
  341. }
  342. }
  343. sort.Sort(byCreationTime(imageObjs))
  344. images := []string{}
  345. for _, imageObj := range imageObjs {
  346. images = append(images, imageObj.name)
  347. previousImages--
  348. if previousImages < 0 {
  349. break
  350. }
  351. }
  352. return images, nil
  353. }
  354. // Provision a gce instance using image and run the tests in archive against the instance.
  355. // Delete the instance afterward.
  356. func testImage(image *internalGCEImage, junitFilePrefix string) *TestResult {
  357. host, err := createInstance(image)
  358. if *deleteInstances {
  359. defer deleteInstance(image.image)
  360. }
  361. if err != nil {
  362. return &TestResult{
  363. err: fmt.Errorf("unable to create gce instance with running docker daemon for image %s. %v", image.image, err),
  364. }
  365. }
  366. // Only delete the files if we are keeping the instance and want it cleaned up.
  367. // If we are going to delete the instance, don't bother with cleaning up the files
  368. deleteFiles := !*deleteInstances && *cleanup
  369. return testHost(host, deleteFiles, junitFilePrefix, *setupNode)
  370. }
  371. // Provision a gce instance using image
  372. func createInstance(image *internalGCEImage) (string, error) {
  373. glog.V(1).Infof("Creating instance %+v", *image)
  374. name := imageToInstanceName(image.image)
  375. i := &compute.Instance{
  376. Name: name,
  377. MachineType: machineType(),
  378. NetworkInterfaces: []*compute.NetworkInterface{
  379. {
  380. AccessConfigs: []*compute.AccessConfig{
  381. {
  382. Type: "ONE_TO_ONE_NAT",
  383. Name: "External NAT",
  384. },
  385. }},
  386. },
  387. Disks: []*compute.AttachedDisk{
  388. {
  389. AutoDelete: true,
  390. Boot: true,
  391. Type: "PERSISTENT",
  392. InitializeParams: &compute.AttachedDiskInitializeParams{
  393. SourceImage: sourceImage(image.image, image.project),
  394. },
  395. },
  396. },
  397. }
  398. i.Metadata = image.metadata
  399. op, err := computeService.Instances.Insert(*project, *zone, i).Do()
  400. if err != nil {
  401. return "", err
  402. }
  403. if op.Error != nil {
  404. return "", fmt.Errorf("could not create instance %s: %+v", name, op.Error)
  405. }
  406. instanceRunning := false
  407. for i := 0; i < 30 && !instanceRunning; i++ {
  408. if i > 0 {
  409. time.Sleep(time.Second * 20)
  410. }
  411. var instance *compute.Instance
  412. instance, err = computeService.Instances.Get(*project, *zone, name).Do()
  413. if err != nil {
  414. continue
  415. }
  416. if strings.ToUpper(instance.Status) != "RUNNING" {
  417. err = fmt.Errorf("instance %s not in state RUNNING, was %s.", name, instance.Status)
  418. continue
  419. }
  420. externalIp := getExternalIp(instance)
  421. if len(externalIp) > 0 {
  422. e2e_node.AddHostnameIp(name, externalIp)
  423. }
  424. var output string
  425. output, err = e2e_node.RunSshCommand("ssh", e2e_node.GetHostnameOrIp(name), "--", "sudo", "docker", "version")
  426. if err != nil {
  427. err = fmt.Errorf("instance %s not running docker daemon - Command failed: %s", name, output)
  428. continue
  429. }
  430. if !strings.Contains(output, "Server") {
  431. err = fmt.Errorf("instance %s not running docker daemon - Server not found: %s", name, output)
  432. continue
  433. }
  434. instanceRunning = true
  435. }
  436. return name, err
  437. }
  438. func getExternalIp(instance *compute.Instance) string {
  439. for i := range instance.NetworkInterfaces {
  440. ni := instance.NetworkInterfaces[i]
  441. for j := range ni.AccessConfigs {
  442. ac := ni.AccessConfigs[j]
  443. if len(ac.NatIP) > 0 {
  444. return ac.NatIP
  445. }
  446. }
  447. }
  448. return ""
  449. }
  450. func getComputeClient() (*compute.Service, error) {
  451. const retries = 10
  452. const backoff = time.Second * 6
  453. // Setup the gce client for provisioning instances
  454. // Getting credentials on gce jenkins is flaky, so try a couple times
  455. var err error
  456. var cs *compute.Service
  457. for i := 0; i < retries; i++ {
  458. if i > 0 {
  459. time.Sleep(backoff)
  460. }
  461. var client *http.Client
  462. client, err = google.DefaultClient(oauth2.NoContext, compute.ComputeScope)
  463. if err != nil {
  464. continue
  465. }
  466. cs, err = compute.New(client)
  467. if err != nil {
  468. continue
  469. }
  470. return cs, nil
  471. }
  472. return nil, err
  473. }
  474. func deleteInstance(image string) {
  475. instanceName := imageToInstanceName(image)
  476. glog.Infof("Deleting instance %q", instanceName)
  477. _, err := computeService.Instances.Delete(*project, *zone, instanceName).Do()
  478. if err != nil {
  479. glog.Errorf("Error deleting instance %q: %v", instanceName, err)
  480. }
  481. }
  482. func parseInstanceMetadata(str string) map[string]string {
  483. metadata := make(map[string]string)
  484. ss := strings.Split(str, ",")
  485. for _, s := range ss {
  486. kv := strings.Split(s, "=")
  487. if len(kv) == 2 {
  488. metadata[kv[0]] = kv[1]
  489. continue
  490. }
  491. kp := strings.Split(s, "<")
  492. if len(kp) != 2 {
  493. glog.Fatalf("Invalid instance metadata: %q", s)
  494. continue
  495. }
  496. v, err := ioutil.ReadFile(kp[1])
  497. if err != nil {
  498. glog.Fatalf("Failed to read metadata file %q: %v", kp[1], err)
  499. continue
  500. }
  501. metadata[kp[0]] = string(v)
  502. }
  503. return metadata
  504. }
  505. func imageToInstanceName(image string) string {
  506. return *instanceNamePrefix + "-" + image
  507. }
  508. func sourceImage(image, imageProject string) string {
  509. return fmt.Sprintf("projects/%s/global/images/%s", imageProject, image)
  510. }
  511. func machineType() string {
  512. return fmt.Sprintf("zones/%s/machineTypes/n1-standard-1", *zone)
  513. }