PageRenderTime 55ms CodeModel.GetById 20ms RepoModel.GetById 1ms app.codeStats 0ms

/test/e2e_node/runner/remote/run_remote.go

https://gitlab.com/unofficial-mirrors/kubernetes
Go | 784 lines | 644 code | 66 blank | 74 comment | 172 complexity | 994f29a2f66bcd9bbdb1191627e11aa1 MD5 | raw file
  1. /*
  2. Copyright 2016 The Kubernetes Authors.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. // To run the node e2e tests remotely against one or more hosts on gce:
  14. // $ go run run_remote.go --logtostderr --v 2 --ssh-env gce --hosts <comma separated hosts>
  15. // To run the node e2e tests remotely against one or more images on gce and provision them:
  16. // $ go run run_remote.go --logtostderr --v 2 --project <project> --zone <zone> --ssh-env gce --images <comma separated images>
  17. package main
  18. import (
  19. "flag"
  20. "fmt"
  21. "io/ioutil"
  22. "math/rand"
  23. "net/http"
  24. "os"
  25. "os/exec"
  26. "path/filepath"
  27. "regexp"
  28. "sort"
  29. "strings"
  30. "sync"
  31. "time"
  32. "k8s.io/kubernetes/test/e2e_node/remote"
  33. "github.com/ghodss/yaml"
  34. "github.com/golang/glog"
  35. "github.com/pborman/uuid"
  36. "golang.org/x/oauth2"
  37. "golang.org/x/oauth2/google"
  38. compute "google.golang.org/api/compute/v0.beta"
  39. )
  40. var testArgs = flag.String("test_args", "", "Space-separated list of arguments to pass to Ginkgo test runner.")
  41. var testSuite = flag.String("test-suite", "default", "Test suite the runner initializes with. Currently support default|conformance")
  42. var instanceNamePrefix = flag.String("instance-name-prefix", "", "prefix for instance names")
  43. var zone = flag.String("zone", "", "gce zone the hosts live in")
  44. var project = flag.String("project", "", "gce project the hosts live in")
  45. var imageConfigFile = flag.String("image-config-file", "", "yaml file describing images to run")
  46. var imageConfigDir = flag.String("image-config-dir", "", "(optional)path to image config files")
  47. var imageProject = flag.String("image-project", "", "gce project the hosts live in")
  48. var images = flag.String("images", "", "images to test")
  49. var hosts = flag.String("hosts", "", "hosts to test")
  50. var cleanup = flag.Bool("cleanup", true, "If true remove files from remote hosts and delete temporary instances")
  51. var deleteInstances = flag.Bool("delete-instances", true, "If true, delete any instances created")
  52. var buildOnly = flag.Bool("build-only", false, "If true, build e2e_node_test.tar.gz and exit.")
  53. var instanceMetadata = flag.String("instance-metadata", "", "key/value metadata for instances separated by '=' or '<', 'k=v' means the key is 'k' and the value is 'v'; 'k<p' means the key is 'k' and the value is extracted from the local path 'p', e.g. k1=v1,k2<p2")
  54. var gubernator = flag.Bool("gubernator", false, "If true, output Gubernator link to view logs")
  55. var ginkgoFlags = flag.String("ginkgo-flags", "", "Passed to ginkgo to specify additional flags such as --skip=.")
  56. var systemSpecName = flag.String("system-spec-name", "", "The name of the system spec used for validating the image in the node conformance test. The specs are at test/e2e_node/system/specs/. If unspecified, the default built-in spec (system.DefaultSpec) will be used.")
  57. // envs is the type used to collect all node envs. The key is the env name,
  58. // and the value is the env value
  59. type envs map[string]string
  60. // String function of flag.Value
  61. func (e *envs) String() string {
  62. return fmt.Sprint(*e)
  63. }
  64. // Set function of flag.Value
  65. func (e *envs) Set(value string) error {
  66. kv := strings.SplitN(value, "=", 2)
  67. if len(kv) != 2 {
  68. return fmt.Errorf("invalid env string")
  69. }
  70. emap := *e
  71. emap[kv[0]] = kv[1]
  72. return nil
  73. }
  74. // nodeEnvs is the node envs from the flag `node-env`.
  75. var nodeEnvs = make(envs)
  76. func init() {
  77. flag.Var(&nodeEnvs, "node-env", "An environment variable passed to instance as metadata, e.g. when '--node-env=PATH=/usr/bin' is specified, there will be an extra instance metadata 'PATH=/usr/bin'.")
  78. }
  79. const (
  80. defaultMachine = "n1-standard-1"
  81. acceleratorTypeResourceFormat = "https://www.googleapis.com/compute/beta/projects/%s/zones/%s/acceleratorTypes/%s"
  82. )
  83. var (
  84. computeService *compute.Service
  85. arc Archive
  86. suite remote.TestSuite
  87. )
  88. type Archive struct {
  89. sync.Once
  90. path string
  91. err error
  92. }
  93. type TestResult struct {
  94. output string
  95. err error
  96. host string
  97. exitOk bool
  98. }
  99. // ImageConfig specifies what images should be run and how for these tests.
  100. // It can be created via the `--images` and `--image-project` flags, or by
  101. // specifying the `--image-config-file` flag, pointing to a json or yaml file
  102. // of the form:
  103. //
  104. // images:
  105. // short-name:
  106. // image: gce-image-name
  107. // project: gce-image-project
  108. // machine: for benchmark only, the machine type (GCE instance) to run test
  109. // tests: for benchmark only, a list of ginkgo focus strings to match tests
  110. // TODO(coufon): replace 'image' with 'node' in configurations
  111. // and we plan to support testing custom machines other than GCE by specifying host
  112. type ImageConfig struct {
  113. Images map[string]GCEImage `json:"images"`
  114. }
  115. type Accelerator struct {
  116. Type string `json:"type,omitempty"`
  117. Count int64 `json:"count,omitempty"`
  118. }
  119. type Resources struct {
  120. Accelerators []Accelerator `json:"accelerators,omitempty"`
  121. }
  122. type GCEImage struct {
  123. Image string `json:"image,omitempty"`
  124. ImageDesc string `json:"image_description,omitempty"`
  125. Project string `json:"project"`
  126. Metadata string `json:"metadata"`
  127. ImageRegex string `json:"image_regex,omitempty"`
  128. // Defaults to using only the latest image. Acceptable values are [0, # of images that match the regex).
  129. // If the number of existing previous images is lesser than what is desired, the test will use that is available.
  130. PreviousImages int `json:"previous_images,omitempty"`
  131. Machine string `json:"machine,omitempty"`
  132. Resources Resources `json:"resources,omitempty"`
  133. // This test is for benchmark (no limit verification, more result log, node name has format 'machine-image-uuid') if 'Tests' is non-empty.
  134. Tests []string `json:"tests,omitempty"`
  135. }
  136. type internalImageConfig struct {
  137. images map[string]internalGCEImage
  138. }
  139. type internalGCEImage struct {
  140. image string
  141. // imageDesc is the description of the image. If empty, the value in the
  142. // 'image' will be used.
  143. imageDesc string
  144. project string
  145. resources Resources
  146. metadata *compute.Metadata
  147. machine string
  148. tests []string
  149. }
  150. func main() {
  151. flag.Parse()
  152. switch *testSuite {
  153. case "conformance":
  154. suite = remote.InitConformanceRemote()
  155. case "cadvisor":
  156. suite = remote.InitCAdvisorE2ERemote()
  157. // TODO: Add subcommand for node soaking, node conformance, cri validation.
  158. case "default":
  159. // Use node e2e suite by default if no subcommand is specified.
  160. suite = remote.InitNodeE2ERemote()
  161. default:
  162. glog.Fatalf("--test-suite must be one of default or conformance")
  163. }
  164. rand.Seed(time.Now().UTC().UnixNano())
  165. if *buildOnly {
  166. // Build the archive and exit
  167. remote.CreateTestArchive(suite, *systemSpecName)
  168. return
  169. }
  170. if *hosts == "" && *imageConfigFile == "" && *images == "" {
  171. glog.Fatalf("Must specify one of --image-config-file, --hosts, --images.")
  172. }
  173. var err error
  174. computeService, err = getComputeClient()
  175. if err != nil {
  176. glog.Fatalf("Unable to create gcloud compute service using defaults. Make sure you are authenticated. %v", err)
  177. }
  178. gceImages := &internalImageConfig{
  179. images: make(map[string]internalGCEImage),
  180. }
  181. if *imageConfigFile != "" {
  182. configPath := *imageConfigFile
  183. if *imageConfigDir != "" {
  184. configPath = filepath.Join(*imageConfigDir, *imageConfigFile)
  185. }
  186. // parse images
  187. imageConfigData, err := ioutil.ReadFile(configPath)
  188. if err != nil {
  189. glog.Fatalf("Could not read image config file provided: %v", err)
  190. }
  191. externalImageConfig := ImageConfig{Images: make(map[string]GCEImage)}
  192. err = yaml.Unmarshal(imageConfigData, &externalImageConfig)
  193. if err != nil {
  194. glog.Fatalf("Could not parse image config file: %v", err)
  195. }
  196. for shortName, imageConfig := range externalImageConfig.Images {
  197. var images []string
  198. isRegex, name := false, shortName
  199. if imageConfig.ImageRegex != "" && imageConfig.Image == "" {
  200. isRegex = true
  201. images, err = getGCEImages(imageConfig.ImageRegex, imageConfig.Project, imageConfig.PreviousImages)
  202. if err != nil {
  203. glog.Fatalf("Could not retrieve list of images based on image prefix %q: %v", imageConfig.ImageRegex, err)
  204. }
  205. } else {
  206. images = []string{imageConfig.Image}
  207. }
  208. for _, image := range images {
  209. metadata := imageConfig.Metadata
  210. if len(strings.TrimSpace(*instanceMetadata)) > 0 {
  211. metadata += "," + *instanceMetadata
  212. }
  213. gceImage := internalGCEImage{
  214. image: image,
  215. imageDesc: imageConfig.ImageDesc,
  216. project: imageConfig.Project,
  217. metadata: getImageMetadata(metadata),
  218. machine: imageConfig.Machine,
  219. tests: imageConfig.Tests,
  220. resources: imageConfig.Resources,
  221. }
  222. if gceImage.imageDesc == "" {
  223. gceImage.imageDesc = gceImage.image
  224. }
  225. if isRegex && len(images) > 1 {
  226. // Use image name when shortName is not unique.
  227. name = image
  228. }
  229. gceImages.images[name] = gceImage
  230. }
  231. }
  232. }
  233. // Allow users to specify additional images via cli flags for local testing
  234. // convenience; merge in with config file
  235. if *images != "" {
  236. if *imageProject == "" {
  237. glog.Fatal("Must specify --image-project if you specify --images")
  238. }
  239. cliImages := strings.Split(*images, ",")
  240. for _, img := range cliImages {
  241. gceImage := internalGCEImage{
  242. image: img,
  243. project: *imageProject,
  244. metadata: getImageMetadata(*instanceMetadata),
  245. }
  246. gceImages.images[img] = gceImage
  247. }
  248. }
  249. if len(gceImages.images) != 0 && *zone == "" {
  250. glog.Fatal("Must specify --zone flag")
  251. }
  252. for shortName, image := range gceImages.images {
  253. if image.project == "" {
  254. glog.Fatalf("Invalid config for %v; must specify a project", shortName)
  255. }
  256. }
  257. if len(gceImages.images) != 0 {
  258. if *project == "" {
  259. glog.Fatal("Must specify --project flag to launch images into")
  260. }
  261. }
  262. if *instanceNamePrefix == "" {
  263. *instanceNamePrefix = "tmp-node-e2e-" + uuid.NewUUID().String()[:8]
  264. }
  265. // Setup coloring
  266. stat, _ := os.Stdout.Stat()
  267. useColor := (stat.Mode() & os.ModeCharDevice) != 0
  268. blue := ""
  269. noColour := ""
  270. if useColor {
  271. blue = "\033[0;34m"
  272. noColour = "\033[0m"
  273. }
  274. go arc.getArchive()
  275. defer arc.deleteArchive()
  276. results := make(chan *TestResult)
  277. running := 0
  278. for shortName := range gceImages.images {
  279. imageConfig := gceImages.images[shortName]
  280. fmt.Printf("Initializing e2e tests using image %s.\n", shortName)
  281. running++
  282. go func(image *internalGCEImage, junitFilePrefix string) {
  283. results <- testImage(image, junitFilePrefix)
  284. }(&imageConfig, shortName)
  285. }
  286. if *hosts != "" {
  287. for _, host := range strings.Split(*hosts, ",") {
  288. fmt.Printf("Initializing e2e tests using host %s.\n", host)
  289. running++
  290. go func(host string, junitFilePrefix string) {
  291. results <- testHost(host, *cleanup, "", junitFilePrefix, *ginkgoFlags)
  292. }(host, host)
  293. }
  294. }
  295. // Wait for all tests to complete and emit the results
  296. errCount := 0
  297. exitOk := true
  298. for i := 0; i < running; i++ {
  299. tr := <-results
  300. host := tr.host
  301. fmt.Println() // Print an empty line
  302. fmt.Printf("%s>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>%s\n", blue, noColour)
  303. fmt.Printf("%s> START TEST >%s\n", blue, noColour)
  304. fmt.Printf("%s>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>%s\n", blue, noColour)
  305. fmt.Printf("Start Test Suite on Host %s\n", host)
  306. fmt.Printf("%s\n", tr.output)
  307. if tr.err != nil {
  308. errCount++
  309. fmt.Printf("Failure Finished Test Suite on Host %s\n%v\n", host, tr.err)
  310. } else {
  311. fmt.Printf("Success Finished Test Suite on Host %s\n", host)
  312. }
  313. exitOk = exitOk && tr.exitOk
  314. fmt.Printf("%s<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<%s\n", blue, noColour)
  315. fmt.Printf("%s< FINISH TEST <%s\n", blue, noColour)
  316. fmt.Printf("%s<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<%s\n", blue, noColour)
  317. fmt.Println() // Print an empty line
  318. }
  319. // Set the exit code if there were failures
  320. if !exitOk {
  321. fmt.Printf("Failure: %d errors encountered.\n", errCount)
  322. callGubernator(*gubernator)
  323. arc.deleteArchive()
  324. os.Exit(1)
  325. }
  326. callGubernator(*gubernator)
  327. }
  328. func callGubernator(gubernator bool) {
  329. if gubernator {
  330. fmt.Println("Running gubernator.sh")
  331. output, err := exec.Command("./test/e2e_node/gubernator.sh", "y").Output()
  332. if err != nil {
  333. fmt.Println("gubernator.sh Failed")
  334. fmt.Println(err)
  335. return
  336. }
  337. fmt.Printf("%s", output)
  338. }
  339. return
  340. }
  341. func (a *Archive) getArchive() (string, error) {
  342. a.Do(func() { a.path, a.err = remote.CreateTestArchive(suite, *systemSpecName) })
  343. return a.path, a.err
  344. }
  345. func (a *Archive) deleteArchive() {
  346. path, err := a.getArchive()
  347. if err != nil {
  348. return
  349. }
  350. os.Remove(path)
  351. }
  352. func getImageMetadata(input string) *compute.Metadata {
  353. if input == "" {
  354. return nil
  355. }
  356. glog.V(3).Infof("parsing instance metadata: %q", input)
  357. raw := parseInstanceMetadata(input)
  358. glog.V(4).Infof("parsed instance metadata: %v", raw)
  359. metadataItems := []*compute.MetadataItems{}
  360. for k, v := range raw {
  361. val := v
  362. metadataItems = append(metadataItems, &compute.MetadataItems{
  363. Key: k,
  364. Value: &val,
  365. })
  366. }
  367. ret := compute.Metadata{Items: metadataItems}
  368. return &ret
  369. }
  370. // Run tests in archive against host
  371. func testHost(host string, deleteFiles bool, imageDesc, junitFilePrefix, ginkgoFlagsStr string) *TestResult {
  372. instance, err := computeService.Instances.Get(*project, *zone, host).Do()
  373. if err != nil {
  374. return &TestResult{
  375. err: err,
  376. host: host,
  377. exitOk: false,
  378. }
  379. }
  380. if strings.ToUpper(instance.Status) != "RUNNING" {
  381. err = fmt.Errorf("instance %s not in state RUNNING, was %s.", host, instance.Status)
  382. return &TestResult{
  383. err: err,
  384. host: host,
  385. exitOk: false,
  386. }
  387. }
  388. externalIp := getExternalIp(instance)
  389. if len(externalIp) > 0 {
  390. remote.AddHostnameIp(host, externalIp)
  391. }
  392. path, err := arc.getArchive()
  393. if err != nil {
  394. // Don't log fatal because we need to do any needed cleanup contained in "defer" statements
  395. return &TestResult{
  396. err: fmt.Errorf("unable to create test archive: %v.", err),
  397. }
  398. }
  399. output, exitOk, err := remote.RunRemote(suite, path, host, deleteFiles, imageDesc, junitFilePrefix, *testArgs, ginkgoFlagsStr, *systemSpecName)
  400. return &TestResult{
  401. output: output,
  402. err: err,
  403. host: host,
  404. exitOk: exitOk,
  405. }
  406. }
  407. type imageObj struct {
  408. creationTime time.Time
  409. name string
  410. }
  411. func (io imageObj) string() string {
  412. return fmt.Sprintf("%q created %q", io.name, io.creationTime.String())
  413. }
  414. type byCreationTime []imageObj
  415. func (a byCreationTime) Len() int { return len(a) }
  416. func (a byCreationTime) Less(i, j int) bool { return a[i].creationTime.After(a[j].creationTime) }
  417. func (a byCreationTime) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
  418. // Returns a list of image names based on regex and number of previous images requested.
  419. func getGCEImages(imageRegex, project string, previousImages int) ([]string, error) {
  420. ilc, err := computeService.Images.List(project).Do()
  421. if err != nil {
  422. return nil, fmt.Errorf("Failed to list images in project %q: %v", project, err)
  423. }
  424. imageObjs := []imageObj{}
  425. imageRe := regexp.MustCompile(imageRegex)
  426. for _, instance := range ilc.Items {
  427. if imageRe.MatchString(instance.Name) {
  428. creationTime, err := time.Parse(time.RFC3339, instance.CreationTimestamp)
  429. if err != nil {
  430. return nil, fmt.Errorf("Failed to parse instance creation timestamp %q: %v", instance.CreationTimestamp, err)
  431. }
  432. io := imageObj{
  433. creationTime: creationTime,
  434. name: instance.Name,
  435. }
  436. glog.V(4).Infof("Found image %q based on regex %q in project %q", io.string(), imageRegex, project)
  437. imageObjs = append(imageObjs, io)
  438. }
  439. }
  440. sort.Sort(byCreationTime(imageObjs))
  441. images := []string{}
  442. for _, imageObj := range imageObjs {
  443. images = append(images, imageObj.name)
  444. previousImages--
  445. if previousImages < 0 {
  446. break
  447. }
  448. }
  449. return images, nil
  450. }
  451. // Provision a gce instance using image and run the tests in archive against the instance.
  452. // Delete the instance afterward.
  453. func testImage(imageConfig *internalGCEImage, junitFilePrefix string) *TestResult {
  454. ginkgoFlagsStr := *ginkgoFlags
  455. // Check whether the test is for benchmark.
  456. if len(imageConfig.tests) > 0 {
  457. // Benchmark needs machine type non-empty.
  458. if imageConfig.machine == "" {
  459. imageConfig.machine = defaultMachine
  460. }
  461. // Use the Ginkgo focus in benchmark config.
  462. ginkgoFlagsStr += (" " + testsToGinkgoFocus(imageConfig.tests))
  463. }
  464. host, err := createInstance(imageConfig)
  465. if *deleteInstances {
  466. defer deleteInstance(host)
  467. }
  468. if err != nil {
  469. return &TestResult{
  470. err: fmt.Errorf("unable to create gce instance with running docker daemon for image %s. %v", imageConfig.image, err),
  471. }
  472. }
  473. // Only delete the files if we are keeping the instance and want it cleaned up.
  474. // If we are going to delete the instance, don't bother with cleaning up the files
  475. deleteFiles := !*deleteInstances && *cleanup
  476. result := testHost(host, deleteFiles, imageConfig.imageDesc, junitFilePrefix, ginkgoFlagsStr)
  477. // This is a temporary solution to collect serial node serial log. Only port 1 contains useful information.
  478. // TODO(random-liu): Extract out and unify log collection logic with cluste e2e.
  479. serialPortOutput, err := computeService.Instances.GetSerialPortOutput(*project, *zone, host).Port(1).Do()
  480. if err != nil {
  481. glog.Errorf("Failed to collect serial output from node %q: %v", host, err)
  482. } else {
  483. logFilename := "serial-1.log"
  484. err := remote.WriteLog(host, logFilename, serialPortOutput.Contents)
  485. if err != nil {
  486. glog.Errorf("Failed to write serial output from node %q to %q: %v", host, logFilename, err)
  487. }
  488. }
  489. return result
  490. }
  491. // Provision a gce instance using image
  492. func createInstance(imageConfig *internalGCEImage) (string, error) {
  493. glog.V(1).Infof("Creating instance %+v", *imageConfig)
  494. name := imageToInstanceName(imageConfig)
  495. i := &compute.Instance{
  496. Name: name,
  497. MachineType: machineType(imageConfig.machine),
  498. NetworkInterfaces: []*compute.NetworkInterface{
  499. {
  500. AccessConfigs: []*compute.AccessConfig{
  501. {
  502. Type: "ONE_TO_ONE_NAT",
  503. Name: "External NAT",
  504. },
  505. }},
  506. },
  507. Disks: []*compute.AttachedDisk{
  508. {
  509. AutoDelete: true,
  510. Boot: true,
  511. Type: "PERSISTENT",
  512. InitializeParams: &compute.AttachedDiskInitializeParams{
  513. SourceImage: sourceImage(imageConfig.image, imageConfig.project),
  514. DiskSizeGb: 20,
  515. },
  516. },
  517. },
  518. }
  519. for _, accelerator := range imageConfig.resources.Accelerators {
  520. if i.GuestAccelerators == nil {
  521. autoRestart := true
  522. i.GuestAccelerators = []*compute.AcceleratorConfig{}
  523. i.Scheduling = &compute.Scheduling{
  524. OnHostMaintenance: "TERMINATE",
  525. AutomaticRestart: &autoRestart,
  526. }
  527. }
  528. aType := fmt.Sprintf(acceleratorTypeResourceFormat, *project, *zone, accelerator.Type)
  529. ac := &compute.AcceleratorConfig{
  530. AcceleratorCount: accelerator.Count,
  531. AcceleratorType: aType,
  532. }
  533. i.GuestAccelerators = append(i.GuestAccelerators, ac)
  534. }
  535. var err error
  536. i.Metadata = imageConfig.metadata
  537. if _, err := computeService.Instances.Get(*project, *zone, i.Name).Do(); err != nil {
  538. op, err := computeService.Instances.Insert(*project, *zone, i).Do()
  539. if err != nil {
  540. ret := fmt.Sprintf("could not create instance %s: API error: %v", name, err)
  541. if op != nil {
  542. ret = fmt.Sprintf("%s: %v", ret, op.Error)
  543. }
  544. return "", fmt.Errorf(ret)
  545. } else if op.Error != nil {
  546. return "", fmt.Errorf("could not create instance %s: %+v", name, op.Error)
  547. }
  548. }
  549. instanceRunning := false
  550. for i := 0; i < 30 && !instanceRunning; i++ {
  551. if i > 0 {
  552. time.Sleep(time.Second * 20)
  553. }
  554. var instance *compute.Instance
  555. instance, err = computeService.Instances.Get(*project, *zone, name).Do()
  556. if err != nil {
  557. continue
  558. }
  559. if strings.ToUpper(instance.Status) != "RUNNING" {
  560. err = fmt.Errorf("instance %s not in state RUNNING, was %s.", name, instance.Status)
  561. continue
  562. }
  563. externalIp := getExternalIp(instance)
  564. if len(externalIp) > 0 {
  565. remote.AddHostnameIp(name, externalIp)
  566. }
  567. // TODO(random-liu): Remove the docker version check. Use some other command to check
  568. // instance readiness.
  569. var output string
  570. output, err = remote.SSH(name, "docker", "version")
  571. if err != nil {
  572. err = fmt.Errorf("instance %s not running docker daemon - Command failed: %s", name, output)
  573. continue
  574. }
  575. if !strings.Contains(output, "Server") {
  576. err = fmt.Errorf("instance %s not running docker daemon - Server not found: %s", name, output)
  577. continue
  578. }
  579. instanceRunning = true
  580. }
  581. // If instance didn't reach running state in time, return with error now.
  582. if err != nil {
  583. return name, err
  584. }
  585. // Instance reached running state in time, make sure that cloud-init is complete
  586. if isCloudInitUsed(imageConfig.metadata) {
  587. cloudInitFinished := false
  588. for i := 0; i < 60 && !cloudInitFinished; i++ {
  589. if i > 0 {
  590. time.Sleep(time.Second * 20)
  591. }
  592. var finished string
  593. finished, err = remote.SSH(name, "ls", "/var/lib/cloud/instance/boot-finished")
  594. if err != nil {
  595. err = fmt.Errorf("instance %s has not finished cloud-init script: %s", name, finished)
  596. continue
  597. }
  598. cloudInitFinished = true
  599. }
  600. }
  601. return name, err
  602. }
  603. func isCloudInitUsed(metadata *compute.Metadata) bool {
  604. if metadata == nil {
  605. return false
  606. }
  607. for _, item := range metadata.Items {
  608. if item.Key == "user-data" && item.Value != nil && strings.HasPrefix(*item.Value, "#cloud-config") {
  609. return true
  610. }
  611. }
  612. return false
  613. }
  614. func getExternalIp(instance *compute.Instance) string {
  615. for i := range instance.NetworkInterfaces {
  616. ni := instance.NetworkInterfaces[i]
  617. for j := range ni.AccessConfigs {
  618. ac := ni.AccessConfigs[j]
  619. if len(ac.NatIP) > 0 {
  620. return ac.NatIP
  621. }
  622. }
  623. }
  624. return ""
  625. }
  626. func getComputeClient() (*compute.Service, error) {
  627. const retries = 10
  628. const backoff = time.Second * 6
  629. // Setup the gce client for provisioning instances
  630. // Getting credentials on gce jenkins is flaky, so try a couple times
  631. var err error
  632. var cs *compute.Service
  633. for i := 0; i < retries; i++ {
  634. if i > 0 {
  635. time.Sleep(backoff)
  636. }
  637. var client *http.Client
  638. client, err = google.DefaultClient(oauth2.NoContext, compute.ComputeScope)
  639. if err != nil {
  640. continue
  641. }
  642. cs, err = compute.New(client)
  643. if err != nil {
  644. continue
  645. }
  646. return cs, nil
  647. }
  648. return nil, err
  649. }
  650. func deleteInstance(host string) {
  651. glog.Infof("Deleting instance %q", host)
  652. _, err := computeService.Instances.Delete(*project, *zone, host).Do()
  653. if err != nil {
  654. glog.Errorf("Error deleting instance %q: %v", host, err)
  655. }
  656. }
  657. func parseInstanceMetadata(str string) map[string]string {
  658. metadata := make(map[string]string)
  659. ss := strings.Split(str, ",")
  660. for _, s := range ss {
  661. kv := strings.Split(s, "=")
  662. if len(kv) == 2 {
  663. metadata[kv[0]] = kv[1]
  664. continue
  665. }
  666. kp := strings.Split(s, "<")
  667. if len(kp) != 2 {
  668. glog.Fatalf("Invalid instance metadata: %q", s)
  669. continue
  670. }
  671. metaPath := kp[1]
  672. if *imageConfigDir != "" {
  673. metaPath = filepath.Join(*imageConfigDir, metaPath)
  674. }
  675. v, err := ioutil.ReadFile(metaPath)
  676. if err != nil {
  677. glog.Fatalf("Failed to read metadata file %q: %v", metaPath, err)
  678. continue
  679. }
  680. metadata[kp[0]] = string(v)
  681. }
  682. for k, v := range nodeEnvs {
  683. metadata[k] = v
  684. }
  685. return metadata
  686. }
  687. func imageToInstanceName(imageConfig *internalGCEImage) string {
  688. if imageConfig.machine == "" {
  689. return *instanceNamePrefix + "-" + imageConfig.image
  690. }
  691. // For benchmark test, node name has the format 'machine-image-uuid' to run
  692. // different machine types with the same image in parallel
  693. return imageConfig.machine + "-" + imageConfig.image + "-" + uuid.NewUUID().String()[:8]
  694. }
  695. func sourceImage(image, imageProject string) string {
  696. return fmt.Sprintf("projects/%s/global/images/%s", imageProject, image)
  697. }
  698. func machineType(machine string) string {
  699. if machine == "" {
  700. machine = defaultMachine
  701. }
  702. return fmt.Sprintf("zones/%s/machineTypes/%s", *zone, machine)
  703. }
  704. // testsToGinkgoFocus converts the test string list to Ginkgo focus
  705. func testsToGinkgoFocus(tests []string) string {
  706. focus := "--focus=\""
  707. for i, test := range tests {
  708. if i == 0 {
  709. focus += test
  710. } else {
  711. focus += ("|" + test)
  712. }
  713. }
  714. return focus + "\""
  715. }