PageRenderTime 60ms CodeModel.GetById 23ms RepoModel.GetById 1ms app.codeStats 0ms

/test/extended/util/framework.go

https://gitlab.com/unofficial-mirrors/openshift-origin
Go | 1389 lines | 1055 code | 141 blank | 193 comment | 315 complexity | a5579c599a7812f9868bd468baec0ac2 MD5 | raw file
  1. package util
  2. import (
  3. "fmt"
  4. "io/ioutil"
  5. "net/http"
  6. "os"
  7. "os/exec"
  8. "path"
  9. "path/filepath"
  10. "regexp"
  11. "strings"
  12. "sync"
  13. "time"
  14. g "github.com/onsi/ginkgo"
  15. o "github.com/onsi/gomega"
  16. "k8s.io/kubernetes/pkg/api/legacyscheme"
  17. batchv1 "k8s.io/api/batch/v1"
  18. kapiv1 "k8s.io/api/core/v1"
  19. "k8s.io/apimachinery/pkg/api/errors"
  20. metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  21. "k8s.io/apimachinery/pkg/fields"
  22. "k8s.io/apimachinery/pkg/labels"
  23. "k8s.io/apimachinery/pkg/selection"
  24. "k8s.io/apimachinery/pkg/util/uuid"
  25. "k8s.io/apimachinery/pkg/util/wait"
  26. kclientset "k8s.io/client-go/kubernetes"
  27. kbatchclient "k8s.io/client-go/kubernetes/typed/batch/v1"
  28. kcoreclient "k8s.io/client-go/kubernetes/typed/core/v1"
  29. "k8s.io/kubernetes/pkg/apis/authorization"
  30. kapi "k8s.io/kubernetes/pkg/apis/core"
  31. kinternalcoreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion"
  32. "k8s.io/kubernetes/pkg/quota"
  33. e2e "k8s.io/kubernetes/test/e2e/framework"
  34. "github.com/openshift/origin/pkg/api/apihelpers"
  35. appsapi "github.com/openshift/origin/pkg/apps/apis/apps"
  36. appstypeclientset "github.com/openshift/origin/pkg/apps/generated/internalclientset/typed/apps/internalversion"
  37. appsutil "github.com/openshift/origin/pkg/apps/util"
  38. buildapi "github.com/openshift/origin/pkg/build/apis/build"
  39. buildtypedclientset "github.com/openshift/origin/pkg/build/generated/internalclientset/typed/build/internalversion"
  40. "github.com/openshift/origin/pkg/git"
  41. imageapi "github.com/openshift/origin/pkg/image/apis/image"
  42. imagetypeclientset "github.com/openshift/origin/pkg/image/generated/internalclientset/typed/image/internalversion"
  43. "github.com/openshift/origin/test/extended/testdata"
  44. "github.com/openshift/origin/test/util"
  45. )
  46. const pvPrefix = "pv-"
  47. const nfsPrefix = "nfs-"
  48. // WaitForOpenShiftNamespaceImageStreams waits for the standard set of imagestreams to be imported
  49. func WaitForOpenShiftNamespaceImageStreams(oc *CLI) error {
  50. langs := []string{"ruby", "nodejs", "perl", "php", "python", "wildfly", "mysql", "postgresql", "mongodb", "jenkins"}
  51. scan := func() bool {
  52. for _, lang := range langs {
  53. e2e.Logf("Checking language %v \n", lang)
  54. is, err := oc.ImageClient().Image().ImageStreams("openshift").Get(lang, metav1.GetOptions{})
  55. if err != nil {
  56. e2e.Logf("ImageStream Error: %#v \n", err)
  57. return false
  58. }
  59. for tag := range is.Spec.Tags {
  60. e2e.Logf("Checking tag %v \n", tag)
  61. if _, ok := is.Status.Tags[tag]; !ok {
  62. e2e.Logf("Tag Error: %#v \n", ok)
  63. return false
  64. }
  65. }
  66. }
  67. return true
  68. }
  69. success := false
  70. for i := 0; i < 10; i++ {
  71. e2e.Logf("Running scan #%v \n", i)
  72. success = scan()
  73. if success {
  74. break
  75. }
  76. e2e.Logf("Sleeping for 3 seconds \n")
  77. time.Sleep(3 * time.Second)
  78. }
  79. if success {
  80. e2e.Logf("Success! \n")
  81. return nil
  82. }
  83. DumpImageStreams(oc)
  84. return fmt.Errorf("Failed to import expected imagestreams")
  85. }
  86. // CheckOpenShiftNamespaceImageStreams is a temporary workaround for the intermittent
  87. // issue seen in extended tests where *something* is deleteing the pre-loaded, languange
  88. // imagestreams from the OpenShift namespace
  89. func CheckOpenShiftNamespaceImageStreams(oc *CLI) {
  90. missing := false
  91. langs := []string{"ruby", "nodejs", "perl", "php", "python", "wildfly", "mysql", "postgresql", "mongodb", "jenkins"}
  92. for _, lang := range langs {
  93. _, err := oc.ImageClient().Image().ImageStreams("openshift").Get(lang, metav1.GetOptions{})
  94. if err != nil {
  95. missing = true
  96. break
  97. }
  98. }
  99. if missing {
  100. fmt.Fprint(g.GinkgoWriter, "\n\n openshift namespace image streams corrupted \n\n")
  101. DumpImageStreams(oc)
  102. out, err := oc.Run("get").Args("is", "-n", "openshift", "--config", KubeConfigPath()).Output()
  103. err = fmt.Errorf("something has tampered with the image streams in the openshift namespace; look at audits in master log; \n%s\n", out)
  104. o.Expect(err).NotTo(o.HaveOccurred())
  105. } else {
  106. fmt.Fprint(g.GinkgoWriter, "\n\n openshift namespace image streams OK \n\n")
  107. }
  108. }
  109. //DumpImageStreams will dump both the openshift namespace and local namespace imagestreams
  110. // as part of debugging when the language imagestreams in the openshift namespace seem to disappear
  111. func DumpImageStreams(oc *CLI) {
  112. out, err := oc.AsAdmin().Run("get").Args("is", "-n", "openshift", "-o", "yaml", "--config", KubeConfigPath()).Output()
  113. if err == nil {
  114. e2e.Logf("\n imagestreams in openshift namespace: \n%s\n", out)
  115. } else {
  116. e2e.Logf("\n error on getting imagestreams in openshift namespace: %+v\n%#v\n", err, out)
  117. }
  118. out, err = oc.AsAdmin().Run("get").Args("is", "-o", "yaml").Output()
  119. if err == nil {
  120. e2e.Logf("\n imagestreams in dynamic test namespace: \n%s\n", out)
  121. } else {
  122. e2e.Logf("\n error on getting imagestreams in dynamic test namespace: %+v\n%#v\n", err, out)
  123. }
  124. ids, err := ListImages()
  125. if err != nil {
  126. e2e.Logf("\n got error on docker images %+v\n", err)
  127. } else {
  128. for _, id := range ids {
  129. e2e.Logf(" found local image %s\n", id)
  130. }
  131. }
  132. }
  133. // DumpBuildLogs will dump the latest build logs for a BuildConfig for debug purposes
  134. func DumpBuildLogs(bc string, oc *CLI) {
  135. buildOutput, err := oc.AsAdmin().Run("logs").Args("-f", "bc/"+bc, "--timestamps").Output()
  136. if err == nil {
  137. e2e.Logf("\n\n build logs : %s\n\n", buildOutput)
  138. } else {
  139. e2e.Logf("\n\n got error on build logs %+v\n\n", err)
  140. }
  141. // if we suspect that we are filling up the registry file system, call ExamineDiskUsage / ExaminePodDiskUsage
  142. // also see if manipulations of the quota around /mnt/openshift-xfs-vol-dir exist in the extended test set up scripts
  143. ExamineDiskUsage()
  144. ExaminePodDiskUsage(oc)
  145. }
  146. // DumpBuilds will dump the yaml for every build in the test namespace; remember, pipeline builds
  147. // don't have build pods so a generic framework dump won't cat our pipeline builds objs in openshift
  148. func DumpBuilds(oc *CLI) {
  149. buildOutput, err := oc.AsAdmin().Run("get").Args("builds", "-o", "yaml").Output()
  150. if err == nil {
  151. e2e.Logf("\n\n builds yaml:\n%s\n\n", buildOutput)
  152. } else {
  153. e2e.Logf("\n\n got error on build yaml dump: %#v\n\n", err)
  154. }
  155. }
  156. func GetDeploymentConfigPods(oc *CLI, dcName string, version int64) (*kapiv1.PodList, error) {
  157. return oc.AdminKubeClient().CoreV1().Pods(oc.Namespace()).List(metav1.ListOptions{LabelSelector: ParseLabelsOrDie(fmt.Sprintf("%s=%s-%d", appsapi.DeployerPodForDeploymentLabel, dcName, version)).String()})
  158. }
  159. func GetApplicationPods(oc *CLI, dcName string) (*kapiv1.PodList, error) {
  160. return oc.AdminKubeClient().CoreV1().Pods(oc.Namespace()).List(metav1.ListOptions{LabelSelector: ParseLabelsOrDie(fmt.Sprintf("deploymentconfig=%s", dcName)).String()})
  161. }
  162. func GetStatefulSetPods(oc *CLI, setName string) (*kapiv1.PodList, error) {
  163. return oc.AdminKubeClient().CoreV1().Pods(oc.Namespace()).List(metav1.ListOptions{LabelSelector: ParseLabelsOrDie(fmt.Sprintf("name=%s", setName)).String()})
  164. }
  165. // DumpDeploymentLogs will dump the latest deployment logs for a DeploymentConfig for debug purposes
  166. func DumpDeploymentLogs(dcName string, version int64, oc *CLI) {
  167. e2e.Logf("Dumping deployment logs for deploymentconfig %q\n", dcName)
  168. pods, err := GetDeploymentConfigPods(oc, dcName, version)
  169. if err != nil {
  170. e2e.Logf("Unable to retrieve pods for deploymentconfig %q: %v\n", dcName, err)
  171. return
  172. }
  173. DumpPodLogs(pods.Items, oc)
  174. }
  175. // DumpApplicationPodLogs will dump the latest application logs for a DeploymentConfig for debug purposes
  176. func DumpApplicationPodLogs(dcName string, oc *CLI) {
  177. e2e.Logf("Dumping application logs for deploymentconfig %q\n", dcName)
  178. pods, err := GetApplicationPods(oc, dcName)
  179. if err != nil {
  180. e2e.Logf("Unable to retrieve pods for deploymentconfig %q: %v\n", dcName, err)
  181. return
  182. }
  183. DumpPodLogs(pods.Items, oc)
  184. }
  185. func DumpPodStates(oc *CLI) {
  186. e2e.Logf("Dumping pod state for namespace %s", oc.Namespace())
  187. out, err := oc.AsAdmin().Run("get").Args("pods", "-o", "yaml").Output()
  188. if err != nil {
  189. e2e.Logf("Error dumping pod states: %v", err)
  190. return
  191. }
  192. e2e.Logf(out)
  193. }
  194. // DumpPodLogsStartingWith will dump any pod starting with the name prefix provided
  195. func DumpPodLogsStartingWith(prefix string, oc *CLI) {
  196. podsToDump := []kapiv1.Pod{}
  197. podList, err := oc.AdminKubeClient().CoreV1().Pods(oc.Namespace()).List(metav1.ListOptions{})
  198. if err != nil {
  199. e2e.Logf("Error listing pods: %v", err)
  200. return
  201. }
  202. for _, pod := range podList.Items {
  203. if strings.HasPrefix(pod.Name, prefix) {
  204. podsToDump = append(podsToDump, pod)
  205. }
  206. }
  207. if len(podsToDump) > 0 {
  208. DumpPodLogs(podsToDump, oc)
  209. }
  210. }
  211. // DumpPodLogsStartingWith will dump any pod starting with the name prefix provided
  212. func DumpPodLogsStartingWithInNamespace(prefix, namespace string, oc *CLI) {
  213. podsToDump := []kapiv1.Pod{}
  214. podList, err := oc.AdminKubeClient().CoreV1().Pods(namespace).List(metav1.ListOptions{})
  215. if err != nil {
  216. e2e.Logf("Error listing pods: %v", err)
  217. return
  218. }
  219. for _, pod := range podList.Items {
  220. if strings.HasPrefix(pod.Name, prefix) {
  221. podsToDump = append(podsToDump, pod)
  222. }
  223. }
  224. if len(podsToDump) > 0 {
  225. DumpPodLogs(podsToDump, oc)
  226. }
  227. }
  228. func DumpPodLogs(pods []kapiv1.Pod, oc *CLI) {
  229. for _, pod := range pods {
  230. descOutput, err := oc.AsAdmin().Run("describe").Args("pod/" + pod.Name).Output()
  231. if err == nil {
  232. e2e.Logf("Describing pod %q\n%s\n\n", pod.Name, descOutput)
  233. } else {
  234. e2e.Logf("Error retrieving description for pod %q: %v\n\n", pod.Name, err)
  235. }
  236. dumpContainer := func(container *kapiv1.Container) {
  237. depOutput, err := oc.AsAdmin().Run("logs").WithoutNamespace().Args("pod/"+pod.Name, "-c", container.Name, "-n", pod.Namespace).Output()
  238. if err == nil {
  239. e2e.Logf("Log for pod %q/%q\n---->\n%s\n<----end of log for %[1]q/%[2]q\n", pod.Name, container.Name, depOutput)
  240. } else {
  241. e2e.Logf("Error retrieving logs for pod %q/%q: %v\n\n", pod.Name, container.Name, err)
  242. }
  243. }
  244. for _, c := range pod.Spec.InitContainers {
  245. dumpContainer(&c)
  246. }
  247. for _, c := range pod.Spec.Containers {
  248. dumpContainer(&c)
  249. }
  250. }
  251. }
  252. // DumpPodsCommand runs the provided command in every pod identified by selector in the provided namespace.
  253. func DumpPodsCommand(c kclientset.Interface, ns string, selector labels.Selector, cmd string) {
  254. podList, err := c.CoreV1().Pods(ns).List(metav1.ListOptions{LabelSelector: selector.String()})
  255. o.Expect(err).NotTo(o.HaveOccurred())
  256. values := make(map[string]string)
  257. for _, pod := range podList.Items {
  258. stdout, err := e2e.RunHostCmdWithRetries(pod.Namespace, pod.Name, cmd, e2e.StatefulSetPoll, e2e.StatefulPodTimeout)
  259. o.Expect(err).NotTo(o.HaveOccurred())
  260. values[pod.Name] = stdout
  261. }
  262. for name, stdout := range values {
  263. stdout = strings.TrimSuffix(stdout, "\n")
  264. e2e.Logf(name + ": " + strings.Join(strings.Split(stdout, "\n"), fmt.Sprintf("\n%s: ", name)))
  265. }
  266. }
  267. // GetMasterThreadDump will get a golang thread stack dump
  268. func GetMasterThreadDump(oc *CLI) {
  269. out, err := oc.AsAdmin().Run("get").Args("--raw", "/debug/pprof/goroutine?debug=2").Output()
  270. if err == nil {
  271. e2e.Logf("\n\n Master thread stack dump:\n\n%s\n\n", string(out))
  272. return
  273. }
  274. e2e.Logf("\n\n got error on oc get --raw /debug/pprof/goroutine?godebug=2: %v\n\n", err)
  275. }
  276. // ExamineDiskUsage will dump df output on the testing system; leveraging this as part of diagnosing
  277. // the registry's disk filling up during external tests on jenkins
  278. func ExamineDiskUsage() {
  279. // disabling this for now, easier to do it here than everywhere that's calling it.
  280. return
  281. /*
  282. out, err := exec.Command("/bin/df", "-m").Output()
  283. if err == nil {
  284. e2e.Logf("\n\n df -m output: %s\n\n", string(out))
  285. } else {
  286. e2e.Logf("\n\n got error on df %v\n\n", err)
  287. }
  288. DumpDockerInfo()
  289. */
  290. }
  291. // DumpDockerInfo runs `docker info` and logs it to the job output
  292. func DumpDockerInfo() {
  293. out, err := exec.Command("/bin/docker", "info").Output()
  294. if err == nil {
  295. e2e.Logf("\n\n docker info output: \n%s\n\n", string(out))
  296. } else {
  297. e2e.Logf("\n\n got error on docker inspect %v\n\n", err)
  298. }
  299. }
  300. // ExaminePodDiskUsage will dump df/du output on registry pod; leveraging this as part of diagnosing
  301. // the registry's disk filling up during external tests on jenkins
  302. func ExaminePodDiskUsage(oc *CLI) {
  303. // disabling this for now, easier to do it here than everywhere that's calling it.
  304. return
  305. /*
  306. out, err := oc.Run("get").Args("pods", "-o", "json", "-n", "default", "--config", KubeConfigPath()).Output()
  307. var podName string
  308. if err == nil {
  309. b := []byte(out)
  310. var list kapiv1.PodList
  311. err = json.Unmarshal(b, &list)
  312. if err == nil {
  313. for _, pod := range list.Items {
  314. e2e.Logf("\n\n looking at pod %s \n\n", pod.ObjectMeta.Name)
  315. if strings.Contains(pod.ObjectMeta.Name, "docker-registry-") && !strings.Contains(pod.ObjectMeta.Name, "deploy") {
  316. podName = pod.ObjectMeta.Name
  317. break
  318. }
  319. }
  320. } else {
  321. e2e.Logf("\n\n got json unmarshal err: %v\n\n", err)
  322. }
  323. } else {
  324. e2e.Logf("\n\n got error on get pods: %v\n\n", err)
  325. }
  326. if len(podName) == 0 {
  327. e2e.Logf("Unable to determine registry pod name, so we can't examine its disk usage.")
  328. return
  329. }
  330. out, err = oc.Run("exec").Args("-n", "default", podName, "df", "--config", KubeConfigPath()).Output()
  331. if err == nil {
  332. e2e.Logf("\n\n df from registry pod: \n%s\n\n", out)
  333. } else {
  334. e2e.Logf("\n\n got error on reg pod df: %v\n", err)
  335. }
  336. out, err = oc.Run("exec").Args("-n", "default", podName, "du", "/registry", "--config", KubeConfigPath()).Output()
  337. if err == nil {
  338. e2e.Logf("\n\n du from registry pod: \n%s\n\n", out)
  339. } else {
  340. e2e.Logf("\n\n got error on reg pod du: %v\n", err)
  341. }
  342. */
  343. }
  344. // VarSubOnFile reads in srcFile, finds instances of ${key} from the map
  345. // and replaces them with their associated values.
  346. func VarSubOnFile(srcFile string, destFile string, vars map[string]string) error {
  347. srcData, err := ioutil.ReadFile(srcFile)
  348. if err == nil {
  349. srcString := string(srcData)
  350. for k, v := range vars {
  351. k = "${" + k + "}"
  352. srcString = strings.Replace(srcString, k, v, -1) // -1 means unlimited replacements
  353. }
  354. err = ioutil.WriteFile(destFile, []byte(srcString), 0644)
  355. }
  356. return err
  357. }
  358. // StartBuild executes OC start-build with the specified arguments. StdOut and StdErr from the process
  359. // are returned as separate strings.
  360. func StartBuild(oc *CLI, args ...string) (stdout, stderr string, err error) {
  361. stdout, stderr, err = oc.Run("start-build").Args(args...).Outputs()
  362. e2e.Logf("\n\nstart-build output with args %v:\nError>%v\nStdOut>\n%s\nStdErr>\n%s\n\n", args, err, stdout, stderr)
  363. return stdout, stderr, err
  364. }
  365. var buildPathPattern = regexp.MustCompile(`^build/([\w\-\._]+)$`)
  366. type LogDumperFunc func(oc *CLI, br *BuildResult) (string, error)
  367. func NewBuildResult(oc *CLI, build *buildapi.Build) *BuildResult {
  368. return &BuildResult{
  369. Oc: oc,
  370. BuildName: build.Name,
  371. BuildPath: "builds/" + build.Name,
  372. }
  373. }
  374. type BuildResult struct {
  375. // BuildPath is a resource qualified name (e.g. "build/test-1").
  376. BuildPath string
  377. // BuildName is the non-resource qualified name.
  378. BuildName string
  379. // StartBuildStdErr is the StdErr output generated by oc start-build.
  380. StartBuildStdErr string
  381. // StartBuildStdOut is the StdOut output generated by oc start-build.
  382. StartBuildStdOut string
  383. // StartBuildErr is the error, if any, returned by the direct invocation of the start-build command.
  384. StartBuildErr error
  385. // The buildconfig which generated this build.
  386. BuildConfigName string
  387. // Build is the resource created. May be nil if there was a timeout.
  388. Build *buildapi.Build
  389. // BuildAttempt represents that a Build resource was created.
  390. // false indicates a severe error unrelated to Build success or failure.
  391. BuildAttempt bool
  392. // BuildSuccess is true if the build was finshed successfully.
  393. BuildSuccess bool
  394. // BuildFailure is true if the build was finished with an error.
  395. BuildFailure bool
  396. // BuildCancelled is true if the build was canceled.
  397. BuildCancelled bool
  398. // BuildTimeout is true if there was a timeout waiting for the build to finish.
  399. BuildTimeout bool
  400. // Alternate log dumper function. If set, this is called instead of 'oc logs'
  401. LogDumper LogDumperFunc
  402. // The openshift client which created this build.
  403. Oc *CLI
  404. }
  405. // DumpLogs sends logs associated with this BuildResult to the GinkgoWriter.
  406. func (t *BuildResult) DumpLogs() {
  407. e2e.Logf("\n\n*****************************************\n")
  408. e2e.Logf("Dumping Build Result: %#v\n", *t)
  409. if t == nil {
  410. e2e.Logf("No build result available!\n\n")
  411. return
  412. }
  413. desc, err := t.Oc.Run("describe").Args(t.BuildPath).Output()
  414. e2e.Logf("\n** Build Description:\n")
  415. if err != nil {
  416. e2e.Logf("Error during description retrieval: %+v\n", err)
  417. } else {
  418. e2e.Logf("%s\n", desc)
  419. }
  420. e2e.Logf("\n** Build Logs:\n")
  421. buildOuput, err := t.Logs()
  422. if err != nil {
  423. e2e.Logf("Error during log retrieval: %+v\n", err)
  424. } else {
  425. e2e.Logf("%s\n", buildOuput)
  426. }
  427. e2e.Logf("\n\n")
  428. t.dumpRegistryLogs()
  429. // if we suspect that we are filling up the registry file system, call ExamineDiskUsage / ExaminePodDiskUsage
  430. // also see if manipulations of the quota around /mnt/openshift-xfs-vol-dir exist in the extended test set up scripts
  431. /*
  432. ExamineDiskUsage()
  433. ExaminePodDiskUsage(t.oc)
  434. e2e.Logf( "\n\n")
  435. */
  436. }
  437. func (t *BuildResult) dumpRegistryLogs() {
  438. var buildStarted *time.Time
  439. oc := t.Oc
  440. e2e.Logf("\n** Registry Logs:\n")
  441. if t.Build != nil && !t.Build.CreationTimestamp.IsZero() {
  442. buildStarted = &t.Build.CreationTimestamp.Time
  443. } else {
  444. proj, err := oc.ProjectClient().Project().Projects().Get(oc.Namespace(), metav1.GetOptions{})
  445. if err != nil {
  446. e2e.Logf("Failed to get project %s: %v\n", oc.Namespace(), err)
  447. } else {
  448. buildStarted = &proj.CreationTimestamp.Time
  449. }
  450. }
  451. if buildStarted == nil {
  452. e2e.Logf("Could not determine test' start time\n\n\n")
  453. return
  454. }
  455. since := time.Now().Sub(*buildStarted)
  456. // Changing the namespace on the derived client still changes it on the original client
  457. // because the kubeFramework field is only copied by reference. Saving the original namespace
  458. // here so we can restore it when done with registry logs
  459. savedNamespace := t.Oc.Namespace()
  460. oadm := t.Oc.AsAdmin().SetNamespace("default")
  461. out, err := oadm.Run("logs").Args("dc/docker-registry", "--since="+since.String()).Output()
  462. if err != nil {
  463. e2e.Logf("Error during log retrieval: %+v\n", err)
  464. } else {
  465. e2e.Logf("%s\n", out)
  466. }
  467. t.Oc.SetNamespace(savedNamespace)
  468. e2e.Logf("\n\n")
  469. }
  470. // Logs returns the logs associated with this build.
  471. func (t *BuildResult) Logs() (string, error) {
  472. if t == nil || t.BuildPath == "" {
  473. return "", fmt.Errorf("Not enough information to retrieve logs for %#v", *t)
  474. }
  475. if t.LogDumper != nil {
  476. return t.LogDumper(t.Oc, t)
  477. }
  478. buildOuput, err := t.Oc.Run("logs").Args("-f", t.BuildPath, "--timestamps").Output()
  479. if err != nil {
  480. return "", fmt.Errorf("Error retrieving logs for %#v: %v", *t, err)
  481. }
  482. return buildOuput, nil
  483. }
  484. // Dumps logs and triggers a Ginkgo assertion if the build did NOT succeed.
  485. func (t *BuildResult) AssertSuccess() *BuildResult {
  486. if !t.BuildSuccess {
  487. t.DumpLogs()
  488. }
  489. o.ExpectWithOffset(1, t.BuildSuccess).To(o.BeTrue())
  490. return t
  491. }
  492. // Dumps logs and triggers a Ginkgo assertion if the build did NOT have an error (this will not assert on timeouts)
  493. func (t *BuildResult) AssertFailure() *BuildResult {
  494. if !t.BuildFailure {
  495. t.DumpLogs()
  496. }
  497. o.ExpectWithOffset(1, t.BuildFailure).To(o.BeTrue())
  498. return t
  499. }
  500. func StartBuildResult(oc *CLI, args ...string) (result *BuildResult, err error) {
  501. args = append(args, "-o=name") // ensure that the build name is the only thing send to stdout
  502. stdout, stderr, err := StartBuild(oc, args...)
  503. // Usually, with -o=name, we only expect the build path.
  504. // However, the caller may have added --follow which can add
  505. // content to stdout. So just grab the first line.
  506. buildPath := strings.TrimSpace(strings.Split(stdout, "\n")[0])
  507. result = &BuildResult{
  508. Build: nil,
  509. BuildPath: buildPath,
  510. StartBuildStdOut: stdout,
  511. StartBuildStdErr: stderr,
  512. StartBuildErr: nil,
  513. BuildAttempt: false,
  514. BuildSuccess: false,
  515. BuildFailure: false,
  516. BuildCancelled: false,
  517. BuildTimeout: false,
  518. Oc: oc,
  519. }
  520. // An error here does not necessarily mean we could not run start-build. For example
  521. // when --wait is specified, start-build returns an error if the build fails. Therefore,
  522. // we continue to collect build information even if we see an error.
  523. result.StartBuildErr = err
  524. matches := buildPathPattern.FindStringSubmatch(buildPath)
  525. if len(matches) != 2 {
  526. return result, fmt.Errorf("Build path output did not match expected format 'build/name' : %q", buildPath)
  527. }
  528. result.BuildName = matches[1]
  529. return result, nil
  530. }
  531. // StartBuildAndWait executes OC start-build with the specified arguments on an existing buildconfig.
  532. // Note that start-build will be run with "-o=name" as a parameter when using this method.
  533. // If no error is returned from this method, it means that the build attempted successfully, NOT that
  534. // the build completed. For completion information, check the BuildResult object.
  535. func StartBuildAndWait(oc *CLI, args ...string) (result *BuildResult, err error) {
  536. result, err = StartBuildResult(oc, args...)
  537. if err != nil {
  538. return result, err
  539. }
  540. return result, WaitForBuildResult(oc.BuildClient().Build().Builds(oc.Namespace()), result)
  541. }
  542. // WaitForBuildResult updates result wit the state of the build
  543. func WaitForBuildResult(c buildtypedclientset.BuildResourceInterface, result *BuildResult) error {
  544. e2e.Logf("Waiting for %s to complete\n", result.BuildName)
  545. err := WaitForABuild(c, result.BuildName,
  546. func(b *buildapi.Build) bool {
  547. result.Build = b
  548. result.BuildSuccess = CheckBuildSuccess(b)
  549. return result.BuildSuccess
  550. },
  551. func(b *buildapi.Build) bool {
  552. result.Build = b
  553. result.BuildFailure = CheckBuildFailed(b)
  554. return result.BuildFailure
  555. },
  556. func(b *buildapi.Build) bool {
  557. result.Build = b
  558. result.BuildCancelled = CheckBuildCancelled(b)
  559. return result.BuildCancelled
  560. },
  561. )
  562. if result.Build == nil {
  563. // We only abort here if the build progress was unobservable. Only known cause would be severe, non-build related error in WaitForABuild.
  564. return fmt.Errorf("Severe error waiting for build: %v", err)
  565. }
  566. result.BuildAttempt = true
  567. result.BuildTimeout = !(result.BuildFailure || result.BuildSuccess || result.BuildCancelled)
  568. e2e.Logf("Done waiting for %s: %#v\n with error: %v\n", result.BuildName, *result, err)
  569. return nil
  570. }
  571. // WaitForABuild waits for a Build object to match either isOK or isFailed conditions.
  572. func WaitForABuild(c buildtypedclientset.BuildResourceInterface, name string, isOK, isFailed, isCanceled func(*buildapi.Build) bool) error {
  573. if isOK == nil {
  574. isOK = CheckBuildSuccess
  575. }
  576. if isFailed == nil {
  577. isFailed = CheckBuildFailed
  578. }
  579. if isCanceled == nil {
  580. isCanceled = CheckBuildCancelled
  581. }
  582. // wait 2 minutes for build to exist
  583. err := wait.Poll(1*time.Second, 2*time.Minute, func() (bool, error) {
  584. if _, err := c.Get(name, metav1.GetOptions{}); err != nil {
  585. return false, nil
  586. }
  587. return true, nil
  588. })
  589. if err == wait.ErrWaitTimeout {
  590. return fmt.Errorf("Timed out waiting for build %q to be created", name)
  591. }
  592. if err != nil {
  593. return err
  594. }
  595. // wait longer for the build to run to completion
  596. err = wait.Poll(5*time.Second, 60*time.Minute, func() (bool, error) {
  597. list, err := c.List(metav1.ListOptions{FieldSelector: fields.Set{"metadata.name": name}.AsSelector().String()})
  598. if err != nil {
  599. e2e.Logf("error listing builds: %v", err)
  600. return false, err
  601. }
  602. for i := range list.Items {
  603. if name == list.Items[i].Name && (isOK(&list.Items[i]) || isCanceled(&list.Items[i])) {
  604. return true, nil
  605. }
  606. if name != list.Items[i].Name {
  607. return false, fmt.Errorf("While listing builds named %s, found unexpected build %#v", name, list.Items[i])
  608. }
  609. if isFailed(&list.Items[i]) {
  610. return false, fmt.Errorf("The build %q status is %q", name, list.Items[i].Status.Phase)
  611. }
  612. }
  613. return false, nil
  614. })
  615. if err != nil {
  616. e2e.Logf("WaitForABuild returning with error: %v", err)
  617. }
  618. if err == wait.ErrWaitTimeout {
  619. return fmt.Errorf("Timed out waiting for build %q to complete", name)
  620. }
  621. return err
  622. }
  623. // CheckBuildSuccess returns true if the build succeeded
  624. func CheckBuildSuccess(b *buildapi.Build) bool {
  625. return b.Status.Phase == buildapi.BuildPhaseComplete
  626. }
  627. // CheckBuildFailed return true if the build failed
  628. func CheckBuildFailed(b *buildapi.Build) bool {
  629. return b.Status.Phase == buildapi.BuildPhaseFailed || b.Status.Phase == buildapi.BuildPhaseError
  630. }
  631. // CheckBuildCancelled return true if the build was canceled
  632. func CheckBuildCancelled(b *buildapi.Build) bool {
  633. return b.Status.Phase == buildapi.BuildPhaseCancelled
  634. }
  635. // WaitForBuilderAccount waits until the builder service account gets fully
  636. // provisioned
  637. func WaitForBuilderAccount(c kcoreclient.ServiceAccountInterface) error {
  638. waitFn := func() (bool, error) {
  639. sc, err := c.Get("builder", metav1.GetOptions{})
  640. if err != nil {
  641. // If we can't access the service accounts, let's wait till the controller
  642. // create it.
  643. if errors.IsForbidden(err) {
  644. return false, nil
  645. }
  646. return false, err
  647. }
  648. for _, s := range sc.Secrets {
  649. if strings.Contains(s.Name, "dockercfg") {
  650. return true, nil
  651. }
  652. }
  653. return false, nil
  654. }
  655. return wait.Poll(time.Duration(100*time.Millisecond), 1*time.Minute, waitFn)
  656. }
  657. // WaitForAnImageStream waits for an ImageStream to fulfill the isOK function
  658. func WaitForAnImageStream(client imagetypeclientset.ImageStreamInterface,
  659. name string,
  660. isOK, isFailed func(*imageapi.ImageStream) bool) error {
  661. for {
  662. list, err := client.List(metav1.ListOptions{FieldSelector: fields.Set{"metadata.name": name}.AsSelector().String()})
  663. if err != nil {
  664. return err
  665. }
  666. for i := range list.Items {
  667. if isOK(&list.Items[i]) {
  668. return nil
  669. }
  670. if isFailed(&list.Items[i]) {
  671. return fmt.Errorf("The image stream %q status is %q",
  672. name, list.Items[i].Annotations[imageapi.DockerImageRepositoryCheckAnnotation])
  673. }
  674. }
  675. rv := list.ResourceVersion
  676. w, err := client.Watch(metav1.ListOptions{FieldSelector: fields.Set{"metadata.name": name}.AsSelector().String(), ResourceVersion: rv})
  677. if err != nil {
  678. return err
  679. }
  680. defer w.Stop()
  681. for {
  682. val, ok := <-w.ResultChan()
  683. if !ok {
  684. // reget and re-watch
  685. break
  686. }
  687. if e, ok := val.Object.(*imageapi.ImageStream); ok {
  688. if isOK(e) {
  689. return nil
  690. }
  691. if isFailed(e) {
  692. return fmt.Errorf("The image stream %q status is %q",
  693. name, e.Annotations[imageapi.DockerImageRepositoryCheckAnnotation])
  694. }
  695. }
  696. }
  697. }
  698. }
  699. // WaitForAnImageStreamTag waits until an image stream with given name has non-empty history for given tag.
  700. // Defaults to waiting for 300 seconds
  701. func WaitForAnImageStreamTag(oc *CLI, namespace, name, tag string) error {
  702. return TimedWaitForAnImageStreamTag(oc, namespace, name, tag, time.Second*300)
  703. }
  704. // TimedWaitForAnImageStreamTag waits until an image stream with given name has non-empty history for given tag.
  705. // Gives up waiting after the specified waitTimeout
  706. func TimedWaitForAnImageStreamTag(oc *CLI, namespace, name, tag string, waitTimeout time.Duration) error {
  707. g.By(fmt.Sprintf("waiting for an is importer to import a tag %s into a stream %s", tag, name))
  708. start := time.Now()
  709. c := make(chan error)
  710. go func() {
  711. err := WaitForAnImageStream(
  712. oc.ImageClient().Image().ImageStreams(namespace),
  713. name,
  714. func(is *imageapi.ImageStream) bool {
  715. if history, exists := is.Status.Tags[tag]; !exists || len(history.Items) == 0 {
  716. return false
  717. }
  718. return true
  719. },
  720. func(is *imageapi.ImageStream) bool {
  721. return time.Now().After(start.Add(waitTimeout))
  722. })
  723. c <- err
  724. }()
  725. select {
  726. case e := <-c:
  727. return e
  728. case <-time.After(waitTimeout):
  729. return fmt.Errorf("timed out while waiting of an image stream tag %s/%s:%s", namespace, name, tag)
  730. }
  731. }
  732. // CheckImageStreamLatestTagPopulated returns true if the imagestream has a ':latest' tag filed
  733. func CheckImageStreamLatestTagPopulated(i *imageapi.ImageStream) bool {
  734. _, ok := i.Status.Tags["latest"]
  735. return ok
  736. }
  737. // CheckImageStreamTagNotFound return true if the imagestream update was not successful
  738. func CheckImageStreamTagNotFound(i *imageapi.ImageStream) bool {
  739. return strings.Contains(i.Annotations[imageapi.DockerImageRepositoryCheckAnnotation], "not") ||
  740. strings.Contains(i.Annotations[imageapi.DockerImageRepositoryCheckAnnotation], "error")
  741. }
  742. // WaitForDeploymentConfig waits for a DeploymentConfig to complete transition
  743. // to a given version and report minimum availability.
  744. func WaitForDeploymentConfig(kc kclientset.Interface, dcClient appstypeclientset.DeploymentConfigsGetter, namespace, name string, version int64, enforceNotProgressing bool, cli *CLI) error {
  745. e2e.Logf("waiting for deploymentconfig %s/%s to be available with version %d\n", namespace, name, version)
  746. var dc *appsapi.DeploymentConfig
  747. start := time.Now()
  748. err := wait.Poll(time.Second, 15*time.Minute, func() (done bool, err error) {
  749. dc, err = dcClient.DeploymentConfigs(namespace).Get(name, metav1.GetOptions{})
  750. if err != nil {
  751. return false, err
  752. }
  753. // TODO re-enable this check once @mfojtik introduces a test that ensures we'll only ever get
  754. // exactly one deployment triggered.
  755. /*
  756. if dc.Status.LatestVersion > version {
  757. return false, fmt.Errorf("latestVersion %d passed %d", dc.Status.LatestVersion, version)
  758. }
  759. */
  760. if dc.Status.LatestVersion < version {
  761. return false, nil
  762. }
  763. var progressing, available *appsapi.DeploymentCondition
  764. for i, condition := range dc.Status.Conditions {
  765. switch condition.Type {
  766. case appsapi.DeploymentProgressing:
  767. progressing = &dc.Status.Conditions[i]
  768. case appsapi.DeploymentAvailable:
  769. available = &dc.Status.Conditions[i]
  770. }
  771. }
  772. if enforceNotProgressing {
  773. if progressing != nil && progressing.Status == kapi.ConditionFalse {
  774. return false, fmt.Errorf("not progressing")
  775. }
  776. }
  777. if progressing != nil &&
  778. progressing.Status == kapi.ConditionTrue &&
  779. progressing.Reason == appsapi.NewRcAvailableReason &&
  780. available != nil &&
  781. available.Status == kapi.ConditionTrue {
  782. return true, nil
  783. }
  784. return false, nil
  785. })
  786. if err != nil {
  787. e2e.Logf("got error %q when waiting for deploymentconfig %s/%s to be available with version %d\n", err, namespace, name, version)
  788. cli.Run("get").Args("dc", dc.Name, "-o", "yaml").Execute()
  789. DumpDeploymentLogs(name, version, cli)
  790. DumpApplicationPodLogs(name, cli)
  791. return err
  792. }
  793. requirement, err := labels.NewRequirement(appsapi.DeploymentLabel, selection.Equals, []string{appsutil.LatestDeploymentNameForConfig(dc)})
  794. if err != nil {
  795. return err
  796. }
  797. podnames, err := GetPodNamesByFilter(kc.CoreV1().Pods(namespace), labels.NewSelector().Add(*requirement), func(kapiv1.Pod) bool { return true })
  798. if err != nil {
  799. return err
  800. }
  801. e2e.Logf("deploymentconfig %s/%s available after %s\npods: %s\n", namespace, name, time.Now().Sub(start), strings.Join(podnames, ", "))
  802. return nil
  803. }
  804. func isUsageSynced(received, expected kapi.ResourceList, expectedIsUpperLimit bool) bool {
  805. resourceNames := quota.ResourceNames(expected)
  806. masked := quota.Mask(received, resourceNames)
  807. if len(masked) != len(expected) {
  808. return false
  809. }
  810. if expectedIsUpperLimit {
  811. if le, _ := quota.LessThanOrEqual(masked, expected); !le {
  812. return false
  813. }
  814. } else {
  815. if le, _ := quota.LessThanOrEqual(expected, masked); !le {
  816. return false
  817. }
  818. }
  819. return true
  820. }
  821. // WaitForResourceQuotaSync watches given resource quota until its usage is updated to desired level or a
  822. // timeout occurs. If successful, used quota values will be returned for expected resources. Otherwise an
  823. // ErrWaitTimeout will be returned. If expectedIsUpperLimit is true, given expected usage must compare greater
  824. // or equal to quota's usage, which is useful for expected usage increment. Otherwise expected usage must
  825. // compare lower or equal to quota's usage, which is useful for expected usage decrement.
  826. func WaitForResourceQuotaSync(
  827. client kinternalcoreclient.ResourceQuotaInterface,
  828. name string,
  829. expectedUsage kapi.ResourceList,
  830. expectedIsUpperLimit bool,
  831. timeout time.Duration,
  832. ) (kapi.ResourceList, error) {
  833. startTime := time.Now()
  834. endTime := startTime.Add(timeout)
  835. expectedResourceNames := quota.ResourceNames(expectedUsage)
  836. list, err := client.List(metav1.ListOptions{FieldSelector: fields.Set{"metadata.name": name}.AsSelector().String()})
  837. if err != nil {
  838. return nil, err
  839. }
  840. for i := range list.Items {
  841. used := quota.Mask(list.Items[i].Status.Used, expectedResourceNames)
  842. if isUsageSynced(used, expectedUsage, expectedIsUpperLimit) {
  843. return used, nil
  844. }
  845. }
  846. rv := list.ResourceVersion
  847. w, err := client.Watch(metav1.ListOptions{FieldSelector: fields.Set{"metadata.name": name}.AsSelector().String(), ResourceVersion: rv})
  848. if err != nil {
  849. return nil, err
  850. }
  851. defer w.Stop()
  852. for time.Now().Before(endTime) {
  853. select {
  854. case val, ok := <-w.ResultChan():
  855. if !ok {
  856. // reget and re-watch
  857. continue
  858. }
  859. if rq, ok := val.Object.(*kapi.ResourceQuota); ok {
  860. used := quota.Mask(rq.Status.Used, expectedResourceNames)
  861. if isUsageSynced(used, expectedUsage, expectedIsUpperLimit) {
  862. return used, nil
  863. }
  864. }
  865. case <-time.After(endTime.Sub(time.Now())):
  866. return nil, wait.ErrWaitTimeout
  867. }
  868. }
  869. return nil, wait.ErrWaitTimeout
  870. }
  871. // GetPodNamesByFilter looks up pods that satisfy the predicate and returns their names.
  872. func GetPodNamesByFilter(c kcoreclient.PodInterface, label labels.Selector, predicate func(kapiv1.Pod) bool) (podNames []string, err error) {
  873. podList, err := c.List(metav1.ListOptions{LabelSelector: label.String()})
  874. if err != nil {
  875. return nil, err
  876. }
  877. for _, pod := range podList.Items {
  878. if predicate(pod) {
  879. podNames = append(podNames, pod.Name)
  880. }
  881. }
  882. return podNames, nil
  883. }
  884. func WaitForAJob(c kbatchclient.JobInterface, name string, timeout time.Duration) error {
  885. return wait.Poll(1*time.Second, timeout, func() (bool, error) {
  886. j, e := c.Get(name, metav1.GetOptions{})
  887. if e != nil {
  888. return true, e
  889. }
  890. // TODO soltysh: replace this with a function once such exist, currently
  891. // it's private in the controller
  892. for _, c := range j.Status.Conditions {
  893. if (c.Type == batchv1.JobComplete || c.Type == batchv1.JobFailed) && c.Status == kapiv1.ConditionTrue {
  894. return true, nil
  895. }
  896. }
  897. return false, nil
  898. })
  899. }
  900. // WaitForPods waits until given number of pods that match the label selector and
  901. // satisfy the predicate are found
  902. func WaitForPods(c kcoreclient.PodInterface, label labels.Selector, predicate func(kapiv1.Pod) bool, count int, timeout time.Duration) ([]string, error) {
  903. var podNames []string
  904. err := wait.Poll(1*time.Second, timeout, func() (bool, error) {
  905. p, e := GetPodNamesByFilter(c, label, predicate)
  906. if e != nil {
  907. return true, e
  908. }
  909. if len(p) != count {
  910. return false, nil
  911. }
  912. podNames = p
  913. return true, nil
  914. })
  915. return podNames, err
  916. }
  917. // CheckPodIsRunning returns true if the pod is running
  918. func CheckPodIsRunning(pod kapiv1.Pod) bool {
  919. return pod.Status.Phase == kapiv1.PodRunning
  920. }
  921. // CheckPodIsSucceeded returns true if the pod status is "Succdeded"
  922. func CheckPodIsSucceeded(pod kapiv1.Pod) bool {
  923. return pod.Status.Phase == kapiv1.PodSucceeded
  924. }
  925. // CheckPodIsReady returns true if the pod's ready probe determined that the pod is ready.
  926. func CheckPodIsReady(pod kapiv1.Pod) bool {
  927. if pod.Status.Phase != kapiv1.PodRunning {
  928. return false
  929. }
  930. for _, cond := range pod.Status.Conditions {
  931. if cond.Type != kapiv1.PodReady {
  932. continue
  933. }
  934. return cond.Status == kapiv1.ConditionTrue
  935. }
  936. return false
  937. }
  938. // CheckPodNoOp always returns true
  939. func CheckPodNoOp(pod kapiv1.Pod) bool {
  940. return true
  941. }
  942. // WaitUntilPodIsGone waits until the named Pod will disappear
  943. func WaitUntilPodIsGone(c kcoreclient.PodInterface, podName string, timeout time.Duration) error {
  944. return wait.Poll(1*time.Second, timeout, func() (bool, error) {
  945. _, err := c.Get(podName, metav1.GetOptions{})
  946. if err != nil {
  947. if strings.Contains(err.Error(), "not found") {
  948. return true, nil
  949. }
  950. return true, err
  951. }
  952. return false, nil
  953. })
  954. }
  955. // GetDockerImageReference retrieves the full Docker pull spec from the given ImageStream
  956. // and tag
  957. func GetDockerImageReference(c imagetypeclientset.ImageStreamInterface, name, tag string) (string, error) {
  958. imageStream, err := c.Get(name, metav1.GetOptions{})
  959. if err != nil {
  960. return "", err
  961. }
  962. isTag, ok := imageStream.Status.Tags[tag]
  963. if !ok {
  964. return "", fmt.Errorf("ImageStream %q does not have tag %q", name, tag)
  965. }
  966. if len(isTag.Items) == 0 {
  967. return "", fmt.Errorf("ImageStreamTag %q is empty", tag)
  968. }
  969. return isTag.Items[0].DockerImageReference, nil
  970. }
  971. // GetPodForContainer creates a new Pod that runs specified container
  972. func GetPodForContainer(container kapiv1.Container) *kapiv1.Pod {
  973. name := apihelpers.GetPodName("test-pod", string(uuid.NewUUID()))
  974. return &kapiv1.Pod{
  975. TypeMeta: metav1.TypeMeta{
  976. Kind: "Pod",
  977. APIVersion: "v1",
  978. },
  979. ObjectMeta: metav1.ObjectMeta{
  980. Name: name,
  981. Labels: map[string]string{"name": name},
  982. },
  983. Spec: kapiv1.PodSpec{
  984. Containers: []kapiv1.Container{container},
  985. RestartPolicy: kapiv1.RestartPolicyNever,
  986. },
  987. }
  988. }
  989. // KubeConfigPath returns the value of KUBECONFIG environment variable
  990. func KubeConfigPath() string {
  991. // can't use gomega in this method since it is used outside of It()
  992. return os.Getenv("KUBECONFIG")
  993. }
  994. //ArtifactDirPath returns the value of ARTIFACT_DIR environment variable
  995. func ArtifactDirPath() string {
  996. path := os.Getenv("ARTIFACT_DIR")
  997. o.Expect(path).NotTo(o.BeNil())
  998. o.Expect(path).NotTo(o.BeEmpty())
  999. return path
  1000. }
  1001. //ArtifactPath returns the absolute path to the fix artifact file
  1002. //The path is relative to ARTIFACT_DIR
  1003. func ArtifactPath(elem ...string) string {
  1004. return filepath.Join(append([]string{ArtifactDirPath()}, elem...)...)
  1005. }
  1006. var (
  1007. fixtureDirLock sync.Once
  1008. fixtureDir string
  1009. )
  1010. // FixturePath returns an absolute path to a fixture file in test/extended/testdata/,
  1011. // test/integration/, or examples/.
  1012. func FixturePath(elem ...string) string {
  1013. switch {
  1014. case len(elem) == 0:
  1015. panic("must specify path")
  1016. case len(elem) > 3 && elem[0] == ".." && elem[1] == ".." && elem[2] == "examples":
  1017. elem = elem[2:]
  1018. case len(elem) > 3 && elem[0] == ".." && elem[1] == ".." && elem[2] == "install":
  1019. elem = elem[2:]
  1020. case len(elem) > 3 && elem[0] == ".." && elem[1] == "integration":
  1021. elem = append([]string{"test"}, elem[1:]...)
  1022. case elem[0] == "testdata":
  1023. elem = append([]string{"test", "extended"}, elem...)
  1024. default:
  1025. panic(fmt.Sprintf("Fixtures must be in test/extended/testdata or examples not %s", path.Join(elem...)))
  1026. }
  1027. fixtureDirLock.Do(func() {
  1028. dir, err := ioutil.TempDir("", "fixture-testdata-dir")
  1029. if err != nil {
  1030. panic(err)
  1031. }
  1032. fixtureDir = dir
  1033. })
  1034. relativePath := path.Join(elem...)
  1035. fullPath := path.Join(fixtureDir, relativePath)
  1036. if err := testdata.RestoreAsset(fixtureDir, relativePath); err != nil {
  1037. if err := testdata.RestoreAssets(fixtureDir, relativePath); err != nil {
  1038. panic(err)
  1039. }
  1040. if err := filepath.Walk(fullPath, func(path string, info os.FileInfo, err error) error {
  1041. if err := os.Chmod(path, 0640); err != nil {
  1042. return err
  1043. }
  1044. if stat, err := os.Lstat(path); err == nil && stat.IsDir() {
  1045. return os.Chmod(path, 0755)
  1046. }
  1047. return nil
  1048. }); err != nil {
  1049. panic(err)
  1050. }
  1051. } else {
  1052. if err := os.Chmod(fullPath, 0640); err != nil {
  1053. panic(err)
  1054. }
  1055. }
  1056. p, err := filepath.Abs(fullPath)
  1057. if err != nil {
  1058. panic(err)
  1059. }
  1060. return p
  1061. }
  1062. // FetchURL grabs the output from the specified url and returns it.
  1063. // It will retry once per second for duration retryTimeout if an error occurs during the request.
  1064. func FetchURL(url string, retryTimeout time.Duration) (response string, err error) {
  1065. waitFn := func() (bool, error) {
  1066. r, err := http.Get(url)
  1067. if err != nil || r.StatusCode != 200 {
  1068. // lie to the poller that we didn't get an error even though we did
  1069. // because otherwise it's going to give up.
  1070. if err != nil {
  1071. e2e.Logf("error fetching url: %v", err)
  1072. }
  1073. if r != nil {
  1074. e2e.Logf("non-200 status code fetching url: %d", r.StatusCode)
  1075. }
  1076. return false, nil
  1077. }
  1078. defer r.Body.Close()
  1079. bytes, err := ioutil.ReadAll(r.Body)
  1080. response = string(bytes)
  1081. return true, nil
  1082. }
  1083. pollErr := wait.Poll(time.Duration(1*time.Second), retryTimeout, waitFn)
  1084. if pollErr == wait.ErrWaitTimeout {
  1085. return "", fmt.Errorf("Timed out while fetching url %q", url)
  1086. }
  1087. if pollErr != nil {
  1088. return "", pollErr
  1089. }
  1090. return
  1091. }
  1092. // ParseLabelsOrDie turns the given string into a label selector or
  1093. // panics; for tests or other cases where you know the string is valid.
  1094. // TODO: Move this to the upstream labels package.
  1095. func ParseLabelsOrDie(str string) labels.Selector {
  1096. ret, err := labels.Parse(str)
  1097. if err != nil {
  1098. panic(fmt.Sprintf("cannot parse '%v': %v", str, err))
  1099. }
  1100. return ret
  1101. }
  1102. // GetEndpointAddress will return an "ip:port" string for the endpoint.
  1103. func GetEndpointAddress(oc *CLI, name string) (string, error) {
  1104. err := e2e.WaitForEndpoint(oc.KubeFramework().ClientSet, oc.Namespace(), name)
  1105. if err != nil {
  1106. return "", err
  1107. }
  1108. endpoint, err := oc.KubeClient().CoreV1().Endpoints(oc.Namespace()).Get(name, metav1.GetOptions{})
  1109. if err != nil {
  1110. return "", err
  1111. }
  1112. return fmt.Sprintf("%s:%d", endpoint.Subsets[0].Addresses[0].IP, endpoint.Subsets[0].Ports[0].Port), nil
  1113. }
  1114. // CreateExecPodOrFail creates a simple busybox pod in a sleep loop used as a
  1115. // vessel for kubectl exec commands.
  1116. // Returns the name of the created pod.
  1117. // TODO: expose upstream
  1118. func CreateExecPodOrFail(client kcoreclient.CoreV1Interface, ns, name string) string {
  1119. e2e.Logf("Creating new exec pod")
  1120. execPod := e2e.NewHostExecPodSpec(ns, name)
  1121. created, err := client.Pods(ns).Create(execPod)
  1122. o.Expect(err).NotTo(o.HaveOccurred())
  1123. err = wait.PollImmediate(e2e.Poll, 5*time.Minute, func() (bool, error) {
  1124. retrievedPod, err := client.Pods(execPod.Namespace).Get(created.Name, metav1.GetOptions{})
  1125. if err != nil {
  1126. return false, nil
  1127. }
  1128. return retrievedPod.Status.Phase == kapiv1.PodRunning, nil
  1129. })
  1130. o.Expect(err).NotTo(o.HaveOccurred())
  1131. return created.Name
  1132. }
  1133. // CheckForBuildEvent will poll a build for up to 1 minute looking for an event with
  1134. // the specified reason and message template.
  1135. func CheckForBuildEvent(client kcoreclient.CoreV1Interface, build *buildapi.Build, reason, message string) {
  1136. var expectedEvent *kapiv1.Event
  1137. err := wait.PollImmediate(e2e.Poll, 1*time.Minute, func() (bool, error) {
  1138. events, err := client.Events(build.Namespace).Search(legacyscheme.Scheme, build)
  1139. if err != nil {
  1140. return false, err
  1141. }
  1142. for _, event := range events.Items {
  1143. e2e.Logf("Found event %#v", event)
  1144. if reason == event.Reason {
  1145. expectedEvent = &event
  1146. return true, nil
  1147. }
  1148. }
  1149. return false, nil
  1150. })
  1151. o.ExpectWithOffset(1, err).NotTo(o.HaveOccurred(), "Should be able to get events from the build")
  1152. o.ExpectWithOffset(1, expectedEvent).NotTo(o.BeNil(), "Did not find a %q event on build %s/%s", reason, build.Namespace, build.Name)
  1153. o.ExpectWithOffset(1, expectedEvent.Message).To(o.Equal(fmt.Sprintf(message, build.Namespace, build.Name)))
  1154. }
  1155. type podExecutor struct {
  1156. client *CLI
  1157. podName string
  1158. }
  1159. // NewPodExecutor returns an executor capable of running commands in a Pod.
  1160. func NewPodExecutor(oc *CLI, name, image string) (*podExecutor, error) {
  1161. out, err := oc.Run("run").Args(name, "--labels", "name="+name, "--image", image, "--restart", "Never", "--command", "--", "/bin/bash", "-c", "sleep infinity").Output()
  1162. if err != nil {
  1163. return nil, fmt.Errorf("error: %v\n(%s)", err, out)
  1164. }
  1165. _, err = WaitForPods(oc.KubeClient().CoreV1().Pods(oc.Namespace()), ParseLabelsOrDie("name="+name), CheckPodIsReady, 1, 3*time.Minute)
  1166. if err != nil {
  1167. return nil, err
  1168. }
  1169. return &podExecutor{client: oc, podName: name}, nil
  1170. }
  1171. // Exec executes a single command or a bash script in the running pod. It returns the
  1172. // command output and error if the command finished with non-zero status code or the
  1173. // command took longer then 3 minutes to run.
  1174. func (r *podExecutor) Exec(script string) (string, error) {
  1175. var out string
  1176. waitErr := wait.PollImmediate(1*time.Second, 3*time.Minute, func() (bool, error) {
  1177. var err error
  1178. out, err = r.client.Run("exec").Args(r.podName, "--", "/bin/bash", "-c", script).Output()
  1179. return true, err
  1180. })
  1181. return out, waitErr
  1182. }
  1183. type GitRepo struct {
  1184. baseTempDir string
  1185. upstream git.Repository
  1186. upstreamPath string
  1187. repo git.Repository
  1188. RepoPath string
  1189. }
  1190. // AddAndCommit commits a file with its content to local repo
  1191. func (r GitRepo) AddAndCommit(file, content string) error {
  1192. if err := ioutil.WriteFile(filepath.Join(r.RepoPath, file), []byte(content), 0666); err != nil {
  1193. return err
  1194. }
  1195. if err := r.repo.Add(r.RepoPath, file); err != nil {
  1196. return err
  1197. }
  1198. if err := r.repo.Commit(r.RepoPath, "added file "+file); err != nil {
  1199. return err
  1200. }
  1201. return nil
  1202. }
  1203. // Remove performs cleanup of no longer needed directories with local and "remote" git repo
  1204. func (r GitRepo) Remove() {
  1205. if r.baseTempDir != "" {
  1206. os.RemoveAll(r.baseTempDir)
  1207. }
  1208. }
  1209. // NewGitRepo creates temporary test directories with local and "remote" git repo
  1210. func NewGitRepo(repoName string) (GitRepo, error) {
  1211. testDir, err := ioutil.TempDir(util.GetBaseDir(), repoName)
  1212. if err != nil {
  1213. return GitRepo{}, err
  1214. }
  1215. repoPath := filepath.Join(testDir, repoName)
  1216. upstreamPath := repoPath + `.git`
  1217. upstream := git.NewRepository()
  1218. if err = upstream.Init(upstreamPath, true); err != nil {
  1219. return GitRepo{baseTempDir: testDir}, err
  1220. }
  1221. repo := git.NewRepository()
  1222. if err = repo.Clone(repoPath, upstreamPath); err != nil {
  1223. return GitRepo{baseTempDir: testDir}, err
  1224. }
  1225. return GitRepo{testDir, upstream, upstreamPath, repo, repoPath}, nil
  1226. }
  1227. // WaitForUserBeAuthorized waits a minute until the cluster bootstrap roles are available
  1228. // and the provided user is authorized to perform the action on the resource.
  1229. func WaitForUserBeAuthorized(oc *CLI, user, verb, resource string) error {
  1230. sar := &authorization.SubjectAccessReview{
  1231. Spec: authorization.SubjectAccessReviewSpec{
  1232. ResourceAttributes: &authorization.ResourceAttributes{
  1233. Namespace: oc.Namespace(),
  1234. Verb: verb,
  1235. Resource: resource,
  1236. },
  1237. User: user,
  1238. },
  1239. }
  1240. return wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) {
  1241. resp, err := oc.InternalAdminKubeClient().Authorization().SubjectAccessReviews().Create(sar)
  1242. if err == nil && resp != nil && resp.Status.Allowed {
  1243. return true, nil
  1244. }
  1245. return false, err
  1246. })
  1247. }