PageRenderTime 62ms CodeModel.GetById 21ms RepoModel.GetById 1ms app.codeStats 0ms

/functional/platform/nspawn.go

https://gitlab.com/github-cloud-corporation/fleet
Go | 785 lines | 631 code | 105 blank | 49 comment | 147 complexity | 760fe7f44c86dbc0d071c733ff3b9885 MD5 | raw file
  1. // Copyright 2014 The fleet Authors
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package platform
  15. import (
  16. "bytes"
  17. "fmt"
  18. "io"
  19. "io/ioutil"
  20. "log"
  21. "net"
  22. "os"
  23. "os/exec"
  24. "path"
  25. "regexp"
  26. "runtime"
  27. "strconv"
  28. "strings"
  29. "testing"
  30. "time"
  31. "github.com/coreos/go-systemd/dbus"
  32. "github.com/coreos/fleet/functional/util"
  33. )
  34. const (
  35. fleetAPIPort = 54728
  36. )
  37. var fleetdBinPath string
  38. var fleetctlBinPath string
  39. func init() {
  40. fleetdBinPath = os.Getenv("FLEETD_BIN")
  41. fleetctlBinPath = os.Getenv("FLEETCTL_BIN")
  42. if fleetdBinPath == "" {
  43. fmt.Println("FLEETD_BIN environment variable must be set")
  44. os.Exit(1)
  45. } else if _, err := os.Stat(fleetdBinPath); err != nil {
  46. fmt.Printf("%v\n", err)
  47. os.Exit(1)
  48. }
  49. if fleetctlBinPath == "" {
  50. fmt.Println("FLEETCTL_BIN environment variable must be set")
  51. os.Exit(1)
  52. } else if _, err := os.Stat(fleetctlBinPath); err != nil {
  53. fmt.Printf("%v\n", err)
  54. os.Exit(1)
  55. }
  56. // sanity check etcd availability
  57. cmd := exec.Command("etcdctl", "ls")
  58. out, err := cmd.CombinedOutput()
  59. if err != nil {
  60. fmt.Printf("Unable to access etcd: %v\n", err)
  61. fmt.Println(string(out))
  62. os.Exit(1)
  63. }
  64. }
  65. type nspawnMember struct {
  66. uuid string
  67. id string
  68. ip string
  69. pid int
  70. }
  71. func (m *nspawnMember) ID() string {
  72. return m.uuid
  73. }
  74. func (m *nspawnMember) IP() string {
  75. return m.ip
  76. }
  77. func (m *nspawnMember) Endpoint() string {
  78. return fmt.Sprintf("http://%s:%d", m.ip, fleetAPIPort)
  79. }
  80. type nspawnCluster struct {
  81. name string
  82. maxID int
  83. members map[string]nspawnMember
  84. }
  85. func (nc *nspawnCluster) nextID() string {
  86. nc.maxID++
  87. return strconv.Itoa(nc.maxID)
  88. }
  89. func (nc *nspawnCluster) keyspace() string {
  90. // TODO(jonboulle): generate this dynamically with atomic in order keys?
  91. return fmt.Sprintf("/fleet_functional/%s", nc.name)
  92. }
  93. // This function either adds --endpoint flag or set env variable
  94. // FLEETCTL_ENDPOINT, if --tunnel flag is not used.
  95. // Useful for "fleetctl fd-forward" tests
  96. func handleEndpointFlag(m Member, useEnv bool, args *[]string) {
  97. result := true
  98. for _, arg := range *args {
  99. if strings.Contains(arg, "-- ") || strings.Contains(arg, "--tunnel") {
  100. result = false
  101. break
  102. }
  103. }
  104. if result {
  105. if useEnv {
  106. os.Setenv("FLEETCTL_ENDPOINT", m.Endpoint())
  107. } else {
  108. *args = append([]string{"--endpoint=" + m.Endpoint()}, *args...)
  109. }
  110. }
  111. }
  112. func (nc *nspawnCluster) Fleetctl(m Member, args ...string) (string, string, error) {
  113. handleEndpointFlag(m, false, &args)
  114. return util.RunFleetctl(args...)
  115. }
  116. func (nc *nspawnCluster) FleetctlWithInput(m Member, input string, args ...string) (string, string, error) {
  117. handleEndpointFlag(m, false, &args)
  118. return util.RunFleetctlWithInput(input, args...)
  119. }
  120. func (nc *nspawnCluster) FleetctlWithEnv(m Member, args ...string) (string, string, error) {
  121. handleEndpointFlag(m, true, &args)
  122. return util.RunFleetctl(args...)
  123. }
  124. // WaitForNUnits runs fleetctl list-units to verify the actual number of units
  125. // matched with the given expected number. It periodically runs list-units
  126. // waiting until list-units actually shows the expected units.
  127. func (nc *nspawnCluster) WaitForNUnits(m Member, expectedUnits int) (map[string][]util.UnitState, error) {
  128. var nUnits int
  129. retStates := make(map[string][]util.UnitState)
  130. checkListUnits := func() bool {
  131. outListUnits, _, err := nc.Fleetctl(m, "list-units", "--no-legend", "--full", "--fields", "unit,active,machine")
  132. if err != nil {
  133. return false
  134. }
  135. // NOTE: There's no need to check if outListUnits is expected to be empty,
  136. // because ParseUnitStates() implicitly filters out such cases.
  137. // However, in case of ParseUnitStates() going away, we should not
  138. // forget about such special cases.
  139. units := strings.Split(strings.TrimSpace(outListUnits), "\n")
  140. allStates := util.ParseUnitStates(units)
  141. nUnits = len(allStates)
  142. if nUnits != expectedUnits {
  143. return false
  144. }
  145. for _, state := range allStates {
  146. name := state.Name
  147. if _, ok := retStates[name]; !ok {
  148. retStates[name] = []util.UnitState{}
  149. }
  150. retStates[name] = append(retStates[name], state)
  151. }
  152. return true
  153. }
  154. timeout, err := util.WaitForState(checkListUnits)
  155. if err != nil {
  156. return nil, fmt.Errorf("failed to find %d units within %v (last found: %d)",
  157. expectedUnits, timeout, nUnits)
  158. }
  159. return retStates, nil
  160. }
  161. func (nc *nspawnCluster) WaitForNActiveUnits(m Member, count int) (map[string][]util.UnitState, error) {
  162. var nactive int
  163. states := make(map[string][]util.UnitState)
  164. timeout, err := util.WaitForState(
  165. func() bool {
  166. stdout, _, err := nc.Fleetctl(m, "list-units", "--no-legend", "--full", "--fields", "unit,active,machine")
  167. stdout = strings.TrimSpace(stdout)
  168. if err != nil {
  169. return false
  170. }
  171. lines := strings.Split(stdout, "\n")
  172. allStates := util.ParseUnitStates(lines)
  173. active := util.FilterActiveUnits(allStates)
  174. nactive = len(active)
  175. if nactive != count {
  176. return false
  177. }
  178. for _, state := range active {
  179. name := state.Name
  180. if _, ok := states[name]; !ok {
  181. states[name] = []util.UnitState{}
  182. }
  183. states[name] = append(states[name], state)
  184. }
  185. return true
  186. },
  187. )
  188. if err != nil {
  189. return nil, fmt.Errorf("failed to find %d active units within %v (last found: %d)", count, timeout, nactive)
  190. }
  191. return states, nil
  192. }
  193. // WaitForNUnitFiles runs fleetctl list-unit-files to verify the actual number of units
  194. // matched with the given expected number. It periodically runs list-unit-files
  195. // waiting until list-unit-files actually shows the expected units.
  196. func (nc *nspawnCluster) WaitForNUnitFiles(m Member, expectedUnits int) (map[string][]util.UnitFileState, error) {
  197. var nUnits int
  198. retStates := make(map[string][]util.UnitFileState)
  199. checkListUnitFiles := func() bool {
  200. outListUnitFiles, _, err := nc.Fleetctl(m, "list-unit-files", "--no-legend", "--full", "--fields", "unit,dstate,state")
  201. if err != nil {
  202. return false
  203. }
  204. // NOTE: There's no need to check if outListUnits is expected to be empty,
  205. // because ParseUnitFileStates() implicitly filters out such cases.
  206. // However, in case of ParseUnitFileStates() going away, we should not
  207. // forget about such special cases.
  208. units := strings.Split(strings.TrimSpace(outListUnitFiles), "\n")
  209. allStates := util.ParseUnitFileStates(units)
  210. nUnits = len(allStates)
  211. if nUnits != expectedUnits {
  212. // retry until number of units matched
  213. return false
  214. }
  215. for _, state := range allStates {
  216. name := state.Name
  217. if _, ok := retStates[name]; !ok {
  218. retStates[name] = []util.UnitFileState{}
  219. }
  220. retStates[name] = append(retStates[name], state)
  221. }
  222. return true
  223. }
  224. timeout, err := util.WaitForState(checkListUnitFiles)
  225. if err != nil {
  226. return nil, fmt.Errorf("failed to find %d units within %v (last found: %d)",
  227. expectedUnits, timeout, nUnits)
  228. }
  229. return retStates, nil
  230. }
  231. func (nc *nspawnCluster) WaitForNMachines(m Member, count int) ([]string, error) {
  232. var machines []string
  233. timeout := 10 * time.Second
  234. alarm := time.After(timeout)
  235. ticker := time.Tick(250 * time.Millisecond)
  236. loop:
  237. for {
  238. select {
  239. case <-alarm:
  240. return machines, fmt.Errorf("failed to find %d machines within %v", count, timeout)
  241. case <-ticker:
  242. stdout, _, err := nc.Fleetctl(m, "list-machines", "--no-legend", "--full", "--fields", "machine")
  243. if err != nil {
  244. continue
  245. }
  246. stdout = strings.TrimSpace(stdout)
  247. found := 0
  248. if stdout != "" {
  249. machines = strings.Split(stdout, "\n")
  250. found = len(machines)
  251. }
  252. if found != count {
  253. continue
  254. }
  255. break loop
  256. }
  257. }
  258. return machines, nil
  259. }
  260. func (nc *nspawnCluster) prepCluster() (err error) {
  261. baseDir := path.Join(os.TempDir(), nc.name)
  262. _, _, err = run(fmt.Sprintf("mkdir -p %s", baseDir))
  263. if err != nil {
  264. return
  265. }
  266. stdout, stderr, err := run("brctl show")
  267. if err != nil {
  268. log.Printf("Failed enumerating bridges: %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
  269. return
  270. }
  271. if !strings.Contains(stdout, "fleet0") {
  272. stdout, stderr, err = run("brctl addbr fleet0")
  273. if err != nil {
  274. log.Printf("Failed adding fleet0 bridge: %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
  275. return
  276. }
  277. } else {
  278. log.Printf("Bridge fleet0 already exists")
  279. }
  280. stdout, stderr, err = run("ip addr list fleet0")
  281. if err != nil {
  282. log.Printf("Failed listing fleet0 addresses: %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
  283. return
  284. }
  285. if !strings.Contains(stdout, "172.18.0.1/16") {
  286. stdout, stderr, err = run("ip addr add 172.18.0.1/16 dev fleet0")
  287. if err != nil {
  288. log.Printf("Failed adding 172.18.0.1/16 to fleet0: %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
  289. return
  290. }
  291. }
  292. stdout, stderr, err = run("ip link set fleet0 up")
  293. if err != nil {
  294. log.Printf("Failed bringing up fleet0 bridge: %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
  295. return
  296. }
  297. return nil
  298. }
  299. func (nc *nspawnCluster) insertBin(src string, dst string) error {
  300. cmd := fmt.Sprintf("mkdir -p %s/opt/fleet", dst)
  301. if _, _, err := run(cmd); err != nil {
  302. return err
  303. }
  304. binDst := path.Join(dst, "opt", "fleet", path.Base(src))
  305. return copyFile(src, binDst, 0755)
  306. }
  307. func (nc *nspawnCluster) buildConfigDrive(dir, ip string) error {
  308. latest := path.Join(dir, "var/lib/coreos-install")
  309. userPath := path.Join(latest, "user_data")
  310. if err := os.MkdirAll(latest, 0755); err != nil {
  311. return err
  312. }
  313. userFile, err := os.OpenFile(userPath, os.O_WRONLY|os.O_CREATE, 0644)
  314. if err != nil {
  315. return err
  316. }
  317. defer userFile.Close()
  318. etcd := "http://172.18.0.1:4001"
  319. return util.BuildCloudConfig(userFile, ip, etcd, nc.keyspace())
  320. }
  321. func (nc *nspawnCluster) Members() []Member {
  322. ms := make([]Member, 0)
  323. for _, nm := range nc.members {
  324. nm := nm
  325. ms = append(ms, Member(&nm))
  326. }
  327. return ms
  328. }
  329. func (nc *nspawnCluster) MemberCommand(m Member, args ...string) (string, error) {
  330. baseArgs := []string{"-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", fmt.Sprintf("core@%s", m.IP())}
  331. args = append(baseArgs, args...)
  332. log.Printf("ssh %s", strings.Join(args, " "))
  333. var stdoutBytes bytes.Buffer
  334. cmd := exec.Command("ssh", args...)
  335. cmd.Stdout = &stdoutBytes
  336. err := cmd.Run()
  337. return stdoutBytes.String(), err
  338. }
  339. func (nc *nspawnCluster) CreateMember() (m Member, err error) {
  340. id := nc.nextID()
  341. log.Printf("Creating nspawn machine %s in cluster %s", id, nc.name)
  342. return nc.createMember(id)
  343. }
  344. func (nc *nspawnCluster) createMember(id string) (m Member, err error) {
  345. nm := nspawnMember{
  346. uuid: util.NewMachineID(),
  347. id: id,
  348. ip: fmt.Sprintf("172.18.1.%s", id),
  349. }
  350. nc.members[nm.ID()] = nm
  351. basedir := path.Join(os.TempDir(), nc.name)
  352. fsdir := path.Join(basedir, nm.ID(), "fs")
  353. cmds := []string{
  354. // set up directory for fleet service
  355. fmt.Sprintf("mkdir -p %s/etc/systemd/system", fsdir),
  356. // minimum requirements for running systemd/coreos in a container.
  357. // NOTE: copying /etc/pam.d is necessary only for such setups with
  358. // sys-auth/pambase installed, for example developer image of CoreOS
  359. // 1053.2.0. It should be fine also for systems where /etc/pam.d is
  360. // empty, because then it should automatically fall back to
  361. // /usr/lib64/pam.d, which belongs to sys-libs/pam.
  362. fmt.Sprintf("mkdir -p %s/usr", fsdir),
  363. fmt.Sprintf("cp /etc/os-release %s/etc", fsdir),
  364. fmt.Sprintf("cp -a /etc/pam.d %s/etc", fsdir),
  365. fmt.Sprintf("ln -s /proc/self/mounts %s/etc/mtab", fsdir),
  366. fmt.Sprintf("ln -s usr/lib64 %s/lib64", fsdir),
  367. fmt.Sprintf("ln -s lib64 %s/lib", fsdir),
  368. fmt.Sprintf("ln -s usr/bin %s/bin", fsdir),
  369. fmt.Sprintf("ln -s usr/sbin %s/sbin", fsdir),
  370. fmt.Sprintf("mkdir -p %s/home/core/.ssh", fsdir),
  371. fmt.Sprintf("install -d -o root -g systemd-journal -m 2755 %s/var/log/journal", fsdir),
  372. fmt.Sprintf("chown -R 500:500 %s/home/core", fsdir),
  373. // We don't need this, and it's slow, so mask it
  374. fmt.Sprintf("ln -s /dev/null %s/etc/systemd/system/systemd-udev-hwdb-update.service", fsdir),
  375. // set up directory for sshd_config (see below)
  376. fmt.Sprintf("mkdir -p %s/etc/ssh", fsdir),
  377. }
  378. for _, cmd := range cmds {
  379. var stderr, stdout string
  380. stdout, stderr, err = run(cmd)
  381. if err != nil {
  382. log.Printf("Command '%s' failed:\nstdout:: %s\nstderr: %s\nerr: %v", cmd, stdout, stderr, err)
  383. return
  384. }
  385. }
  386. filesContents := []struct {
  387. path string
  388. contents string
  389. mode os.FileMode
  390. }{
  391. {
  392. "/etc/ssh/sshd_config",
  393. `# Use most defaults for sshd configuration.
  394. UsePrivilegeSeparation sandbox
  395. Subsystem sftp internal-sftp
  396. UseDNS no
  397. `,
  398. 0644,
  399. },
  400. // For expediency, generate the minimal viable SSH keys for the host, instead of the default set
  401. {
  402. "/etc/systemd/system/sshd-keygen.service",
  403. `[Unit]
  404. Description=Generate sshd host keys
  405. Before=sshd.service
  406. [Service]
  407. Type=oneshot
  408. RemainAfterExit=yes
  409. ExecStart=/usr/bin/ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key -N "" -b 1024`,
  410. 0644,
  411. },
  412. {
  413. "/etc/passwd",
  414. "core:x:500:500:CoreOS Admin:/home/core:/bin/bash",
  415. 0644,
  416. },
  417. {
  418. "/etc/group",
  419. "core:x:500:",
  420. 0644,
  421. },
  422. {
  423. "/etc/machine-id",
  424. nm.ID(),
  425. 0644,
  426. },
  427. {
  428. "/home/core/.bash_profile",
  429. "export PATH=/opt/fleet:$PATH",
  430. 0644,
  431. },
  432. }
  433. for _, file := range filesContents {
  434. if err = ioutil.WriteFile(path.Join(fsdir, file.path), []byte(file.contents), file.mode); err != nil {
  435. log.Printf("Failed writing %s: %v", path.Join(fsdir, file.path), err)
  436. return
  437. }
  438. }
  439. if err = nc.insertBin(fleetdBinPath, fsdir); err != nil {
  440. log.Printf("Failed preparing fleetd in filesystem: %v", err)
  441. return
  442. }
  443. if err = nc.insertBin(fleetctlBinPath, fsdir); err != nil {
  444. log.Printf("Failed preparing fleetctl in filesystem: %v", err)
  445. return
  446. }
  447. if err = nc.buildConfigDrive(fsdir, nm.IP()); err != nil {
  448. log.Printf("Failed building config drive: %v", err)
  449. return
  450. }
  451. exec := strings.Join([]string{
  452. "/usr/bin/systemd-nspawn",
  453. "--bind-ro=/usr",
  454. "-b",
  455. "--uuid=" + nm.uuid,
  456. fmt.Sprintf("-M %s%s", nc.name, nm.ID()),
  457. "--capability=CAP_NET_BIND_SERVICE,CAP_SYS_TIME", // needed for ntpd
  458. "--network-bridge fleet0",
  459. fmt.Sprintf("-D %s", fsdir),
  460. }, " ")
  461. log.Printf("Creating nspawn container: %s", exec)
  462. err = nc.systemd(fmt.Sprintf("%s%s.service", nc.name, nm.ID()), exec)
  463. if err != nil {
  464. log.Printf("Failed creating nspawn container: %v", err)
  465. return
  466. }
  467. nm.pid, err = nc.machinePID(nm.ID())
  468. if err != nil {
  469. log.Printf("Failed detecting machine %s%s PID: %v", nc.name, nm.ID(), err)
  470. return
  471. }
  472. alarm := time.After(10 * time.Second)
  473. addr := fmt.Sprintf("%s:%d", nm.IP(), fleetAPIPort)
  474. for {
  475. select {
  476. case <-alarm:
  477. err = fmt.Errorf("Timed out waiting for machine to start")
  478. log.Printf("Starting %s%s failed: %v", nc.name, nm.ID(), err)
  479. return
  480. default:
  481. }
  482. log.Printf("Dialing machine: %s", addr)
  483. c, err := net.DialTimeout("tcp", addr, 100*time.Millisecond)
  484. if err == nil {
  485. c.Close()
  486. break
  487. }
  488. time.Sleep(100 * time.Millisecond)
  489. }
  490. return Member(&nm), nil
  491. }
  492. func (nc *nspawnCluster) Destroy(t *testing.T) error {
  493. re := regexp.MustCompile(`/functional\.([a-zA-Z0-9]+)$`)
  494. for _, m := range nc.Members() {
  495. log.Printf("Destroying nspawn machine %s", m.ID())
  496. if t.Failed() {
  497. log.Printf("Failed tests, fetching logs from %s machine", m.ID())
  498. wd, err := os.Getwd()
  499. if err != nil {
  500. log.Printf("Failed to get working directory, skipping journal logs fetch: %v", err)
  501. } else {
  502. var logPath string
  503. containerDir := path.Join(os.TempDir(), nc.name, m.ID(), "fs")
  504. stdout, _, _ := run(fmt.Sprintf("journalctl --directory=%s/var/log/journal --root=%s --no-pager", containerDir, containerDir))
  505. pc := make([]uintptr, 10)
  506. runtime.Callers(6, pc)
  507. f := runtime.FuncForPC(pc[0])
  508. match := re.FindStringSubmatch(f.Name())
  509. if len(match) == 2 {
  510. logPath = fmt.Sprintf("%s/%s_smoke%s.log", wd, match[1], m.ID())
  511. } else {
  512. logPath = fmt.Sprintf("%s/TestUnknown_smoke%s.log", wd, m.ID())
  513. }
  514. err = ioutil.WriteFile(logPath, []byte(stdout), 0644)
  515. if err != nil {
  516. log.Printf("Failed to write journal logs (%s): %v", logPath, err)
  517. } else {
  518. log.Printf("Wrote smoke%s logs to %s", m.ID(), logPath)
  519. }
  520. }
  521. }
  522. nc.DestroyMember(m)
  523. }
  524. dir := path.Join(os.TempDir(), nc.name)
  525. if _, _, err := run(fmt.Sprintf("rm -fr %s", dir)); err != nil {
  526. log.Printf("Failed cleaning up cluster workspace: %v", err)
  527. }
  528. // TODO(bcwaldon): This returns 4 on success, but we can't easily
  529. // ignore just that return code. Ignore the returned error
  530. // altogether until this is fixed.
  531. run("etcdctl rm --recursive " + nc.keyspace())
  532. run("ip link del fleet0")
  533. return nil
  534. }
  535. func (nc *nspawnCluster) ReplaceMember(m Member) (Member, error) {
  536. count := len(nc.members)
  537. label := fmt.Sprintf("%s%s", nc.name, m.ID())
  538. cmd := fmt.Sprintf("machinectl poweroff %s", label)
  539. if stdout, stderr, err := run(cmd); err != nil {
  540. return nil, fmt.Errorf("poweroff failed: %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
  541. }
  542. var mN Member
  543. for id, nm := range nc.members {
  544. if id != m.ID() {
  545. mN = Member(&nm)
  546. break
  547. }
  548. }
  549. if _, err := nc.WaitForNMachines(mN, count-1); err != nil {
  550. return nil, err
  551. }
  552. if err := nc.DestroyMember(m); err != nil {
  553. return nil, err
  554. }
  555. m, err := nc.createMember(m.(*nspawnMember).id)
  556. if err != nil {
  557. return nil, err
  558. }
  559. if _, err := nc.WaitForNMachines(mN, count); err != nil {
  560. return nil, err
  561. }
  562. return m, nil
  563. }
  564. func (nc *nspawnCluster) DestroyMember(m Member) error {
  565. dir := path.Join(os.TempDir(), nc.name, m.ID())
  566. label := fmt.Sprintf("%s%s", nc.name, m.ID())
  567. cmds := []string{
  568. fmt.Sprintf("machinectl terminate %s", label),
  569. fmt.Sprintf("rm -f /run/systemd/system/machine-%s.scope", label),
  570. fmt.Sprintf("rm -f /run/systemd/system/%s.service", label),
  571. fmt.Sprintf("rm -fr /run/systemd/system/%s.service.d", label),
  572. fmt.Sprintf("rm -r %s", dir),
  573. }
  574. for _, cmd := range cmds {
  575. _, _, err := run(cmd)
  576. if err != nil {
  577. log.Printf("Command '%s' failed, but operation will continue: %v", cmd, err)
  578. }
  579. }
  580. // Unfortunately nspawn doesn't always seem to tear down the interfaces
  581. // in time, which can result in subsequent tests failing
  582. run(fmt.Sprintf("ip link del vb-%s", label))
  583. if err := nc.systemdReload(); err != nil {
  584. log.Printf("Failed systemd daemon-reload: %v", err)
  585. }
  586. delete(nc.members, m.ID())
  587. return nil
  588. }
  589. func (nc *nspawnCluster) systemdReload() error {
  590. conn, err := dbus.New()
  591. if err != nil {
  592. return err
  593. }
  594. conn.Reload()
  595. return nil
  596. }
  597. func (nc *nspawnCluster) systemd(unitName, exec string) error {
  598. conn, err := dbus.New()
  599. if err != nil {
  600. return err
  601. }
  602. props := []dbus.Property{
  603. dbus.PropExecStart(strings.Split(exec, " "), false),
  604. }
  605. log.Printf("Creating transient systemd unit %s", unitName)
  606. res1 := make(chan string)
  607. if _, err = conn.StartTransientUnit(unitName, "replace", props, res1); err != nil {
  608. log.Printf("Failed creating transient unit %s: %v", unitName, err)
  609. return err
  610. }
  611. <-res1
  612. res2 := make(chan string)
  613. _, err = conn.StartUnit(unitName, "replace", res2)
  614. if err != nil {
  615. log.Printf("Failed starting transient unit %s: %v", unitName, err)
  616. return err
  617. }
  618. <-res2
  619. return nil
  620. }
  621. // wait up to 10s for a machine to be started
  622. func (nc *nspawnCluster) machinePID(name string) (int, error) {
  623. for i := 0; i < 100; i++ {
  624. mach := fmt.Sprintf("%s%s", nc.name, name)
  625. stdout, stderr, err := run(fmt.Sprintf("machinectl show -p Leader %s", mach))
  626. if err != nil {
  627. if i != -1 {
  628. time.Sleep(100 * time.Millisecond)
  629. continue
  630. }
  631. return -1, fmt.Errorf("failed detecting machine %s status: %v\nstdout: %s\nstderr: %s", mach, err, stdout, stderr)
  632. }
  633. out := strings.SplitN(strings.TrimSpace(stdout), "=", 2)
  634. return strconv.Atoi(out[1])
  635. }
  636. return -1, fmt.Errorf("unable to detect machine PID")
  637. }
  638. func (nc *nspawnCluster) nsenter(pid int, cmd string) (string, string, error) {
  639. cmd = fmt.Sprintf("nsenter -t %d -m -n -p -- %s", pid, cmd)
  640. return run(cmd)
  641. }
  642. func NewNspawnCluster(name string) (Cluster, error) {
  643. nc := &nspawnCluster{name: name, members: map[string]nspawnMember{}}
  644. err := nc.prepCluster()
  645. return nc, err
  646. }
  647. func run(command string) (string, string, error) {
  648. log.Printf(command)
  649. var stdoutBytes, stderrBytes bytes.Buffer
  650. parts := strings.Split(command, " ")
  651. cmd := exec.Command(parts[0], parts[1:]...)
  652. cmd.Stdout = &stdoutBytes
  653. cmd.Stderr = &stderrBytes
  654. err := cmd.Run()
  655. return stdoutBytes.String(), stderrBytes.String(), err
  656. }
  657. func copyFile(src, dst string, mode int) error {
  658. log.Printf("Copying %s -> %s", src, dst)
  659. in, err := os.Open(src)
  660. if err != nil {
  661. return err
  662. }
  663. defer in.Close()
  664. out, err := os.Create(dst)
  665. if err != nil {
  666. return err
  667. }
  668. defer out.Close()
  669. if _, err = io.Copy(out, in); err != nil {
  670. return err
  671. }
  672. if err = out.Sync(); err != nil {
  673. return err
  674. }
  675. if err = os.Chmod(dst, os.FileMode(mode)); err != nil {
  676. return err
  677. }
  678. return nil
  679. }