PageRenderTime 51ms CodeModel.GetById 25ms RepoModel.GetById 0ms app.codeStats 0ms

/pkg/volume/rbd/rbd.go

https://gitlab.com/unofficial-mirrors/kubernetes
Go | 1127 lines | 924 code | 123 blank | 80 comment | 284 complexity | 520cafd5cd51f2b3d1f34aa707f19b21 MD5 | raw file
  1. /*
  2. Copyright 2014 The Kubernetes Authors.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. package rbd
  14. import (
  15. "fmt"
  16. "os"
  17. "path/filepath"
  18. "regexp"
  19. dstrings "strings"
  20. "github.com/golang/glog"
  21. "k8s.io/api/core/v1"
  22. "k8s.io/apimachinery/pkg/api/resource"
  23. metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  24. "k8s.io/apimachinery/pkg/types"
  25. "k8s.io/apimachinery/pkg/util/sets"
  26. "k8s.io/apimachinery/pkg/util/uuid"
  27. clientset "k8s.io/client-go/kubernetes"
  28. "k8s.io/kubernetes/pkg/util/mount"
  29. "k8s.io/kubernetes/pkg/util/strings"
  30. "k8s.io/kubernetes/pkg/volume"
  31. volutil "k8s.io/kubernetes/pkg/volume/util"
  32. "k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
  33. )
  34. var (
  35. supportedFeatures = sets.NewString("layering")
  36. )
  37. // This is the primary entrypoint for volume plugins.
  38. func ProbeVolumePlugins() []volume.VolumePlugin {
  39. return []volume.VolumePlugin{&rbdPlugin{}}
  40. }
  41. // rbdPlugin implements Volume.VolumePlugin.
  42. type rbdPlugin struct {
  43. host volume.VolumeHost
  44. }
  45. var _ volume.VolumePlugin = &rbdPlugin{}
  46. var _ volume.PersistentVolumePlugin = &rbdPlugin{}
  47. var _ volume.DeletableVolumePlugin = &rbdPlugin{}
  48. var _ volume.ProvisionableVolumePlugin = &rbdPlugin{}
  49. var _ volume.AttachableVolumePlugin = &rbdPlugin{}
  50. var _ volume.ExpandableVolumePlugin = &rbdPlugin{}
  51. var _ volume.BlockVolumePlugin = &rbdPlugin{}
  52. const (
  53. rbdPluginName = "kubernetes.io/rbd"
  54. secretKeyName = "key" // key name used in secret
  55. rbdImageFormat1 = "1"
  56. rbdImageFormat2 = "2"
  57. rbdDefaultAdminId = "admin"
  58. rbdDefaultAdminSecretNamespace = "default"
  59. rbdDefaultPool = "rbd"
  60. rbdDefaultUserId = rbdDefaultAdminId
  61. )
  62. func getPath(uid types.UID, volName string, host volume.VolumeHost) string {
  63. return host.GetPodVolumeDir(uid, strings.EscapeQualifiedNameForDisk(rbdPluginName), volName)
  64. }
  65. func (plugin *rbdPlugin) Init(host volume.VolumeHost) error {
  66. plugin.host = host
  67. return nil
  68. }
  69. func (plugin *rbdPlugin) GetPluginName() string {
  70. return rbdPluginName
  71. }
  72. func (plugin *rbdPlugin) GetVolumeName(spec *volume.Spec) (string, error) {
  73. pool, err := getVolumeSourcePool(spec)
  74. if err != nil {
  75. return "", err
  76. }
  77. img, err := getVolumeSourceImage(spec)
  78. if err != nil {
  79. return "", err
  80. }
  81. return fmt.Sprintf(
  82. "%v:%v",
  83. pool,
  84. img), nil
  85. }
  86. func (plugin *rbdPlugin) CanSupport(spec *volume.Spec) bool {
  87. return (spec.Volume != nil && spec.Volume.RBD != nil) || (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.RBD != nil)
  88. }
  89. func (plugin *rbdPlugin) RequiresRemount() bool {
  90. return false
  91. }
  92. func (plugin *rbdPlugin) SupportsMountOption() bool {
  93. return true
  94. }
  95. func (plugin *rbdPlugin) SupportsBulkVolumeVerification() bool {
  96. return false
  97. }
  98. func (plugin *rbdPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
  99. return []v1.PersistentVolumeAccessMode{
  100. v1.ReadWriteOnce,
  101. v1.ReadOnlyMany,
  102. }
  103. }
  104. type rbdVolumeExpander struct {
  105. *rbdMounter
  106. }
  107. func (plugin *rbdPlugin) getAdminAndSecret(spec *volume.Spec) (string, string, error) {
  108. class, err := volutil.GetClassForVolume(plugin.host.GetKubeClient(), spec.PersistentVolume)
  109. if err != nil {
  110. return "", "", err
  111. }
  112. adminSecretName := ""
  113. adminSecretNamespace := rbdDefaultAdminSecretNamespace
  114. admin := ""
  115. for k, v := range class.Parameters {
  116. switch dstrings.ToLower(k) {
  117. case "adminid":
  118. admin = v
  119. case "adminsecretname":
  120. adminSecretName = v
  121. case "adminsecretnamespace":
  122. adminSecretNamespace = v
  123. }
  124. }
  125. if admin == "" {
  126. admin = rbdDefaultAdminId
  127. }
  128. secret, err := parsePVSecret(adminSecretNamespace, adminSecretName, plugin.host.GetKubeClient())
  129. if err != nil {
  130. return admin, "", fmt.Errorf("failed to get admin secret from [%q/%q]: %v", adminSecretNamespace, adminSecretName, err)
  131. }
  132. return admin, secret, nil
  133. }
  134. func (plugin *rbdPlugin) ExpandVolumeDevice(spec *volume.Spec, newSize resource.Quantity, oldSize resource.Quantity) (resource.Quantity, error) {
  135. if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.RBD == nil {
  136. return oldSize, fmt.Errorf("spec.PersistentVolumeSource.Spec.RBD is nil")
  137. }
  138. // get admin and secret
  139. admin, secret, err := plugin.getAdminAndSecret(spec)
  140. if err != nil {
  141. return oldSize, err
  142. }
  143. expander := &rbdVolumeExpander{
  144. rbdMounter: &rbdMounter{
  145. rbd: &rbd{
  146. volName: spec.Name(),
  147. Image: spec.PersistentVolume.Spec.RBD.RBDImage,
  148. Pool: spec.PersistentVolume.Spec.RBD.RBDPool,
  149. plugin: plugin,
  150. manager: &RBDUtil{},
  151. mounter: &mount.SafeFormatAndMount{Interface: plugin.host.GetMounter(plugin.GetPluginName())},
  152. exec: plugin.host.GetExec(plugin.GetPluginName()),
  153. },
  154. Mon: spec.PersistentVolume.Spec.RBD.CephMonitors,
  155. adminId: admin,
  156. adminSecret: secret,
  157. },
  158. }
  159. expandedSize, err := expander.ResizeImage(oldSize, newSize)
  160. if err != nil {
  161. return oldSize, err
  162. } else {
  163. return expandedSize, nil
  164. }
  165. }
  166. func (expander *rbdVolumeExpander) ResizeImage(oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) {
  167. return expander.manager.ExpandImage(expander, oldSize, newSize)
  168. }
  169. func (plugin *rbdPlugin) RequiresFSResize() bool {
  170. return true
  171. }
  172. func (plugin *rbdPlugin) createMounterFromVolumeSpecAndPod(spec *volume.Spec, pod *v1.Pod) (*rbdMounter, error) {
  173. var err error
  174. mon, err := getVolumeSourceMonitors(spec)
  175. if err != nil {
  176. return nil, err
  177. }
  178. img, err := getVolumeSourceImage(spec)
  179. if err != nil {
  180. return nil, err
  181. }
  182. fstype, err := getVolumeSourceFSType(spec)
  183. if err != nil {
  184. return nil, err
  185. }
  186. pool, err := getVolumeSourcePool(spec)
  187. if err != nil {
  188. return nil, err
  189. }
  190. id, err := getVolumeSourceUser(spec)
  191. if err != nil {
  192. return nil, err
  193. }
  194. keyring, err := getVolumeSourceKeyRing(spec)
  195. if err != nil {
  196. return nil, err
  197. }
  198. ro, err := getVolumeSourceReadOnly(spec)
  199. if err != nil {
  200. return nil, err
  201. }
  202. ams, err := getVolumeAccessModes(spec)
  203. if err != nil {
  204. return nil, err
  205. }
  206. secretName, secretNs, err := getSecretNameAndNamespace(spec, pod.Namespace)
  207. if err != nil {
  208. return nil, err
  209. }
  210. secret := ""
  211. if len(secretName) > 0 && len(secretNs) > 0 {
  212. // if secret is provideded, retrieve it
  213. kubeClient := plugin.host.GetKubeClient()
  214. if kubeClient == nil {
  215. return nil, fmt.Errorf("Cannot get kube client")
  216. }
  217. secrets, err := kubeClient.CoreV1().Secrets(secretNs).Get(secretName, metav1.GetOptions{})
  218. if err != nil {
  219. err = fmt.Errorf("Couldn't get secret %v/%v err: %v", secretNs, secretName, err)
  220. return nil, err
  221. }
  222. for _, data := range secrets.Data {
  223. secret = string(data)
  224. }
  225. }
  226. return &rbdMounter{
  227. rbd: newRBD("", spec.Name(), img, pool, ro, plugin, &RBDUtil{}),
  228. Mon: mon,
  229. Id: id,
  230. Keyring: keyring,
  231. Secret: secret,
  232. fsType: fstype,
  233. accessModes: ams,
  234. }, nil
  235. }
  236. func (plugin *rbdPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
  237. secretName, secretNs, err := getSecretNameAndNamespace(spec, pod.Namespace)
  238. if err != nil {
  239. return nil, err
  240. }
  241. secret := ""
  242. if len(secretName) > 0 && len(secretNs) > 0 {
  243. // if secret is provideded, retrieve it
  244. kubeClient := plugin.host.GetKubeClient()
  245. if kubeClient == nil {
  246. return nil, fmt.Errorf("Cannot get kube client")
  247. }
  248. secrets, err := kubeClient.CoreV1().Secrets(secretNs).Get(secretName, metav1.GetOptions{})
  249. if err != nil {
  250. err = fmt.Errorf("Couldn't get secret %v/%v err: %v", secretNs, secretName, err)
  251. return nil, err
  252. }
  253. for _, data := range secrets.Data {
  254. secret = string(data)
  255. }
  256. }
  257. // Inject real implementations here, test through the internal function.
  258. return plugin.newMounterInternal(spec, pod.UID, &RBDUtil{}, secret)
  259. }
  260. func (plugin *rbdPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager diskManager, secret string) (volume.Mounter, error) {
  261. mon, err := getVolumeSourceMonitors(spec)
  262. if err != nil {
  263. return nil, err
  264. }
  265. img, err := getVolumeSourceImage(spec)
  266. if err != nil {
  267. return nil, err
  268. }
  269. fstype, err := getVolumeSourceFSType(spec)
  270. if err != nil {
  271. return nil, err
  272. }
  273. pool, err := getVolumeSourcePool(spec)
  274. if err != nil {
  275. return nil, err
  276. }
  277. id, err := getVolumeSourceUser(spec)
  278. if err != nil {
  279. return nil, err
  280. }
  281. keyring, err := getVolumeSourceKeyRing(spec)
  282. if err != nil {
  283. return nil, err
  284. }
  285. ro, err := getVolumeSourceReadOnly(spec)
  286. if err != nil {
  287. return nil, err
  288. }
  289. ams, err := getVolumeAccessModes(spec)
  290. if err != nil {
  291. return nil, err
  292. }
  293. return &rbdMounter{
  294. rbd: newRBD(podUID, spec.Name(), img, pool, ro, plugin, manager),
  295. Mon: mon,
  296. Id: id,
  297. Keyring: keyring,
  298. Secret: secret,
  299. fsType: fstype,
  300. mountOptions: volutil.MountOptionFromSpec(spec),
  301. accessModes: ams,
  302. }, nil
  303. }
  304. func (plugin *rbdPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
  305. // Inject real implementations here, test through the internal function.
  306. return plugin.newUnmounterInternal(volName, podUID, &RBDUtil{})
  307. }
  308. func (plugin *rbdPlugin) newUnmounterInternal(volName string, podUID types.UID, manager diskManager) (volume.Unmounter, error) {
  309. return &rbdUnmounter{
  310. rbdMounter: &rbdMounter{
  311. rbd: newRBD(podUID, volName, "", "", false, plugin, manager),
  312. Mon: make([]string, 0),
  313. },
  314. }, nil
  315. }
  316. func (plugin *rbdPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
  317. mounter := plugin.host.GetMounter(plugin.GetPluginName())
  318. pluginDir := plugin.host.GetPluginDir(plugin.GetPluginName())
  319. sourceName, err := mounter.GetDeviceNameFromMount(mountPath, pluginDir)
  320. if err != nil {
  321. return nil, err
  322. }
  323. s := dstrings.Split(sourceName, "-image-")
  324. if len(s) != 2 {
  325. // The mountPath parameter is the volume mount path for a specific pod, its format
  326. // is /var/lib/kubelet/pods/{podUID}/volumes/{volumePluginName}/{volumeName}.
  327. // mounter.GetDeviceNameFromMount will find the device path(such as /dev/rbd0) by
  328. // mountPath first, and then try to find the global device mount path from the mounted
  329. // path list of this device. sourceName is extracted from this global device mount path.
  330. // mounter.GetDeviceNameFromMount expects the global device mount path conforms to canonical
  331. // format: /var/lib/kubelet/plugins/kubernetes.io/rbd/mounts/{pool}-image-{image}.
  332. // If this assertion failed, it means that the global device mount path is created by
  333. // the deprecated format: /var/lib/kubelet/plugins/kubernetes.io/rbd/rbd/{pool}-image-{image}.
  334. // So we will try to check whether this old style global device mount path exist or not.
  335. // If existed, extract the sourceName from this old style path, otherwise return an error.
  336. glog.V(3).Infof("SourceName %s wrong, fallback to old format", sourceName)
  337. sourceName, err = plugin.getDeviceNameFromOldMountPath(mounter, mountPath)
  338. if err != nil {
  339. return nil, err
  340. }
  341. s = dstrings.Split(sourceName, "-image-")
  342. if len(s) != 2 {
  343. return nil, fmt.Errorf("sourceName %s wrong, should be pool+\"-image-\"+imageName", sourceName)
  344. }
  345. }
  346. rbdVolume := &v1.Volume{
  347. Name: volumeName,
  348. VolumeSource: v1.VolumeSource{
  349. RBD: &v1.RBDVolumeSource{
  350. RBDPool: s[0],
  351. RBDImage: s[1],
  352. },
  353. },
  354. }
  355. return volume.NewSpecFromVolume(rbdVolume), nil
  356. }
  357. func (plugin *rbdPlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName, mapPath string) (*volume.Spec, error) {
  358. pluginDir := plugin.host.GetVolumeDevicePluginDir(rbdPluginName)
  359. blkutil := volumepathhandler.NewBlockVolumePathHandler()
  360. globalMapPathUUID, err := blkutil.FindGlobalMapPathUUIDFromPod(pluginDir, mapPath, podUID)
  361. if err != nil {
  362. return nil, err
  363. }
  364. glog.V(5).Infof("globalMapPathUUID: %v, err: %v", globalMapPathUUID, err)
  365. globalMapPath := filepath.Dir(globalMapPathUUID)
  366. if len(globalMapPath) == 1 {
  367. return nil, fmt.Errorf("failed to retrieve volume plugin information from globalMapPathUUID: %v", globalMapPathUUID)
  368. }
  369. return getVolumeSpecFromGlobalMapPath(globalMapPath)
  370. }
  371. func getVolumeSpecFromGlobalMapPath(globalMapPath string) (*volume.Spec, error) {
  372. // Retrieve volume spec information from globalMapPath
  373. // globalMapPath example:
  374. // plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}
  375. pool, image, err := getPoolAndImageFromMapPath(globalMapPath)
  376. if err != nil {
  377. return nil, err
  378. }
  379. block := v1.PersistentVolumeBlock
  380. rbdVolume := &v1.PersistentVolume{
  381. Spec: v1.PersistentVolumeSpec{
  382. PersistentVolumeSource: v1.PersistentVolumeSource{
  383. RBD: &v1.RBDPersistentVolumeSource{
  384. RBDImage: image,
  385. RBDPool: pool,
  386. },
  387. },
  388. VolumeMode: &block,
  389. },
  390. }
  391. return volume.NewSpecFromPersistentVolume(rbdVolume, true), nil
  392. }
  393. func (plugin *rbdPlugin) NewBlockVolumeMapper(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.BlockVolumeMapper, error) {
  394. var uid types.UID
  395. if pod != nil {
  396. uid = pod.UID
  397. }
  398. secret := ""
  399. if pod != nil {
  400. secretName, secretNs, err := getSecretNameAndNamespace(spec, pod.Namespace)
  401. if err != nil {
  402. return nil, err
  403. }
  404. if len(secretName) > 0 && len(secretNs) > 0 {
  405. // if secret is provideded, retrieve it
  406. kubeClient := plugin.host.GetKubeClient()
  407. if kubeClient == nil {
  408. return nil, fmt.Errorf("Cannot get kube client")
  409. }
  410. secrets, err := kubeClient.Core().Secrets(secretNs).Get(secretName, metav1.GetOptions{})
  411. if err != nil {
  412. err = fmt.Errorf("Couldn't get secret %v/%v err: %v", secretNs, secretName, err)
  413. return nil, err
  414. }
  415. for _, data := range secrets.Data {
  416. secret = string(data)
  417. }
  418. }
  419. }
  420. return plugin.newBlockVolumeMapperInternal(spec, uid, &RBDUtil{}, secret, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName()))
  421. }
  422. func (plugin *rbdPlugin) newBlockVolumeMapperInternal(spec *volume.Spec, podUID types.UID, manager diskManager, secret string, mounter mount.Interface, exec mount.Exec) (volume.BlockVolumeMapper, error) {
  423. mon, err := getVolumeSourceMonitors(spec)
  424. if err != nil {
  425. return nil, err
  426. }
  427. img, err := getVolumeSourceImage(spec)
  428. if err != nil {
  429. return nil, err
  430. }
  431. pool, err := getVolumeSourcePool(spec)
  432. if err != nil {
  433. return nil, err
  434. }
  435. id, err := getVolumeSourceUser(spec)
  436. if err != nil {
  437. return nil, err
  438. }
  439. keyring, err := getVolumeSourceKeyRing(spec)
  440. if err != nil {
  441. return nil, err
  442. }
  443. ro, err := getVolumeSourceReadOnly(spec)
  444. if err != nil {
  445. return nil, err
  446. }
  447. return &rbdDiskMapper{
  448. rbd: newRBD(podUID, spec.Name(), img, pool, ro, plugin, manager),
  449. mon: mon,
  450. id: id,
  451. keyring: keyring,
  452. secret: secret,
  453. }, nil
  454. }
  455. func (plugin *rbdPlugin) NewBlockVolumeUnmapper(volName string, podUID types.UID) (volume.BlockVolumeUnmapper, error) {
  456. return plugin.newUnmapperInternal(volName, podUID, &RBDUtil{})
  457. }
  458. func (plugin *rbdPlugin) newUnmapperInternal(volName string, podUID types.UID, manager diskManager) (volume.BlockVolumeUnmapper, error) {
  459. return &rbdDiskUnmapper{
  460. rbdDiskMapper: &rbdDiskMapper{
  461. rbd: newRBD(podUID, volName, "", "", false, plugin, manager),
  462. mon: make([]string, 0),
  463. },
  464. }, nil
  465. }
  466. func (plugin *rbdPlugin) getDeviceNameFromOldMountPath(mounter mount.Interface, mountPath string) (string, error) {
  467. refs, err := mounter.GetMountRefs(mountPath)
  468. if err != nil {
  469. return "", err
  470. }
  471. // baseMountPath is the prefix of deprecated device global mounted path,
  472. // such as: /var/lib/kubelet/plugins/kubernetes.io/rbd/rbd
  473. baseMountPath := filepath.Join(plugin.host.GetPluginDir(rbdPluginName), "rbd")
  474. for _, ref := range refs {
  475. if dstrings.HasPrefix(ref, baseMountPath) {
  476. return filepath.Rel(baseMountPath, ref)
  477. }
  478. }
  479. return "", fmt.Errorf("can't find source name from mounted path: %s", mountPath)
  480. }
  481. func (plugin *rbdPlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, error) {
  482. if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.RBD == nil {
  483. return nil, fmt.Errorf("spec.PersistentVolumeSource.Spec.RBD is nil")
  484. }
  485. admin, secret, err := plugin.getAdminAndSecret(spec)
  486. if err != nil {
  487. return nil, err
  488. }
  489. return plugin.newDeleterInternal(spec, admin, secret, &RBDUtil{})
  490. }
  491. func (plugin *rbdPlugin) newDeleterInternal(spec *volume.Spec, admin, secret string, manager diskManager) (volume.Deleter, error) {
  492. return &rbdVolumeDeleter{
  493. rbdMounter: &rbdMounter{
  494. rbd: newRBD("", spec.Name(), spec.PersistentVolume.Spec.RBD.RBDImage, spec.PersistentVolume.Spec.RBD.RBDPool, false, plugin, manager),
  495. Mon: spec.PersistentVolume.Spec.RBD.CephMonitors,
  496. adminId: admin,
  497. adminSecret: secret,
  498. }}, nil
  499. }
  500. func (plugin *rbdPlugin) NewProvisioner(options volume.VolumeOptions) (volume.Provisioner, error) {
  501. return plugin.newProvisionerInternal(options, &RBDUtil{})
  502. }
  503. func (plugin *rbdPlugin) newProvisionerInternal(options volume.VolumeOptions, manager diskManager) (volume.Provisioner, error) {
  504. return &rbdVolumeProvisioner{
  505. rbdMounter: &rbdMounter{
  506. rbd: newRBD("", "", "", "", false, plugin, manager),
  507. },
  508. options: options,
  509. }, nil
  510. }
  511. // rbdVolumeProvisioner implements volume.Provisioner interface.
  512. type rbdVolumeProvisioner struct {
  513. *rbdMounter
  514. options volume.VolumeOptions
  515. }
  516. var _ volume.Provisioner = &rbdVolumeProvisioner{}
  517. func (r *rbdVolumeProvisioner) Provision() (*v1.PersistentVolume, error) {
  518. if !volutil.AccessModesContainedInAll(r.plugin.GetAccessModes(), r.options.PVC.Spec.AccessModes) {
  519. return nil, fmt.Errorf("invalid AccessModes %v: only AccessModes %v are supported", r.options.PVC.Spec.AccessModes, r.plugin.GetAccessModes())
  520. }
  521. if r.options.PVC.Spec.Selector != nil {
  522. return nil, fmt.Errorf("claim Selector is not supported")
  523. }
  524. var err error
  525. adminSecretName := ""
  526. adminSecretNamespace := rbdDefaultAdminSecretNamespace
  527. secret := ""
  528. secretName := ""
  529. secretNamespace := ""
  530. keyring := ""
  531. imageFormat := rbdImageFormat2
  532. fstype := ""
  533. for k, v := range r.options.Parameters {
  534. switch dstrings.ToLower(k) {
  535. case "monitors":
  536. arr := dstrings.Split(v, ",")
  537. r.Mon = append(r.Mon, arr...)
  538. case "adminid":
  539. r.adminId = v
  540. case "adminsecretname":
  541. adminSecretName = v
  542. case "adminsecretnamespace":
  543. adminSecretNamespace = v
  544. case "userid":
  545. r.Id = v
  546. case "pool":
  547. r.Pool = v
  548. case "usersecretname":
  549. secretName = v
  550. case "usersecretnamespace":
  551. secretNamespace = v
  552. case "keyring":
  553. keyring = v
  554. case "imageformat":
  555. imageFormat = v
  556. case "imagefeatures":
  557. arr := dstrings.Split(v, ",")
  558. for _, f := range arr {
  559. if !supportedFeatures.Has(f) {
  560. return nil, fmt.Errorf("invalid feature %q for volume plugin %s, supported features are: %v", f, r.plugin.GetPluginName(), supportedFeatures)
  561. } else {
  562. r.imageFeatures = append(r.imageFeatures, f)
  563. }
  564. }
  565. case volume.VolumeParameterFSType:
  566. fstype = v
  567. default:
  568. return nil, fmt.Errorf("invalid option %q for volume plugin %s", k, r.plugin.GetPluginName())
  569. }
  570. }
  571. // sanity check
  572. if imageFormat != rbdImageFormat1 && imageFormat != rbdImageFormat2 {
  573. return nil, fmt.Errorf("invalid ceph imageformat %s, expecting %s or %s",
  574. imageFormat, rbdImageFormat1, rbdImageFormat2)
  575. }
  576. r.imageFormat = imageFormat
  577. if adminSecretName == "" {
  578. return nil, fmt.Errorf("missing Ceph admin secret name")
  579. }
  580. if secret, err = parsePVSecret(adminSecretNamespace, adminSecretName, r.plugin.host.GetKubeClient()); err != nil {
  581. return nil, fmt.Errorf("failed to get admin secret from [%q/%q]: %v", adminSecretNamespace, adminSecretName, err)
  582. }
  583. r.adminSecret = secret
  584. if len(r.Mon) < 1 {
  585. return nil, fmt.Errorf("missing Ceph monitors")
  586. }
  587. if secretName == "" && keyring == "" {
  588. return nil, fmt.Errorf("must specify either keyring or user secret name")
  589. }
  590. if r.adminId == "" {
  591. r.adminId = rbdDefaultAdminId
  592. }
  593. if r.Pool == "" {
  594. r.Pool = rbdDefaultPool
  595. }
  596. if r.Id == "" {
  597. r.Id = r.adminId
  598. }
  599. // create random image name
  600. image := fmt.Sprintf("kubernetes-dynamic-pvc-%s", uuid.NewUUID())
  601. r.rbdMounter.Image = image
  602. rbd, sizeMB, err := r.manager.CreateImage(r)
  603. if err != nil {
  604. glog.Errorf("rbd: create volume failed, err: %v", err)
  605. return nil, err
  606. }
  607. glog.Infof("successfully created rbd image %q", image)
  608. pv := new(v1.PersistentVolume)
  609. metav1.SetMetaDataAnnotation(&pv.ObjectMeta, volutil.VolumeDynamicallyCreatedByKey, "rbd-dynamic-provisioner")
  610. if secretName != "" {
  611. rbd.SecretRef = new(v1.SecretReference)
  612. rbd.SecretRef.Name = secretName
  613. rbd.SecretRef.Namespace = secretNamespace
  614. } else {
  615. var filePathRegex = regexp.MustCompile(`^(?:/[^/!;` + "`" + ` ]+)+$`)
  616. if keyring != "" && !filePathRegex.MatchString(keyring) {
  617. return nil, fmt.Errorf("keyring field must contain a path to a file")
  618. }
  619. rbd.Keyring = keyring
  620. }
  621. rbd.RadosUser = r.Id
  622. rbd.FSType = fstype
  623. pv.Spec.PersistentVolumeSource.RBD = rbd
  624. pv.Spec.PersistentVolumeReclaimPolicy = r.options.PersistentVolumeReclaimPolicy
  625. pv.Spec.AccessModes = r.options.PVC.Spec.AccessModes
  626. if len(pv.Spec.AccessModes) == 0 {
  627. pv.Spec.AccessModes = r.plugin.GetAccessModes()
  628. }
  629. pv.Spec.Capacity = v1.ResourceList{
  630. v1.ResourceName(v1.ResourceStorage): resource.MustParse(fmt.Sprintf("%dMi", sizeMB)),
  631. }
  632. pv.Spec.MountOptions = r.options.MountOptions
  633. return pv, nil
  634. }
  635. // rbdVolumeDeleter implements volume.Deleter interface.
  636. type rbdVolumeDeleter struct {
  637. *rbdMounter
  638. }
  639. var _ volume.Deleter = &rbdVolumeDeleter{}
  640. func (r *rbdVolumeDeleter) GetPath() string {
  641. return getPath(r.podUID, r.volName, r.plugin.host)
  642. }
  643. func (r *rbdVolumeDeleter) Delete() error {
  644. return r.manager.DeleteImage(r)
  645. }
  646. // rbd implmenets volume.Volume interface.
  647. // It's embedded in Mounter/Unmounter/Deleter.
  648. type rbd struct {
  649. volName string
  650. podUID types.UID
  651. Pool string
  652. Image string
  653. ReadOnly bool
  654. plugin *rbdPlugin
  655. mounter *mount.SafeFormatAndMount
  656. exec mount.Exec
  657. // Utility interface that provides API calls to the provider to attach/detach disks.
  658. manager diskManager
  659. volume.MetricsProvider `json:"-"`
  660. }
  661. var _ volume.Volume = &rbd{}
  662. func (rbd *rbd) GetPath() string {
  663. // safe to use PodVolumeDir now: volume teardown occurs before pod is cleaned up
  664. return getPath(rbd.podUID, rbd.volName, rbd.plugin.host)
  665. }
  666. // newRBD creates a new rbd.
  667. func newRBD(podUID types.UID, volName string, image string, pool string, readOnly bool, plugin *rbdPlugin, manager diskManager) *rbd {
  668. return &rbd{
  669. podUID: podUID,
  670. volName: volName,
  671. Image: image,
  672. Pool: pool,
  673. ReadOnly: readOnly,
  674. plugin: plugin,
  675. mounter: volutil.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host),
  676. exec: plugin.host.GetExec(plugin.GetPluginName()),
  677. manager: manager,
  678. MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host)),
  679. }
  680. }
  681. // rbdMounter implements volume.Mounter interface.
  682. // It contains information which need to be persisted in whole life cycle of PV
  683. // on the node. It is persisted at the very beginning in the pod mount point
  684. // directory.
  685. // Note: Capitalized field names of this struct determines the information
  686. // persisted on the disk, DO NOT change them. (TODO: refactoring to use a dedicated struct?)
  687. type rbdMounter struct {
  688. *rbd
  689. // capitalized so they can be exported in persistRBD()
  690. Mon []string
  691. Id string
  692. Keyring string
  693. Secret string
  694. fsType string
  695. adminSecret string
  696. adminId string
  697. mountOptions []string
  698. imageFormat string
  699. imageFeatures []string
  700. accessModes []v1.PersistentVolumeAccessMode
  701. }
  702. var _ volume.Mounter = &rbdMounter{}
  703. func (b *rbd) GetAttributes() volume.Attributes {
  704. return volume.Attributes{
  705. ReadOnly: b.ReadOnly,
  706. Managed: !b.ReadOnly,
  707. SupportsSELinux: true,
  708. }
  709. }
  710. // Checks prior to mount operations to verify that the required components (binaries, etc.)
  711. // to mount the volume are available on the underlying node.
  712. // If not, it returns an error
  713. func (b *rbdMounter) CanMount() error {
  714. return nil
  715. }
  716. func (b *rbdMounter) SetUp(fsGroup *int64) error {
  717. return b.SetUpAt(b.GetPath(), fsGroup)
  718. }
  719. func (b *rbdMounter) SetUpAt(dir string, fsGroup *int64) error {
  720. // diskSetUp checks mountpoints and prevent repeated calls
  721. glog.V(4).Infof("rbd: attempting to setup at %s", dir)
  722. err := diskSetUp(b.manager, *b, dir, b.mounter, fsGroup)
  723. if err != nil {
  724. glog.Errorf("rbd: failed to setup at %s %v", dir, err)
  725. }
  726. glog.V(3).Infof("rbd: successfully setup at %s", dir)
  727. return err
  728. }
  729. // rbdUnmounter implements volume.Unmounter interface.
  730. type rbdUnmounter struct {
  731. *rbdMounter
  732. }
  733. var _ volume.Unmounter = &rbdUnmounter{}
  734. // Unmounts the bind mount, and detaches the disk only if the disk
  735. // resource was the last reference to that disk on the kubelet.
  736. func (c *rbdUnmounter) TearDown() error {
  737. return c.TearDownAt(c.GetPath())
  738. }
  739. func (c *rbdUnmounter) TearDownAt(dir string) error {
  740. glog.V(4).Infof("rbd: attempting to teardown at %s", dir)
  741. if pathExists, pathErr := volutil.PathExists(dir); pathErr != nil {
  742. return fmt.Errorf("Error checking if path exists: %v", pathErr)
  743. } else if !pathExists {
  744. glog.Warningf("Warning: Unmount skipped because path does not exist: %v", dir)
  745. return nil
  746. }
  747. err := diskTearDown(c.manager, *c, dir, c.mounter)
  748. if err != nil {
  749. return err
  750. }
  751. glog.V(3).Infof("rbd: successfully teardown at %s", dir)
  752. return nil
  753. }
  754. var _ volume.BlockVolumeMapper = &rbdDiskMapper{}
  755. type rbdDiskMapper struct {
  756. *rbd
  757. mon []string
  758. id string
  759. keyring string
  760. secret string
  761. adminSecret string
  762. adminId string
  763. imageFormat string
  764. imageFeatures []string
  765. }
  766. var _ volume.BlockVolumeUnmapper = &rbdDiskUnmapper{}
  767. // GetGlobalMapPath returns global map path and error
  768. // path: plugins/kubernetes.io/{PluginName}/volumeDevices/{rbd pool}-image-{rbd image-name}/{podUid}
  769. func (rbd *rbd) GetGlobalMapPath(spec *volume.Spec) (string, error) {
  770. return rbd.rbdGlobalMapPath(spec)
  771. }
  772. // GetPodDeviceMapPath returns pod device map path and volume name
  773. // path: pods/{podUid}/volumeDevices/kubernetes.io~rbd
  774. // volumeName: pv0001
  775. func (rbd *rbd) GetPodDeviceMapPath() (string, string) {
  776. return rbd.rbdPodDeviceMapPath()
  777. }
  778. func (rbd *rbdDiskMapper) SetUpDevice() (string, error) {
  779. return "", nil
  780. }
  781. func (rbd *rbd) rbdGlobalMapPath(spec *volume.Spec) (string, error) {
  782. mon, err := getVolumeSourceMonitors(spec)
  783. if err != nil {
  784. return "", err
  785. }
  786. img, err := getVolumeSourceImage(spec)
  787. if err != nil {
  788. return "", err
  789. }
  790. pool, err := getVolumeSourcePool(spec)
  791. if err != nil {
  792. return "", err
  793. }
  794. ro, err := getVolumeSourceReadOnly(spec)
  795. if err != nil {
  796. return "", err
  797. }
  798. mounter := &rbdMounter{
  799. rbd: newRBD("", spec.Name(), img, pool, ro, rbd.plugin, &RBDUtil{}),
  800. Mon: mon,
  801. }
  802. return rbd.manager.MakeGlobalVDPDName(*mounter.rbd), nil
  803. }
  804. func (rbd *rbd) rbdPodDeviceMapPath() (string, string) {
  805. name := rbdPluginName
  806. return rbd.plugin.host.GetPodVolumeDeviceDir(rbd.podUID, strings.EscapeQualifiedNameForDisk(name)), rbd.volName
  807. }
  808. type rbdDiskUnmapper struct {
  809. *rbdDiskMapper
  810. }
  811. func getPoolAndImageFromMapPath(mapPath string) (string, string, error) {
  812. pathParts := dstrings.Split(mapPath, "/")
  813. if len(pathParts) < 2 {
  814. return "", "", fmt.Errorf("corrupted mapPath")
  815. }
  816. rbdParts := dstrings.Split(pathParts[len(pathParts)-1], "-image-")
  817. if len(rbdParts) < 2 {
  818. return "", "", fmt.Errorf("corrupted mapPath")
  819. }
  820. return string(rbdParts[0]), string(rbdParts[1]), nil
  821. }
  822. func getBlockVolumeDevice(mapPath string) (string, error) {
  823. pool, image, err := getPoolAndImageFromMapPath(mapPath)
  824. if err != nil {
  825. return "", err
  826. }
  827. // Getting full device path
  828. device, found := getDevFromImageAndPool(pool, image)
  829. if !found {
  830. return "", err
  831. }
  832. return device, nil
  833. }
  834. func (rbd *rbdDiskUnmapper) TearDownDevice(mapPath, _ string) error {
  835. device, err := getBlockVolumeDevice(mapPath)
  836. if err != nil {
  837. return fmt.Errorf("rbd: failed to get loopback for device: %v, err: %v", device, err)
  838. }
  839. // Get loopback device which takes fd lock for device beofore detaching a volume from node.
  840. // TODO: This is a workaround for issue #54108
  841. // Currently local attach plugins such as FC, iSCSI, RBD can't obtain devicePath during
  842. // GenerateUnmapDeviceFunc() in operation_generator. As a result, these plugins fail to get
  843. // and remove loopback device then it will be remained on kubelet node. To avoid the problem,
  844. // local attach plugins needs to remove loopback device during TearDownDevice().
  845. blkUtil := volumepathhandler.NewBlockVolumePathHandler()
  846. loop, err := volumepathhandler.BlockVolumePathHandler.GetLoopDevice(blkUtil, device)
  847. if err != nil {
  848. if err.Error() != volumepathhandler.ErrDeviceNotFound {
  849. return fmt.Errorf("rbd: failed to get loopback for device: %v, err: %v", device, err)
  850. }
  851. glog.Warning("rbd: loopback for device: % not found", device)
  852. } else {
  853. if len(loop) != 0 {
  854. // Remove loop device before detaching volume since volume detach operation gets busy if volume is opened by loopback.
  855. err = volumepathhandler.BlockVolumePathHandler.RemoveLoopDevice(blkUtil, loop)
  856. if err != nil {
  857. return fmt.Errorf("rbd: failed to remove loopback :%v, err: %v", loop, err)
  858. }
  859. glog.V(4).Infof("rbd: successfully removed loop device: %s", loop)
  860. }
  861. }
  862. err = rbd.manager.DetachBlockDisk(*rbd, mapPath)
  863. if err != nil {
  864. return fmt.Errorf("rbd: failed to detach disk: %s\nError: %v", mapPath, err)
  865. }
  866. glog.V(4).Infof("rbd: %q is unmapped, deleting the directory", mapPath)
  867. err = os.RemoveAll(mapPath)
  868. if err != nil {
  869. return fmt.Errorf("rbd: failed to delete the directory: %s\nError: %v", mapPath, err)
  870. }
  871. glog.V(4).Infof("rbd: successfully detached disk: %s", mapPath)
  872. return nil
  873. }
  874. func getVolumeSourceMonitors(spec *volume.Spec) ([]string, error) {
  875. if spec.Volume != nil && spec.Volume.RBD != nil {
  876. return spec.Volume.RBD.CephMonitors, nil
  877. } else if spec.PersistentVolume != nil &&
  878. spec.PersistentVolume.Spec.RBD != nil {
  879. return spec.PersistentVolume.Spec.RBD.CephMonitors, nil
  880. }
  881. return nil, fmt.Errorf("Spec does not reference a RBD volume type")
  882. }
  883. func getVolumeSourceImage(spec *volume.Spec) (string, error) {
  884. if spec.Volume != nil && spec.Volume.RBD != nil {
  885. return spec.Volume.RBD.RBDImage, nil
  886. } else if spec.PersistentVolume != nil &&
  887. spec.PersistentVolume.Spec.RBD != nil {
  888. return spec.PersistentVolume.Spec.RBD.RBDImage, nil
  889. }
  890. return "", fmt.Errorf("Spec does not reference a RBD volume type")
  891. }
  892. func getVolumeSourceFSType(spec *volume.Spec) (string, error) {
  893. if spec.Volume != nil && spec.Volume.RBD != nil {
  894. return spec.Volume.RBD.FSType, nil
  895. } else if spec.PersistentVolume != nil &&
  896. spec.PersistentVolume.Spec.RBD != nil {
  897. return spec.PersistentVolume.Spec.RBD.FSType, nil
  898. }
  899. return "", fmt.Errorf("Spec does not reference a RBD volume type")
  900. }
  901. func getVolumeSourcePool(spec *volume.Spec) (string, error) {
  902. if spec.Volume != nil && spec.Volume.RBD != nil {
  903. return spec.Volume.RBD.RBDPool, nil
  904. } else if spec.PersistentVolume != nil &&
  905. spec.PersistentVolume.Spec.RBD != nil {
  906. return spec.PersistentVolume.Spec.RBD.RBDPool, nil
  907. }
  908. return "", fmt.Errorf("Spec does not reference a RBD volume type")
  909. }
  910. func getVolumeSourceUser(spec *volume.Spec) (string, error) {
  911. if spec.Volume != nil && spec.Volume.RBD != nil {
  912. return spec.Volume.RBD.RadosUser, nil
  913. } else if spec.PersistentVolume != nil &&
  914. spec.PersistentVolume.Spec.RBD != nil {
  915. return spec.PersistentVolume.Spec.RBD.RadosUser, nil
  916. }
  917. return "", fmt.Errorf("Spec does not reference a RBD volume type")
  918. }
  919. func getVolumeSourceKeyRing(spec *volume.Spec) (string, error) {
  920. if spec.Volume != nil && spec.Volume.RBD != nil {
  921. return spec.Volume.RBD.Keyring, nil
  922. } else if spec.PersistentVolume != nil &&
  923. spec.PersistentVolume.Spec.RBD != nil {
  924. return spec.PersistentVolume.Spec.RBD.Keyring, nil
  925. }
  926. return "", fmt.Errorf("Spec does not reference a RBD volume type")
  927. }
  928. func getVolumeSourceReadOnly(spec *volume.Spec) (bool, error) {
  929. if spec.Volume != nil && spec.Volume.RBD != nil {
  930. return spec.Volume.RBD.ReadOnly, nil
  931. } else if spec.PersistentVolume != nil &&
  932. spec.PersistentVolume.Spec.RBD != nil {
  933. // rbd volumes used as a PersistentVolume gets the ReadOnly flag indirectly through
  934. // the persistent-claim volume used to mount the PV
  935. return spec.ReadOnly, nil
  936. }
  937. return false, fmt.Errorf("Spec does not reference a RBD volume type")
  938. }
  939. func getVolumeAccessModes(spec *volume.Spec) ([]v1.PersistentVolumeAccessMode, error) {
  940. // Only PersistentVolumeSpec has AccessModes
  941. if spec.PersistentVolume != nil {
  942. if spec.PersistentVolume.Spec.RBD != nil {
  943. return spec.PersistentVolume.Spec.AccessModes, nil
  944. } else {
  945. return nil, fmt.Errorf("Spec does not reference a RBD volume type")
  946. }
  947. }
  948. return nil, nil
  949. }
  950. func parsePodSecret(pod *v1.Pod, secretName string, kubeClient clientset.Interface) (string, error) {
  951. secret, err := volutil.GetSecretForPod(pod, secretName, kubeClient)
  952. if err != nil {
  953. glog.Errorf("failed to get secret from [%q/%q]", pod.Namespace, secretName)
  954. return "", fmt.Errorf("failed to get secret from [%q/%q]", pod.Namespace, secretName)
  955. }
  956. return parseSecretMap(secret)
  957. }
  958. func parsePVSecret(namespace, secretName string, kubeClient clientset.Interface) (string, error) {
  959. secret, err := volutil.GetSecretForPV(namespace, secretName, rbdPluginName, kubeClient)
  960. if err != nil {
  961. glog.Errorf("failed to get secret from [%q/%q]", namespace, secretName)
  962. return "", fmt.Errorf("failed to get secret from [%q/%q]", namespace, secretName)
  963. }
  964. return parseSecretMap(secret)
  965. }
  966. // parseSecretMap locates the secret by key name.
  967. func parseSecretMap(secretMap map[string]string) (string, error) {
  968. if len(secretMap) == 0 {
  969. return "", fmt.Errorf("empty secret map")
  970. }
  971. secret := ""
  972. for k, v := range secretMap {
  973. if k == secretKeyName {
  974. return v, nil
  975. }
  976. secret = v
  977. }
  978. // If not found, the last secret in the map wins as done before
  979. return secret, nil
  980. }
  981. func getSecretNameAndNamespace(spec *volume.Spec, defaultNamespace string) (string, string, error) {
  982. if spec.Volume != nil && spec.Volume.RBD != nil {
  983. localSecretRef := spec.Volume.RBD.SecretRef
  984. if localSecretRef != nil {
  985. return localSecretRef.Name, defaultNamespace, nil
  986. }
  987. return "", "", nil
  988. } else if spec.PersistentVolume != nil &&
  989. spec.PersistentVolume.Spec.RBD != nil {
  990. secretRef := spec.PersistentVolume.Spec.RBD.SecretRef
  991. secretNs := defaultNamespace
  992. if secretRef != nil {
  993. if len(secretRef.Namespace) != 0 {
  994. secretNs = secretRef.Namespace
  995. }
  996. return secretRef.Name, secretNs, nil
  997. }
  998. return "", "", nil
  999. }
  1000. return "", "", fmt.Errorf("Spec does not reference an RBD volume type")
  1001. }