PageRenderTime 43ms CodeModel.GetById 17ms RepoModel.GetById 0ms app.codeStats 0ms

/pkg/volume/rbd/rbd.go

https://bitbucket.org/Jake-Qu/kubernetes-mirror
Go | 1134 lines | 929 code | 125 blank | 80 comment | 285 complexity | ee9bb0bc9243915d295c517ff74f64c7 MD5 | raw file
Possible License(s): MIT, MPL-2.0-no-copyleft-exception, 0BSD, CC0-1.0, BSD-2-Clause, Apache-2.0, BSD-3-Clause
  1. /*
  2. Copyright 2014 The Kubernetes Authors.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. package rbd
  14. import (
  15. "fmt"
  16. "os"
  17. "path/filepath"
  18. "regexp"
  19. dstrings "strings"
  20. "github.com/golang/glog"
  21. "k8s.io/api/core/v1"
  22. "k8s.io/apimachinery/pkg/api/resource"
  23. metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  24. "k8s.io/apimachinery/pkg/types"
  25. "k8s.io/apimachinery/pkg/util/sets"
  26. "k8s.io/apimachinery/pkg/util/uuid"
  27. utilfeature "k8s.io/apiserver/pkg/util/feature"
  28. clientset "k8s.io/client-go/kubernetes"
  29. "k8s.io/kubernetes/pkg/features"
  30. "k8s.io/kubernetes/pkg/util/mount"
  31. "k8s.io/kubernetes/pkg/util/strings"
  32. "k8s.io/kubernetes/pkg/volume"
  33. volutil "k8s.io/kubernetes/pkg/volume/util"
  34. "k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
  35. )
  36. var (
  37. supportedFeatures = sets.NewString("layering")
  38. )
  39. // This is the primary entrypoint for volume plugins.
  40. func ProbeVolumePlugins() []volume.VolumePlugin {
  41. return []volume.VolumePlugin{&rbdPlugin{}}
  42. }
  43. // rbdPlugin implements Volume.VolumePlugin.
  44. type rbdPlugin struct {
  45. host volume.VolumeHost
  46. }
  47. var _ volume.VolumePlugin = &rbdPlugin{}
  48. var _ volume.PersistentVolumePlugin = &rbdPlugin{}
  49. var _ volume.DeletableVolumePlugin = &rbdPlugin{}
  50. var _ volume.ProvisionableVolumePlugin = &rbdPlugin{}
  51. var _ volume.AttachableVolumePlugin = &rbdPlugin{}
  52. var _ volume.ExpandableVolumePlugin = &rbdPlugin{}
  53. var _ volume.BlockVolumePlugin = &rbdPlugin{}
  54. const (
  55. rbdPluginName = "kubernetes.io/rbd"
  56. secretKeyName = "key" // key name used in secret
  57. rbdImageFormat1 = "1"
  58. rbdImageFormat2 = "2"
  59. rbdDefaultAdminId = "admin"
  60. rbdDefaultAdminSecretNamespace = "default"
  61. rbdDefaultPool = "rbd"
  62. rbdDefaultUserId = rbdDefaultAdminId
  63. )
  64. func getPath(uid types.UID, volName string, host volume.VolumeHost) string {
  65. return host.GetPodVolumeDir(uid, strings.EscapeQualifiedNameForDisk(rbdPluginName), volName)
  66. }
  67. func (plugin *rbdPlugin) Init(host volume.VolumeHost) error {
  68. plugin.host = host
  69. return nil
  70. }
  71. func (plugin *rbdPlugin) GetPluginName() string {
  72. return rbdPluginName
  73. }
  74. func (plugin *rbdPlugin) GetVolumeName(spec *volume.Spec) (string, error) {
  75. pool, err := getVolumeSourcePool(spec)
  76. if err != nil {
  77. return "", err
  78. }
  79. img, err := getVolumeSourceImage(spec)
  80. if err != nil {
  81. return "", err
  82. }
  83. return fmt.Sprintf(
  84. "%v:%v",
  85. pool,
  86. img), nil
  87. }
  88. func (plugin *rbdPlugin) CanSupport(spec *volume.Spec) bool {
  89. return (spec.Volume != nil && spec.Volume.RBD != nil) || (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.RBD != nil)
  90. }
  91. func (plugin *rbdPlugin) RequiresRemount() bool {
  92. return false
  93. }
  94. func (plugin *rbdPlugin) SupportsMountOption() bool {
  95. return true
  96. }
  97. func (plugin *rbdPlugin) SupportsBulkVolumeVerification() bool {
  98. return false
  99. }
  100. func (plugin *rbdPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
  101. return []v1.PersistentVolumeAccessMode{
  102. v1.ReadWriteOnce,
  103. v1.ReadOnlyMany,
  104. }
  105. }
  106. type rbdVolumeExpander struct {
  107. *rbdMounter
  108. }
  109. func (plugin *rbdPlugin) getAdminAndSecret(spec *volume.Spec) (string, string, error) {
  110. class, err := volutil.GetClassForVolume(plugin.host.GetKubeClient(), spec.PersistentVolume)
  111. if err != nil {
  112. return "", "", err
  113. }
  114. adminSecretName := ""
  115. adminSecretNamespace := rbdDefaultAdminSecretNamespace
  116. admin := ""
  117. for k, v := range class.Parameters {
  118. switch dstrings.ToLower(k) {
  119. case "adminid":
  120. admin = v
  121. case "adminsecretname":
  122. adminSecretName = v
  123. case "adminsecretnamespace":
  124. adminSecretNamespace = v
  125. }
  126. }
  127. if admin == "" {
  128. admin = rbdDefaultAdminId
  129. }
  130. secret, err := parsePVSecret(adminSecretNamespace, adminSecretName, plugin.host.GetKubeClient())
  131. if err != nil {
  132. return admin, "", fmt.Errorf("failed to get admin secret from [%q/%q]: %v", adminSecretNamespace, adminSecretName, err)
  133. }
  134. return admin, secret, nil
  135. }
  136. func (plugin *rbdPlugin) ExpandVolumeDevice(spec *volume.Spec, newSize resource.Quantity, oldSize resource.Quantity) (resource.Quantity, error) {
  137. if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.RBD == nil {
  138. return oldSize, fmt.Errorf("spec.PersistentVolumeSource.Spec.RBD is nil")
  139. }
  140. // get admin and secret
  141. admin, secret, err := plugin.getAdminAndSecret(spec)
  142. if err != nil {
  143. return oldSize, err
  144. }
  145. expander := &rbdVolumeExpander{
  146. rbdMounter: &rbdMounter{
  147. rbd: &rbd{
  148. volName: spec.Name(),
  149. Image: spec.PersistentVolume.Spec.RBD.RBDImage,
  150. Pool: spec.PersistentVolume.Spec.RBD.RBDPool,
  151. plugin: plugin,
  152. manager: &RBDUtil{},
  153. mounter: &mount.SafeFormatAndMount{Interface: plugin.host.GetMounter(plugin.GetPluginName())},
  154. exec: plugin.host.GetExec(plugin.GetPluginName()),
  155. },
  156. Mon: spec.PersistentVolume.Spec.RBD.CephMonitors,
  157. adminId: admin,
  158. adminSecret: secret,
  159. },
  160. }
  161. expandedSize, err := expander.ResizeImage(oldSize, newSize)
  162. if err != nil {
  163. return oldSize, err
  164. } else {
  165. return expandedSize, nil
  166. }
  167. }
  168. func (expander *rbdVolumeExpander) ResizeImage(oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) {
  169. return expander.manager.ExpandImage(expander, oldSize, newSize)
  170. }
  171. func (plugin *rbdPlugin) RequiresFSResize() bool {
  172. return true
  173. }
  174. func (plugin *rbdPlugin) createMounterFromVolumeSpecAndPod(spec *volume.Spec, pod *v1.Pod) (*rbdMounter, error) {
  175. var err error
  176. mon, err := getVolumeSourceMonitors(spec)
  177. if err != nil {
  178. return nil, err
  179. }
  180. img, err := getVolumeSourceImage(spec)
  181. if err != nil {
  182. return nil, err
  183. }
  184. fstype, err := getVolumeSourceFSType(spec)
  185. if err != nil {
  186. return nil, err
  187. }
  188. pool, err := getVolumeSourcePool(spec)
  189. if err != nil {
  190. return nil, err
  191. }
  192. id, err := getVolumeSourceUser(spec)
  193. if err != nil {
  194. return nil, err
  195. }
  196. keyring, err := getVolumeSourceKeyRing(spec)
  197. if err != nil {
  198. return nil, err
  199. }
  200. ro, err := getVolumeSourceReadOnly(spec)
  201. if err != nil {
  202. return nil, err
  203. }
  204. ams, err := getVolumeAccessModes(spec)
  205. if err != nil {
  206. return nil, err
  207. }
  208. secretName, secretNs, err := getSecretNameAndNamespace(spec, pod.Namespace)
  209. if err != nil {
  210. return nil, err
  211. }
  212. secret := ""
  213. if len(secretName) > 0 && len(secretNs) > 0 {
  214. // if secret is provideded, retrieve it
  215. kubeClient := plugin.host.GetKubeClient()
  216. if kubeClient == nil {
  217. return nil, fmt.Errorf("Cannot get kube client")
  218. }
  219. secrets, err := kubeClient.CoreV1().Secrets(secretNs).Get(secretName, metav1.GetOptions{})
  220. if err != nil {
  221. err = fmt.Errorf("Couldn't get secret %v/%v err: %v", secretNs, secretName, err)
  222. return nil, err
  223. }
  224. for _, data := range secrets.Data {
  225. secret = string(data)
  226. }
  227. }
  228. return &rbdMounter{
  229. rbd: newRBD("", spec.Name(), img, pool, ro, plugin, &RBDUtil{}),
  230. Mon: mon,
  231. Id: id,
  232. Keyring: keyring,
  233. Secret: secret,
  234. fsType: fstype,
  235. accessModes: ams,
  236. }, nil
  237. }
  238. func (plugin *rbdPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
  239. secretName, secretNs, err := getSecretNameAndNamespace(spec, pod.Namespace)
  240. if err != nil {
  241. return nil, err
  242. }
  243. secret := ""
  244. if len(secretName) > 0 && len(secretNs) > 0 {
  245. // if secret is provideded, retrieve it
  246. kubeClient := plugin.host.GetKubeClient()
  247. if kubeClient == nil {
  248. return nil, fmt.Errorf("Cannot get kube client")
  249. }
  250. secrets, err := kubeClient.CoreV1().Secrets(secretNs).Get(secretName, metav1.GetOptions{})
  251. if err != nil {
  252. err = fmt.Errorf("Couldn't get secret %v/%v err: %v", secretNs, secretName, err)
  253. return nil, err
  254. }
  255. for _, data := range secrets.Data {
  256. secret = string(data)
  257. }
  258. }
  259. // Inject real implementations here, test through the internal function.
  260. return plugin.newMounterInternal(spec, pod.UID, &RBDUtil{}, secret)
  261. }
  262. func (plugin *rbdPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager diskManager, secret string) (volume.Mounter, error) {
  263. mon, err := getVolumeSourceMonitors(spec)
  264. if err != nil {
  265. return nil, err
  266. }
  267. img, err := getVolumeSourceImage(spec)
  268. if err != nil {
  269. return nil, err
  270. }
  271. fstype, err := getVolumeSourceFSType(spec)
  272. if err != nil {
  273. return nil, err
  274. }
  275. pool, err := getVolumeSourcePool(spec)
  276. if err != nil {
  277. return nil, err
  278. }
  279. id, err := getVolumeSourceUser(spec)
  280. if err != nil {
  281. return nil, err
  282. }
  283. keyring, err := getVolumeSourceKeyRing(spec)
  284. if err != nil {
  285. return nil, err
  286. }
  287. ro, err := getVolumeSourceReadOnly(spec)
  288. if err != nil {
  289. return nil, err
  290. }
  291. ams, err := getVolumeAccessModes(spec)
  292. if err != nil {
  293. return nil, err
  294. }
  295. return &rbdMounter{
  296. rbd: newRBD(podUID, spec.Name(), img, pool, ro, plugin, manager),
  297. Mon: mon,
  298. Id: id,
  299. Keyring: keyring,
  300. Secret: secret,
  301. fsType: fstype,
  302. mountOptions: volutil.MountOptionFromSpec(spec),
  303. accessModes: ams,
  304. }, nil
  305. }
  306. func (plugin *rbdPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
  307. // Inject real implementations here, test through the internal function.
  308. return plugin.newUnmounterInternal(volName, podUID, &RBDUtil{})
  309. }
  310. func (plugin *rbdPlugin) newUnmounterInternal(volName string, podUID types.UID, manager diskManager) (volume.Unmounter, error) {
  311. return &rbdUnmounter{
  312. rbdMounter: &rbdMounter{
  313. rbd: newRBD(podUID, volName, "", "", false, plugin, manager),
  314. Mon: make([]string, 0),
  315. },
  316. }, nil
  317. }
  318. func (plugin *rbdPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
  319. mounter := plugin.host.GetMounter(plugin.GetPluginName())
  320. pluginDir := plugin.host.GetPluginDir(plugin.GetPluginName())
  321. sourceName, err := mounter.GetDeviceNameFromMount(mountPath, pluginDir)
  322. if err != nil {
  323. return nil, err
  324. }
  325. s := dstrings.Split(sourceName, "-image-")
  326. if len(s) != 2 {
  327. // The mountPath parameter is the volume mount path for a specific pod, its format
  328. // is /var/lib/kubelet/pods/{podUID}/volumes/{volumePluginName}/{volumeName}.
  329. // mounter.GetDeviceNameFromMount will find the device path(such as /dev/rbd0) by
  330. // mountPath first, and then try to find the global device mount path from the mounted
  331. // path list of this device. sourceName is extracted from this global device mount path.
  332. // mounter.GetDeviceNameFromMount expects the global device mount path conforms to canonical
  333. // format: /var/lib/kubelet/plugins/kubernetes.io/rbd/mounts/{pool}-image-{image}.
  334. // If this assertion failed, it means that the global device mount path is created by
  335. // the deprecated format: /var/lib/kubelet/plugins/kubernetes.io/rbd/rbd/{pool}-image-{image}.
  336. // So we will try to check whether this old style global device mount path exist or not.
  337. // If existed, extract the sourceName from this old style path, otherwise return an error.
  338. glog.V(3).Infof("SourceName %s wrong, fallback to old format", sourceName)
  339. sourceName, err = plugin.getDeviceNameFromOldMountPath(mounter, mountPath)
  340. if err != nil {
  341. return nil, err
  342. }
  343. s = dstrings.Split(sourceName, "-image-")
  344. if len(s) != 2 {
  345. return nil, fmt.Errorf("sourceName %s wrong, should be pool+\"-image-\"+imageName", sourceName)
  346. }
  347. }
  348. rbdVolume := &v1.Volume{
  349. Name: volumeName,
  350. VolumeSource: v1.VolumeSource{
  351. RBD: &v1.RBDVolumeSource{
  352. RBDPool: s[0],
  353. RBDImage: s[1],
  354. },
  355. },
  356. }
  357. return volume.NewSpecFromVolume(rbdVolume), nil
  358. }
  359. func (plugin *rbdPlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName, mapPath string) (*volume.Spec, error) {
  360. pluginDir := plugin.host.GetVolumeDevicePluginDir(rbdPluginName)
  361. blkutil := volumepathhandler.NewBlockVolumePathHandler()
  362. globalMapPathUUID, err := blkutil.FindGlobalMapPathUUIDFromPod(pluginDir, mapPath, podUID)
  363. if err != nil {
  364. return nil, err
  365. }
  366. glog.V(5).Infof("globalMapPathUUID: %v, err: %v", globalMapPathUUID, err)
  367. globalMapPath := filepath.Dir(globalMapPathUUID)
  368. if len(globalMapPath) == 1 {
  369. return nil, fmt.Errorf("failed to retrieve volume plugin information from globalMapPathUUID: %v", globalMapPathUUID)
  370. }
  371. return getVolumeSpecFromGlobalMapPath(globalMapPath)
  372. }
  373. func getVolumeSpecFromGlobalMapPath(globalMapPath string) (*volume.Spec, error) {
  374. // Retrieve volume spec information from globalMapPath
  375. // globalMapPath example:
  376. // plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}
  377. pool, image, err := getPoolAndImageFromMapPath(globalMapPath)
  378. if err != nil {
  379. return nil, err
  380. }
  381. block := v1.PersistentVolumeBlock
  382. rbdVolume := &v1.PersistentVolume{
  383. Spec: v1.PersistentVolumeSpec{
  384. PersistentVolumeSource: v1.PersistentVolumeSource{
  385. RBD: &v1.RBDPersistentVolumeSource{
  386. RBDImage: image,
  387. RBDPool: pool,
  388. },
  389. },
  390. VolumeMode: &block,
  391. },
  392. }
  393. return volume.NewSpecFromPersistentVolume(rbdVolume, true), nil
  394. }
  395. func (plugin *rbdPlugin) NewBlockVolumeMapper(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.BlockVolumeMapper, error) {
  396. var uid types.UID
  397. if pod != nil {
  398. uid = pod.UID
  399. }
  400. secret := ""
  401. if pod != nil {
  402. secretName, secretNs, err := getSecretNameAndNamespace(spec, pod.Namespace)
  403. if err != nil {
  404. return nil, err
  405. }
  406. if len(secretName) > 0 && len(secretNs) > 0 {
  407. // if secret is provideded, retrieve it
  408. kubeClient := plugin.host.GetKubeClient()
  409. if kubeClient == nil {
  410. return nil, fmt.Errorf("Cannot get kube client")
  411. }
  412. secrets, err := kubeClient.Core().Secrets(secretNs).Get(secretName, metav1.GetOptions{})
  413. if err != nil {
  414. err = fmt.Errorf("Couldn't get secret %v/%v err: %v", secretNs, secretName, err)
  415. return nil, err
  416. }
  417. for _, data := range secrets.Data {
  418. secret = string(data)
  419. }
  420. }
  421. }
  422. return plugin.newBlockVolumeMapperInternal(spec, uid, &RBDUtil{}, secret, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName()))
  423. }
  424. func (plugin *rbdPlugin) newBlockVolumeMapperInternal(spec *volume.Spec, podUID types.UID, manager diskManager, secret string, mounter mount.Interface, exec mount.Exec) (volume.BlockVolumeMapper, error) {
  425. mon, err := getVolumeSourceMonitors(spec)
  426. if err != nil {
  427. return nil, err
  428. }
  429. img, err := getVolumeSourceImage(spec)
  430. if err != nil {
  431. return nil, err
  432. }
  433. pool, err := getVolumeSourcePool(spec)
  434. if err != nil {
  435. return nil, err
  436. }
  437. id, err := getVolumeSourceUser(spec)
  438. if err != nil {
  439. return nil, err
  440. }
  441. keyring, err := getVolumeSourceKeyRing(spec)
  442. if err != nil {
  443. return nil, err
  444. }
  445. ro, err := getVolumeSourceReadOnly(spec)
  446. if err != nil {
  447. return nil, err
  448. }
  449. return &rbdDiskMapper{
  450. rbd: newRBD(podUID, spec.Name(), img, pool, ro, plugin, manager),
  451. mon: mon,
  452. id: id,
  453. keyring: keyring,
  454. secret: secret,
  455. }, nil
  456. }
  457. func (plugin *rbdPlugin) NewBlockVolumeUnmapper(volName string, podUID types.UID) (volume.BlockVolumeUnmapper, error) {
  458. return plugin.newUnmapperInternal(volName, podUID, &RBDUtil{})
  459. }
  460. func (plugin *rbdPlugin) newUnmapperInternal(volName string, podUID types.UID, manager diskManager) (volume.BlockVolumeUnmapper, error) {
  461. return &rbdDiskUnmapper{
  462. rbdDiskMapper: &rbdDiskMapper{
  463. rbd: newRBD(podUID, volName, "", "", false, plugin, manager),
  464. mon: make([]string, 0),
  465. },
  466. }, nil
  467. }
  468. func (plugin *rbdPlugin) getDeviceNameFromOldMountPath(mounter mount.Interface, mountPath string) (string, error) {
  469. refs, err := mounter.GetMountRefs(mountPath)
  470. if err != nil {
  471. return "", err
  472. }
  473. // baseMountPath is the prefix of deprecated device global mounted path,
  474. // such as: /var/lib/kubelet/plugins/kubernetes.io/rbd/rbd
  475. baseMountPath := filepath.Join(plugin.host.GetPluginDir(rbdPluginName), "rbd")
  476. for _, ref := range refs {
  477. if dstrings.HasPrefix(ref, baseMountPath) {
  478. return filepath.Rel(baseMountPath, ref)
  479. }
  480. }
  481. return "", fmt.Errorf("can't find source name from mounted path: %s", mountPath)
  482. }
  483. func (plugin *rbdPlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, error) {
  484. if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.RBD == nil {
  485. return nil, fmt.Errorf("spec.PersistentVolumeSource.Spec.RBD is nil")
  486. }
  487. admin, secret, err := plugin.getAdminAndSecret(spec)
  488. if err != nil {
  489. return nil, err
  490. }
  491. return plugin.newDeleterInternal(spec, admin, secret, &RBDUtil{})
  492. }
  493. func (plugin *rbdPlugin) newDeleterInternal(spec *volume.Spec, admin, secret string, manager diskManager) (volume.Deleter, error) {
  494. return &rbdVolumeDeleter{
  495. rbdMounter: &rbdMounter{
  496. rbd: newRBD("", spec.Name(), spec.PersistentVolume.Spec.RBD.RBDImage, spec.PersistentVolume.Spec.RBD.RBDPool, false, plugin, manager),
  497. Mon: spec.PersistentVolume.Spec.RBD.CephMonitors,
  498. adminId: admin,
  499. adminSecret: secret,
  500. }}, nil
  501. }
  502. func (plugin *rbdPlugin) NewProvisioner(options volume.VolumeOptions) (volume.Provisioner, error) {
  503. return plugin.newProvisionerInternal(options, &RBDUtil{})
  504. }
  505. func (plugin *rbdPlugin) newProvisionerInternal(options volume.VolumeOptions, manager diskManager) (volume.Provisioner, error) {
  506. return &rbdVolumeProvisioner{
  507. rbdMounter: &rbdMounter{
  508. rbd: newRBD("", "", "", "", false, plugin, manager),
  509. },
  510. options: options,
  511. }, nil
  512. }
  513. // rbdVolumeProvisioner implements volume.Provisioner interface.
  514. type rbdVolumeProvisioner struct {
  515. *rbdMounter
  516. options volume.VolumeOptions
  517. }
  518. var _ volume.Provisioner = &rbdVolumeProvisioner{}
  519. func (r *rbdVolumeProvisioner) Provision() (*v1.PersistentVolume, error) {
  520. if !volutil.AccessModesContainedInAll(r.plugin.GetAccessModes(), r.options.PVC.Spec.AccessModes) {
  521. return nil, fmt.Errorf("invalid AccessModes %v: only AccessModes %v are supported", r.options.PVC.Spec.AccessModes, r.plugin.GetAccessModes())
  522. }
  523. if r.options.PVC.Spec.Selector != nil {
  524. return nil, fmt.Errorf("claim Selector is not supported")
  525. }
  526. var err error
  527. adminSecretName := ""
  528. adminSecretNamespace := rbdDefaultAdminSecretNamespace
  529. secret := ""
  530. secretName := ""
  531. secretNamespace := ""
  532. keyring := ""
  533. imageFormat := rbdImageFormat2
  534. fstype := ""
  535. for k, v := range r.options.Parameters {
  536. switch dstrings.ToLower(k) {
  537. case "monitors":
  538. arr := dstrings.Split(v, ",")
  539. r.Mon = append(r.Mon, arr...)
  540. case "adminid":
  541. r.adminId = v
  542. case "adminsecretname":
  543. adminSecretName = v
  544. case "adminsecretnamespace":
  545. adminSecretNamespace = v
  546. case "userid":
  547. r.Id = v
  548. case "pool":
  549. r.Pool = v
  550. case "usersecretname":
  551. secretName = v
  552. case "usersecretnamespace":
  553. secretNamespace = v
  554. case "keyring":
  555. keyring = v
  556. case "imageformat":
  557. imageFormat = v
  558. case "imagefeatures":
  559. arr := dstrings.Split(v, ",")
  560. for _, f := range arr {
  561. if !supportedFeatures.Has(f) {
  562. return nil, fmt.Errorf("invalid feature %q for volume plugin %s, supported features are: %v", f, r.plugin.GetPluginName(), supportedFeatures)
  563. } else {
  564. r.imageFeatures = append(r.imageFeatures, f)
  565. }
  566. }
  567. case volume.VolumeParameterFSType:
  568. fstype = v
  569. default:
  570. return nil, fmt.Errorf("invalid option %q for volume plugin %s", k, r.plugin.GetPluginName())
  571. }
  572. }
  573. // sanity check
  574. if imageFormat != rbdImageFormat1 && imageFormat != rbdImageFormat2 {
  575. return nil, fmt.Errorf("invalid ceph imageformat %s, expecting %s or %s",
  576. imageFormat, rbdImageFormat1, rbdImageFormat2)
  577. }
  578. r.imageFormat = imageFormat
  579. if adminSecretName == "" {
  580. return nil, fmt.Errorf("missing Ceph admin secret name")
  581. }
  582. if secret, err = parsePVSecret(adminSecretNamespace, adminSecretName, r.plugin.host.GetKubeClient()); err != nil {
  583. return nil, fmt.Errorf("failed to get admin secret from [%q/%q]: %v", adminSecretNamespace, adminSecretName, err)
  584. }
  585. r.adminSecret = secret
  586. if len(r.Mon) < 1 {
  587. return nil, fmt.Errorf("missing Ceph monitors")
  588. }
  589. if secretName == "" && keyring == "" {
  590. return nil, fmt.Errorf("must specify either keyring or user secret name")
  591. }
  592. if r.adminId == "" {
  593. r.adminId = rbdDefaultAdminId
  594. }
  595. if r.Pool == "" {
  596. r.Pool = rbdDefaultPool
  597. }
  598. if r.Id == "" {
  599. r.Id = r.adminId
  600. }
  601. // create random image name
  602. image := fmt.Sprintf("kubernetes-dynamic-pvc-%s", uuid.NewUUID())
  603. r.rbdMounter.Image = image
  604. rbd, sizeMB, err := r.manager.CreateImage(r)
  605. if err != nil {
  606. glog.Errorf("rbd: create volume failed, err: %v", err)
  607. return nil, err
  608. }
  609. glog.Infof("successfully created rbd image %q", image)
  610. pv := new(v1.PersistentVolume)
  611. metav1.SetMetaDataAnnotation(&pv.ObjectMeta, volutil.VolumeDynamicallyCreatedByKey, "rbd-dynamic-provisioner")
  612. if secretName != "" {
  613. rbd.SecretRef = new(v1.SecretReference)
  614. rbd.SecretRef.Name = secretName
  615. rbd.SecretRef.Namespace = secretNamespace
  616. } else {
  617. var filePathRegex = regexp.MustCompile(`^(?:/[^/!;` + "`" + ` ]+)+$`)
  618. if keyring != "" && !filePathRegex.MatchString(keyring) {
  619. return nil, fmt.Errorf("keyring field must contain a path to a file")
  620. }
  621. rbd.Keyring = keyring
  622. }
  623. rbd.RadosUser = r.Id
  624. rbd.FSType = fstype
  625. pv.Spec.PersistentVolumeSource.RBD = rbd
  626. pv.Spec.PersistentVolumeReclaimPolicy = r.options.PersistentVolumeReclaimPolicy
  627. pv.Spec.AccessModes = r.options.PVC.Spec.AccessModes
  628. if len(pv.Spec.AccessModes) == 0 {
  629. pv.Spec.AccessModes = r.plugin.GetAccessModes()
  630. }
  631. pv.Spec.Capacity = v1.ResourceList{
  632. v1.ResourceName(v1.ResourceStorage): resource.MustParse(fmt.Sprintf("%dMi", sizeMB)),
  633. }
  634. pv.Spec.MountOptions = r.options.MountOptions
  635. if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) {
  636. pv.Spec.VolumeMode = r.options.PVC.Spec.VolumeMode
  637. }
  638. return pv, nil
  639. }
  640. // rbdVolumeDeleter implements volume.Deleter interface.
  641. type rbdVolumeDeleter struct {
  642. *rbdMounter
  643. }
  644. var _ volume.Deleter = &rbdVolumeDeleter{}
  645. func (r *rbdVolumeDeleter) GetPath() string {
  646. return getPath(r.podUID, r.volName, r.plugin.host)
  647. }
  648. func (r *rbdVolumeDeleter) Delete() error {
  649. return r.manager.DeleteImage(r)
  650. }
  651. // rbd implmenets volume.Volume interface.
  652. // It's embedded in Mounter/Unmounter/Deleter.
  653. type rbd struct {
  654. volName string
  655. podUID types.UID
  656. Pool string
  657. Image string
  658. ReadOnly bool
  659. plugin *rbdPlugin
  660. mounter *mount.SafeFormatAndMount
  661. exec mount.Exec
  662. // Utility interface that provides API calls to the provider to attach/detach disks.
  663. manager diskManager
  664. volume.MetricsProvider `json:"-"`
  665. }
  666. var _ volume.Volume = &rbd{}
  667. func (rbd *rbd) GetPath() string {
  668. // safe to use PodVolumeDir now: volume teardown occurs before pod is cleaned up
  669. return getPath(rbd.podUID, rbd.volName, rbd.plugin.host)
  670. }
  671. // newRBD creates a new rbd.
  672. func newRBD(podUID types.UID, volName string, image string, pool string, readOnly bool, plugin *rbdPlugin, manager diskManager) *rbd {
  673. return &rbd{
  674. podUID: podUID,
  675. volName: volName,
  676. Image: image,
  677. Pool: pool,
  678. ReadOnly: readOnly,
  679. plugin: plugin,
  680. mounter: volutil.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host),
  681. exec: plugin.host.GetExec(plugin.GetPluginName()),
  682. manager: manager,
  683. MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host)),
  684. }
  685. }
  686. // rbdMounter implements volume.Mounter interface.
  687. // It contains information which need to be persisted in whole life cycle of PV
  688. // on the node. It is persisted at the very beginning in the pod mount point
  689. // directory.
  690. // Note: Capitalized field names of this struct determines the information
  691. // persisted on the disk, DO NOT change them. (TODO: refactoring to use a dedicated struct?)
  692. type rbdMounter struct {
  693. *rbd
  694. // capitalized so they can be exported in persistRBD()
  695. Mon []string
  696. Id string
  697. Keyring string
  698. Secret string
  699. fsType string
  700. adminSecret string
  701. adminId string
  702. mountOptions []string
  703. imageFormat string
  704. imageFeatures []string
  705. accessModes []v1.PersistentVolumeAccessMode
  706. }
  707. var _ volume.Mounter = &rbdMounter{}
  708. func (b *rbd) GetAttributes() volume.Attributes {
  709. return volume.Attributes{
  710. ReadOnly: b.ReadOnly,
  711. Managed: !b.ReadOnly,
  712. SupportsSELinux: true,
  713. }
  714. }
  715. // Checks prior to mount operations to verify that the required components (binaries, etc.)
  716. // to mount the volume are available on the underlying node.
  717. // If not, it returns an error
  718. func (b *rbdMounter) CanMount() error {
  719. return nil
  720. }
  721. func (b *rbdMounter) SetUp(fsGroup *int64) error {
  722. return b.SetUpAt(b.GetPath(), fsGroup)
  723. }
  724. func (b *rbdMounter) SetUpAt(dir string, fsGroup *int64) error {
  725. // diskSetUp checks mountpoints and prevent repeated calls
  726. glog.V(4).Infof("rbd: attempting to setup at %s", dir)
  727. err := diskSetUp(b.manager, *b, dir, b.mounter, fsGroup)
  728. if err != nil {
  729. glog.Errorf("rbd: failed to setup at %s %v", dir, err)
  730. }
  731. glog.V(3).Infof("rbd: successfully setup at %s", dir)
  732. return err
  733. }
  734. // rbdUnmounter implements volume.Unmounter interface.
  735. type rbdUnmounter struct {
  736. *rbdMounter
  737. }
  738. var _ volume.Unmounter = &rbdUnmounter{}
  739. // Unmounts the bind mount, and detaches the disk only if the disk
  740. // resource was the last reference to that disk on the kubelet.
  741. func (c *rbdUnmounter) TearDown() error {
  742. return c.TearDownAt(c.GetPath())
  743. }
  744. func (c *rbdUnmounter) TearDownAt(dir string) error {
  745. glog.V(4).Infof("rbd: attempting to teardown at %s", dir)
  746. if pathExists, pathErr := volutil.PathExists(dir); pathErr != nil {
  747. return fmt.Errorf("Error checking if path exists: %v", pathErr)
  748. } else if !pathExists {
  749. glog.Warningf("Warning: Unmount skipped because path does not exist: %v", dir)
  750. return nil
  751. }
  752. err := diskTearDown(c.manager, *c, dir, c.mounter)
  753. if err != nil {
  754. return err
  755. }
  756. glog.V(3).Infof("rbd: successfully teardown at %s", dir)
  757. return nil
  758. }
  759. var _ volume.BlockVolumeMapper = &rbdDiskMapper{}
  760. type rbdDiskMapper struct {
  761. *rbd
  762. mon []string
  763. id string
  764. keyring string
  765. secret string
  766. adminSecret string
  767. adminId string
  768. imageFormat string
  769. imageFeatures []string
  770. }
  771. var _ volume.BlockVolumeUnmapper = &rbdDiskUnmapper{}
  772. // GetGlobalMapPath returns global map path and error
  773. // path: plugins/kubernetes.io/{PluginName}/volumeDevices/{rbd pool}-image-{rbd image-name}/{podUid}
  774. func (rbd *rbd) GetGlobalMapPath(spec *volume.Spec) (string, error) {
  775. return rbd.rbdGlobalMapPath(spec)
  776. }
  777. // GetPodDeviceMapPath returns pod device map path and volume name
  778. // path: pods/{podUid}/volumeDevices/kubernetes.io~rbd
  779. // volumeName: pv0001
  780. func (rbd *rbd) GetPodDeviceMapPath() (string, string) {
  781. return rbd.rbdPodDeviceMapPath()
  782. }
  783. func (rbd *rbdDiskMapper) SetUpDevice() (string, error) {
  784. return "", nil
  785. }
  786. func (rbd *rbd) rbdGlobalMapPath(spec *volume.Spec) (string, error) {
  787. mon, err := getVolumeSourceMonitors(spec)
  788. if err != nil {
  789. return "", err
  790. }
  791. img, err := getVolumeSourceImage(spec)
  792. if err != nil {
  793. return "", err
  794. }
  795. pool, err := getVolumeSourcePool(spec)
  796. if err != nil {
  797. return "", err
  798. }
  799. ro, err := getVolumeSourceReadOnly(spec)
  800. if err != nil {
  801. return "", err
  802. }
  803. mounter := &rbdMounter{
  804. rbd: newRBD("", spec.Name(), img, pool, ro, rbd.plugin, &RBDUtil{}),
  805. Mon: mon,
  806. }
  807. return rbd.manager.MakeGlobalVDPDName(*mounter.rbd), nil
  808. }
  809. func (rbd *rbd) rbdPodDeviceMapPath() (string, string) {
  810. name := rbdPluginName
  811. return rbd.plugin.host.GetPodVolumeDeviceDir(rbd.podUID, strings.EscapeQualifiedNameForDisk(name)), rbd.volName
  812. }
  813. type rbdDiskUnmapper struct {
  814. *rbdDiskMapper
  815. }
  816. func getPoolAndImageFromMapPath(mapPath string) (string, string, error) {
  817. pathParts := dstrings.Split(mapPath, "/")
  818. if len(pathParts) < 2 {
  819. return "", "", fmt.Errorf("corrupted mapPath")
  820. }
  821. rbdParts := dstrings.Split(pathParts[len(pathParts)-1], "-image-")
  822. if len(rbdParts) < 2 {
  823. return "", "", fmt.Errorf("corrupted mapPath")
  824. }
  825. return string(rbdParts[0]), string(rbdParts[1]), nil
  826. }
  827. func getBlockVolumeDevice(mapPath string) (string, error) {
  828. pool, image, err := getPoolAndImageFromMapPath(mapPath)
  829. if err != nil {
  830. return "", err
  831. }
  832. // Getting full device path
  833. device, found := getDevFromImageAndPool(pool, image)
  834. if !found {
  835. return "", err
  836. }
  837. return device, nil
  838. }
  839. func (rbd *rbdDiskUnmapper) TearDownDevice(mapPath, _ string) error {
  840. device, err := getBlockVolumeDevice(mapPath)
  841. if err != nil {
  842. return fmt.Errorf("rbd: failed to get loopback for device: %v, err: %v", device, err)
  843. }
  844. // Get loopback device which takes fd lock for device beofore detaching a volume from node.
  845. // TODO: This is a workaround for issue #54108
  846. // Currently local attach plugins such as FC, iSCSI, RBD can't obtain devicePath during
  847. // GenerateUnmapDeviceFunc() in operation_generator. As a result, these plugins fail to get
  848. // and remove loopback device then it will be remained on kubelet node. To avoid the problem,
  849. // local attach plugins needs to remove loopback device during TearDownDevice().
  850. blkUtil := volumepathhandler.NewBlockVolumePathHandler()
  851. loop, err := volumepathhandler.BlockVolumePathHandler.GetLoopDevice(blkUtil, device)
  852. if err != nil {
  853. if err.Error() != volumepathhandler.ErrDeviceNotFound {
  854. return fmt.Errorf("rbd: failed to get loopback for device: %v, err: %v", device, err)
  855. }
  856. glog.Warning("rbd: loopback for device: % not found", device)
  857. } else {
  858. if len(loop) != 0 {
  859. // Remove loop device before detaching volume since volume detach operation gets busy if volume is opened by loopback.
  860. err = volumepathhandler.BlockVolumePathHandler.RemoveLoopDevice(blkUtil, loop)
  861. if err != nil {
  862. return fmt.Errorf("rbd: failed to remove loopback :%v, err: %v", loop, err)
  863. }
  864. glog.V(4).Infof("rbd: successfully removed loop device: %s", loop)
  865. }
  866. }
  867. err = rbd.manager.DetachBlockDisk(*rbd, mapPath)
  868. if err != nil {
  869. return fmt.Errorf("rbd: failed to detach disk: %s\nError: %v", mapPath, err)
  870. }
  871. glog.V(4).Infof("rbd: %q is unmapped, deleting the directory", mapPath)
  872. err = os.RemoveAll(mapPath)
  873. if err != nil {
  874. return fmt.Errorf("rbd: failed to delete the directory: %s\nError: %v", mapPath, err)
  875. }
  876. glog.V(4).Infof("rbd: successfully detached disk: %s", mapPath)
  877. return nil
  878. }
  879. func getVolumeSourceMonitors(spec *volume.Spec) ([]string, error) {
  880. if spec.Volume != nil && spec.Volume.RBD != nil {
  881. return spec.Volume.RBD.CephMonitors, nil
  882. } else if spec.PersistentVolume != nil &&
  883. spec.PersistentVolume.Spec.RBD != nil {
  884. return spec.PersistentVolume.Spec.RBD.CephMonitors, nil
  885. }
  886. return nil, fmt.Errorf("Spec does not reference a RBD volume type")
  887. }
  888. func getVolumeSourceImage(spec *volume.Spec) (string, error) {
  889. if spec.Volume != nil && spec.Volume.RBD != nil {
  890. return spec.Volume.RBD.RBDImage, nil
  891. } else if spec.PersistentVolume != nil &&
  892. spec.PersistentVolume.Spec.RBD != nil {
  893. return spec.PersistentVolume.Spec.RBD.RBDImage, nil
  894. }
  895. return "", fmt.Errorf("Spec does not reference a RBD volume type")
  896. }
  897. func getVolumeSourceFSType(spec *volume.Spec) (string, error) {
  898. if spec.Volume != nil && spec.Volume.RBD != nil {
  899. return spec.Volume.RBD.FSType, nil
  900. } else if spec.PersistentVolume != nil &&
  901. spec.PersistentVolume.Spec.RBD != nil {
  902. return spec.PersistentVolume.Spec.RBD.FSType, nil
  903. }
  904. return "", fmt.Errorf("Spec does not reference a RBD volume type")
  905. }
  906. func getVolumeSourcePool(spec *volume.Spec) (string, error) {
  907. if spec.Volume != nil && spec.Volume.RBD != nil {
  908. return spec.Volume.RBD.RBDPool, nil
  909. } else if spec.PersistentVolume != nil &&
  910. spec.PersistentVolume.Spec.RBD != nil {
  911. return spec.PersistentVolume.Spec.RBD.RBDPool, nil
  912. }
  913. return "", fmt.Errorf("Spec does not reference a RBD volume type")
  914. }
  915. func getVolumeSourceUser(spec *volume.Spec) (string, error) {
  916. if spec.Volume != nil && spec.Volume.RBD != nil {
  917. return spec.Volume.RBD.RadosUser, nil
  918. } else if spec.PersistentVolume != nil &&
  919. spec.PersistentVolume.Spec.RBD != nil {
  920. return spec.PersistentVolume.Spec.RBD.RadosUser, nil
  921. }
  922. return "", fmt.Errorf("Spec does not reference a RBD volume type")
  923. }
  924. func getVolumeSourceKeyRing(spec *volume.Spec) (string, error) {
  925. if spec.Volume != nil && spec.Volume.RBD != nil {
  926. return spec.Volume.RBD.Keyring, nil
  927. } else if spec.PersistentVolume != nil &&
  928. spec.PersistentVolume.Spec.RBD != nil {
  929. return spec.PersistentVolume.Spec.RBD.Keyring, nil
  930. }
  931. return "", fmt.Errorf("Spec does not reference a RBD volume type")
  932. }
  933. func getVolumeSourceReadOnly(spec *volume.Spec) (bool, error) {
  934. if spec.Volume != nil && spec.Volume.RBD != nil {
  935. return spec.Volume.RBD.ReadOnly, nil
  936. } else if spec.PersistentVolume != nil &&
  937. spec.PersistentVolume.Spec.RBD != nil {
  938. // rbd volumes used as a PersistentVolume gets the ReadOnly flag indirectly through
  939. // the persistent-claim volume used to mount the PV
  940. return spec.ReadOnly, nil
  941. }
  942. return false, fmt.Errorf("Spec does not reference a RBD volume type")
  943. }
  944. func getVolumeAccessModes(spec *volume.Spec) ([]v1.PersistentVolumeAccessMode, error) {
  945. // Only PersistentVolumeSpec has AccessModes
  946. if spec.PersistentVolume != nil {
  947. if spec.PersistentVolume.Spec.RBD != nil {
  948. return spec.PersistentVolume.Spec.AccessModes, nil
  949. } else {
  950. return nil, fmt.Errorf("Spec does not reference a RBD volume type")
  951. }
  952. }
  953. return nil, nil
  954. }
  955. func parsePodSecret(pod *v1.Pod, secretName string, kubeClient clientset.Interface) (string, error) {
  956. secret, err := volutil.GetSecretForPod(pod, secretName, kubeClient)
  957. if err != nil {
  958. glog.Errorf("failed to get secret from [%q/%q]", pod.Namespace, secretName)
  959. return "", fmt.Errorf("failed to get secret from [%q/%q]", pod.Namespace, secretName)
  960. }
  961. return parseSecretMap(secret)
  962. }
  963. func parsePVSecret(namespace, secretName string, kubeClient clientset.Interface) (string, error) {
  964. secret, err := volutil.GetSecretForPV(namespace, secretName, rbdPluginName, kubeClient)
  965. if err != nil {
  966. glog.Errorf("failed to get secret from [%q/%q]", namespace, secretName)
  967. return "", fmt.Errorf("failed to get secret from [%q/%q]", namespace, secretName)
  968. }
  969. return parseSecretMap(secret)
  970. }
  971. // parseSecretMap locates the secret by key name.
  972. func parseSecretMap(secretMap map[string]string) (string, error) {
  973. if len(secretMap) == 0 {
  974. return "", fmt.Errorf("empty secret map")
  975. }
  976. secret := ""
  977. for k, v := range secretMap {
  978. if k == secretKeyName {
  979. return v, nil
  980. }
  981. secret = v
  982. }
  983. // If not found, the last secret in the map wins as done before
  984. return secret, nil
  985. }
  986. func getSecretNameAndNamespace(spec *volume.Spec, defaultNamespace string) (string, string, error) {
  987. if spec.Volume != nil && spec.Volume.RBD != nil {
  988. localSecretRef := spec.Volume.RBD.SecretRef
  989. if localSecretRef != nil {
  990. return localSecretRef.Name, defaultNamespace, nil
  991. }
  992. return "", "", nil
  993. } else if spec.PersistentVolume != nil &&
  994. spec.PersistentVolume.Spec.RBD != nil {
  995. secretRef := spec.PersistentVolume.Spec.RBD.SecretRef
  996. secretNs := defaultNamespace
  997. if secretRef != nil {
  998. if len(secretRef.Namespace) != 0 {
  999. secretNs = secretRef.Namespace
  1000. }
  1001. return secretRef.Name, secretNs, nil
  1002. }
  1003. return "", "", nil
  1004. }
  1005. return "", "", fmt.Errorf("Spec does not reference an RBD volume type")
  1006. }