/vendor/github.com/Mellanox/sriovnet/sriovnet.go

https://github.com/kubevirt/ovs-cni · Go · 455 lines · 378 code · 62 blank · 15 comment · 103 complexity · 20127b0775846e6913911e08c5887d85 MD5 · raw file

  1. package sriovnet
  2. import (
  3. "fmt"
  4. "log"
  5. "net"
  6. "os"
  7. "path"
  8. "path/filepath"
  9. "regexp"
  10. "strconv"
  11. "strings"
  12. "github.com/google/uuid"
  13. "github.com/vishvananda/netlink"
  14. utilfs "github.com/Mellanox/sriovnet/pkg/utils/filesystem"
  15. )
  16. const (
  17. // Used locally
  18. etherEncapType = "ether"
  19. ibEncapType = "infiniband"
  20. )
  21. var virtFnRe = regexp.MustCompile(`virtfn(\d+)`)
  22. type VfObj struct {
  23. Index int
  24. PciAddress string
  25. Bound bool
  26. Allocated bool
  27. }
  28. type PfNetdevHandle struct {
  29. PfNetdevName string
  30. pfLinkHandle netlink.Link
  31. List []*VfObj
  32. }
  33. func SetPFLinkUp(pfNetdevName string) error {
  34. handle, err := netlink.LinkByName(pfNetdevName)
  35. if err != nil {
  36. return err
  37. }
  38. return netlink.LinkSetUp(handle)
  39. }
  40. func IsSriovSupported(netdevName string) bool {
  41. maxvfs, err := getMaxVfCount(netdevName)
  42. if maxvfs == 0 || err != nil {
  43. return false
  44. }
  45. return true
  46. }
  47. func IsSriovEnabled(netdevName string) bool {
  48. curvfs, err := getCurrentVfCount(netdevName)
  49. if curvfs == 0 || err != nil {
  50. return false
  51. }
  52. return true
  53. }
  54. func EnableSriov(pfNetdevName string) error {
  55. var maxVfCount int
  56. var err error
  57. devDirName := netDevDeviceDir(pfNetdevName)
  58. devExist := dirExists(devDirName)
  59. if !devExist {
  60. return fmt.Errorf("device %s not found", pfNetdevName)
  61. }
  62. maxVfCount, err = getMaxVfCount(pfNetdevName)
  63. if err != nil {
  64. log.Println("Fail to read max vf count of PF", pfNetdevName)
  65. return err
  66. }
  67. if maxVfCount == 0 {
  68. return fmt.Errorf("sriov unsupported for device: %s", pfNetdevName)
  69. }
  70. curVfCount, err2 := getCurrentVfCount(pfNetdevName)
  71. if err2 != nil {
  72. log.Println("Fail to read current vf count of PF", pfNetdevName)
  73. return err
  74. }
  75. if curVfCount == 0 {
  76. return setMaxVfCount(pfNetdevName, maxVfCount)
  77. }
  78. return nil
  79. }
  80. func DisableSriov(pfNetdevName string) error {
  81. devDirName := netDevDeviceDir(pfNetdevName)
  82. devExist := dirExists(devDirName)
  83. if !devExist {
  84. return fmt.Errorf("device %s not found", pfNetdevName)
  85. }
  86. return setMaxVfCount(pfNetdevName, 0)
  87. }
  88. func GetPfNetdevHandle(pfNetdevName string) (*PfNetdevHandle, error) {
  89. pfLinkHandle, err := netlink.LinkByName(pfNetdevName)
  90. if err != nil {
  91. return nil, err
  92. }
  93. handle := PfNetdevHandle{
  94. PfNetdevName: pfNetdevName,
  95. pfLinkHandle: pfLinkHandle,
  96. }
  97. list, err := GetVfPciDevList(pfNetdevName)
  98. if err != nil {
  99. return nil, err
  100. }
  101. for _, vfDir := range list {
  102. vfIndexStr := strings.TrimPrefix(vfDir, netDevVfDevicePrefix)
  103. vfIndex, _ := strconv.Atoi(vfIndexStr)
  104. vfNetdevName := vfNetdevNameFromParent(pfNetdevName, vfIndex)
  105. pciAddress, err := vfPCIDevNameFromVfIndex(pfNetdevName, vfIndex)
  106. if err != nil {
  107. log.Printf("Failed to read PCI Address for VF %v from PF %v: %v\n",
  108. vfNetdevName, pfNetdevName, err)
  109. continue
  110. }
  111. vfObj := VfObj{
  112. Index: vfIndex,
  113. PciAddress: pciAddress,
  114. }
  115. if vfNetdevName != "" {
  116. vfObj.Bound = true
  117. } else {
  118. vfObj.Bound = false
  119. }
  120. vfObj.Allocated = false
  121. handle.List = append(handle.List, &vfObj)
  122. }
  123. return &handle, nil
  124. }
  125. func UnbindVf(handle *PfNetdevHandle, vf *VfObj) error {
  126. cmdFile := filepath.Join(NetSysDir, handle.PfNetdevName, netdevDriverDir, netdevUnbindFile)
  127. cmdFileObj := fileObject{
  128. Path: cmdFile,
  129. }
  130. err := cmdFileObj.Write(vf.PciAddress)
  131. if err != nil {
  132. vf.Bound = false
  133. }
  134. return err
  135. }
  136. func BindVf(handle *PfNetdevHandle, vf *VfObj) error {
  137. cmdFile := filepath.Join(NetSysDir, handle.PfNetdevName, netdevDriverDir, netdevBindFile)
  138. cmdFileObj := fileObject{
  139. Path: cmdFile,
  140. }
  141. err := cmdFileObj.Write(vf.PciAddress)
  142. if err != nil {
  143. vf.Bound = true
  144. }
  145. return err
  146. }
  147. func GetVfDefaultMacAddr(vfNetdevName string) (string, error) {
  148. ethHandle, err1 := netlink.LinkByName(vfNetdevName)
  149. if err1 != nil {
  150. return "", err1
  151. }
  152. ethAttr := ethHandle.Attrs()
  153. return ethAttr.HardwareAddr.String(), nil
  154. }
  155. func SetVfDefaultMacAddress(handle *PfNetdevHandle, vf *VfObj) error {
  156. netdevName := vfNetdevNameFromParent(handle.PfNetdevName, vf.Index)
  157. ethHandle, err1 := netlink.LinkByName(netdevName)
  158. if err1 != nil {
  159. return err1
  160. }
  161. ethAttr := ethHandle.Attrs()
  162. return netlink.LinkSetVfHardwareAddr(handle.pfLinkHandle, vf.Index, ethAttr.HardwareAddr)
  163. }
  164. func SetVfVlan(handle *PfNetdevHandle, vf *VfObj, vlan int) error {
  165. return netlink.LinkSetVfVlan(handle.pfLinkHandle, vf.Index, vlan)
  166. }
  167. func setVfNodeGUID(handle *PfNetdevHandle, vf *VfObj, guid []byte) error {
  168. var err error
  169. nodeGUIDHwAddr := net.HardwareAddr(guid)
  170. err = ibSetNodeGUID(handle.PfNetdevName, vf.Index, nodeGUIDHwAddr)
  171. if err == nil {
  172. return nil
  173. }
  174. err = netlink.LinkSetVfNodeGUID(handle.pfLinkHandle, vf.Index, guid)
  175. return err
  176. }
  177. func setVfPortGUID(handle *PfNetdevHandle, vf *VfObj, guid []byte) error {
  178. var err error
  179. portGUIDHwAddr := net.HardwareAddr(guid)
  180. err = ibSetPortGUID(handle.PfNetdevName, vf.Index, portGUIDHwAddr)
  181. if err == nil {
  182. return nil
  183. }
  184. err = netlink.LinkSetVfPortGUID(handle.pfLinkHandle, vf.Index, guid)
  185. return err
  186. }
  187. func SetVfDefaultGUID(handle *PfNetdevHandle, vf *VfObj) error {
  188. randUUID, err := uuid.NewRandom()
  189. if err != nil {
  190. return err
  191. }
  192. guid := randUUID[0:8]
  193. guid[7] = byte(vf.Index)
  194. err = setVfNodeGUID(handle, vf, guid)
  195. if err != nil {
  196. return err
  197. }
  198. err = setVfPortGUID(handle, vf, guid)
  199. return err
  200. }
  201. func SetVfPrivileged(handle *PfNetdevHandle, vf *VfObj, privileged bool) error {
  202. var spoofChk bool
  203. var trusted bool
  204. ethAttr := handle.pfLinkHandle.Attrs()
  205. if ethAttr.EncapType != etherEncapType {
  206. return nil
  207. }
  208. // Only ether type is supported
  209. if privileged {
  210. spoofChk = false
  211. trusted = true
  212. } else {
  213. spoofChk = true
  214. trusted = false
  215. }
  216. /* do not check for error status as older kernels doesn't
  217. * have support for it.
  218. * golangci-lint complains on missing error check. ignore it
  219. * with nolint comment until we update the code to ignore ENOTSUP error
  220. */
  221. netlink.LinkSetVfTrust(handle.pfLinkHandle, vf.Index, trusted) //nolint
  222. netlink.LinkSetVfSpoofchk(handle.pfLinkHandle, vf.Index, spoofChk) //nolint
  223. return nil
  224. }
  225. func setDefaultHwAddr(handle *PfNetdevHandle, vf *VfObj) error {
  226. var err error
  227. ethAttr := handle.pfLinkHandle.Attrs()
  228. if ethAttr.EncapType == etherEncapType {
  229. err = SetVfDefaultMacAddress(handle, vf)
  230. } else if ethAttr.EncapType == ibEncapType {
  231. err = SetVfDefaultGUID(handle, vf)
  232. }
  233. return err
  234. }
  235. func setPortAdminState(handle *PfNetdevHandle, vf *VfObj) error {
  236. ethAttr := handle.pfLinkHandle.Attrs()
  237. if ethAttr.EncapType == ibEncapType {
  238. state, err2 := ibGetPortAdminState(handle.PfNetdevName, vf.Index)
  239. // Ignore the error where this file is not available
  240. if err2 != nil {
  241. return nil
  242. }
  243. log.Printf("Admin state = %v", state)
  244. err2 = ibSetPortAdminState(handle.PfNetdevName, vf.Index, ibSriovPortAdminStateFollow)
  245. if err2 != nil {
  246. // If file exist, we must be able to write
  247. log.Printf("Admin state setting error = %v", err2)
  248. return err2
  249. }
  250. }
  251. return nil
  252. }
  253. func ConfigVfs(handle *PfNetdevHandle, privileged bool) error {
  254. var err error
  255. for _, vf := range handle.List {
  256. log.Printf("vf = %v\n", vf)
  257. err = setPortAdminState(handle, vf)
  258. if err != nil {
  259. break
  260. }
  261. // skip VFs in another namespace
  262. netdevName := vfNetdevNameFromParent(handle.PfNetdevName, vf.Index)
  263. if _, err = netlink.LinkByName(netdevName); err != nil {
  264. continue
  265. }
  266. err = setDefaultHwAddr(handle, vf)
  267. if err != nil {
  268. break
  269. }
  270. _ = SetVfPrivileged(handle, vf, privileged)
  271. }
  272. if err != nil {
  273. return err
  274. }
  275. for _, vf := range handle.List {
  276. if !vf.Bound {
  277. continue
  278. }
  279. err = UnbindVf(handle, vf)
  280. if err != nil {
  281. log.Printf("Fail to unbind err=%v\n", err)
  282. break
  283. }
  284. err = BindVf(handle, vf)
  285. if err != nil {
  286. log.Printf("Fail to bind err=%v\n", err)
  287. break
  288. }
  289. log.Printf("vf = %v unbind/bind completed", vf)
  290. }
  291. return nil
  292. }
  293. func AllocateVf(handle *PfNetdevHandle) (*VfObj, error) {
  294. for _, vf := range handle.List {
  295. if vf.Allocated {
  296. continue
  297. }
  298. vf.Allocated = true
  299. log.Printf("Allocated vf = %v\n", *vf)
  300. return vf, nil
  301. }
  302. return nil, fmt.Errorf("all Vfs for %v are allocated", handle.PfNetdevName)
  303. }
  304. func AllocateVfByMacAddress(handle *PfNetdevHandle, vfMacAddress string) (*VfObj, error) {
  305. for _, vf := range handle.List {
  306. if vf.Allocated {
  307. continue
  308. }
  309. netdevName := vfNetdevNameFromParent(handle.PfNetdevName, vf.Index)
  310. macAddr, _ := GetVfDefaultMacAddr(netdevName)
  311. if macAddr != vfMacAddress {
  312. continue
  313. }
  314. vf.Allocated = true
  315. log.Printf("Allocated vf by mac = %v\n", *vf)
  316. return vf, nil
  317. }
  318. return nil, fmt.Errorf("all Vfs for %v are allocated for mac address %v",
  319. handle.PfNetdevName, vfMacAddress)
  320. }
  321. func FreeVf(handle *PfNetdevHandle, vf *VfObj) {
  322. vf.Allocated = false
  323. log.Printf("Free vf = %v\n", *vf)
  324. }
  325. func FreeVfByNetdevName(handle *PfNetdevHandle, vfIndex int) error {
  326. vfNetdevName := fmt.Sprintf("%s%v", netDevVfDevicePrefix, vfIndex)
  327. for _, vf := range handle.List {
  328. netdevName := vfNetdevNameFromParent(handle.PfNetdevName, vf.Index)
  329. if vf.Allocated && netdevName == vfNetdevName {
  330. vf.Allocated = true
  331. return nil
  332. }
  333. }
  334. return fmt.Errorf("vf netdev %v not found", vfNetdevName)
  335. }
  336. func GetVfNetdevName(handle *PfNetdevHandle, vf *VfObj) string {
  337. return vfNetdevNameFromParent(handle.PfNetdevName, vf.Index)
  338. }
  339. // GetVfIndexByPciAddress gets a VF PCI address (e.g '0000:03:00.4') and
  340. // returns the correlate VF index.
  341. func GetVfIndexByPciAddress(vfPciAddress string) (int, error) {
  342. vfPath := filepath.Join(PciSysDir, vfPciAddress, "physfn/virtfn*")
  343. matches, err := filepath.Glob(vfPath)
  344. if err != nil {
  345. return -1, err
  346. }
  347. for _, match := range matches {
  348. tmp, err := os.Readlink(match)
  349. if err != nil {
  350. continue
  351. }
  352. if strings.Contains(tmp, vfPciAddress) {
  353. result := virtFnRe.FindStringSubmatch(match)
  354. vfIndex, err := strconv.Atoi(result[1])
  355. if err != nil {
  356. continue
  357. }
  358. return vfIndex, nil
  359. }
  360. }
  361. return -1, fmt.Errorf("vf index for %s not found", vfPciAddress)
  362. }
  363. // GetNetDevicesFromPci gets a PCI address (e.g '0000:03:00.1') and
  364. // returns the correlate list of netdevices
  365. func GetNetDevicesFromPci(pciAddress string) ([]string, error) {
  366. pciDir := filepath.Join(PciSysDir, pciAddress, "net")
  367. _, err := utilfs.Fs.Stat(pciDir)
  368. if err != nil {
  369. return nil, fmt.Errorf("cannot get a network device with pci address %v %v", pciAddress, err)
  370. }
  371. netDevicesFiles, err := utilfs.Fs.ReadDir(pciDir)
  372. if err != nil {
  373. return nil, fmt.Errorf("failed to get network device name in %v %v", pciDir, err)
  374. }
  375. netDevices := make([]string, 0, len(netDevicesFiles))
  376. for _, netDeviceFile := range netDevicesFiles {
  377. netDevices = append(netDevices, strings.TrimSpace(netDeviceFile.Name()))
  378. }
  379. return netDevices, nil
  380. }
  381. // GetPfPciFromVfPci retrieves the parent PF PCI address of the provided VF PCI address in D:B:D.f format
  382. func GetPfPciFromVfPci(vfPciAddress string) (string, error) {
  383. pfPath := filepath.Join(PciSysDir, vfPciAddress, "physfn")
  384. pciDevDir, err := utilfs.Fs.Readlink(pfPath)
  385. if err != nil {
  386. return "", fmt.Errorf("failed to read physfn link, provided address may not be a VF. %v", err)
  387. }
  388. pf := path.Base(pciDevDir)
  389. if pf == "" {
  390. return pf, fmt.Errorf("could not find PF PCI Address")
  391. }
  392. return pf, err
  393. }