PageRenderTime 35ms CodeModel.GetById 8ms RepoModel.GetById 1ms app.codeStats 0ms

/pkg/cloudprovider/providers/azure/azure_standard.go

https://gitlab.com/unofficial-mirrors/kubernetes
Go | 729 lines | 549 code | 95 blank | 85 comment | 137 complexity | 690f442259fd311de22dc048ad78d983 MD5 | raw file
  1. /*
  2. Copyright 2016 The Kubernetes Authors.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. package azure
  14. import (
  15. "errors"
  16. "fmt"
  17. "hash/crc32"
  18. "regexp"
  19. "sort"
  20. "strconv"
  21. "strings"
  22. "k8s.io/api/core/v1"
  23. "k8s.io/kubernetes/pkg/cloudprovider"
  24. "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2017-12-01/compute"
  25. "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2017-09-01/network"
  26. "github.com/Azure/go-autorest/autorest/to"
  27. "github.com/golang/glog"
  28. "k8s.io/apimachinery/pkg/types"
  29. utilerrors "k8s.io/apimachinery/pkg/util/errors"
  30. "k8s.io/apimachinery/pkg/util/sets"
  31. "k8s.io/apimachinery/pkg/util/uuid"
  32. )
  33. const (
  34. loadBalancerMinimumPriority = 500
  35. loadBalancerMaximumPriority = 4096
  36. machineIDTemplate = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/virtualMachines/%s"
  37. availabilitySetIDTemplate = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/availabilitySets/%s"
  38. frontendIPConfigIDTemplate = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/frontendIPConfigurations/%s"
  39. backendPoolIDTemplate = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/backendAddressPools/%s"
  40. loadBalancerProbeIDTemplate = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/probes/%s"
  41. // InternalLoadBalancerNameSuffix is load balancer posfix
  42. InternalLoadBalancerNameSuffix = "-internal"
  43. // nodeLabelRole specifies the role of a node
  44. nodeLabelRole = "kubernetes.io/role"
  45. storageAccountNameMaxLength = 24
  46. )
  47. var errNotInVMSet = errors.New("vm is not in the vmset")
  48. var providerIDRE = regexp.MustCompile(`^` + CloudProviderName + `://(?:.*)/Microsoft.Compute/virtualMachines/(.+)$`)
  49. var backendPoolIDRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Network/loadBalancers/(.+)/backendAddressPools/(?:.*)`)
  50. // getStandardMachineID returns the full identifier of a virtual machine.
  51. func (az *Cloud) getStandardMachineID(machineName string) string {
  52. return fmt.Sprintf(
  53. machineIDTemplate,
  54. az.SubscriptionID,
  55. az.ResourceGroup,
  56. machineName)
  57. }
  58. // returns the full identifier of an availabilitySet
  59. func (az *Cloud) getAvailabilitySetID(availabilitySetName string) string {
  60. return fmt.Sprintf(
  61. availabilitySetIDTemplate,
  62. az.SubscriptionID,
  63. az.ResourceGroup,
  64. availabilitySetName)
  65. }
  66. // returns the full identifier of a loadbalancer frontendipconfiguration.
  67. func (az *Cloud) getFrontendIPConfigID(lbName, backendPoolName string) string {
  68. return fmt.Sprintf(
  69. frontendIPConfigIDTemplate,
  70. az.SubscriptionID,
  71. az.ResourceGroup,
  72. lbName,
  73. backendPoolName)
  74. }
  75. // returns the full identifier of a loadbalancer backendpool.
  76. func (az *Cloud) getBackendPoolID(lbName, backendPoolName string) string {
  77. return fmt.Sprintf(
  78. backendPoolIDTemplate,
  79. az.SubscriptionID,
  80. az.ResourceGroup,
  81. lbName,
  82. backendPoolName)
  83. }
  84. // returns the full identifier of a loadbalancer probe.
  85. func (az *Cloud) getLoadBalancerProbeID(lbName, lbRuleName string) string {
  86. return fmt.Sprintf(
  87. loadBalancerProbeIDTemplate,
  88. az.SubscriptionID,
  89. az.ResourceGroup,
  90. lbName,
  91. lbRuleName)
  92. }
  93. func (az *Cloud) mapLoadBalancerNameToVMSet(lbName string, clusterName string) (vmSetName string) {
  94. vmSetName = strings.TrimSuffix(lbName, InternalLoadBalancerNameSuffix)
  95. if strings.EqualFold(clusterName, vmSetName) {
  96. vmSetName = az.vmSet.GetPrimaryVMSetName()
  97. }
  98. return vmSetName
  99. }
  100. // For a load balancer, all frontend ip should reference either a subnet or publicIpAddress.
  101. // Thus Azure do not allow mixed type (public and internal) load balancer.
  102. // So we'd have a separate name for internal load balancer.
  103. // This would be the name for Azure LoadBalancer resource.
  104. func (az *Cloud) getLoadBalancerName(clusterName string, vmSetName string, isInternal bool) string {
  105. lbNamePrefix := vmSetName
  106. if strings.EqualFold(vmSetName, az.vmSet.GetPrimaryVMSetName()) || az.useStandardLoadBalancer() {
  107. lbNamePrefix = clusterName
  108. }
  109. if isInternal {
  110. return fmt.Sprintf("%s%s", lbNamePrefix, InternalLoadBalancerNameSuffix)
  111. }
  112. return lbNamePrefix
  113. }
  114. // isMasterNode returns true if the node has a master role label.
  115. // The master role is determined by looking for:
  116. // * a kubernetes.io/role="master" label
  117. func isMasterNode(node *v1.Node) bool {
  118. if val, ok := node.Labels[nodeLabelRole]; ok && val == "master" {
  119. return true
  120. }
  121. return false
  122. }
  123. // returns the deepest child's identifier from a full identifier string.
  124. func getLastSegment(ID string) (string, error) {
  125. parts := strings.Split(ID, "/")
  126. name := parts[len(parts)-1]
  127. if len(name) == 0 {
  128. return "", fmt.Errorf("resource name was missing from identifier")
  129. }
  130. return name, nil
  131. }
  132. // returns the equivalent LoadBalancerRule, SecurityRule and LoadBalancerProbe
  133. // protocol types for the given Kubernetes protocol type.
  134. func getProtocolsFromKubernetesProtocol(protocol v1.Protocol) (*network.TransportProtocol, *network.SecurityRuleProtocol, *network.ProbeProtocol, error) {
  135. var transportProto network.TransportProtocol
  136. var securityProto network.SecurityRuleProtocol
  137. var probeProto network.ProbeProtocol
  138. switch protocol {
  139. case v1.ProtocolTCP:
  140. transportProto = network.TransportProtocolTCP
  141. securityProto = network.SecurityRuleProtocolTCP
  142. probeProto = network.ProbeProtocolTCP
  143. return &transportProto, &securityProto, &probeProto, nil
  144. case v1.ProtocolUDP:
  145. transportProto = network.TransportProtocolUDP
  146. securityProto = network.SecurityRuleProtocolUDP
  147. return &transportProto, &securityProto, nil, nil
  148. default:
  149. return &transportProto, &securityProto, &probeProto, fmt.Errorf("Only TCP and UDP are supported for Azure LoadBalancers")
  150. }
  151. }
  152. // This returns the full identifier of the primary NIC for the given VM.
  153. func getPrimaryInterfaceID(machine compute.VirtualMachine) (string, error) {
  154. if len(*machine.NetworkProfile.NetworkInterfaces) == 1 {
  155. return *(*machine.NetworkProfile.NetworkInterfaces)[0].ID, nil
  156. }
  157. for _, ref := range *machine.NetworkProfile.NetworkInterfaces {
  158. if *ref.Primary {
  159. return *ref.ID, nil
  160. }
  161. }
  162. return "", fmt.Errorf("failed to find a primary nic for the vm. vmname=%q", *machine.Name)
  163. }
  164. func getPrimaryIPConfig(nic network.Interface) (*network.InterfaceIPConfiguration, error) {
  165. if nic.IPConfigurations == nil {
  166. return nil, fmt.Errorf("nic.IPConfigurations for nic (nicname=%q) is nil", *nic.Name)
  167. }
  168. if len(*nic.IPConfigurations) == 1 {
  169. return &((*nic.IPConfigurations)[0]), nil
  170. }
  171. for _, ref := range *nic.IPConfigurations {
  172. if *ref.Primary {
  173. return &ref, nil
  174. }
  175. }
  176. return nil, fmt.Errorf("failed to determine the primary ipconfig. nicname=%q", *nic.Name)
  177. }
  178. func isInternalLoadBalancer(lb *network.LoadBalancer) bool {
  179. return strings.HasSuffix(*lb.Name, InternalLoadBalancerNameSuffix)
  180. }
  181. func getBackendPoolName(clusterName string) string {
  182. return clusterName
  183. }
  184. func getLoadBalancerRuleName(service *v1.Service, port v1.ServicePort, subnetName *string) string {
  185. if subnetName == nil {
  186. return fmt.Sprintf("%s-%s-%d", getRulePrefix(service), port.Protocol, port.Port)
  187. }
  188. return fmt.Sprintf("%s-%s-%s-%d", getRulePrefix(service), *subnetName, port.Protocol, port.Port)
  189. }
  190. func getSecurityRuleName(service *v1.Service, port v1.ServicePort, sourceAddrPrefix string) string {
  191. if useSharedSecurityRule(service) {
  192. safePrefix := strings.Replace(sourceAddrPrefix, "/", "_", -1)
  193. return fmt.Sprintf("shared-%s-%d-%s", port.Protocol, port.Port, safePrefix)
  194. }
  195. safePrefix := strings.Replace(sourceAddrPrefix, "/", "_", -1)
  196. return fmt.Sprintf("%s-%s-%d-%s", getRulePrefix(service), port.Protocol, port.Port, safePrefix)
  197. }
  198. // This returns a human-readable version of the Service used to tag some resources.
  199. // This is only used for human-readable convenience, and not to filter.
  200. func getServiceName(service *v1.Service) string {
  201. return fmt.Sprintf("%s/%s", service.Namespace, service.Name)
  202. }
  203. // This returns a prefix for loadbalancer/security rules.
  204. func getRulePrefix(service *v1.Service) string {
  205. return cloudprovider.GetLoadBalancerName(service)
  206. }
  207. func getPublicIPName(clusterName string, service *v1.Service) string {
  208. return fmt.Sprintf("%s-%s", clusterName, cloudprovider.GetLoadBalancerName(service))
  209. }
  210. func serviceOwnsRule(service *v1.Service, rule string) bool {
  211. prefix := getRulePrefix(service)
  212. return strings.HasPrefix(strings.ToUpper(rule), strings.ToUpper(prefix))
  213. }
  214. func serviceOwnsFrontendIP(fip network.FrontendIPConfiguration, service *v1.Service) bool {
  215. baseName := cloudprovider.GetLoadBalancerName(service)
  216. return strings.HasPrefix(*fip.Name, baseName)
  217. }
  218. func getFrontendIPConfigName(service *v1.Service, subnetName *string) string {
  219. baseName := cloudprovider.GetLoadBalancerName(service)
  220. if subnetName != nil {
  221. return fmt.Sprintf("%s-%s", baseName, *subnetName)
  222. }
  223. return baseName
  224. }
  225. // This returns the next available rule priority level for a given set of security rules.
  226. func getNextAvailablePriority(rules []network.SecurityRule) (int32, error) {
  227. var smallest int32 = loadBalancerMinimumPriority
  228. var spread int32 = 1
  229. outer:
  230. for smallest < loadBalancerMaximumPriority {
  231. for _, rule := range rules {
  232. if *rule.Priority == smallest {
  233. smallest += spread
  234. continue outer
  235. }
  236. }
  237. // no one else had it
  238. return smallest, nil
  239. }
  240. return -1, fmt.Errorf("SecurityGroup priorities are exhausted")
  241. }
  242. func (az *Cloud) getIPForMachine(nodeName types.NodeName) (string, string, error) {
  243. return az.vmSet.GetIPByNodeName(string(nodeName))
  244. }
  245. var polyTable = crc32.MakeTable(crc32.Koopman)
  246. //MakeCRC32 : convert string to CRC32 format
  247. func MakeCRC32(str string) string {
  248. crc := crc32.New(polyTable)
  249. crc.Write([]byte(str))
  250. hash := crc.Sum32()
  251. return strconv.FormatUint(uint64(hash), 10)
  252. }
  253. //ExtractVMData : extract dataDisks, storageProfile from a map struct
  254. func ExtractVMData(vmData map[string]interface{}) (dataDisks []interface{},
  255. storageProfile map[string]interface{},
  256. hardwareProfile map[string]interface{}, err error) {
  257. props, ok := vmData["properties"].(map[string]interface{})
  258. if !ok {
  259. return nil, nil, nil, fmt.Errorf("convert vmData(properties) to map error")
  260. }
  261. storageProfile, ok = props["storageProfile"].(map[string]interface{})
  262. if !ok {
  263. return nil, nil, nil, fmt.Errorf("convert vmData(storageProfile) to map error")
  264. }
  265. hardwareProfile, ok = props["hardwareProfile"].(map[string]interface{})
  266. if !ok {
  267. return nil, nil, nil, fmt.Errorf("convert vmData(hardwareProfile) to map error")
  268. }
  269. dataDisks, ok = storageProfile["dataDisks"].([]interface{})
  270. if !ok {
  271. return nil, nil, nil, fmt.Errorf("convert vmData(dataDisks) to map error")
  272. }
  273. return dataDisks, storageProfile, hardwareProfile, nil
  274. }
  275. //ExtractDiskData : extract provisioningState, diskState from a map struct
  276. func ExtractDiskData(diskData interface{}) (provisioningState string, diskState string, err error) {
  277. fragment, ok := diskData.(map[string]interface{})
  278. if !ok {
  279. return "", "", fmt.Errorf("convert diskData to map error")
  280. }
  281. properties, ok := fragment["properties"].(map[string]interface{})
  282. if !ok {
  283. return "", "", fmt.Errorf("convert diskData(properties) to map error")
  284. }
  285. provisioningState, ok = properties["provisioningState"].(string) // if there is a disk, provisioningState property will be there
  286. if ref, ok := properties["diskState"]; ok {
  287. diskState = ref.(string)
  288. }
  289. return provisioningState, diskState, nil
  290. }
  291. // availabilitySet implements VMSet interface for Azure availability sets.
  292. type availabilitySet struct {
  293. *Cloud
  294. }
  295. // newStandardSet creates a new availabilitySet.
  296. func newAvailabilitySet(az *Cloud) VMSet {
  297. return &availabilitySet{
  298. Cloud: az,
  299. }
  300. }
  301. // GetInstanceIDByNodeName gets the cloud provider ID by node name.
  302. // It must return ("", cloudprovider.InstanceNotFound) if the instance does
  303. // not exist or is no longer running.
  304. func (as *availabilitySet) GetInstanceIDByNodeName(name string) (string, error) {
  305. var machine compute.VirtualMachine
  306. var err error
  307. machine, err = as.getVirtualMachine(types.NodeName(name))
  308. if err == cloudprovider.InstanceNotFound {
  309. return "", cloudprovider.InstanceNotFound
  310. }
  311. if err != nil {
  312. if as.CloudProviderBackoff {
  313. glog.V(2).Infof("InstanceID(%s) backing off", name)
  314. machine, err = as.GetVirtualMachineWithRetry(types.NodeName(name))
  315. if err != nil {
  316. glog.V(2).Infof("InstanceID(%s) abort backoff", name)
  317. return "", err
  318. }
  319. } else {
  320. return "", err
  321. }
  322. }
  323. return *machine.ID, nil
  324. }
  325. // GetNodeNameByProviderID gets the node name by provider ID.
  326. func (as *availabilitySet) GetNodeNameByProviderID(providerID string) (types.NodeName, error) {
  327. // NodeName is part of providerID for standard instances.
  328. matches := providerIDRE.FindStringSubmatch(providerID)
  329. if len(matches) != 2 {
  330. return "", errors.New("error splitting providerID")
  331. }
  332. return types.NodeName(matches[1]), nil
  333. }
  334. // GetInstanceTypeByNodeName gets the instance type by node name.
  335. func (as *availabilitySet) GetInstanceTypeByNodeName(name string) (string, error) {
  336. machine, err := as.getVirtualMachine(types.NodeName(name))
  337. if err != nil {
  338. glog.Errorf("error: as.GetInstanceTypeByNodeName(%s), as.getVirtualMachine(%s) err=%v", name, name, err)
  339. return "", err
  340. }
  341. return string(machine.HardwareProfile.VMSize), nil
  342. }
  343. // GetZoneByNodeName gets zone from instance view.
  344. func (as *availabilitySet) GetZoneByNodeName(name string) (cloudprovider.Zone, error) {
  345. vm, err := as.getVirtualMachine(types.NodeName(name))
  346. if err != nil {
  347. return cloudprovider.Zone{}, err
  348. }
  349. failureDomain := strconv.Itoa(int(*vm.VirtualMachineProperties.InstanceView.PlatformFaultDomain))
  350. zone := cloudprovider.Zone{
  351. FailureDomain: failureDomain,
  352. Region: *(vm.Location),
  353. }
  354. return zone, nil
  355. }
  356. // GetPrimaryVMSetName returns the VM set name depending on the configured vmType.
  357. // It returns config.PrimaryScaleSetName for vmss and config.PrimaryAvailabilitySetName for standard vmType.
  358. func (as *availabilitySet) GetPrimaryVMSetName() string {
  359. return as.Config.PrimaryAvailabilitySetName
  360. }
  361. // GetIPByNodeName gets machine private IP and public IP by node name.
  362. func (as *availabilitySet) GetIPByNodeName(name string) (string, string, error) {
  363. nic, err := as.GetPrimaryInterface(name)
  364. if err != nil {
  365. return "", "", err
  366. }
  367. ipConfig, err := getPrimaryIPConfig(nic)
  368. if err != nil {
  369. glog.Errorf("error: as.GetIPByNodeName(%s), getPrimaryIPConfig(%v), err=%v", name, nic, err)
  370. return "", "", err
  371. }
  372. privateIP := *ipConfig.PrivateIPAddress
  373. publicIP := ""
  374. if ipConfig.PublicIPAddress != nil && ipConfig.PublicIPAddress.ID != nil {
  375. pipID := *ipConfig.PublicIPAddress.ID
  376. pipName, err := getLastSegment(pipID)
  377. if err != nil {
  378. return "", "", fmt.Errorf("failed to publicIP name for node %q with pipID %q", name, pipID)
  379. }
  380. pip, existsPip, err := as.getPublicIPAddress(as.ResourceGroup, pipName)
  381. if err != nil {
  382. return "", "", err
  383. }
  384. if existsPip {
  385. publicIP = *pip.IPAddress
  386. }
  387. }
  388. return privateIP, publicIP, nil
  389. }
  390. // getAgentPoolAvailabiliySets lists the virtual machines for the resource group and then builds
  391. // a list of availability sets that match the nodes available to k8s.
  392. func (as *availabilitySet) getAgentPoolAvailabiliySets(nodes []*v1.Node) (agentPoolAvailabilitySets *[]string, err error) {
  393. vms, err := as.VirtualMachineClientListWithRetry()
  394. if err != nil {
  395. glog.Errorf("as.getNodeAvailabilitySet - VirtualMachineClientListWithRetry failed, err=%v", err)
  396. return nil, err
  397. }
  398. vmNameToAvailabilitySetID := make(map[string]string, len(vms))
  399. for vmx := range vms {
  400. vm := vms[vmx]
  401. if vm.AvailabilitySet != nil {
  402. vmNameToAvailabilitySetID[*vm.Name] = *vm.AvailabilitySet.ID
  403. }
  404. }
  405. availabilitySetIDs := sets.NewString()
  406. agentPoolAvailabilitySets = &[]string{}
  407. for nx := range nodes {
  408. nodeName := (*nodes[nx]).Name
  409. if isMasterNode(nodes[nx]) {
  410. continue
  411. }
  412. asID, ok := vmNameToAvailabilitySetID[nodeName]
  413. if !ok {
  414. glog.Errorf("as.getNodeAvailabilitySet - Node(%s) has no availability sets", nodeName)
  415. return nil, fmt.Errorf("Node (%s) - has no availability sets", nodeName)
  416. }
  417. if availabilitySetIDs.Has(asID) {
  418. // already added in the list
  419. continue
  420. }
  421. asName, err := getLastSegment(asID)
  422. if err != nil {
  423. glog.Errorf("as.getNodeAvailabilitySet - Node (%s)- getLastSegment(%s), err=%v", nodeName, asID, err)
  424. return nil, err
  425. }
  426. // AvailabilitySet ID is currently upper cased in a indeterministic way
  427. // We want to keep it lower case, before the ID get fixed
  428. asName = strings.ToLower(asName)
  429. *agentPoolAvailabilitySets = append(*agentPoolAvailabilitySets, asName)
  430. }
  431. return agentPoolAvailabilitySets, nil
  432. }
  433. // GetVMSetNames selects all possible availability sets or scale sets
  434. // (depending vmType configured) for service load balancer, if the service has
  435. // no loadbalancer mode annotaion returns the primary VMSet. If service annotation
  436. // for loadbalancer exists then return the eligible VMSet.
  437. func (as *availabilitySet) GetVMSetNames(service *v1.Service, nodes []*v1.Node) (availabilitySetNames *[]string, err error) {
  438. hasMode, isAuto, serviceAvailabilitySetNames := getServiceLoadBalancerMode(service)
  439. if !hasMode {
  440. // no mode specified in service annotation default to PrimaryAvailabilitySetName
  441. availabilitySetNames = &[]string{as.Config.PrimaryAvailabilitySetName}
  442. return availabilitySetNames, nil
  443. }
  444. availabilitySetNames, err = as.getAgentPoolAvailabiliySets(nodes)
  445. if err != nil {
  446. glog.Errorf("as.GetVMSetNames - getAgentPoolAvailabiliySets failed err=(%v)", err)
  447. return nil, err
  448. }
  449. if len(*availabilitySetNames) == 0 {
  450. glog.Errorf("as.GetVMSetNames - No availability sets found for nodes in the cluster, node count(%d)", len(nodes))
  451. return nil, fmt.Errorf("No availability sets found for nodes, node count(%d)", len(nodes))
  452. }
  453. // sort the list to have deterministic selection
  454. sort.Strings(*availabilitySetNames)
  455. if !isAuto {
  456. if serviceAvailabilitySetNames == nil || len(serviceAvailabilitySetNames) == 0 {
  457. return nil, fmt.Errorf("service annotation for LoadBalancerMode is empty, it should have __auto__ or availability sets value")
  458. }
  459. // validate availability set exists
  460. var found bool
  461. for sasx := range serviceAvailabilitySetNames {
  462. for asx := range *availabilitySetNames {
  463. if strings.EqualFold((*availabilitySetNames)[asx], serviceAvailabilitySetNames[sasx]) {
  464. found = true
  465. serviceAvailabilitySetNames[sasx] = (*availabilitySetNames)[asx]
  466. break
  467. }
  468. }
  469. if !found {
  470. glog.Errorf("as.GetVMSetNames - Availability set (%s) in service annotation not found", serviceAvailabilitySetNames[sasx])
  471. return nil, fmt.Errorf("availability set (%s) - not found", serviceAvailabilitySetNames[sasx])
  472. }
  473. }
  474. availabilitySetNames = &serviceAvailabilitySetNames
  475. }
  476. return availabilitySetNames, nil
  477. }
  478. // GetPrimaryInterface gets machine primary network interface by node name.
  479. func (as *availabilitySet) GetPrimaryInterface(nodeName string) (network.Interface, error) {
  480. return as.getPrimaryInterfaceWithVMSet(nodeName, "")
  481. }
  482. // getPrimaryInterfaceWithVMSet gets machine primary network interface by node name and vmSet.
  483. func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName string) (network.Interface, error) {
  484. var machine compute.VirtualMachine
  485. machine, err := as.GetVirtualMachineWithRetry(types.NodeName(nodeName))
  486. if err != nil {
  487. glog.V(2).Infof("GetPrimaryInterface(%s, %s) abort backoff", nodeName, vmSetName)
  488. return network.Interface{}, err
  489. }
  490. primaryNicID, err := getPrimaryInterfaceID(machine)
  491. if err != nil {
  492. return network.Interface{}, err
  493. }
  494. nicName, err := getLastSegment(primaryNicID)
  495. if err != nil {
  496. return network.Interface{}, err
  497. }
  498. // Check availability set name. Note that vmSetName is empty string when getting
  499. // the Node's IP address. While vmSetName is not empty, it should be checked with
  500. // Node's real availability set name:
  501. // - For basic SKU load balancer, errNotInVMSet should be returned if the node's
  502. // availability set is mismatched with vmSetName.
  503. // - For standard SKU load balancer, backend could belong to multiple VMAS, so we
  504. // don't check vmSet for it.
  505. if vmSetName != "" && !as.useStandardLoadBalancer() {
  506. expectedAvailabilitySetName := as.getAvailabilitySetID(vmSetName)
  507. if machine.AvailabilitySet == nil || !strings.EqualFold(*machine.AvailabilitySet.ID, expectedAvailabilitySetName) {
  508. glog.V(3).Infof(
  509. "GetPrimaryInterface: nic (%s) is not in the availabilitySet(%s)", nicName, vmSetName)
  510. return network.Interface{}, errNotInVMSet
  511. }
  512. }
  513. ctx, cancel := getContextWithCancel()
  514. defer cancel()
  515. nic, err := as.InterfacesClient.Get(ctx, as.ResourceGroup, nicName, "")
  516. if err != nil {
  517. return network.Interface{}, err
  518. }
  519. return nic, nil
  520. }
  521. // ensureHostInPool ensures the given VM's Primary NIC's Primary IP Configuration is
  522. // participating in the specified LoadBalancer Backend Pool.
  523. func (as *availabilitySet) ensureHostInPool(serviceName string, nodeName types.NodeName, backendPoolID string, vmSetName string, isInternal bool) error {
  524. vmName := mapNodeNameToVMName(nodeName)
  525. nic, err := as.getPrimaryInterfaceWithVMSet(vmName, vmSetName)
  526. if err != nil {
  527. if err == errNotInVMSet {
  528. glog.V(3).Infof("ensureHostInPool skips node %s because it is not in the vmSet %s", nodeName, vmSetName)
  529. return nil
  530. }
  531. glog.Errorf("error: az.ensureHostInPool(%s), az.vmSet.GetPrimaryInterface.Get(%s, %s), err=%v", nodeName, vmName, vmSetName, err)
  532. return err
  533. }
  534. var primaryIPConfig *network.InterfaceIPConfiguration
  535. primaryIPConfig, err = getPrimaryIPConfig(nic)
  536. if err != nil {
  537. return err
  538. }
  539. foundPool := false
  540. newBackendPools := []network.BackendAddressPool{}
  541. if primaryIPConfig.LoadBalancerBackendAddressPools != nil {
  542. newBackendPools = *primaryIPConfig.LoadBalancerBackendAddressPools
  543. }
  544. for _, existingPool := range newBackendPools {
  545. if strings.EqualFold(backendPoolID, *existingPool.ID) {
  546. foundPool = true
  547. break
  548. }
  549. }
  550. if !foundPool {
  551. if as.useStandardLoadBalancer() && len(newBackendPools) > 0 {
  552. // Although standard load balancer supports backends from multiple availability
  553. // sets, the same network interface couldn't be added to more than one load balancer of
  554. // the same type. Omit those nodes (e.g. masters) so Azure ARM won't complain
  555. // about this.
  556. for _, pool := range newBackendPools {
  557. backendPool := *pool.ID
  558. matches := backendPoolIDRE.FindStringSubmatch(backendPool)
  559. if len(matches) == 2 {
  560. lbName := matches[1]
  561. if strings.HasSuffix(lbName, InternalLoadBalancerNameSuffix) == isInternal {
  562. glog.V(4).Infof("Node %q has already been added to LB %q, omit adding it to a new one", nodeName, lbName)
  563. return nil
  564. }
  565. }
  566. }
  567. }
  568. newBackendPools = append(newBackendPools,
  569. network.BackendAddressPool{
  570. ID: to.StringPtr(backendPoolID),
  571. })
  572. primaryIPConfig.LoadBalancerBackendAddressPools = &newBackendPools
  573. nicName := *nic.Name
  574. glog.V(3).Infof("nicupdate(%s): nic(%s) - updating", serviceName, nicName)
  575. ctx, cancel := getContextWithCancel()
  576. defer cancel()
  577. resp, err := as.InterfacesClient.CreateOrUpdate(ctx, as.ResourceGroup, *nic.Name, nic)
  578. glog.V(10).Infof("InterfacesClient.CreateOrUpdate(%q): end", *nic.Name)
  579. if as.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) {
  580. glog.V(2).Infof("nicupdate(%s) backing off: nic(%s) - updating, err=%v", serviceName, nicName, err)
  581. retryErr := as.CreateOrUpdateInterfaceWithRetry(nic)
  582. if retryErr != nil {
  583. err = retryErr
  584. glog.V(2).Infof("nicupdate(%s) abort backoff: nic(%s) - updating", serviceName, nicName)
  585. }
  586. }
  587. if err != nil {
  588. return err
  589. }
  590. }
  591. return nil
  592. }
  593. // EnsureHostsInPool ensures the given Node's primary IP configurations are
  594. // participating in the specified LoadBalancer Backend Pool.
  595. func (as *availabilitySet) EnsureHostsInPool(serviceName string, nodes []*v1.Node, backendPoolID string, vmSetName string, isInternal bool) error {
  596. hostUpdates := make([]func() error, 0, len(nodes))
  597. for _, node := range nodes {
  598. localNodeName := node.Name
  599. if as.useStandardLoadBalancer() && as.excludeMasterNodesFromStandardLB() && isMasterNode(node) {
  600. glog.V(4).Infof("Excluding master node %q from load balancer backendpool %q", localNodeName, backendPoolID)
  601. continue
  602. }
  603. f := func() error {
  604. err := as.ensureHostInPool(serviceName, types.NodeName(localNodeName), backendPoolID, vmSetName, isInternal)
  605. if err != nil {
  606. return fmt.Errorf("ensure(%s): backendPoolID(%s) - failed to ensure host in pool: %q", serviceName, backendPoolID, err)
  607. }
  608. return nil
  609. }
  610. hostUpdates = append(hostUpdates, f)
  611. }
  612. errs := utilerrors.AggregateGoroutines(hostUpdates...)
  613. if errs != nil {
  614. return utilerrors.Flatten(errs)
  615. }
  616. return nil
  617. }
  618. // EnsureBackendPoolDeleted ensures the loadBalancer backendAddressPools deleted from the specified vmSet.
  619. func (as *availabilitySet) EnsureBackendPoolDeleted(poolID, vmSetName string, backendAddressPools *[]network.BackendAddressPool) error {
  620. // Do nothing for availability set.
  621. return nil
  622. }
  623. // get a storage account by UUID
  624. func generateStorageAccountName(accountNamePrefix string) string {
  625. uniqueID := strings.Replace(string(uuid.NewUUID()), "-", "", -1)
  626. accountName := strings.ToLower(accountNamePrefix + uniqueID)
  627. if len(accountName) > storageAccountNameMaxLength {
  628. return accountName[:storageAccountNameMaxLength-1]
  629. }
  630. return accountName
  631. }