Skip to content

Commit

Permalink
Enhance: support custom health checks for AlibabaCloud-NLB (openkruis…
Browse files Browse the repository at this point in the history 
…e#147)

Signed-off-by: ChrisLiu <chrisliu1995@163.com>
  • Loading branch information
@chrisliu1995
chrisliu1995 authored May 22, 2024
1 parent cafaab3 commit b6fdc23
Show file tree
Hide file tree
Showing 4 changed files with 669 additions and 44 deletions.
227 changes: 209 additions & 18 deletions cloudprovider/alibabacloud/nlb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package alibabacloud

import (
"context"
"fmt"
gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
"github.com/openkruise/kruise-game/cloudprovider"
cperrors "github.com/openkruise/kruise-game/cloudprovider/errors"
Expand All @@ -30,6 +31,7 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
log "k8s.io/klog/v2"
"regexp"
"sigs.k8s.io/controller-runtime/pkg/client"
"strconv"
"strings"
Expand All @@ -39,6 +41,30 @@ import (
const (
NlbNetwork = "AlibabaCloud-NLB"
AliasNLB = "NLB-Network"

// annotations provided by AlibabaCloud Cloud Controller Manager
LBHealthCheckFlagAnnotationKey = "service.beta.kubernetes.io/alibaba-cloud-loadbalancer-health-check-flag"
LBHealthCheckTypeAnnotationKey = "service.beta.kubernetes.io/alibaba-cloud-loadbalancer-health-check-type"
LBHealthCheckConnectPortAnnotationKey = "service.beta.kubernetes.io/alibaba-cloud-loadbalancer-health-check-connect-port"
LBHealthCheckConnectTimeoutAnnotationKey = "service.beta.kubernetes.io/alibaba-cloud-loadbalancer-health-check-connect-timeout"
LBHealthyThresholdAnnotationKey = "service.beta.kubernetes.io/alibaba-cloud-loadbalancer-healthy-threshold"
LBUnhealthyThresholdAnnotationKey = "service.beta.kubernetes.io/alibaba-cloud-loadbalancer-unhealthy-threshold"
LBHealthCheckIntervalAnnotationKey = "service.beta.kubernetes.io/alibaba-cloud-loadbalancer-health-check-interval"
LBHealthCheckUriAnnotationKey = "service.beta.kubernetes.io/alibaba-cloud-loadbalancer-health-check-uri"
LBHealthCheckDomainAnnotationKey = "service.beta.kubernetes.io/alibaba-cloud-loadbalancer-health-check-domain"
LBHealthCheckMethodAnnotationKey = "service.beta.kubernetes.io/alibaba-cloud-loadbalancer-health-check-method"

// ConfigNames defined by OKG
LBHealthCheckFlagConfigName = "LBHealthCheckFlag"
LBHealthCheckTypeConfigName = "LBHealthCheckType"
LBHealthCheckConnectPortConfigName = "LBHealthCheckConnectPort"
LBHealthCheckConnectTimeoutConfigName = "LBHealthCheckConnectTimeout"
LBHealthCheckIntervalConfigName = "LBHealthCheckInterval"
LBHealthCheckUriConfigName = "LBHealthCheckUri"
LBHealthCheckDomainConfigName = "LBHealthCheckDomain"
LBHealthCheckMethodConfigName = "LBHealthCheckMethod"
LBHealthyThresholdConfigName = "LBHealthyThreshold"
LBUnhealthyThresholdConfigName = "LBUnhealthyThreshold"
)

type NlbPlugin struct {
Expand All @@ -50,10 +76,20 @@ type NlbPlugin struct {
}

type nlbConfig struct {
lbIds []string
targetPorts []int
protocols []corev1.Protocol
isFixed bool
lbIds []string
targetPorts []int
protocols []corev1.Protocol
isFixed bool
lBHealthCheckFlag string
lBHealthCheckType string
lBHealthCheckConnectPort string
lBHealthCheckConnectTimeout string
lBHealthCheckInterval string
lBHealthCheckUri string
lBHealthCheckDomain string
lBHealthCheckMethod string
lBHealthyThreshold string
lBUnhealthyThreshold string
}

func (n *NlbPlugin) Name() string {
Expand Down Expand Up @@ -91,7 +127,10 @@ func (n *NlbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.C

networkStatus, _ := networkManager.GetNetworkStatus()
networkConfig := networkManager.GetNetworkConfig()
sc := parseNlbConfig(networkConfig)
sc, err := parseNlbConfig(networkConfig)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.ParameterError, err.Error())
}
if networkStatus == nil {
pod, err := networkManager.UpdateNetworkStatus(gamekruiseiov1alpha1.NetworkStatus{
CurrentNetworkState: gamekruiseiov1alpha1.NetworkNotReady,
Expand All @@ -101,7 +140,7 @@ func (n *NlbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.C

// get svc
svc := &corev1.Service{}
err := c.Get(ctx, types.NamespacedName{
err = c.Get(ctx, types.NamespacedName{
Name: pod.GetName(),
Namespace: pod.GetNamespace(),
}, svc)
Expand Down Expand Up @@ -258,15 +297,31 @@ func (n *NlbPlugin) consSvc(nc *nlbConfig, pod *corev1.Pod, c client.Client, ctx

loadBalancerClass := "alibabacloud.com/nlb"

svcAnnotations := map[string]string{
SlbListenerOverrideKey: "true",
SlbIdAnnotationKey: lbId,
SlbConfigHashKey: util.GetHash(nc),
LBHealthCheckFlagAnnotationKey: nc.lBHealthCheckFlag,
}
if nc.lBHealthCheckFlag == "on" {
svcAnnotations[LBHealthCheckTypeAnnotationKey] = nc.lBHealthCheckType
svcAnnotations[LBHealthCheckConnectPortAnnotationKey] = nc.lBHealthCheckConnectPort
svcAnnotations[LBHealthCheckConnectTimeoutAnnotationKey] = nc.lBHealthCheckConnectTimeout
svcAnnotations[LBHealthCheckIntervalAnnotationKey] = nc.lBHealthCheckInterval
svcAnnotations[LBHealthyThresholdAnnotationKey] = nc.lBHealthyThreshold
svcAnnotations[LBUnhealthyThresholdAnnotationKey] = nc.lBUnhealthyThreshold
if nc.lBHealthCheckType == "http" {
svcAnnotations[LBHealthCheckDomainAnnotationKey] = nc.lBHealthCheckDomain
svcAnnotations[LBHealthCheckUriAnnotationKey] = nc.lBHealthCheckUri
svcAnnotations[LBHealthCheckMethodAnnotationKey] = nc.lBHealthCheckMethod
}
}

svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: pod.GetName(),
Namespace: pod.GetNamespace(),
Annotations: map[string]string{
SlbListenerOverrideKey: "true",
SlbIdAnnotationKey: lbId,
SlbConfigHashKey: util.GetHash(nc),
},
Name: pod.GetName(),
Namespace: pod.GetNamespace(),
Annotations: svcAnnotations,
OwnerReferences: getSvcOwnerReference(c, ctx, pod, nc.isFixed),
},
Spec: corev1.ServiceSpec{
Expand Down Expand Up @@ -348,11 +403,21 @@ func (n *NlbPlugin) deAllocate(nsName string) {
log.Infof("pod %s deallocate nlb %s ports %v", nsName, lbId, ports)
}

func parseNlbConfig(conf []gamekruiseiov1alpha1.NetworkConfParams) *nlbConfig {
func parseNlbConfig(conf []gamekruiseiov1alpha1.NetworkConfParams) (*nlbConfig, error) {
var lbIds []string
ports := make([]int, 0)
protocols := make([]corev1.Protocol, 0)
isFixed := false
lBHealthCheckFlag := "on"
lBHealthCheckType := "tcp"
lBHealthCheckConnectPort := "0"
lBHealthCheckConnectTimeout := "5"
lBHealthCheckInterval := "10"
lBUnhealthyThreshold := "2"
lBHealthyThreshold := "2"
lBHealthCheckUri := ""
lBHealthCheckDomain := ""
lBHealthCheckMethod := ""
for _, c := range conf {
switch c.Name {
case NlbIdsConfigName:
Expand Down Expand Up @@ -381,12 +446,138 @@ func parseNlbConfig(conf []gamekruiseiov1alpha1.NetworkConfParams) *nlbConfig {
continue
}
isFixed = v
case LBHealthCheckFlagConfigName:
flag := strings.ToLower(c.Value)
if flag != "on" && flag != "off" {
return nil, fmt.Errorf("invalid lb health check flag value: %s", c.Value)
}
lBHealthCheckFlag = flag
case LBHealthCheckTypeConfigName:
checkType := strings.ToLower(c.Value)
if checkType != "tcp" && checkType != "http" {
return nil, fmt.Errorf("invalid lb health check type: %s", c.Value)
}
lBHealthCheckType = checkType
case LBHealthCheckConnectPortConfigName:
portInt, err := strconv.Atoi(c.Value)
if err != nil {
return nil, fmt.Errorf("invalid lb health check connect port: %s", c.Value)
}
if portInt < 0 || portInt > 65535 {
return nil, fmt.Errorf("invalid lb health check connect port: %d", portInt)
}
lBHealthCheckConnectPort = c.Value
case LBHealthCheckConnectTimeoutConfigName:
timeoutInt, err := strconv.Atoi(c.Value)
if err != nil {
return nil, fmt.Errorf("invalid lb health check connect timeout: %s", c.Value)
}
if timeoutInt < 1 || timeoutInt > 300 {
return nil, fmt.Errorf("invalid lb health check connect timeout: %d", timeoutInt)
}
lBHealthCheckConnectTimeout = c.Value
case LBHealthCheckIntervalConfigName:
intervalInt, err := strconv.Atoi(c.Value)
if err != nil {
return nil, fmt.Errorf("invalid lb health check interval: %s", c.Value)
}
if intervalInt < 1 || intervalInt > 50 {
return nil, fmt.Errorf("invalid lb health check interval: %d", intervalInt)
}
lBHealthCheckInterval = c.Value
case LBHealthyThresholdConfigName:
thresholdInt, err := strconv.Atoi(c.Value)
if err != nil {
return nil, fmt.Errorf("invalid lb healthy threshold: %s", c.Value)
}
if thresholdInt < 2 || thresholdInt > 10 {
return nil, fmt.Errorf("invalid lb healthy threshold: %d", thresholdInt)
}
lBHealthyThreshold = c.Value
case LBUnhealthyThresholdConfigName:
thresholdInt, err := strconv.Atoi(c.Value)
if err != nil {
return nil, fmt.Errorf("invalid lb unhealthy threshold: %s", c.Value)
}
if thresholdInt < 2 || thresholdInt > 10 {
return nil, fmt.Errorf("invalid lb unhealthy threshold: %d", thresholdInt)
}
lBUnhealthyThreshold = c.Value
case LBHealthCheckUriConfigName:
if validateUri(c.Value) != nil {
return nil, fmt.Errorf("invalid lb health check uri: %s", c.Value)
}
lBHealthCheckUri = c.Value
case LBHealthCheckDomainConfigName:
if validateDomain(c.Value) != nil {
return nil, fmt.Errorf("invalid lb health check domain: %s", c.Value)
}
lBHealthCheckDomain = c.Value
case LBHealthCheckMethodConfigName:
method := strings.ToLower(c.Value)
if method != "get" && method != "head" {
return nil, fmt.Errorf("invalid lb health check method: %s", c.Value)
}
lBHealthCheckMethod = method
}
}
return &nlbConfig{
lbIds: lbIds,
protocols: protocols,
targetPorts: ports,
isFixed: isFixed,
lbIds: lbIds,
protocols: protocols,
targetPorts: ports,
isFixed: isFixed,
lBHealthCheckFlag: lBHealthCheckFlag,
lBHealthCheckType: lBHealthCheckType,
lBHealthCheckConnectPort: lBHealthCheckConnectPort,
lBHealthCheckConnectTimeout: lBHealthCheckConnectTimeout,
lBHealthCheckInterval: lBHealthCheckInterval,
lBHealthCheckUri: lBHealthCheckUri,
lBHealthCheckDomain: lBHealthCheckDomain,
lBHealthCheckMethod: lBHealthCheckMethod,
lBHealthyThreshold: lBHealthyThreshold,
lBUnhealthyThreshold: lBUnhealthyThreshold,
}, nil
}

func validateDomain(domain string) error {
if len(domain) < 1 || len(domain) > 80 {
return fmt.Errorf("the domain length must be between 1 and 80 characters")
}

// Regular expression matches lowercase letters, numbers, dashes and periods
domainRegex := regexp.MustCompile(`^[a-z0-9-.]+$`)
if !domainRegex.MatchString(domain) {
return fmt.Errorf("the domain must only contain lowercase letters, numbers, hyphens, and periods")
}

// make sure the domain name does not start or end with a dash or period
if domain[0] == '-' || domain[0] == '.' || domain[len(domain)-1] == '-' || domain[len(domain)-1] == '.' {
return fmt.Errorf("the domain must not start or end with a hyphen or period")
}

// make sure the domain name does not contain consecutive dots or dashes
if regexp.MustCompile(`(--|\.\.)`).MatchString(domain) {
return fmt.Errorf("the domain must not contain consecutive hyphens or periods")
}

return nil
}

func validateUri(uri string) error {
if len(uri) < 1 || len(uri) > 80 {
return fmt.Errorf("string length must be between 1 and 80 characters")
}

regexPattern := `^/[0-9a-zA-Z.!$%&'*+/=?^_` + "`" + `{|}~-]*$`
matched, err := regexp.MatchString(regexPattern, uri)

if err != nil {
return fmt.Errorf("regex error: %v", err)
}

if !matched {
return fmt.Errorf("string does not match the required pattern")
}

return nil
}
Loading

0 comments on commit b6fdc23

Please sign in to comment.