diff --git a/cloudprovider/alibabacloud/nlb.go b/cloudprovider/alibabacloud/nlb.go index c3d6db89..48ff9ce1 100644 --- a/cloudprovider/alibabacloud/nlb.go +++ b/cloudprovider/alibabacloud/nlb.go @@ -18,6 +18,7 @@ package alibabacloud import ( "context" + "fmt" gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" "github.com/openkruise/kruise-game/cloudprovider" cperrors "github.com/openkruise/kruise-game/cloudprovider/errors" @@ -30,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" log "k8s.io/klog/v2" + "regexp" "sigs.k8s.io/controller-runtime/pkg/client" "strconv" "strings" @@ -39,6 +41,30 @@ import ( const ( NlbNetwork = "AlibabaCloud-NLB" AliasNLB = "NLB-Network" + + // annotations provided by AlibabaCloud Cloud Controller Manager + LBHealthCheckFlagAnnotationKey = "service.beta.kubernetes.io/alibaba-cloud-loadbalancer-health-check-flag" + LBHealthCheckTypeAnnotationKey = "service.beta.kubernetes.io/alibaba-cloud-loadbalancer-health-check-type" + LBHealthCheckConnectPortAnnotationKey = "service.beta.kubernetes.io/alibaba-cloud-loadbalancer-health-check-connect-port" + LBHealthCheckConnectTimeoutAnnotationKey = "service.beta.kubernetes.io/alibaba-cloud-loadbalancer-health-check-connect-timeout" + LBHealthyThresholdAnnotationKey = "service.beta.kubernetes.io/alibaba-cloud-loadbalancer-healthy-threshold" + LBUnhealthyThresholdAnnotationKey = "service.beta.kubernetes.io/alibaba-cloud-loadbalancer-unhealthy-threshold" + LBHealthCheckIntervalAnnotationKey = "service.beta.kubernetes.io/alibaba-cloud-loadbalancer-health-check-interval" + LBHealthCheckUriAnnotationKey = "service.beta.kubernetes.io/alibaba-cloud-loadbalancer-health-check-uri" + LBHealthCheckDomainAnnotationKey = "service.beta.kubernetes.io/alibaba-cloud-loadbalancer-health-check-domain" + LBHealthCheckMethodAnnotationKey = "service.beta.kubernetes.io/alibaba-cloud-loadbalancer-health-check-method" + + // ConfigNames defined by OKG + LBHealthCheckFlagConfigName = "LBHealthCheckFlag" + LBHealthCheckTypeConfigName = "LBHealthCheckType" + LBHealthCheckConnectPortConfigName = "LBHealthCheckConnectPort" + LBHealthCheckConnectTimeoutConfigName = "LBHealthCheckConnectTimeout" + LBHealthCheckIntervalConfigName = "LBHealthCheckInterval" + LBHealthCheckUriConfigName = "LBHealthCheckUri" + LBHealthCheckDomainConfigName = "LBHealthCheckDomain" + LBHealthCheckMethodConfigName = "LBHealthCheckMethod" + LBHealthyThresholdConfigName = "LBHealthyThreshold" + LBUnhealthyThresholdConfigName = "LBUnhealthyThreshold" ) type NlbPlugin struct { @@ -50,10 +76,20 @@ type NlbPlugin struct { } type nlbConfig struct { - lbIds []string - targetPorts []int - protocols []corev1.Protocol - isFixed bool + lbIds []string + targetPorts []int + protocols []corev1.Protocol + isFixed bool + lBHealthCheckFlag string + lBHealthCheckType string + lBHealthCheckConnectPort string + lBHealthCheckConnectTimeout string + lBHealthCheckInterval string + lBHealthCheckUri string + lBHealthCheckDomain string + lBHealthCheckMethod string + lBHealthyThreshold string + lBUnhealthyThreshold string } func (n *NlbPlugin) Name() string { @@ -91,7 +127,10 @@ func (n *NlbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.C networkStatus, _ := networkManager.GetNetworkStatus() networkConfig := networkManager.GetNetworkConfig() - sc := parseNlbConfig(networkConfig) + sc, err := parseNlbConfig(networkConfig) + if err != nil { + return pod, cperrors.NewPluginError(cperrors.ParameterError, err.Error()) + } if networkStatus == nil { pod, err := networkManager.UpdateNetworkStatus(gamekruiseiov1alpha1.NetworkStatus{ CurrentNetworkState: gamekruiseiov1alpha1.NetworkNotReady, @@ -101,7 +140,7 @@ func (n *NlbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.C // get svc svc := &corev1.Service{} - err := c.Get(ctx, types.NamespacedName{ + err = c.Get(ctx, types.NamespacedName{ Name: pod.GetName(), Namespace: pod.GetNamespace(), }, svc) @@ -258,15 +297,31 @@ func (n *NlbPlugin) consSvc(nc *nlbConfig, pod *corev1.Pod, c client.Client, ctx loadBalancerClass := "alibabacloud.com/nlb" + svcAnnotations := map[string]string{ + SlbListenerOverrideKey: "true", + SlbIdAnnotationKey: lbId, + SlbConfigHashKey: util.GetHash(nc), + LBHealthCheckFlagAnnotationKey: nc.lBHealthCheckFlag, + } + if nc.lBHealthCheckFlag == "on" { + svcAnnotations[LBHealthCheckTypeAnnotationKey] = nc.lBHealthCheckType + svcAnnotations[LBHealthCheckConnectPortAnnotationKey] = nc.lBHealthCheckConnectPort + svcAnnotations[LBHealthCheckConnectTimeoutAnnotationKey] = nc.lBHealthCheckConnectTimeout + svcAnnotations[LBHealthCheckIntervalAnnotationKey] = nc.lBHealthCheckInterval + svcAnnotations[LBHealthyThresholdAnnotationKey] = nc.lBHealthyThreshold + svcAnnotations[LBUnhealthyThresholdAnnotationKey] = nc.lBUnhealthyThreshold + if nc.lBHealthCheckType == "http" { + svcAnnotations[LBHealthCheckDomainAnnotationKey] = nc.lBHealthCheckDomain + svcAnnotations[LBHealthCheckUriAnnotationKey] = nc.lBHealthCheckUri + svcAnnotations[LBHealthCheckMethodAnnotationKey] = nc.lBHealthCheckMethod + } + } + svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: pod.GetName(), - Namespace: pod.GetNamespace(), - Annotations: map[string]string{ - SlbListenerOverrideKey: "true", - SlbIdAnnotationKey: lbId, - SlbConfigHashKey: util.GetHash(nc), - }, + Name: pod.GetName(), + Namespace: pod.GetNamespace(), + Annotations: svcAnnotations, OwnerReferences: getSvcOwnerReference(c, ctx, pod, nc.isFixed), }, Spec: corev1.ServiceSpec{ @@ -348,11 +403,21 @@ func (n *NlbPlugin) deAllocate(nsName string) { log.Infof("pod %s deallocate nlb %s ports %v", nsName, lbId, ports) } -func parseNlbConfig(conf []gamekruiseiov1alpha1.NetworkConfParams) *nlbConfig { +func parseNlbConfig(conf []gamekruiseiov1alpha1.NetworkConfParams) (*nlbConfig, error) { var lbIds []string ports := make([]int, 0) protocols := make([]corev1.Protocol, 0) isFixed := false + lBHealthCheckFlag := "on" + lBHealthCheckType := "tcp" + lBHealthCheckConnectPort := "0" + lBHealthCheckConnectTimeout := "5" + lBHealthCheckInterval := "10" + lBUnhealthyThreshold := "2" + lBHealthyThreshold := "2" + lBHealthCheckUri := "" + lBHealthCheckDomain := "" + lBHealthCheckMethod := "" for _, c := range conf { switch c.Name { case NlbIdsConfigName: @@ -381,12 +446,138 @@ func parseNlbConfig(conf []gamekruiseiov1alpha1.NetworkConfParams) *nlbConfig { continue } isFixed = v + case LBHealthCheckFlagConfigName: + flag := strings.ToLower(c.Value) + if flag != "on" && flag != "off" { + return nil, fmt.Errorf("invalid lb health check flag value: %s", c.Value) + } + lBHealthCheckFlag = flag + case LBHealthCheckTypeConfigName: + checkType := strings.ToLower(c.Value) + if checkType != "tcp" && checkType != "http" { + return nil, fmt.Errorf("invalid lb health check type: %s", c.Value) + } + lBHealthCheckType = checkType + case LBHealthCheckConnectPortConfigName: + portInt, err := strconv.Atoi(c.Value) + if err != nil { + return nil, fmt.Errorf("invalid lb health check connect port: %s", c.Value) + } + if portInt < 0 || portInt > 65535 { + return nil, fmt.Errorf("invalid lb health check connect port: %d", portInt) + } + lBHealthCheckConnectPort = c.Value + case LBHealthCheckConnectTimeoutConfigName: + timeoutInt, err := strconv.Atoi(c.Value) + if err != nil { + return nil, fmt.Errorf("invalid lb health check connect timeout: %s", c.Value) + } + if timeoutInt < 1 || timeoutInt > 300 { + return nil, fmt.Errorf("invalid lb health check connect timeout: %d", timeoutInt) + } + lBHealthCheckConnectTimeout = c.Value + case LBHealthCheckIntervalConfigName: + intervalInt, err := strconv.Atoi(c.Value) + if err != nil { + return nil, fmt.Errorf("invalid lb health check interval: %s", c.Value) + } + if intervalInt < 1 || intervalInt > 50 { + return nil, fmt.Errorf("invalid lb health check interval: %d", intervalInt) + } + lBHealthCheckInterval = c.Value + case LBHealthyThresholdConfigName: + thresholdInt, err := strconv.Atoi(c.Value) + if err != nil { + return nil, fmt.Errorf("invalid lb healthy threshold: %s", c.Value) + } + if thresholdInt < 2 || thresholdInt > 10 { + return nil, fmt.Errorf("invalid lb healthy threshold: %d", thresholdInt) + } + lBHealthyThreshold = c.Value + case LBUnhealthyThresholdConfigName: + thresholdInt, err := strconv.Atoi(c.Value) + if err != nil { + return nil, fmt.Errorf("invalid lb unhealthy threshold: %s", c.Value) + } + if thresholdInt < 2 || thresholdInt > 10 { + return nil, fmt.Errorf("invalid lb unhealthy threshold: %d", thresholdInt) + } + lBUnhealthyThreshold = c.Value + case LBHealthCheckUriConfigName: + if validateUri(c.Value) != nil { + return nil, fmt.Errorf("invalid lb health check uri: %s", c.Value) + } + lBHealthCheckUri = c.Value + case LBHealthCheckDomainConfigName: + if validateDomain(c.Value) != nil { + return nil, fmt.Errorf("invalid lb health check domain: %s", c.Value) + } + lBHealthCheckDomain = c.Value + case LBHealthCheckMethodConfigName: + method := strings.ToLower(c.Value) + if method != "get" && method != "head" { + return nil, fmt.Errorf("invalid lb health check method: %s", c.Value) + } + lBHealthCheckMethod = method } } return &nlbConfig{ - lbIds: lbIds, - protocols: protocols, - targetPorts: ports, - isFixed: isFixed, + lbIds: lbIds, + protocols: protocols, + targetPorts: ports, + isFixed: isFixed, + lBHealthCheckFlag: lBHealthCheckFlag, + lBHealthCheckType: lBHealthCheckType, + lBHealthCheckConnectPort: lBHealthCheckConnectPort, + lBHealthCheckConnectTimeout: lBHealthCheckConnectTimeout, + lBHealthCheckInterval: lBHealthCheckInterval, + lBHealthCheckUri: lBHealthCheckUri, + lBHealthCheckDomain: lBHealthCheckDomain, + lBHealthCheckMethod: lBHealthCheckMethod, + lBHealthyThreshold: lBHealthyThreshold, + lBUnhealthyThreshold: lBUnhealthyThreshold, + }, nil +} + +func validateDomain(domain string) error { + if len(domain) < 1 || len(domain) > 80 { + return fmt.Errorf("the domain length must be between 1 and 80 characters") + } + + // Regular expression matches lowercase letters, numbers, dashes and periods + domainRegex := regexp.MustCompile(`^[a-z0-9-.]+$`) + if !domainRegex.MatchString(domain) { + return fmt.Errorf("the domain must only contain lowercase letters, numbers, hyphens, and periods") + } + + // make sure the domain name does not start or end with a dash or period + if domain[0] == '-' || domain[0] == '.' || domain[len(domain)-1] == '-' || domain[len(domain)-1] == '.' { + return fmt.Errorf("the domain must not start or end with a hyphen or period") + } + + // make sure the domain name does not contain consecutive dots or dashes + if regexp.MustCompile(`(--|\.\.)`).MatchString(domain) { + return fmt.Errorf("the domain must not contain consecutive hyphens or periods") + } + + return nil +} + +func validateUri(uri string) error { + if len(uri) < 1 || len(uri) > 80 { + return fmt.Errorf("string length must be between 1 and 80 characters") + } + + regexPattern := `^/[0-9a-zA-Z.!$%&'*+/=?^_` + "`" + `{|}~-]*$` + matched, err := regexp.MatchString(regexPattern, uri) + + if err != nil { + return fmt.Errorf("regex error: %v", err) + } + + if !matched { + return fmt.Errorf("string does not match the required pattern") } + + return nil } diff --git a/cloudprovider/alibabacloud/nlb_test.go b/cloudprovider/alibabacloud/nlb_test.go index 0337133b..0776ca27 100644 --- a/cloudprovider/alibabacloud/nlb_test.go +++ b/cloudprovider/alibabacloud/nlb_test.go @@ -59,10 +59,7 @@ func TestNLBAllocateDeAllocate(t *testing.T) { func TestParseNlbConfig(t *testing.T) { tests := []struct { conf []gamekruiseiov1alpha1.NetworkConfParams - lbIds []string - ports []int - protocols []corev1.Protocol - isFixed bool + nlbConfig *nlbConfig }{ { conf: []gamekruiseiov1alpha1.NetworkConfParams{ @@ -74,11 +71,63 @@ func TestParseNlbConfig(t *testing.T) { Name: PortProtocolsConfigName, Value: "80", }, + { + Name: LBHealthCheckFlagConfigName, + Value: "On", + }, + { + Name: LBHealthCheckTypeConfigName, + Value: "HTTP", + }, + { + Name: LBHealthCheckConnectPortConfigName, + Value: "6000", + }, + { + Name: LBHealthCheckConnectTimeoutConfigName, + Value: "100", + }, + { + Name: LBHealthCheckIntervalConfigName, + Value: "30", + }, + { + Name: LBHealthCheckUriConfigName, + Value: "/another?valid", + }, + { + Name: LBHealthCheckDomainConfigName, + Value: "www.test.com", + }, + { + Name: LBHealthCheckMethodConfigName, + Value: "HEAD", + }, + { + Name: LBHealthyThresholdConfigName, + Value: "5", + }, + { + Name: LBUnhealthyThresholdConfigName, + Value: "5", + }, + }, + nlbConfig: &nlbConfig{ + lbIds: []string{"xxx-A"}, + targetPorts: []int{80}, + protocols: []corev1.Protocol{corev1.ProtocolTCP}, + isFixed: false, + lBHealthCheckFlag: "on", + lBHealthCheckType: "http", + lBHealthCheckConnectPort: "6000", + lBHealthCheckConnectTimeout: "100", + lBHealthCheckInterval: "30", + lBHealthCheckUri: "/another?valid", + lBHealthCheckDomain: "www.test.com", + lBHealthCheckMethod: "head", + lBHealthyThreshold: "5", + lBUnhealthyThreshold: "5", }, - lbIds: []string{"xxx-A"}, - ports: []int{80}, - protocols: []corev1.Protocol{corev1.ProtocolTCP}, - isFixed: false, }, { conf: []gamekruiseiov1alpha1.NetworkConfParams{ @@ -95,26 +144,32 @@ func TestParseNlbConfig(t *testing.T) { Value: "true", }, }, - lbIds: []string{"xxx-A", "xxx-B"}, - ports: []int{81, 82, 83}, - protocols: []corev1.Protocol{corev1.ProtocolUDP, corev1.ProtocolTCP, corev1.ProtocolTCP}, - isFixed: true, + nlbConfig: &nlbConfig{ + lbIds: []string{"xxx-A", "xxx-B"}, + targetPorts: []int{81, 82, 83}, + protocols: []corev1.Protocol{corev1.ProtocolUDP, corev1.ProtocolTCP, corev1.ProtocolTCP}, + isFixed: true, + lBHealthCheckFlag: "on", + lBHealthCheckType: "tcp", + lBHealthCheckConnectPort: "0", + lBHealthCheckConnectTimeout: "5", + lBHealthCheckInterval: "10", + lBUnhealthyThreshold: "2", + lBHealthyThreshold: "2", + lBHealthCheckUri: "", + lBHealthCheckDomain: "", + lBHealthCheckMethod: "", + }, }, } - for _, test := range tests { - sc := parseNlbConfig(test.conf) - if !reflect.DeepEqual(test.lbIds, sc.lbIds) { - t.Errorf("lbId expect: %v, actual: %v", test.lbIds, sc.lbIds) - } - if !util.IsSliceEqual(test.ports, sc.targetPorts) { - t.Errorf("ports expect: %v, actual: %v", test.ports, sc.targetPorts) - } - if !reflect.DeepEqual(test.protocols, sc.protocols) { - t.Errorf("protocols expect: %v, actual: %v", test.protocols, sc.protocols) + for i, test := range tests { + sc, err := parseNlbConfig(test.conf) + if err != nil { + t.Error(err) } - if test.isFixed != sc.isFixed { - t.Errorf("isFixed expect: %v, actual: %v", test.isFixed, sc.isFixed) + if !reflect.DeepEqual(test.nlbConfig, sc) { + t.Errorf("case %d: lbId expect: %v, actual: %v", i, test.nlbConfig, sc) } } } @@ -158,7 +213,17 @@ func TestNlbPlugin_consSvc(t *testing.T) { protocols: []corev1.Protocol{ corev1.ProtocolTCP, }, - isFixed: false, + isFixed: false, + lBHealthCheckFlag: "on", + lBHealthCheckType: "tcp", + lBHealthCheckConnectPort: "0", + lBHealthCheckConnectTimeout: "5", + lBHealthCheckInterval: "10", + lBUnhealthyThreshold: "2", + lBHealthyThreshold: "2", + lBHealthCheckUri: "", + lBHealthCheckDomain: "", + lBHealthCheckMethod: "", }, pod: &corev1.Pod{ TypeMeta: metav1.TypeMeta{ @@ -187,8 +252,25 @@ func TestNlbPlugin_consSvc(t *testing.T) { protocols: []corev1.Protocol{ corev1.ProtocolTCP, }, - isFixed: false, + isFixed: false, + lBHealthCheckFlag: "on", + lBHealthCheckType: "tcp", + lBHealthCheckConnectPort: "0", + lBHealthCheckConnectTimeout: "5", + lBHealthCheckInterval: "10", + lBUnhealthyThreshold: "2", + lBHealthyThreshold: "2", + lBHealthCheckUri: "", + lBHealthCheckDomain: "", + lBHealthCheckMethod: "", }), + LBHealthCheckFlagAnnotationKey: "on", + LBHealthCheckTypeAnnotationKey: "tcp", + LBHealthCheckConnectPortAnnotationKey: "0", + LBHealthCheckConnectTimeoutAnnotationKey: "5", + LBHealthCheckIntervalAnnotationKey: "10", + LBUnhealthyThresholdAnnotationKey: "2", + LBHealthyThresholdAnnotationKey: "2", }, OwnerReferences: []metav1.OwnerReference{ { diff --git a/docs/en/user_manuals/network.md b/docs/en/user_manuals/network.md index 5c9d9ee3..4cb223e8 100644 --- a/docs/en/user_manuals/network.md +++ b/docs/en/user_manuals/network.md @@ -489,6 +489,178 @@ AllowNotReadyContainers None +--- + +### AlibabaCloud-NLB +#### Plugin name + +`AlibabaCloud-NLB` + +#### Cloud Provider + +AlibabaCloud + +#### Plugin description + +- AlibabaCloud-NLB enables game servers to be accessed from the Internet by using Layer 4 Network Load Balancer (NLB) of Alibaba Cloud. AlibabaCloud-NLB uses different ports of the same NLB instance to forward Internet traffic to different game servers. The NLB instance only forwards traffic, but does not implement load balancing. + +- This network plugin supports network isolation. + +#### Network parameters + +NlbIds + +- Meaning: the NLB instance ID. You can fill in multiple ids. +- Value: in the format of nlbId-0,nlbId-1,... An example value can be "nlb-ji8l844c0qzii1x6mc,nlb-26jbknebrjlejt5abu" +- Configuration change supported or not: yes. You can add new nlbIds at the end. However, it is recommended not to change existing nlbId that is in use. + +PortProtocols + +- Meaning: the ports in the pod to be exposed and the protocols. You can specify multiple ports and protocols. +- Value: in the format of port1/protocol1,port2/protocol2,... The protocol names must be in uppercase letters. +- Configuration change supported or not: yes. + +Fixed + +- Meaning: whether the mapping relationship is fixed. If the mapping relationship is fixed, the mapping relationship remains unchanged even if the pod is deleted and recreated. +- Value: false or true. +- Configuration change supported or not: yes. + +AllowNotReadyContainers + +- Meaning: the container names that are allowed not ready when inplace updating, when traffic will not be cut. +- Value: {containerName_0},{containerName_1},... Example：sidecar +- Configuration change supported or not: It cannot be changed during the in-place updating process. + +LBHealthCheckFlag + +- Meaning: Whether to enable health check +- Format: "on" means on, "off" means off. Default is on +- Whether to support changes: Yes + +LBHealthCheckType + +- Meaning: Health Check Protocol +- Format: fill in "tcp" or "http", the default is tcp +- Whether to support changes: Yes + +LBHealthCheckConnectPort + +- Meaning: Server port for health check. +- Format: Value range [0, 65535]. Default value is "0" +- Whether to support changes: Yes + +LBHealthCheckConnectTimeout + +- Meaning: Maximum timeout for health check response. +- Format: Unit: seconds. The value range is [1, 300]. The default value is "5" +- Whether to support changes: Yes + +LBHealthyThreshold + +- Meaning: After the number of consecutive successful health checks, the health check status of the server will be determined from failure to success. +- Format: Value range [2, 10]. Default value is "2" +- Whether to support changes: Yes + +LBUnhealthyThreshold + +- Meaning: After the number of consecutive health check failures, the health check status of the server will be determined from success to failure. +- Format: Value range [2, 10]. The default value is "2" +- Whether to support changes: Yes + +LBHealthCheckInterval + +- Meaning: health check interval. +- Format: Unit: seconds. The value range is [1, 50]. The default value is "10" +- Whether to support changes: Yes + +LBHealthCheckUri + +- Meaning: The corresponding uri when the health check type is HTTP. +- Format: The length is 1~80 characters, only letters, numbers, and characters can be used. Must start with a forward slash (/). Such as "/test/index.html" +- Whether to support changes: Yes + +LBHealthCheckDomain + +- Meaning: The corresponding domain name when the health check type is HTTP. +- Format: The length of a specific domain name is limited to 1~80 characters. Only lowercase letters, numbers, dashes (-), and half-width periods (.) can be used. +- Whether to support changes: Yes + +LBHealthCheckMethod + +- Meaning: The corresponding method when the health check type is HTTP. +- Format: "GET" or "HEAD" +- Whether to support changes: Yes + +#### Plugin configuration +``` +[alibabacloud] +enable = true +[alibabacloud.nlb] +# Specify the range of available ports of the NLB instance. Ports in this range can be used to forward Internet traffic to pods. In this example, the range includes 500 ports. +max_port = 1500 +min_port = 1000 +``` + +#### Example + +``` +cat <