feat(*): add volcengine provider and clb plugin (openkruise#127)

Co-authored-by: 李志朋 <lizhipeng.629@bytedance.com>
CloudNativeGame · Jan 29, 2024 · 9c203d0 · 9c203d0
1 parent 2b6ce6f
commit 9c203d0
Show file tree

Hide file tree

Showing 9 changed files with 1,140 additions and 0 deletions.
diff --git a/cloudprovider/config.go b/cloudprovider/config.go
@@ -45,11 +45,13 @@ type ConfigFile struct {
 type CloudProviderConfig struct {
 	KubernetesOptions   CloudProviderOptions
 	AlibabaCloudOptions CloudProviderOptions
+	VolcengineOptions   CloudProviderOptions
 }
 
 type tomlConfigs struct {
 	Kubernetes   options.KubernetesOptions   `toml:"kubernetes"`
 	AlibabaCloud options.AlibabaCloudOptions `toml:"alibabacloud"`
+	Volcengine   options.VolcengineOptions   `toml:"volcengine"`
 }
 
 func (cf *ConfigFile) Parse() *CloudProviderConfig {
@@ -62,6 +64,7 @@ func (cf *ConfigFile) Parse() *CloudProviderConfig {
 	return &CloudProviderConfig{
 		KubernetesOptions:   config.Kubernetes,
 		AlibabaCloudOptions: config.AlibabaCloud,
+		VolcengineOptions:   config.Volcengine,
 	}
 }
 

diff --git a/cloudprovider/manager/provider_manager.go b/cloudprovider/manager/provider_manager.go
@@ -22,6 +22,7 @@ import (
 	"github.com/openkruise/kruise-game/cloudprovider"
 	"github.com/openkruise/kruise-game/cloudprovider/alibabacloud"
 	"github.com/openkruise/kruise-game/cloudprovider/kubernetes"
+	volcengine "github.com/openkruise/kruise-game/cloudprovider/volcengine"
 	corev1 "k8s.io/api/core/v1"
 	log "k8s.io/klog/v2"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -116,5 +117,15 @@ func NewProviderManager() (*ProviderManager, error) {
 		}
 	}
 
+	if configs.VolcengineOptions.Valid() && configs.VolcengineOptions.Enabled() {
+		// build and register volcengine cloud provider
+		vcp, err := volcengine.NewVolcengineProvider()
+		if err != nil {
+			log.Errorf("Failed to initialize volcengine provider.because of %s", err.Error())
+		} else {
+			pm.RegisterCloudProvider(vcp, configs.VolcengineOptions)
+		}
+	}
+
 	return pm, nil
 }
diff --git a/cloudprovider/options/volcenginecloud_options.go b/cloudprovider/options/volcenginecloud_options.go
@@ -0,0 +1,31 @@
+package options
+
+type VolcengineOptions struct {
+	Enable     bool       `toml:"enable"`
+	CLBOptions CLBOptions `toml:"clb"`
+}
+
+type CLBOptions struct {
+	MaxPort int32 `toml:"max_port"`
+	MinPort int32 `toml:"min_port"`
+}
+
+func (v VolcengineOptions) Valid() bool {
+	clbOptions := v.CLBOptions
+	if clbOptions.MaxPort-clbOptions.MinPort > 200 {
+		return false
+	}
+
+	if clbOptions.MaxPort > 65535 {
+		return false
+	}
+
+	if clbOptions.MinPort < 1 {
+		return false
+	}
+	return true
+}
+
+func (v VolcengineOptions) Enabled() bool {
+	return v.Enable
+}
diff --git a/cloudprovider/volcengine/README.md b/cloudprovider/volcengine/README.md
@@ -0,0 +1,108 @@
+English | [中文](./README.zh_CN.md)
+
+The Volcaengine Kubernetes Engine supports the CLB reuse mechanism in k8s. Different SVCs can use different ports of the same CLB. Therefore, the Volcengine-CLB network plugin will record the port allocation corresponding to each CLB. For the specified network type as Volcengine-CLB, the Volcengine-CLB network plugin will automatically allocate a port and create a service object. Wait for the svc ingress field. After the public network IP is successfully created, the GameServer network is in the Ready state and the process is completed.
+![image](https://github.com/lizhipeng629/kruise-game/assets/110802158/209de309-b9b7-4ba8-b2fb-da0d299e2edb)
+
+## Volcengine-CLB configuration
+### plugin configuration
+```toml
+[volcengine]
+enable = true
+[volcengine.clb]
+#Fill in the free port segment that clb can use to allocate external access ports to pods, The maximum port range is 200.
+max_port = 700
+min_port = 500
+```
+### Parameter
+#### ClbIds
+- Meaning：fill in the id of the clb. You can fill in more than one. You need to create the clb in [Volcano Engine].
+- Value：each clbId is divided by `,` . For example: `clb-9zeo7prq1m25ctpfrw1m7`,`clb-bp1qz7h50yd3w58h2f8je`,...
+- Configurable：Y
+
+#### PortProtocols
+- Meaning：the ports and protocols exposed by the pod, support filling in multiple ports/protocols
+- Value：`port1/protocol1`,`port2/protocol2`,... The protocol names must be in uppercase letters.
+- Configurable：Y
+
+#### Fixed
+- Meaning：whether the mapping relationship is fixed. If the mapping relationship is fixed, the mapping relationship remains unchanged even if the pod is deleted and recreated.
+- Value：false / true
+- Configurable：Y
+
+#### AllowNotReadyContainers
+- Meaning：the container names that are allowed not ready when inplace updating, when traffic will not be cut.
+- Value：{containerName_0},{containerName_1},... eg：sidecar
+- Configurable：It cannot be changed during the in-place updating process.
+
+
+### Example
+```yaml
+cat <<EOF | kubectl apply -f -
+apiVersion: game.kruise.io/v1alpha1
+kind: GameServerSet
+metadata:
+  name: gss-2048-clb
+  namespace: default
+spec:
+  replicas: 3
+  updateStrategy:
+    rollingUpdate:
+      podUpdatePolicy: InPlaceIfPossible
+  network:
+    networkType: Volcengine-CLB
+    networkConf:
+      - name: ClbIds
+        #Fill in Volcengine Cloud LoadBalancer Id here
+        value: clb-xxxxx
+      - name: PortProtocols
+        #Fill in the exposed ports and their corresponding protocols here. 
+        #If there are multiple ports, the format is as follows: {port1}/{protocol1},{port2}/{protocol2}...
+        #If the protocol is not filled in, the default is TCP
+        value: 80/TCP
+      - name: Fixed
+        #Fill in here whether a fixed IP is required [optional] ; Default is false
+        value: "false"
+  gameServerTemplate:
+    spec:
+      containers:
+        - image: cr-helm2-cn-beijing.cr.volces.com/kruise/2048:v1.0
+          name: app-2048
+          volumeMounts:
+            - name: shared-dir
+              mountPath: /var/www/html/js
+        - image: cr-helm2-cn-beijing.cr.volces.com/kruise/2048-sidecar:v1.0
+          name: sidecar
+          args:
+            - bash
+            - -c
+            - rsync -aP /app/js/* /app/scripts/ && while true; do echo 11;sleep 2; done
+          volumeMounts:
+            - name: shared-dir
+              mountPath: /app/scripts
+      volumes:
+        - name: shared-dir
+          emptyDir: {}
+EOF
+```
+
+Check the network status in GameServer:
+```
+networkStatus:
+    createTime: "2024-01-19T08:19:49Z"
+    currentNetworkState: Ready
+    desiredNetworkState: Ready
+    externalAddresses:
+    - ip: xxx.xxx.xx.xxx
+      ports:
+      - name: "80"
+        port: 6611
+        protocol: TCP
+    internalAddresses:
+    - ip: 172.16.200.60
+      ports:
+      - name: "80"
+        port: 80
+        protocol: TCP
+    lastTransitionTime: "2024-01-19T08:19:49Z"
+    networkType: Volcengine-CLB
+```
diff --git a/cloudprovider/volcengine/README.zh_CN.md b/cloudprovider/volcengine/README.zh_CN.md
@@ -0,0 +1,108 @@
+中文 | [English](./README.md)
+
+火山引擎容器服务支持在k8s中对CLB复用的机制，不同的svc可以使用同一个CLB的不同端口。由此，Volcengine-CLB network plugin将记录各CLB对应的端口分配情况，对于指定了网络类型为Volcengine-CLB，Volcengine-CLB网络插件将会自动分配一个端口并创建一个service对象，待svc ingress字段的公网IP创建成功后，GameServer的网络处于Ready状态，该过程执行完成。
+![image](https://github.com/lizhipeng629/kruise-game/assets/110802158/209de309-b9b7-4ba8-b2fb-da0d299e2edb)
+
+## Volcengine-CLB 相关配置
+### plugin配置
+```toml
+[volcengine]
+enable = true
+[volcengine.clb]
+#填写clb可使用的空闲端口段，用于为pod分配外部接入端口，范围最大为200
+max_port = 700
+min_port = 500
+```
+### 参数
+#### ClbIds
+- 含义：填写clb的id，可填写多个，需要现在【火山引擎】中创建好clb。
+- 填写格式：各个clbId用,分割。例如：clb-9zeo7prq1m25ctpfrw1m7,clb-bp1qz7h50yd3w58h2f8je,...
+- 是否支持变更：是
+
+#### PortProtocols
+- 含义：pod暴露的端口及协议，支持填写多个端口/协议
+- 填写格式：port1/protocol1,port2/protocol2,...（协议需大写）
+- 是否支持变更：是
+
+#### Fixed
+- 含义：是否固定访问IP/端口。若是，即使pod删除重建，网络内外映射关系不会改变
+- 填写格式：false / true
+- 是否支持变更：是
+
+#### AllowNotReadyContainers
+- 含义：在容器原地升级时允许不断流的对应容器名称，可填写多个
+- 填写格式：{containerName_0},{containerName_1},... 例如：sidecar
+- 是否支持变更：在原地升级过程中不可变更。
+
+
+### 使用示例
+```yaml
+cat <<EOF | kubectl apply -f -
+apiVersion: game.kruise.io/v1alpha1
+kind: GameServerSet
+metadata:
+  name: gss-2048-clb
+  namespace: default
+spec:
+  replicas: 3
+  updateStrategy:
+    rollingUpdate:
+      podUpdatePolicy: InPlaceIfPossible
+  network:
+    networkType: Volcengine-CLB
+    networkConf:
+      - name: ClbIds
+        #Fill in Volcengine Cloud LoadBalancer Id here
+        value: clb-xxxxx
+      - name: PortProtocols
+        #Fill in the exposed ports and their corresponding protocols here. 
+        #If there are multiple ports, the format is as follows: {port1}/{protocol1},{port2}/{protocol2}...
+        #If the protocol is not filled in, the default is TCP
+        value: 80/TCP
+      - name: Fixed
+        #Fill in here whether a fixed IP is required [optional] ; Default is false
+        value: "false"
+  gameServerTemplate:
+    spec:
+      containers:
+        - image: cr-helm2-cn-beijing.cr.volces.com/kruise/2048:v1.0
+          name: app-2048
+          volumeMounts:
+            - name: shared-dir
+              mountPath: /var/www/html/js
+        - image: cr-helm2-cn-beijing.cr.volces.com/kruise/2048-sidecar:v1.0
+          name: sidecar
+          args:
+            - bash
+            - -c
+            - rsync -aP /app/js/* /app/scripts/ && while true; do echo 11;sleep 2; done
+          volumeMounts:
+            - name: shared-dir
+              mountPath: /app/scripts
+      volumes:
+        - name: shared-dir
+          emptyDir: {}
+EOF
+```
+
+检查GameServer中的网络状态:
+```
+networkStatus:
+    createTime: "2024-01-19T08:19:49Z"
+    currentNetworkState: Ready
+    desiredNetworkState: Ready
+    externalAddresses:
+    - ip: xxx.xxx.xx.xxx
+      ports:
+      - name: "80"
+        port: 6611
+        protocol: TCP
+    internalAddresses:
+    - ip: 172.16.200.60
+      ports:
+      - name: "80"
+        port: 80
+        protocol: TCP
+    lastTransitionTime: "2024-01-19T08:19:49Z"
+    networkType: Volcengine-CLB
+```