perf: 分库分表改造前置准备 #2991

src/backend/build.gradle

@@ -152,6 +152,9 @@ ext {
     set('mockitoVersion', "4.0.0")
     set('embeddedRedisVersion', "0.6")
     set('openai4jVersion', "0.18.0")
+    set('shardingJdbcVersion', "5.4.1")
+    set('snakeyamlVersion', "1.33")
+    set('bkDevOpsLeafVersion', "1.0.2-RELEASE")
 }
 
 group "com.tencent.bk.job"
@@ -348,6 +351,9 @@ subprojects {
             dependency "org.mockito:mockito-inline:$mockitoVersion"
             dependency "com.github.kstyrc:embedded-redis:$embeddedRedisVersion"
             dependency "dev.ai4j:openai4j:$openai4jVersion"
+            dependency "org.apache.shardingsphere:shardingsphere-jdbc-core:$shardingJdbcVersion"
+            dependency "org.yaml:snakeyaml:$snakeyamlVersion"
+            dependency "com.tencent.devops.leaf:leaf-boot-starter:$bkDevOpsLeafVersion"
         }
     }
     dependencies {

src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/config/JobSpringCloudKubernetesInformerFactoryProcessor.java

@@ -37,7 +37,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeansException;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.config.ConfigurableListableBeanFactory;
 import org.springframework.beans.factory.support.AbstractBeanDefinition;
 import org.springframework.beans.factory.support.BeanDefinitionRegistry;

src/backend/commons/common-mysql-sharding/build.gradle

@@ -0,0 +1,42 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available.
+ *
+ * Copyright (C) 2021 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License.
+ *
+ * License for BK-JOB蓝鲸智云作业平台:
+ * --------------------------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+ * to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+ * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+ * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+dependencies {
+    api project(":commons:common")
+    implementation 'io.micrometer:micrometer-registry-prometheus'
+    implementation("org.springframework.boot:spring-boot-autoconfigure")
+    implementation 'org.springframework:spring-web'
+    implementation 'org.aspectj:aspectjrt'
+    implementation 'org.aspectj:aspectjweaver'
+    api 'org.apache.shardingsphere:shardingsphere-jdbc-core'
+    api 'org.yaml:snakeyaml'
+    api 'com.tencent.devops.leaf:leaf-boot-starter'
+    api 'org.jooq:jooq'
+    api 'org.springframework.boot:spring-boot-starter-jdbc'
+    compileOnly 'org.projectlombok:lombok'
+    annotationProcessor 'org.projectlombok:lombok'
+    testImplementation 'org.junit.jupiter:junit-jupiter'
+    testImplementation 'org.springframework.boot:spring-boot-starter-test'
+}
+apply from: "./leaf_gen_jooq.gradle"

src/backend/commons/common-mysql-sharding/leaf_gen_jooq.gradle

@@ -0,0 +1,78 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available.
+ *
+ * Copyright (C) 2021 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License.
+ *
+ * License for BK-JOB蓝鲸智云作业平台:
+ * --------------------------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+ * to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+ * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+ * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+apply plugin: 'nu.studer.jooq'
+
+dependencies {
+    api "org.jooq:jooq"
+    jooqRuntime "mysql:mysql-connector-java"
+}
+
+def databaseName = "job_leaf"
+def targetPackageName = "com.tencent.bk.job.common.sharding.mysql.jooq.model"
+
+jooq {
+    genenrate(sourceSets.main) {
+
+        jdbc {
+            driver = 'com.mysql.cj.jdbc.Driver'
+            def mysqlURL = System.getProperty("mysqlURL")
+            def mysqlUser = System.getProperty("mysqlUser")
+            def mysqlPasswd = System.getProperty("mysqlPasswd")
+
+            if (mysqlURL == null) {
+                // gradle.properties 中的 Jooq DB 配置
+                mysqlURL = project.findProperty("DB_HOST")
+                mysqlUser = project.findProperty("DB_USERNAME")
+                mysqlPasswd = project.findProperty("DB_PASSWORD")
+            }
+
+            println("mysqlURL=" + mysqlURL)
+            println("mysqlUser=" + mysqlUser)
+            url = "jdbc:mysql://${mysqlURL}/${databaseName}?useSSL=false&serverTimezone=UTC"
+            user = mysqlUser
+            password = mysqlPasswd
+        }
+
+        generator {
+            name = 'org.jooq.codegen.DefaultGenerator'
+            database {
+                name = 'org.jooq.meta.mysql.MySQLDatabase'
+                inputSchema = "${databaseName}"
+            }
+
+            generate {
+                relations = false
+                deprecated = false
+                fluentSetters = true
+                generatedAnnotation = false
+                javaTimeTypes = true
+            }
+
+            target {
+                packageName = "${targetPackageName}"
+                encoding = 'UTF-8'
+            }
+        }
+    }
+}

src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/JooqLeafIdAllocator.java

@@ -0,0 +1,119 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available.
+ *
+ * Copyright (C) 2021 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License.
+ *
+ * License for BK-JOB蓝鲸智云作业平台:
+ * --------------------------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+ * to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+ * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+ * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+package com.tencent.bk.job.common.sharding.mysql;
+
+import com.tencent.bk.job.common.sharding.mysql.jooq.model.tables.TLeafAlloc;
+import com.tencent.bk.job.common.sharding.mysql.jooq.model.tables.records.TLeafAllocRecord;
+import com.tencent.bk.job.common.util.date.DateUtils;
+import com.tencent.devops.leaf.segment.dao.IDAllocDao;
+import com.tencent.devops.leaf.segment.model.LeafAlloc;
+import org.jooq.DSLContext;
+import org.jooq.Record1;
+import org.jooq.Result;
+import org.jooq.impl.DSL;
+
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * 基于 JOOQ+美团 Leaf分布式 ID 组件，生成分布式 ID
+ */
+public class JooqLeafIdAllocator implements IDAllocDao {
+    private final DSLContext dslContext;
+
+    private static final TLeafAlloc T_LEAF_ALLOC = TLeafAlloc.T_LEAF_ALLOC;
+
+
+    public JooqLeafIdAllocator(DSLContext dslContext) {
+        this.dslContext = dslContext;
+    }
+
+    @Override
+    public List<LeafAlloc> getAllLeafAllocs() {
+        Result<TLeafAllocRecord> leafAllocRecords = dslContext.selectFrom(T_LEAF_ALLOC).fetch();
+        List<LeafAlloc> leafAllocs = new ArrayList<>();
+        leafAllocRecords.forEach(leafAllocRecord -> {
+            LeafAlloc leafAlloc = generateLeafAlloc(leafAllocRecord);
+            leafAllocs.add(leafAlloc);
+        });
+        return leafAllocs;
+    }
+
+    private LeafAlloc generateLeafAlloc(TLeafAllocRecord leafAllocRecord) {
+        LeafAlloc leafAlloc = new LeafAlloc();
+        leafAlloc.setKey(leafAllocRecord.getBizTag());
+        leafAlloc.setMaxId(leafAllocRecord.getMaxId());
+        leafAlloc.setStep(leafAllocRecord.getStep());
+        leafAlloc.setUpdateTime(DateUtils.defaultLocalDateTime(leafAllocRecord.getUpdateTime()));
+        return leafAlloc;
+    }
+
+    @Override
+    public LeafAlloc updateMaxIdAndGetLeafAlloc(String tag) {
+        TLeafAllocRecord leafAllocRecord;
+        leafAllocRecord = dslContext.transactionResult(t -> {
+            DSLContext context = DSL.using(t);
+            context.update(T_LEAF_ALLOC)
+                .set(T_LEAF_ALLOC.MAX_ID, T_LEAF_ALLOC.MAX_ID.add(T_LEAF_ALLOC.STEP))
+                .set(T_LEAF_ALLOC.UPDATE_TIME, LocalDateTime.now())
+                .where(T_LEAF_ALLOC.BIZ_TAG.eq(tag))
+                .execute();
+            return context.selectFrom(T_LEAF_ALLOC)
+                .where(T_LEAF_ALLOC.BIZ_TAG
+                    .eq(tag))
+                .fetchOne();
+        });
+        return generateLeafAlloc(leafAllocRecord);
+    }
+
+    @Override
+    public LeafAlloc updateMaxIdByCustomStepAndGetLeafAlloc(LeafAlloc leafAlloc) {
+        TLeafAllocRecord leafAllocRecord;
+        leafAllocRecord = dslContext.transactionResult(t -> {
+            DSLContext context = DSL.using(t);
+            context.update(T_LEAF_ALLOC)
+                .set(T_LEAF_ALLOC.MAX_ID, T_LEAF_ALLOC.MAX_ID.add(leafAlloc.getStep()))
+                .set(T_LEAF_ALLOC.UPDATE_TIME, LocalDateTime.now())
+                .where(T_LEAF_ALLOC.BIZ_TAG.eq(leafAlloc.getKey()))
+                .execute();
+            return context.selectFrom(T_LEAF_ALLOC)
+                .where(T_LEAF_ALLOC.BIZ_TAG.eq(leafAlloc.getKey()))
+                .fetchOne();
+        });
+        return generateLeafAlloc(leafAllocRecord);
+    }
+
+    @Override
+    public List<String> getAllTags() {
+        Result<Record1<String>> tagRecords = dslContext
+            .select(T_LEAF_ALLOC.BIZ_TAG)
+            .from(T_LEAF_ALLOC)
+            .fetch();
+        List<String> tags = new ArrayList<>();
+        tagRecords.forEach(record -> tags.add(record.value1()));
+        return tags;
+    }
+}

src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/SegmentIdKeys.java

@@ -0,0 +1,45 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available.
+ *
+ * Copyright (C) 2021 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License.
+ *
+ * License for BK-JOB蓝鲸智云作业平台:
+ * --------------------------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+ * to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+ * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+ * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+package com.tencent.bk.job.common.sharding.mysql;
+
+/**
+ * 分布式 ID KEY
+ */
+public class SegmentIdKeys {
+
+    public static final String KEY_JOB_EXECUTE_TASK_INSTANCE = "job_execute.task_instance";
+    public static final String KEY_JOB_EXECUTE_STEP_INSTANCE = "job_execute.step_instance";
+    public static final String KEY_JOB_EXECUTE_GSE_TASK = "job_execute.gse_task";
+    public static final String KEY_JOB_EXECUTE_OPERATION_LOG = "job_execute.operation_log";
+    public static final String KEY_JOB_EXECUTE_FILE_SOURCE_TASK_LOG = "job_execute.file_source_task_log";
+    public static final String KEY_JOB_EXECUTE_GSE_FILE_EXECUTE_OBJ_TASK = "job_execute.gse_file_execute_obj_task";
+    public static final String KEY_JOB_EXECUTE_GSE_SCRIPT_EXECUTE_OBJ_TASK =
+        "job_execute.gse_script_execute_obj_task";
+    public static final String KEY_JOB_EXECUTE_ROLLING_CONFIG = "job_execute.rolling_config";
+    public static final String KEY_JOB_EXECUTE_STEP_INSTANCE_ROLLING_TASK =
+        "job_execute.step_instance_rolling_task";
+    public static final String KEY_JOB_EXECUTE_STEP_INSTANCE_VARIABLE = "job_execute.step_instance_variable";
+    public static final String KEY_JOB_EXECUTE_TASK_INSTANCE_VARIABLE = "job_execute.task_instance_variable";
+}

src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/api/inner/ServiceArchiveResource.java → src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/ShardingConfigParseException.java

@@ -22,22 +22,16 @@
  * IN THE SOFTWARE.
  */
 
-package com.tencent.bk.job.backup.api.inner;
+package com.tencent.bk.job.common.sharding.mysql;
 
-import com.tencent.bk.job.backup.model.inner.ServiceArchiveDBRequest;
-import com.tencent.bk.job.common.annotation.InternalAPI;
-import com.tencent.bk.job.common.model.InternalResponse;
-import io.swagger.annotations.Api;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
+import com.tencent.bk.job.common.exception.InternalException;
 
-@Api(tags = {"DB_Archive"})
-@RequestMapping("/service/archiveDB")
-@RestController
-@InternalAPI
-public interface ServiceArchiveResource {
-    @PostMapping
-    InternalResponse<?> archive(@RequestBody ServiceArchiveDBRequest request);
+/**
+ * 分片配置解析异常
+ */
+public class ShardingConfigParseException extends InternalException {
+
+    public ShardingConfigParseException(String message) {
+        super(message);
+    }
 }