Merge pull request #3335 from wangyu096/issue_3332

fix: 修复分库分表引入的问题 #3332
TencentBlueKing · Dec 18, 2024 · 27ce025 · 27ce025
2 parents faf0b8b + 71430be
commit 27ce025
Show file tree

Hide file tree

Showing 33 changed files with 596 additions and 575 deletions.
diff --git a/...com/tencent/bk/job/common/web/filter/RepeatableReadWriteServletRequestResponseFilter.java b/...com/tencent/bk/job/common/web/filter/RepeatableReadWriteServletRequestResponseFilter.java
@@ -27,6 +27,7 @@
 import com.tencent.bk.job.common.web.model.RepeatableReadHttpServletResponse;
 import com.tencent.bk.job.common.web.model.RepeatableReadWriteHttpServletRequest;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
 
 import javax.servlet.Filter;
 import javax.servlet.FilterChain;
@@ -48,11 +49,24 @@ public void init(FilterConfig filterConfig) {
     @Override
     public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException,
         ServletException {
+        if (!isJsonRequest(request)) {
+            chain.doFilter(request, response);
+            return;
+        }
         ServletRequest requestWrapper = new RepeatableReadWriteHttpServletRequest((HttpServletRequest) request);
         ServletResponse responseWrapper = new RepeatableReadHttpServletResponse((HttpServletResponse) response);
         chain.doFilter(requestWrapper, responseWrapper);
     }
 
+    private boolean isJsonRequest(ServletRequest request) {
+        String contentType = request.getContentType();
+        if (StringUtils.isBlank(contentType)) {
+            return false;
+        }
+        contentType = contentType.trim().toLowerCase();
+        return contentType.startsWith("application/json");
+    }
+
     @Override
     public void destroy() {
 

diff --git a/.../resources/init_archive_progress_data.sql → ...nt/bk/job/backup/BootIntegrationTest.java b/.../resources/init_archive_progress_data.sql → ...nt/bk/job/backup/BootIntegrationTest.java
@@ -22,10 +22,28 @@
  * IN THE SOFTWARE.
  */
 
-truncate table db_archive_progress;
+package com.tencent.bk.job.backup;
 
-insert into job_backup.db_archive_progress (table_name,progress,last_modify_time) values ('task_instance','{\"lastArchivedId\":1000}',1621166442000);
-insert into job_backup.db_archive_progress (table_name,progress,last_modify_time) values ('step_instance','{\"lastArchivedId\":2000}',1621166442000);
-insert into job_backup.db_archive_progress (table_name,progress,last_modify_time) values ('gse_task_log','{\"lastArchivedId\":2000}',1621166442000);
-insert into job_backup.db_archive_progress (table_name,progress,last_modify_time) values ('gse_task_ip_log','{\"lastArchivedId\":2000}',1621166442000);
-insert into job_backup.db_archive_progress (table_name,progress,last_modify_time) values ('task_instance_variable','{\"lastArchivedId\":1000}',1621166442000);
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.TestPropertySource;
+import org.springframework.test.context.jdbc.SqlConfig;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+@ExtendWith(SpringExtension.class)
+@SpringBootTest
+@ActiveProfiles("test")
+@TestPropertySource(locations = "classpath:test.properties")
+@SqlConfig(encoding = "utf-8")
+public class BootIntegrationTest {
+
+
+    @Test
+    @DisplayName("测试 job-backup 启动")
+    public void bootTest() {
+        // do nothing
+    }
+}
diff --git a/src/backend/job-backup/boot-job-backup/src/test/resources/application-test.yml b/src/backend/job-backup/boot-job-backup/src/test/resources/application-test.yml
@@ -12,7 +12,7 @@ spring:
         enabled: false
   datasource:
     job-backup:
-      driver-class-name: com.mysql.cj.jdbc.Driver
+      driver-class-name: org.h2.Driver
       type: com.zaxxer.hikari.HikariDataSource
       jdbc-url: ${job.backup.db.url}
       username: ${job.backup.db.username}
@@ -23,7 +23,7 @@ spring:
       poolName: "job-backup"
       validationTimeout: 5000
     job-execute:
-      driver-class-name: com.mysql.cj.jdbc.Driver
+      driver-class-name: org.h2.Driver
       type: com.zaxxer.hikari.HikariDataSource
       jdbc-url: ${job.execute.db.dr.url:}
       username: ${job.execute.db.dr.username:}
@@ -34,7 +34,7 @@ spring:
       poolName: "job-execute"
       validationTimeout: 5000
     job-execute-archive:
-      driver-class-name: com.mysql.cj.jdbc.Driver
+      driver-class-name: org.h2.Driver
       type: com.zaxxer.hikari.HikariDataSource
       jdbc-url: ${job.execute.db.archive.url:}
       username: ${job.execute.db.archive.username:}
@@ -49,6 +49,12 @@ job:
     service:
       public-key-base64: LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUFyTXN3emh3QkVUTktQOThTcElna0VjZThPNVlQWm5aUDVHVW1ZUVhhSHdHWmg5UHJwNDFseS9xbWk5enEwTlRCODRyaXhSZHZyU1c0cCtCUjk4TVRzRmpYaUlRYVBxRTdRUitpdDdoT1VQNVJMQW9KcWlvL3NlZlRIdlZxVW1GcitYU3NST05rek5selRVaVcvMFRNY0cwNVdIS3hORno5YTB2aXo1T01PeE5lUzhqS1ZiRStIMHlQS0JhR1JHMEdZNTZTS1BLbXh1RHk5a2tyVUtGSXZycldSZkh2bzVsK0xsT3IyQmVGT3FHcEhmeW1FaXlkbFRxWHNlbGlsYTRxMUoxbnBZRDYwY3JIY1ZRV3d2Ri9CZUkrZytxVkF5YzNaWmhYeEhoNWNMcmZsK3pqMktsUEFEZVIvZktubHFqKzFBQm4yaFRoVWJ2WjRCdXhTdWxOa1FJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0t
       private-key-base64: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV1d0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktVd2dnU2hBZ0VBQW9JQkFRQ3N5ekRPSEFFUk0wby8zeEtraUNRUng3dzdsZzltZGsva1pTWmhCZG9mQVptSDArdW5qV1hMK3FhTDNPclExTUh6aXVMRkYyK3RKYmluNEZIM3d4T3dXTmVJaEJvK29UdEJINkszdUU1US9sRXNDZ21xS2oreDU5TWU5V3BTWVd2NWRLeEU0MlRNMlhOTlNKYi9STXh3YlRsWWNyRTBYUDFyUytMUGs0dzdFMTVMeU1wVnNUNGZUSThvRm9aRWJRWmpucElvOHFiRzRQTDJTU3RRb1VpK3V0WkY4ZStqbVg0dVU2dllGNFU2b2FrZC9LWVNMSjJWT3BleDZXS1ZyaXJVbldlbGdQclJ5c2R4VkJiQzhYOEY0ajZENnBVREp6ZGxtRmZFZUhsd3V0K1g3T1BZcVU4QU41SDk4cWVXcVA3VUFHZmFGT0ZSdTluZ0c3Rks2VTJSQWdNQkFBRUNnZ0VBZlkvZGREaUhVZjlxV2dWNG41VEtXUjFDSEM2TUhGR205bm5HTE1TNjNzTWoxTDdiUkZNTWdQQXY2L3hwcWJ5Zy81K1I3OVI2ODJBdzBkVEEyNENuSXdNOEE5aXkxWVlGNytuVUxSckIxanNuc3cvTVBCb2RWT0wzMkV4UHFYTmZxZEx0aDlXUm53WDRzbFVvWEhDYStxdHorMkRFZ2g5OGkyYUdkMzVwYlZjT2p0czlWc3FtZE5uQTdLNFVQWkd4aVFheG1rLzBOVEVNY2RqMUl4bWNlZjNqL1RUVkFiR05IdWVtYTJKeUdZdk52N09BZUN3VGJSdW5TNW1PSmptRHlFSmp4dndaVWZwanZoM0w0SkZzMGRJUm5wZ2VtTkY3NytqUmFTTy9WUVlqc1FLelB0SnEwcHFneHUwd0RoQUVBeXpjYWI2NW8wSm1hZzNJUjZNQzdRS0JnUUR1MThHNFRVaTlSVk8yZS9YdWtDSUNiOVZISHlMeVFIUDh4NDhBVFZmVENtSVBEMUduTWp5TlFmSnhsUlNma3UyUjJIM1IyRC81YUNpNUYwTi9USG51OWVqYlMvS1NoYjFxMFl5TWF1TDkxQlZjMkEyeDZlRzRPNFFpT2daT0gzaE0raUVSREJxOExNQ1dILzVtYVJDeDhVRmRQTnpHMUtXVVNwTEJXM3l2ZXdLQmdRQzVOTS9yN09Wc3F6OGlDUjBXa2JramN2eWU4MkJNcGFsZVkyRHNWQVMwRHNMNlMvZlAzY2VUdEJQVmdJL244Sk16b1ZNZE5OK3htbXM4Wi9rMGM5OGlLTzRUeE0zN2daYkIwUE8rWGRyeUZkRFhTRDZXTm1heWpYUG5MUndBZmRwbFlWdFNqVGxWMWJzWWIySnhXNXY2RUVSaVlLYnNnRGNpWTA5ZnYva0RZd0ovUnVmR212QldwOW50QUQvTXd3WUhFcklnbnZ5ZlkwdS9JMHdiSi92T0Z0aitRM3BJdzFvbW44ajBNTVFSVzA1RE9Ra01VVC9odmlrdDdjVDZSTkJ2WW9HZW4zdnNoNU1zcUltTk1DS0xRTSsxaDlxY05qVTR6WGpkd1V0NGs3akQvaFpEdXN2ZEpBQWxMR3hUR2hRVzRMeStxdTltbTZDRmg0Q2NFTXNZT1FLQmdCa0pEWTRyd0x1V3pucndWbkZGazR3NkwvSGdua1RlSSszeGF4bW5KZGVkSXpnb2FzQTdqNDNreXNXU2ttRmRBTmo2MXJ4YytGd09ycTVFczhnbVhhUkZlaUZMK2pna1JWZS93aU1hbHMwZ3E5RmVINis0Zk50b0NhSFhWREN3MG9QZnpETjFpYW1Uc2EzVTZLN0FIbTZJSW1aV0FGcDUzbm9VaUpjOTRSQlhBb0dCQUxMcmQrbnljNzVEbjZ0ZWdVT0JQRy9HR3FYWkZGdmd6emJRRnR0SmR0NTlMZ0NBanUxMjJMRExsRjZreTB6Q0JsK0h6Q0kyUXJXVEp4cXUxSTZvbndaZDlOMllKVlNWQlFOV285NTJUamRkOHcvSkExYlNqK3JpTU1WQ0txT1BNRHNiOENEaGlEbVM3RlBEM0trTVlmcUxJOVhNRTVvNVUvNFJzcEZUZWozRQotLS0tLUVORCBQUklWQVRFIEtFWS0tLS0t
+  backup:
+    archive:
+      execute:
+        enabled: false
+        mariadb:
+          dataSourceMode: standalone
 bk-api-gateway:
   gse:
     url: gse.apigw.com

diff --git a/src/backend/job-backup/boot-job-backup/src/test/resources/init_schema.sql b/src/backend/job-backup/boot-job-backup/src/test/resources/init_schema.sql
@@ -23,52 +23,31 @@
  */
 
 SET NAMES utf8mb4;
-CREATE SCHEMA IF NOT EXISTS job_execute;
 CREATE SCHEMA IF NOT EXISTS job_backup;
+USE job_backup;
 
-CREATE TABLE IF NOT EXISTS `job_execute.task_instance`
-(
-    `id`               bigint(20)   NOT NULL AUTO_INCREMENT,
-    `app_id`           bigint(20)   NOT NULL,
-    `task_id`          bigint(20)   NOT NULL,
-    `task_template_id` bigint(20)   NOT NULL,
-    `name`             varchar(512) NOT NULL,
-    `type`             tinyint(4)   NOT NULL,
-    `operator`         varchar(128) NOT NULL,
-    `status`           tinyint(4)   NOT NULL DEFAULT '0',
-    `current_step_id`  bigint(20)   NOT NULL DEFAULT '0',
-    `startup_mode`     tinyint(4)   NOT NULL,
-    `total_time`       bigint(20)            DEFAULT NULL,
-    `callback_url`     varchar(1024)         DEFAULT NULL,
-    `is_debug_task`    tinyint(4)   NOT NULL DEFAULT '0',
-    `cron_task_id`     bigint(20)   NOT NULL DEFAULT '0',
-    `create_time`      bigint(20)            DEFAULT NULL,
-    `start_time`       bigint(20)            DEFAULT NULL,
-    `end_time`         bigint(20)            DEFAULT NULL,
-    `app_code`         varchar(128)          DEFAULT NULL,
-    `row_create_time`  DATETIME     NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    `row_update_time`  DATETIME     NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-    PRIMARY KEY (`id`),
-    KEY (`app_id`),
-    KEY (`operator`),
-    KEY (`task_id`),
-    KEY (`status`),
-    KEY (`create_time`)
-) ENGINE = InnoDB
-  AUTO_INCREMENT = 1000000
-  DEFAULT CHARSET = utf8mb4;
-
-  USE `job_backup`;
-
-SET NAMES utf8mb4;
-
-CREATE TABLE `job_backup.db_archive_progress` (
+CREATE TABLE IF NOT EXISTS `archive_task` (
   `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
   `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-  `table_name` varchar(256) NOT NULL,
-  `progress` text,
-  `last_modify_time` BIGINT(20) NOT NULL DEFAULT '0',
-  PRIMARY KEY (`table_name`)
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `task_type` tinyint(2) DEFAULT NULL,
+  `data_node` varchar(128) NOT NULL,
+  `db_node` varchar(64) NOT NULL,
+  `day` int(8) DEFAULT NULL,
+  `hour` tinyint(2) DEFAULT NULL,
+  `from_timestamp` bigint(20) NOT NULL,
+  `to_timestamp` bigint(20) NOT NULL,
+  `process` varchar(256) DEFAULT NULL,
+  `status` tinyint(2) NOT NULL DEFAULT '0',
+  `create_time` bigint(20) NOT NULL,
+  `last_update_time` bigint(20) NOT NULL,
+  `task_start_time` bigint(20) DEFAULT NULL,
+  `task_end_time` bigint(20) DEFAULT NULL,
+  `task_cost` bigint(20) DEFAULT NULL,
+  `detail` text,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY (`task_type`,`data_node`,`day`,`hour`),
+  KEY (`task_type`,`status`,`db_node`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
 
diff --git a/src/backend/job-backup/boot-job-backup/src/test/resources/logback-spring.xml b/src/backend/job-backup/boot-job-backup/src/test/resources/logback-spring.xml
@@ -2,7 +2,7 @@
 <configuration scan="true" scanPeriod="30 seconds">
     <property name="LOG_PATTERN"
               value="[%date{yyyy-MM-dd HH:mm:ss.SSS}][%X{traceId}|%X{spanId}][%thread] %-5level %logger{36}:%method:%line - %msg%n"/>
-    <property name="BK_LOG_DIR" value="${job.log.dir:-/data/bkee/logs/job}"/>
+    <property name="BK_LOG_DIR" value="${job.log.dir:-logs/job}"/>
     <property name="BK_LOG_DIR_BACKUP" value="${BK_LOG_DIR}/job-backup"/>
     <property name="BACKUP_LOG_FILE" value="${BK_LOG_DIR_BACKUP}/backup.log"/>
     <property name="ERROR_LOG_FILE" value="${BK_LOG_DIR_BACKUP}/error.log"/>

diff --git a/src/backend/job-backup/boot-job-backup/src/test/resources/test.properties b/src/backend/job-backup/boot-job-backup/src/test/resources/test.properties
@@ -1,28 +1,4 @@
-#
-# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available.
-#
-# Copyright (C) 2021 THL A29 Limited, a Tencent company.  All rights reserved.
-#
-# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License.
-#
-# License for BK-JOB蓝鲸智云作业平台:
-# --------------------------------------------------------------------
-# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
-# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
-# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
-# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
-# the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
-# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
-# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-# IN THE SOFTWARE.
-#
-
-job.backup.db.url=jdbc:h2:mem:job-backup;MODE=MYSQL;INIT=RUNSCRIPT FROM 'classpath:init_schema.sql'
+job.backup.db.url=jdbc:h2:mem:job-execute;MODE=MYSQL;INIT=RUNSCRIPT FROM 'classpath:init_schema.sql'
 job.backup.db.username=job
 job.backup.db.password=job_db_password
 job.execute.db.dr.url=jdbc:h2:mem:job-execute;MODE=MYSQL;INIT=RUNSCRIPT FROM 'classpath:init_schema.sql'

diff --git a/...-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTablePropsStorage.java b/...-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTablePropsStorage.java
@@ -26,22 +26,17 @@
 
 import com.tencent.bk.job.backup.archive.model.ArchiveTableProps;
 import com.tencent.bk.job.backup.config.ArchiveProperties;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
 
 import java.util.HashMap;
 import java.util.Map;
 
 /**
  * 归档配置存储
  */
-@Component
 public class ArchiveTablePropsStorage {
     private final ArchiveProperties archiveProperties;
     private final Map<String, ArchiveTableProps> tablePropsMap = new HashMap<>();
 
-
-    @Autowired
     public ArchiveTablePropsStorage(ArchiveProperties archiveProperties) {
         this.archiveProperties = archiveProperties;
         if (archiveProperties.getTableConfigs() != null && !archiveProperties.getTableConfigs().isEmpty()) {