From b85b871b0c51fc79b76313233d5591c6f365e305 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Mon, 20 May 2024 20:34:32 +0800 Subject: [PATCH 01/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/web/WebExecuteTaskResource.java | 5 +- .../web/WebTaskExecutionResultResource.java | 68 --- .../StepInstanceDAOImplIntegrationTest.java | 113 ++-- .../src/test/resources/init_schema.sql | 7 + .../resources/init_step_instance_data.sql | 35 +- .../EsbGetStepInstanceStatusResourceImpl.java | 3 +- .../EsbOperateStepInstanceResourceImpl.java | 1 + ...atchGetJobInstanceIpLogV3ResourceImpl.java | 6 +- ...bInstanceExecuteObjectLogResourceImpl.java | 6 +- .../EsbGetJobInstanceIpLogV3ResourceImpl.java | 6 +- ...sbGetStepInstanceDetailV3ResourceImpl.java | 16 - ...sbGetStepInstanceStatusV3ResourceImpl.java | 18 +- .../EsbOperateStepInstanceV3ResourceImpl.java | 1 + .../web/impl/WebExecuteTaskResourceImpl.java | 2 + .../web/impl/WebSearchToolsResourceImpl.java | 11 +- .../WebTaskExecutionResultResourceImpl.java | 81 +-- .../api/web/impl/WebTaskLogResourceImpl.java | 32 +- .../bk/job/execute/dao/StepInstanceDAO.java | 85 +-- .../execute/dao/impl/StepInstanceDAOImpl.java | 552 +++++++----------- .../engine/evict/TaskEvictPolicyExecutor.java | 1 + .../executor/AbstractGseTaskStartCommand.java | 26 +- .../executor/FileGseTaskStartCommand.java | 7 +- .../engine/executor/GseTaskManager.java | 19 +- .../executor/ScriptGseTaskStartCommand.java | 1 + .../listener/ConfirmStepEventHandler.java | 12 +- .../engine/listener/GseStepEventHandler.java | 99 ++-- .../execute/engine/listener/JobListener.java | 55 +- .../listener/ResultHandleResumeListener.java | 3 +- .../execute/engine/listener/StepListener.java | 3 +- .../engine/listener/event/EventSource.java | 9 +- .../engine/listener/event/GseTaskEvent.java | 24 +- .../event/ResultHandleTaskResumeEvent.java | 8 +- .../engine/listener/event/StepEvent.java | 57 +- .../DefaultFilePrepareTaskResultHandler.java | 13 +- .../prepare/FilePrepareControlTask.java | 6 +- .../engine/prepare/FilePrepareService.java | 3 +- .../prepare/FilePrepareServiceImpl.java | 6 +- .../local/LocalFilePrepareService.java | 5 +- .../third/ThirdFilePrepareService.java | 8 +- .../prepare/third/ThirdFilePrepareTask.java | 8 +- .../result/AbstractResultHandleTask.java | 21 +- .../execute/model/ConfirmStepInstanceDTO.java | 1 + .../execute/model/FileStepInstanceDTO.java | 1 + .../bk/job/execute/model/GseTaskDTO.java | 4 + .../job/execute/model/GseTaskSimpleDTO.java | 4 + .../execute/model/ScriptStepInstanceDTO.java | 5 +- .../job/execute/model/StepOperationDTO.java | 4 + .../job/execute/service/LogExportService.java | 1 + .../bk/job/execute/service/LogService.java | 10 +- .../execute/service/StepInstanceService.java | 98 ++-- .../service/StepInstanceValidateService.java | 9 - .../execute/service/TaskExecuteService.java | 13 +- .../service/impl/LogExportServiceImpl.java | 10 +- .../execute/service/impl/LogServiceImpl.java | 13 +- .../service/impl/NotifyServiceImpl.java | 6 +- .../service/impl/StepInstanceServiceImpl.java | 145 ++--- .../impl/StepInstanceValidateServiceImpl.java | 33 -- .../service/impl/TaskExecuteServiceImpl.java | 45 +- .../service/impl/TaskResultServiceImpl.java | 3 +- .../statistics/StatisticsServiceImpl.java | 5 +- ...ob_execute_20240520-1000_V3.10.0_mysql.sql | 143 +++++ 61 files changed, 979 insertions(+), 1016 deletions(-) delete mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceValidateService.java delete mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceValidateServiceImpl.java create mode 100644 support-files/sql/job-execute/0026_job_execute_20240520-1000_V3.10.0_mysql.sql diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebExecuteTaskResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebExecuteTaskResource.java index 41bbb25ebe..c702bb5610 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebExecuteTaskResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebExecuteTaskResource.java @@ -140,7 +140,7 @@ Response fastPushFile( ); @ApiOperation(value = "执行作业步骤操作", produces = "application/json") - @PostMapping(value = {"/do-step-operation/stepInstanceId/{stepInstanceId}"}) + @PostMapping(value = {"/do-step-operation/taskInstanceId/{taskInstanceId}/stepInstanceId/{stepInstanceId}"}) Response doStepOperation( @ApiParam("用户名，网关自动传入") @RequestHeader("username") @@ -154,6 +154,9 @@ Response doStepOperation( @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") String scopeId, + @ApiParam(value = "作业实例ID", required = true, example = "1") + @PathVariable("taskInstanceId") + Long taskInstanceId, @ApiParam(value = "步骤实例ID", required = true, example = "1") @PathVariable("stepInstanceId") Long stepInstanceId, diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java index 95dc7aa0b7..a8471f8988 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java @@ -37,8 +37,6 @@ import com.tencent.bk.job.execute.model.web.vo.ExecuteObjectVO; import com.tencent.bk.job.execute.model.web.vo.ExecuteVariableVO; import com.tencent.bk.job.execute.model.web.vo.FileDistributionDetailV2VO; -import com.tencent.bk.job.execute.model.web.vo.FileDistributionDetailVO; -import com.tencent.bk.job.execute.model.web.vo.IpFileLogContentVO; import com.tencent.bk.job.execute.model.web.vo.IpScriptLogContentVO; import com.tencent.bk.job.execute.model.web.vo.StepExecutionDetailV2VO; import com.tencent.bk.job.execute.model.web.vo.StepExecutionDetailVO; @@ -318,40 +316,6 @@ Response getScriptLogContentByExecuteObject( Integer batch ); - @ApiOperation(value = "获取文件分发步骤主机对应的日志(废弃）", produces = "application/json") - @GetMapping(value = {"/step-execution-result/log-content/file/{stepInstanceId}/{executeCount}/host/{hostId}"}) - @Deprecated - @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, - explain = "发布完成后可以删除") - Response getFileLogContentByHost( - @ApiParam("用户名，网关自动传入") - @RequestHeader("username") - String username, - @ApiIgnore - @RequestAttribute(value = "appResourceScope") - AppResourceScope appResourceScope, - @ApiParam(value = "资源范围类型", required = true) - @PathVariable(value = "scopeType") - String scopeType, - @ApiParam(value = "资源范围ID", required = true) - @PathVariable(value = "scopeId") - String scopeId, - @ApiParam(value = "步骤实例ID", name = "stepInstanceId", required = true) - @PathVariable("stepInstanceId") - Long stepInstanceId, - @ApiParam(value = "执行次数，首次传0", name = "executeCount", required = true) - @PathVariable("executeCount") - Integer executeCount, - @ApiParam(value = "主机ID", name = "hostId") - @PathVariable(value = "hostId", required = false) Long hostId, - @ApiParam(value = "文件任务上传下载标识,upload-上传,download-下载", name = "mode", required = true) - @RequestParam(value = "mode") - String mode, - @ApiParam(value = "滚动批次，非滚动步骤不需要传入", name = "batch") - @RequestParam(value = "batch", required = false) - Integer batch - ); - @ApiOperation(value = "获取文件分发步骤执行对象对应的日志", produces = "application/json") @GetMapping(value = {"/taskInstance/{taskInstanceId}/stepInstance/{stepInstanceId}/executeObject" + "/{executeObjectType}/{executeObjectResourceId}/fileLog"}) @@ -391,38 +355,6 @@ Response getFileLogContentByExecuteObject( Integer mode ); - @ApiOperation(value = "获取文件分发步骤文件任务ID对应的执行日志（废弃）", produces = "application/json") - @PostMapping(value = {"/step-execution-result/log-content/file/{stepInstanceId}/{executeCount}/query-by-ids"}) - @Deprecated - @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, - explain = "发布完成后可以删除") - Response> getFileLogContentByFileTaskIds( - @ApiParam("用户名，网关自动传入") - @RequestHeader("username") - String username, - @ApiIgnore - @RequestAttribute(value = "appResourceScope") - AppResourceScope appResourceScope, - @ApiParam(value = "资源范围类型", required = true) - @PathVariable(value = "scopeType") - String scopeType, - @ApiParam(value = "资源范围ID", required = true) - @PathVariable(value = "scopeId") - String scopeId, - @ApiParam(value = "步骤实例ID", name = "stepInstanceId", required = true) - @PathVariable("stepInstanceId") - Long stepInstanceId, - @ApiParam(value = "执行次数，首次传0", name = "executeCount", required = true) - @PathVariable("executeCount") - Integer executeCount, - @ApiParam(value = "滚动批次，非滚动步骤不需要传入", name = "batch") - @RequestParam(value = "batch", required = false) - Integer batch, - @ApiParam(value = "文件任务ID列表", name = "taskIds", required = true) - @RequestBody - List taskIds - ); - @ApiOperation(value = "获取文件分发步骤文件任务ID对应的执行日志", produces = "application/json") @PostMapping(value = {"/taskInstance/{taskInstanceId}/stepInstance/{stepInstanceId}/fileLog/queryByIds"}) Response> getFileLogContentByFileTaskIds( diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImplIntegrationTest.java index ace68f6e71..bb0e9e35b2 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImplIntegrationTest.java @@ -67,8 +67,9 @@ public class StepInstanceDAOImplIntegrationTest { @Test public void testGetStepInstanceBase() { + long taskInstanceId = 1L; long stepInstanceId = 1L; - StepInstanceBaseDTO stepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + StepInstanceBaseDTO stepInstance = stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); assertThat(stepInstance.getId()).isEqualTo(stepInstanceId); assertThat(stepInstance.getAppId()).isEqualTo(2L); @@ -154,12 +155,14 @@ public void testListStepInstanceBaseByTaskInstanceId() { @Test public void testResetStepStatus() { + long taskInstanceId = 1L; long stepInstanceId = 1L; - stepInstanceDAO.resetStepStatus(stepInstanceId); + stepInstanceDAO.resetStepStatus(taskInstanceId, stepInstanceId); - StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); assertThat(returnStepInstance.getId()).isEqualTo(stepInstanceId); + assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(taskInstanceId); assertThat(returnStepInstance.getStartTime()).isNull(); assertThat(returnStepInstance.getEndTime()).isNull(); assertThat(returnStepInstance.getTotalTime()).isNull(); @@ -167,90 +170,107 @@ public void testResetStepStatus() { @Test public void testAddStepRetryCount() { + long taskInstanceId = 1L; long stepInstanceId = 1L; - stepInstanceDAO.addStepExecuteCount(stepInstanceId); + stepInstanceDAO.addStepExecuteCount(taskInstanceId, stepInstanceId); - StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); + assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(taskInstanceId); assertThat(returnStepInstance.getId()).isEqualTo(stepInstanceId); assertThat(returnStepInstance.getExecuteCount()).isEqualTo(1); } @Test public void testUpdateStepStatus() { + long taskInstanceId = 1L; long stepInstanceId = 1L; - stepInstanceDAO.updateStepStatus(stepInstanceId, RunStatusEnum.RUNNING.getValue()); + stepInstanceDAO.updateStepStatus(taskInstanceId, stepInstanceId, RunStatusEnum.RUNNING.getValue()); StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + assertThat(returnStepInstance.getId()).isEqualTo(stepInstanceId); assertThat(returnStepInstance.getId()).isEqualTo(stepInstanceId); assertThat(returnStepInstance.getStatus()).isEqualTo(RunStatusEnum.RUNNING); } @Test public void testUpdateStepStartTime() { + long taskInstanceId = 3L; long stepInstanceId = 4L; long startTime = 1573041600000L; - stepInstanceDAO.updateStepStartTimeIfNull(stepInstanceId, startTime); - StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + stepInstanceDAO.updateStepStartTimeIfNull(taskInstanceId, stepInstanceId, startTime); + StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); + assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(taskInstanceId); assertThat(returnStepInstance.getId()).isEqualTo(stepInstanceId); assertThat(returnStepInstance.getStartTime()).isEqualTo(startTime); stepInstanceId = 1L; + taskInstanceId = 1L; startTime = 1573041700000L; - stepInstanceDAO.updateStepStartTimeIfNull(stepInstanceId, startTime); - returnStepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + stepInstanceDAO.updateStepStartTimeIfNull(taskInstanceId, stepInstanceId, startTime); + returnStepInstance = stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); + assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(taskInstanceId); assertThat(returnStepInstance.getId()).isEqualTo(stepInstanceId); assertThat(returnStepInstance.getStartTime()).isEqualTo(1572868800000L); } @Test void testUpdateStepStartTimeIfNull() { + long taskInstanceId = 1L; long stepInstanceId = 1L; long startTime = 1573041600000L; - stepInstanceDAO.updateStepStartTime(stepInstanceId, startTime); + stepInstanceDAO.updateStepStartTime(taskInstanceId, stepInstanceId, startTime); - StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); + assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(taskInstanceId); assertThat(returnStepInstance.getId()).isEqualTo(stepInstanceId); assertThat(returnStepInstance.getStartTime()).isEqualTo(startTime); } @Test public void testUpdateStepEndTime() { + long taskInstanceId = 1L; long stepInstanceId = 1L; long endTime = 1573041600000L; - stepInstanceDAO.updateStepEndTime(stepInstanceId, endTime); + stepInstanceDAO.updateStepEndTime(taskInstanceId, stepInstanceId, endTime); - StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); + assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(taskInstanceId); assertThat(returnStepInstance.getId()).isEqualTo(stepInstanceId); assertThat(returnStepInstance.getEndTime()).isEqualTo(endTime); } @Test public void testUpdateStepTotalTime() { + long taskInstanceId = 1L; long stepInstanceId = 1L; long totalTime = 1234L; - stepInstanceDAO.updateStepTotalTime(stepInstanceId, totalTime); + stepInstanceDAO.updateStepTotalTime(taskInstanceId, stepInstanceId, totalTime); - StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); + assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(taskInstanceId); assertThat(returnStepInstance.getId()).isEqualTo(stepInstanceId); assertThat(returnStepInstance.getTotalTime()).isEqualTo(totalTime); } @Test public void testResetStepExecuteInfoForRetry() { + long taskInstanceId = 1L; long stepInstanceId = 1L; - stepInstanceDAO.resetStepExecuteInfoForRetry(stepInstanceId); - StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + stepInstanceDAO.resetStepExecuteInfoForRetry(taskInstanceId, stepInstanceId); + StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); assertThat(returnStepInstance).isNotNull(); + assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(taskInstanceId); + assertThat(returnStepInstance.getId()).isEqualTo(stepInstanceId); assertThat(returnStepInstance.getStartTime()).isNotNull(); assertThat(returnStepInstance.getEndTime()).isNull(); assertThat(returnStepInstance.getStatus()).isEqualTo(RunStatusEnum.RUNNING); @@ -259,8 +279,12 @@ public void testResetStepExecuteInfoForRetry() { @Test public void testGetScriptStepInstance() { - ScriptStepInstanceDTO returnStepInstance = stepInstanceDAO.getScriptStepInstance(1L); - assertThat(returnStepInstance.getStepInstanceId()).isEqualTo(1L); + long taskInstanceId = 1L; + long stepInstanceId = 1L; + ScriptStepInstanceDTO returnStepInstance = stepInstanceDAO.getScriptStepInstance(taskInstanceId, + stepInstanceId); + assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(taskInstanceId); + assertThat(returnStepInstance.getStepInstanceId()).isEqualTo(stepInstanceId); assertThat(returnStepInstance.getAccount()).isEqualTo("root"); assertThat(returnStepInstance.getAccountId()).isEqualTo(1L); assertThat(returnStepInstance.getDbPass()).isEqualTo("db_password"); @@ -280,6 +304,7 @@ public void testGetScriptStepInstance() { public void testAddScriptStepInstance() { StepInstanceDTO scriptStepInstance = new StepInstanceDTO(); scriptStepInstance.setId(100L); + scriptStepInstance.setTaskInstanceId(100L); scriptStepInstance.setAccount("root"); scriptStepInstance.setAccountId(100L); scriptStepInstance.setAccountAlias("root"); @@ -296,7 +321,8 @@ public void testAddScriptStepInstance() { stepInstanceDAO.addScriptStepInstance(scriptStepInstance); - ScriptStepInstanceDTO savedStepInstance = stepInstanceDAO.getScriptStepInstance(100L); + ScriptStepInstanceDTO savedStepInstance = stepInstanceDAO.getScriptStepInstance(100L, 100L); + assertThat(savedStepInstance.getTaskInstanceId()).isEqualTo(100L); assertThat(savedStepInstance.getStepInstanceId()).isEqualTo(100L); assertThat(savedStepInstance.getAccount()).isEqualTo("root"); assertThat(savedStepInstance.getAccountId()).isEqualTo(100L); @@ -316,6 +342,7 @@ public void testAddScriptStepInstance() { public void testAddFileStepInstance() { StepInstanceDTO fileStepInstance = new StepInstanceDTO(); fileStepInstance.setId(101L); + fileStepInstance.setTaskInstanceId(101L); fileStepInstance.setAccount("root"); fileStepInstance.setAccountId(1L); fileStepInstance.setAccountAlias("root"); @@ -346,7 +373,8 @@ public void testAddFileStepInstance() { stepInstanceDAO.addFileStepInstance(fileStepInstance); - FileStepInstanceDTO savedStepInstance = stepInstanceDAO.getFileStepInstance(101L); + FileStepInstanceDTO savedStepInstance = stepInstanceDAO.getFileStepInstance(101L, 101L); + assertThat(savedStepInstance.getTaskInstanceId()).isEqualTo(101L); assertThat(savedStepInstance.getStepInstanceId()).isEqualTo(101L); assertThat(savedStepInstance.getAccount()).isEqualTo("root"); assertThat(savedStepInstance.getAccountId()).isEqualTo(1L); @@ -373,8 +401,9 @@ public void testAddFileStepInstance() { @Test public void testGetFileStepInstance() { - FileStepInstanceDTO returnStepInstance = stepInstanceDAO.getFileStepInstance(2L); + FileStepInstanceDTO returnStepInstance = stepInstanceDAO.getFileStepInstance(1L, 2L); assertThat(returnStepInstance).isNotNull(); + assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(1L); assertThat(returnStepInstance.getStepInstanceId()).isEqualTo(2L); assertThat(returnStepInstance.getFileSourceList()).isNotEmpty(); assertThat(returnStepInstance.getFileSourceList().get(0).getFiles().get(0).getFilePath()).isEqualTo("/$" + @@ -394,9 +423,10 @@ public void testGetFileStepInstance() { @Test public void testGetConfirmStepInstance() { - ConfirmStepInstanceDTO returnStepInstance = stepInstanceDAO.getConfirmStepInstance(3L); + ConfirmStepInstanceDTO returnStepInstance = stepInstanceDAO.getConfirmStepInstance(13L, 15L); assertThat(returnStepInstance).isNotNull(); - assertThat(returnStepInstance.getStepInstanceId()).isEqualTo(3L); + assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(13L); + assertThat(returnStepInstance.getStepInstanceId()).isEqualTo(15L); assertThat(returnStepInstance.getConfirmMessage()).isEqualTo("confirm_message"); assertThat(returnStepInstance.getConfirmReason()).isEqualTo("confirm_reason"); assertThat(returnStepInstance.getConfirmRoles()).containsOnly("JOB_RESOURCE_TRIGGER_USER"); @@ -408,6 +438,7 @@ public void testGetConfirmStepInstance() { public void testAddConfirmStepInstance() { StepInstanceDTO confirmStepInstance = new StepInstanceDTO(); confirmStepInstance.setId(102L); + confirmStepInstance.setTaskInstanceId(102L); confirmStepInstance.setConfirmMessage("confirm_message"); confirmStepInstance.setConfirmUsers(Lists.newArrayList("admin")); confirmStepInstance.setConfirmRoles(Lists.newArrayList("JOB_RESOURCE_TRIGGER_USER")); @@ -415,8 +446,9 @@ public void testAddConfirmStepInstance() { stepInstanceDAO.addConfirmStepInstance(confirmStepInstance); - ConfirmStepInstanceDTO returnStepInstance = stepInstanceDAO.getConfirmStepInstance(102L); + ConfirmStepInstanceDTO returnStepInstance = stepInstanceDAO.getConfirmStepInstance(102L, 102L); assertThat(returnStepInstance).isNotNull(); + assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(102L); assertThat(returnStepInstance.getStepInstanceId()).isEqualTo(102L); assertThat(returnStepInstance.getConfirmMessage()).isEqualTo("confirm_message"); assertThat(returnStepInstance.getConfirmRoles()).containsOnly("JOB_RESOURCE_TRIGGER_USER"); @@ -427,8 +459,8 @@ public void testAddConfirmStepInstance() { @Test void testUpdateResolvedScriptParam() { - stepInstanceDAO.updateResolvedScriptParam(1L, true, "resolved_var"); - ScriptStepInstanceDTO updatedStepInstance = stepInstanceDAO.getScriptStepInstance(1L); + stepInstanceDAO.updateResolvedScriptParam(1L, 1L, true, "resolved_var"); + ScriptStepInstanceDTO updatedStepInstance = stepInstanceDAO.getScriptStepInstance(1L, 1L); assertThat(updatedStepInstance).isNotNull(); assertThat(updatedStepInstance.getResolvedScriptParam()).isEqualTo("resolved_var"); @@ -458,8 +490,8 @@ void testUpdateResolvedFileSource() { fileSources.add(fileSourceDTO); - stepInstanceDAO.updateResolvedSourceFile(2L, fileSources); - FileStepInstanceDTO updatedStepInstance = stepInstanceDAO.getFileStepInstance(2L); + stepInstanceDAO.updateResolvedSourceFile(1L, 2L, fileSources); + FileStepInstanceDTO updatedStepInstance = stepInstanceDAO.getFileStepInstance(1L, 2L); assertThat(updatedStepInstance).isNotNull(); assertThat(updatedStepInstance.getFileSourceList()).hasSize(1); @@ -475,8 +507,8 @@ void testUpdateResolvedFileSource() { @Test void testUpdateResolvedFileTargetPath() { - stepInstanceDAO.updateResolvedTargetPath(2L, "/data/bkee/"); - FileStepInstanceDTO updatedStepInstance = stepInstanceDAO.getFileStepInstance(2L); + stepInstanceDAO.updateResolvedTargetPath(1L, 2L, "/data/bkee/"); + FileStepInstanceDTO updatedStepInstance = stepInstanceDAO.getFileStepInstance(1L, 2L); assertThat(updatedStepInstance).isNotNull(); assertThat(updatedStepInstance.getResolvedFileTargetPath()).isEqualTo("/data/bkee/"); @@ -484,8 +516,8 @@ void testUpdateResolvedFileTargetPath() { @Test void testUpdateConfirmReason() { - stepInstanceDAO.updateConfirmReason(3L, "ok"); - ConfirmStepInstanceDTO updatedStepInstance = stepInstanceDAO.getConfirmStepInstance(3L); + stepInstanceDAO.updateConfirmReason(13L, 15L, "ok"); + ConfirmStepInstanceDTO updatedStepInstance = stepInstanceDAO.getConfirmStepInstance(13L, 15L); assertThat(updatedStepInstance).isNotNull(); assertThat(updatedStepInstance.getConfirmReason()).isEqualTo("ok"); @@ -493,8 +525,8 @@ void testUpdateConfirmReason() { @Test void testUpdateStepOperator() { - stepInstanceDAO.updateStepOperator(3L, "test"); - StepInstanceBaseDTO updatedStepInstance = stepInstanceDAO.getStepInstanceBase(3L); + stepInstanceDAO.updateStepOperator(3L, 4L, "test"); + StepInstanceBaseDTO updatedStepInstance = stepInstanceDAO.getStepInstanceBase(3L, 4L); assertThat(updatedStepInstance).isNotNull(); assertThat(updatedStepInstance.getOperator()).isEqualTo("test"); @@ -504,22 +536,23 @@ void testUpdateStepOperator() { void testGetPreExecutableStepInstance() { StepInstanceBaseDTO preStepInstance = stepInstanceDAO.getPreExecutableStepInstance(1L, 2L); assertThat(preStepInstance).isNotNull(); + assertThat(preStepInstance.getTaskInstanceId()).isEqualTo(1L); assertThat(preStepInstance.getId()).isEqualTo(1L); } @Test void updateStepCurrentBatch() { - stepInstanceDAO.updateStepCurrentBatch(1L, 1); + stepInstanceDAO.updateStepCurrentBatch(1L, 1L, 1); - StepInstanceBaseDTO stepInstance = stepInstanceDAO.getStepInstanceBase(1L); + StepInstanceBaseDTO stepInstance = stepInstanceDAO.getStepInstanceBase(1L, 1L); assertThat(stepInstance.getBatch()).isEqualTo(1); } @Test void updateStepRollingConfigId() { - stepInstanceDAO.updateStepRollingConfigId(1L, 1000L); + stepInstanceDAO.updateStepRollingConfigId(1L, 1L, 1000L); - StepInstanceBaseDTO stepInstance = stepInstanceDAO.getStepInstanceBase(1L); + StepInstanceBaseDTO stepInstance = stepInstanceDAO.getStepInstanceBase(1L, 1L); assertThat(stepInstance.getRollingConfigId()).isEqualTo(1000L); } diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_schema.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_schema.sql index 39cb8ba7b6..788e17efa9 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/init_schema.sql +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_schema.sql @@ -90,6 +90,7 @@ CREATE TABLE IF NOT EXISTS `step_instance` CREATE TABLE IF NOT EXISTS `step_instance_script` ( `step_instance_id` bigint(20) NOT NULL, + `task_instance_id` bigint(20) NOT NULL DEFAULT 0, `script_content` mediumtext, `script_type` tinyint(4) DEFAULT NULL, `script_param` text, @@ -115,6 +116,7 @@ CREATE TABLE IF NOT EXISTS `step_instance_script` CREATE TABLE IF NOT EXISTS `step_instance_file` ( `step_instance_id` bigint(20) NOT NULL, + `task_instance_id` bigint(20) NOT NULL DEFAULT 0, `file_source` mediumtext NOT NULL, `resolved_file_source` mediumtext DEFAULT NULL, `file_target_path` varchar(512) DEFAULT NULL, @@ -136,6 +138,7 @@ CREATE TABLE IF NOT EXISTS `step_instance_file` CREATE TABLE IF NOT EXISTS `step_instance_confirm` ( `step_instance_id` bigint(20) NOT NULL, + `task_instance_id` bigint(20) NOT NULL DEFAULT 0, `confirm_message` text NOT NULL, `confirm_reason` varchar(256) DEFAULT NULL, `confirm_users` varchar(1024) DEFAULT NULL, @@ -190,6 +193,7 @@ CREATE TABLE IF NOT EXISTS `gse_task_ip_log` CREATE TABLE IF NOT EXISTS `gse_task` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, + `task_instance_id` bigint(20) NOT NULL DEFAULT 0, `step_instance_id` bigint(20) NOT NULL DEFAULT '0', `execute_count` smallint(6) NOT NULL DEFAULT '0', `batch` smallint(6) NOT NULL DEFAULT '0', @@ -209,6 +213,7 @@ CREATE TABLE IF NOT EXISTS `gse_task` CREATE TABLE IF NOT EXISTS `gse_script_agent_task` ( `id` bigint(20) NOT NULL AUTO_INCREMENT PRIMARY KEY, + `task_instance_id` bigint(20) NOT NULL DEFAULT 0, `step_instance_id` bigint(20) NOT NULL, `execute_count` smallint(6) NOT NULL DEFAULT '0', `actual_execute_count` smallint(6) DEFAULT NULL, @@ -234,6 +239,7 @@ CREATE TABLE IF NOT EXISTS `gse_script_agent_task` CREATE TABLE IF NOT EXISTS `gse_file_agent_task` ( `id` bigint(20) NOT NULL AUTO_INCREMENT PRIMARY KEY, + `task_instance_id` bigint(20) NOT NULL DEFAULT 0, `step_instance_id` bigint(20) NOT NULL, `execute_count` smallint(6) NOT NULL DEFAULT '0', `actual_execute_count` smallint(6) DEFAULT NULL, @@ -345,6 +351,7 @@ CREATE TABLE IF NOT EXISTS `rolling_config` CREATE TABLE IF NOT EXISTS `step_instance_rolling_task` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, + `task_instance_id` bigint(20) NOT NULL DEFAULT 0, `step_instance_id` bigint(20) NOT NULL DEFAULT '0', `batch` smallint(6) NOT NULL DEFAULT '0', `execute_count` smallint(6) NOT NULL DEFAULT '0', diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_step_instance_data.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_step_instance_data.sql index cb6d196b68..e1930001d0 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/init_step_instance_data.sql +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_step_instance_data.sql @@ -27,28 +27,23 @@ truncate table step_instance_script; truncate table step_instance_file; truncate table step_instance_confirm; -insert into job_execute.step_instance (id,app_id,task_instance_id,step_id,name,type,target_servers,operator,status,execute_count,start_time,end_time,total_time, - create_time,step_num,step_order) values (1,2,1,1,'task1-step1',1,'{"ipList":[{"cloudAreaId":0,"ip":"127.0.0.1"}]}','admin',3,0,1572868800000,1572868801000,1111,1572868800000,2,1); - -insert into job_execute.step_instance (id,app_id,task_instance_id,step_id,name,type,target_servers,operator,status,execute_count,start_time,end_time,total_time, - create_time,step_num,step_order) values (2,2,1,2,'task1-step2',2,'{"ipList":[{"cloudAreaId":0,"ip":"127.0.0.1"}]}','admin',3,0,1572868801000,1572868802000,1112,1572868800000,2,2); - -insert into job_execute.step_instance (id,app_id,task_instance_id,step_id,name,type,target_servers,operator,status,execute_count,start_time,end_time,total_time, - create_time,step_num,step_order) values (3,2,2,-1,'fast_execute_task_name',1,'{"ipList":[{"cloudAreaId":0,"ip":"127.0.0.1"}]}','admin',3,0,1572868800000,1572868801000,1111,1572868800000,1,1); - -insert into job_execute.step_instance (id,app_id,task_instance_id,step_id,name,type,target_servers,operator,status,execute_count,start_time,end_time,total_time, - create_time,step_num,step_order) values (4,2,3,-1,'fast_execute_task_name',1,'{"ipList":[{"cloudAreaId":0,"ip":"127.0.0.1"}]}','admin',3,0,null,1572868801000,0,null,1,1); - - -insert into job_execute.step_instance_script(step_instance_id,script_content,script_type,script_param,resolved_script_param,execution_timeout,system_account_id,system_account, - db_account_id,db_type,db_account,db_password,db_port,script_source,script_id,script_version_id,is_secure_param) values (1,'script_content',1,'${var1}','var1',1000,1,'root',11,1,'root','ESKsXn+pF9hACG3BSYG38ZnUjQQ8bUcOylREiEnDTPU=',3306,1,NULL,NULL,1); - -insert into job_execute.step_instance_file(step_instance_id,file_source,resolved_file_source,file_target_path,file_target_name,resolved_file_target_path,file_upload_speed_limit,file_download_speed_limit, - file_duplicate_handle,not_exist_path_handler,execution_timeout,system_account_id,system_account) values (2,'[{"files":[{ "filePath":"/${log_dir}/1.log" }],"localUpload":false}]', +insert into job_execute.step_instance ( +id,app_id,task_instance_id,step_id,name,type,target_servers,operator,status,execute_count,start_time,end_time,total_time,create_time,step_num,step_order) +values + (1,2,1,1,'task1-step1',1,'{"ipList":[{"cloudAreaId":0,"ip":"127.0.0.1"}]}','admin',3,0,1572868800000,1572868801000,1111,1572868800000,2,1), + (2,2,1,2,'task1-step2',2,'{"ipList":[{"cloudAreaId":0,"ip":"127.0.0.1"}]}','admin',3,0,1572868801000,1572868802000,1112,1572868800000,2,2), + (3,2,2,-1,'fast_execute_task_name',1,'{"ipList":[{"cloudAreaId":0,"ip":"127.0.0.1"}]}','admin',3,0,1572868800000,1572868801000,1111,1572868800000,1,1), + (4,2,3,-1,'fast_execute_task_name',1,'{"ipList":[{"cloudAreaId":0,"ip":"127.0.0.1"}]}','admin',3,0,null,1572868801000,0,null,1,1); + +insert into job_execute.step_instance_script(step_instance_id,task_instance_id,script_content,script_type,script_param,resolved_script_param,execution_timeout,system_account_id,system_account, + db_account_id,db_type,db_account,db_password,db_port,script_source,script_id,script_version_id,is_secure_param) values (1,1,'script_content',1,'${var1}','var1',1000,1,'root',11,1,'root','ESKsXn+pF9hACG3BSYG38ZnUjQQ8bUcOylREiEnDTPU=',3306,1,NULL,NULL,1); + +insert into job_execute.step_instance_file(step_instance_id,task_instance_id,file_source,resolved_file_source,file_target_path,file_target_name,resolved_file_target_path,file_upload_speed_limit,file_download_speed_limit, + file_duplicate_handle,not_exist_path_handler,execution_timeout,system_account_id,system_account) values (2,1,'[{"files":[{ "filePath":"/${log_dir}/1.log" }],"localUpload":false}]', '[{"files":[{ "resolvedFilePath":"/tmp/1.log", "filePath":"/${log_dir}/1.log" }],"localUpload":false}]','/${log_dir}/','2.log','/tmp/', 100,100,1,1,1000,1,'root'); -insert into job_execute.step_instance_confirm(step_instance_id,confirm_message,confirm_users,confirm_roles,notify_channels,confirm_reason) values ( - 3,'confirm_message','admin,test','JOB_RESOURCE_TRIGGER_USER','weixin','confirm_reason'); +insert into job_execute.step_instance_confirm(step_instance_id,task_instance_id,confirm_message,confirm_users,confirm_roles,notify_channels,confirm_reason) values ( + 15,13,'confirm_message','admin,test','JOB_RESOURCE_TRIGGER_USER','weixin','confirm_reason'); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetStepInstanceStatusResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetStepInstanceStatusResourceImpl.java index 41520ba2a5..f075363a05 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetStepInstanceStatusResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetStepInstanceStatusResourceImpl.java @@ -100,7 +100,8 @@ public EsbResp getJobStepInstanceStatus( resultData.setIsFinished(stepExecutionDetail.isFinished()); resultData.setAyalyseResult(convertToStandardAnalyseResult(stepExecutionDetail)); - StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance(request.getStepInstanceId()); + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance(request.getTaskInstanceId(), + request.getStepInstanceId()); if (stepInstance == null) { log.warn("Get step instance status by taskInstanceId:{}, stepInstanceId:{}, stepInstance is null!", request.getTaskInstanceId(), request.getStepInstanceId()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbOperateStepInstanceResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbOperateStepInstanceResourceImpl.java index 765906a858..e01d03779b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbOperateStepInstanceResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbOperateStepInstanceResourceImpl.java @@ -59,6 +59,7 @@ public EsbResp operateStepInstance(String username, } StepOperationEnum operationEnum = StepOperationEnum.getStepOperation(request.getOperationCode()); StepOperationDTO stepOperation = new StepOperationDTO(); + stepOperation.setTaskInstanceId(request.getTaskInstanceId()); stepOperation.setStepInstanceId(request.getStepInstanceId()); stepOperation.setOperation(operationEnum); taskExecuteService.doStepOperation(request.getAppId(), username, stepOperation); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBatchGetJobInstanceIpLogV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBatchGetJobInstanceIpLogV3ResourceImpl.java index 85dcdff9a7..6031109e34 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBatchGetJobInstanceIpLogV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBatchGetJobInstanceIpLogV3ResourceImpl.java @@ -97,7 +97,8 @@ public EsbResp batchGetJobInstanceIpLogs( taskInstanceAccessProcessor.processBeforeAccess(username, request.getAppResourceScope().getAppId(), taskInstanceId); - StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance(request.getStepInstanceId()); + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance( + request.getTaskInstanceId(), request.getStepInstanceId()); if (stepInstance == null) { throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); } @@ -176,7 +177,8 @@ private void buildFileLogs(EsbIpLogsV3DTO esbIpLogs, esbIpLogs.setLogType(LogTypeEnum.FILE.getValue()); List ipLogs = logService.batchGetFileExecuteObjectLogContent( - stepInstance.getId(), stepInstance.getExecuteCount(), null, null, hostKeys); + stepInstance.getTaskInstanceId(), stepInstance.getId(), stepInstance.getExecuteCount(), + null, null, hostKeys); if (CollectionUtils.isEmpty(ipLogs)) { return; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginBatchGetJobInstanceExecuteObjectLogResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginBatchGetJobInstanceExecuteObjectLogResourceImpl.java index 8a6176f709..6f162d2cf0 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginBatchGetJobInstanceExecuteObjectLogResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginBatchGetJobInstanceExecuteObjectLogResourceImpl.java @@ -92,7 +92,8 @@ public EsbResp batchGetJobInstanceExecuteObjectLogs( taskInstanceAccessProcessor.processBeforeAccess(username, request.getAppResourceScope().getAppId(), taskInstanceId); - StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance(request.getStepInstanceId()); + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance( + request.getTaskInstanceId(), request.getStepInstanceId()); if (stepInstance == null) { throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); } @@ -144,7 +145,8 @@ private void buildFileLogs(EsbExecuteObjectLogsDTO esbExecuteObjectLogs, esbExecuteObjectLogs.setLogType(LogTypeEnum.FILE.getValue()); List executeObjectLogs = logService.batchGetFileExecuteObjectLogContent( - stepInstance.getId(), stepInstance.getExecuteCount(), null, null, executeObjectCompositeKeys); + stepInstance.getTaskInstanceId(), stepInstance.getId(), stepInstance.getExecuteCount(), + null, null, executeObjectCompositeKeys); if (CollectionUtils.isEmpty(executeObjectLogs)) { return; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceIpLogV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceIpLogV3ResourceImpl.java index fdd8332d73..db4c64a6a4 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceIpLogV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceIpLogV3ResourceImpl.java @@ -96,7 +96,8 @@ public EsbResp getJobInstanceIpLogUsingPost( taskInstanceAccessProcessor.processBeforeAccess(username, request.getAppResourceScope().getAppId(), taskInstanceId); - StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance(request.getStepInstanceId()); + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance( + request.getTaskInstanceId(), request.getStepInstanceId()); if (stepInstance == null) { throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); } @@ -152,7 +153,8 @@ private void buildFileLog(EsbIpLogV3DTO ipLog, FileExecuteObjectLogContent downloadIpLog = logService.getFileExecuteObjectLogContent(stepInstance, executeCount, null, hostKey, FileDistModeEnum.DOWNLOAD.getValue()); List uploadExecuteObjectLogs = - logService.batchGetFileSourceExecuteObjectLogContent(stepInstance.getId(), executeCount, null); + logService.batchGetFileSourceExecuteObjectLogContent( + stepInstance.getTaskInstanceId(), stepInstance.getId(), executeCount, null); List fileLogs = new ArrayList<>(); fileLogs.addAll(buildDownloadFileLogs(downloadIpLog)); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceDetailV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceDetailV3ResourceImpl.java index ffb5574128..29dbbc4e22 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceDetailV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceDetailV3ResourceImpl.java @@ -33,15 +33,12 @@ import com.tencent.bk.job.common.esb.model.job.v3.resp.EsbFileStepV3DTO; import com.tencent.bk.job.common.esb.model.job.v3.resp.EsbScriptStepV3DTO; import com.tencent.bk.job.common.esb.model.job.v3.resp.EsbStepV3DTO; -import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.iam.constant.ActionId; -import com.tencent.bk.job.common.model.ValidateResult; import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.execute.model.FileDetailDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.service.StepInstanceService; -import com.tencent.bk.job.execute.service.StepInstanceValidateService; import com.tencent.bk.job.execute.service.TaskInstanceAccessProcessor; import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.execute.util.FileTransferModeUtil; @@ -61,18 +58,15 @@ public class EsbGetStepInstanceDetailV3ResourceImpl implements EsbGetStepInstanc private final TaskInstanceService taskInstanceService; private final AppScopeMappingService appScopeMappingService; private final TaskInstanceAccessProcessor taskInstanceAccessProcessor; - private final StepInstanceValidateService stepInstanceValidateService; private final StepInstanceService stepInstanceService; public EsbGetStepInstanceDetailV3ResourceImpl(TaskInstanceService taskInstanceService, AppScopeMappingService appScopeMappingService, TaskInstanceAccessProcessor taskInstanceAccessProcessor, - StepInstanceValidateService stepInstanceValidateService, StepInstanceService stepInstanceService) { this.taskInstanceService = taskInstanceService; this.appScopeMappingService = appScopeMappingService; this.taskInstanceAccessProcessor = taskInstanceAccessProcessor; - this.stepInstanceValidateService = stepInstanceValidateService; this.stepInstanceService = stepInstanceService; } @@ -86,16 +80,6 @@ public EsbResp getStepInstanceDetail(String username, Long stepInstanceId) { long appId = appScopeMappingService.getAppIdByScope(scopeType, scopeId); - ValidateResult checkResult = stepInstanceValidateService.checkStepInstance( - appId, - taskInstanceId, - stepInstanceId - ); - if (!checkResult.isPass()) { - log.warn("Get step instance detail request is illegal!"); - throw new InvalidParamException(checkResult); - } - StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail(appId, stepInstanceId); taskInstanceAccessProcessor.processBeforeAccess(username, appId, stepInstance.getTaskInstanceId()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceStatusV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceStatusV3ResourceImpl.java index d7af95bc49..cd3f9450ab 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceStatusV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceStatusV3ResourceImpl.java @@ -28,12 +28,10 @@ import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; import com.tencent.bk.job.common.esb.model.EsbResp; -import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.i18n.service.MessageI18nService; import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.metrics.CommonMetricNames; -import com.tencent.bk.job.common.model.ValidateResult; import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; @@ -43,7 +41,6 @@ import com.tencent.bk.job.execute.model.StepExecutionResultQuery; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.esb.v3.EsbStepInstanceStatusV3DTO; -import com.tencent.bk.job.execute.service.StepInstanceValidateService; import com.tencent.bk.job.execute.service.TaskResultService; import lombok.extern.slf4j.Slf4j; import org.springframework.web.bind.annotation.RestController; @@ -55,16 +52,13 @@ @Slf4j public class EsbGetStepInstanceStatusV3ResourceImpl implements EsbGetStepInstanceStatusV3Resource { - private final StepInstanceValidateService stepInstanceValidateService; private final AppScopeMappingService appScopeMappingService; private final TaskResultService taskResultService; private final MessageI18nService messageI18nService; - public EsbGetStepInstanceStatusV3ResourceImpl(StepInstanceValidateService stepInstanceValidateService, - AppScopeMappingService appScopeMappingService, + public EsbGetStepInstanceStatusV3ResourceImpl(AppScopeMappingService appScopeMappingService, TaskResultService taskResultService, MessageI18nService messageI18nService) { - this.stepInstanceValidateService = stepInstanceValidateService; this.appScopeMappingService = appScopeMappingService; this.taskResultService = taskResultService; this.messageI18nService = messageI18nService; @@ -145,16 +139,6 @@ public EsbResp getStepInstanceStatus(String username String tag) { long appId = appScopeMappingService.getAppIdByScope(scopeType, scopeId); - ValidateResult checkResult = stepInstanceValidateService.checkStepInstance( - appId, - taskInstanceId, - stepInstanceId - ); - if (!checkResult.isPass()) { - log.warn("Get step instance status request is illegal!"); - throw new InvalidParamException(checkResult); - } - StepExecutionResultQuery query = StepExecutionResultQuery.builder() .stepInstanceId(stepInstanceId) .executeCount(executeCount) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbOperateStepInstanceV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbOperateStepInstanceV3ResourceImpl.java index 442d38eb27..b6b5662590 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbOperateStepInstanceV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbOperateStepInstanceV3ResourceImpl.java @@ -59,6 +59,7 @@ public EsbResp operateStepInstance(String username, } StepOperationEnum operationEnum = StepOperationEnum.getStepOperation(request.getOperationCode()); StepOperationDTO stepOperation = new StepOperationDTO(); + stepOperation.setTaskInstanceId(request.getTaskInstanceId()); stepOperation.setStepInstanceId(request.getStepInstanceId()); stepOperation.setOperation(operationEnum); taskExecuteService.doStepOperation(request.getAppId(), username, stepOperation); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java index 3088105c78..970b2d80b7 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java @@ -531,6 +531,7 @@ public Response doStepOperation(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, + Long taskInstanceId, Long stepInstanceId, WebStepOperation operation) { StepOperationEnum stepOperationEnum = StepOperationEnum.getStepOperation(operation.getOperationCode()); @@ -538,6 +539,7 @@ public Response doStepOperation(String username, throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } StepOperationDTO stepOperation = new StepOperationDTO(); + stepOperation.setTaskInstanceId(taskInstanceId); stepOperation.setStepInstanceId(stepInstanceId); stepOperation.setOperation(stepOperationEnum); stepOperation.setConfirmReason(operation.getConfirmReason()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebSearchToolsResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebSearchToolsResourceImpl.java index 999e3be680..67a70e3487 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebSearchToolsResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebSearchToolsResourceImpl.java @@ -69,12 +69,13 @@ public Response getTaskLink(String username, String gseTaskId) { GseTaskSimpleDTO gseTaskSimpleInfo = gseTaskService.getGseTaskSimpleInfo(gseTaskId); if (gseTaskSimpleInfo == null) { - String errorMsg = "not found gseTask by "+gseTaskId; + String errorMsg = "not found gseTask by " + gseTaskId; log.warn(errorMsg); return Response.buildCommonFailResp(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, new String[]{String.valueOf(gseTaskId), errorMsg}); } - StepInstanceBaseDTO stepInstanceBase = stepInstanceService.getStepInstanceBase(gseTaskSimpleInfo.getStepInstanceId()); + StepInstanceBaseDTO stepInstanceBase = stepInstanceService.getBaseStepInstance( + gseTaskSimpleInfo.getTaskInstanceId(), gseTaskSimpleInfo.getStepInstanceId()); Long appId = stepInstanceBase.getAppId(); ResourceScope resourceScope = appScopeMappingService.getScopeByAppId(appId); AppResourceScope appResourceScope = new AppResourceScope(appId, resourceScope); @@ -91,9 +92,9 @@ public Response getTaskLink(String username, @Override public Response> getTaskLinkByStepId(String username, Long stepInstanceId) { - StepInstanceBaseDTO stepInstanceBase = stepInstanceService.getStepInstanceBase(stepInstanceId); + StepInstanceBaseDTO stepInstanceBase = stepInstanceService.getBaseStepInstance(stepInstanceId); if (stepInstanceBase == null) { - String errorMsg = "not found StepInstance by "+stepInstanceId; + String errorMsg = "not found StepInstance by " + stepInstanceId; log.warn(errorMsg); return Response.buildCommonFailResp(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, new String[]{String.valueOf(stepInstanceId), errorMsg}); @@ -177,7 +178,7 @@ private TaskLinkVO convertToTaskLinVO(ResourceScope resourceScope, private List buildLink(TaskLinkVO taskLinkVO, StepInstanceBaseDTO stepInstanceBase) { List links = new ArrayList(); String linkTemplate = FAST_LINK; - if(stepInstanceBase.getStepId() != -1L){ + if (stepInstanceBase.getStepId() != -1L) { linkTemplate = TASK_LINK; } if (jobWebUrl.indexOf(",") != -1) { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java index f5c0e8b6b3..00561a86c3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java @@ -89,7 +89,6 @@ import com.tencent.bk.job.execute.model.web.vo.ExecutionResultGroupVO; import com.tencent.bk.job.execute.model.web.vo.FileDistributionDetailV2VO; import com.tencent.bk.job.execute.model.web.vo.FileDistributionDetailVO; -import com.tencent.bk.job.execute.model.web.vo.IpFileLogContentVO; import com.tencent.bk.job.execute.model.web.vo.IpScriptLogContentVO; import com.tencent.bk.job.execute.model.web.vo.RollingStepBatchTaskVO; import com.tencent.bk.job.execute.model.web.vo.StepExecutionDetailV2VO; @@ -847,60 +846,6 @@ private ExecuteVariableVO convertToTaskVariableVO(TaskVariableDTO taskVariable) return vo; } - @Override - @AuditEntry(actionId = ActionId.VIEW_HISTORY) - public Response getFileLogContentByHost(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - Long stepInstanceId, - Integer executeCount, - Long hostId, - String mode, - Integer batch) { - StepInstanceDTO stepInstance = - stepInstanceService.getStepInstanceDetail(appResourceScope.getAppId(), stepInstanceId); - auditAndAuthViewStepInstance(username, appResourceScope, stepInstance); - - IpFileLogContentVO result = new IpFileLogContentVO(); - List fileDistDetailVOS = new ArrayList<>(); - - if ("download".equals(mode)) { - FileExecuteObjectLogContent downloadLog = logService.getFileExecuteObjectLogContent(stepInstance, - executeCount, batch, ExecuteObjectCompositeKey.ofHostId(hostId), FileDistModeEnum.DOWNLOAD.getValue()); - // downloadLog为null说明步骤还未下发至GSE就被终止 - if (downloadLog != null && CollectionUtils.isNotEmpty(downloadLog.getFileTaskLogs())) { - fileDistDetailVOS = - downloadLog.getFileTaskLogs().stream() - .map(this::convertToFileDistributionDetailVO) - .collect(Collectors.toList()); - result.setFinished(downloadLog.isFinished()); - } - } else { - List executeObjectLogContents = - logService.batchGetFileSourceExecuteObjectLogContent(stepInstanceId, - executeCount, batch); - if (CollectionUtils.isNotEmpty(executeObjectLogContents)) { - fileDistDetailVOS = - executeObjectLogContents.stream() - .flatMap(executeObjectLogContent -> executeObjectLogContent.getFileTaskLogs().stream()) - .map(this::convertToFileDistributionDetailVO) - .collect(Collectors.toList()); - result.setFinished( - executeObjectLogContents.stream() - .flatMap(executeObjectLogContent -> executeObjectLogContent.getFileTaskLogs().stream()) - .allMatch(fileLog -> FileDistStatusEnum.isFinishedStatus(fileLog.getStatus()))); - } - } - Collections.sort(fileDistDetailVOS); - result.setFileDistributionDetails(fileDistDetailVOS); - - boolean includingLogContent = !removeFileLogContentIfResultIsLarge(fileDistDetailVOS); - result.setIncludingLogContent(includingLogContent); - - return Response.buildSuccessResp(result); - } - private boolean removeFileLogContentIfResultIsLarge(List fileDistDetailVOS) { // 超过128K boolean removeFileLogContent = calculateFileLogContentLength(fileDistDetailVOS) > 131072L; @@ -991,28 +936,6 @@ private FileDistributionDetailV2VO convertToFileDistributionDetailV2VO(AtomicFil return fileDistDetailVO; } - @Override - @AuditEntry(actionId = ActionId.VIEW_HISTORY) - public Response> getFileLogContentByFileTaskIds(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - Long stepInstanceId, - Integer executeCount, - Integer batch, - List taskIds) { - auditAndAuthViewStepInstance(username, appResourceScope, stepInstanceId); - - List fileTaskLogs = logService.getAtomicFileTaskLogByTaskIds(stepInstanceId, executeCount, - batch, taskIds); - if (CollectionUtils.isEmpty(fileTaskLogs)) { - return Response.buildSuccessResp(null); - } - List fileDistDetailVOS = new ArrayList<>(); - fileTaskLogs.forEach(fileLog -> fileDistDetailVOS.add(convertToFileDistributionDetailVO(fileLog))); - return Response.buildSuccessResp(fileDistDetailVOS); - } - private void auditAndAuthViewStepInstance(String username, AppResourceScope appResourceScope, StepInstanceBaseDTO stepInstance) { @@ -1190,7 +1113,7 @@ public Response getFileLogContentByExecuteObject(String break; case UPLOAD: List executeObjectLogContents = - logService.batchGetFileSourceExecuteObjectLogContent(stepInstanceId, + logService.batchGetFileSourceExecuteObjectLogContent(taskInstanceId, stepInstanceId, actualExecuteCount, batch); if (CollectionUtils.isNotEmpty(executeObjectLogContents)) { fileDistDetailVOS = @@ -1230,7 +1153,7 @@ public Response> getFileLogContentByFileTaskIds auditAndAuthViewStepInstance(username, appResourceScope, stepInstance); int actualExecuteCount = computeActualExecuteCount(stepInstance, executeCount); - List fileTaskLogs = logService.getAtomicFileTaskLogByTaskIds(stepInstanceId, + List fileTaskLogs = logService.getAtomicFileTaskLogByTaskIds(taskInstanceId, stepInstanceId, actualExecuteCount, batch, taskIds); if (CollectionUtils.isEmpty(fileTaskLogs)) { return Response.buildSuccessResp(null); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskLogResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskLogResourceImpl.java index 812246d221..3fa9e10f31 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskLogResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskLogResourceImpl.java @@ -133,7 +133,7 @@ public Response requestDownloadLogFile(String username, repackage = false; } - StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance(stepInstanceId); + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance(taskInstanceId, stepInstanceId); if (!stepInstance.getAppId().equals(appResourceScope.getAppId())) { throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); } @@ -193,8 +193,18 @@ public Response requestDownloadLogFile(String username, throw new InternalException(ErrorCode.EXPORT_STEP_EXECUTION_LOG_FAIL); } - LogExportJobInfoDTO exportInfo = logExportService.packageLogFile(username, appId, stepInstanceId, - executeObjectTypeEnum, executeObjectResourceId, executeCount, logFileDir, logFileName, repackage); + LogExportJobInfoDTO exportInfo = logExportService.packageLogFile( + username, + appId, + taskInstanceId, + stepInstanceId, + executeObjectTypeEnum, + executeObjectResourceId, + executeCount, + logFileDir, + logFileName, + repackage + ); return Response.buildSuccessResp(LogExportJobInfoDTO.toVO(exportInfo)); } @@ -276,7 +286,7 @@ public ResponseEntity downloadLogFile(HttpServletResponse Long executeObjectResourceId) { Long appId = appResourceScope.getAppId(); - StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance(stepInstanceId); + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance(taskInstanceId, stepInstanceId); if (!stepInstance.getAppId().equals(appId)) { log.info("StepInstance: {} is not in app: {}", stepInstance.getId(), appResourceScope.getAppId()); return ResponseEntity.notFound().build(); @@ -297,8 +307,18 @@ public ResponseEntity downloadLogFile(HttpServletResponse stepInstanceId, executeObjectType, executeObjectResourceId, executeCount); return ResponseEntity.notFound().build(); } - exportInfo = logExportService.packageLogFile(username, appId, stepInstanceId, executeObjectTypeEnum, - executeObjectResourceId, executeCount, logFileDir, logFileName, false); + exportInfo = logExportService.packageLogFile( + username, + appId, + taskInstanceId, + stepInstanceId, + executeObjectTypeEnum, + executeObjectResourceId, + executeCount, + logFileDir, + logFileName, + false + ); } else { exportInfo = logExportService.getExportInfo(appId, stepInstanceId, null, null); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceDAO.java index 3d9cbacb65..ef9c5702d3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceDAO.java @@ -24,20 +24,19 @@ package com.tencent.bk.job.execute.dao; -import com.tencent.bk.job.common.constant.DuplicateHandlerEnum; -import com.tencent.bk.job.common.constant.NotExistPathHandlerEnum; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; import com.tencent.bk.job.execute.model.ConfirmStepInstanceDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.FileStepInstanceDTO; import com.tencent.bk.job.execute.model.ScriptStepInstanceDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; -import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import java.util.List; +/** + * 任务步骤实例 DAO + */ public interface StepInstanceDAO { Long addStepInstanceBase(StepInstanceBaseDTO stepInstance); @@ -47,6 +46,8 @@ public interface StepInstanceDAO { void addConfirmStepInstance(StepInstanceDTO stepInstance); + StepInstanceBaseDTO getStepInstanceBase(long taskInstanceId, long stepInstanceId); + StepInstanceBaseDTO getStepInstanceBase(long stepInstanceId); /** @@ -66,23 +67,23 @@ public interface StepInstanceDAO { */ StepInstanceBaseDTO getNextStepInstance(long taskInstanceId, int currentStepOrder); - ScriptStepInstanceDTO getScriptStepInstance(long stepInstanceId); + ScriptStepInstanceDTO getScriptStepInstance(long taskInstanceId, long stepInstanceId); - FileStepInstanceDTO getFileStepInstance(long stepInstanceId); + FileStepInstanceDTO getFileStepInstance(long taskInstanceId, long stepInstanceId); - ConfirmStepInstanceDTO getConfirmStepInstance(long stepInstanceId); + ConfirmStepInstanceDTO getConfirmStepInstance(long taskInstanceId, long stepInstanceId); List listStepInstanceBaseByTaskInstanceId(long taskInstanceId); - void resetStepStatus(long stepInstanceId); + void resetStepStatus(long taskInstanceId, long stepInstanceId); - void resetStepExecuteInfoForRetry(long stepInstanceId); + void resetStepExecuteInfoForRetry(long taskInstanceId, long stepInstanceId); - void addStepExecuteCount(long stepInstanceId); + void addStepExecuteCount(long taskInstanceId, long stepInstanceId); - void updateStepStatus(long stepInstanceId, int status); + void updateStepStatus(long taskInstanceId, long stepInstanceId, int status); - void updateStepStartTime(long stepInstanceId, Long startTime); + void updateStepStartTime(long taskInstanceId, long stepInstanceId, Long startTime); /** * 更新步骤启动时间 - 仅当启动时间为空的的场景 @@ -90,79 +91,81 @@ public interface StepInstanceDAO { * @param stepInstanceId 步骤实例ID * @param startTime 启动时间 */ - void updateStepStartTimeIfNull(long stepInstanceId, Long startTime); + void updateStepStartTimeIfNull(long taskInstanceId, long stepInstanceId, Long startTime); - void updateStepEndTime(long stepInstanceId, Long endTime); + void updateStepEndTime(long taskInstanceId, long stepInstanceId, Long endTime); - void addStepInstanceExecuteCount(long stepInstanceId); + void addStepInstanceExecuteCount(long taskInstanceId, long stepInstanceId); - void updateStepTotalTime(long stepInstanceId, long totalTime); + void updateStepTotalTime(long taskInstanceId, long stepInstanceId, long totalTime); /** * 更新步骤的执行信息 * + * @param taskInstanceId 任务实例 ID * @param stepInstanceId 步骤实例ID * @param status 步骤执行状态 * @param startTime 开始时间 * @param endTime 结束时间 * @param totalTime 总耗时 */ - void updateStepExecutionInfo(long stepInstanceId, RunStatusEnum status, Long startTime, Long endTime, + void updateStepExecutionInfo(long taskInstanceId, + long stepInstanceId, + RunStatusEnum status, + Long startTime, + Long endTime, Long totalTime); /** * 更新解析之后的脚本参数 * + * @param taskInstanceId 任务实例 ID * @param stepInstanceId 步骤实例ID * @param isSecureParam 是否为敏感参数 * @param resolvedScriptParam 解析之后的脚本参数 */ - void updateResolvedScriptParam(long stepInstanceId, boolean isSecureParam, String resolvedScriptParam); + void updateResolvedScriptParam(long taskInstanceId, + long stepInstanceId, + boolean isSecureParam, + String resolvedScriptParam); /** * 更新变量解析之后的源文件 * + * @param taskInstanceId 任务实例 ID * @param stepInstanceId 步骤实例ID * @param resolvedFileSources 解析后的源文件信息 */ - void updateResolvedSourceFile(long stepInstanceId, List resolvedFileSources); + void updateResolvedSourceFile(long taskInstanceId, + long stepInstanceId, + List resolvedFileSources); /** * 更新变量解析之后的目标路径 * + * @param taskInstanceId 任务实例 ID * @param stepInstanceId 步骤实例ID * @param resolvedTargetPath 解析之后的目标路径 */ - void updateResolvedTargetPath(long stepInstanceId, String resolvedTargetPath); + void updateResolvedTargetPath(long taskInstanceId, long stepInstanceId, String resolvedTargetPath); /** * 更新确认理由 * + * @param taskInstanceId 任务实例 ID * @param stepInstanceId 步骤实例ID * @param confirmReason 确认理由 */ - void updateConfirmReason(long stepInstanceId, String confirmReason); + void updateConfirmReason(long taskInstanceId, long stepInstanceId, String confirmReason); /** * 更新步骤实例操作者 * + * @param taskInstanceId 任务实例 ID * @param stepInstanceId 步骤实例ID * @param operator 操作者 */ - void updateStepOperator(long stepInstanceId, String operator); - - Integer count(Long appId, List stepIdList, StepExecuteTypeEnum stepExecuteType, ScriptTypeEnum scriptType, - RunStatusEnum runStatus, Long fromTime, Long toTime); - - Integer countFastPushFile(Long appId, DuplicateHandlerEnum fileDupliateHandle, Boolean fileDupliateHandleNull, - NotExistPathHandlerEnum notExistPathHandler, Boolean notExistPathHandlerNull, - RunStatusEnum runStatus, Long fromTime, Long toTime); - - List> listFastPushFileSource(Long appId, DuplicateHandlerEnum fileDupliateHandle, - Boolean fileDupliateHandleNull, - NotExistPathHandlerEnum notExistPathHandler, - Boolean notExistPathHandlerNull, RunStatusEnum runStatus, - Long fromTime, Long toTime); + void updateStepOperator(long taskInstanceId, long stepInstanceId, String operator); /** * 获取前一个可执行步骤实例 @@ -193,34 +196,38 @@ List> listFastPushFileSource(Long appId, DuplicateHandlerEnu /** * 根据stepInstanceId获取脚本类型 * + * @param taskInstanceId 任务实例ID * @param stepInstanceId 步骤实例ID * @return 脚本类型 */ - Byte getScriptTypeByStepInstanceId(long stepInstanceId); + Byte getScriptTypeByStepInstanceId(long taskInstanceId, long stepInstanceId); /** * 更新步骤实例的当前滚动执行批次 * + * @param taskInstanceId 任务实例ID * @param stepInstanceId 步骤实例ID * @param batch 滚动执行批次 */ - void updateStepCurrentBatch(long stepInstanceId, int batch); + void updateStepCurrentBatch(long taskInstanceId, long stepInstanceId, int batch); /** * 更新步骤实例的当前滚动执行批次 * + * @param taskInstanceId 任务实例ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 */ - void updateStepCurrentExecuteCount(long stepInstanceId, int executeCount); + void updateStepCurrentExecuteCount(long taskInstanceId, long stepInstanceId, int executeCount); /** * 更新步骤实例的滚动配置ID * + * @param taskInstanceId 任务实例ID * @param stepInstanceId 步骤实例ID * @param rollingConfigId 滚动配置ID */ - void updateStepRollingConfigId(long stepInstanceId, long rollingConfigId); + void updateStepRollingConfigId(long taskInstanceId, long stepInstanceId, long rollingConfigId); List getTaskStepInstanceIdList(long taskInstanceId); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImpl.java index e58c5bfc55..542df7c6f9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImpl.java @@ -26,8 +26,6 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.tencent.bk.job.common.constant.AccountCategoryEnum; -import com.tencent.bk.job.common.constant.DuplicateHandlerEnum; -import com.tencent.bk.job.common.constant.NotExistPathHandlerEnum; import com.tencent.bk.job.common.crypto.scenario.DbPasswordCryptoService; import com.tencent.bk.job.common.crypto.scenario.SensitiveParamCryptoService; import com.tencent.bk.job.common.util.Utils; @@ -50,25 +48,19 @@ import com.tencent.bk.job.execute.model.tables.records.StepInstanceRecord; import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import lombok.extern.slf4j.Slf4j; -import lombok.val; import org.apache.commons.lang3.StringUtils; -import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.Record; import org.jooq.Record1; import org.jooq.Result; import org.jooq.TableField; import org.jooq.UpdateSetMoreStep; -import org.jooq.conf.ParamType; import org.jooq.types.UByte; -import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; import java.util.List; @Slf4j @@ -76,8 +68,10 @@ public class StepInstanceDAOImpl implements StepInstanceDAO { private static final StepInstance T_STEP_INSTANCE = StepInstance.STEP_INSTANCE; - private static final StepInstanceScript TABLE_STEP_INSTANCE_SCRIPT = StepInstanceScript.STEP_INSTANCE_SCRIPT; - private static final StepInstanceFile TABLE_STEP_INSTANCE_FILE = StepInstanceFile.STEP_INSTANCE_FILE; + private static final StepInstanceScript T_STEP_INSTANCE_SCRIPT = StepInstanceScript.STEP_INSTANCE_SCRIPT; + private static final StepInstanceFile T_STEP_INSTANCE_FILE = StepInstanceFile.STEP_INSTANCE_FILE; + private static final StepInstanceConfirm T_STEP_INSTANCE_CONFIRM = StepInstanceConfirm.STEP_INSTANCE_CONFIRM; + private static final TableField[] T_STEP_INSTANCE_ALL_FIELDS = { T_STEP_INSTANCE.ID, T_STEP_INSTANCE.STEP_ID, @@ -99,6 +93,50 @@ public class StepInstanceDAOImpl implements StepInstanceDAO { T_STEP_INSTANCE.BATCH, T_STEP_INSTANCE.ROLLING_CONFIG_ID }; + private static final TableField[] T_STEP_INSTANCE_SCRIPT_ALL_FIELDS = { + T_STEP_INSTANCE_SCRIPT.STEP_INSTANCE_ID, + T_STEP_INSTANCE_SCRIPT.TASK_INSTANCE_ID, + T_STEP_INSTANCE_SCRIPT.SCRIPT_CONTENT, + T_STEP_INSTANCE_SCRIPT.SCRIPT_TYPE, + T_STEP_INSTANCE_SCRIPT.SCRIPT_PARAM, + T_STEP_INSTANCE_SCRIPT.RESOLVED_SCRIPT_PARAM, + T_STEP_INSTANCE_SCRIPT.EXECUTION_TIMEOUT, + T_STEP_INSTANCE_SCRIPT.SYSTEM_ACCOUNT_ID, + T_STEP_INSTANCE_SCRIPT.SYSTEM_ACCOUNT, + T_STEP_INSTANCE_SCRIPT.DB_ACCOUNT_ID, + T_STEP_INSTANCE_SCRIPT.DB_ACCOUNT, + T_STEP_INSTANCE_SCRIPT.DB_TYPE, + T_STEP_INSTANCE_SCRIPT.DB_PASSWORD, + T_STEP_INSTANCE_SCRIPT.DB_PORT, + T_STEP_INSTANCE_SCRIPT.SCRIPT_SOURCE, + T_STEP_INSTANCE_SCRIPT.SCRIPT_ID, + T_STEP_INSTANCE_SCRIPT.SCRIPT_VERSION_ID, + T_STEP_INSTANCE_SCRIPT.IS_SECURE_PARAM + }; + private static final TableField[] T_STEP_INSTANCE_FILE_ALL_FIELDS = { + T_STEP_INSTANCE_FILE.TASK_INSTANCE_ID, + T_STEP_INSTANCE_FILE.STEP_INSTANCE_ID, + T_STEP_INSTANCE_FILE.FILE_SOURCE, + T_STEP_INSTANCE_FILE.FILE_TARGET_PATH, + T_STEP_INSTANCE_FILE.FILE_TARGET_NAME, + T_STEP_INSTANCE_FILE.RESOLVED_FILE_TARGET_PATH, + T_STEP_INSTANCE_FILE.FILE_UPLOAD_SPEED_LIMIT, + T_STEP_INSTANCE_FILE.FILE_DOWNLOAD_SPEED_LIMIT, + T_STEP_INSTANCE_FILE.FILE_DUPLICATE_HANDLE, + T_STEP_INSTANCE_FILE.NOT_EXIST_PATH_HANDLER, + T_STEP_INSTANCE_FILE.EXECUTION_TIMEOUT, + T_STEP_INSTANCE_FILE.SYSTEM_ACCOUNT_ID, + T_STEP_INSTANCE_FILE.SYSTEM_ACCOUNT + }; + private static final TableField[] T_STEP_INSTANCE_CONFIRM_ALL_FIELDS = { + T_STEP_INSTANCE_CONFIRM.TASK_INSTANCE_ID, + T_STEP_INSTANCE_CONFIRM.STEP_INSTANCE_ID, + T_STEP_INSTANCE_CONFIRM.CONFIRM_MESSAGE, + T_STEP_INSTANCE_CONFIRM.CONFIRM_REASON, + T_STEP_INSTANCE_CONFIRM.CONFIRM_USERS, + T_STEP_INSTANCE_CONFIRM.CONFIRM_ROLES, + T_STEP_INSTANCE_CONFIRM.NOTIFY_CHANNELS + }; private final DSLContext CTX; private final SensitiveParamCryptoService sensitiveParamCryptoService; @@ -168,6 +206,7 @@ public void addScriptStepInstance(StepInstanceDTO stepInstance) { StepInstanceScript t = StepInstanceScript.STEP_INSTANCE_SCRIPT; CTX.insertInto(t, t.STEP_INSTANCE_ID, + t.TASK_INSTANCE_ID, t.SCRIPT_CONTENT, t.SCRIPT_TYPE, t.SCRIPT_PARAM, @@ -186,6 +225,7 @@ public void addScriptStepInstance(StepInstanceDTO stepInstance) { t.IS_SECURE_PARAM ).values( stepInstance.getId(), + stepInstance.getTaskInstanceId(), stepInstance.getScriptContent(), stepInstance.getScriptType().getValue().byteValue(), sensitiveParamCryptoService.encryptParamIfNeeded( @@ -213,6 +253,7 @@ public void addFileStepInstance(StepInstanceDTO stepInstance) { StepInstanceFile t = StepInstanceFile.STEP_INSTANCE_FILE; CTX.insertInto(t, t.STEP_INSTANCE_ID, + t.TASK_INSTANCE_ID, t.FILE_SOURCE, t.FILE_TARGET_PATH, t.FILE_TARGET_NAME, @@ -225,6 +266,7 @@ public void addFileStepInstance(StepInstanceDTO stepInstance) { t.SYSTEM_ACCOUNT ).values( stepInstance.getId(), + stepInstance.getTaskInstanceId(), JsonUtils.toJson(stepInstance.getFileSourceList()), stepInstance.getFileTargetPath(), stepInstance.getFileTargetName(), @@ -241,8 +283,16 @@ public void addFileStepInstance(StepInstanceDTO stepInstance) { @Override public void addConfirmStepInstance(StepInstanceDTO stepInstance) { StepInstanceConfirm t = StepInstanceConfirm.STEP_INSTANCE_CONFIRM; - CTX.insertInto(t, t.STEP_INSTANCE_ID, t.CONFIRM_MESSAGE, t.CONFIRM_USERS, t.CONFIRM_ROLES, t.NOTIFY_CHANNELS) - .values(stepInstance.getId(), + CTX.insertInto(t, + t.STEP_INSTANCE_ID, + t.TASK_INSTANCE_ID, + t.CONFIRM_MESSAGE, + t.CONFIRM_USERS, + t.CONFIRM_ROLES, + t.NOTIFY_CHANNELS) + .values( + stepInstance.getId(), + stepInstance.getTaskInstanceId(), stepInstance.getConfirmMessage(), stepInstance.getConfirmUsers() == null ? null : Utils.concatStringWithSeperator(stepInstance.getConfirmUsers(), ","), @@ -254,28 +304,12 @@ public void addConfirmStepInstance(StepInstanceDTO stepInstance) { } @Override - public ScriptStepInstanceDTO getScriptStepInstance(long stepInstanceId) { - StepInstanceScript t = StepInstanceScript.STEP_INSTANCE_SCRIPT; - Record record = CTX.select( - t.STEP_INSTANCE_ID, - t.SCRIPT_CONTENT, - t.SCRIPT_TYPE, - t.SCRIPT_PARAM, - t.RESOLVED_SCRIPT_PARAM, - t.EXECUTION_TIMEOUT, - t.SYSTEM_ACCOUNT_ID, - t.SYSTEM_ACCOUNT, - t.DB_ACCOUNT_ID, - t.DB_ACCOUNT, - t.DB_TYPE, - t.DB_PASSWORD, - t.DB_PORT, - t.SCRIPT_SOURCE, - t.SCRIPT_ID, - t.SCRIPT_VERSION_ID, - t.IS_SECURE_PARAM - ).from(t) - .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)).fetchOne(); + public ScriptStepInstanceDTO getScriptStepInstance(long taskInstanceId, long stepInstanceId) { + Record record = CTX.select(T_STEP_INSTANCE_SCRIPT_ALL_FIELDS) + .from(T_STEP_INSTANCE_SCRIPT) + .where(T_STEP_INSTANCE_SCRIPT.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(T_STEP_INSTANCE_SCRIPT.STEP_INSTANCE_ID.eq(stepInstanceId)) + .fetchOne(); return extractScriptInfo(record); } @@ -285,6 +319,7 @@ private ScriptStepInstanceDTO extractScriptInfo(Record record) { } StepInstanceScript t = StepInstanceScript.STEP_INSTANCE_SCRIPT; ScriptStepInstanceDTO stepInstance = new ScriptStepInstanceDTO(); + stepInstance.setTaskInstanceId(record.get(t.TASK_INSTANCE_ID)); stepInstance.setStepInstanceId(record.get(t.STEP_INSTANCE_ID)); stepInstance.setScriptContent(record.get(t.SCRIPT_CONTENT)); stepInstance.setScriptType(ScriptTypeEnum.valOf(record.get(t.SCRIPT_TYPE).intValue())); @@ -332,23 +367,12 @@ private ScriptStepInstanceDTO extractScriptInfo(Record record) { } @Override - public FileStepInstanceDTO getFileStepInstance(long stepInstanceId) { - StepInstanceFile t = StepInstanceFile.STEP_INSTANCE_FILE; - Record record = CTX.select( - t.STEP_INSTANCE_ID, - t.FILE_SOURCE, - t.FILE_TARGET_PATH, - t.FILE_TARGET_NAME, - t.RESOLVED_FILE_TARGET_PATH, - t.FILE_UPLOAD_SPEED_LIMIT, - t.FILE_DOWNLOAD_SPEED_LIMIT, - t.FILE_DUPLICATE_HANDLE, - t.NOT_EXIST_PATH_HANDLER, - t.EXECUTION_TIMEOUT, - t.SYSTEM_ACCOUNT_ID, - t.SYSTEM_ACCOUNT - ).from(t) - .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)).fetchOne(); + public FileStepInstanceDTO getFileStepInstance(long taskInstanceId, long stepInstanceId) { + Record record = CTX.select(T_STEP_INSTANCE_FILE_ALL_FIELDS) + .from(T_STEP_INSTANCE_FILE) + .where(T_STEP_INSTANCE_FILE.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(T_STEP_INSTANCE_FILE.STEP_INSTANCE_ID.eq(stepInstanceId)) + .fetchOne(); return extractFileInfo(record); } @@ -358,6 +382,7 @@ private FileStepInstanceDTO extractFileInfo(Record record) { } StepInstanceFile t = StepInstanceFile.STEP_INSTANCE_FILE; FileStepInstanceDTO stepInstance = new FileStepInstanceDTO(); + stepInstance.setTaskInstanceId(record.get(t.TASK_INSTANCE_ID)); stepInstance.setStepInstanceId(record.get(t.STEP_INSTANCE_ID)); stepInstance.setTimeout(record.get(t.EXECUTION_TIMEOUT)); List fileSourceList = JsonUtils.fromJson(record.get(t.FILE_SOURCE), @@ -383,17 +408,12 @@ private FileStepInstanceDTO extractFileInfo(Record record) { } @Override - public ConfirmStepInstanceDTO getConfirmStepInstance(long stepInstanceId) { - StepInstanceConfirm t = StepInstanceConfirm.STEP_INSTANCE_CONFIRM; - Record record = CTX.select( - t.STEP_INSTANCE_ID, - t.CONFIRM_MESSAGE, - t.CONFIRM_REASON, - t.CONFIRM_USERS, - t.CONFIRM_ROLES, - t.NOTIFY_CHANNELS - ).from(t) - .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)).fetchOne(); + public ConfirmStepInstanceDTO getConfirmStepInstance(long taskInstanceId, long stepInstanceId) { + Record record = CTX.select(T_STEP_INSTANCE_CONFIRM_ALL_FIELDS) + .from(T_STEP_INSTANCE_CONFIRM) + .where(T_STEP_INSTANCE_CONFIRM.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(T_STEP_INSTANCE_CONFIRM.STEP_INSTANCE_ID.eq(stepInstanceId)) + .fetchOne(); return extractConfirmInfo(record); } @@ -403,6 +423,7 @@ private ConfirmStepInstanceDTO extractConfirmInfo(Record record) { } StepInstanceConfirm t = StepInstanceConfirm.STEP_INSTANCE_CONFIRM; ConfirmStepInstanceDTO stepInstanceDTO = new ConfirmStepInstanceDTO(); + stepInstanceDTO.setTaskInstanceId(record.get(t.TASK_INSTANCE_ID)); stepInstanceDTO.setStepInstanceId(record.get(t.STEP_INSTANCE_ID)); stepInstanceDTO.setConfirmMessage(record.get(t.CONFIRM_MESSAGE)); stepInstanceDTO.setConfirmReason(record.get(t.CONFIRM_REASON)); @@ -416,11 +437,12 @@ private ConfirmStepInstanceDTO extractConfirmInfo(Record record) { } @Override - public StepInstanceBaseDTO getStepInstanceBase(long stepInstanceId) { + public StepInstanceBaseDTO getStepInstanceBase(long taskInstanceId, long stepInstanceId) { Record record = CTX .select(T_STEP_INSTANCE_ALL_FIELDS) .from(T_STEP_INSTANCE) - .where(T_STEP_INSTANCE.ID.eq(stepInstanceId)) + .where(T_STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(T_STEP_INSTANCE.ID.eq(stepInstanceId)) .fetchOne(); return extractBaseInfo(record); } @@ -457,6 +479,16 @@ private StepInstanceBaseDTO extractBaseInfo(Record record) { return stepInstance; } + @Override + public StepInstanceBaseDTO getStepInstanceBase(long stepInstanceId) { + Record record = CTX + .select(T_STEP_INSTANCE_ALL_FIELDS) + .from(T_STEP_INSTANCE) + .where(T_STEP_INSTANCE.ID.eq(stepInstanceId)) + .fetchOne(); + return extractBaseInfo(record); + } + @Override public StepInstanceBaseDTO getFirstStepInstanceBase(long taskInstanceId) { Record record = CTX @@ -496,76 +528,106 @@ public List listStepInstanceBaseByTaskInstanceId(long taskI } @Override - public void resetStepStatus(long stepInstanceId) { + public void resetStepStatus(long taskInstanceId, long stepInstanceId) { StepInstance t = StepInstance.STEP_INSTANCE; - CTX.update(t).setNull(t.START_TIME).setNull(t.END_TIME).setNull(t.TOTAL_TIME) - .where(t.ID.eq(stepInstanceId)).execute(); + CTX.update(t) + .setNull(t.START_TIME) + .setNull(t.END_TIME) + .setNull(t.TOTAL_TIME) + .where(t.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(t.ID.eq(stepInstanceId)) + .execute(); } @Override - public void resetStepExecuteInfoForRetry(long stepInstanceId) { + public void resetStepExecuteInfoForRetry(long taskInstanceId, long stepInstanceId) { StepInstance t = StepInstance.STEP_INSTANCE; CTX.update(t) .set(t.STATUS, RunStatusEnum.RUNNING.getValue().byteValue()) .setNull(t.END_TIME) .setNull(t.TOTAL_TIME) - .where(t.ID.eq(stepInstanceId)).execute(); + .where(t.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(t.ID.eq(stepInstanceId)) + .execute(); } @Override - public void addStepExecuteCount(long stepInstanceId) { + public void addStepExecuteCount(long taskInstanceId, long stepInstanceId) { StepInstance t = StepInstance.STEP_INSTANCE; - CTX.update(t).set(t.EXECUTE_COUNT, t.EXECUTE_COUNT.plus(1)) - .where(t.ID.eq(stepInstanceId)).execute(); + CTX.update(t) + .set(t.EXECUTE_COUNT, t.EXECUTE_COUNT.plus(1)) + .where(t.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(t.ID.eq(stepInstanceId)) + .execute(); } @Override - public void updateStepStatus(long stepInstanceId, int status) { + public void updateStepStatus(long taskInstanceId, long stepInstanceId, int status) { StepInstance t = StepInstance.STEP_INSTANCE; - CTX.update(t).set(t.STATUS, JooqDataTypeUtil.toByte(status)) - .where(t.ID.eq(stepInstanceId)).execute(); + CTX.update(t) + .set(t.STATUS, JooqDataTypeUtil.toByte(status)) + .where(t.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(t.ID.eq(stepInstanceId)) + .execute(); } @Override - public void updateStepStartTime(long stepInstanceId, Long startTime) { + public void updateStepStartTime(long taskInstanceId, long stepInstanceId, Long startTime) { StepInstance t = StepInstance.STEP_INSTANCE; - CTX.update(t).set(t.START_TIME, startTime) - .where(t.ID.eq(stepInstanceId)).execute(); + CTX.update(t) + .set(t.START_TIME, startTime) + .where(t.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(t.ID.eq(stepInstanceId)) + .execute(); } @Override - public void updateStepStartTimeIfNull(long stepInstanceId, Long startTime) { + public void updateStepStartTimeIfNull(long taskInstanceId, long stepInstanceId, Long startTime) { StepInstance t = StepInstance.STEP_INSTANCE; - CTX.update(t).set(t.START_TIME, startTime) - .where(t.ID.eq(stepInstanceId)) + CTX.update(t) + .set(t.START_TIME, startTime) + .where(t.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(t.ID.eq(stepInstanceId)) .and(t.START_TIME.isNull()) .execute(); } @Override - public void updateStepEndTime(long stepInstanceId, Long endTime) { + public void updateStepEndTime(long taskInstanceId, long stepInstanceId, Long endTime) { StepInstance t = StepInstance.STEP_INSTANCE; - CTX.update(t).set(t.END_TIME, endTime) - .where(t.ID.eq(stepInstanceId)) + CTX.update(t) + .set(t.END_TIME, endTime) + .where(t.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(t.ID.eq(stepInstanceId)) .execute(); } @Override - public void addStepInstanceExecuteCount(long stepInstanceId) { + public void addStepInstanceExecuteCount(long taskInstanceId, long stepInstanceId) { StepInstance t = StepInstance.STEP_INSTANCE; - CTX.update(t).set(t.EXECUTE_COUNT, t.EXECUTE_COUNT.plus(1)) - .where(t.ID.eq(stepInstanceId)).execute(); + CTX.update(t) + .set(t.EXECUTE_COUNT, t.EXECUTE_COUNT.plus(1)) + .where(t.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(t.ID.eq(stepInstanceId)) + .execute(); } @Override - public void updateStepTotalTime(long stepInstanceId, long totalTime) { + public void updateStepTotalTime(long taskInstanceId, long stepInstanceId, long totalTime) { StepInstance t = StepInstance.STEP_INSTANCE; - CTX.update(t).set(t.TOTAL_TIME, totalTime) - .where(t.ID.eq(stepInstanceId)).execute(); + CTX.update(t) + .set(t.TOTAL_TIME, totalTime) + .where(t.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(t.ID.eq(stepInstanceId)) + .execute(); } @Override - public void updateStepExecutionInfo(long stepInstanceId, RunStatusEnum status, Long startTime, Long endTime, + public void updateStepExecutionInfo(long taskInstanceId, + long stepInstanceId, + RunStatusEnum status, + Long startTime, + Long endTime, Long totalTime) { StepInstance t = StepInstance.STEP_INSTANCE; UpdateSetMoreStep updateSetMoreStep = buildBasicUpdateSetMoreStep(status, @@ -573,11 +635,16 @@ public void updateStepExecutionInfo(long stepInstanceId, RunStatusEnum status, L if (updateSetMoreStep == null) { return; } - updateSetMoreStep.where(t.ID.eq(stepInstanceId)).execute(); + updateSetMoreStep + .where(t.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(t.ID.eq(stepInstanceId)) + .execute(); } - private UpdateSetMoreStep buildBasicUpdateSetMoreStep(RunStatusEnum status, Long startTime, - Long endTime, Long totalTime) { + private UpdateSetMoreStep buildBasicUpdateSetMoreStep(RunStatusEnum status, + Long startTime, + Long endTime, + Long totalTime) { StepInstance t = StepInstance.STEP_INSTANCE; UpdateSetMoreStep updateSetMoreStep = null; if (status != null) { @@ -608,44 +675,57 @@ private UpdateSetMoreStep buildBasicUpdateSetMoreStep(RunSta } @Override - public void updateResolvedScriptParam(long stepInstanceId, boolean isSecureParam, String resolvedScriptParam) { + public void updateResolvedScriptParam(long taskInstanceId, + long stepInstanceId, + boolean isSecureParam, + String resolvedScriptParam) { StepInstanceScript t = StepInstanceScript.STEP_INSTANCE_SCRIPT; CTX.update(t) .set(t.RESOLVED_SCRIPT_PARAM, sensitiveParamCryptoService.encryptParamIfNeeded( isSecureParam, resolvedScriptParam - )).where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) + )) + .where(t.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(t.STEP_INSTANCE_ID.eq(stepInstanceId)) .execute(); } @Override - public void updateResolvedSourceFile(long stepInstanceId, List resolvedFileSources) { + public void updateResolvedSourceFile(long taskInstanceId, + long stepInstanceId, + List resolvedFileSources) { StepInstanceFile t = StepInstanceFile.STEP_INSTANCE_FILE; - CTX.update(t).set(t.FILE_SOURCE, JsonUtils.toJson(resolvedFileSources)) - .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) + CTX.update(t) + .set(t.FILE_SOURCE, JsonUtils.toJson(resolvedFileSources)) + .where(t.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(t.STEP_INSTANCE_ID.eq(stepInstanceId)) .execute(); } @Override - public void updateResolvedTargetPath(long stepInstanceId, String resolvedTargetPath) { + public void updateResolvedTargetPath(long taskInstanceId, long stepInstanceId, String resolvedTargetPath) { StepInstanceFile t = StepInstanceFile.STEP_INSTANCE_FILE; - CTX.update(t).set(t.RESOLVED_FILE_TARGET_PATH, resolvedTargetPath) - .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) + CTX.update(t) + .set(t.RESOLVED_FILE_TARGET_PATH, resolvedTargetPath) + .where(t.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(t.STEP_INSTANCE_ID.eq(stepInstanceId)) .execute(); } @Override - public void updateConfirmReason(long stepInstanceId, String confirmReason) { + public void updateConfirmReason(long taskInstanceId, long stepInstanceId, String confirmReason) { StepInstanceConfirm t = StepInstanceConfirm.STEP_INSTANCE_CONFIRM; CTX.update(t).set(t.CONFIRM_REASON, confirmReason) - .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) + .where(t.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(t.STEP_INSTANCE_ID.eq(stepInstanceId)) .execute(); } @Override - public void updateStepOperator(long stepInstanceId, String operator) { + public void updateStepOperator(long taskInstanceId, long stepInstanceId, String operator) { StepInstance t = StepInstance.STEP_INSTANCE; CTX.update(t).set(t.OPERATOR, operator) - .where(t.ID.eq(stepInstanceId)) + .where(t.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(t.ID.eq(stepInstanceId)) .execute(); } @@ -693,262 +773,44 @@ public Long getTaskInstanceId(long appId, long stepInstanceId) { } @Override - public Byte getScriptTypeByStepInstanceId(long stepInstanceId) { - Result> records = CTX.select(TABLE_STEP_INSTANCE_SCRIPT.SCRIPT_TYPE) - .from(TABLE_STEP_INSTANCE_SCRIPT) - .where(TABLE_STEP_INSTANCE_SCRIPT.STEP_INSTANCE_ID.eq(stepInstanceId)) + public Byte getScriptTypeByStepInstanceId(long taskInstanceId, long stepInstanceId) { + Result> records = CTX.select(T_STEP_INSTANCE_SCRIPT.SCRIPT_TYPE) + .from(T_STEP_INSTANCE_SCRIPT) + .where(T_STEP_INSTANCE_SCRIPT.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(T_STEP_INSTANCE_SCRIPT.STEP_INSTANCE_ID.eq(stepInstanceId)) .limit(1) .fetch(); if (records.isEmpty()) { return null; } else { - return records.get(0).get(TABLE_STEP_INSTANCE_SCRIPT.SCRIPT_TYPE); - } - } - - public Integer countStepInstanceByConditions(Collection conditions) { - Record1 record = CTX.selectCount().from(T_STEP_INSTANCE) - .where(conditions).fetchOne(); - assert record != null; - return record.value1(); - } - - public Integer countStepInstanceScriptByConditions(Collection conditions) { - Record1 record = CTX.selectCount().from(TABLE_STEP_INSTANCE_SCRIPT) - .where(conditions).fetchOne(); - assert record != null; - return record.value1(); - } - - public Integer countStepInstanceFileByConditions(Collection conditions) { - Record1 record = CTX.selectCount().from(TABLE_STEP_INSTANCE_FILE) - .where(conditions).fetchOne(); - assert record != null; - return record.value1(); - } - - @Override - public Integer count(Long appId, List stepIdList, StepExecuteTypeEnum stepExecuteType, - ScriptTypeEnum scriptType, RunStatusEnum runStatus, Long fromTime, Long toTime) { - List stepInstanceConditions = genStepInstanceConditions(appId, stepIdList, stepExecuteType, - runStatus, fromTime, toTime); - if (StepExecuteTypeEnum.EXECUTE_SCRIPT == stepExecuteType && scriptType != null) { - int totalCount = 0; - int offset = 0; - int limit = 10000; - List stepInstanceIds; - do { - // 1.查stepInstanceId - stepInstanceIds = listStepInstanceIds(stepInstanceConditions, offset, limit); - if (stepInstanceIds == null || stepInstanceIds.isEmpty()) { - break; - } - // 2.分批统计脚本步骤实例 - List stepInstanceScriptConditions = new ArrayList<>(); - stepInstanceScriptConditions.add(TABLE_STEP_INSTANCE_SCRIPT.STEP_INSTANCE_ID.in(stepInstanceIds)); - stepInstanceScriptConditions.add(TABLE_STEP_INSTANCE_SCRIPT.SCRIPT_TYPE.eq(scriptType.getValue().byteValue())); - totalCount += countStepInstanceScriptByConditions(stepInstanceScriptConditions); - offset += limit; - } while (stepInstanceIds.size() == limit); - return totalCount; - } else { - return countStepInstanceByConditions(stepInstanceConditions); - } - } - - private List listStepInstanceIds(Collection conditions, int offset, int limit) { - Result> records = CTX.select(T_STEP_INSTANCE.ID) - .from(T_STEP_INSTANCE) - .where(conditions) - .limit(offset, limit) - .fetch(); - if (records.isEmpty()) { - return Collections.emptyList(); - } else { - return records.map(it -> it.get(T_STEP_INSTANCE.ID)); - } - } - - private List genStepInstanceConditions(Long appId, List stepIdList, - StepExecuteTypeEnum stepExecuteType, RunStatusEnum runStatus, - Long fromTime, Long toTime) { - List conditions = new ArrayList<>(); - if (appId != null) { - conditions.add(T_STEP_INSTANCE.APP_ID.eq(appId)); - } - if (stepIdList != null && !stepIdList.isEmpty()) { - conditions.add(T_STEP_INSTANCE.STEP_ID.in(stepIdList)); - } - if (stepExecuteType != null) { - conditions.add(T_STEP_INSTANCE.TYPE.eq(stepExecuteType.getValue().byteValue())); - } - if (runStatus != null) { - conditions.add(T_STEP_INSTANCE.STATUS.eq(runStatus.getValue().byteValue())); - } - if (fromTime != null) { - conditions.add(T_STEP_INSTANCE.CREATE_TIME.greaterOrEqual(fromTime)); - } - if (toTime != null) { - conditions.add(T_STEP_INSTANCE.CREATE_TIME.lessThan(toTime)); - } - return conditions; - } - - private List genStepInstanceFileConditions(Collection stepInstanceIds, - DuplicateHandlerEnum fileDupliateHandle, - Boolean fileDupliateHandleNull, - NotExistPathHandlerEnum notExistPathHandler, - Boolean notExistPathHandlerNull) { - List conditions = new ArrayList<>(); - if (stepInstanceIds != null) { - conditions.add(TABLE_STEP_INSTANCE_FILE.STEP_INSTANCE_ID.in(stepInstanceIds)); - } - if (fileDupliateHandle != null) { - conditions.add(TABLE_STEP_INSTANCE_FILE.FILE_DUPLICATE_HANDLE.eq((byte) fileDupliateHandle.getId())); - } else { - if (fileDupliateHandleNull != null) { - if (fileDupliateHandleNull) { - conditions.add(TABLE_STEP_INSTANCE_FILE.FILE_DUPLICATE_HANDLE.isNull()); - } else { - conditions.add(TABLE_STEP_INSTANCE_FILE.FILE_DUPLICATE_HANDLE.isNotNull()); - } - } - // fileDupliateHandle与fileDupliateHandleNull同时为null表示FILE_DUPLICATE_HANDLE字段不作任何筛选 + return records.get(0).get(T_STEP_INSTANCE_SCRIPT.SCRIPT_TYPE); } - if (notExistPathHandler != null) { - conditions.add(TABLE_STEP_INSTANCE_FILE.NOT_EXIST_PATH_HANDLER.eq(UByte.valueOf(notExistPathHandler.getValue()))); - } else { - if (notExistPathHandlerNull != null) { - if (notExistPathHandlerNull) { - conditions.add(TABLE_STEP_INSTANCE_FILE.NOT_EXIST_PATH_HANDLER.isNull()); - } else { - conditions.add(TABLE_STEP_INSTANCE_FILE.NOT_EXIST_PATH_HANDLER.isNotNull()); - } - } - // notExistPathHandler与notExistPathHandlerNull同时为null表示NOT_EXIST_PATH_HANDLER字段不作任何筛选 - } - return conditions; - } - - private List genConditions(Long appId, DuplicateHandlerEnum fileDupliateHandle, - Boolean fileDupliateHandleNull, NotExistPathHandlerEnum notExistPathHandler - , Boolean notExistPathHandlerNull, RunStatusEnum runStatus, Long fromTime, Long toTime) { - List conditions = new ArrayList<>(); - if (appId != null) { - conditions.add(T_STEP_INSTANCE.APP_ID.eq(appId)); - } - if (fileDupliateHandle != null) { - conditions.add(TABLE_STEP_INSTANCE_FILE.FILE_DUPLICATE_HANDLE.eq((byte) fileDupliateHandle.getId())); - } else { - if (fileDupliateHandleNull != null) { - if (fileDupliateHandleNull) { - conditions.add(TABLE_STEP_INSTANCE_FILE.FILE_DUPLICATE_HANDLE.isNull()); - } else { - conditions.add(TABLE_STEP_INSTANCE_FILE.FILE_DUPLICATE_HANDLE.isNotNull()); - } - } - // fileDupliateHandle与fileDupliateHandleNull同时为null表示FILE_DUPLICATE_HANDLE字段不作任何筛选 - } - if (notExistPathHandler != null) { - conditions.add(TABLE_STEP_INSTANCE_FILE.NOT_EXIST_PATH_HANDLER.eq(UByte.valueOf(notExistPathHandler.getValue()))); - } else { - if (notExistPathHandlerNull != null) { - if (notExistPathHandlerNull) { - conditions.add(TABLE_STEP_INSTANCE_FILE.NOT_EXIST_PATH_HANDLER.isNull()); - } else { - conditions.add(TABLE_STEP_INSTANCE_FILE.NOT_EXIST_PATH_HANDLER.isNotNull()); - } - } - // notExistPathHandler与notExistPathHandlerNull同时为null表示NOT_EXIST_PATH_HANDLER字段不作任何筛选 - } - if (runStatus != null) { - conditions.add(T_STEP_INSTANCE.STATUS.eq(runStatus.getValue().byteValue())); - } - if (fromTime != null) { - conditions.add(T_STEP_INSTANCE.CREATE_TIME.greaterOrEqual(fromTime)); - } - if (toTime != null) { - conditions.add(T_STEP_INSTANCE.CREATE_TIME.lessThan(toTime)); - } - return conditions; - } - - @Override - public List> listFastPushFileSource(Long appId, DuplicateHandlerEnum fileDupliateHandle, - Boolean fileDupliateHandleNull, - NotExistPathHandlerEnum notExistPathHandler, - Boolean notExistPathHandlerNull, RunStatusEnum runStatus, - Long fromTime, Long toTime) { - List conditions = genConditions(appId, fileDupliateHandle, fileDupliateHandleNull, - notExistPathHandler, notExistPathHandlerNull, runStatus, fromTime, toTime); - val query = - CTX.select(TABLE_STEP_INSTANCE_FILE.FILE_SOURCE).from(T_STEP_INSTANCE).join(TABLE_STEP_INSTANCE_FILE) - .on(T_STEP_INSTANCE.ID.eq(TABLE_STEP_INSTANCE_FILE.STEP_INSTANCE_ID)) - .where(conditions); - String sql = query.getSQL(ParamType.INLINED); - try { - Result> records = query.fetch(); - List> resultList = new ArrayList<>(); - records.forEach(record -> resultList.add(convertStringToFileSourceDTO((String) record.get(0)))); - return resultList; - } catch (Exception e) { - String msg = MessageFormatter.format( - "Fail to query:{}", - sql - ).getMessage(); - log.error(msg, e); - throw new RuntimeException("Fail to listFastPushFileSource", e); - } - } - - @Override - public Integer countFastPushFile(Long appId, DuplicateHandlerEnum fileDupliateHandle, - Boolean fileDupliateHandleNull, NotExistPathHandlerEnum notExistPathHandler, - Boolean notExistPathHandlerNull, RunStatusEnum runStatus, Long fromTime, - Long toTime) { - // 1.查stepInstanceId - int totalCount = 0; - int offset = 0; - int limit = 10000; - List stepInstanceIds; - do { - stepInstanceIds = listStepInstanceIds(genStepInstanceConditions(appId, null, null, null, fromTime, - toTime), offset, limit); - if (stepInstanceIds == null || stepInstanceIds.isEmpty()) { - break; - } - // 2.分批统计文件步骤实例 - List stepInstanceFileConditions = genStepInstanceFileConditions(stepInstanceIds, - fileDupliateHandle, fileDupliateHandleNull, notExistPathHandler, notExistPathHandlerNull); - totalCount += countStepInstanceFileByConditions(stepInstanceFileConditions); - offset += limit; - } while (stepInstanceIds.size() == limit); - return totalCount; - } - - private List convertStringToFileSourceDTO(String str) { - return JsonUtils.fromJson(str, new TypeReference>() { - }); } @Override - public void updateStepCurrentBatch(long stepInstanceId, int batch) { - CTX.update(T_STEP_INSTANCE).set(T_STEP_INSTANCE.BATCH, JooqDataTypeUtil.toShort(batch)) - .where(T_STEP_INSTANCE.ID.eq(stepInstanceId)) + public void updateStepCurrentBatch(long taskInstanceId, long stepInstanceId, int batch) { + CTX.update(T_STEP_INSTANCE) + .set(T_STEP_INSTANCE.BATCH, JooqDataTypeUtil.toShort(batch)) + .where(T_STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(T_STEP_INSTANCE.ID.eq(stepInstanceId)) .execute(); } @Override - public void updateStepCurrentExecuteCount(long stepInstanceId, int executeCount) { - CTX.update(T_STEP_INSTANCE).set(T_STEP_INSTANCE.EXECUTE_COUNT, executeCount) - .where(T_STEP_INSTANCE.ID.eq(stepInstanceId)) + public void updateStepCurrentExecuteCount(long taskInstanceId, long stepInstanceId, int executeCount) { + CTX.update(T_STEP_INSTANCE) + .set(T_STEP_INSTANCE.EXECUTE_COUNT, executeCount) + .where(T_STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(T_STEP_INSTANCE.ID.eq(stepInstanceId)) .execute(); } @Override - public void updateStepRollingConfigId(long stepInstanceId, long rollingConfigId) { - CTX.update(T_STEP_INSTANCE).set(T_STEP_INSTANCE.ROLLING_CONFIG_ID, rollingConfigId) - .where(T_STEP_INSTANCE.ID.eq(stepInstanceId)) + public void updateStepRollingConfigId(long taskInstanceId, long stepInstanceId, long rollingConfigId) { + CTX.update(T_STEP_INSTANCE) + .set(T_STEP_INSTANCE.ROLLING_CONFIG_ID, rollingConfigId) + .where(T_STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) + .and(T_STEP_INSTANCE.ID.eq(stepInstanceId)) .execute(); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/TaskEvictPolicyExecutor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/TaskEvictPolicyExecutor.java index 81da35dbca..e400c4ec4d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/TaskEvictPolicyExecutor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/TaskEvictPolicyExecutor.java @@ -85,6 +85,7 @@ public void updateEvictedTaskStatus(TaskInstanceDTO taskInstance, StepInstanceBa stepInstance.getTotalTime() ); stepInstanceService.updateStepExecutionInfo( + taskInstanceId, stepInstance.getId(), RunStatusEnum.ABANDONED, null, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskStartCommand.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskStartCommand.java index ed873fe357..3bbd92acb2 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskStartCommand.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskStartCommand.java @@ -215,8 +215,17 @@ private boolean startGseTaskIfNotAvailable(StopWatch watch) { log.error("[{}] Start gse task fail, response: {}", this.gseTaskInfo, gseTaskResponse); handleStartGseTaskError(gseTaskResponse); gseTasksExceptionCounter.increment(); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.refreshStep(stepInstanceId, - EventSource.buildGseTaskEventSource(stepInstanceId, executeCount, batch, gseTask.getId()))); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.refreshStep( + taskInstanceId, + stepInstanceId, + EventSource.buildGseTaskEventSource( + taskInstanceId, + stepInstanceId, + executeCount, + batch, + gseTask.getId())) + ); watch.stop(); return false; } else { @@ -329,8 +338,17 @@ protected Map buildStringGlobalVarKV(StepInstanceVariableValuesD private void finishGseTask(RunStatusEnum gseTaskStatus) { updateGseTaskExecutionInfo(null, gseTaskStatus, DateUtils.currentTimeMillis()); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.refreshStep(stepInstanceId, - EventSource.buildGseTaskEventSource(stepInstanceId, executeCount, batch, gseTask.getId()))); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.refreshStep( + taskInstanceId, + stepInstanceId, + EventSource.buildGseTaskEventSource( + taskInstanceId, + stepInstanceId, + executeCount, + batch, + gseTask.getId() + ))); } /** diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStartCommand.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStartCommand.java index a5009c57d3..eecb5d007e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStartCommand.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStartCommand.java @@ -204,8 +204,8 @@ private void resolveFileSource() { // 解析源文件路径中的全局变量 resolveVariableForSourceFilePath(fileSourceList, buildStringGlobalVarKV(stepInputVariables)); - stepInstanceService.updateResolvedSourceFile(stepInstance.getId(), - stepInstance.getFileSourceList()); + stepInstanceService.updateResolvedSourceFile(stepInstance.getTaskInstanceId(), + stepInstance.getId(), stepInstance.getFileSourceList()); } } @@ -265,7 +265,8 @@ private void resolvedTargetPathWithVariable() { resolvedTargetPath = MacroUtil.resolveDateWithStrfTime(resolvedTargetPath); stepInstance.setResolvedFileTargetPath(resolvedTargetPath); if (!resolvedTargetPath.equals(stepInstance.getFileTargetPath())) { - stepInstanceService.updateResolvedTargetPath(stepInstance.getId(), resolvedTargetPath); + stepInstanceService.updateResolvedTargetPath(stepInstance.getTaskInstanceId(), + stepInstance.getId(), resolvedTargetPath); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/GseTaskManager.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/GseTaskManager.java index eaaa5e2640..58b5d455ef 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/GseTaskManager.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/GseTaskManager.java @@ -209,7 +209,8 @@ public void startTask(GseTaskDTO gseTask, String requestId) { watch.stop(); watch.start("loadTaskAndCheck"); - StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail(stepInstanceId); + StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail( + gseTask.getTaskInstanceId(), stepInstanceId); TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(stepInstance.getTaskInstanceId()); // 如果任务应当被驱逐，直接置为被丢弃状态 @@ -226,9 +227,16 @@ public void startTask(GseTaskDTO gseTask, String requestId) { + "stepInstanceId:{}", taskInstance.getId(), stepInstance.getId()); gseTask.setStatus(RunStatusEnum.STOP_SUCCESS.getValue()); gseTaskService.updateGseTask(gseTask); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.refreshStep(stepInstanceId, - EventSource.buildGseTaskEventSource(stepInstanceId, stepInstance.getExecuteCount(), - stepInstance.getBatch(), gseTask.getId()))); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.refreshStep( + gseTask.getTaskInstanceId(), + stepInstanceId, + EventSource.buildGseTaskEventSource( + gseTask.getTaskInstanceId(), + stepInstanceId, + stepInstance.getExecuteCount(), + stepInstance.getBatch(), + gseTask.getId()))); watch.stop(); return; } @@ -431,7 +439,8 @@ public void stopTask(GseTaskDTO gseTask) { GseTaskCommand stopCommand; try { watch.start("loadTaskAndCheck"); - StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail(stepInstanceId); + StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail( + gseTask.getTaskInstanceId(), stepInstanceId); TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(stepInstance.getTaskInstanceId()); watch.stop(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStartCommand.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStartCommand.java index 6e3ac2f742..de7068fbc7 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStartCommand.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStartCommand.java @@ -248,6 +248,7 @@ private void updateResolvedScriptParamIfNecessary(String originParam, String res // 只有存在变量解析之后才需要update if (!resolvedScriptParam.equals(originParam)) { stepInstanceService.updateResolvedScriptParam( + stepInstance.getTaskInstanceId(), stepInstance.getId(), stepInstance.isSecureParam(), resolvedScriptParam diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ConfirmStepEventHandler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ConfirmStepEventHandler.java index 5c701fa4d3..4a19136296 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ConfirmStepEventHandler.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ConfirmStepEventHandler.java @@ -108,8 +108,8 @@ private void confirmStepTerminate(StepInstanceDTO stepInstance) { null, endTime, taskTotalTime); long stepTotalTime = TaskCostCalculator.calculate(stepInstance.getStartTime(), endTime, stepInstance.getTotalTime()); - stepInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.CONFIRM_TERMINATED, null, - endTime, stepTotalTime); + stepInstanceService.updateStepExecutionInfo(stepInstance.getTaskInstanceId(), stepInstanceId, + RunStatusEnum.CONFIRM_TERMINATED, null, endTime, stepTotalTime); } else { log.warn("Unsupported step instance status for confirm step terminate action, stepInstanceId:{}, " + "status:{}", stepInstanceId, stepInstance.getStatus()); @@ -147,7 +147,7 @@ private void executeConfirmStep(StepInstanceBaseDTO stepInstance) { stepOperator = taskInstance.getOperator(); stepInstance.setOperator(stepOperator); } - stepInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.WAITING_USER, + stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, RunStatusEnum.WAITING_USER, System.currentTimeMillis(), null, null); taskInstanceService.updateTaskStatus(taskInstanceId, RunStatusEnum.WAITING_USER.getValue()); notifyService.asyncSendMQConfirmNotification(taskInstance, stepInstance); @@ -169,10 +169,10 @@ private void confirmStepContinue(StepInstanceBaseDTO stepInstance) { long totalTime = TaskCostCalculator.calculate(stepInstance.getStartTime(), endTime, stepInstance.getTotalTime()); // 人工确认通过，该步骤状态标识为成功；终止成功的步骤保持状态不变 - stepInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.SUCCESS, null, endTime, - totalTime); + stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, RunStatusEnum.SUCCESS, + null, endTime, totalTime); taskExecuteMQEventDispatcher.dispatchJobEvent( - JobEvent.refreshJob(taskInstanceId, EventSource.buildStepEventSource(stepInstanceId))); + JobEvent.refreshJob(taskInstanceId, EventSource.buildStepEventSource(taskInstanceId, stepInstanceId))); } else { log.warn("Unsupported step instance status for confirm-step-continue step action, stepInstanceId:{}, " + "status:{}", stepInstanceId, stepInstance.getStatus()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java index 7a3e2b8958..6b550035e8 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java @@ -154,6 +154,7 @@ public void handleEvent(StepEvent stepEvent, } private void startStep(StepEvent stepEvent, StepInstanceDTO stepInstance) { + long taskInstanceId = stepInstance.getTaskInstanceId(); long stepInstanceId = stepInstance.getId(); boolean isRollingStep = stepInstance.isRollingStep(); if (isRollingStep) { @@ -179,7 +180,7 @@ private void startStep(StepEvent stepEvent, StepInstanceDTO stepInstance) { rollingConfig = rollingConfigService.getRollingConfig(stepInstance.getRollingConfigId()); log.info("Rolling config: {}", rollingConfig); // 更新滚动进度 - stepInstanceService.updateStepCurrentBatch(stepInstanceId, stepInstance.getBatch()); + stepInstanceService.updateStepCurrentBatch(taskInstanceId, stepInstanceId, stepInstance.getBatch()); // 初始化步骤滚动任务 saveInitialStepInstanceRollingTask(stepInstance); } @@ -187,7 +188,7 @@ private void startStep(StepEvent stepEvent, StepInstanceDTO stepInstance) { Long gseTaskId = saveInitialGseTask(stepInstance); saveExecuteObjectTasksForStartStep(gseTaskId, stepInstance, rollingConfig); - stepInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.RUNNING, + stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, RunStatusEnum.RUNNING, stepInstance.getStartTime() == null ? DateUtils.currentTimeMillis() : null, null, null); if (isRollingStep) { stepInstanceRollingTaskService.updateRollingTask(stepInstanceId, stepInstance.getExecuteCount(), @@ -209,15 +210,26 @@ private void startStep(StepEvent stepEvent, StepInstanceDTO stepInstance) { */ private void startGseTask(StepInstanceDTO stepInstance, Long gseTaskId) { if (stepInstance.isScriptStep()) { - taskExecuteMQEventDispatcher.dispatchGseTaskEvent(GseTaskEvent.startGseTask( - stepInstance.getId(), stepInstance.getExecuteCount(), stepInstance.getBatch(), gseTaskId, null)); + taskExecuteMQEventDispatcher.dispatchGseTaskEvent( + GseTaskEvent.startGseTask( + stepInstance.getTaskInstanceId(), + stepInstance.getId(), + stepInstance.getExecuteCount(), + stepInstance.getBatch(), + gseTaskId, + null)); } else if (stepInstance.isFileStep()) { if (filePrepareService.needToPrepareSourceFilesForGseTask(stepInstance)) { filePrepareService.prepareFileForGseTask(stepInstance); } else { - taskExecuteMQEventDispatcher.dispatchGseTaskEvent(GseTaskEvent.startGseTask( - stepInstance.getId(), stepInstance.getExecuteCount(), stepInstance.getBatch(), gseTaskId, - null)); + taskExecuteMQEventDispatcher.dispatchGseTaskEvent( + GseTaskEvent.startGseTask( + stepInstance.getTaskInstanceId(), + stepInstance.getId(), + stepInstance.getExecuteCount(), + stepInstance.getBatch(), + gseTaskId, + null)); } } } @@ -380,10 +392,15 @@ private void nextStep(StepInstanceDTO stepInstance) { long totalTime = TaskCostCalculator.calculate(stepInstance.getStartTime(), endTime, stepInstance.getTotalTime()); // 终止成功，进入下一步，该步骤设置为“跳过” - stepInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.SKIPPED, null, endTime, + stepInstanceService.updateStepExecutionInfo( + taskInstanceId, + stepInstanceId, + RunStatusEnum.SKIPPED, + null, + endTime, totalTime); taskExecuteMQEventDispatcher.dispatchJobEvent( - JobEvent.refreshJob(taskInstanceId, EventSource.buildStepEventSource(stepInstanceId))); + JobEvent.refreshJob(taskInstanceId, EventSource.buildStepEventSource(taskInstanceId, stepInstanceId))); } else { log.warn("Unsupported step instance status for next step action, stepInstanceId:{}, status:{}", stepInstanceId, stepInstance.getStatus()); @@ -399,6 +416,7 @@ private void ignoreError(StepInstanceDTO stepInstance) { return; } + long taskInstanceId = stepInstance.getTaskInstanceId(); long stepInstanceId = stepInstance.getId(); boolean isRollingStep = stepInstance.isRollingStep(); if (isRollingStep) { @@ -408,7 +426,8 @@ private void ignoreError(StepInstanceDTO stepInstance) { log.info("Retry-fail for step, stepInstanceId={}", stepInstanceId); } - stepInstanceService.updateStepStatus(stepInstance.getId(), RunStatusEnum.IGNORE_ERROR.getValue()); + stepInstanceService.updateStepStatus(taskInstanceId, stepInstance.getId(), + RunStatusEnum.IGNORE_ERROR.getValue()); taskInstanceService.resetTaskExecuteInfoForRetry(stepInstance.getTaskInstanceId()); if (isRollingStep) { StepInstanceRollingTaskDTO stepInstanceRollingTask = @@ -422,7 +441,7 @@ private void ignoreError(StepInstanceDTO stepInstance) { taskExecuteMQEventDispatcher.dispatchJobEvent( JobEvent.refreshJob(stepInstance.getTaskInstanceId(), - EventSource.buildStepEventSource(stepInstance.getId()))); + EventSource.buildStepEventSource(taskInstanceId, stepInstance.getId()))); } @@ -442,13 +461,13 @@ private void skipStep(StepInstanceDTO stepInstance) { // 只有当步骤状态为'终止中'时可以跳过步骤 if (RunStatusEnum.STOPPING == stepStatus) { long now = DateUtils.currentTimeMillis(); - stepInstanceService.updateStepStartTimeIfNull(stepInstanceId, now); - stepInstanceService.updateStepStatus(stepInstanceId, RunStatusEnum.SKIPPED.getValue()); - stepInstanceService.updateStepEndTime(stepInstanceId, now); + stepInstanceService.updateStepStartTimeIfNull(taskInstanceId, stepInstanceId, now); + stepInstanceService.updateStepStatus(taskInstanceId, stepInstanceId, RunStatusEnum.SKIPPED.getValue()); + stepInstanceService.updateStepEndTime(taskInstanceId, stepInstanceId, now); taskInstanceService.updateTaskStatus(taskInstanceId, RunStatusEnum.RUNNING.getValue()); taskExecuteMQEventDispatcher.dispatchJobEvent( - JobEvent.refreshJob(taskInstanceId, EventSource.buildStepEventSource(stepInstanceId))); + JobEvent.refreshJob(taskInstanceId, EventSource.buildStepEventSource(taskInstanceId, stepInstanceId))); } else { log.warn("Unsupported step instance run status for skipping step, stepInstanceId={}, status={}", stepInstanceId, stepStatus); @@ -456,6 +475,7 @@ private void skipStep(StepInstanceDTO stepInstance) { } private void stopStep(StepInstanceDTO stepInstance) { + long taskInstanceId = stepInstance.getTaskInstanceId(); long stepInstanceId = stepInstance.getId(); log.info("Force stop step, stepInstanceId={}", stepInstanceId); @@ -464,14 +484,19 @@ private void stopStep(StepInstanceDTO stepInstance) { log.info("Step status is WAITING_USER, set step status stop_success directly!"); // 等待用户的步骤可以直接结束 long endTime = DateUtils.currentTimeMillis(); - stepInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.STOP_SUCCESS, - null, endTime, TaskCostCalculator.calculate(stepInstance.getStartTime(), endTime, - stepInstance.getTotalTime())); + stepInstanceService.updateStepExecutionInfo( + taskInstanceId, + stepInstanceId, + RunStatusEnum.STOP_SUCCESS, + null, + endTime, + TaskCostCalculator.calculate(stepInstance.getStartTime(), endTime, stepInstance.getTotalTime())); + taskExecuteMQEventDispatcher.dispatchJobEvent(JobEvent.refreshJob(stepInstance.getTaskInstanceId(), - EventSource.buildStepEventSource(stepInstanceId))); + EventSource.buildStepEventSource(taskInstanceId, stepInstanceId))); } else if (stepStatus == RunStatusEnum.RUNNING) { // 正在运行中的任务无法立即结束，需要等待任务调度引擎检测到停止状态;这里只需要处理设置步骤状态即可 - stepInstanceService.updateStepStatus(stepInstanceId, RunStatusEnum.STOPPING.getValue()); + stepInstanceService.updateStepStatus(taskInstanceId, stepInstanceId, RunStatusEnum.STOPPING.getValue()); } } @@ -486,7 +511,8 @@ private void continueGseFileStep(StepInstanceDTO stepInstance) { GseTaskDTO gseTask = gseTaskService.getGseTask(stepInstance.getId(), stepInstance.getExecuteCount(), stepInstance.getBatch()); taskExecuteMQEventDispatcher.dispatchGseTaskEvent(GseTaskEvent.startGseTask( - gseTask.getStepInstanceId(), gseTask.getExecuteCount(), gseTask.getBatch(), gseTask.getId(), null)); + stepInstance.getTaskInstanceId(), gseTask.getStepInstanceId(), gseTask.getExecuteCount(), + gseTask.getBatch(), gseTask.getId(), null)); } /** @@ -629,7 +655,7 @@ private void clearStep(StepInstanceDTO stepInstance) { // 当前仅有文件分发类步骤需要清理中间文件 if (stepInstance.isFileStep()) { log.info("Clear file step, stepInstanceId={}", stepInstance.getId()); - filePrepareService.clearPreparedTmpFile(stepInstance.getId()); + filePrepareService.clearPreparedTmpFile(stepInstance.getTaskInstanceId(), stepInstance.getId()); } } @@ -642,7 +668,7 @@ private void resetExecutionInfoForRetry(StepInstanceDTO stepInstance) { long stepInstanceId = stepInstance.getId(); long taskInstanceId = stepInstance.getTaskInstanceId(); - stepInstanceService.resetStepExecuteInfoForRetry(stepInstanceId); + stepInstanceService.resetStepExecuteInfoForRetry(taskInstanceId, stepInstanceId); taskInstanceService.resetTaskExecuteInfoForRetry(taskInstanceId); } @@ -683,10 +709,12 @@ private void refreshStep(StepEvent stepEvent, StepInstanceDTO stepInstance) { // 更新作业状态 taskExecuteMQEventDispatcher.dispatchJobEvent( - JobEvent.refreshJob(stepInstance.getTaskInstanceId(), EventSource.buildStepEventSource(stepInstanceId))); + JobEvent.refreshJob(stepInstance.getTaskInstanceId(), + EventSource.buildStepEventSource(stepInstance.getTaskInstanceId(), stepInstanceId))); } private void onSuccess(StepInstanceDTO stepInstance) { + long taskInstanceId = stepInstance.getTaskInstanceId(); long stepInstanceId = stepInstance.getId(); long endTime = System.currentTimeMillis(); long startTime = stepInstance.getStartTime(); @@ -700,16 +728,16 @@ private void onSuccess(StepInstanceDTO stepInstance) { int totalBatch = rollingConfig.getConfigDetail().getTotalBatch(); boolean isLastBatch = totalBatch == stepInstance.getBatch(); if (isLastBatch) { - stepInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.SUCCESS, + stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, RunStatusEnum.SUCCESS, startTime, endTime, totalTime); // 步骤执行成功后清理产生的临时文件 clearStep(stepInstance); } else { - stepInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.ROLLING_WAITING, - startTime, endTime, totalTime); + stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, + RunStatusEnum.ROLLING_WAITING, startTime, endTime, totalTime); } } else { - stepInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.SUCCESS, + stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, RunStatusEnum.SUCCESS, startTime, endTime, totalTime); // 步骤执行成功后清理产生的临时文件 clearStep(stepInstance); @@ -717,13 +745,14 @@ private void onSuccess(StepInstanceDTO stepInstance) { } private void onFail(StepInstanceDTO stepInstance) { + long taskInstanceId = stepInstance.getTaskInstanceId(); long stepInstanceId = stepInstance.getId(); long endTime = System.currentTimeMillis(); long startTime = stepInstance.getStartTime(); long totalTime = endTime - startTime; if (stepInstance.isIgnoreError()) { log.info("Ignore error for step: {}", stepInstanceId); - stepInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.IGNORE_ERROR, + stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, RunStatusEnum.IGNORE_ERROR, startTime, endTime, totalTime); if (stepInstance.isRollingStep()) { finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), @@ -741,26 +770,27 @@ private void onFail(StepInstanceDTO stepInstance) { log.info("Ignore error for rolling step, rollingMode: {}", rollingMode); finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), RunStatusEnum.IGNORE_ERROR); - stepInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.IGNORE_ERROR, - startTime, endTime, totalTime); + stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, + RunStatusEnum.IGNORE_ERROR, startTime, endTime, totalTime); break; case PAUSE_IF_FAIL: case MANUAL: finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), RunStatusEnum.FAIL); - stepInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.FAIL, + stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, RunStatusEnum.FAIL, startTime, endTime, totalTime); break; default: log.error("Invalid rolling mode: {}", rollingMode); } } else { - stepInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.FAIL, + stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, RunStatusEnum.FAIL, startTime, endTime, totalTime); } } private void onStopSuccess(StepInstanceDTO stepInstance) { + long taskInstanceId = stepInstance.getTaskInstanceId(); long stepInstanceId = stepInstance.getId(); long endTime = System.currentTimeMillis(); long startTime = stepInstance.getStartTime(); @@ -768,7 +798,7 @@ private void onStopSuccess(StepInstanceDTO stepInstance) { RunStatusEnum stepStatus = stepInstance.getStatus(); if (stepStatus == RunStatusEnum.STOPPING || stepStatus == RunStatusEnum.RUNNING) { - stepInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.STOP_SUCCESS, + stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, RunStatusEnum.STOP_SUCCESS, startTime, endTime, totalTime); if (stepInstance.isRollingStep()) { finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), @@ -794,6 +824,7 @@ private void finishStep(StepInstanceDTO stepInstance, RunStatusEnum status) { long totalTime = TaskCostCalculator.calculate(stepInstance.getStartTime(), endTime, stepInstance.getTotalTime()); stepInstanceService.updateStepExecutionInfo( + stepInstance.getTaskInstanceId(), stepInstance.getId(), status, null, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java index f7889522fd..e6aa1763de 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java @@ -99,9 +99,6 @@ public void handleEvent(JobEvent jobEvent) { case STOP: stopJob(taskInstance); break; - case RESTART: - restartJob(taskInstance); - break; case REFRESH: refreshJob(taskInstance); break; @@ -148,42 +145,14 @@ private void stopJob(TaskInstanceDTO taskInstance) { if (RunStatusEnum.RUNNING == taskStatus || RunStatusEnum.WAITING_USER == taskStatus) { taskInstanceService.updateTaskStatus(jobInstanceId, RunStatusEnum.STOPPING.getValue()); long currentStepInstanceId = taskInstance.getCurrentStepInstanceId(); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.stopStep(currentStepInstanceId)); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.stopStep(jobInstanceId, currentStepInstanceId)); } else { log.warn("Unsupported job instance run status for stop task, jobInstanceId={}, status={}", jobInstanceId, taskInstance.getStatus()); } } - /** - * 重头执行作业 - * - * @param taskInstance 作业实例 - */ - private void restartJob(TaskInstanceDTO taskInstance) { - long jobInstanceId = taskInstance.getId(); - RunStatusEnum taskStatus = taskInstance.getStatus(); - // 验证作业状态，只有“执行失败”和“等待用户”的作业可以重头执行 - if (RunStatusEnum.WAITING_USER == taskStatus || RunStatusEnum.FAIL == taskStatus) { - - // 重置作业状态 - taskInstanceService.resetTaskStatus(jobInstanceId); - stepInstanceService.addStepInstanceExecuteCount(jobInstanceId); - - // 重置作业下步骤的状态、开始时间和结束时间等。 - List stepInstanceIdList = stepInstanceService.getTaskStepIdList(jobInstanceId); - for (long stepInstanceId : stepInstanceIdList) { - stepInstanceService.updateStepStatus(stepInstanceId, RunStatusEnum.BLANK.getValue()); - stepInstanceService.resetStepStatus(stepInstanceId); - } - - taskExecuteMQEventDispatcher.dispatchJobEvent(JobEvent.startJob(jobInstanceId)); - } else { - log.warn("Unsupported job instance run status for restart task, jobInstanceId={}, status={}", - jobInstanceId, taskInstance.getStatus()); - } - } - /** * 作业状态流转 * @@ -194,7 +163,8 @@ private void refreshJob(TaskInstanceDTO taskInstance) { RunStatusEnum taskStatus = taskInstance.getStatus(); long currentStepInstanceId = taskInstance.getCurrentStepInstanceId(); - StepInstanceBaseDTO currentStepInstance = stepInstanceService.getBaseStepInstance(currentStepInstanceId); + StepInstanceBaseDTO currentStepInstance = stepInstanceService.getBaseStepInstance( + jobInstanceId, currentStepInstanceId); RunStatusEnum stepStatus = currentStepInstance.getStatus(); // 验证作业状态，只有'正在执行'、'强制终止中'的作业可以刷新状态进入下一步或者结束 @@ -242,7 +212,7 @@ private void nextStep(TaskInstanceDTO taskInstance, StepInstanceBaseDTO currentS && rollingConfig.isFirstRollingStep(nextStepInstance.getId())) { log.info("Manual mode for rolling step[{}], pause and wait for user confirmation", nextStepInstance.getId()); - stepInstanceService.updateStepStatus(nextStepInstance.getId(), + stepInstanceService.updateStepStatus(taskInstance.getId(), nextStepInstance.getId(), RunStatusEnum.WAITING_USER.getValue()); taskInstanceService.updateTaskStatus(taskInstance.getId(), RunStatusEnum.WAITING_USER.getValue()); } else { @@ -320,7 +290,8 @@ private StepInstanceBaseDTO getNextStepInstance(TaskInstanceDTO taskInstance, currentStepInstance.getId()); } if (nextRollingStepInstanceId != null) { - nextStepInstance = stepInstanceService.getBaseStepInstance(nextRollingStepInstanceId); + nextStepInstance = stepInstanceService.getBaseStepInstance(taskInstance.getId(), + nextRollingStepInstanceId); } } } else { @@ -342,12 +313,16 @@ private Long getNextStepInstanceId(List stepInstanceIdList, long currentSt private void startStep(StepInstanceBaseDTO stepInstance) { taskInstanceService.updateTaskCurrentStepId(stepInstance.getTaskInstanceId(), stepInstance.getId()); if (stepInstance.isRollingStep()) { - stepInstanceService.updateStepStatus(stepInstance.getId(), RunStatusEnum.ROLLING_WAITING.getValue()); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.startStep(stepInstance.getId(), + stepInstanceService.updateStepStatus(stepInstance.getTaskInstanceId(), + stepInstance.getId(), RunStatusEnum.ROLLING_WAITING.getValue()); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.startStep(stepInstance.getTaskInstanceId(), stepInstance.getId(), stepInstance.getBatch() + 1)); } else { - stepInstanceService.updateStepStatus(stepInstance.getId(), RunStatusEnum.BLANK.getValue()); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.startStep(stepInstance.getId(), null)); + stepInstanceService.updateStepStatus(stepInstance.getTaskInstanceId(), + stepInstance.getId(), RunStatusEnum.BLANK.getValue()); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.startStep(stepInstance.getTaskInstanceId(), stepInstance.getId(), null)); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ResultHandleResumeListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ResultHandleResumeListener.java index c0e1f66299..c7bb452a2a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ResultHandleResumeListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ResultHandleResumeListener.java @@ -140,7 +140,8 @@ public void handleEvent(ResultHandleTaskResumeEvent event) { : UUID.randomUUID().toString(); try { - StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail(stepInstanceId); + StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail( + event.getJobInstanceId(), stepInstanceId); TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(stepInstance.getTaskInstanceId()); if (!checkIsTaskResumeable(stepInstance, gseTask)) { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepListener.java index caffa87264..bb9203efdd 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepListener.java @@ -60,7 +60,8 @@ public void handleEvent(StepEvent stepEvent) { log.info("Handle step event: {}, duration: {}ms", stepEvent, stepEvent.duration()); long stepInstanceId = stepEvent.getStepInstanceId(); try { - StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail(stepInstanceId); + StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail( + stepEvent.getJobInstanceId(), stepInstanceId); dispatchEvent(stepEvent, stepInstance); } catch (Throwable e) { String errorMsg = "Handling step event error,stepInstanceId:" + stepInstanceId; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/EventSource.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/EventSource.java index 475c3fb06e..8fe7ec1131 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/EventSource.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/EventSource.java @@ -67,11 +67,13 @@ public class EventSource { /** * 构造事件源 - 步骤 * + * @param jobInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @return 执行引擎事件源 */ - public static EventSource buildStepEventSource(long stepInstanceId) { + public static EventSource buildStepEventSource(Long jobInstanceId, long stepInstanceId) { EventSource eventSource = new EventSource(); + eventSource.setJobInstanceId(jobInstanceId); eventSource.setStepInstanceId(stepInstanceId); eventSource.setSourceType(EventSourceTypeEnum.STEP); return eventSource; @@ -80,17 +82,20 @@ public static EventSource buildStepEventSource(long stepInstanceId) { /** * 构造事件源 - GSE 任务 * + * @param jobInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 步骤执行次数 * @param batch 滚动执行批次 * @param gseTaskId GSE 任务ID * @return 执行引擎事件源 */ - public static EventSource buildGseTaskEventSource(Long stepInstanceId, + public static EventSource buildGseTaskEventSource(Long jobInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, Long gseTaskId) { EventSource eventSource = new EventSource(); + eventSource.setJobInstanceId(jobInstanceId); eventSource.setStepInstanceId(stepInstanceId); eventSource.setExecuteCount(executeCount); eventSource.setBatch(batch); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/GseTaskEvent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/GseTaskEvent.java index dbd4494e00..5ad7072cf4 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/GseTaskEvent.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/GseTaskEvent.java @@ -49,6 +49,10 @@ public class GseTaskEvent extends Event { * @see com.tencent.bk.job.execute.engine.consts.GseTaskActionEnum */ private int action; + /** + * 步骤实例ID + */ + private Long jobInstanceId; /** * 步骤实例ID */ @@ -73,6 +77,7 @@ public class GseTaskEvent extends Event { /** * 构造启动 GSE 任务 事件 * + * @param jobInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动批次 @@ -80,12 +85,13 @@ public class GseTaskEvent extends Event { * @param requestId 请求ID,防止重复下发任务 * @return 事件 */ - public static GseTaskEvent startGseTask(Long stepInstanceId, + public static GseTaskEvent startGseTask(long jobInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, Long gseTaskId, String requestId) { - GseTaskEvent gseTaskEvent = buildGseTaskEvent(stepInstanceId, executeCount, batch, gseTaskId); + GseTaskEvent gseTaskEvent = buildGseTaskEvent(jobInstanceId, stepInstanceId, executeCount, batch, gseTaskId); if (StringUtils.isNotEmpty(requestId)) { gseTaskEvent.setRequestId(requestId); } else { @@ -98,27 +104,31 @@ public static GseTaskEvent startGseTask(Long stepInstanceId, /** * 构造停止 GSE 任务 事件 * + * @param jobInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动批次 * @param gseTaskId GSE任务ID * @return 事件 */ - public static GseTaskEvent stopGseTask(Long stepInstanceId, + public static GseTaskEvent stopGseTask(Long jobInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, Long gseTaskId) { - GseTaskEvent gseTaskEvent = buildGseTaskEvent(stepInstanceId, executeCount, batch, gseTaskId); + GseTaskEvent gseTaskEvent = buildGseTaskEvent(jobInstanceId, stepInstanceId, executeCount, batch, gseTaskId); gseTaskEvent.setAction(GseTaskActionEnum.STOP.getValue()); return gseTaskEvent; } - private static GseTaskEvent buildGseTaskEvent(Long stepInstanceId, + private static GseTaskEvent buildGseTaskEvent(Long jobInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, Long gseTaskId) { GseTaskEvent gseTaskEvent = new GseTaskEvent(); gseTaskEvent.setGseTaskId(gseTaskId); + gseTaskEvent.setJobInstanceId(jobInstanceId); gseTaskEvent.setStepInstanceId(stepInstanceId); gseTaskEvent.setExecuteCount(executeCount); gseTaskEvent.setBatch(batch); @@ -133,6 +143,10 @@ public String toString() { .add("action=" + action) .add("actionDesc=" + GseTaskActionEnum.valueOf(action)) .add("gseTaskId=" + gseTaskId) + .add("jobInstanceId=" + jobInstanceId) + .add("stepInstanceId=" + stepInstanceId) + .add("executeCount=" + executeCount) + .add("batch=" + batch) .add("requestId='" + requestId + "'") .add("source=" + source) .add("time=" + time) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/ResultHandleTaskResumeEvent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/ResultHandleTaskResumeEvent.java index 8ec4909866..d52c694ee1 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/ResultHandleTaskResumeEvent.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/ResultHandleTaskResumeEvent.java @@ -41,6 +41,10 @@ @NoArgsConstructor @JsonInclude(JsonInclude.Include.NON_NULL) public class ResultHandleTaskResumeEvent extends Event { + /** + * 作业实例ID + */ + private Long jobInstanceId; /** * 步骤实例ID */ @@ -72,12 +76,14 @@ public class ResultHandleTaskResumeEvent extends Event { * @param requestId 请求ID,防止重复下发任务 * @return 事件 */ - public static ResultHandleTaskResumeEvent resume(Long stepInstanceId, + public static ResultHandleTaskResumeEvent resume(Long jobInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, Long gseTaskId, String requestId) { ResultHandleTaskResumeEvent event = new ResultHandleTaskResumeEvent(); + event.setJobInstanceId(jobInstanceId); event.setStepInstanceId(stepInstanceId); event.setExecuteCount(executeCount); event.setBatch(batch); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/StepEvent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/StepEvent.java index 635dad2bcc..6f2261e485 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/StepEvent.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/StepEvent.java @@ -47,6 +47,10 @@ public class StepEvent extends Event { * @see com.tencent.bk.job.execute.engine.consts.StepActionEnum */ private int action; + /** + * 作业实例ID + */ + private long jobInstanceId; /** * 步骤实例ID */ @@ -59,11 +63,13 @@ public class StepEvent extends Event { /** * 构造`忽略错误`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent ignoreError(long stepInstanceId) { + public static StepEvent ignoreError(long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setAction(StepActionEnum.IGNORE_ERROR.getValue()); stepEvent.setTime(LocalDateTime.now()); @@ -73,11 +79,13 @@ public static StepEvent ignoreError(long stepInstanceId) { /** * 构造`下一步`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent nextStep(long stepInstanceId) { + public static StepEvent nextStep(long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setAction(StepActionEnum.NEXT_STEP.getValue()); stepEvent.setTime(LocalDateTime.now()); @@ -87,11 +95,13 @@ public static StepEvent nextStep(long stepInstanceId) { /** * 构造`人工确认-重新发起确认`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent confirmStepContinue(long stepInstanceId) { + public static StepEvent confirmStepContinue(long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setAction(StepActionEnum.CONFIRM_CONTINUE.getValue()); stepEvent.setTime(LocalDateTime.now()); @@ -101,11 +111,13 @@ public static StepEvent confirmStepContinue(long stepInstanceId) { /** * 构造`人工确认-终止流程`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent confirmStepTerminate(long stepInstanceId) { + public static StepEvent confirmStepTerminate(long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setAction(StepActionEnum.CONFIRM_TERMINATE.getValue()); stepEvent.setTime(LocalDateTime.now()); @@ -115,11 +127,13 @@ public static StepEvent confirmStepTerminate(long stepInstanceId) { /** * 构造`人工确认-重新发起确认`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent confirmStepRestart(long stepInstanceId) { + public static StepEvent confirmStepRestart(long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setAction(StepActionEnum.CONFIRM_RESTART.getValue()); stepEvent.setTime(LocalDateTime.now()); @@ -129,12 +143,14 @@ public static StepEvent confirmStepRestart(long stepInstanceId) { /** * 构造`启动步骤`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @param batch 滚动执行批次 * @return 事件 */ - public static StepEvent startStep(long stepInstanceId, Integer batch) { + public static StepEvent startStep(long jobInstanceId, long stepInstanceId, Integer batch) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setBatch(batch); stepEvent.setAction(StepActionEnum.START.getValue()); @@ -145,11 +161,13 @@ public static StepEvent startStep(long stepInstanceId, Integer batch) { /** * 构造`跳过步骤`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent skipStep(long stepInstanceId) { + public static StepEvent skipStep(long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setAction(StepActionEnum.SKIP.getValue()); stepEvent.setTime(LocalDateTime.now()); @@ -159,11 +177,13 @@ public static StepEvent skipStep(long stepInstanceId) { /** * 构造`停止步骤`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent stopStep(long stepInstanceId) { + public static StepEvent stopStep(long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setAction(StepActionEnum.STOP.getValue()); stepEvent.setTime(LocalDateTime.now()); @@ -173,11 +193,13 @@ public static StepEvent stopStep(long stepInstanceId) { /** * 构造`失败IP重试`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent retryStepFail(long stepInstanceId) { + public static StepEvent retryStepFail(long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setAction(StepActionEnum.RETRY_FAIL.getValue()); stepEvent.setTime(LocalDateTime.now()); @@ -187,11 +209,13 @@ public static StepEvent retryStepFail(long stepInstanceId) { /** * 构造`全部重试`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent retryStepAll(long stepInstanceId) { + public static StepEvent retryStepAll(long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setAction(StepActionEnum.RETRY_ALL.getValue()); stepEvent.setTime(LocalDateTime.now()); @@ -201,11 +225,13 @@ public static StepEvent retryStepAll(long stepInstanceId) { /** * 构造`文件准备完成后继续GSE分发`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent continueGseFileStep(long stepInstanceId) { + public static StepEvent continueGseFileStep(long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setAction(StepActionEnum.CONTINUE_FILE_PUSH.getValue()); stepEvent.setTime(LocalDateTime.now()); @@ -215,11 +241,13 @@ public static StepEvent continueGseFileStep(long stepInstanceId) { /** * 构造`准备文件`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent prepareFile(long stepInstanceId) { + public static StepEvent prepareFile(long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setAction(StepActionEnum.PREPARE_FILE.getValue()); stepEvent.setTime(LocalDateTime.now()); @@ -229,11 +257,13 @@ public static StepEvent prepareFile(long stepInstanceId) { /** * 构造`刷新步骤状态`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent refreshStep(long stepInstanceId, EventSource eventSource) { + public static StepEvent refreshStep(long jobInstanceId, long stepInstanceId, EventSource eventSource) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setAction(StepActionEnum.REFRESH.getValue()); stepEvent.setSource(eventSource); @@ -246,6 +276,7 @@ public String toString() { return new StringJoiner(", ", StepEvent.class.getSimpleName() + "[", "]") .add("action=" + action) .add("actionDesc=" + StepActionEnum.valueOf(action)) + .add("jobInstanceId=" + jobInstanceId) .add("stepInstanceId=" + stepInstanceId) .add("batch=" + batch) .add("source=" + source) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/DefaultFilePrepareTaskResultHandler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/DefaultFilePrepareTaskResultHandler.java index 97dcb2d60a..66af70ad67 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/DefaultFilePrepareTaskResultHandler.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/DefaultFilePrepareTaskResultHandler.java @@ -105,25 +105,28 @@ private FilePrepareTaskResult combineTaskResult(List resu private void onSuccess(StepInstanceDTO stepInstance, FilePrepareTaskResult finalResult) { if (!finalResult.getTaskContext().isForRetry()) { // 直接进行下一步 - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.continueGseFileStep(stepInstance.getId())); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.continueGseFileStep(stepInstance.getTaskInstanceId(), stepInstance.getId())); } } private void onStopped(StepInstanceDTO stepInstance, FilePrepareTaskResult finalResult) { // 步骤状态变更 - stepInstanceService.updateStepStatus(stepInstance.getId(), RunStatusEnum.STOP_SUCCESS.getValue()); + stepInstanceService.updateStepStatus(stepInstance.getTaskInstanceId(), + stepInstance.getId(), RunStatusEnum.STOP_SUCCESS.getValue()); // 任务状态变更 taskExecuteMQEventDispatcher.dispatchJobEvent( JobEvent.refreshJob(stepInstance.getTaskInstanceId(), - EventSource.buildStepEventSource(stepInstance.getId()))); + EventSource.buildStepEventSource(stepInstance.getTaskInstanceId(), stepInstance.getId()))); } private void onFailed(StepInstanceDTO stepInstance, FilePrepareTaskResult finalResult) { // 文件源文件下载失败 - stepInstanceService.updateStepStatus(stepInstance.getId(), RunStatusEnum.FAIL.getValue()); + stepInstanceService.updateStepStatus(stepInstance.getTaskInstanceId(), + stepInstance.getId(), RunStatusEnum.FAIL.getValue()); taskExecuteMQEventDispatcher.dispatchJobEvent( JobEvent.refreshJob(stepInstance.getTaskInstanceId(), - EventSource.buildStepEventSource(stepInstance.getId()))); + EventSource.buildStepEventSource(stepInstance.getTaskInstanceId(), stepInstance.getId()))); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareControlTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareControlTask.java index e42fe8b867..8915b2f86e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareControlTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareControlTask.java @@ -130,7 +130,8 @@ private void doExecute() { private boolean needToStop(StepInstanceDTO stepInstance) { TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(stepInstance.getTaskInstanceId()); // 刷新步骤状态 - stepInstance = stepInstanceService.getStepInstanceDetail(stepInstance.getId()); + stepInstance = stepInstanceService.getStepInstanceDetail( + stepInstance.getTaskInstanceId(), stepInstance.getId()); // 如果任务处于“终止中”状态，触发任务终止 if (taskInstance.getStatus() == RunStatusEnum.STOPPING) { // 已经发送过停止命令的就不再重复发送了 @@ -161,7 +162,8 @@ private void gracefulStop() { // 1.停止正在进行的所有文件准备任务 filePrepareService.stopPrepareFile(stepInstance); // 2.MQ消息通知其他实例准备文件 - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.prepareFile(stepInstance.getId())); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.prepareFile(stepInstance.getTaskInstanceId(), stepInstance.getId())); this.isStopped = true; StopTaskCounter.getInstance().decrement(getTaskId()); log.info("gracefulStop end:{}", getTaskId()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareService.java index eb71e66545..941cba64d6 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareService.java @@ -44,9 +44,10 @@ public interface FilePrepareService { /** * 清理已准备完成的临时文件 * + * @param taskInstanceId 作业实例Id * @param stepInstanceId 步骤实例Id */ - void clearPreparedTmpFile(long stepInstanceId); + void clearPreparedTmpFile(long taskInstanceId, long stepInstanceId); /** * 是否需要准备源文件 diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareServiceImpl.java index 9b9eb45a11..a5c3fd7e72 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareServiceImpl.java @@ -75,9 +75,9 @@ public FilePrepareServiceImpl(LocalFilePrepareService localFilePrepareService, } @Override - public void clearPreparedTmpFile(long stepInstanceId) { - localFilePrepareService.clearPreparedTmpFile(stepInstanceId); - thirdFilePrepareService.clearPreparedTmpFile(stepInstanceId); + public void clearPreparedTmpFile(long taskInstanceId, long stepInstanceId) { + localFilePrepareService.clearPreparedTmpFile(taskInstanceId, stepInstanceId); + thirdFilePrepareService.clearPreparedTmpFile(taskInstanceId, stepInstanceId); } private void startPrepareLocalFileTask(StepInstanceDTO stepInstance, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/LocalFilePrepareService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/LocalFilePrepareService.java index 3ebb4b6a46..c738e3c17b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/LocalFilePrepareService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/LocalFilePrepareService.java @@ -139,10 +139,11 @@ private void fillLocalFileSourceHost(List fileSourceList, StepIns } }); // 更新本地文件任务内容 - stepInstanceService.updateResolvedSourceFile(stepInstance.getId(), fileSourceList); + stepInstanceService.updateResolvedSourceFile(stepInstance.getTaskInstanceId(), + stepInstance.getId(), fileSourceList); } - public void clearPreparedTmpFile(long stepInstanceId) { + public void clearPreparedTmpFile(long taskInstanceId, long stepInstanceId) { // 本地文件暂不支持实时清理，依赖定时清理 } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareService.java index ceaa15566b..cd7eb928b0 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareService.java @@ -230,7 +230,8 @@ public void prepareThirdFileAsync( FileSourceTaskLogDTO fileSourceTaskLogDTO = buildInitFileSourceTaskLog(stepInstance, batchTaskInfoDTO); fileSourceTaskLogDAO.insertOrUpdateFileSourceTaskLog(fileSourceTaskLogDTO); // 更新文件源任务状态 - stepInstanceService.updateResolvedSourceFile(stepInstance.getId(), fileSourceList); + stepInstanceService.updateResolvedSourceFile(stepInstance.getTaskInstanceId(), + stepInstance.getId(), fileSourceList); // 异步轮询文件下载任务 ThirdFilePrepareTask task = asyncWatchThirdFilePulling( stepInstance, @@ -250,6 +251,7 @@ public void prepareThirdFileAsync( private void continueStepAtOnce(StepInstanceDTO stepInstance) { taskExecuteMQEventDispatcher.dispatchGseTaskEvent( GseTaskEvent.startGseTask( + stepInstance.getTaskInstanceId(), stepInstance.getId(), stepInstance.getExecuteCount(), stepInstance.getBatch(), @@ -284,8 +286,8 @@ public void stopPrepareThirdFileAsync(StepInstanceDTO stepInstance) { } } - public void clearPreparedTmpFile(long stepInstanceId) { - StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail(stepInstanceId); + public void clearPreparedTmpFile(long taskInstanceId, long stepInstanceId) { + StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail(taskInstanceId, stepInstanceId); // 找出所有第三方文件源的TaskId进行清理 List fileSourceList = stepInstance.getFileSourceList(); List fileSourceTaskIdList = findFileSourceTaskIds(stepInstanceId, fileSourceList); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareTask.java index c49385d621..b9d313a4a9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareTask.java @@ -331,10 +331,11 @@ private void onSuccess(StepInstanceDTO stepInstance, List getAtomicFileTaskLogByTaskIds(long stepInstanceId, + List getAtomicFileTaskLogByTaskIds(long taskInstanceId, + long stepInstanceId, int executeCount, Integer batch, List taskIds); @@ -193,18 +195,21 @@ List getAtomicFileTaskLogByTaskIds(long stepInstanceId, /** * 获取文件任务文件源日志 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例 ID * @param executeCount 执行次数 * @param batch 滚动执行批次;非滚动步骤传入null * @return 日志内容 */ - List batchGetFileSourceExecuteObjectLogContent(long stepInstanceId, + List batchGetFileSourceExecuteObjectLogContent(long taskInstanceId, + long stepInstanceId, int executeCount, Integer batch); /** * 获取文件任务文件日志 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例 ID * @param executeCount 执行次数 * @param batch 滚动执行批次;非滚动步骤传入null @@ -213,6 +218,7 @@ List batchGetFileSourceExecuteObjectLogContent(long * @return 日志内容 */ List batchGetFileExecuteObjectLogContent( + long taskInstanceId, long stepInstanceId, int executeCount, Integer batch, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceService.java index bdc75460d6..8103eae29b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceService.java @@ -44,26 +44,20 @@ public interface StepInstanceService { /** * 更新步骤实例的当前滚动执行批次 * + * @param taskInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @param batch 滚动执行批次 */ - void updateStepCurrentBatch(long stepInstanceId, int batch); - - /** - * 更新步骤实例的当前滚动执行批次 - * - * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - */ - void updateStepCurrentExecuteCount(long stepInstanceId, int executeCount); + void updateStepCurrentBatch(long taskInstanceId, long stepInstanceId, int batch); /** * 更新步骤实例的滚动配置ID * + * @param taskInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @param rollingConfigId 滚动配置ID */ - void updateStepRollingConfigId(long stepInstanceId, long rollingConfigId); + void updateStepRollingConfigId(long taskInstanceId, long stepInstanceId, long rollingConfigId); /** * 获取下一个步骤实例 @@ -83,14 +77,6 @@ public interface StepInstanceService { Map computeStepExecuteObjects(StepInstanceBaseDTO stepInstance, Function keyMapper); - /** - * 获取步骤实例 - * - * @param stepInstanceId 步骤实例ID - * @return 步骤实例 - */ - StepInstanceBaseDTO getStepInstanceBase(long stepInstanceId); - /** * 保存步骤实例 * @@ -118,19 +104,19 @@ Map computeStepExecuteObjects(StepInstanceBaseDTO stepInst /** * 获取步骤基本信息 * + * @param taskInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 步骤基本信息 */ - StepInstanceBaseDTO getBaseStepInstance(long stepInstanceId); + StepInstanceBaseDTO getBaseStepInstance(long taskInstanceId, long stepInstanceId); /** * 获取步骤基本信息 * - * @param appId 业务 ID * @param stepInstanceId 步骤实例ID * @return 步骤基本信息 */ - StepInstanceBaseDTO getBaseStepInstance(long appId, long stepInstanceId); + StepInstanceBaseDTO getBaseStepInstance(long stepInstanceId); /** * 获取步骤基本信息 @@ -140,25 +126,16 @@ Map computeStepExecuteObjects(StepInstanceBaseDTO stepInst * @param stepInstanceId 步骤实例ID * @return 步骤基本信息 */ - StepInstanceBaseDTO getBaseStepInstance(long appId, long taskInstanceId, long stepInstanceId) - throws NotFoundException; - - /** - * 获取步骤基本信息 - * - * @param stepInstanceId 步骤实例ID - * @return 步骤基本信息 - */ - StepInstanceDTO getStepInstanceDetail(long stepInstanceId) throws NotFoundException; + StepInstanceBaseDTO getBaseStepInstance(long appId, long taskInstanceId, long stepInstanceId); /** * 获取步骤基本信息 * - * @param appId 业务 ID + * @param taskInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 步骤基本信息 */ - StepInstanceDTO getStepInstanceDetail(long appId, long stepInstanceId) throws NotFoundException; + StepInstanceDTO getStepInstanceDetail(long taskInstanceId, long stepInstanceId) throws NotFoundException; /** * 获取步骤基本信息 @@ -168,8 +145,9 @@ StepInstanceBaseDTO getBaseStepInstance(long appId, long taskInstanceId, long st * @param stepInstanceId 步骤实例ID * @return 步骤基本信息 */ - StepInstanceDTO getStepInstanceDetail(long appId, long taskInstanceId, long stepInstanceId) - throws NotFoundException; + StepInstanceDTO getStepInstanceDetail(long appId, + long taskInstanceId, + long stepInstanceId) throws NotFoundException; /** * 获取作业的第一个步骤实例 @@ -181,48 +159,49 @@ StepInstanceDTO getStepInstanceDetail(long appId, long taskInstanceId, long step List getTaskStepIdList(long taskInstanceId); - void updateStepStatus(long stepInstanceId, int status); + void updateStepStatus(long taskInstanceId, long stepInstanceId, int status); /** * 重试步骤操作-重置步骤执行状态 * + * @param taskInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID */ - void resetStepExecuteInfoForRetry(long stepInstanceId); - - void resetStepStatus(long stepInstanceId); + void resetStepExecuteInfoForRetry(long taskInstanceId, long stepInstanceId); - void updateStepStartTime(long stepInstanceId, Long startTime); + void resetStepStatus(long taskInstanceId, long stepInstanceId); /** * 更新步骤启动时间 - 仅当启动时间为空 * + * @param taskInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @param startTime 启动时间 */ - void updateStepStartTimeIfNull(long stepInstanceId, Long startTime); + void updateStepStartTimeIfNull(long taskInstanceId, long stepInstanceId, Long startTime); - void updateStepEndTime(long stepInstanceId, Long endTime); + void updateStepEndTime(long taskInstanceId, long stepInstanceId, Long endTime); /** * 步骤重试次数+1 * + * @param taskInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID */ - void addStepInstanceExecuteCount(long stepInstanceId); - - void updateStepTotalTime(long stepInstanceId, long totalTime); + void addStepInstanceExecuteCount(long taskInstanceId, long stepInstanceId); /** * 更新步骤的执行信息 * + * @param taskInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @param status 步骤执行状态 * @param startTime 开始时间 * @param endTime 结束时间 * @param totalTime 总耗时 */ - void updateStepExecutionInfo(long stepInstanceId, + void updateStepExecutionInfo(long taskInstanceId, + long stepInstanceId, RunStatusEnum status, Long startTime, Long endTime, @@ -232,43 +211,51 @@ void updateStepExecutionInfo(long stepInstanceId, /** * 更新解析之后的脚本参数 * + * @param taskInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @param isSecureParam 是否为敏感参数 * @param resolvedScriptParam 解析之后的脚本参数 */ - void updateResolvedScriptParam(long stepInstanceId, boolean isSecureParam, String resolvedScriptParam); + void updateResolvedScriptParam(long taskInstanceId, + long stepInstanceId, + boolean isSecureParam, + String resolvedScriptParam); /** * 更新变量解析之后的源文件 * + * @param taskInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @param resolvedFileSources 解析后的源文件信息 */ - void updateResolvedSourceFile(long stepInstanceId, List resolvedFileSources); + void updateResolvedSourceFile(long taskInstanceId, long stepInstanceId, List resolvedFileSources); /** * 更新变量解析之后的目标路径 * + * @param taskInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @param resolvedTargetPath 解析之后的目标路径 */ - void updateResolvedTargetPath(long stepInstanceId, String resolvedTargetPath); + void updateResolvedTargetPath(long taskInstanceId, long stepInstanceId, String resolvedTargetPath); /** * 更新确认理由 * + * @param taskInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @param confirmReason 确认理由 */ - void updateConfirmReason(long stepInstanceId, String confirmReason); + void updateConfirmReason(long taskInstanceId, long stepInstanceId, String confirmReason); /** * 更新步骤操作人 * + * @param taskInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @param operator 操作人 */ - void updateStepOperator(long stepInstanceId, String operator); + void updateStepOperator(long taskInstanceId, long stepInstanceId, String operator); /** * 获取上一步骤实例(可执行的，不包含人工确认这种) @@ -306,13 +293,4 @@ List findExecuteObjectByCompositeKeys( StepInstanceBaseDTO stepInstance, Collection executeObjectCompositeKeys ); - - /** - * 根据 appId,stepInstanceId 获取所属任务实例ID - * - * @param appId Job业务ID - * @param stepInstanceId 步骤实例ID - * @return 任务实例ID - */ - Long getStepTaskInstanceId(long appId, long stepInstanceId); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceValidateService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceValidateService.java deleted file mode 100644 index ed73db6ce6..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceValidateService.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.tencent.bk.job.execute.service; - -import com.tencent.bk.job.common.model.ValidateResult; - -public interface StepInstanceValidateService { - - ValidateResult checkStepInstance(long appId, Long taskInstanceId, Long stepInstanceId); - -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskExecuteService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskExecuteService.java index f6fa668021..468efb4d4e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskExecuteService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskExecuteService.java @@ -76,11 +76,10 @@ public interface TaskExecuteService { * @param taskInstanceId 作业实例 ID * @param operator 操作者 * @param executeVariableValues 全局变量 - * @return */ TaskInstanceDTO createTaskInstanceForRedo(Long appId, Long taskInstanceId, String operator, List executeVariableValues) - throws ServiceException; + ; /** * 步骤操作 @@ -91,7 +90,7 @@ TaskInstanceDTO createTaskInstanceForRedo(Long appId, Long taskInstanceId, Strin * @return 执行次数 * @throws ServiceException */ - Integer doStepOperation(Long appId, String operator, StepOperationDTO stepOperation) throws ServiceException; + Integer doStepOperation(Long appId, String operator, StepOperationDTO stepOperation); /** * 终止作业 @@ -99,9 +98,8 @@ TaskInstanceDTO createTaskInstanceForRedo(Long appId, Long taskInstanceId, Strin * @param username 操作者 * @param appId 业务ID * @param taskInstanceId 作业实例ID - * @throws ServiceException 终止操作异常的时候抛出 */ - void terminateJob(String username, Long appId, Long taskInstanceId) throws ServiceException; + void terminateJob(String username, Long appId, Long taskInstanceId); /** * 作业操作 @@ -110,15 +108,14 @@ TaskInstanceDTO createTaskInstanceForRedo(Long appId, Long taskInstanceId, Strin * @param operator 操作者 * @param taskInstanceId 作业实例 ID * @param operation 操作类型 - * @throws ServiceException */ void doTaskOperation(Long appId, String operator, long taskInstanceId, - TaskOperationEnum operation) throws ServiceException; + TaskOperationEnum operation); /** * 作业执行方案执行鉴权 * * @param executeParam 作业执行参数 */ - void authExecuteJobPlan(TaskExecuteParam executeParam) throws ServiceException; + void authExecuteJobPlan(TaskExecuteParam executeParam); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogExportServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogExportServiceImpl.java index 295a6c7106..28019faf1c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogExportServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogExportServiceImpl.java @@ -104,6 +104,7 @@ public LogExportServiceImpl(LogService logService, @Override public LogExportJobInfoDTO packageLogFile(String username, Long appId, + Long taskInstanceId, Long stepInstanceId, ExecuteObjectTypeEnum executeObjectType, Long executeObjectResourceId, @@ -129,7 +130,7 @@ public LogExportJobInfoDTO packageLogFile(String username, boolean isGetByExecuteObject = executeObjectResourceId != null; if (isGetByExecuteObject) { - doPackage(exportJobInfo, stepInstanceId, executeObjectType, executeObjectResourceId, + doPackage(exportJobInfo, taskInstanceId, stepInstanceId, executeObjectType, executeObjectResourceId, executeCount, logFileDir, logFileName); } else { String requestId = JobContextUtil.getRequestId(); @@ -143,8 +144,8 @@ public LogExportJobInfoDTO packageLogFile(String username, exportJobInfo.setStatus(LogExportStatusEnum.PROCESSING); saveExportInfo(exportJobInfo); - doPackage(exportJobInfo, stepInstanceId, executeObjectType, executeObjectResourceId, - executeCount, logFileDir, logFileName); + doPackage(exportJobInfo, taskInstanceId, stepInstanceId, executeObjectType, + executeObjectResourceId, executeCount, logFileDir, logFileName); } else { log.error("Job already running!|appId={}|stepInstanceId={}", appId, stepInstanceId); } @@ -204,13 +205,14 @@ private String getExportJobKey(Long appId, } private void doPackage(LogExportJobInfoDTO exportJobInfo, + long taskInstanceId, long stepInstanceId, ExecuteObjectTypeEnum executeObjectType, Long executeObjectResourceId, int executeCount, String logFileDir, String logFileName) { - StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance(stepInstanceId); + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance(taskInstanceId, stepInstanceId); File logFile = new File(logFileDir + logFileName); StopWatch watch = new StopWatch("exportJobLog"); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java index f99c8a7e4e..815e95f36c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java @@ -414,11 +414,12 @@ public FileExecuteObjectLogContent getFileExecuteObjectLogContent(StepInstanceBa } @Override - public List getAtomicFileTaskLogByTaskIds(long stepInstanceId, + public List getAtomicFileTaskLogByTaskIds(long taskInstanceId, + long stepInstanceId, int executeCount, Integer batch, List taskIds) { - StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance(stepInstanceId); + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance(taskInstanceId, stepInstanceId); String taskCreateDateStr = buildTaskCreateDateStr(stepInstance); InternalResponse> resp = logResource.listTaskFileLogsByTaskIds( taskCreateDateStr, stepInstanceId, executeCount, batch, taskIds); @@ -481,10 +482,11 @@ private List batchConvertToAtomicFileTaskLog( } @Override - public List batchGetFileSourceExecuteObjectLogContent(long stepInstanceId, + public List batchGetFileSourceExecuteObjectLogContent(long taskInstanceId, + long stepInstanceId, int executeCount, Integer batch) { - return batchGetFileExecuteObjectLogContent(stepInstanceId, executeCount, batch, + return batchGetFileExecuteObjectLogContent(taskInstanceId, stepInstanceId, executeCount, batch, FileTaskModeEnum.UPLOAD, null); } @@ -540,13 +542,14 @@ private List batchConvertToFileExecuteObjectLogCont @Override public List batchGetFileExecuteObjectLogContent( + long taskInstanceId, long stepInstanceId, int executeCount, Integer batch, FileTaskModeEnum mode, List executeObjectCompositeKeys) { - StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail(stepInstanceId); + StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail(taskInstanceId, stepInstanceId); String taskCreateDateStr = buildTaskCreateDateStr(stepInstance); ServiceFileLogQueryRequest request = new ServiceFileLogQueryRequest(); request.setStepInstanceId(stepInstance.getId()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/NotifyServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/NotifyServiceImpl.java index 2e753af7f1..ad6e4dc966 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/NotifyServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/NotifyServiceImpl.java @@ -256,7 +256,8 @@ private Map getTemplateVariablesMap(TaskNotifyDTO taskNotifyDTO, variablesMap.put("task.step.total_seq_cnt", "" + stepIdList.size()); long currentStepId = taskInstanceDTO.getCurrentStepInstanceId(); variablesMap.put("task.step.current_seq_id", "" + (stepIdList.indexOf(currentStepId) + 1)); - StepInstanceDTO stepInstanceDTO = stepInstanceService.getStepInstanceDetail(currentStepId); + StepInstanceDTO stepInstanceDTO = stepInstanceService.getStepInstanceDetail( + taskInstanceDTO.getId(), currentStepId); if (executeStatus == ExecuteStatusEnum.FAIL || executeStatus == ExecuteStatusEnum.SUCCESS) { if (stepInstanceDTO.getTotalTime() != null) { variablesMap.put("task.step.duration", "" + stepInstanceDTO.getTotalTime() / 1000.0); @@ -383,7 +384,8 @@ public void asyncSendMQSuccessTaskNotification(TaskInstanceDTO taskInstance, @Override public void asyncSendMQConfirmNotification(TaskInstanceDTO taskInstance, StepInstanceBaseDTO stepInstance) { - StepInstanceDTO stepInstanceDetail = stepInstanceService.getStepInstanceDetail(stepInstance.getId()); + StepInstanceDTO stepInstanceDetail = stepInstanceService.getStepInstanceDetail( + taskInstance.getId(), stepInstance.getId()); if (stepInstanceDetail == null) { log.warn("StepInstance is not exist, stepInstanceId: {}", stepInstance.getId()); return; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java index 227a9ca99e..5959811cd1 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java @@ -72,18 +72,13 @@ public StepInstanceServiceImpl(StepInstanceDAO stepInstanceDAO) { } @Override - public void updateStepCurrentBatch(long stepInstanceId, int batch) { - stepInstanceDAO.updateStepCurrentBatch(stepInstanceId, batch); + public void updateStepCurrentBatch(long taskInstanceId, long stepInstanceId, int batch) { + stepInstanceDAO.updateStepCurrentBatch(taskInstanceId, stepInstanceId, batch); } @Override - public void updateStepCurrentExecuteCount(long stepInstanceId, int executeCount) { - stepInstanceDAO.updateStepCurrentExecuteCount(stepInstanceId, executeCount); - } - - @Override - public void updateStepRollingConfigId(long stepInstanceId, long rollingConfigId) { - stepInstanceDAO.updateStepRollingConfigId(stepInstanceId, rollingConfigId); + public void updateStepRollingConfigId(long taskInstanceId, long stepInstanceId, long rollingConfigId) { + stepInstanceDAO.updateStepRollingConfigId(taskInstanceId, stepInstanceId, rollingConfigId); } @Override @@ -110,14 +105,15 @@ public long addStepInstance(StepInstanceDTO stepInstance) { } private void fillStepInstanceDetail(StepInstanceDTO stepInstance) { + long taskInstanceId = stepInstance.getTaskInstanceId(); long stepInstanceId = stepInstance.getId(); TaskStepTypeEnum stepType = stepInstance.getStepType(); if (stepType == TaskStepTypeEnum.SCRIPT) { - stepInstance.fillScriptStepInfo(stepInstanceDAO.getScriptStepInstance(stepInstanceId)); + stepInstance.fillScriptStepInfo(stepInstanceDAO.getScriptStepInstance(taskInstanceId, stepInstanceId)); } else if (stepType == TaskStepTypeEnum.FILE) { - stepInstance.fillFileStepInfo(stepInstanceDAO.getFileStepInstance(stepInstanceId)); + stepInstance.fillFileStepInfo(stepInstanceDAO.getFileStepInstance(taskInstanceId, stepInstanceId)); } else if (stepType == TaskStepTypeEnum.APPROVAL) { - stepInstance.fillConfirmStepInfo(stepInstanceDAO.getConfirmStepInstance(stepInstanceId)); + stepInstance.fillConfirmStepInfo(stepInstanceDAO.getConfirmStepInstance(taskInstanceId, stepInstanceId)); } } @@ -141,36 +137,33 @@ public List listStepInstanceByTaskInstanceId(long taskInstanceI } @Override - public StepInstanceBaseDTO getBaseStepInstance(long stepInstanceId) { - return stepInstanceDAO.getStepInstanceBase(stepInstanceId); + public StepInstanceBaseDTO getBaseStepInstance(long taskInstanceId, long stepInstanceId) { + return stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); } @Override - public StepInstanceBaseDTO getBaseStepInstance(long appId, long stepInstanceId) { - StepInstanceBaseDTO stepInstance = getBaseStepInstance(stepInstanceId); - if (!stepInstance.getAppId().equals(appId)) { - log.warn("StepInstance:{} is not in app:{}", stepInstanceId, appId); - throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); - } - return stepInstance; + public StepInstanceBaseDTO getBaseStepInstance(long stepInstanceId) { + return stepInstanceDAO.getStepInstanceBase(stepInstanceId); } @Override - public StepInstanceBaseDTO getBaseStepInstance(long appId, - long taskInstanceId, - long stepInstanceId) throws NotFoundException { - StepInstanceBaseDTO stepInstance = getBaseStepInstance(appId, stepInstanceId); - if (!stepInstance.getTaskInstanceId().equals(taskInstanceId)) { + public StepInstanceBaseDTO getBaseStepInstance(long appId, long taskInstanceId, long stepInstanceId) { + StepInstanceBaseDTO stepInstance = getBaseStepInstance(taskInstanceId, stepInstanceId); + if (stepInstance == null) { log.warn("Step instance is not exist, taskInstanceId={}, stepInstanceId: {}", taskInstanceId, stepInstanceId); throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); } + if (!stepInstance.getAppId().equals(appId)) { + log.warn("StepInstance:{} is not in app:{}", stepInstanceId, appId); + throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); + } return stepInstance; } @Override - public StepInstanceDTO getStepInstanceDetail(long stepInstanceId) throws NotFoundException { - StepInstanceBaseDTO stepInstanceBase = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + public StepInstanceDTO getStepInstanceDetail(long taskInstanceId, long stepInstanceId) throws NotFoundException { + StepInstanceBaseDTO stepInstanceBase = stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); if (stepInstanceBase == null) { log.warn("StepInstance:{} not exist", stepInstanceId); throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); @@ -181,8 +174,9 @@ public StepInstanceDTO getStepInstanceDetail(long stepInstanceId) throws NotFoun } @Override - public StepInstanceDTO getStepInstanceDetail(long appId, long stepInstanceId) throws NotFoundException { - StepInstanceDTO stepInstance = getStepInstanceDetail(stepInstanceId); + public StepInstanceDTO getStepInstanceDetail(long appId, long taskInstanceId, + long stepInstanceId) throws NotFoundException { + StepInstanceDTO stepInstance = getStepInstanceDetail(taskInstanceId, stepInstanceId); if (!stepInstance.getAppId().equals(appId)) { log.warn("StepInstance:{} is not in app:{}", stepInstanceId, appId); throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); @@ -190,18 +184,6 @@ public StepInstanceDTO getStepInstanceDetail(long appId, long stepInstanceId) th return stepInstance; } - @Override - public StepInstanceDTO getStepInstanceDetail(long appId, - long taskInstanceId, - long stepInstanceId) throws NotFoundException { - StepInstanceDTO stepInstance = getStepInstanceDetail(appId, stepInstanceId); - if (!stepInstance.getTaskInstanceId().equals(taskInstanceId)) { - log.warn("StepInstance:{} is not belong to taskInstance:{}", stepInstanceId, taskInstanceId); - throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); - } - return stepInstance; - } - @Override public StepInstanceBaseDTO getFirstStepInstance(long taskInstanceId) { return stepInstanceDAO.getFirstStepInstanceBase(taskInstanceId); @@ -213,80 +195,76 @@ public List getTaskStepIdList(long taskInstanceId) { } @Override - public void updateStepStatus(long stepInstanceId, int status) { - stepInstanceDAO.updateStepStatus(stepInstanceId, status); - } - - @Override - public void resetStepExecuteInfoForRetry(long stepInstanceId) { - stepInstanceDAO.resetStepExecuteInfoForRetry(stepInstanceId); - } - - @Override - public void resetStepStatus(long stepInstanceId) { - stepInstanceDAO.resetStepStatus(stepInstanceId); + public void updateStepStatus(long taskInstanceId, long stepInstanceId, int status) { + stepInstanceDAO.updateStepStatus(taskInstanceId, stepInstanceId, status); } @Override - public void updateStepStartTime(long stepInstanceId, Long startTime) { - stepInstanceDAO.updateStepStartTime(stepInstanceId, startTime); + public void resetStepExecuteInfoForRetry(long taskInstanceId, long stepInstanceId) { + stepInstanceDAO.resetStepExecuteInfoForRetry(taskInstanceId, stepInstanceId); } @Override - public void updateStepStartTimeIfNull(long stepInstanceId, Long startTime) { - stepInstanceDAO.updateStepStartTimeIfNull(stepInstanceId, startTime); + public void resetStepStatus(long taskInstanceId, long stepInstanceId) { + stepInstanceDAO.resetStepStatus(taskInstanceId, stepInstanceId); } @Override - public void updateStepEndTime(long stepInstanceId, Long endTime) { - stepInstanceDAO.updateStepEndTime(stepInstanceId, endTime); + public void updateStepStartTimeIfNull(long taskInstanceId, long stepInstanceId, Long startTime) { + stepInstanceDAO.updateStepStartTimeIfNull(taskInstanceId, stepInstanceId, startTime); } @Override - public void addStepInstanceExecuteCount(long stepInstanceId) { - stepInstanceDAO.addStepInstanceExecuteCount(stepInstanceId); + public void updateStepEndTime(long taskInstanceId, long stepInstanceId, Long endTime) { + stepInstanceDAO.updateStepEndTime(taskInstanceId, stepInstanceId, endTime); } @Override - public void updateStepTotalTime(long stepInstanceId, long totalTime) { - stepInstanceDAO.updateStepTotalTime(stepInstanceId, totalTime); + public void addStepInstanceExecuteCount(long taskInstanceId, long stepInstanceId) { + stepInstanceDAO.addStepInstanceExecuteCount(taskInstanceId, stepInstanceId); } @Override - public void updateStepExecutionInfo(long stepInstanceId, + public void updateStepExecutionInfo(long taskInstanceId, + long stepInstanceId, RunStatusEnum status, Long startTime, Long endTime, Long totalTime) { - stepInstanceDAO.updateStepExecutionInfo(stepInstanceId, status, startTime, endTime, totalTime); + stepInstanceDAO.updateStepExecutionInfo(taskInstanceId, stepInstanceId, status, startTime, endTime, totalTime); } @Override - public void updateResolvedScriptParam(long stepInstanceId, boolean isSecureParam, String resolvedScriptParam) { - stepInstanceDAO.updateResolvedScriptParam(stepInstanceId, isSecureParam, resolvedScriptParam); + public void updateResolvedScriptParam(long taskInstanceId, + long stepInstanceId, + boolean isSecureParam, + String resolvedScriptParam) { + stepInstanceDAO.updateResolvedScriptParam(taskInstanceId, stepInstanceId, isSecureParam, resolvedScriptParam); } @Override - public void updateResolvedSourceFile(long stepInstanceId, List resolvedFileSources) { + public void updateResolvedSourceFile(long taskInstanceId, + long stepInstanceId, + List resolvedFileSources) { if (log.isDebugEnabled()) { log.debug("updateResolvedSourceFile={}", JsonUtils.toJson(resolvedFileSources)); } - stepInstanceDAO.updateResolvedSourceFile(stepInstanceId, resolvedFileSources); + stepInstanceDAO.updateResolvedSourceFile(taskInstanceId, stepInstanceId, resolvedFileSources); } @Override - public void updateResolvedTargetPath(long stepInstanceId, String resolvedTargetPath) { - stepInstanceDAO.updateResolvedTargetPath(stepInstanceId, resolvedTargetPath); + public void updateResolvedTargetPath(long taskInstanceId, long stepInstanceId, String resolvedTargetPath) { + stepInstanceDAO.updateResolvedTargetPath(taskInstanceId, stepInstanceId, resolvedTargetPath); } @Override - public void updateConfirmReason(long stepInstanceId, String confirmReason) { - stepInstanceDAO.updateConfirmReason(stepInstanceId, confirmReason); + public void updateConfirmReason(long taskInstanceId, long stepInstanceId, String confirmReason) { + stepInstanceDAO.updateConfirmReason(taskInstanceId, stepInstanceId, confirmReason); } @Override - public void updateStepOperator(long stepInstanceId, String operator) { - stepInstanceDAO.updateStepOperator(stepInstanceId, operator); + public void updateStepOperator(long taskInstanceId, long stepInstanceId, String operator) { + stepInstanceDAO.updateStepOperator(taskInstanceId, stepInstanceId, operator); } @Override @@ -331,7 +309,8 @@ public Map computeStepExecuteObjects(StepInstanceBaseDTO s if (stepInstance instanceof StepInstanceDTO) { fileSourceList = ((StepInstanceDTO) stepInstance).getFileSourceList(); } else { - FileStepInstanceDTO fileStepInstance = stepInstanceDAO.getFileStepInstance(stepInstance.getId()); + FileStepInstanceDTO fileStepInstance = stepInstanceDAO.getFileStepInstance( + stepInstance.getTaskInstanceId(), stepInstance.getId()); fileSourceList = fileStepInstance.getFileSourceList(); } @@ -355,11 +334,6 @@ public Map computeStepExecuteObjects(StepInstanceBaseDTO s return executeObjects; } - @Override - public StepInstanceBaseDTO getStepInstanceBase(long stepInstanceId) { - return stepInstanceDAO.getStepInstanceBase(stepInstanceId); - } - @Override public ExecuteObject findExecuteObjectByCompositeKey(StepInstanceBaseDTO stepInstance, ExecuteObjectCompositeKey executeObjectCompositeKey) { @@ -392,7 +366,7 @@ private StepInstanceDTO castToStepInstanceDTO(StepInstanceBaseDTO stepInstance) if (stepInstance instanceof StepInstanceDTO) { return (StepInstanceDTO) stepInstance; } else { - return getStepInstanceDetail(stepInstance.getId()); + return getStepInstanceDetail(stepInstance.getTaskInstanceId(), stepInstance.getId()); } } @@ -425,9 +399,4 @@ public List findExecuteObjectByCompositeKeys( throw new InternalException("Not support method invoke for step", ErrorCode.INTERNAL_ERROR); } } - - @Override - public Long getStepTaskInstanceId(long appId, long stepInstanceId) { - return stepInstanceDAO.getTaskInstanceId(appId, stepInstanceId); - } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceValidateServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceValidateServiceImpl.java deleted file mode 100644 index cdbb920aa9..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceValidateServiceImpl.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.tencent.bk.job.execute.service.impl; - -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.exception.NotFoundException; -import com.tencent.bk.job.common.model.ValidateResult; -import com.tencent.bk.job.execute.service.StepInstanceService; -import com.tencent.bk.job.execute.service.StepInstanceValidateService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; - -@Slf4j -@Service -public class StepInstanceValidateServiceImpl implements StepInstanceValidateService { - - private final StepInstanceService stepInstanceService; - - public StepInstanceValidateServiceImpl(StepInstanceService stepInstanceService) { - this.stepInstanceService = stepInstanceService; - } - - @Override - public ValidateResult checkStepInstance(long appId, Long taskInstanceId, Long stepInstanceId) { - // 检查taskInstanceId与stepInstanceId关联关系的正确性 - Long realTaskInstanceId = stepInstanceService.getStepTaskInstanceId(appId, stepInstanceId); - if (realTaskInstanceId == null) { - throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); - } else if (!realTaskInstanceId.equals(taskInstanceId)) { - log.info("stepInstance {} does not belong to taskInstance {}", stepInstanceId, taskInstanceId); - return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, "job_instance_id"); - } - return ValidateResult.pass(); - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java index 36c726846d..66387ad101 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java @@ -325,7 +325,7 @@ private void saveTaskInstance(StopWatch watch, watch.start("saveRollingConfig"); RollingConfigDTO rollingConfig = rollingConfigService.saveRollingConfigForFastJob(fastTask); long rollingConfigId = rollingConfig.getId(); - stepInstanceService.updateStepRollingConfigId(stepInstanceId, rollingConfigId); + stepInstanceService.updateStepRollingConfigId(taskInstanceId, stepInstanceId, rollingConfigId); watch.stop(); } @@ -1698,7 +1698,8 @@ public Integer doStepOperation(Long appId, String operator, StepOperationEnum operation = stepOperation.getOperation(); log.info("Operate step, appId:{}, stepInstanceId:{}, operator:{}, operation:{}", appId, stepInstanceId, operator, operation.getValue()); - StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail(stepInstanceId); + StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail( + stepOperation.getTaskInstanceId(),stepInstanceId); if (stepInstance == null) { log.warn("Step instance {} is not exist", stepInstanceId); throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); @@ -1760,8 +1761,8 @@ private void continueRolling(StepInstanceDTO stepInstance) { // 需要同步设置任务状态为RUNNING，保证客户端可以在操作完之后立马获取到运行状态，开启同步刷新 taskInstanceService.updateTaskStatus(stepInstance.getTaskInstanceId(), RunStatusEnum.RUNNING.getValue()); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.startStep(stepInstance.getId(), - stepInstance.getBatch() + 1)); + taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.startStep(stepInstance.getTaskInstanceId(), + stepInstance.getId(), stepInstance.getBatch() + 1)); } private void confirmTerminate(StepInstanceDTO stepInstance, String operator, String reason) { @@ -1784,10 +1785,11 @@ private void confirmTerminate(StepInstanceDTO stepInstance, String operator, Str operationLog.getDetail().setConfirmReason(reason); taskOperationLogService.saveOperationLog(operationLog); - stepInstanceService.updateConfirmReason(stepInstance.getId(), reason); - stepInstanceService.updateStepOperator(stepInstance.getId(), operator); + stepInstanceService.updateConfirmReason(stepInstance.getTaskInstanceId(), stepInstance.getId(), reason); + stepInstanceService.updateStepOperator(stepInstance.getTaskInstanceId(), stepInstance.getId(), operator); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.confirmStepTerminate(stepInstance.getId())); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.confirmStepTerminate(stepInstance.getTaskInstanceId(), stepInstance.getId())); } private void confirmRestart(StepInstanceDTO stepInstance, String operator) { @@ -1804,7 +1806,8 @@ private void confirmRestart(StepInstanceDTO stepInstance, String operator) { } taskOperationLogService.saveOperationLog(buildCommonStepOperationLog(stepInstance, operator, UserOperationEnum.CONFIRM_RESTART)); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.confirmStepRestart(stepInstance.getId())); + taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.confirmStepRestart( + stepInstance.getTaskInstanceId(), stepInstance.getId())); } private void checkConfirmUser(TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance, @@ -1879,13 +1882,14 @@ private void confirmContinue(StepInstanceDTO stepInstance, String operator, Stri operationLog.getDetail().setConfirmReason(reason); taskOperationLogService.saveOperationLog(operationLog); - stepInstanceService.updateConfirmReason(stepInstance.getId(), reason); - stepInstanceService.updateStepOperator(stepInstance.getId(), operator); + stepInstanceService.updateConfirmReason(stepInstance.getTaskInstanceId(), stepInstance.getId(), reason); + stepInstanceService.updateStepOperator(stepInstance.getTaskInstanceId(), stepInstance.getId(), operator); // 需要同步设置任务状态为RUNNING，保证客户端可以在操作完之后立马获取到运行状态，开启同步刷新 taskInstanceService.updateTaskStatus(taskInstance.getId(), RunStatusEnum.RUNNING.getValue()); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.confirmStepContinue(stepInstance.getId())); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.confirmStepContinue(stepInstance.getTaskInstanceId(), stepInstance.getId())); } private void nextStep(StepInstanceDTO stepInstance, String operator) { @@ -1901,7 +1905,8 @@ private void nextStep(StepInstanceDTO stepInstance, String operator) { // 需要同步设置任务状态为RUNNING，保证客户端可以在操作完之后立马获取到运行状态，开启同步刷新 taskInstanceService.updateTaskStatus(taskInstance.getId(), RunStatusEnum.RUNNING.getValue()); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.nextStep(stepInstance.getId())); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.nextStep(stepInstance.getTaskInstanceId(), stepInstance.getId())); } private void retryStepFail(StepInstanceDTO stepInstance, String operator) { @@ -1912,8 +1917,9 @@ private void retryStepFail(StepInstanceDTO stepInstance, String operator) { } // 需要同步设置任务状态为RUNNING，保证客户端可以在操作完之后立马获取到运行状态，开启同步刷新 taskInstanceService.updateTaskStatus(stepInstance.getTaskInstanceId(), RunStatusEnum.RUNNING.getValue()); - stepInstanceService.addStepInstanceExecuteCount(stepInstance.getId()); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.retryStepFail(stepInstance.getId())); + stepInstanceService.addStepInstanceExecuteCount( + stepInstance.getTaskInstanceId(), stepInstance.getId()); + taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.retryStepFail(stepInstance.getTaskInstanceId(), stepInstance.getId())); OperationLogDTO operationLog = buildCommonStepOperationLog(stepInstance, operator, UserOperationEnum.RETRY_STEP_FAIL); taskOperationLogService.saveOperationLog(operationLog); @@ -1927,8 +1933,9 @@ private void retryStepAll(StepInstanceDTO stepInstance, String operator) { } // 需要同步设置任务状态为RUNNING，保证客户端可以在操作完之后立马获取到运行状态，开启同步刷新 taskInstanceService.updateTaskStatus(stepInstance.getTaskInstanceId(), RunStatusEnum.RUNNING.getValue()); - stepInstanceService.addStepInstanceExecuteCount(stepInstance.getId()); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.retryStepAll(stepInstance.getId())); + stepInstanceService.addStepInstanceExecuteCount(stepInstance.getTaskInstanceId(), stepInstance.getId()); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.retryStepAll(stepInstance.getTaskInstanceId(), stepInstance.getId())); OperationLogDTO operationLog = buildCommonStepOperationLog(stepInstance, operator, UserOperationEnum.RETRY_STEP_ALL); taskOperationLogService.saveOperationLog(operationLog); @@ -1951,7 +1958,8 @@ private void ignoreError(StepInstanceDTO stepInstance, String operator) { } // 需要同步设置任务状态为RUNNING，保证客户端可以在操作完之后立马获取到运行状态，开启同步刷新 taskInstanceService.updateTaskStatus(stepInstance.getTaskInstanceId(), RunStatusEnum.RUNNING.getValue()); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.ignoreError(stepInstance.getId())); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.ignoreError(stepInstance.getTaskInstanceId(), stepInstance.getId())); OperationLogDTO operationLog = buildCommonStepOperationLog(stepInstance, operator, UserOperationEnum.IGNORE_ERROR); taskOperationLogService.saveOperationLog(operationLog); @@ -1964,7 +1972,8 @@ private void skipStep(StepInstanceDTO stepInstance, String operator) { stepInstance.getId(), StepOperationEnum.SKIP.name(), stepInstance.getStatus().name()); throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); } - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.skipStep(stepInstance.getId())); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.skipStep(stepInstance.getTaskInstanceId(), stepInstance.getId())); OperationLogDTO operationLog = buildCommonStepOperationLog(stepInstance, operator, UserOperationEnum.SKIP_STEP); taskOperationLogService.saveOperationLog(operationLog); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java index c042795fb8..14536412e1 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java @@ -216,7 +216,8 @@ private StepExecutionDTO buildStepExecutionDTO(StepInstanceBaseDTO stepInstance) stepInstance.getEndTime(), stepInstance.getTotalTime())); stepExecution.setLastStep(stepInstance.isLastStep()); if (stepInstance.getExecuteType() == StepExecuteTypeEnum.MANUAL_CONFIRM) { - ConfirmStepInstanceDTO confirmStepInstance = stepInstanceDAO.getConfirmStepInstance(stepInstance.getId()); + ConfirmStepInstanceDTO confirmStepInstance = stepInstanceDAO.getConfirmStepInstance( + stepInstance.getTaskInstanceId(), stepInstance.getId()); if (confirmStepInstance != null) { stepExecution.setConfirmMessage(confirmStepInstance.getConfirmMessage()); stepExecution.setConfirmReason(confirmStepInstance.getConfirmReason()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/StatisticsServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/StatisticsServiceImpl.java index 34c9d0f09f..547c01d428 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/StatisticsServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/StatisticsServiceImpl.java @@ -229,7 +229,7 @@ private void updateExecutedFastScriptStatistics(TaskInstanceDTO taskInstanceDTO, return; } // 查StepInstanceScript - Byte scriptType = stepInstanceDAO.getScriptTypeByStepInstanceId(stepInstanceId); + Byte scriptType = stepInstanceDAO.getScriptTypeByStepInstanceId(taskInstanceDTO.getId(), stepInstanceId); // 更新统计数据 // 快速执行脚本：按脚本类型统计 String scriptTypeName = ScriptTypeEnum.getName(scriptType.intValue()); @@ -254,7 +254,8 @@ private void updateExecutedFastFileStatistics(TaskInstanceDTO taskInstanceDTO, return; } // 查StepInstanceFile - FileStepInstanceDTO fileStepInstanceDTO = stepInstanceDAO.getFileStepInstance(stepInstanceId); + FileStepInstanceDTO fileStepInstanceDTO = stepInstanceDAO.getFileStepInstance( + taskInstanceDTO.getId(), stepInstanceId); // 更新统计数据 // 快速分发文件：按传输模式统计 Integer notExistPathHandler = fileStepInstanceDTO.getNotExistPathHandler(); diff --git a/support-files/sql/job-execute/0026_job_execute_20240520-1000_V3.10.0_mysql.sql b/support-files/sql/job-execute/0026_job_execute_20240520-1000_V3.10.0_mysql.sql new file mode 100644 index 0000000000..c56970b340 --- /dev/null +++ b/support-files/sql/job-execute/0026_job_execute_20240520-1000_V3.10.0_mysql.sql @@ -0,0 +1,143 @@ +USE job_execute; + +SET NAMES utf8mb4; + +DROP PROCEDURE IF EXISTS job_schema_update; + +DELIMITER + +CREATE PROCEDURE job_schema_update() +BEGIN + + DECLARE db VARCHAR(100); + SET AUTOCOMMIT = 0; + SELECT DATABASE() INTO db; + + IF NOT EXISTS(SELECT 1 + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'step_instance_script' + AND COLUMN_NAME = 'task_instance_id') THEN + ALTER TABLE step_instance_script ADD COLUMN `task_instance_id` bigint(20) NOT NULL DEFAULT '0'; + END IF; + IF NOT EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'step_instance_script' + AND INDEX_NAME = 'idx_task_instance_id') THEN + ALTER TABLE step_instance_script ADD INDEX `idx_task_instance_id` (`task_instance_id`); + END IF; + + IF NOT EXISTS(SELECT 1 + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'step_instance_file' + AND COLUMN_NAME = 'task_instance_id') THEN + ALTER TABLE step_instance_file ADD COLUMN `task_instance_id` bigint(20) NOT NULL DEFAULT '0'; + END IF; + IF NOT EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'step_instance_file' + AND INDEX_NAME = 'idx_task_instance_id') THEN + ALTER TABLE step_instance_file ADD INDEX `idx_task_instance_id` (`task_instance_id`); + END IF; + + IF NOT EXISTS(SELECT 1 + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'step_instance_confirm' + AND COLUMN_NAME = 'task_instance_id') THEN + ALTER TABLE step_instance_confirm ADD COLUMN `task_instance_id` bigint(20) NOT NULL DEFAULT '0'; + END IF; + IF NOT EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'step_instance_confirm' + AND INDEX_NAME = 'idx_task_instance_id') THEN + ALTER TABLE step_instance_confirm ADD INDEX `idx_task_instance_id` (`task_instance_id`); + END IF; + + IF NOT EXISTS(SELECT 1 + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'gse_task' + AND COLUMN_NAME = 'task_instance_id') THEN + ALTER TABLE gse_task ADD COLUMN `task_instance_id` bigint(20) NOT NULL DEFAULT '0'; + END IF; + IF NOT EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'gse_task' + AND INDEX_NAME = 'idx_task_instance_id') THEN + ALTER TABLE gse_task ADD INDEX `idx_task_instance_id` (`task_instance_id`); + END IF; + + IF NOT EXISTS(SELECT 1 + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'gse_script_agent_task' + AND COLUMN_NAME = 'task_instance_id') THEN + ALTER TABLE gse_script_agent_task ADD COLUMN `task_instance_id` bigint(20) NOT NULL DEFAULT '0'; + END IF; + IF NOT EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'gse_script_agent_task' + AND INDEX_NAME = 'idx_task_instance_id') THEN + ALTER TABLE gse_script_agent_task ADD INDEX `idx_task_instance_id` (`task_instance_id`); + END IF; + + IF NOT EXISTS(SELECT 1 + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'gse_file_agent_task' + AND COLUMN_NAME = 'task_instance_id') THEN + ALTER TABLE gse_file_agent_task ADD COLUMN `task_instance_id` bigint(20) NOT NULL DEFAULT '0'; + END IF; + IF NOT EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'gse_file_agent_task' + AND INDEX_NAME = 'idx_task_instance_id') THEN + ALTER TABLE gse_file_agent_task ADD INDEX `idx_task_instance_id` (`task_instance_id`); + END IF; + + IF NOT EXISTS(SELECT 1 + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'file_source_task_log' + AND COLUMN_NAME = 'task_instance_id') THEN + ALTER TABLE file_source_task_log ADD COLUMN `task_instance_id` bigint(20) NOT NULL DEFAULT '0'; + END IF; + IF NOT EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'file_source_task_log' + AND INDEX_NAME = 'idx_task_instance_id') THEN + ALTER TABLE file_source_task_log ADD INDEX `idx_task_instance_id` (`task_instance_id`); + END IF; + + IF NOT EXISTS(SELECT 1 + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'step_instance_rolling_task' + AND COLUMN_NAME = 'task_instance_id') THEN + ALTER TABLE step_instance_rolling_task ADD COLUMN `task_instance_id` bigint(20) NOT NULL DEFAULT '0'; + END IF; + IF NOT EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'step_instance_rolling_task' + AND INDEX_NAME = 'idx_task_instance_id') THEN + ALTER TABLE step_instance_rolling_task ADD INDEX `idx_task_instance_id` (`task_instance_id`); + END IF; + + COMMIT; +END +DELIMITER ; +COMMIT; + +CALL job_schema_update(); + +DROP PROCEDURE IF EXISTS job_schema_update; \ No newline at end of file From 10f85d5cee7f436d5d16ab7a9fed291e47688d2e Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Tue, 21 May 2024 15:00:04 +0800 Subject: [PATCH 02/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../util/feature/FeatureIdConstants.java | 4 + ...ecuteObjectTaskDAOImplIntegrationTest.java | 24 +++-- .../impl/GseTaskDAOImplIntegrationTest.java | 8 +- ...ecuteObjectTaskDAOImplIntegrationTest.java | 21 +++-- ...anceRollingTaskDAOImplIntegrationTest.java | 11 ++- .../src/test/resources/init_gse_task_data.sql | 8 +- .../init_step_instance_rolling_task_data.sql | 13 +-- .../web/impl/WebSearchToolsResourceImpl.java | 2 +- .../execute/dao/FileExecuteObjectTaskDAO.java | 32 +++++-- .../job/execute/dao/FileSourceTaskLogDAO.java | 10 +- .../bk/job/execute/dao/GseTaskDAO.java | 12 ++- .../dao/ScriptExecuteObjectTaskDAO.java | 36 ++++++-- .../bk/job/execute/dao/StepInstanceDAO.java | 58 ++++++------ .../dao/StepInstanceRollingTaskDAO.java | 12 ++- .../impl/FileExecuteObjectTaskDAOImpl.java | 45 +++++++-- .../dao/impl/FileSourceTaskLogDAOImpl.java | 53 ++++++++--- .../job/execute/dao/impl/GseTaskDAOImpl.java | 42 +++++++-- .../impl/ScriptExecuteObjectTaskDAOImpl.java | 52 ++++++++--- .../execute/dao/impl/StepInstanceDAOImpl.java | 91 ++++++++++++------- .../impl/StepInstanceRollingTaskDAOImpl.java | 28 +++++- .../impl/TaskInstanceIdDynamicCondition.java | 67 ++++++++++++++ .../engine/listener/GseStepEventHandler.java | 64 ++++++++----- .../engine/listener/GseTaskListener.java | 2 +- .../listener/ResultHandleResumeListener.java | 2 +- .../engine/listener/event/GseTaskEvent.java | 2 +- .../engine/listener/event/StepEvent.java | 28 +++--- .../third/ThirdFilePrepareService.java | 1 + .../prepare/third/ThirdFilePrepareTask.java | 7 +- .../execute/model/FileSourceTaskLogDTO.java | 2 + .../bk/job/execute/model/GseTaskDTO.java | 6 +- .../model/StepInstanceRollingTaskDTO.java | 5 + .../service/ExecuteObjectTaskService.java | 3 +- .../job/execute/service/GseTaskService.java | 12 ++- .../StepInstanceRollingTaskService.java | 17 +++- .../execute/service/StepInstanceService.java | 50 +++++----- .../FileExecuteObjectTaskServiceImpl.java | 48 +++++++--- .../service/impl/GseTaskServiceImpl.java | 8 +- .../service/impl/NotifyServiceImpl.java | 18 ++-- .../ScriptExecuteObjectTaskServiceImpl.java | 58 ++++++++---- .../StepInstanceRollingTaskServiceImpl.java | 31 ++++--- .../service/impl/StepInstanceServiceImpl.java | 52 +++++------ .../service/impl/TaskResultServiceImpl.java | 10 +- ...ob_execute_20240520-1000_V3.10.0_mysql.sql | 58 ++++++------ 43 files changed, 736 insertions(+), 377 deletions(-) create mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/feature/FeatureIdConstants.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/feature/FeatureIdConstants.java index 7922d13123..afddc2002d 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/feature/FeatureIdConstants.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/feature/FeatureIdConstants.java @@ -57,4 +57,8 @@ public interface FeatureIdConstants { * 特性: 容器执行 */ String FEATURE_CONTAINER_EXECUTE = "containerExecution"; + /** + * 特性: 作业执行增加db 所有表增加 task_instance_id 字段作为分库分表 shard_key + */ + String DAO_ADD_TASK_INSTANCE_ID = "daoAddTaskInstanceId"; } diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileExecuteObjectTaskDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileExecuteObjectTaskDAOImplIntegrationTest.java index 3530de9df8..6e1fc66b69 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileExecuteObjectTaskDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileExecuteObjectTaskDAOImplIntegrationTest.java @@ -60,13 +60,15 @@ public class FileExecuteObjectTaskDAOImplIntegrationTest { @DisplayName("根据执行对象ID获取任务") public void testGetTaskByExecuteObjectId() { String executeObjectId = "1:101"; + long taskInstanceId = 1L; long stepInstanceId = 1L; int executeCount = 0; int batch = 1; FileTaskModeEnum mode = FileTaskModeEnum.UPLOAD; - ExecuteObjectTask executeObjectTask = fileExecuteObjectTaskDAO.getTaskByExecuteObjectId(stepInstanceId, - executeCount, batch, mode, executeObjectId); + ExecuteObjectTask executeObjectTask = fileExecuteObjectTaskDAO.getTaskByExecuteObjectId( + taskInstanceId, stepInstanceId, executeCount, batch, mode, executeObjectId); + assertThat(executeObjectTask.getTaskInstanceId()).isEqualTo(taskInstanceId); assertThat(executeObjectTask.getStepInstanceId()).isEqualTo(stepInstanceId); assertThat(executeObjectTask.getExecuteCount()).isEqualTo(executeCount); assertThat(executeObjectTask.getBatch()).isEqualTo(batch); @@ -86,6 +88,7 @@ public void testGetTaskByExecuteObjectId() { public void testBatchSaveTasks() { List executeObjectTaskList = new ArrayList<>(); ExecuteObjectTask executeObjectTask1 = new ExecuteObjectTask(); + executeObjectTask1.setTaskInstanceId(100L); executeObjectTask1.setStepInstanceId(100L); executeObjectTask1.setExecuteCount(1); executeObjectTask1.setActualExecuteCount(1); @@ -102,6 +105,7 @@ public void testBatchSaveTasks() { executeObjectTaskList.add(executeObjectTask1); ExecuteObjectTask executeObjectTask2 = new ExecuteObjectTask(); + executeObjectTask2.setTaskInstanceId(100L); executeObjectTask2.setStepInstanceId(100L); executeObjectTask2.setExecuteCount(1); executeObjectTask2.setActualExecuteCount(1); @@ -120,8 +124,9 @@ public void testBatchSaveTasks() { fileExecuteObjectTaskDAO.batchSaveTasks(executeObjectTaskList); - ExecuteObjectTask executeObjectTask1Return = fileExecuteObjectTaskDAO.getTaskByExecuteObjectId(100L, 1, 1, + ExecuteObjectTask executeObjectTask1Return = fileExecuteObjectTaskDAO.getTaskByExecuteObjectId(100L, 100L, 1, 1, FileTaskModeEnum.UPLOAD, "1:101"); + assertThat(executeObjectTask1Return.getTaskInstanceId()).isEqualTo(100L); assertThat(executeObjectTask1Return.getStepInstanceId()).isEqualTo(100L); assertThat(executeObjectTask1Return.getExecuteCount()).isEqualTo(1L); assertThat(executeObjectTask1Return.getActualExecuteCount()).isEqualTo(1L); @@ -137,8 +142,9 @@ public void testBatchSaveTasks() { assertThat(executeObjectTask1Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.AGENT_ERROR); - ExecuteObjectTask executeObjectTask2Return = fileExecuteObjectTaskDAO.getTaskByExecuteObjectId(100L, 1, 1, + ExecuteObjectTask executeObjectTask2Return = fileExecuteObjectTaskDAO.getTaskByExecuteObjectId(100L, 100L, 1, 1, FileTaskModeEnum.DOWNLOAD, "1:102"); + assertThat(executeObjectTask2Return.getTaskInstanceId()).isEqualTo(100L); assertThat(executeObjectTask2Return.getStepInstanceId()).isEqualTo(100L); assertThat(executeObjectTask2Return.getExecuteCount()).isEqualTo(1L); assertThat(executeObjectTask2Return.getActualExecuteCount()).isEqualTo(1L); @@ -193,7 +199,7 @@ public void testBatchUpdateAgentTasks() { fileExecuteObjectTaskDAO.batchUpdateTasks(executeObjectTaskList); - ExecuteObjectTask executeObjectTask1Return = fileExecuteObjectTaskDAO.getTaskByExecuteObjectId(1L, 0, 2, + ExecuteObjectTask executeObjectTask1Return = fileExecuteObjectTaskDAO.getTaskByExecuteObjectId(1L, 1L, 0, 2, FileTaskModeEnum.UPLOAD, "1:101"); assertThat(executeObjectTask1Return.getStepInstanceId()).isEqualTo(1L); assertThat(executeObjectTask1Return.getExecuteCount()).isEqualTo(0L); @@ -208,7 +214,7 @@ public void testBatchUpdateAgentTasks() { assertThat(executeObjectTask1Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.AGENT_ERROR); - ExecuteObjectTask executeObjectTask2Return = fileExecuteObjectTaskDAO.getTaskByExecuteObjectId(1L, 0, 2, + ExecuteObjectTask executeObjectTask2Return = fileExecuteObjectTaskDAO.getTaskByExecuteObjectId(1L, 1L, 0, 2, FileTaskModeEnum.DOWNLOAD, "1:103"); assertThat(executeObjectTask2Return.getStepInstanceId()).isEqualTo(1L); assertThat(executeObjectTask2Return.getExecuteCount()).isEqualTo(0L); @@ -226,7 +232,7 @@ public void testBatchUpdateAgentTasks() { @Test @DisplayName("任务结果分组") public void listResultGroups() { - List resultGroups = fileExecuteObjectTaskDAO.listResultGroups(1L, 0, null); + List resultGroups = fileExecuteObjectTaskDAO.listResultGroups(1L, 1L, 0, null); assertThat(resultGroups.size()).isEqualTo(2); assertThat(resultGroups).extracting("status").containsOnly(9, 11); @@ -239,7 +245,7 @@ public void listResultGroups() { } // 根据滚动执行批次查询 - resultGroups = fileExecuteObjectTaskDAO.listResultGroups(1L, 0, 2); + resultGroups = fileExecuteObjectTaskDAO.listResultGroups(1L, 1L, 0, 2); assertThat(resultGroups.size()).isEqualTo(2); assertThat(resultGroups).extracting("status").containsOnly(9, 11); @@ -254,7 +260,7 @@ public void listResultGroups() { @Test public void testListAgentTaskByResultGroup() { - List executeObjectTasks = fileExecuteObjectTaskDAO.listTaskByResultGroup(1L, 0, 2, 9); + List executeObjectTasks = fileExecuteObjectTaskDAO.listTaskByResultGroup(1L, 1L, 0, 2, 9); assertThat(executeObjectTasks.size()).isEqualTo(1); assertThat(executeObjectTasks.get(0).getStepInstanceId()).isEqualTo(1L); assertThat(executeObjectTasks.get(0).getExecuteCount()).isEqualTo(0); diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImplIntegrationTest.java index 159bea4f45..57525454c2 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImplIntegrationTest.java @@ -55,6 +55,7 @@ public class GseTaskDAOImplIntegrationTest { @DisplayName("保存GseTask") void saveGseTask() { GseTaskDTO gseTask = new GseTaskDTO(); + gseTask.setTaskInstanceId(9L); gseTask.setStepInstanceId(10L); gseTask.setExecuteCount(0); gseTask.setBatch(1); @@ -67,8 +68,9 @@ void saveGseTask() { long id = gseTaskDAO.saveGseTask(gseTask); assertThat(id).isGreaterThan(0); - GseTaskDTO savedGseTask = gseTaskDAO.getGseTask(id); + GseTaskDTO savedGseTask = gseTaskDAO.getGseTask(9L, id); + assertThat(savedGseTask.getTaskInstanceId()).isEqualTo(9L); assertThat(savedGseTask.getStepInstanceId()).isEqualTo(10L); assertThat(savedGseTask.getExecuteCount()).isEqualTo(0); assertThat(savedGseTask.getBatch()).isEqualTo(1); @@ -84,6 +86,7 @@ void saveGseTask() { void updateGseTask() { GseTaskDTO gseTask = new GseTaskDTO(); gseTask.setId(1L); + gseTask.setTaskInstanceId(1L); gseTask.setStepInstanceId(1L); gseTask.setExecuteCount(0); gseTask.setBatch(0); @@ -96,8 +99,9 @@ void updateGseTask() { boolean result = gseTaskDAO.updateGseTask(gseTask); assertThat(result).isTrue(); - GseTaskDTO savedGseTask = gseTaskDAO.getGseTask(1L); + GseTaskDTO savedGseTask = gseTaskDAO.getGseTask(1L, 1L); + assertThat(savedGseTask.getTaskInstanceId()).isEqualTo(1L); assertThat(savedGseTask.getStepInstanceId()).isEqualTo(1L); assertThat(savedGseTask.getExecuteCount()).isEqualTo(0); assertThat(savedGseTask.getBatch()).isEqualTo(0); diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptExecuteObjectTaskDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptExecuteObjectTaskDAOImplIntegrationTest.java index 61e5d2d07a..16dd78b2f8 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptExecuteObjectTaskDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptExecuteObjectTaskDAOImplIntegrationTest.java @@ -63,8 +63,8 @@ public void testGetTaskByExecuteObjectId() { long taskInstanceId = 1L; int executeCount = 0; int batch = 1; - ExecuteObjectTask executeObjectTask = scriptExecuteObjectTaskDAO.getTaskByExecuteObjectId(stepInstanceId, - executeCount, batch, executeObjectId); + ExecuteObjectTask executeObjectTask = scriptExecuteObjectTaskDAO.getTaskByExecuteObjectId( + taskInstanceId, stepInstanceId, executeCount, batch, executeObjectId); assertThat(executeObjectTask.getStepInstanceId()).isEqualTo(stepInstanceId); assertThat(executeObjectTask.getTaskInstanceId()).isEqualTo(taskInstanceId); @@ -130,7 +130,7 @@ public void testBatchSaveTasks() { scriptExecuteObjectTaskDAO.batchSaveTasks(executeObjectTaskList); ExecuteObjectTask executeObjectTask1Return = scriptExecuteObjectTaskDAO - .getTaskByExecuteObjectId(100L, 1, 1, "1:101"); + .getTaskByExecuteObjectId(100L, 100L, 1, 1, "1:101"); assertThat(executeObjectTask1Return.getTaskInstanceId()).isEqualTo(100L); assertThat(executeObjectTask1Return.getStepInstanceId()).isEqualTo(100L); assertThat(executeObjectTask1Return.getExecuteCount()).isEqualTo(1L); @@ -149,8 +149,9 @@ public void testBatchSaveTasks() { ExecuteObjectTask executeObjectTask2Return = scriptExecuteObjectTaskDAO - .getTaskByExecuteObjectId(100L, 1, 1, "1:102"); + .getTaskByExecuteObjectId(100L, 100L, 1, 1, "1:102"); assertThat(executeObjectTask2Return.getStepInstanceId()).isEqualTo(100L); + assertThat(executeObjectTask2Return.getTaskInstanceId()).isEqualTo(100L); assertThat(executeObjectTask2Return.getExecuteCount()).isEqualTo(1L); assertThat(executeObjectTask2Return.getActualExecuteCount()).isEqualTo(1L); assertThat(executeObjectTask2Return.getBatch()).isEqualTo(1); @@ -209,7 +210,7 @@ public void testBatchUpdateTasks() { scriptExecuteObjectTaskDAO.batchUpdateTasks(executeObjectTaskList); ExecuteObjectTask executeObjectTask1Return = - scriptExecuteObjectTaskDAO.getTaskByExecuteObjectId(1L, 0, 3, "1:103"); + scriptExecuteObjectTaskDAO.getTaskByExecuteObjectId(1L, 1L, 0, 3, "1:103"); assertThat(executeObjectTask1Return.getStepInstanceId()).isEqualTo(1L); assertThat(executeObjectTask1Return.getExecuteCount()).isEqualTo(0L); assertThat(executeObjectTask1Return.getBatch()).isEqualTo(3); @@ -226,7 +227,7 @@ public void testBatchUpdateTasks() { ExecuteObjectTask executeObjectTask2Return = - scriptExecuteObjectTaskDAO.getTaskByExecuteObjectId(1L, 0, 3, "1:104"); + scriptExecuteObjectTaskDAO.getTaskByExecuteObjectId(1L, 1L, 0, 3, "1:104"); assertThat(executeObjectTask2Return.getStepInstanceId()).isEqualTo(1L); assertThat(executeObjectTask2Return.getExecuteCount()).isEqualTo(0L); assertThat(executeObjectTask2Return.getBatch()).isEqualTo(3); @@ -244,14 +245,14 @@ public void testBatchUpdateTasks() { @Test public void testGetSuccessIpCount() { - Integer count = scriptExecuteObjectTaskDAO.getSuccessTaskCount(1L, 0); + Integer count = scriptExecuteObjectTaskDAO.getSuccessTaskCount(1L, 1L, 0); assertThat(count).isEqualTo(4); } @Test @DisplayName("查询任务结果分组") public void listResultGroups() { - List resultGroups = scriptExecuteObjectTaskDAO.listResultGroups(1L, 0, null); + List resultGroups = scriptExecuteObjectTaskDAO.listResultGroups(1L, 1L, 0, null); assertThat(resultGroups.size()).isEqualTo(2); assertThat(resultGroups).extracting("status").containsOnly(9, 11); @@ -265,7 +266,7 @@ public void listResultGroups() { } // 根据滚动执行批次查询 - resultGroups = scriptExecuteObjectTaskDAO.listResultGroups(1L, 0, 3); + resultGroups = scriptExecuteObjectTaskDAO.listResultGroups(1L, 1L, 0, 3); assertThat(resultGroups.size()).isEqualTo(2); assertThat(resultGroups).extracting("status").containsOnly(9, 11); @@ -281,7 +282,7 @@ public void listResultGroups() { @Test public void testListAgentTaskByResultGroup() { - List executeObjectTasks = scriptExecuteObjectTaskDAO.listTasksByResultGroup(1L, 0, 1, 9, + List executeObjectTasks = scriptExecuteObjectTaskDAO.listTasksByResultGroup(1L, 1L, 0, 1, 9, "succ"); assertThat(executeObjectTasks.size()).isEqualTo(1); assertThat(executeObjectTasks.get(0).getStepInstanceId()).isEqualTo(1L); diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImplIntegrationTest.java index 31378cb33d..d6baf56544 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImplIntegrationTest.java @@ -56,7 +56,7 @@ public class StepInstanceRollingTaskDAOImplIntegrationTest { @Test @DisplayName("根据步骤实例ID、执行次数、滚动批次查询步骤滚动任务") void queryRollingTask() { - StepInstanceRollingTaskDTO rollingTask = stepInstanceRollingTaskDAO.queryRollingTask(1, 0, 1); + StepInstanceRollingTaskDTO rollingTask = stepInstanceRollingTaskDAO.queryRollingTask(1L, 1, 0, 1); assertThat(rollingTask).isNotNull(); assertThat(rollingTask.getStepInstanceId()).isEqualTo(1L); assertThat(rollingTask.getExecuteCount()).isEqualTo(0); @@ -70,7 +70,7 @@ void queryRollingTask() { @Test @DisplayName("根据步骤实例ID查询步骤滚动任务") void listRollingTasks() { - List rollingTasks = stepInstanceRollingTaskDAO.listRollingTasks(1L, null, null); + List rollingTasks = stepInstanceRollingTaskDAO.listRollingTasks(1L, 1L, null, null); assertThat(rollingTasks).hasSize(2); StepInstanceRollingTaskDTO rollingTask1 = rollingTasks.get(0); @@ -105,7 +105,7 @@ void saveRollingTask() { rollingTask.setStartTime(1642247802000L); stepInstanceRollingTaskDAO.saveRollingTask(rollingTask); - StepInstanceRollingTaskDTO savedRollingTask = stepInstanceRollingTaskDAO.queryRollingTask(100L, 0, 1); + StepInstanceRollingTaskDTO savedRollingTask = stepInstanceRollingTaskDAO.queryRollingTask(100L, 100L, 0, 1); assertThat(savedRollingTask).isNotNull(); assertThat(savedRollingTask.getStepInstanceId()).isEqualTo(100L); assertThat(savedRollingTask.getExecuteCount()).isEqualTo(0); @@ -121,9 +121,10 @@ void updateRollingTask() { long startTime = System.currentTimeMillis(); Long endTime = startTime + 1000L; Long totalTime = 1000L; - stepInstanceRollingTaskDAO.updateRollingTask(1L, 0, 1, RunStatusEnum.SUCCESS, startTime, endTime, totalTime); + stepInstanceRollingTaskDAO.updateRollingTask(1L, 1L, 0, 1, RunStatusEnum.SUCCESS, startTime, endTime, + totalTime); - StepInstanceRollingTaskDTO savedRollingTask = stepInstanceRollingTaskDAO.queryRollingTask(1L, 0, 1); + StepInstanceRollingTaskDTO savedRollingTask = stepInstanceRollingTaskDAO.queryRollingTask(1L, 1L, 0, 1); assertThat(savedRollingTask).isNotNull(); assertThat(savedRollingTask.getStepInstanceId()).isEqualTo(1L); diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_gse_task_data.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_gse_task_data.sql index 69c531371a..ed653c5cc7 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/init_gse_task_data.sql +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_gse_task_data.sql @@ -24,8 +24,8 @@ TRUNCATE TABLE job_execute.gse_task; -INSERT INTO job_execute.gse_task (id,step_instance_id,execute_count,batch,status,start_time,end_time,total_time,gse_task_id) +INSERT INTO job_execute.gse_task (id,task_instance_id,step_instance_id,execute_count,batch,status,start_time,end_time,total_time,gse_task_id) VALUES -(1,1,0,0,4,1660639901000,1660639902000,1000,'GSE_TASK_1'), -(2,2,0,1,3,1660639903000,1660639904000,1000,'GSE_TASK_2'), -(3,2,0,2,4,1660639905000,1660639906000,1000,'GSE_TASK_3'); +(1,1,1,0,0,4,1660639901000,1660639902000,1000,'GSE_TASK_1'), +(2,2,2,0,1,3,1660639903000,1660639904000,1000,'GSE_TASK_2'), +(3,2,2,0,2,4,1660639905000,1660639906000,1000,'GSE_TASK_3'); diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_step_instance_rolling_task_data.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_step_instance_rolling_task_data.sql index 780a6c7e77..1d18771356 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/init_step_instance_rolling_task_data.sql +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_step_instance_rolling_task_data.sql @@ -25,7 +25,8 @@ TRUNCATE TABLE step_instance_rolling_task; INSERT INTO job_execute.step_instance_rolling_task - (step_instance_id, + (task_instance_id, + step_instance_id, execute_count, batch, status, @@ -33,9 +34,9 @@ INSERT INTO job_execute.step_instance_rolling_task end_time, total_time) VALUES - (1,0,1,1,1642247802000,1642247803000,1000), - (1,0,2,1,1642247804000,1642247805000,1000), - (2,0,1,1,1642247804000,1642247805000,1000), - (2,0,2,1,1642247805000,1642247806000,1000), - (2,0,3,1,1642247806000,1642247807000,1000); + (1,1,0,1,1,1642247802000,1642247803000,1000), + (1,1,0,2,1,1642247804000,1642247805000,1000), + (2,2,0,1,1,1642247804000,1642247805000,1000), + (2,2,0,2,1,1642247805000,1642247806000,1000), + (2,2,0,3,1,1642247806000,1642247807000,1000); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebSearchToolsResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebSearchToolsResourceImpl.java index 67a70e3487..d4e5cc8993 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebSearchToolsResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebSearchToolsResourceImpl.java @@ -106,7 +106,7 @@ public Response> getTaskLinkByStepId(String username, auth(username, appResourceScope); List stepInstanceRollingTaskDTOS = - stepInstanceRollingTaskService.listRollingTasksByStep(stepInstanceId); + stepInstanceRollingTaskService.listRollingTasksByStep(stepInstanceBase.getTaskInstanceId(), stepInstanceId); List gseTaskDTOList = new ArrayList<>(); // 是否滚动执行 if (CollectionUtils.isNotEmpty(stepInstanceRollingTaskDTOS)) { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileExecuteObjectTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileExecuteObjectTaskDAO.java index bebb532bce..6d5ce23e40 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileExecuteObjectTaskDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileExecuteObjectTaskDAO.java @@ -53,21 +53,26 @@ public interface FileExecuteObjectTaskDAO { /** * 获取步骤成功执行的任务数量 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @return 步骤成功执行的任务数量 */ - int getSuccessTaskCount(long stepInstanceId, int executeCount); + int getSuccessTaskCount(Long taskInstanceId, long stepInstanceId, int executeCount); /** * 查询执行结果分组 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 * @return 执行结果分组 */ - List listResultGroups(long stepInstanceId, int executeCount, Integer batch); + List listResultGroups(Long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer batch); /** * 根据执行结果查询任务 @@ -78,7 +83,8 @@ public interface FileExecuteObjectTaskDAO { * @param status 任务状态 * @return 任务 */ - List listTaskByResultGroup(Long stepInstanceId, + List listTaskByResultGroup(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, Integer status); @@ -86,6 +92,7 @@ List listTaskByResultGroup(Long stepInstanceId, /** * 根据执行结果查询任务(排序、限制返回数量) * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 @@ -95,7 +102,8 @@ List listTaskByResultGroup(Long stepInstanceId, * @param order 排序方式 * @return 任务 */ - List listTaskByResultGroup(Long stepInstanceId, + List listTaskByResultGroup(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, Integer status, @@ -106,13 +114,15 @@ List listTaskByResultGroup(Long stepInstanceId, /** * 获取agent任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；传入null或者0将忽略该参数 * @param fileTaskMode 文件分发任务模式;传入null表示忽略该过滤条件 * @return agent任务 */ - List listTasks(Long stepInstanceId, + List listTasks(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, FileTaskModeEnum fileTaskMode); @@ -123,11 +133,12 @@ List listTasks(Long stepInstanceId, * @param gseTaskId GSE任务ID * @return agent任务 */ - List listTasksByGseTaskId(Long gseTaskId); + List listTasksByGseTaskId(Long taskInstanceId, Long gseTaskId); /** * 根据执行对象ID查询任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；传入null或者0将忽略该参数 @@ -135,7 +146,8 @@ List listTasks(Long stepInstanceId, * @param executeObjectId 执行对象ID * @return 任务 */ - ExecuteObjectTask getTaskByExecuteObjectId(Long stepInstanceId, + ExecuteObjectTask getTaskByExecuteObjectId(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, FileTaskModeEnum mode, @@ -147,18 +159,20 @@ ExecuteObjectTask getTaskByExecuteObjectId(Long stepInstanceId, * * @param stepInstanceId 步骤实例ID */ - boolean isStepInstanceRecordExist(long stepInstanceId); + boolean isStepInstanceRecordExist(Long taskInstanceId, long stepInstanceId); /** * 批量更新Task的字段 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 条件 - 步骤实例ID * @param executeCount 条件 - 重试次数 * @param batch 条件 - 滚动执行批次；传入null将忽略该条件 * @param actualExecuteCount 值 - 任务实际执行的步骤重试次数；如果传入null，则不更新 * @param gseTaskId 值 - 任务对应的GSE_TASK_ID；如果传入null，则不更新 */ - void updateTaskFields(long stepInstanceId, + void updateTaskFields(Long taskInstanceId, + long stepInstanceId, int executeCount, Integer batch, Integer actualExecuteCount, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileSourceTaskLogDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileSourceTaskLogDAO.java index e9ebecdfcb..7f6fc10a30 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileSourceTaskLogDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileSourceTaskLogDAO.java @@ -30,10 +30,14 @@ public interface FileSourceTaskLogDAO { void insertOrUpdateFileSourceTaskLog(FileSourceTaskLogDTO fileSourceTaskLog); - FileSourceTaskLogDTO getFileSourceTaskLog(long stepInstanceId, int executeCount); + FileSourceTaskLogDTO getFileSourceTaskLog(Long taskInstanceId, long stepInstanceId, int executeCount); - FileSourceTaskLogDTO getFileSourceTaskLogByBatchTaskId(String fileSourceBatchTaskId); + FileSourceTaskLogDTO getFileSourceTaskLogByBatchTaskId(Long taskInstanceId, String fileSourceBatchTaskId); - int updateTimeConsumingByBatchTaskId(String fileSourceBatchTaskId, Long startTime, Long endTime, Long totalTime); + int updateTimeConsumingByBatchTaskId(Long taskInstanceId, + String fileSourceBatchTaskId, + Long startTime, + Long endTime, + Long totalTime); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/GseTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/GseTaskDAO.java index bf1be9d6e2..1ff39d5f04 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/GseTaskDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/GseTaskDAO.java @@ -53,20 +53,22 @@ public interface GseTaskDAO { /** * 获取 GSE 任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 步骤执行次数 * @param batch 滚动执行批次 * @return GSE 任务 */ - GseTaskDTO getGseTask(long stepInstanceId, int executeCount, Integer batch); + GseTaskDTO getGseTask(Long taskInstanceId, long stepInstanceId, int executeCount, Integer batch); /** * 获取 GSE 任务 * - * @param gseTaskId GSE任务ID + * @param taskInstanceId 作业实例 ID + * @param gseTaskId GSE任务ID * @return GSE 任务 */ - GseTaskDTO getGseTask(long gseTaskId); + GseTaskDTO getGseTask(Long taskInstanceId, long gseTaskId); /** * 获取 GSE 任务 @@ -80,8 +82,8 @@ public interface GseTaskDAO { * 获取 GSE 任务列表 * * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - * @param batch 批次 + * @param executeCount 执行次数 + * @param batch 批次 * @return GSE 任务列表 */ List ListGseTaskSimpleInfo(Long stepInstanceId, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptExecuteObjectTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptExecuteObjectTaskDAO.java index 089f7ea2af..85560e8024 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptExecuteObjectTaskDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptExecuteObjectTaskDAO.java @@ -56,21 +56,26 @@ public interface ScriptExecuteObjectTaskDAO { * @param executeCount 执行次数 * @return 步骤成功执行的任务数量 */ - int getSuccessTaskCount(long stepInstanceId, int executeCount); + int getSuccessTaskCount(Long taskInstanceId, long stepInstanceId, int executeCount); /** * 查询执行结果分组 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 * @return 执行结果分组 */ - List listResultGroups(long stepInstanceId, int executeCount, Integer batch); + List listResultGroups(Long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer batch); /** * 根据执行结果查询任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 @@ -78,7 +83,8 @@ public interface ScriptExecuteObjectTaskDAO { * @param tag 用户自定义分组标签 * @return 任务 */ - List listTasksByResultGroup(Long stepInstanceId, + List listTasksByResultGroup(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, Integer status, @@ -87,6 +93,7 @@ List listTasksByResultGroup(Long stepInstanceId, /** * 根据执行结果查询任务(排序、限制返回数量) * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 @@ -97,7 +104,8 @@ List listTasksByResultGroup(Long stepInstanceId, * @param order 排序方式 * @return 任务 */ - List listTasksByResultGroup(Long stepInstanceId, + List listTasksByResultGroup(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, Integer status, @@ -109,33 +117,38 @@ List listTasksByResultGroup(Long stepInstanceId, /** * 获取任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；传入null或者0将忽略该参数 * @return 任务信息 */ - List listTasks(Long stepInstanceId, + List listTasks(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch); /** * 根据GSE任务ID获取任务 * - * @param gseTaskId GSE任务ID + * @param taskInstanceId 作业实例 ID + * @param gseTaskId GSE任务ID * @return 任务 */ - List listTasksByGseTaskId(Long gseTaskId); + List listTasksByGseTaskId(Long taskInstanceId, Long gseTaskId); /** * 根据hostId查询任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；传入null或者0将忽略该参数 * @param executeObjectId 执行对象ID * @return 任务 */ - ExecuteObjectTask getTaskByExecuteObjectId(Long stepInstanceId, + ExecuteObjectTask getTaskByExecuteObjectId(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, String executeObjectId); @@ -143,20 +156,23 @@ ExecuteObjectTask getTaskByExecuteObjectId(Long stepInstanceId, /** * 判断步骤实例的执行对象任务记录是否存在 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID */ - boolean isStepInstanceRecordExist(long stepInstanceId); + boolean isStepInstanceRecordExist(Long taskInstanceId, long stepInstanceId); /** * 部分更新任务的字段 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 条件 - 步骤实例ID * @param executeCount 条件 - 重试次数 * @param batch 条件 - 滚动执行批次；传入null将忽略该条件 * @param actualExecuteCount 值 - 任务实际执行的步骤重试次数；如果传入null，则不更新 * @param gseTaskId 值 - 任务对应的GSE_TASK_ID；如果传入null，则不更新 */ - void updateTaskFields(long stepInstanceId, + void updateTaskFields(Long taskInstanceId, + long stepInstanceId, int executeCount, Integer batch, Integer actualExecuteCount, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceDAO.java index ef9c5702d3..18bab32cb9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceDAO.java @@ -46,7 +46,7 @@ public interface StepInstanceDAO { void addConfirmStepInstance(StepInstanceDTO stepInstance); - StepInstanceBaseDTO getStepInstanceBase(long taskInstanceId, long stepInstanceId); + StepInstanceBaseDTO getStepInstanceBase(Long taskInstanceId, long stepInstanceId); StepInstanceBaseDTO getStepInstanceBase(long stepInstanceId); @@ -56,7 +56,7 @@ public interface StepInstanceDAO { * @param taskInstanceId 作业实例ID * @return 作业第一个步骤实例 */ - StepInstanceBaseDTO getFirstStepInstanceBase(long taskInstanceId); + StepInstanceBaseDTO getFirstStepInstanceBase(Long taskInstanceId); /** * 获取下一个步骤实例 @@ -65,25 +65,25 @@ public interface StepInstanceDAO { * @param currentStepOrder 当前步骤的顺序 * @return 步骤实例；如果当前为最后一个步骤实例，那么返回null */ - StepInstanceBaseDTO getNextStepInstance(long taskInstanceId, int currentStepOrder); + StepInstanceBaseDTO getNextStepInstance(Long taskInstanceId, int currentStepOrder); - ScriptStepInstanceDTO getScriptStepInstance(long taskInstanceId, long stepInstanceId); + ScriptStepInstanceDTO getScriptStepInstance(Long taskInstanceId, long stepInstanceId); - FileStepInstanceDTO getFileStepInstance(long taskInstanceId, long stepInstanceId); + FileStepInstanceDTO getFileStepInstance(Long taskInstanceId, long stepInstanceId); - ConfirmStepInstanceDTO getConfirmStepInstance(long taskInstanceId, long stepInstanceId); + ConfirmStepInstanceDTO getConfirmStepInstance(Long taskInstanceId, long stepInstanceId); - List listStepInstanceBaseByTaskInstanceId(long taskInstanceId); + List listStepInstanceBaseByTaskInstanceId(Long taskInstanceId); - void resetStepStatus(long taskInstanceId, long stepInstanceId); + void resetStepStatus(Long taskInstanceId, long stepInstanceId); - void resetStepExecuteInfoForRetry(long taskInstanceId, long stepInstanceId); + void resetStepExecuteInfoForRetry(Long taskInstanceId, long stepInstanceId); - void addStepExecuteCount(long taskInstanceId, long stepInstanceId); + void addStepExecuteCount(Long taskInstanceId, long stepInstanceId); - void updateStepStatus(long taskInstanceId, long stepInstanceId, int status); + void updateStepStatus(Long taskInstanceId, long stepInstanceId, int status); - void updateStepStartTime(long taskInstanceId, long stepInstanceId, Long startTime); + void updateStepStartTime(Long taskInstanceId, long stepInstanceId, Long startTime); /** * 更新步骤启动时间 - 仅当启动时间为空的的场景 @@ -91,13 +91,13 @@ public interface StepInstanceDAO { * @param stepInstanceId 步骤实例ID * @param startTime 启动时间 */ - void updateStepStartTimeIfNull(long taskInstanceId, long stepInstanceId, Long startTime); + void updateStepStartTimeIfNull(Long taskInstanceId, long stepInstanceId, Long startTime); - void updateStepEndTime(long taskInstanceId, long stepInstanceId, Long endTime); + void updateStepEndTime(Long taskInstanceId, long stepInstanceId, Long endTime); - void addStepInstanceExecuteCount(long taskInstanceId, long stepInstanceId); + void addStepInstanceExecuteCount(Long taskInstanceId, long stepInstanceId); - void updateStepTotalTime(long taskInstanceId, long stepInstanceId, long totalTime); + void updateStepTotalTime(Long taskInstanceId, long stepInstanceId, long totalTime); /** * 更新步骤的执行信息 @@ -109,7 +109,7 @@ public interface StepInstanceDAO { * @param endTime 结束时间 * @param totalTime 总耗时 */ - void updateStepExecutionInfo(long taskInstanceId, + void updateStepExecutionInfo(Long taskInstanceId, long stepInstanceId, RunStatusEnum status, Long startTime, @@ -124,7 +124,7 @@ void updateStepExecutionInfo(long taskInstanceId, * @param isSecureParam 是否为敏感参数 * @param resolvedScriptParam 解析之后的脚本参数 */ - void updateResolvedScriptParam(long taskInstanceId, + void updateResolvedScriptParam(Long taskInstanceId, long stepInstanceId, boolean isSecureParam, String resolvedScriptParam); @@ -136,7 +136,7 @@ void updateResolvedScriptParam(long taskInstanceId, * @param stepInstanceId 步骤实例ID * @param resolvedFileSources 解析后的源文件信息 */ - void updateResolvedSourceFile(long taskInstanceId, + void updateResolvedSourceFile(Long taskInstanceId, long stepInstanceId, List resolvedFileSources); @@ -147,7 +147,7 @@ void updateResolvedSourceFile(long taskInstanceId, * @param stepInstanceId 步骤实例ID * @param resolvedTargetPath 解析之后的目标路径 */ - void updateResolvedTargetPath(long taskInstanceId, long stepInstanceId, String resolvedTargetPath); + void updateResolvedTargetPath(Long taskInstanceId, long stepInstanceId, String resolvedTargetPath); /** * 更新确认理由 @@ -156,7 +156,7 @@ void updateResolvedSourceFile(long taskInstanceId, * @param stepInstanceId 步骤实例ID * @param confirmReason 确认理由 */ - void updateConfirmReason(long taskInstanceId, long stepInstanceId, String confirmReason); + void updateConfirmReason(Long taskInstanceId, long stepInstanceId, String confirmReason); /** * 更新步骤实例操作者 @@ -165,7 +165,7 @@ void updateResolvedSourceFile(long taskInstanceId, * @param stepInstanceId 步骤实例ID * @param operator 操作者 */ - void updateStepOperator(long taskInstanceId, long stepInstanceId, String operator); + void updateStepOperator(Long taskInstanceId, long stepInstanceId, String operator); /** * 获取前一个可执行步骤实例 @@ -174,7 +174,7 @@ void updateResolvedSourceFile(long taskInstanceId, * @param stepInstanceId 当前步骤实例ID * @return 可执行步骤实例 */ - StepInstanceBaseDTO getPreExecutableStepInstance(long taskInstanceId, long stepInstanceId); + StepInstanceBaseDTO getPreExecutableStepInstance(Long taskInstanceId, long stepInstanceId); /** * 根据taskInstanceId获取一个stepInstanceId，用于快速脚本/文件任务 @@ -182,7 +182,7 @@ void updateResolvedSourceFile(long taskInstanceId, * @param taskInstanceId 作业实例ID * @return 步骤实例ID */ - Long getStepInstanceId(long taskInstanceId); + Long getStepInstanceId(Long taskInstanceId); /** * 根据 appId,stepInstanceId 获取所属任务实例ID @@ -200,7 +200,7 @@ void updateResolvedSourceFile(long taskInstanceId, * @param stepInstanceId 步骤实例ID * @return 脚本类型 */ - Byte getScriptTypeByStepInstanceId(long taskInstanceId, long stepInstanceId); + Byte getScriptTypeByStepInstanceId(Long taskInstanceId, long stepInstanceId); /** * 更新步骤实例的当前滚动执行批次 @@ -209,7 +209,7 @@ void updateResolvedSourceFile(long taskInstanceId, * @param stepInstanceId 步骤实例ID * @param batch 滚动执行批次 */ - void updateStepCurrentBatch(long taskInstanceId, long stepInstanceId, int batch); + void updateStepCurrentBatch(Long taskInstanceId, long stepInstanceId, int batch); /** * 更新步骤实例的当前滚动执行批次 @@ -218,7 +218,7 @@ void updateResolvedSourceFile(long taskInstanceId, * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 */ - void updateStepCurrentExecuteCount(long taskInstanceId, long stepInstanceId, int executeCount); + void updateStepCurrentExecuteCount(Long taskInstanceId, long stepInstanceId, int executeCount); /** * 更新步骤实例的滚动配置ID @@ -227,7 +227,7 @@ void updateResolvedSourceFile(long taskInstanceId, * @param stepInstanceId 步骤实例ID * @param rollingConfigId 滚动配置ID */ - void updateStepRollingConfigId(long taskInstanceId, long stepInstanceId, long rollingConfigId); + void updateStepRollingConfigId(Long taskInstanceId, long stepInstanceId, long rollingConfigId); - List getTaskStepInstanceIdList(long taskInstanceId); + List getTaskStepInstanceIdList(Long taskInstanceId); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceRollingTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceRollingTaskDAO.java index d7a0f9f3e6..f69deb0c2f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceRollingTaskDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceRollingTaskDAO.java @@ -37,24 +37,28 @@ public interface StepInstanceRollingTaskDAO { /** * 查询步骤滚动任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 步骤执行次数 * @param batch 滚动批次 * @return 步骤滚动任务 */ - StepInstanceRollingTaskDTO queryRollingTask(long stepInstanceId, + StepInstanceRollingTaskDTO queryRollingTask(Long taskInstanceId, + long stepInstanceId, int executeCount, int batch); /** * 查询步骤滚动任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数;如果不为null，会根据executeCount过滤 * @param batch 滚动执行批次;如果不为null，会根据batch过滤 * @return 步骤滚动任务 */ - List listRollingTasks(long stepInstanceId, + List listRollingTasks(Long taskInstanceId, + long stepInstanceId, Integer executeCount, Integer batch); @@ -69,6 +73,7 @@ List listRollingTasks(long stepInstanceId, /** * 更新滚动任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 步骤执行次数 * @param batch 滚动执行批次 @@ -77,7 +82,8 @@ List listRollingTasks(long stepInstanceId, * @param endTime 任务结束时间；如果不更新传入null * @param totalTime 任务执行总时间；如果不更新传入null */ - void updateRollingTask(long stepInstanceId, + void updateRollingTask(Long taskInstanceId, + long stepInstanceId, int executeCount, int batch, RunStatusEnum status, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileExecuteObjectTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileExecuteObjectTaskDAOImpl.java index 8045c294fe..9177ea548a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileExecuteObjectTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileExecuteObjectTaskDAOImpl.java @@ -146,7 +146,7 @@ public void batchUpdateTasks(Collection tasks) { } @Override - public int getSuccessTaskCount(long stepInstanceId, int executeCount) { + public int getSuccessTaskCount(Long taskInstanceId, long stepInstanceId, int executeCount) { Integer count = CTX.selectCount() .from(T) .where(T.STATUS.in(ExecuteObjectTaskStatusEnum.LAST_SUCCESS.getValue(), @@ -154,18 +154,28 @@ public int getSuccessTaskCount(long stepInstanceId, int executeCount) { .and(T.STEP_INSTANCE_ID.eq(stepInstanceId)) .and(T.EXECUTE_COUNT.eq((short) executeCount)) .and(T.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .fetchOne(0, Integer.class); return count == null ? 0 : count; } + private Condition buildTaskInstanceIdQueryCondition(Long taskInstanceId) { + return TaskInstanceIdDynamicCondition.build( + taskInstanceId, + T.TASK_INSTANCE_ID::eq + ); + } + @Override - public List listResultGroups(long stepInstanceId, + public List listResultGroups(Long taskInstanceId, + long stepInstanceId, int executeCount, Integer batch) { SelectConditionStep selectConditionStep = CTX.select(T.STATUS, count().as("ip_count")) .from(T) .where(T.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .and(T.EXECUTE_COUNT.eq((short) executeCount)) .and(T.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())); if (batch != null && batch > 0) { @@ -189,13 +199,15 @@ public List listResultGroups(long stepInstanceId, } @Override - public List listTaskByResultGroup(Long stepInstanceId, + public List listTaskByResultGroup(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, Integer status) { SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) .from(T) .where(T.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .and(T.EXECUTE_COUNT.eq(executeCount.shortValue())) .and(T.STATUS.eq(status)) .and(T.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())); @@ -212,7 +224,8 @@ public List listTaskByResultGroup(Long stepInstanceId, } @Override - public List listTaskByResultGroup(Long stepInstanceId, + public List listTaskByResultGroup(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, Integer status, @@ -220,6 +233,7 @@ public List listTaskByResultGroup(Long stepInstanceId, String orderField, Order order) { List conditions = new ArrayList<>(); + conditions.add(buildTaskInstanceIdQueryCondition(taskInstanceId)); conditions.add(T.STEP_INSTANCE_ID.eq(stepInstanceId)); conditions.add(T.EXECUTE_COUNT.eq(executeCount.shortValue())); conditions.add(T.STATUS.eq(status)); @@ -279,13 +293,15 @@ private OrderField buildOrderField(String field, Order order) { } @Override - public List listTasks(Long stepInstanceId, + public List listTasks(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, FileTaskModeEnum fileTaskMode) { SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) .from(T) .where(T.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .and(T.EXECUTE_COUNT.eq(executeCount.shortValue())); if (batch != null && batch > 0) { selectConditionStep.and(T.BATCH.eq(batch.shortValue())); @@ -328,7 +344,7 @@ private ExecuteObjectTask extract(Record record) { } @Override - public List listTasksByGseTaskId(Long gseTaskId) { + public List listTasksByGseTaskId(Long taskInstanceId, Long gseTaskId) { if (gseTaskId == null || gseTaskId <= 0) { return Collections.emptyList(); } @@ -338,6 +354,7 @@ public List listTasksByGseTaskId(Long gseTaskId) { Result result = CTX.select(ALL_FIELDS) .from(T) .where(T.GSE_TASK_ID.eq(gseTaskId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .fetch(); if (result.size() > 0) { result.forEach(record -> executeObjectList.add(extract(record))); @@ -346,7 +363,8 @@ public List listTasksByGseTaskId(Long gseTaskId) { } @Override - public ExecuteObjectTask getTaskByExecuteObjectId(Long stepInstanceId, + public ExecuteObjectTask getTaskByExecuteObjectId(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, FileTaskModeEnum mode, @@ -355,6 +373,7 @@ public ExecuteObjectTask getTaskByExecuteObjectId(Long stepInstanceId, CTX.select(ALL_FIELDS) .from(T) .where(T.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .and(T.EXECUTE_COUNT.eq(executeCount.shortValue())) .and(T.MODE.eq(mode.getValue().byteValue())) .and(T.EXECUTE_OBJ_ID.eq(executeObjectId)); @@ -368,12 +387,17 @@ public ExecuteObjectTask getTaskByExecuteObjectId(Long stepInstanceId, } @Override - public boolean isStepInstanceRecordExist(long stepInstanceId) { - return CTX.fetchExists(T, T.STEP_INSTANCE_ID.eq(stepInstanceId)); + public boolean isStepInstanceRecordExist(Long taskInstanceId, long stepInstanceId) { + return CTX.fetchExists( + T, + T.STEP_INSTANCE_ID.eq(stepInstanceId), + buildTaskInstanceIdQueryCondition(taskInstanceId) + ); } @Override - public void updateTaskFields(long stepInstanceId, + public void updateTaskFields(Long taskInstanceId, + long stepInstanceId, int executeCount, Integer batch, Integer actualExecuteCount, @@ -400,6 +424,7 @@ public void updateTaskFields(long stepInstanceId, UpdateConditionStep updateConditionStep = updateSetMoreStep .where(T.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .and(T.EXECUTE_COUNT.eq((short) executeCount)); if (batch != null) { updateConditionStep.and(T.BATCH.eq(batch.shortValue())); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileSourceTaskLogDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileSourceTaskLogDAOImpl.java index 39d120ac3e..f0f7f269de 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileSourceTaskLogDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileSourceTaskLogDAOImpl.java @@ -29,8 +29,10 @@ import com.tencent.bk.job.execute.model.FileSourceTaskLogDTO; import com.tencent.bk.job.execute.model.tables.FileSourceTaskLog; import com.tencent.bk.job.execute.model.tables.records.FileSourceTaskLogRecord; +import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.Record; +import org.jooq.TableField; import org.jooq.UpdateSetFirstStep; import org.jooq.UpdateSetMoreStep; import org.springframework.beans.factory.annotation.Autowired; @@ -39,8 +41,19 @@ @Repository public class FileSourceTaskLogDAOImpl implements FileSourceTaskLogDAO { - FileSourceTaskLog defaultTable = FileSourceTaskLog.FILE_SOURCE_TASK_LOG; + private static final FileSourceTaskLog defaultTable = FileSourceTaskLog.FILE_SOURCE_TASK_LOG; private final DSLContext defaultContext; + private static final TableField[] ALL_FIELDS = { + defaultTable.TASK_INSTANCE_ID, + defaultTable.STEP_INSTANCE_ID, + defaultTable.EXECUTE_COUNT, + defaultTable.START_TIME, + defaultTable.END_TIME, + defaultTable.TOTAL_TIME, + defaultTable.STATUS, + defaultTable.FILE_SOURCE_BATCH_TASK_ID + }; + @Autowired public FileSourceTaskLogDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext defaultContext) { @@ -54,6 +67,7 @@ private FileSourceTaskLogDTO extractInfo(Record record) { FileSourceTaskLogDTO fileSourceTaskLogDTO = new FileSourceTaskLogDTO(); FileSourceTaskLog t = FileSourceTaskLog.FILE_SOURCE_TASK_LOG; + fileSourceTaskLogDTO.setTaskInstanceId(record.get(t.TASK_INSTANCE_ID)); fileSourceTaskLogDTO.setStepInstanceId(record.get(t.STEP_INSTANCE_ID)); fileSourceTaskLogDTO.setExecuteCount(record.get(t.EXECUTE_COUNT)); fileSourceTaskLogDTO.setStartTime(record.get(t.START_TIME)); @@ -69,6 +83,7 @@ public void insertOrUpdateFileSourceTaskLog(FileSourceTaskLogDTO fileSourceTaskL FileSourceTaskLog t = FileSourceTaskLog.FILE_SOURCE_TASK_LOG; defaultContext.insertInto( t, + t.TASK_INSTANCE_ID, t.STEP_INSTANCE_ID, t.EXECUTE_COUNT, t.START_TIME, @@ -77,6 +92,7 @@ public void insertOrUpdateFileSourceTaskLog(FileSourceTaskLogDTO fileSourceTaskL t.STATUS, t.FILE_SOURCE_BATCH_TASK_ID ).values( + fileSourceTaskLog.getTaskInstanceId(), fileSourceTaskLog.getStepInstanceId(), fileSourceTaskLog.getExecuteCount(), fileSourceTaskLog.getStartTime(), @@ -94,35 +110,40 @@ public void insertOrUpdateFileSourceTaskLog(FileSourceTaskLogDTO fileSourceTaskL } @Override - public FileSourceTaskLogDTO getFileSourceTaskLog(long stepInstanceId, int executeCount) { + public FileSourceTaskLogDTO getFileSourceTaskLog(Long taskInstanceId, long stepInstanceId, int executeCount) { FileSourceTaskLog t = FileSourceTaskLog.FILE_SOURCE_TASK_LOG; Record record = defaultContext.select( - t.STEP_INSTANCE_ID, - t.EXECUTE_COUNT, - t.START_TIME, - t.END_TIME, - t.TOTAL_TIME, - t.STATUS, - t.FILE_SOURCE_BATCH_TASK_ID + ALL_FIELDS ).from(t) .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .and(t.EXECUTE_COUNT.eq(executeCount)) .fetchOne(); return extractInfo(record); } + private Condition buildTaskInstanceIdQueryCondition(Long taskInstanceId) { + return TaskInstanceIdDynamicCondition.build( + taskInstanceId, + FileSourceTaskLog.FILE_SOURCE_TASK_LOG.TASK_INSTANCE_ID::eq + ); + } + @Override - public FileSourceTaskLogDTO getFileSourceTaskLogByBatchTaskId(String fileSourceBatchTaskId) { - Record record = defaultContext.select(defaultTable.STEP_INSTANCE_ID, defaultTable.EXECUTE_COUNT, - defaultTable.START_TIME, defaultTable.END_TIME, defaultTable.TOTAL_TIME, - defaultTable.STATUS, defaultTable.FILE_SOURCE_BATCH_TASK_ID).from(defaultTable) + public FileSourceTaskLogDTO getFileSourceTaskLogByBatchTaskId(Long taskInstanceId, String fileSourceBatchTaskId) { + Record record = defaultContext.select(ALL_FIELDS) + .from(defaultTable) .where(defaultTable.FILE_SOURCE_BATCH_TASK_ID.eq(fileSourceBatchTaskId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .fetchOne(); return extractInfo(record); } @Override - public int updateTimeConsumingByBatchTaskId(String fileSourceBatchTaskId, Long startTime, Long endTime, + public int updateTimeConsumingByBatchTaskId(Long taskInstanceId, + String fileSourceBatchTaskId, + Long startTime, + Long endTime, Long totalTime) { UpdateSetFirstStep firstStep = defaultContext.update(defaultTable); UpdateSetMoreStep moreStep = null; @@ -144,7 +165,9 @@ public int updateTimeConsumingByBatchTaskId(String fileSourceBatchTaskId, Long s } } if (moreStep != null) { - return moreStep.where(defaultTable.FILE_SOURCE_BATCH_TASK_ID.eq(fileSourceBatchTaskId)) + return moreStep.where( + defaultTable.FILE_SOURCE_BATCH_TASK_ID.eq(fileSourceBatchTaskId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .execute(); } else { return 0; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImpl.java index 288b2058f8..8f55e93113 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImpl.java @@ -47,11 +47,26 @@ public class GseTaskDAOImpl implements GseTaskDAO { private final DSLContext dslContext; private static final GseTask TABLE = GseTask.GSE_TASK; - private static final TableField[] ALL_FIELDS = {TABLE.ID, TABLE.STEP_INSTANCE_ID, TABLE.EXECUTE_COUNT, - TABLE.BATCH, TABLE.START_TIME, TABLE.END_TIME, TABLE.TOTAL_TIME, TABLE.STATUS, TABLE.GSE_TASK_ID}; - - private static final TableField[] SIMPLE_FIELDS = {TABLE.STEP_INSTANCE_ID, TABLE.EXECUTE_COUNT, - TABLE.BATCH, TABLE.GSE_TASK_ID}; + private static final TableField[] ALL_FIELDS = { + TABLE.ID, + TABLE.STEP_INSTANCE_ID, + TABLE.EXECUTE_COUNT, + TABLE.BATCH, + TABLE.START_TIME, + TABLE.END_TIME, + TABLE.TOTAL_TIME, + TABLE.STATUS, + TABLE.GSE_TASK_ID, + TABLE.TASK_INSTANCE_ID + }; + + private static final TableField[] SIMPLE_FIELDS = { + TABLE.STEP_INSTANCE_ID, + TABLE.EXECUTE_COUNT, + TABLE.BATCH, + TABLE.GSE_TASK_ID, + TABLE.TASK_INSTANCE_ID + }; @Autowired public GseTaskDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext dslContext) { @@ -65,6 +80,7 @@ private GseTaskDTO extractInfo(Record record) { GseTaskDTO gseTaskDTO = new GseTaskDTO(); gseTaskDTO.setId(record.get(TABLE.ID)); + gseTaskDTO.setTaskInstanceId(record.get(TABLE.TASK_INSTANCE_ID)); gseTaskDTO.setStepInstanceId(record.get(TABLE.STEP_INSTANCE_ID)); gseTaskDTO.setExecuteCount(record.get(TABLE.EXECUTE_COUNT).intValue()); gseTaskDTO.setBatch(record.get(TABLE.BATCH)); @@ -81,6 +97,7 @@ private GseTaskSimpleDTO extractSimpleInfo(Record record) { return null; } GseTaskSimpleDTO gseTaskSimpleDTO = new GseTaskSimpleDTO(); + gseTaskSimpleDTO.setTaskInstanceId(record.get(TABLE.TASK_INSTANCE_ID)); gseTaskSimpleDTO.setStepInstanceId(record.get(TABLE.STEP_INSTANCE_ID)); gseTaskSimpleDTO.setExecuteCount(record.get(TABLE.EXECUTE_COUNT).intValue()); gseTaskSimpleDTO.setBatch(record.get(TABLE.BATCH).intValue()); @@ -99,7 +116,8 @@ public long saveGseTask(GseTaskDTO gseTask) { TABLE.END_TIME, TABLE.TOTAL_TIME, TABLE.STATUS, - TABLE.GSE_TASK_ID) + TABLE.GSE_TASK_ID, + TABLE.TASK_INSTANCE_ID) .values( gseTask.getStepInstanceId(), gseTask.getExecuteCount().shortValue(), @@ -108,7 +126,8 @@ public long saveGseTask(GseTaskDTO gseTask) { gseTask.getEndTime(), gseTask.getTotalTime(), JooqDataTypeUtil.toByte(gseTask.getStatus()), - gseTask.getGseTaskId()) + gseTask.getGseTaskId(), + gseTask.getTaskInstanceId()) .returning(TABLE.ID) .fetchOne(); @@ -124,16 +143,18 @@ public boolean updateGseTask(GseTaskDTO gseTask) { .set(TABLE.STATUS, gseTask.getStatus().byteValue()) .set(TABLE.GSE_TASK_ID, gseTask.getGseTaskId()) .where(TABLE.ID.eq(gseTask.getId())) + .and(TaskInstanceIdDynamicCondition.build(gseTask.getTaskInstanceId(), TABLE.TASK_INSTANCE_ID::eq)) .execute(); return affectRows > 0; } @Override - public GseTaskDTO getGseTask(long stepInstanceId, int executeCount, Integer batch) { + public GseTaskDTO getGseTask(Long taskInstanceId, long stepInstanceId, int executeCount, Integer batch) { SelectConditionStep selectConditionStep = dslContext.select(ALL_FIELDS).from(TABLE) .where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(TABLE.EXECUTE_COUNT.eq((short) executeCount)); + .and(TABLE.EXECUTE_COUNT.eq((short) executeCount)) + .and(TaskInstanceIdDynamicCondition.build(taskInstanceId, TABLE.TASK_INSTANCE_ID::eq)); if (batch != null && batch > 0) { // 滚动执行批次，传入null或者0将忽略该参数 selectConditionStep.and(TABLE.BATCH.eq(batch.shortValue())); @@ -144,9 +165,10 @@ public GseTaskDTO getGseTask(long stepInstanceId, int executeCount, Integer batc } @Override - public GseTaskDTO getGseTask(long gseTaskId) { + public GseTaskDTO getGseTask(Long taskInstanceId, long gseTaskId) { Record record = dslContext.select(ALL_FIELDS).from(TABLE) .where(TABLE.ID.eq(gseTaskId)) + .and(TaskInstanceIdDynamicCondition.build(taskInstanceId, TABLE.TASK_INSTANCE_ID::eq)) .fetchOne(); return extractInfo(record); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptExecuteObjectTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptExecuteObjectTaskDAOImpl.java index f25467dc85..f6fabce2c4 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptExecuteObjectTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptExecuteObjectTaskDAOImpl.java @@ -155,26 +155,36 @@ public void batchUpdateTasks(Collection tasks) { } @Override - public int getSuccessTaskCount(long stepInstanceId, int executeCount) { + public int getSuccessTaskCount(Long taskInstanceId, long stepInstanceId, int executeCount) { Integer count = CTX.selectCount() .from(T) .where(T.STATUS.in(ExecuteObjectTaskStatusEnum.LAST_SUCCESS.getValue(), ExecuteObjectTaskStatusEnum.SUCCESS.getValue())) .and(T.STEP_INSTANCE_ID.eq(stepInstanceId)) .and(T.EXECUTE_COUNT.eq((short) executeCount)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .fetchOne(0, Integer.class); return count == null ? 0 : count; } + private Condition buildTaskInstanceIdQueryCondition(Long taskInstanceId) { + return TaskInstanceIdDynamicCondition.build( + taskInstanceId, + T.TASK_INSTANCE_ID::eq + ); + } + @Override - public List listResultGroups(long stepInstanceId, + public List listResultGroups(Long taskInstanceId, + long stepInstanceId, int executeCount, Integer batch) { SelectConditionStep selectConditionStep = CTX.select(T.STATUS, T.TAG, count().as("task_count")) .from(T) .where(T.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T.EXECUTE_COUNT.eq((short) executeCount)); + .and(T.EXECUTE_COUNT.eq((short) executeCount)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); if (batch != null && batch > 0) { selectConditionStep.and(T.BATCH.eq(batch.shortValue())); } @@ -196,7 +206,8 @@ public List listResultGroups(long stepInstanceId, } @Override - public List listTasksByResultGroup(Long stepInstanceId, + public List listTasksByResultGroup(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, Integer status, @@ -206,7 +217,8 @@ public List listTasksByResultGroup(Long stepInstanceId, .where(T.STEP_INSTANCE_ID.eq(stepInstanceId)) .and(T.EXECUTE_COUNT.eq(executeCount.shortValue())) .and(T.STATUS.eq(status)) - .and(T.TAG.eq(tag == null ? "" : tag)); + .and(T.TAG.eq(tag == null ? "" : tag)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); if (batch != null && batch > 0) { selectConditionStep.and(T.BATCH.eq(batch.shortValue())); } @@ -245,7 +257,8 @@ private ExecuteObjectTask extract(Record record) { } @Override - public List listTasksByResultGroup(Long stepInstanceId, + public List listTasksByResultGroup(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, Integer status, @@ -258,6 +271,7 @@ public List listTasksByResultGroup(Long stepInstanceId, conditions.add(T.EXECUTE_COUNT.eq(executeCount.shortValue())); conditions.add(T.STATUS.eq(status)); conditions.add(T.TAG.eq(tag == null ? "" : tag)); + conditions.add(buildTaskInstanceIdQueryCondition(taskInstanceId)); SelectConditionStep select = CTX.select(ALL_FIELDS) .from(T) @@ -319,13 +333,15 @@ private OrderField buildOrderField(String field, Order order) { } @Override - public List listTasks(Long stepInstanceId, + public List listTasks(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch) { SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) .from(T) .where(T.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T.EXECUTE_COUNT.eq(executeCount.shortValue())); + .and(T.EXECUTE_COUNT.eq(executeCount.shortValue())) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); if (batch != null && batch > 0) { selectConditionStep.and(T.BATCH.eq(batch.shortValue())); } @@ -341,7 +357,7 @@ public List listTasks(Long stepInstanceId, } @Override - public List listTasksByGseTaskId(Long gseTaskId) { + public List listTasksByGseTaskId(Long taskInstanceId, Long gseTaskId) { if (gseTaskId == null || gseTaskId <= 0) { return Collections.emptyList(); } @@ -351,6 +367,7 @@ public List listTasksByGseTaskId(Long gseTaskId) { Result result = CTX.select(ALL_FIELDS) .from(T) .where(T.GSE_TASK_ID.eq(gseTaskId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .fetch(); if (result.size() > 0) { result.forEach(record -> executeObjectList.add(extract(record))); @@ -359,14 +376,16 @@ public List listTasksByGseTaskId(Long gseTaskId) { } @Override - public ExecuteObjectTask getTaskByExecuteObjectId(Long stepInstanceId, + public ExecuteObjectTask getTaskByExecuteObjectId(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, String executeObjectId) { SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) .from(T) - .where(T.STEP_INSTANCE_ID.eq(stepInstanceId)) + .where(buildTaskInstanceIdQueryCondition(taskInstanceId)) + .and(T.STEP_INSTANCE_ID.eq(stepInstanceId)) .and(T.EXECUTE_COUNT.eq(executeCount.shortValue())) .and(T.EXECUTE_OBJ_ID.eq(executeObjectId)); if (batch != null && batch > 0) { @@ -379,12 +398,16 @@ public ExecuteObjectTask getTaskByExecuteObjectId(Long stepInstanceId, } @Override - public boolean isStepInstanceRecordExist(long stepInstanceId) { - return CTX.fetchExists(T, T.STEP_INSTANCE_ID.eq(stepInstanceId)); + public boolean isStepInstanceRecordExist(Long taskInstanceId, long stepInstanceId) { + return CTX.fetchExists( + T, + T.STEP_INSTANCE_ID.eq(stepInstanceId), + buildTaskInstanceIdQueryCondition(taskInstanceId)); } @Override - public void updateTaskFields(long stepInstanceId, + public void updateTaskFields(Long taskInstanceId, + long stepInstanceId, int executeCount, Integer batch, Integer actualExecuteCount, @@ -411,6 +434,7 @@ public void updateTaskFields(long stepInstanceId, UpdateConditionStep updateConditionStep = updateSetMoreStep .where(T.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .and(T.EXECUTE_COUNT.eq((short) executeCount)); if (batch != null) { updateConditionStep.and(T.BATCH.eq(batch.shortValue())); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImpl.java index 542df7c6f9..df7907a328 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImpl.java @@ -49,10 +49,12 @@ import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.Record; import org.jooq.Record1; import org.jooq.Result; +import org.jooq.Table; import org.jooq.TableField; import org.jooq.UpdateSetMoreStep; import org.jooq.types.UByte; @@ -304,15 +306,38 @@ public void addConfirmStepInstance(StepInstanceDTO stepInstance) { } @Override - public ScriptStepInstanceDTO getScriptStepInstance(long taskInstanceId, long stepInstanceId) { + public ScriptStepInstanceDTO getScriptStepInstance(Long taskInstanceId, long stepInstanceId) { Record record = CTX.select(T_STEP_INSTANCE_SCRIPT_ALL_FIELDS) .from(T_STEP_INSTANCE_SCRIPT) - .where(T_STEP_INSTANCE_SCRIPT.TASK_INSTANCE_ID.eq(taskInstanceId)) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE_SCRIPT, taskInstanceId)) .and(T_STEP_INSTANCE_SCRIPT.STEP_INSTANCE_ID.eq(stepInstanceId)) .fetchOne(); return extractScriptInfo(record); } + private Condition buildTaskInstanceIdQueryCondition(Table table, + Long taskInstanceId) { + if (table instanceof StepInstanceScript) { + return TaskInstanceIdDynamicCondition.build( + taskInstanceId, + T_STEP_INSTANCE_SCRIPT.TASK_INSTANCE_ID::eq + ); + } else if (table instanceof StepInstanceFile) { + return TaskInstanceIdDynamicCondition.build( + taskInstanceId, + T_STEP_INSTANCE_FILE.TASK_INSTANCE_ID::eq + ); + } else if (table instanceof StepInstanceConfirm) { + return TaskInstanceIdDynamicCondition.build( + taskInstanceId, + T_STEP_INSTANCE_CONFIRM.TASK_INSTANCE_ID::eq + ); + } else { + throw new IllegalArgumentException("Invalid table for building task_instance_id query condition"); + } + } + + private ScriptStepInstanceDTO extractScriptInfo(Record record) { if (record == null) { return null; @@ -367,10 +392,10 @@ private ScriptStepInstanceDTO extractScriptInfo(Record record) { } @Override - public FileStepInstanceDTO getFileStepInstance(long taskInstanceId, long stepInstanceId) { + public FileStepInstanceDTO getFileStepInstance(Long taskInstanceId, long stepInstanceId) { Record record = CTX.select(T_STEP_INSTANCE_FILE_ALL_FIELDS) .from(T_STEP_INSTANCE_FILE) - .where(T_STEP_INSTANCE_FILE.TASK_INSTANCE_ID.eq(taskInstanceId)) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE_FILE, taskInstanceId)) .and(T_STEP_INSTANCE_FILE.STEP_INSTANCE_ID.eq(stepInstanceId)) .fetchOne(); return extractFileInfo(record); @@ -408,10 +433,10 @@ private FileStepInstanceDTO extractFileInfo(Record record) { } @Override - public ConfirmStepInstanceDTO getConfirmStepInstance(long taskInstanceId, long stepInstanceId) { + public ConfirmStepInstanceDTO getConfirmStepInstance(Long taskInstanceId, long stepInstanceId) { Record record = CTX.select(T_STEP_INSTANCE_CONFIRM_ALL_FIELDS) .from(T_STEP_INSTANCE_CONFIRM) - .where(T_STEP_INSTANCE_CONFIRM.TASK_INSTANCE_ID.eq(taskInstanceId)) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE_CONFIRM, taskInstanceId)) .and(T_STEP_INSTANCE_CONFIRM.STEP_INSTANCE_ID.eq(stepInstanceId)) .fetchOne(); return extractConfirmInfo(record); @@ -437,7 +462,7 @@ private ConfirmStepInstanceDTO extractConfirmInfo(Record record) { } @Override - public StepInstanceBaseDTO getStepInstanceBase(long taskInstanceId, long stepInstanceId) { + public StepInstanceBaseDTO getStepInstanceBase(Long taskInstanceId, long stepInstanceId) { Record record = CTX .select(T_STEP_INSTANCE_ALL_FIELDS) .from(T_STEP_INSTANCE) @@ -490,7 +515,7 @@ public StepInstanceBaseDTO getStepInstanceBase(long stepInstanceId) { } @Override - public StepInstanceBaseDTO getFirstStepInstanceBase(long taskInstanceId) { + public StepInstanceBaseDTO getFirstStepInstanceBase(Long taskInstanceId) { Record record = CTX .select(T_STEP_INSTANCE_ALL_FIELDS) .from(T_STEP_INSTANCE) @@ -502,7 +527,7 @@ public StepInstanceBaseDTO getFirstStepInstanceBase(long taskInstanceId) { } @Override - public StepInstanceBaseDTO getNextStepInstance(long taskInstanceId, int currentStepOrder) { + public StepInstanceBaseDTO getNextStepInstance(Long taskInstanceId, int currentStepOrder) { Record record = CTX .select(T_STEP_INSTANCE_ALL_FIELDS) .from(T_STEP_INSTANCE) @@ -515,7 +540,7 @@ public StepInstanceBaseDTO getNextStepInstance(long taskInstanceId, int currentS } @Override - public List listStepInstanceBaseByTaskInstanceId(long taskInstanceId) { + public List listStepInstanceBaseByTaskInstanceId(Long taskInstanceId) { Result result = CTX .select(T_STEP_INSTANCE_ALL_FIELDS) .from(T_STEP_INSTANCE) @@ -528,7 +553,7 @@ public List listStepInstanceBaseByTaskInstanceId(long taskI } @Override - public void resetStepStatus(long taskInstanceId, long stepInstanceId) { + public void resetStepStatus(Long taskInstanceId, long stepInstanceId) { StepInstance t = StepInstance.STEP_INSTANCE; CTX.update(t) .setNull(t.START_TIME) @@ -540,7 +565,7 @@ public void resetStepStatus(long taskInstanceId, long stepInstanceId) { } @Override - public void resetStepExecuteInfoForRetry(long taskInstanceId, long stepInstanceId) { + public void resetStepExecuteInfoForRetry(Long taskInstanceId, long stepInstanceId) { StepInstance t = StepInstance.STEP_INSTANCE; CTX.update(t) .set(t.STATUS, RunStatusEnum.RUNNING.getValue().byteValue()) @@ -552,7 +577,7 @@ public void resetStepExecuteInfoForRetry(long taskInstanceId, long stepInstanceI } @Override - public void addStepExecuteCount(long taskInstanceId, long stepInstanceId) { + public void addStepExecuteCount(Long taskInstanceId, long stepInstanceId) { StepInstance t = StepInstance.STEP_INSTANCE; CTX.update(t) .set(t.EXECUTE_COUNT, t.EXECUTE_COUNT.plus(1)) @@ -562,7 +587,7 @@ public void addStepExecuteCount(long taskInstanceId, long stepInstanceId) { } @Override - public void updateStepStatus(long taskInstanceId, long stepInstanceId, int status) { + public void updateStepStatus(Long taskInstanceId, long stepInstanceId, int status) { StepInstance t = StepInstance.STEP_INSTANCE; CTX.update(t) .set(t.STATUS, JooqDataTypeUtil.toByte(status)) @@ -572,7 +597,7 @@ public void updateStepStatus(long taskInstanceId, long stepInstanceId, int statu } @Override - public void updateStepStartTime(long taskInstanceId, long stepInstanceId, Long startTime) { + public void updateStepStartTime(Long taskInstanceId, long stepInstanceId, Long startTime) { StepInstance t = StepInstance.STEP_INSTANCE; CTX.update(t) .set(t.START_TIME, startTime) @@ -582,7 +607,7 @@ public void updateStepStartTime(long taskInstanceId, long stepInstanceId, Long s } @Override - public void updateStepStartTimeIfNull(long taskInstanceId, long stepInstanceId, Long startTime) { + public void updateStepStartTimeIfNull(Long taskInstanceId, long stepInstanceId, Long startTime) { StepInstance t = StepInstance.STEP_INSTANCE; CTX.update(t) .set(t.START_TIME, startTime) @@ -593,7 +618,7 @@ public void updateStepStartTimeIfNull(long taskInstanceId, long stepInstanceId, } @Override - public void updateStepEndTime(long taskInstanceId, long stepInstanceId, Long endTime) { + public void updateStepEndTime(Long taskInstanceId, long stepInstanceId, Long endTime) { StepInstance t = StepInstance.STEP_INSTANCE; CTX.update(t) .set(t.END_TIME, endTime) @@ -603,7 +628,7 @@ public void updateStepEndTime(long taskInstanceId, long stepInstanceId, Long end } @Override - public void addStepInstanceExecuteCount(long taskInstanceId, long stepInstanceId) { + public void addStepInstanceExecuteCount(Long taskInstanceId, long stepInstanceId) { StepInstance t = StepInstance.STEP_INSTANCE; CTX.update(t) .set(t.EXECUTE_COUNT, t.EXECUTE_COUNT.plus(1)) @@ -613,7 +638,7 @@ public void addStepInstanceExecuteCount(long taskInstanceId, long stepInstanceId } @Override - public void updateStepTotalTime(long taskInstanceId, long stepInstanceId, long totalTime) { + public void updateStepTotalTime(Long taskInstanceId, long stepInstanceId, long totalTime) { StepInstance t = StepInstance.STEP_INSTANCE; CTX.update(t) .set(t.TOTAL_TIME, totalTime) @@ -623,7 +648,7 @@ public void updateStepTotalTime(long taskInstanceId, long stepInstanceId, long t } @Override - public void updateStepExecutionInfo(long taskInstanceId, + public void updateStepExecutionInfo(Long taskInstanceId, long stepInstanceId, RunStatusEnum status, Long startTime, @@ -675,7 +700,7 @@ private UpdateSetMoreStep buildBasicUpdateSetMoreStep(RunSta } @Override - public void updateResolvedScriptParam(long taskInstanceId, + public void updateResolvedScriptParam(Long taskInstanceId, long stepInstanceId, boolean isSecureParam, String resolvedScriptParam) { @@ -690,7 +715,7 @@ public void updateResolvedScriptParam(long taskInstanceId, } @Override - public void updateResolvedSourceFile(long taskInstanceId, + public void updateResolvedSourceFile(Long taskInstanceId, long stepInstanceId, List resolvedFileSources) { StepInstanceFile t = StepInstanceFile.STEP_INSTANCE_FILE; @@ -702,7 +727,7 @@ public void updateResolvedSourceFile(long taskInstanceId, } @Override - public void updateResolvedTargetPath(long taskInstanceId, long stepInstanceId, String resolvedTargetPath) { + public void updateResolvedTargetPath(Long taskInstanceId, long stepInstanceId, String resolvedTargetPath) { StepInstanceFile t = StepInstanceFile.STEP_INSTANCE_FILE; CTX.update(t) .set(t.RESOLVED_FILE_TARGET_PATH, resolvedTargetPath) @@ -712,7 +737,7 @@ public void updateResolvedTargetPath(long taskInstanceId, long stepInstanceId, S } @Override - public void updateConfirmReason(long taskInstanceId, long stepInstanceId, String confirmReason) { + public void updateConfirmReason(Long taskInstanceId, long stepInstanceId, String confirmReason) { StepInstanceConfirm t = StepInstanceConfirm.STEP_INSTANCE_CONFIRM; CTX.update(t).set(t.CONFIRM_REASON, confirmReason) .where(t.TASK_INSTANCE_ID.eq(taskInstanceId)) @@ -721,7 +746,7 @@ public void updateConfirmReason(long taskInstanceId, long stepInstanceId, String } @Override - public void updateStepOperator(long taskInstanceId, long stepInstanceId, String operator) { + public void updateStepOperator(Long taskInstanceId, long stepInstanceId, String operator) { StepInstance t = StepInstance.STEP_INSTANCE; CTX.update(t).set(t.OPERATOR, operator) .where(t.TASK_INSTANCE_ID.eq(taskInstanceId)) @@ -730,7 +755,7 @@ public void updateStepOperator(long taskInstanceId, long stepInstanceId, String } @Override - public StepInstanceBaseDTO getPreExecutableStepInstance(long taskInstanceId, long stepInstanceId) { + public StepInstanceBaseDTO getPreExecutableStepInstance(Long taskInstanceId, long stepInstanceId) { Record record = CTX .select(T_STEP_INSTANCE_ALL_FIELDS) .from(T_STEP_INSTANCE) @@ -744,7 +769,7 @@ public StepInstanceBaseDTO getPreExecutableStepInstance(long taskInstanceId, lon } @Override - public Long getStepInstanceId(long taskInstanceId) { + public Long getStepInstanceId(Long taskInstanceId) { Result> records = CTX.select(T_STEP_INSTANCE.ID) .from(T_STEP_INSTANCE) .where(T_STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) @@ -773,10 +798,10 @@ public Long getTaskInstanceId(long appId, long stepInstanceId) { } @Override - public Byte getScriptTypeByStepInstanceId(long taskInstanceId, long stepInstanceId) { + public Byte getScriptTypeByStepInstanceId(Long taskInstanceId, long stepInstanceId) { Result> records = CTX.select(T_STEP_INSTANCE_SCRIPT.SCRIPT_TYPE) .from(T_STEP_INSTANCE_SCRIPT) - .where(T_STEP_INSTANCE_SCRIPT.TASK_INSTANCE_ID.eq(taskInstanceId)) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE_SCRIPT, taskInstanceId)) .and(T_STEP_INSTANCE_SCRIPT.STEP_INSTANCE_ID.eq(stepInstanceId)) .limit(1) .fetch(); @@ -788,7 +813,7 @@ public Byte getScriptTypeByStepInstanceId(long taskInstanceId, long stepInstance } @Override - public void updateStepCurrentBatch(long taskInstanceId, long stepInstanceId, int batch) { + public void updateStepCurrentBatch(Long taskInstanceId, long stepInstanceId, int batch) { CTX.update(T_STEP_INSTANCE) .set(T_STEP_INSTANCE.BATCH, JooqDataTypeUtil.toShort(batch)) .where(T_STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) @@ -797,7 +822,7 @@ public void updateStepCurrentBatch(long taskInstanceId, long stepInstanceId, int } @Override - public void updateStepCurrentExecuteCount(long taskInstanceId, long stepInstanceId, int executeCount) { + public void updateStepCurrentExecuteCount(Long taskInstanceId, long stepInstanceId, int executeCount) { CTX.update(T_STEP_INSTANCE) .set(T_STEP_INSTANCE.EXECUTE_COUNT, executeCount) .where(T_STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) @@ -806,7 +831,7 @@ public void updateStepCurrentExecuteCount(long taskInstanceId, long stepInstance } @Override - public void updateStepRollingConfigId(long taskInstanceId, long stepInstanceId, long rollingConfigId) { + public void updateStepRollingConfigId(Long taskInstanceId, long stepInstanceId, long rollingConfigId) { CTX.update(T_STEP_INSTANCE) .set(T_STEP_INSTANCE.ROLLING_CONFIG_ID, rollingConfigId) .where(T_STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) @@ -815,7 +840,7 @@ public void updateStepRollingConfigId(long taskInstanceId, long stepInstanceId, } @Override - public List getTaskStepInstanceIdList(long taskInstanceId) { + public List getTaskStepInstanceIdList(Long taskInstanceId) { Result result = CTX.select(StepInstance.STEP_INSTANCE.ID).from(StepInstance.STEP_INSTANCE) .where(StepInstance.STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) .orderBy(StepInstance.STEP_INSTANCE.ID.asc()) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImpl.java index 078eecf0fb..6c5d2a25bc 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImpl.java @@ -31,6 +31,7 @@ import com.tencent.bk.job.execute.model.tables.StepInstanceRollingTask; import com.tencent.bk.job.execute.model.tables.records.StepInstanceRollingTaskRecord; import lombok.extern.slf4j.Slf4j; +import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.Record; import org.jooq.Result; @@ -51,6 +52,7 @@ public class StepInstanceRollingTaskDAOImpl implements StepInstanceRollingTaskDA private static final StepInstanceRollingTask TABLE = StepInstanceRollingTask.STEP_INSTANCE_ROLLING_TASK; private static final TableField[] ALL_FIELDS = { TABLE.ID, + TABLE.TASK_INSTANCE_ID, TABLE.STEP_INSTANCE_ID, TABLE.EXECUTE_COUNT, TABLE.BATCH, @@ -67,22 +69,34 @@ public StepInstanceRollingTaskDAOImpl(@Qualifier("job-execute-dsl-context") DSLC } @Override - public StepInstanceRollingTaskDTO queryRollingTask(long stepInstanceId, int executeCount, int batch) { + public StepInstanceRollingTaskDTO queryRollingTask(Long taskInstanceId, + long stepInstanceId, + int executeCount, + int batch) { Record record = CTX.select(ALL_FIELDS) .from(TABLE) .where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .and(TABLE.EXECUTE_COUNT.eq(JooqDataTypeUtil.toShort(executeCount))) .and(TABLE.BATCH.eq(JooqDataTypeUtil.toShort(batch))) .fetchOne(); return extract(record); } + private Condition buildTaskInstanceIdQueryCondition(Long taskInstanceId) { + return TaskInstanceIdDynamicCondition.build( + taskInstanceId, + TABLE.TASK_INSTANCE_ID::eq + ); + } + private StepInstanceRollingTaskDTO extract(Record record) { if (record == null) { return null; } StepInstanceRollingTaskDTO stepInstanceRollingTask = new StepInstanceRollingTaskDTO(); stepInstanceRollingTask.setId(record.get(TABLE.ID)); + stepInstanceRollingTask.setTaskInstanceId(record.get(TABLE.TASK_INSTANCE_ID)); stepInstanceRollingTask.setStepInstanceId(record.get(TABLE.STEP_INSTANCE_ID)); stepInstanceRollingTask.setExecuteCount(record.get(TABLE.EXECUTE_COUNT).intValue()); stepInstanceRollingTask.setBatch(record.get(TABLE.BATCH).intValue()); @@ -94,12 +108,14 @@ private StepInstanceRollingTaskDTO extract(Record record) { } @Override - public List listRollingTasks(long stepInstanceId, + public List listRollingTasks(Long taskInstanceId, + long stepInstanceId, Integer executeCount, Integer batch) { SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) .from(TABLE) - .where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)); + .where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); if (executeCount != null) { selectConditionStep.and(TABLE.EXECUTE_COUNT.eq(executeCount.shortValue())); } @@ -120,6 +136,7 @@ public List listRollingTasks(long stepInstanceId, public long saveRollingTask(StepInstanceRollingTaskDTO rollingTask) { Record record = CTX.insertInto( TABLE, + TABLE.TASK_INSTANCE_ID, TABLE.STEP_INSTANCE_ID, TABLE.EXECUTE_COUNT, TABLE.BATCH, @@ -128,6 +145,7 @@ public long saveRollingTask(StepInstanceRollingTaskDTO rollingTask) { TABLE.END_TIME, TABLE.TOTAL_TIME) .values( + rollingTask.getTaskInstanceId(), rollingTask.getStepInstanceId(), JooqDataTypeUtil.toShort(rollingTask.getExecuteCount()), JooqDataTypeUtil.toShort(rollingTask.getBatch()), @@ -142,7 +160,8 @@ assert record != null; } @Override - public void updateRollingTask(long stepInstanceId, + public void updateRollingTask(Long taskInstanceId, + long stepInstanceId, int executeCount, int batch, RunStatusEnum status, @@ -181,6 +200,7 @@ public void updateRollingTask(long stepInstanceId, updateSetMoreStep.where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) .and(TABLE.EXECUTE_COUNT.eq(JooqDataTypeUtil.toByte(executeCount).shortValue())) .and(TABLE.BATCH.eq(JooqDataTypeUtil.toShort(batch))) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .execute(); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java new file mode 100644 index 0000000000..b2b19b3b8d --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java @@ -0,0 +1,67 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.impl; + +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.common.util.feature.FeatureExecutionContext; +import com.tencent.bk.job.common.util.feature.FeatureIdConstants; +import com.tencent.bk.job.common.util.feature.FeatureToggle; +import com.tencent.bk.job.common.util.feature.ToggleStrategyContextParams; +import lombok.extern.slf4j.Slf4j; +import org.jooq.Condition; +import org.jooq.impl.DSL; + +import java.util.function.Function; + +@Slf4j +public class TaskInstanceIdDynamicCondition { + + public static Condition build(Long taskInstanceId, + Function taskInstanceIdConditionBuilder) { + ResourceScope resourceScope = JobContextUtil.getAppResourceScope(); + if (resourceScope == null) { + log.warn("Empty resource scope, expected!"); + // 为了不影响兼容性，忽略错误 + return DSL.trueCondition(); + } + if (FeatureToggle.checkFeature( + FeatureIdConstants.DAO_ADD_TASK_INSTANCE_ID, + FeatureExecutionContext.builder() + .addContextParam(ToggleStrategyContextParams.CTX_PARAM_RESOURCE_SCOPE, + JobContextUtil.getAppResourceScope()))) { + if (taskInstanceId == null || taskInstanceId <= 0L) { + log.warn("Invalid taskInstanceId, ignore task_instance_id condition. taskInstanceId: {}", + taskInstanceId); + // 为了不影响兼容性，忽略错误 + return DSL.trueCondition(); + } else { + return taskInstanceIdConditionBuilder.apply(taskInstanceId); + } + } else { + return DSL.trueCondition(); + } + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java index 6b550035e8..4e2ecd9a6a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java @@ -191,8 +191,16 @@ private void startStep(StepEvent stepEvent, StepInstanceDTO stepInstance) { stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, RunStatusEnum.RUNNING, stepInstance.getStartTime() == null ? DateUtils.currentTimeMillis() : null, null, null); if (isRollingStep) { - stepInstanceRollingTaskService.updateRollingTask(stepInstanceId, stepInstance.getExecuteCount(), - stepInstance.getBatch(), RunStatusEnum.RUNNING, System.currentTimeMillis(), null, null); + stepInstanceRollingTaskService.updateRollingTask( + taskInstanceId, + stepInstanceId, + stepInstance.getExecuteCount(), + stepInstance.getBatch(), + RunStatusEnum.RUNNING, + System.currentTimeMillis(), + null, + null + ); } startGseTask(stepInstance, gseTaskId); @@ -250,7 +258,7 @@ private void prepareFile(StepInstanceDTO stepInstance) { * @return GSE 任务ID */ private Long saveInitialGseTask(StepInstanceDTO stepInstance) { - GseTaskDTO gseTask = new GseTaskDTO(stepInstance.getId(), + GseTaskDTO gseTask = new GseTaskDTO(stepInstance.getTaskInstanceId(), stepInstance.getId(), stepInstance.getExecuteCount(), stepInstance.getBatch()); gseTask.setStatus(RunStatusEnum.WAITING_USER.getValue()); @@ -371,6 +379,7 @@ private List buildInitialExecuteObjectTasks(long taskInstance */ private void saveInitialStepInstanceRollingTask(StepInstanceDTO stepInstance) { StepInstanceRollingTaskDTO stepInstanceRollingTask = new StepInstanceRollingTaskDTO(); + stepInstanceRollingTask.setTaskInstanceId(stepInstance.getTaskInstanceId()); stepInstanceRollingTask.setStepInstanceId(stepInstance.getId()); stepInstanceRollingTask.setBatch(stepInstance.getBatch()); stepInstanceRollingTask.setExecuteCount(stepInstance.getExecuteCount()); @@ -431,11 +440,11 @@ private void ignoreError(StepInstanceDTO stepInstance) { taskInstanceService.resetTaskExecuteInfoForRetry(stepInstance.getTaskInstanceId()); if (isRollingStep) { StepInstanceRollingTaskDTO stepInstanceRollingTask = - stepInstanceRollingTaskService.queryRollingTask(stepInstanceId, stepInstance.getExecuteCount(), - stepInstance.getBatch()); + stepInstanceRollingTaskService.queryRollingTask(taskInstanceId, stepInstanceId, + stepInstance.getExecuteCount(), stepInstance.getBatch()); if (stepInstanceRollingTask != null) { - finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), - RunStatusEnum.IGNORE_ERROR); + finishRollingTask(taskInstanceId, stepInstanceId, stepInstance.getExecuteCount(), + stepInstance.getBatch(), RunStatusEnum.IGNORE_ERROR); } } @@ -509,7 +518,8 @@ private void continueGseFileStep(StepInstanceDTO stepInstance) { log.info("Continue file push step, stepInstanceId={}", stepInstance.getId()); GseTaskDTO gseTask = - gseTaskService.getGseTask(stepInstance.getId(), stepInstance.getExecuteCount(), stepInstance.getBatch()); + gseTaskService.getGseTask(stepInstance.getTaskInstanceId(), stepInstance.getId(), + stepInstance.getExecuteCount(), stepInstance.getBatch()); taskExecuteMQEventDispatcher.dispatchGseTaskEvent(GseTaskEvent.startGseTask( stepInstance.getTaskInstanceId(), gseTask.getStepInstanceId(), gseTask.getExecuteCount(), gseTask.getBatch(), gseTask.getId(), null)); @@ -677,7 +687,7 @@ private void refreshStep(StepEvent stepEvent, StepInstanceDTO stepInstance) { long stepInstanceId = stepInstance.getId(); EventSource eventSource = stepEvent.getSource(); - GseTaskDTO gseTask = gseTaskService.getGseTask(eventSource.getGseTaskId()); + GseTaskDTO gseTask = gseTaskService.getGseTask(eventSource.getJobInstanceId(), eventSource.getGseTaskId()); RunStatusEnum gseTaskStatus = RunStatusEnum.valueOf(gseTask.getStatus()); log.info("Refresh step according to gse task status, stepInstanceId: {}, gseTaskStatus: {}", @@ -723,7 +733,7 @@ private void onSuccess(StepInstanceDTO stepInstance) { if (stepInstance.isRollingStep()) { RollingConfigDTO rollingConfig = rollingConfigService.getRollingConfig(stepInstance.getRollingConfigId()); - finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), + finishRollingTask(taskInstanceId, stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), RunStatusEnum.SUCCESS); int totalBatch = rollingConfig.getConfigDetail().getTotalBatch(); boolean isLastBatch = totalBatch == stepInstance.getBatch(); @@ -755,8 +765,8 @@ private void onFail(StepInstanceDTO stepInstance) { stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, RunStatusEnum.IGNORE_ERROR, startTime, endTime, totalTime); if (stepInstance.isRollingStep()) { - finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), - RunStatusEnum.IGNORE_ERROR); + finishRollingTask(taskInstanceId, stepInstanceId, stepInstance.getExecuteCount(), + stepInstance.getBatch(), RunStatusEnum.IGNORE_ERROR); } return; } @@ -768,15 +778,15 @@ private void onFail(StepInstanceDTO stepInstance) { switch (rollingMode) { case IGNORE_ERROR: log.info("Ignore error for rolling step, rollingMode: {}", rollingMode); - finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), - RunStatusEnum.IGNORE_ERROR); + finishRollingTask(taskInstanceId, stepInstanceId, stepInstance.getExecuteCount(), + stepInstance.getBatch(), RunStatusEnum.IGNORE_ERROR); stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, RunStatusEnum.IGNORE_ERROR, startTime, endTime, totalTime); break; case PAUSE_IF_FAIL: case MANUAL: - finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), - RunStatusEnum.FAIL); + finishRollingTask(taskInstanceId, stepInstanceId, stepInstance.getExecuteCount(), + stepInstance.getBatch(), RunStatusEnum.FAIL); stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, RunStatusEnum.FAIL, startTime, endTime, totalTime); break; @@ -801,8 +811,8 @@ private void onStopSuccess(StepInstanceDTO stepInstance) { stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, RunStatusEnum.STOP_SUCCESS, startTime, endTime, totalTime); if (stepInstance.isRollingStep()) { - finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), - RunStatusEnum.STOP_SUCCESS); + finishRollingTask(taskInstanceId, stepInstanceId, stepInstance.getExecuteCount(), + stepInstance.getBatch(), RunStatusEnum.STOP_SUCCESS); } } else { log.error("Refresh step fail, stepInstanceId: {}, stepStatus: {}, gseTaskStatus: {}", @@ -813,8 +823,8 @@ private void onStopSuccess(StepInstanceDTO stepInstance) { private void onAbnormalState(StepInstanceDTO stepInstance) { finishStep(stepInstance, RunStatusEnum.ABNORMAL_STATE); if (stepInstance.isRollingStep()) { - finishRollingTask(stepInstance.getId(), stepInstance.getExecuteCount(), stepInstance.getBatch(), - RunStatusEnum.ABNORMAL_STATE); + finishRollingTask(stepInstance.getTaskInstanceId(), stepInstance.getId(), stepInstance.getExecuteCount(), + stepInstance.getBatch(), RunStatusEnum.ABNORMAL_STATE); } } @@ -843,14 +853,18 @@ private void finishStep(StepInstanceDTO stepInstance, RunStatusEnum status) { private void onAbandonState(StepInstanceDTO stepInstance) { finishStep(stepInstance, RunStatusEnum.ABANDONED); if (stepInstance.isRollingStep()) { - finishRollingTask(stepInstance.getId(), stepInstance.getExecuteCount(), stepInstance.getBatch(), - RunStatusEnum.ABANDONED); + finishRollingTask(stepInstance.getTaskInstanceId(), stepInstance.getId(), stepInstance.getExecuteCount(), + stepInstance.getBatch(), RunStatusEnum.ABANDONED); } } - private void finishRollingTask(long stepInstanceId, int executeCount, int batch, RunStatusEnum status) { + private void finishRollingTask(Long taskInstanceId, + long stepInstanceId, + int executeCount, + int batch, + RunStatusEnum status) { StepInstanceRollingTaskDTO rollingTask = - stepInstanceRollingTaskService.queryRollingTask(stepInstanceId, executeCount, batch); + stepInstanceRollingTaskService.queryRollingTask(taskInstanceId, stepInstanceId, executeCount, batch); if (rollingTask == null) { log.error("Rolling task is not exist, skip update! stepInstanceId: {}, executeCount: {}, batch: {}", stepInstanceId, executeCount, batch); @@ -859,7 +873,7 @@ private void finishRollingTask(long stepInstanceId, int executeCount, int batch, long now = System.currentTimeMillis(); long startTime = rollingTask.getStartTime() != null ? rollingTask.getStartTime() : now; - stepInstanceRollingTaskService.updateRollingTask(stepInstanceId, executeCount, + stepInstanceRollingTaskService.updateRollingTask(taskInstanceId, stepInstanceId, executeCount, batch, status, startTime, now, now - startTime); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseTaskListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseTaskListener.java index fedb2465fb..ded0d99b5f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseTaskListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseTaskListener.java @@ -62,7 +62,7 @@ public GseTaskListener(GseTaskManager gseTaskManager, */ public void handleEvent(GseTaskEvent gseTaskEvent) { log.info("Handle gse task event: {}, duration: {}ms", gseTaskEvent, gseTaskEvent.duration()); - GseTaskDTO gseTask = gseTaskService.getGseTask(gseTaskEvent.getGseTaskId()); + GseTaskDTO gseTask = gseTaskService.getGseTask(gseTaskEvent.getJobInstanceId(), gseTaskEvent.getGseTaskId()); String requestId = gseTaskEvent.getRequestId(); try { int action = gseTaskEvent.getAction(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ResultHandleResumeListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ResultHandleResumeListener.java index c7bb452a2a..23a2f5c1a0 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ResultHandleResumeListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ResultHandleResumeListener.java @@ -134,7 +134,7 @@ public ResultHandleResumeListener(TaskInstanceService taskInstanceService, */ public void handleEvent(ResultHandleTaskResumeEvent event) { log.info("Receive gse task result handle task resume event: {}, duration: {}ms", event, event.duration()); - GseTaskDTO gseTask = gseTaskService.getGseTask(event.getGseTaskId()); + GseTaskDTO gseTask = gseTaskService.getGseTask(event.getJobInstanceId(), event.getGseTaskId()); long stepInstanceId = gseTask.getStepInstanceId(); String requestId = StringUtils.isNotEmpty(event.getRequestId()) ? event.getRequestId() : UUID.randomUUID().toString(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/GseTaskEvent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/GseTaskEvent.java index 5ad7072cf4..982b7b21f3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/GseTaskEvent.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/GseTaskEvent.java @@ -85,7 +85,7 @@ public class GseTaskEvent extends Event { * @param requestId 请求ID,防止重复下发任务 * @return 事件 */ - public static GseTaskEvent startGseTask(long jobInstanceId, + public static GseTaskEvent startGseTask(Long jobInstanceId, Long stepInstanceId, Integer executeCount, Integer batch, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/StepEvent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/StepEvent.java index 6f2261e485..3c0e30ef59 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/StepEvent.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/StepEvent.java @@ -50,7 +50,7 @@ public class StepEvent extends Event { /** * 作业实例ID */ - private long jobInstanceId; + private Long jobInstanceId; /** * 步骤实例ID */ @@ -67,7 +67,7 @@ public class StepEvent extends Event { * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent ignoreError(long jobInstanceId, long stepInstanceId) { + public static StepEvent ignoreError(Long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); @@ -83,7 +83,7 @@ public static StepEvent ignoreError(long jobInstanceId, long stepInstanceId) { * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent nextStep(long jobInstanceId, long stepInstanceId) { + public static StepEvent nextStep(Long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); @@ -99,7 +99,7 @@ public static StepEvent nextStep(long jobInstanceId, long stepInstanceId) { * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent confirmStepContinue(long jobInstanceId, long stepInstanceId) { + public static StepEvent confirmStepContinue(Long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); @@ -115,7 +115,7 @@ public static StepEvent confirmStepContinue(long jobInstanceId, long stepInstanc * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent confirmStepTerminate(long jobInstanceId, long stepInstanceId) { + public static StepEvent confirmStepTerminate(Long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); @@ -131,7 +131,7 @@ public static StepEvent confirmStepTerminate(long jobInstanceId, long stepInstan * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent confirmStepRestart(long jobInstanceId, long stepInstanceId) { + public static StepEvent confirmStepRestart(Long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); @@ -148,7 +148,7 @@ public static StepEvent confirmStepRestart(long jobInstanceId, long stepInstance * @param batch 滚动执行批次 * @return 事件 */ - public static StepEvent startStep(long jobInstanceId, long stepInstanceId, Integer batch) { + public static StepEvent startStep(Long jobInstanceId, long stepInstanceId, Integer batch) { StepEvent stepEvent = new StepEvent(); stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); @@ -165,7 +165,7 @@ public static StepEvent startStep(long jobInstanceId, long stepInstanceId, Integ * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent skipStep(long jobInstanceId, long stepInstanceId) { + public static StepEvent skipStep(Long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); @@ -181,7 +181,7 @@ public static StepEvent skipStep(long jobInstanceId, long stepInstanceId) { * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent stopStep(long jobInstanceId, long stepInstanceId) { + public static StepEvent stopStep(Long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); @@ -197,7 +197,7 @@ public static StepEvent stopStep(long jobInstanceId, long stepInstanceId) { * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent retryStepFail(long jobInstanceId, long stepInstanceId) { + public static StepEvent retryStepFail(Long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); @@ -213,7 +213,7 @@ public static StepEvent retryStepFail(long jobInstanceId, long stepInstanceId) { * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent retryStepAll(long jobInstanceId, long stepInstanceId) { + public static StepEvent retryStepAll(Long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); @@ -229,7 +229,7 @@ public static StepEvent retryStepAll(long jobInstanceId, long stepInstanceId) { * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent continueGseFileStep(long jobInstanceId, long stepInstanceId) { + public static StepEvent continueGseFileStep(Long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); @@ -245,7 +245,7 @@ public static StepEvent continueGseFileStep(long jobInstanceId, long stepInstanc * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent prepareFile(long jobInstanceId, long stepInstanceId) { + public static StepEvent prepareFile(Long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); @@ -261,7 +261,7 @@ public static StepEvent prepareFile(long jobInstanceId, long stepInstanceId) { * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent refreshStep(long jobInstanceId, long stepInstanceId, EventSource eventSource) { + public static StepEvent refreshStep(Long jobInstanceId, long stepInstanceId, EventSource eventSource) { StepEvent stepEvent = new StepEvent(); stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareService.java index cd7eb928b0..1014818357 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareService.java @@ -271,6 +271,7 @@ private void continueStepAtOnce(StepInstanceDTO stepInstance) { private FileSourceTaskLogDTO buildInitFileSourceTaskLog(StepInstanceDTO stepInstance, BatchTaskInfoDTO batchTaskInfoDTO) { FileSourceTaskLogDTO fileSourceTaskLogDTO = new FileSourceTaskLogDTO(); + fileSourceTaskLogDTO.setTaskInstanceId(stepInstance.getTaskInstanceId()); fileSourceTaskLogDTO.setStepInstanceId(stepInstance.getId()); fileSourceTaskLogDTO.setExecuteCount(stepInstance.getExecuteCount()); fileSourceTaskLogDTO.setFileSourceBatchTaskId(batchTaskInfoDTO.getBatchTaskId()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareTask.java index b9d313a4a9..7e983928a8 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareTask.java @@ -255,11 +255,12 @@ private void handleFileSourceTaskResult( BatchTaskStatusDTO batchTaskStatusDTO ) { // 更新文件源拉取任务耗时数据 - FileSourceTaskLogDTO fileSourceTaskLogDTO = fileSourceTaskLogDAO.getFileSourceTaskLogByBatchTaskId(batchTaskId); + FileSourceTaskLogDTO fileSourceTaskLogDTO = fileSourceTaskLogDAO.getFileSourceTaskLogByBatchTaskId( + stepInstance.getTaskInstanceId(), batchTaskId); if (fileSourceTaskLogDTO != null) { Long endTime = System.currentTimeMillis(); - fileSourceTaskLogDAO.updateTimeConsumingByBatchTaskId(batchTaskId, null, endTime, - endTime - fileSourceTaskLogDTO.getStartTime()); + fileSourceTaskLogDAO.updateTimeConsumingByBatchTaskId(stepInstance.getTaskInstanceId(), batchTaskId, + null, endTime, endTime - fileSourceTaskLogDTO.getStartTime()); } List taskStatusList = batchTaskStatusDTO.getFileSourceTaskStatusInfoList(); if (taskStatusList.isEmpty()) { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileSourceTaskLogDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileSourceTaskLogDTO.java index 9d8dcec80f..b3654e8e36 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileSourceTaskLogDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileSourceTaskLogDTO.java @@ -31,6 +31,8 @@ */ @Data public class FileSourceTaskLogDTO { + private Long taskInstanceId; + private Long stepInstanceId; private Integer executeCount; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/GseTaskDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/GseTaskDTO.java index 49c00bbb5d..67c05c9db5 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/GseTaskDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/GseTaskDTO.java @@ -78,7 +78,11 @@ public class GseTaskDTO { */ private String taskUniqueName; - public GseTaskDTO(Long stepInstanceId, Integer executeCount, int batch) { + public GseTaskDTO(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + int batch) { + this.taskInstanceId = taskInstanceId; this.stepInstanceId = stepInstanceId; this.executeCount = executeCount; this.batch = batch; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceRollingTaskDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceRollingTaskDTO.java index 92d3b1c7fe..4011313b75 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceRollingTaskDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceRollingTaskDTO.java @@ -37,6 +37,11 @@ public class StepInstanceRollingTaskDTO { */ private Long id; + /** + * 作业实例ID + */ + private Long taskInstanceId; + /** * 步骤实例ID */ diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ExecuteObjectTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ExecuteObjectTaskService.java index 0c91e7a7bb..063d4cc8d8 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ExecuteObjectTaskService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ExecuteObjectTaskService.java @@ -55,11 +55,12 @@ public interface ExecuteObjectTaskService { /** * 获取执行成功的执行对象任务数量 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 步骤执行次数 * @return 执行成功的任务数量 */ - int getSuccessTaskCount(long stepInstanceId, int executeCount); + int getSuccessTaskCount(Long taskInstanceId, long stepInstanceId, int executeCount); /** diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/GseTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/GseTaskService.java index 1ae08048c5..fe53160c1f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/GseTaskService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/GseTaskService.java @@ -52,20 +52,22 @@ public interface GseTaskService { /** * 获取 GSE 任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 步骤执行次数 * @param batch 滚动执行批次 * @return GSE 任务 */ - GseTaskDTO getGseTask(long stepInstanceId, int executeCount, Integer batch); + GseTaskDTO getGseTask(Long taskInstanceId, long stepInstanceId, int executeCount, Integer batch); /** * 获取 GSE 任务 * - * @param gseTaskId GSE任务ID + * @param taskInstanceId 作业实例 ID + * @param gseTaskId GSE任务ID * @return GSE 任务 */ - GseTaskDTO getGseTask(long gseTaskId); + GseTaskDTO getGseTask(Long taskInstanceId, long gseTaskId); /** * 获取 GSE 任务 @@ -79,8 +81,8 @@ public interface GseTaskService { * 获取 GSE 任务列表 * * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - * @param batch 批次 + * @param executeCount 执行次数 + * @param batch 批次 * @return GSE 任务列表 */ List ListGseTaskSimpleInfo(Long stepInstanceId, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceRollingTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceRollingTaskService.java index 05ba7bc69e..9e496b771b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceRollingTaskService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceRollingTaskService.java @@ -37,32 +37,36 @@ public interface StepInstanceRollingTaskService { /** * 查询步骤滚动任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 步骤执行次数 * @param batch 滚动批次 * @return 步骤滚动任务 */ - StepInstanceRollingTaskDTO queryRollingTask(long stepInstanceId, + StepInstanceRollingTaskDTO queryRollingTask(Long taskInstanceId, + long stepInstanceId, int executeCount, int batch); /** * 查询最新的步骤滚动任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 步骤执行次数 * @return 步骤滚动任务 */ - List listLatestRollingTasks(long stepInstanceId, int executeCount); + List listLatestRollingTasks(Long taskInstanceId, long stepInstanceId, int executeCount); /** * 根据批次查询滚动任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param batch 滚动批次 * @return 步骤滚动任务 */ - List listRollingTasksByBatch(long stepInstanceId, Integer batch); + List listRollingTasksByBatch(Long taskInstanceId, long stepInstanceId, Integer batch); /** * 保存步骤滚动任务 @@ -75,6 +79,7 @@ StepInstanceRollingTaskDTO queryRollingTask(long stepInstanceId, /** * 更新滚动任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 步骤执行次数 * @param batch 滚动执行批次 @@ -83,7 +88,8 @@ StepInstanceRollingTaskDTO queryRollingTask(long stepInstanceId, * @param endTime 任务结束时间；如果不更新传入null * @param totalTime 任务执行总时间；如果不更新传入null */ - void updateRollingTask(long stepInstanceId, + void updateRollingTask(Long taskInstanceId, + long stepInstanceId, int executeCount, int batch, RunStatusEnum status, @@ -95,8 +101,9 @@ void updateRollingTask(long stepInstanceId, /** * 根据步骤实例ID查询滚动任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @return 步骤滚动任务 */ - List listRollingTasksByStep(long stepInstanceId); + List listRollingTasksByStep(Long taskInstanceId, long stepInstanceId); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceService.java index 8103eae29b..14e0676314 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceService.java @@ -48,7 +48,7 @@ public interface StepInstanceService { * @param stepInstanceId 步骤实例ID * @param batch 滚动执行批次 */ - void updateStepCurrentBatch(long taskInstanceId, long stepInstanceId, int batch); + void updateStepCurrentBatch(Long taskInstanceId, long stepInstanceId, int batch); /** * 更新步骤实例的滚动配置ID @@ -57,7 +57,7 @@ public interface StepInstanceService { * @param stepInstanceId 步骤实例ID * @param rollingConfigId 滚动配置ID */ - void updateStepRollingConfigId(long taskInstanceId, long stepInstanceId, long rollingConfigId); + void updateStepRollingConfigId(Long taskInstanceId, long stepInstanceId, long rollingConfigId); /** * 获取下一个步骤实例 @@ -66,7 +66,7 @@ public interface StepInstanceService { * @param currentStepOrder 当前步骤的顺序 * @return 步骤实例；如果当前为最后一个步骤实例，那么返回null */ - StepInstanceBaseDTO getNextStepInstance(long taskInstanceId, int currentStepOrder); + StepInstanceBaseDTO getNextStepInstance(Long taskInstanceId, int currentStepOrder); /** * 获取步骤包含的执行对象(源+目标) @@ -91,7 +91,7 @@ Map computeStepExecuteObjects(StepInstanceBaseDTO stepInst * @param taskInstanceId 作业实例 ID * @return 步骤实例列表 */ - List listBaseStepInstanceByTaskInstanceId(long taskInstanceId); + List listBaseStepInstanceByTaskInstanceId(Long taskInstanceId); /** * 获取作业实例下的所有步骤实例(详细) @@ -99,7 +99,7 @@ Map computeStepExecuteObjects(StepInstanceBaseDTO stepInst * @param taskInstanceId 作业实例 ID * @return 步骤实例列表 */ - List listStepInstanceByTaskInstanceId(long taskInstanceId); + List listStepInstanceByTaskInstanceId(Long taskInstanceId); /** * 获取步骤基本信息 @@ -108,7 +108,7 @@ Map computeStepExecuteObjects(StepInstanceBaseDTO stepInst * @param stepInstanceId 步骤实例ID * @return 步骤基本信息 */ - StepInstanceBaseDTO getBaseStepInstance(long taskInstanceId, long stepInstanceId); + StepInstanceBaseDTO getBaseStepInstance(Long taskInstanceId, long stepInstanceId); /** * 获取步骤基本信息 @@ -126,7 +126,7 @@ Map computeStepExecuteObjects(StepInstanceBaseDTO stepInst * @param stepInstanceId 步骤实例ID * @return 步骤基本信息 */ - StepInstanceBaseDTO getBaseStepInstance(long appId, long taskInstanceId, long stepInstanceId); + StepInstanceBaseDTO getBaseStepInstance(long appId, Long taskInstanceId, long stepInstanceId); /** * 获取步骤基本信息 @@ -135,7 +135,7 @@ Map computeStepExecuteObjects(StepInstanceBaseDTO stepInst * @param stepInstanceId 步骤实例ID * @return 步骤基本信息 */ - StepInstanceDTO getStepInstanceDetail(long taskInstanceId, long stepInstanceId) throws NotFoundException; + StepInstanceDTO getStepInstanceDetail(Long taskInstanceId, long stepInstanceId) throws NotFoundException; /** * 获取步骤基本信息 @@ -146,7 +146,7 @@ Map computeStepExecuteObjects(StepInstanceBaseDTO stepInst * @return 步骤基本信息 */ StepInstanceDTO getStepInstanceDetail(long appId, - long taskInstanceId, + Long taskInstanceId, long stepInstanceId) throws NotFoundException; /** @@ -155,11 +155,11 @@ StepInstanceDTO getStepInstanceDetail(long appId, * @param taskInstanceId 作业实例ID * @return 作业第一个步骤实例 */ - StepInstanceBaseDTO getFirstStepInstance(long taskInstanceId); + StepInstanceBaseDTO getFirstStepInstance(Long taskInstanceId); - List getTaskStepIdList(long taskInstanceId); + List getTaskStepIdList(Long taskInstanceId); - void updateStepStatus(long taskInstanceId, long stepInstanceId, int status); + void updateStepStatus(Long taskInstanceId, long stepInstanceId, int status); /** * 重试步骤操作-重置步骤执行状态 @@ -167,9 +167,9 @@ StepInstanceDTO getStepInstanceDetail(long appId, * @param taskInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID */ - void resetStepExecuteInfoForRetry(long taskInstanceId, long stepInstanceId); + void resetStepExecuteInfoForRetry(Long taskInstanceId, long stepInstanceId); - void resetStepStatus(long taskInstanceId, long stepInstanceId); + void resetStepStatus(Long taskInstanceId, long stepInstanceId); /** * 更新步骤启动时间 - 仅当启动时间为空 @@ -178,9 +178,9 @@ StepInstanceDTO getStepInstanceDetail(long appId, * @param stepInstanceId 步骤实例ID * @param startTime 启动时间 */ - void updateStepStartTimeIfNull(long taskInstanceId, long stepInstanceId, Long startTime); + void updateStepStartTimeIfNull(Long taskInstanceId, long stepInstanceId, Long startTime); - void updateStepEndTime(long taskInstanceId, long stepInstanceId, Long endTime); + void updateStepEndTime(Long taskInstanceId, long stepInstanceId, Long endTime); /** * 步骤重试次数+1 @@ -188,7 +188,7 @@ StepInstanceDTO getStepInstanceDetail(long appId, * @param taskInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID */ - void addStepInstanceExecuteCount(long taskInstanceId, long stepInstanceId); + void addStepInstanceExecuteCount(Long taskInstanceId, long stepInstanceId); /** * 更新步骤的执行信息 @@ -200,7 +200,7 @@ StepInstanceDTO getStepInstanceDetail(long appId, * @param endTime 结束时间 * @param totalTime 总耗时 */ - void updateStepExecutionInfo(long taskInstanceId, + void updateStepExecutionInfo(Long taskInstanceId, long stepInstanceId, RunStatusEnum status, Long startTime, @@ -216,7 +216,7 @@ void updateStepExecutionInfo(long taskInstanceId, * @param isSecureParam 是否为敏感参数 * @param resolvedScriptParam 解析之后的脚本参数 */ - void updateResolvedScriptParam(long taskInstanceId, + void updateResolvedScriptParam(Long taskInstanceId, long stepInstanceId, boolean isSecureParam, String resolvedScriptParam); @@ -228,7 +228,7 @@ void updateResolvedScriptParam(long taskInstanceId, * @param stepInstanceId 步骤实例ID * @param resolvedFileSources 解析后的源文件信息 */ - void updateResolvedSourceFile(long taskInstanceId, long stepInstanceId, List resolvedFileSources); + void updateResolvedSourceFile(Long taskInstanceId, long stepInstanceId, List resolvedFileSources); /** * 更新变量解析之后的目标路径 @@ -237,7 +237,7 @@ void updateResolvedScriptParam(long taskInstanceId, * @param stepInstanceId 步骤实例ID * @param resolvedTargetPath 解析之后的目标路径 */ - void updateResolvedTargetPath(long taskInstanceId, long stepInstanceId, String resolvedTargetPath); + void updateResolvedTargetPath(Long taskInstanceId, long stepInstanceId, String resolvedTargetPath); /** * 更新确认理由 @@ -246,7 +246,7 @@ void updateResolvedScriptParam(long taskInstanceId, * @param stepInstanceId 步骤实例ID * @param confirmReason 确认理由 */ - void updateConfirmReason(long taskInstanceId, long stepInstanceId, String confirmReason); + void updateConfirmReason(Long taskInstanceId, long stepInstanceId, String confirmReason); /** * 更新步骤操作人 @@ -255,7 +255,7 @@ void updateResolvedScriptParam(long taskInstanceId, * @param stepInstanceId 步骤实例ID * @param operator 操作人 */ - void updateStepOperator(long taskInstanceId, long stepInstanceId, String operator); + void updateStepOperator(Long taskInstanceId, long stepInstanceId, String operator); /** * 获取上一步骤实例(可执行的，不包含人工确认这种) @@ -264,7 +264,7 @@ void updateResolvedScriptParam(long taskInstanceId, * @param stepInstanceId 当前步骤实例ID * @return 上一步骤实例 */ - StepInstanceDTO getPreExecutableStepInstance(long taskInstanceId, long stepInstanceId); + StepInstanceDTO getPreExecutableStepInstance(Long taskInstanceId, long stepInstanceId); /** * 根据 taskInstanceId 获取快速任务步骤实例详情 @@ -272,7 +272,7 @@ void updateResolvedScriptParam(long taskInstanceId, * @param taskInstanceId 任务实例ID * @return 步骤详情 */ - StepInstanceDTO getStepInstanceByTaskInstanceId(long taskInstanceId); + StepInstanceDTO getStepInstanceByTaskInstanceId(Long taskInstanceId); /** * 根据 executeObjectCompositeKey 获取步骤实例中的执行对象 diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileExecuteObjectTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileExecuteObjectTaskServiceImpl.java index c6aa0d8c16..afed5e2c06 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileExecuteObjectTaskServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileExecuteObjectTaskServiceImpl.java @@ -71,9 +71,9 @@ public void batchUpdateTasks(Collection tasks) { } @Override - public int getSuccessTaskCount(long stepInstanceId, int executeCount) { - if (isStepInstanceRecordExist(stepInstanceId)) { - return fileExecuteObjectTaskDAO.getSuccessTaskCount(stepInstanceId, executeCount); + public int getSuccessTaskCount(Long taskInstanceId, long stepInstanceId, int executeCount) { + if (isStepInstanceRecordExist(taskInstanceId, stepInstanceId)) { + return fileExecuteObjectTaskDAO.getSuccessTaskCount(taskInstanceId, stepInstanceId, executeCount); } else { return fileAgentTaskDAO.getSuccessAgentTaskCount(stepInstanceId, executeCount); } @@ -88,7 +88,8 @@ public List listTasks(StepInstanceBaseDTO stepInstance, Integ public List listTasksByGseTaskId(StepInstanceBaseDTO stepInstance, Long gseTaskId) { List executeObjectTasks; if (stepInstance.isSupportExecuteObjectFeature()) { - executeObjectTasks = fileExecuteObjectTaskDAO.listTasksByGseTaskId(gseTaskId); + executeObjectTasks = fileExecuteObjectTaskDAO.listTasksByGseTaskId( + stepInstance.getTaskInstanceId(), gseTaskId); } else { // 兼容老版本数据 executeObjectTasks = fileAgentTaskDAO.listAgentTasksByGseTaskId(gseTaskId); @@ -113,7 +114,8 @@ public ExecuteObjectTask getTaskByExecuteObjectCompositeKey(StepInstanceBaseDTO long stepInstanceId = stepInstance.getId(); ExecuteObjectTask executeObjectTask; if (stepInstance.isSupportExecuteObjectFeature()) { - executeObjectTask = fileExecuteObjectTaskDAO.getTaskByExecuteObjectId(stepInstanceId, executeCount, + executeObjectTask = fileExecuteObjectTaskDAO.getTaskByExecuteObjectId( + stepInstance.getTaskInstanceId(), stepInstanceId, executeCount, batch, fileTaskMode, executeObject.getId()); } else { // 兼容老版本不使用执行对象的场景(仅支持主机） @@ -151,7 +153,8 @@ public List listResultGroups(StepInstanceBaseDTO stepInstanc long stepInstanceId = stepInstance.getId(); if (stepInstance.isSupportExecuteObjectFeature()) { - resultGroups = fileExecuteObjectTaskDAO.listResultGroups(stepInstanceId, executeCount, batch); + resultGroups = fileExecuteObjectTaskDAO.listResultGroups( + stepInstance.getTaskInstanceId(), stepInstanceId, executeCount, batch); } else { // 兼容历史数据 resultGroups = fileAgentTaskDAO.listResultGroups(stepInstanceId, executeCount, batch); @@ -168,8 +171,8 @@ public List listTaskByResultGroup(StepInstanceBaseDTO stepIns List executeObjectTasks; if (stepInstance.isSupportExecuteObjectFeature()) { - executeObjectTasks = fileExecuteObjectTaskDAO.listTaskByResultGroup(stepInstance.getId(), - executeCount, batch, status); + executeObjectTasks = fileExecuteObjectTaskDAO.listTaskByResultGroup( + stepInstance.getTaskInstanceId(), stepInstance.getId(), executeCount, batch, status); } else { // 兼容历史数据 executeObjectTasks = fileAgentTaskDAO.listAgentTaskByResultGroup( @@ -193,8 +196,16 @@ public List listTaskByResultGroup(StepInstanceBaseDTO stepIns List executeObjectTasks; if (stepInstance.isSupportExecuteObjectFeature()) { - executeObjectTasks = fileExecuteObjectTaskDAO.listTaskByResultGroup(stepInstance.getId(), - executeCount, batch, status, limit, orderField, order); + executeObjectTasks = fileExecuteObjectTaskDAO.listTaskByResultGroup( + stepInstance.getTaskInstanceId(), + stepInstance.getId(), + executeCount, + batch, + status, + limit, + orderField, + order + ); } else { // 兼容历史数据 executeObjectTasks = fileAgentTaskDAO.listAgentTaskByResultGroup(stepInstance.getId(), executeCount, @@ -205,8 +216,8 @@ public List listTaskByResultGroup(StepInstanceBaseDTO stepIns return executeObjectTasks; } - private boolean isStepInstanceRecordExist(long stepInstanceId) { - return fileExecuteObjectTaskDAO.isStepInstanceRecordExist(stepInstanceId); + private boolean isStepInstanceRecordExist(Long taskInstanceId, long stepInstanceId) { + return fileExecuteObjectTaskDAO.isStepInstanceRecordExist(taskInstanceId, stepInstanceId); } @Override @@ -216,8 +227,14 @@ public void updateTaskFields(StepInstanceBaseDTO stepInstance, Integer actualExecuteCount, Long gseTaskId) { if (stepInstance.isSupportExecuteObjectFeature()) { - fileExecuteObjectTaskDAO.updateTaskFields(stepInstance.getId(), executeCount, batch, - actualExecuteCount, gseTaskId); + fileExecuteObjectTaskDAO.updateTaskFields( + stepInstance.getTaskInstanceId(), + stepInstance.getId(), + executeCount, + batch, + actualExecuteCount, + gseTaskId + ); } else { // 兼容老版本方式 fileAgentTaskDAO.updateAgentTaskFields(stepInstance.getId(), executeCount, batch, @@ -233,7 +250,8 @@ public List listTasks(StepInstanceBaseDTO stepInstance, List executeObjectTasks; long stepInstanceId = stepInstance.getId(); if (stepInstance.isSupportExecuteObjectFeature()) { - executeObjectTasks = fileExecuteObjectTaskDAO.listTasks(stepInstanceId, executeCount, batch, fileTaskMode); + executeObjectTasks = fileExecuteObjectTaskDAO.listTasks( + stepInstance.getTaskInstanceId(), stepInstanceId, executeCount, batch, fileTaskMode); } else { // 兼容老版本数据 executeObjectTasks = fileAgentTaskDAO.listAgentTasks(stepInstanceId, executeCount, batch, fileTaskMode); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/GseTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/GseTaskServiceImpl.java index 25bd16c0b0..97e05078f3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/GseTaskServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/GseTaskServiceImpl.java @@ -56,13 +56,13 @@ public boolean updateGseTask(GseTaskDTO gseTask) { } @Override - public GseTaskDTO getGseTask(long stepInstanceId, int executeCount, Integer batch) { - return gseTaskDAO.getGseTask(stepInstanceId, executeCount, batch); + public GseTaskDTO getGseTask(Long taskInstanceId, long stepInstanceId, int executeCount, Integer batch) { + return gseTaskDAO.getGseTask(taskInstanceId, stepInstanceId, executeCount, batch); } @Override - public GseTaskDTO getGseTask(long gseTaskId) { - return gseTaskDAO.getGseTask(gseTaskId); + public GseTaskDTO getGseTask(Long taskInstanceId, long gseTaskId) { + return gseTaskDAO.getGseTask(taskInstanceId, gseTaskId); } @Override diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/NotifyServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/NotifyServiceImpl.java index ad6e4dc966..58f1bddf62 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/NotifyServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/NotifyServiceImpl.java @@ -38,7 +38,6 @@ import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.TaskNotifyDTO; import com.tencent.bk.job.execute.service.ApplicationService; -import com.tencent.bk.job.execute.service.FileExecuteObjectTaskService; import com.tencent.bk.job.execute.service.NotifyService; import com.tencent.bk.job.execute.service.ScriptExecuteObjectTaskService; import com.tencent.bk.job.execute.service.StepInstanceService; @@ -92,7 +91,6 @@ public class NotifyServiceImpl implements NotifyService { private final TaskInstanceService taskInstanceService; private final MessageI18nService i18nService; private final ScriptExecuteObjectTaskService scriptExecuteObjectTaskService; - private final FileExecuteObjectTaskService fileExecuteObjectTaskService; private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; private final StepInstanceService stepInstanceService; @@ -104,7 +102,6 @@ public NotifyServiceImpl(JobCommonConfig jobCommonConfig, TaskInstanceService taskInstanceService, MessageI18nService i18nService, ScriptExecuteObjectTaskService scriptExecuteObjectTaskService, - FileExecuteObjectTaskService fileExecuteObjectTaskService, TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, StepInstanceService stepInstanceService) { this.jobCommonConfig = jobCommonConfig; @@ -114,7 +111,6 @@ public NotifyServiceImpl(JobCommonConfig jobCommonConfig, this.taskInstanceService = taskInstanceService; this.i18nService = i18nService; this.scriptExecuteObjectTaskService = scriptExecuteObjectTaskService; - this.fileExecuteObjectTaskService = fileExecuteObjectTaskService; this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; this.stepInstanceService = stepInstanceService; } @@ -272,11 +268,17 @@ private Map getTemplateVariablesMap(TaskNotifyDTO taskNotifyDTO, } else { int successIpCount = 0; if (stepInstanceDTO.isScriptStep()) { - successIpCount = scriptExecuteObjectTaskService.getSuccessTaskCount(stepInstanceDTO.getId(), - stepInstanceDTO.getExecuteCount()); + successIpCount = scriptExecuteObjectTaskService.getSuccessTaskCount( + stepInstanceDTO.getTaskInstanceId(), + stepInstanceDTO.getId(), + stepInstanceDTO.getExecuteCount() + ); } else if (stepInstanceDTO.isFileStep()) { - successIpCount = scriptExecuteObjectTaskService.getSuccessTaskCount(stepInstanceDTO.getId(), - stepInstanceDTO.getExecuteCount()); + successIpCount = scriptExecuteObjectTaskService.getSuccessTaskCount( + stepInstanceDTO.getTaskInstanceId(), + stepInstanceDTO.getId(), + stepInstanceDTO.getExecuteCount() + ); } variablesMap.put("task.step.failed_cnt", String.valueOf(totalTargetIpCount - successIpCount)); variablesMap.put("task.step.success_cnt", String.valueOf(successIpCount)); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptExecuteObjectTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptExecuteObjectTaskServiceImpl.java index bb9c494e4e..1d00daf72e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptExecuteObjectTaskServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptExecuteObjectTaskServiceImpl.java @@ -65,9 +65,9 @@ public void batchUpdateTasks(Collection tasks) { } @Override - public int getSuccessTaskCount(long stepInstanceId, int executeCount) { - if (isStepInstanceRecordExist(stepInstanceId)) { - return scriptExecuteObjectTaskDAO.getSuccessTaskCount(stepInstanceId, executeCount); + public int getSuccessTaskCount(Long taskInstanceId, long stepInstanceId, int executeCount) { + if (isStepInstanceRecordExist(taskInstanceId, stepInstanceId)) { + return scriptExecuteObjectTaskDAO.getSuccessTaskCount(taskInstanceId, stepInstanceId, executeCount); } else { return scriptAgentTaskDAO.getSuccessAgentTaskCount(stepInstanceId, executeCount); } @@ -78,7 +78,8 @@ public List listTasks(StepInstanceBaseDTO stepInstance, Integ List executeObjectTasks; long stepInstanceId = stepInstance.getId(); if (stepInstance.isSupportExecuteObjectFeature()) { - executeObjectTasks = scriptExecuteObjectTaskDAO.listTasks(stepInstanceId, executeCount, batch); + executeObjectTasks = scriptExecuteObjectTaskDAO.listTasks(stepInstance.getTaskInstanceId(), + stepInstanceId, executeCount, batch); } else { // 兼容老版本数据 executeObjectTasks = scriptAgentTaskDAO.listAgentTasks(stepInstanceId, executeCount, batch); @@ -92,7 +93,8 @@ public List listTasksByGseTaskId(StepInstanceBaseDTO stepInst List executeObjectTasks; if (stepInstance.isSupportExecuteObjectFeature()) { - executeObjectTasks = scriptExecuteObjectTaskDAO.listTasksByGseTaskId(gseTaskId); + executeObjectTasks = scriptExecuteObjectTaskDAO.listTasksByGseTaskId( + stepInstance.getTaskInstanceId(), gseTaskId); } else { // 兼容老版本数据 executeObjectTasks = scriptAgentTaskDAO.listAgentTasksByGseTaskId(gseTaskId); @@ -115,8 +117,12 @@ public ExecuteObjectTask getTaskByExecuteObjectCompositeKey(StepInstanceBaseDTO long stepInstanceId = stepInstance.getId(); ExecuteObjectTask executeObjectTask; if (stepInstance.isSupportExecuteObjectFeature()) { - executeObjectTask = scriptExecuteObjectTaskDAO.getTaskByExecuteObjectId(stepInstanceId, executeCount, - batch, executeObject.getId()); + executeObjectTask = scriptExecuteObjectTaskDAO.getTaskByExecuteObjectId( + stepInstance.getTaskInstanceId(), + stepInstanceId, executeCount, + batch, + executeObject.getId() + ); } else { // 兼容老版本不使用执行对象的场景(仅支持主机） Long hostId = executeObject.getHost().getHostId(); @@ -151,7 +157,12 @@ public List listResultGroups(StepInstanceBaseDTO stepInstanc long stepInstanceId = stepInstance.getId(); if (stepInstance.isSupportExecuteObjectFeature()) { - resultGroups = scriptExecuteObjectTaskDAO.listResultGroups(stepInstanceId, executeCount, batch); + resultGroups = scriptExecuteObjectTaskDAO.listResultGroups( + stepInstance.getTaskInstanceId(), + stepInstanceId, + executeCount, + batch + ); } else { // 兼容历史数据 resultGroups = scriptAgentTaskDAO.listResultGroups(stepInstanceId, executeCount, batch); @@ -168,8 +179,14 @@ public List listTaskByResultGroup(StepInstanceBaseDTO stepIns List executeObjectTasks; if (stepInstance.isSupportExecuteObjectFeature()) { - executeObjectTasks = scriptExecuteObjectTaskDAO.listTasksByResultGroup(stepInstance.getId(), - executeCount, batch, status, tag); + executeObjectTasks = scriptExecuteObjectTaskDAO.listTasksByResultGroup( + stepInstance.getTaskInstanceId(), + stepInstance.getId(), + executeCount, + batch, + status, + tag + ); } else { // 兼容历史数据 executeObjectTasks = scriptAgentTaskDAO.listAgentTaskByResultGroup( @@ -192,8 +209,17 @@ public List listTaskByResultGroup(StepInstanceBaseDTO stepIns List executeObjectTasks; if (stepInstance.isSupportExecuteObjectFeature()) { - executeObjectTasks = scriptExecuteObjectTaskDAO.listTasksByResultGroup(stepInstance.getId(), - executeCount, batch, status, tag, limit, orderField, order); + executeObjectTasks = scriptExecuteObjectTaskDAO.listTasksByResultGroup( + stepInstance.getTaskInstanceId(), + stepInstance.getId(), + executeCount, + batch, + status, + tag, + limit, + orderField, + order + ); } else { // 兼容历史数据 executeObjectTasks = scriptAgentTaskDAO.listAgentTaskByResultGroup(stepInstance.getId(), executeCount, @@ -204,8 +230,8 @@ public List listTaskByResultGroup(StepInstanceBaseDTO stepIns return executeObjectTasks; } - private boolean isStepInstanceRecordExist(long stepInstanceId) { - return scriptExecuteObjectTaskDAO.isStepInstanceRecordExist(stepInstanceId); + private boolean isStepInstanceRecordExist(Long taskInstanceId, long stepInstanceId) { + return scriptExecuteObjectTaskDAO.isStepInstanceRecordExist(taskInstanceId, stepInstanceId); } @Override @@ -215,8 +241,8 @@ public void updateTaskFields(StepInstanceBaseDTO stepInstance, Integer actualExecuteCount, Long gseTaskId) { if (stepInstance.isSupportExecuteObjectFeature()) { - scriptExecuteObjectTaskDAO.updateTaskFields(stepInstance.getId(), executeCount, batch, - actualExecuteCount, gseTaskId); + scriptExecuteObjectTaskDAO.updateTaskFields(stepInstance.getTaskInstanceId(), stepInstance.getId(), + executeCount, batch, actualExecuteCount, gseTaskId); } else { // 兼容老版本方式 scriptAgentTaskDAO.updateAgentTaskFields(stepInstance.getId(), executeCount, batch, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceRollingTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceRollingTaskServiceImpl.java index ac0652a97f..d08d670e63 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceRollingTaskServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceRollingTaskServiceImpl.java @@ -52,15 +52,19 @@ public StepInstanceRollingTaskServiceImpl( } @Override - public StepInstanceRollingTaskDTO queryRollingTask(long stepInstanceId, - int executeCount, int batch) { - return stepInstanceRollingTaskDAO.queryRollingTask(stepInstanceId, executeCount, batch); + public StepInstanceRollingTaskDTO queryRollingTask(Long taskInstanceId, + long stepInstanceId, + int executeCount, + int batch) { + return stepInstanceRollingTaskDAO.queryRollingTask(taskInstanceId, stepInstanceId, executeCount, batch); } @Override - public List listLatestRollingTasks(long stepInstanceId, int executeCount) { + public List listLatestRollingTasks(Long taskInstanceId, + long stepInstanceId, + int executeCount) { List stepInstanceRollingTasks = - stepInstanceRollingTaskDAO.listRollingTasks(stepInstanceId, null, null); + stepInstanceRollingTaskDAO.listRollingTasks(taskInstanceId, stepInstanceId, null, null); if (CollectionUtils.isEmpty(stepInstanceRollingTasks)) { return stepInstanceRollingTasks; } @@ -92,8 +96,10 @@ public List listLatestRollingTasks(long stepInstance } @Override - public List listRollingTasksByBatch(long stepInstanceId, Integer batch) { - return stepInstanceRollingTaskDAO.listRollingTasks(stepInstanceId, null, batch); + public List listRollingTasksByBatch(Long taskInstanceId, + long stepInstanceId, + Integer batch) { + return stepInstanceRollingTaskDAO.listRollingTasks(taskInstanceId, stepInstanceId, null, batch); } @Override @@ -102,19 +108,20 @@ public long saveRollingTask(StepInstanceRollingTaskDTO rollingTask) { } @Override - public void updateRollingTask(long stepInstanceId, + public void updateRollingTask(Long taskInstanceId, + long stepInstanceId, int executeCount, int batch, RunStatusEnum status, Long startTime, Long endTime, Long totalTime) { - stepInstanceRollingTaskDAO.updateRollingTask(stepInstanceId, executeCount, batch, status, startTime, - endTime, totalTime); + stepInstanceRollingTaskDAO.updateRollingTask(taskInstanceId, stepInstanceId, executeCount, batch, status, + startTime, endTime, totalTime); } @Override - public List listRollingTasksByStep(long stepInstanceId) { - return stepInstanceRollingTaskDAO.listRollingTasks(stepInstanceId, null, null); + public List listRollingTasksByStep(Long taskInstanceId, long stepInstanceId) { + return stepInstanceRollingTaskDAO.listRollingTasks(taskInstanceId, stepInstanceId, null, null); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java index 5959811cd1..33af9df86c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java @@ -72,17 +72,17 @@ public StepInstanceServiceImpl(StepInstanceDAO stepInstanceDAO) { } @Override - public void updateStepCurrentBatch(long taskInstanceId, long stepInstanceId, int batch) { + public void updateStepCurrentBatch(Long taskInstanceId, long stepInstanceId, int batch) { stepInstanceDAO.updateStepCurrentBatch(taskInstanceId, stepInstanceId, batch); } @Override - public void updateStepRollingConfigId(long taskInstanceId, long stepInstanceId, long rollingConfigId) { + public void updateStepRollingConfigId(Long taskInstanceId, long stepInstanceId, long rollingConfigId) { stepInstanceDAO.updateStepRollingConfigId(taskInstanceId, stepInstanceId, rollingConfigId); } @Override - public StepInstanceBaseDTO getNextStepInstance(long taskInstanceId, + public StepInstanceBaseDTO getNextStepInstance(Long taskInstanceId, int currentStepOrder) { return stepInstanceDAO.getNextStepInstance(taskInstanceId, currentStepOrder); } @@ -105,7 +105,7 @@ public long addStepInstance(StepInstanceDTO stepInstance) { } private void fillStepInstanceDetail(StepInstanceDTO stepInstance) { - long taskInstanceId = stepInstance.getTaskInstanceId(); + Long taskInstanceId = stepInstance.getTaskInstanceId(); long stepInstanceId = stepInstance.getId(); TaskStepTypeEnum stepType = stepInstance.getStepType(); if (stepType == TaskStepTypeEnum.SCRIPT) { @@ -118,12 +118,12 @@ private void fillStepInstanceDetail(StepInstanceDTO stepInstance) { } @Override - public List listBaseStepInstanceByTaskInstanceId(long taskInstanceId) { + public List listBaseStepInstanceByTaskInstanceId(Long taskInstanceId) { return stepInstanceDAO.listStepInstanceBaseByTaskInstanceId(taskInstanceId); } @Override - public List listStepInstanceByTaskInstanceId(long taskInstanceId) { + public List listStepInstanceByTaskInstanceId(Long taskInstanceId) { List stepInstanceList = listBaseStepInstanceByTaskInstanceId(taskInstanceId); if (CollectionUtils.isEmpty(stepInstanceList)) { return Collections.emptyList(); @@ -137,7 +137,7 @@ public List listStepInstanceByTaskInstanceId(long taskInstanceI } @Override - public StepInstanceBaseDTO getBaseStepInstance(long taskInstanceId, long stepInstanceId) { + public StepInstanceBaseDTO getBaseStepInstance(Long taskInstanceId, long stepInstanceId) { return stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); } @@ -147,7 +147,7 @@ public StepInstanceBaseDTO getBaseStepInstance(long stepInstanceId) { } @Override - public StepInstanceBaseDTO getBaseStepInstance(long appId, long taskInstanceId, long stepInstanceId) { + public StepInstanceBaseDTO getBaseStepInstance(long appId, Long taskInstanceId, long stepInstanceId) { StepInstanceBaseDTO stepInstance = getBaseStepInstance(taskInstanceId, stepInstanceId); if (stepInstance == null) { log.warn("Step instance is not exist, taskInstanceId={}, stepInstanceId: {}", @@ -162,7 +162,7 @@ public StepInstanceBaseDTO getBaseStepInstance(long appId, long taskInstanceId, } @Override - public StepInstanceDTO getStepInstanceDetail(long taskInstanceId, long stepInstanceId) throws NotFoundException { + public StepInstanceDTO getStepInstanceDetail(Long taskInstanceId, long stepInstanceId) throws NotFoundException { StepInstanceBaseDTO stepInstanceBase = stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); if (stepInstanceBase == null) { log.warn("StepInstance:{} not exist", stepInstanceId); @@ -174,7 +174,7 @@ public StepInstanceDTO getStepInstanceDetail(long taskInstanceId, long stepInsta } @Override - public StepInstanceDTO getStepInstanceDetail(long appId, long taskInstanceId, + public StepInstanceDTO getStepInstanceDetail(long appId, Long taskInstanceId, long stepInstanceId) throws NotFoundException { StepInstanceDTO stepInstance = getStepInstanceDetail(taskInstanceId, stepInstanceId); if (!stepInstance.getAppId().equals(appId)) { @@ -185,47 +185,47 @@ public StepInstanceDTO getStepInstanceDetail(long appId, long taskInstanceId, } @Override - public StepInstanceBaseDTO getFirstStepInstance(long taskInstanceId) { + public StepInstanceBaseDTO getFirstStepInstance(Long taskInstanceId) { return stepInstanceDAO.getFirstStepInstanceBase(taskInstanceId); } @Override - public List getTaskStepIdList(long taskInstanceId) { + public List getTaskStepIdList(Long taskInstanceId) { return stepInstanceDAO.getTaskStepInstanceIdList(taskInstanceId); } @Override - public void updateStepStatus(long taskInstanceId, long stepInstanceId, int status) { + public void updateStepStatus(Long taskInstanceId, long stepInstanceId, int status) { stepInstanceDAO.updateStepStatus(taskInstanceId, stepInstanceId, status); } @Override - public void resetStepExecuteInfoForRetry(long taskInstanceId, long stepInstanceId) { + public void resetStepExecuteInfoForRetry(Long taskInstanceId, long stepInstanceId) { stepInstanceDAO.resetStepExecuteInfoForRetry(taskInstanceId, stepInstanceId); } @Override - public void resetStepStatus(long taskInstanceId, long stepInstanceId) { + public void resetStepStatus(Long taskInstanceId, long stepInstanceId) { stepInstanceDAO.resetStepStatus(taskInstanceId, stepInstanceId); } @Override - public void updateStepStartTimeIfNull(long taskInstanceId, long stepInstanceId, Long startTime) { + public void updateStepStartTimeIfNull(Long taskInstanceId, long stepInstanceId, Long startTime) { stepInstanceDAO.updateStepStartTimeIfNull(taskInstanceId, stepInstanceId, startTime); } @Override - public void updateStepEndTime(long taskInstanceId, long stepInstanceId, Long endTime) { + public void updateStepEndTime(Long taskInstanceId, long stepInstanceId, Long endTime) { stepInstanceDAO.updateStepEndTime(taskInstanceId, stepInstanceId, endTime); } @Override - public void addStepInstanceExecuteCount(long taskInstanceId, long stepInstanceId) { + public void addStepInstanceExecuteCount(Long taskInstanceId, long stepInstanceId) { stepInstanceDAO.addStepInstanceExecuteCount(taskInstanceId, stepInstanceId); } @Override - public void updateStepExecutionInfo(long taskInstanceId, + public void updateStepExecutionInfo(Long taskInstanceId, long stepInstanceId, RunStatusEnum status, Long startTime, @@ -235,7 +235,7 @@ public void updateStepExecutionInfo(long taskInstanceId, } @Override - public void updateResolvedScriptParam(long taskInstanceId, + public void updateResolvedScriptParam(Long taskInstanceId, long stepInstanceId, boolean isSecureParam, String resolvedScriptParam) { @@ -243,7 +243,7 @@ public void updateResolvedScriptParam(long taskInstanceId, } @Override - public void updateResolvedSourceFile(long taskInstanceId, + public void updateResolvedSourceFile(Long taskInstanceId, long stepInstanceId, List resolvedFileSources) { if (log.isDebugEnabled()) { @@ -253,22 +253,22 @@ public void updateResolvedSourceFile(long taskInstanceId, } @Override - public void updateResolvedTargetPath(long taskInstanceId, long stepInstanceId, String resolvedTargetPath) { + public void updateResolvedTargetPath(Long taskInstanceId, long stepInstanceId, String resolvedTargetPath) { stepInstanceDAO.updateResolvedTargetPath(taskInstanceId, stepInstanceId, resolvedTargetPath); } @Override - public void updateConfirmReason(long taskInstanceId, long stepInstanceId, String confirmReason) { + public void updateConfirmReason(Long taskInstanceId, long stepInstanceId, String confirmReason) { stepInstanceDAO.updateConfirmReason(taskInstanceId, stepInstanceId, confirmReason); } @Override - public void updateStepOperator(long taskInstanceId, long stepInstanceId, String operator) { + public void updateStepOperator(Long taskInstanceId, long stepInstanceId, String operator) { stepInstanceDAO.updateStepOperator(taskInstanceId, stepInstanceId, operator); } @Override - public StepInstanceDTO getPreExecutableStepInstance(long taskInstanceId, long stepInstanceId) { + public StepInstanceDTO getPreExecutableStepInstance(Long taskInstanceId, long stepInstanceId) { StepInstanceBaseDTO preStepInstance = stepInstanceDAO.getPreExecutableStepInstance(taskInstanceId, stepInstanceId); if (preStepInstance == null) { @@ -280,7 +280,7 @@ public StepInstanceDTO getPreExecutableStepInstance(long taskInstanceId, long st } @Override - public StepInstanceDTO getStepInstanceByTaskInstanceId(long taskInstanceId) { + public StepInstanceDTO getStepInstanceByTaskInstanceId(Long taskInstanceId) { List stepInstanceList = stepInstanceDAO.listStepInstanceBaseByTaskInstanceId(taskInstanceId); if (CollectionUtils.isEmpty(stepInstanceList)) { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java index 14536412e1..48b331b369 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java @@ -459,6 +459,7 @@ public StepExecutionDetailDTO getStepExecutionResult(String username, Long appId watch.start("involveFileSourceTaskLog"); FileSourceTaskLogDTO fileSourceTaskLog = fileSourceTaskLogDAO.getFileSourceTaskLog( + stepInstance.getTaskInstanceId(), stepInstance.getId(), queryExecuteCount ); @@ -777,6 +778,7 @@ private void setRollingInfoForStep(StepInstanceBaseDTO stepInstance, Map latestStepInstanceRollingTasks = stepInstanceRollingTaskService.listLatestRollingTasks( + stepInstance.getTaskInstanceId(), stepExecutionDetail.getStepInstanceId(), stepExecutionDetail.getExecuteCount()) .stream() @@ -997,7 +999,7 @@ public List listStepExecutionHistory(String username, records = queryStepRetryRecords(stepInstance); } else { // 获取滚动任务维度的重试记录 - records = queryStepRollingTaskRetryRecords(stepInstanceId, batch); + records = queryStepRollingTaskRetryRecords(stepInstance.getTaskInstanceId(), stepInstanceId, batch); } records.sort(Comparator.comparingInt(StepExecutionRecordDTO::getRetryCount).reversed()); @@ -1037,10 +1039,12 @@ private List queryStepRetryRecords(StepInstanceBaseDTO s return records; } - private List queryStepRollingTaskRetryRecords(long stepInstanceId, int batch) { + private List queryStepRollingTaskRetryRecords(Long taskInstanceId, + long stepInstanceId, + int batch) { List records = new ArrayList<>(); List rollingTasks = - stepInstanceRollingTaskService.listRollingTasksByBatch(stepInstanceId, batch); + stepInstanceRollingTaskService.listRollingTasksByBatch(taskInstanceId, stepInstanceId, batch); rollingTasks.forEach(rollingTask -> { StepExecutionRecordDTO record = new StepExecutionRecordDTO(); record.setStepInstanceId(stepInstanceId); diff --git a/support-files/sql/job-execute/0026_job_execute_20240520-1000_V3.10.0_mysql.sql b/support-files/sql/job-execute/0026_job_execute_20240520-1000_V3.10.0_mysql.sql index c56970b340..340aa91f3e 100644 --- a/support-files/sql/job-execute/0026_job_execute_20240520-1000_V3.10.0_mysql.sql +++ b/support-files/sql/job-execute/0026_job_execute_20240520-1000_V3.10.0_mysql.sql @@ -73,35 +73,35 @@ BEGIN ALTER TABLE gse_task ADD INDEX `idx_task_instance_id` (`task_instance_id`); END IF; - IF NOT EXISTS(SELECT 1 - FROM information_schema.COLUMNS - WHERE TABLE_SCHEMA = db - AND TABLE_NAME = 'gse_script_agent_task' - AND COLUMN_NAME = 'task_instance_id') THEN - ALTER TABLE gse_script_agent_task ADD COLUMN `task_instance_id` bigint(20) NOT NULL DEFAULT '0'; - END IF; - IF NOT EXISTS(SELECT 1 - FROM information_schema.statistics - WHERE TABLE_SCHEMA = db - AND TABLE_NAME = 'gse_script_agent_task' - AND INDEX_NAME = 'idx_task_instance_id') THEN - ALTER TABLE gse_script_agent_task ADD INDEX `idx_task_instance_id` (`task_instance_id`); - END IF; - - IF NOT EXISTS(SELECT 1 - FROM information_schema.COLUMNS - WHERE TABLE_SCHEMA = db - AND TABLE_NAME = 'gse_file_agent_task' - AND COLUMN_NAME = 'task_instance_id') THEN - ALTER TABLE gse_file_agent_task ADD COLUMN `task_instance_id` bigint(20) NOT NULL DEFAULT '0'; - END IF; - IF NOT EXISTS(SELECT 1 - FROM information_schema.statistics - WHERE TABLE_SCHEMA = db - AND TABLE_NAME = 'gse_file_agent_task' - AND INDEX_NAME = 'idx_task_instance_id') THEN - ALTER TABLE gse_file_agent_task ADD INDEX `idx_task_instance_id` (`task_instance_id`); - END IF; + -- IF NOT EXISTS(SELECT 1 + -- FROM information_schema.COLUMNS + -- WHERE TABLE_SCHEMA = db + -- AND TABLE_NAME = 'gse_script_agent_task' + -- AND COLUMN_NAME = 'task_instance_id') THEN + -- ALTER TABLE gse_script_agent_task ADD COLUMN `task_instance_id` bigint(20) NOT NULL DEFAULT '0'; + -- END IF; + -- IF NOT EXISTS(SELECT 1 + -- FROM information_schema.statistics + -- WHERE TABLE_SCHEMA = db + -- AND TABLE_NAME = 'gse_script_agent_task' + -- AND INDEX_NAME = 'idx_task_instance_id') THEN + -- ALTER TABLE gse_script_agent_task ADD INDEX `idx_task_instance_id` (`task_instance_id`); + -- END IF; + + -- IF NOT EXISTS(SELECT 1 + -- FROM information_schema.COLUMNS + -- WHERE TABLE_SCHEMA = db + -- AND TABLE_NAME = 'gse_file_agent_task' + -- AND COLUMN_NAME = 'task_instance_id') THEN + -- ALTER TABLE gse_file_agent_task ADD COLUMN `task_instance_id` bigint(20) NOT NULL DEFAULT '0'; + -- END IF; + -- IF NOT EXISTS(SELECT 1 + -- FROM information_schema.statistics + -- WHERE TABLE_SCHEMA = db + -- AND TABLE_NAME = 'gse_file_agent_task' + -- AND INDEX_NAME = 'idx_task_instance_id') THEN + -- ALTER TABLE gse_file_agent_task ADD INDEX `idx_task_instance_id` (`task_instance_id`); + -- END IF; IF NOT EXISTS(SELECT 1 FROM information_schema.COLUMNS From b355c29983010613072a94cd6ac7ca87a1901ab7 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Tue, 21 May 2024 15:03:33 +0800 Subject: [PATCH 03/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../job/execute/dao/impl/TaskInstanceIdDynamicCondition.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java index b2b19b3b8d..194e435d66 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java @@ -43,7 +43,7 @@ public static Condition build(Long taskInstanceId, Function taskInstanceIdConditionBuilder) { ResourceScope resourceScope = JobContextUtil.getAppResourceScope(); if (resourceScope == null) { - log.warn("Empty resource scope, expected!"); + log.warn("TaskInstanceIdDynamicCondition : Empty resource scope!"); // 为了不影响兼容性，忽略错误 return DSL.trueCondition(); } @@ -53,7 +53,8 @@ public static Condition build(Long taskInstanceId, .addContextParam(ToggleStrategyContextParams.CTX_PARAM_RESOURCE_SCOPE, JobContextUtil.getAppResourceScope()))) { if (taskInstanceId == null || taskInstanceId <= 0L) { - log.warn("Invalid taskInstanceId, ignore task_instance_id condition. taskInstanceId: {}", + log.warn("TaskInstanceIdDynamicCondition : Invalid taskInstanceId for building query condition. " + + "taskInstanceId : {}", taskInstanceId); // 为了不影响兼容性，忽略错误 return DSL.trueCondition(); From a169879f9a9153fe55d6c4cb7cc90d8a2ad694df Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Tue, 21 May 2024 16:27:48 +0800 Subject: [PATCH 04/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../web/WebTaskExecutionResultResource.java | 30 ++++++++++++ .../model/web/vo/StepExecutionRecordVO.java | 4 ++ .../WebTaskExecutionResultResourceImpl.java | 48 +++++++++++++++++-- .../bk/job/execute/dao/StepInstanceDAO.java | 5 ++ .../execute/dao/impl/StepInstanceDAOImpl.java | 9 +++- .../model/StepExecutionResultQuery.java | 5 +- .../execute/service/TaskResultService.java | 4 ++ .../service/impl/TaskResultServiceImpl.java | 23 +++++---- 8 files changed, 113 insertions(+), 15 deletions(-) diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java index a8471f8988..868f7f4129 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java @@ -520,6 +520,9 @@ Response> getExecuteObjectsByResultType( @ApiOperation(value = "获取步骤执行历史", produces = "application/json") @GetMapping(value = {"/step-execution-history/{stepInstanceId}"}) + @Deprecated + @CompatibleImplementation(name = "add_task_instance_id", deprecatedVersion = "3.10.x", type = CompatibleType.DEPLOY, + explain = "发布完成后可以删除") Response> listStepExecutionHistory( @ApiParam("用户名，网关自动传入") @RequestHeader("username") @@ -541,4 +544,31 @@ Response> listStepExecutionHistory( Integer batch ); + + @ApiOperation(value = "获取步骤执行历史", produces = "application/json") + @GetMapping(value = {"/taskInstance/{taskInstanceId}/stepInstance/{stepInstanceId}/stepExecutionHistory"}) + Response> listStepExecutionHistoryV2( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "作业实例ID", name = "stepInstanceId", required = true) + @PathVariable("taskInstanceId") + Long taskInstanceId, + @ApiParam(value = "步骤实例ID", name = "stepInstanceId", required = true) + @PathVariable("stepInstanceId") + Long stepInstanceId, + @ApiParam(value = "滚动批次，非滚动步骤不需要传入", name = "batch") + @RequestParam(value = "batch", required = false) + Integer batch + ); + } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionRecordVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionRecordVO.java index 0941371053..8292cbf414 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionRecordVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionRecordVO.java @@ -35,6 +35,10 @@ @ApiModel("步骤执行记录") @Data public class StepExecutionRecordVO { + + @ApiModelProperty(value = "作业实例ID") + private Long taskInstanceId; + @ApiModelProperty(value = "步骤实例ID") private Long stepInstanceId; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java index 00561a86c3..776e22c027 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java @@ -966,7 +966,16 @@ public Response> getHostsByResultType(String username, String tag, String keyword) { List executeObjects = taskResultService.getExecuteObjectsByResultType( - username, appResourceScope.getAppId(), stepInstanceId, executeCount, batch, resultType, tag, keyword); + username, + appResourceScope.getAppId(), + null, + stepInstanceId, + executeCount, + batch, + resultType, + tag, + keyword + ); if (CollectionUtils.isEmpty(executeObjects)) { return Response.buildSuccessResp(Collections.emptyList()); } @@ -987,10 +996,31 @@ public Response> listStepExecutionHistory(String use Integer batch) { List stepExecutionRecords = taskResultService.listStepExecutionHistory(username, - appResourceScope.getAppId(), stepInstanceId, batch); + appResourceScope.getAppId(), null, stepInstanceId, batch); + + return Response.buildSuccessResp(stepExecutionRecords.stream().map(stepExecutionRecord -> { + StepExecutionRecordVO vo = new StepExecutionRecordVO(); + vo.setStepInstanceId(stepInstanceId); + vo.setRetryCount(stepExecutionRecord.getRetryCount()); + vo.setExecuteCount(stepExecutionRecord.getRetryCount()); + vo.setCreateTime(stepExecutionRecord.getCreateTime()); + return vo; + }).collect(Collectors.toList())); + } + + @Override + public Response> listStepExecutionHistoryV2(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + Long taskInstanceId, + Long stepInstanceId, Integer batch) { + List stepExecutionRecords = taskResultService.listStepExecutionHistory(username, + appResourceScope.getAppId(), taskInstanceId, stepInstanceId, batch); return Response.buildSuccessResp(stepExecutionRecords.stream().map(stepExecutionRecord -> { StepExecutionRecordVO vo = new StepExecutionRecordVO(); + vo.setTaskInstanceId(taskInstanceId); vo.setStepInstanceId(stepInstanceId); vo.setRetryCount(stepExecutionRecord.getRetryCount()); vo.setExecuteCount(stepExecutionRecord.getRetryCount()); @@ -1200,8 +1230,18 @@ public Response> getExecuteObjectsByResultType(String user Integer resultType, String tag, String keyword) { - List executeObjects = taskResultService.getExecuteObjectsByResultType( - username, appResourceScope.getAppId(), stepInstanceId, executeCount, batch, resultType, tag, keyword); + List executeObjects = + taskResultService.getExecuteObjectsByResultType( + username, + appResourceScope.getAppId(), + taskInstanceId, + stepInstanceId, + executeCount, + batch, + resultType, + tag, + keyword + ); if (CollectionUtils.isEmpty(executeObjects)) { return Response.buildSuccessResp(Collections.emptyList()); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceDAO.java index 18bab32cb9..eb3e85c62a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceDAO.java @@ -48,6 +48,11 @@ public interface StepInstanceDAO { StepInstanceBaseDTO getStepInstanceBase(Long taskInstanceId, long stepInstanceId); + /** + * 获取步骤信息。注意，在分库分表的部署架构下，由于缺少 taskInstanceId （分片键）作为查询条件，该 SQL 执行性能较低；慎用！！ + * + * @param stepInstanceId 步骤实例 ID + */ StepInstanceBaseDTO getStepInstanceBase(long stepInstanceId); /** diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImpl.java index df7907a328..348305186e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImpl.java @@ -317,7 +317,12 @@ public ScriptStepInstanceDTO getScriptStepInstance(Long taskInstanceId, long ste private Condition buildTaskInstanceIdQueryCondition(Table table, Long taskInstanceId) { - if (table instanceof StepInstanceScript) { + if (table instanceof StepInstance) { + return TaskInstanceIdDynamicCondition.build( + taskInstanceId, + T_STEP_INSTANCE.TASK_INSTANCE_ID::eq + ); + } else if (table instanceof StepInstanceScript) { return TaskInstanceIdDynamicCondition.build( taskInstanceId, T_STEP_INSTANCE_SCRIPT.TASK_INSTANCE_ID::eq @@ -466,7 +471,7 @@ public StepInstanceBaseDTO getStepInstanceBase(Long taskInstanceId, long stepIns Record record = CTX .select(T_STEP_INSTANCE_ALL_FIELDS) .from(T_STEP_INSTANCE) - .where(T_STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE, taskInstanceId)) .and(T_STEP_INSTANCE.ID.eq(stepInstanceId)) .fetchOne(); return extractBaseInfo(record); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionResultQuery.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionResultQuery.java index 75814ea195..822b355cce 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionResultQuery.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionResultQuery.java @@ -48,7 +48,10 @@ public class StepExecutionResultQuery { ORDER_FIELD_IN_DB.put(ORDER_FIELD_TOTAL_TIME, "total_time"); ORDER_FIELD_IN_DB.put(ORDER_FIELD_EXIT_CODE, "exit_code"); } - + /** + * 作业实例ID + */ + private Long taskInstanceId; /** * 步骤实例ID */ diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskResultService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskResultService.java index 4864ace549..1df9b1df71 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskResultService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskResultService.java @@ -89,6 +89,7 @@ Map getCronTaskExecuteResultStatis * * @param username 用户名 * @param appId 业务ID + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次;如果传入null或0，忽略该参数 @@ -99,6 +100,7 @@ Map getCronTaskExecuteResultStatis */ List getExecuteObjectsByResultType(String username, Long appId, + Long taskInstanceId, Long stepInstanceId, Integer batch, Integer executeCount, @@ -111,12 +113,14 @@ List getExecuteObjectsByResultType(String username, * * @param username 用户名 * @param appId 业务ID + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param batch 滚动执行批次;如果传入，则会按照batch过滤步骤执行历史 * @return 执行历史 */ List listStepExecutionHistory(String username, Long appId, + Long taskInstanceId, Long stepInstanceId, Integer batch); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java index 48b331b369..67123192a5 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java @@ -28,7 +28,6 @@ import com.tencent.bk.job.common.constant.Order; import com.tencent.bk.job.common.exception.FailedPreconditionException; import com.tencent.bk.job.common.exception.NotFoundException; -import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; @@ -164,8 +163,9 @@ private void computeTotalTime(List pageData) { } @Override - public TaskExecuteResultDTO getTaskExecutionResult(String username, Long appId, - Long taskInstanceId) throws ServiceException { + public TaskExecuteResultDTO getTaskExecutionResult(String username, + Long appId, + Long taskInstanceId) { TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(username, appId, taskInstanceId); TaskExecutionDTO taskExecution = buildTaskExecutionDTO(taskInstance); @@ -415,14 +415,16 @@ private List filterTargetExecuteObjectsByBatch(StepInstanceBaseDT } @Override - public StepExecutionDetailDTO getStepExecutionResult(String username, Long appId, - StepExecutionResultQuery query) throws ServiceException { + public StepExecutionDetailDTO getStepExecutionResult(String username, + Long appId, + StepExecutionResultQuery query) { StopWatch watch = new StopWatch("getStepExecutionResult"); try { Long stepInstanceId = query.getStepInstanceId(); watch.start("getAndCheckStepInstance"); - StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance(appId, stepInstanceId); + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance( + appId, query.getTaskInstanceId(), stepInstanceId); preProcessViewStepExecutionResult(username, appId, stepInstance); int queryExecuteCount = query.getExecuteCount() == null ? stepInstance.getExecuteCount() : @@ -922,16 +924,19 @@ private List convertToCronTaskExecuteResult(List getExecuteObjectsByResultType(String username, Long appId, + Long taskInstanceId, Long stepInstanceId, Integer executeCount, Integer batch, Integer resultType, String tag, String keyword) { - StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance(appId, stepInstanceId); + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance( + appId, taskInstanceId, stepInstanceId); preProcessViewStepExecutionResult(username, appId, stepInstance); StepExecutionResultQuery query = StepExecutionResultQuery.builder() + .taskInstanceId(taskInstanceId) .stepInstanceId(stepInstanceId) .executeCount(executeCount) .batch(batch) @@ -979,9 +984,11 @@ public List getExecuteObjectsByResultType(String username, @Override public List listStepExecutionHistory(String username, Long appId, + Long taskInstanceId, Long stepInstanceId, Integer batch) { - StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance(appId, stepInstanceId); + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance( + appId, taskInstanceId, stepInstanceId); preProcessViewStepExecutionResult(username, appId, stepInstance); // 步骤没有重试执行过 From b50f2ea5a2057becbcac40545259eef0d443c3a7 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Wed, 22 May 2024 14:32:27 +0800 Subject: [PATCH 05/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bk/job/common/constant/ArchiveFlag.java | 43 +++++++++ .../colddata/JobInstanceRecordQuery.java | 37 ++++++++ .../JobInstanceRecordQueryAspect.java | 91 +++++++++++++++++++ .../TaskInstanceRecordStateCache.java | 76 ++++++++++++++++ ...askInstanceRecordStateThreadLocalRepo.java | 47 ++++++++++ .../config/JobFunctionConfiguration.java | 7 +- .../execute/engine/listener/JobListener.java | 6 +- .../event/TaskExecuteMQEventDispatcher.java | 9 +- .../model/TaskInstanceRecordStateDO.java | 53 +++++++++++ 9 files changed, 364 insertions(+), 5 deletions(-) create mode 100644 src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ArchiveFlag.java create mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobInstanceRecordQuery.java create mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobInstanceRecordQueryAspect.java create mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/TaskInstanceRecordStateCache.java create mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/TaskInstanceRecordStateThreadLocalRepo.java create mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInstanceRecordStateDO.java diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ArchiveFlag.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ArchiveFlag.java new file mode 100644 index 0000000000..42a7e3c5e4 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ArchiveFlag.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.constant; + +/** + * 数据归档状态 + */ +public enum ArchiveFlag { + HOT(1), + COLD(2); + + ArchiveFlag(int value) { + this.value = value; + } + + private final int value; + + public int getValue() { + return value; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobInstanceRecordQuery.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobInstanceRecordQuery.java new file mode 100644 index 0000000000..2ce32842b8 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobInstanceRecordQuery.java @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.colddata; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target({ElementType.TYPE, ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +@Inherited +public @interface JobInstanceRecordQuery { +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobInstanceRecordQueryAspect.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobInstanceRecordQueryAspect.java new file mode 100644 index 0000000000..ab2caf2fcd --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobInstanceRecordQueryAspect.java @@ -0,0 +1,91 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.colddata; + +import lombok.extern.slf4j.Slf4j; + + +//@Aspect +@Slf4j +public class JobInstanceRecordQueryAspect { +// private final MeterRegistry registry; +// +// public JobInstanceRecordQueryAspect(MeterRegistry registry) { +// log.info("Init JobInstanceRecordQueryAspect"); +// this.registry = registry; +// } +// +// @Around("within(com.tencent.bk.job..*) && execution (@com.tencent.bk.job.execute.colddata.JobInstanceRecordQuery * *.*(..))") +// public Object process(ProceedingJoinPoint pjp) throws Throwable { +// RequestAttributes requestAttributes = RequestContextHolder.getRequestAttributes(); +// HttpServletRequest request = null; +// if (requestAttributes != null) { +// request = ((ServletRequestAttributes)requestAttributes).getRequest(); +// } +// if (request == null) { +// return pjp.proceed(); +// } +// +// JobContextUtil.set +// +// String appCode = request.getHeader(JobCommonHeaders.APP_CODE); +// Tags tags = Tags.of("app_code", StringUtils.isNotBlank(appCode) ? appCode : "None"); +// +// Method method = ((MethodSignature) pjp.getSignature()).getMethod(); +// JobInstanceRecordQuery timed = method.getAnnotation(JobInstanceRecordQuery.class); +// final String metricName = timed.value(); +// return processWithTimer(pjp, timed, metricName, tags); +// } +// +// private Object processWithTimer(ProceedingJoinPoint pjp, JobInstanceRecordQuery timed, String metricName, +// Tags tags) throws Throwable { +// +// Timer.Sample sample = Timer.start(registry); +// Exception exception = null; +// try { +// return pjp.proceed(); +// } catch (Exception ex) { +// exception = ex; +// throw ex; +// } finally { +// tags.and(exception(exception)); +// record(timed, metricName, sample, tags); +// } +// } +// +// private void record(JobInstanceRecordQuery timed, String metricName, Timer.Sample sample, Tags tags) { +// try { +// sample.stop(Timer.builder(metricName) +// .description(timed.description().isEmpty() ? null : timed.description()) +// .tags(tags) +// .tags(timed.extraTags()) +// .publishPercentileHistogram(timed.histogram()) +// .publishPercentiles(timed.percentiles().length == 0 ? null : timed.percentiles()) +// .register(registry)); +// } catch (Exception e) { +// // ignoring on purpose +// } +// } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/TaskInstanceRecordStateCache.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/TaskInstanceRecordStateCache.java new file mode 100644 index 0000000000..bd1db06613 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/TaskInstanceRecordStateCache.java @@ -0,0 +1,76 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.colddata; + +import com.tencent.bk.job.common.redis.BaseRedisCache; +import com.tencent.bk.job.execute.model.TaskInstanceRecordStateDO; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.stereotype.Component; + +import java.util.concurrent.TimeUnit; + +/** + * 作业实例记录状态缓存 + */ +@Slf4j +@Component +public class TaskInstanceRecordStateCache extends BaseRedisCache { + + private final RedisTemplate redisTemplate; + + // 1小时过期 + private static final int EXPIRE_HOURS = 1; + + @Autowired + public TaskInstanceRecordStateCache(@Qualifier("jsonRedisTemplate") RedisTemplate redisTemplate, + MeterRegistry meterRegistry) { + super(meterRegistry, "TaskInstanceRecordStateCache"); + this.redisTemplate = redisTemplate; + } + + /** + * 更新 + * + * @param recordStateDO 更新数据 + */ + public void addOrUpdate(TaskInstanceRecordStateDO recordStateDO) { + String key = buildKey(recordStateDO.getTaskInstanceId()); + redisTemplate.opsForValue().set(key, recordStateDO, EXPIRE_HOURS, TimeUnit.HOURS); + } + + public TaskInstanceRecordStateDO get(Long taskInstanceId) { + String key = buildKey(taskInstanceId); + Object cacheObj = redisTemplate.opsForValue().get(key); + return cacheObj != null ? (TaskInstanceRecordStateDO) cacheObj : null; + } + + private String buildKey(Long taskInstanceId) { + return "job:taskInstance:state:" + taskInstanceId; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/TaskInstanceRecordStateThreadLocalRepo.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/TaskInstanceRecordStateThreadLocalRepo.java new file mode 100644 index 0000000000..641066ce0b --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/TaskInstanceRecordStateThreadLocalRepo.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.colddata; + +import com.tencent.bk.job.execute.model.TaskInstanceRecordStateDO; + +/** + * 作业托管（ThreadLocal方式) + */ +public class TaskInstanceRecordStateThreadLocalRepo { + + public static final ThreadLocal HOLDER = new ThreadLocal<>(); + + public static void set(TaskInstanceRecordStateDO taskInstanceRecordStateDO) { + HOLDER.set(taskInstanceRecordStateDO); + } + + public static void unset() { + HOLDER.remove(); + } + + public static TaskInstanceRecordStateDO get() { + return HOLDER.get(); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobFunctionConfiguration.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobFunctionConfiguration.java index 0da133f870..d3caac0495 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobFunctionConfiguration.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobFunctionConfiguration.java @@ -35,11 +35,13 @@ import com.tencent.bk.job.execute.engine.listener.event.ResultHandleTaskResumeEvent; import com.tencent.bk.job.execute.engine.listener.event.StepEvent; import com.tencent.bk.job.execute.engine.model.JobCallbackDTO; +import com.tencent.bk.job.execute.model.TaskInstanceRecordStateDO; import com.tencent.bk.job.execute.model.TaskNotifyDTO; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.messaging.Message; import java.util.function.Consumer; @@ -52,9 +54,10 @@ @Slf4j public class JobFunctionConfiguration { @Bean - public Consumer handleJobEvent(@Autowired JobListener jobListener) { + public Consumer> handleJobEvent(@Autowired JobListener jobListener) { log.info("Init handleJobEvent consumer"); - return jobListener::handleEvent; + return message -> jobListener.handleEvent(message.getPayload(), message.getHeaders().get( + "TaskInstanceRecordStateDO", TaskInstanceRecordStateDO.class)); } @Bean diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java index e6aa1763de..59b726cd63 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java @@ -37,6 +37,7 @@ import com.tencent.bk.job.execute.model.RollingConfigDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.execute.model.TaskInstanceRecordStateDO; import com.tencent.bk.job.execute.model.db.RollingConfigDetailDO; import com.tencent.bk.job.execute.service.NotifyService; import com.tencent.bk.job.execute.service.RollingConfigService; @@ -86,8 +87,9 @@ public JobListener(TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, * * @param jobEvent 作业执行相关的事件 */ - public void handleEvent(JobEvent jobEvent) { - log.info("Handle job event, event: {}, duration: {}ms", jobEvent, jobEvent.duration()); + public void handleEvent(JobEvent jobEvent, TaskInstanceRecordStateDO taskInstanceRecordStateDO) { + log.info("Handle job event, event: {}, duration: {}ms, taskInstanceRecordStateDO: {}", + jobEvent, jobEvent.duration(), taskInstanceRecordStateDO); long jobInstanceId = jobEvent.getJobInstanceId(); JobActionEnum action = JobActionEnum.valueOf(jobEvent.getAction()); try { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/TaskExecuteMQEventDispatcher.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/TaskExecuteMQEventDispatcher.java index 88d0cfcdc7..8bb92fe7e4 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/TaskExecuteMQEventDispatcher.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/TaskExecuteMQEventDispatcher.java @@ -26,10 +26,13 @@ import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.execute.engine.model.JobCallbackDTO; +import com.tencent.bk.job.execute.model.TaskInstanceRecordStateDO; import com.tencent.bk.job.execute.model.TaskNotifyDTO; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cloud.stream.function.StreamBridge; +import org.springframework.messaging.Message; +import org.springframework.messaging.support.MessageBuilder; import org.springframework.stereotype.Component; /** @@ -57,7 +60,11 @@ public TaskExecuteMQEventDispatcher(StreamBridge streamBridge) { public void dispatchJobEvent(JobEvent jobEvent) { log.info("Begin to dispatch job event, event: {}", jobEvent); String taskOutput = "task-out-0"; - streamBridge.send(taskOutput, jobEvent); + TaskInstanceRecordStateDO taskInstanceRecordStateDO = new TaskInstanceRecordStateDO(); + taskInstanceRecordStateDO.setTaskInstanceId(jobEvent.getJobInstanceId()); + Message message = MessageBuilder.withPayload(jobEvent) + .setHeader("TaskInstanceRecordStateDO", taskInstanceRecordStateDO).build(); + streamBridge.send(taskOutput, message); log.info("Dispatch job event successfully, event: {}", jobEvent); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInstanceRecordStateDO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInstanceRecordStateDO.java new file mode 100644 index 0000000000..8e519e5cfc --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInstanceRecordStateDO.java @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.constant.ArchiveFlag; +import lombok.Data; + +/** + * 作业实例数据状态 + */ +@Data +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public class TaskInstanceRecordStateDO { + /** + * 数据归档状态 + * + * @see com.tencent.bk.job.common.constant.ArchiveFlag + */ + private int archiveFlag = ArchiveFlag.HOT.getValue(); + + /** + * 所属业务 ID + */ + private Long appId; + + /** + * 作业实例 ID + */ + private Long taskInstanceId; +} From 91137121163c37d37b6289b0d1d5c7a1133e5ab2 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Wed, 22 May 2024 15:36:07 +0800 Subject: [PATCH 06/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../job/execute/config/JobFunctionConfiguration.java | 10 ++++++++-- .../listener/event/TaskExecuteMQEventDispatcher.java | 3 ++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobFunctionConfiguration.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobFunctionConfiguration.java index d3caac0495..81c1e57f4d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobFunctionConfiguration.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobFunctionConfiguration.java @@ -24,6 +24,7 @@ package com.tencent.bk.job.execute.config; +import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.execute.engine.listener.CallbackListener; import com.tencent.bk.job.execute.engine.listener.GseTaskListener; import com.tencent.bk.job.execute.engine.listener.JobListener; @@ -56,8 +57,13 @@ public class JobFunctionConfiguration { @Bean public Consumer> handleJobEvent(@Autowired JobListener jobListener) { log.info("Init handleJobEvent consumer"); - return message -> jobListener.handleEvent(message.getPayload(), message.getHeaders().get( - "TaskInstanceRecordStateDO", TaskInstanceRecordStateDO.class)); + + return message -> { + TaskInstanceRecordStateDO taskInstanceRecordStateDO = JsonUtils.fromJson( + message.getHeaders().get("TaskInstanceRecordStateDO", String.class), + TaskInstanceRecordStateDO.class); + jobListener.handleEvent(message.getPayload(), taskInstanceRecordStateDO); + }; } @Bean diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/TaskExecuteMQEventDispatcher.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/TaskExecuteMQEventDispatcher.java index 8bb92fe7e4..39c572d447 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/TaskExecuteMQEventDispatcher.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/TaskExecuteMQEventDispatcher.java @@ -63,7 +63,8 @@ public void dispatchJobEvent(JobEvent jobEvent) { TaskInstanceRecordStateDO taskInstanceRecordStateDO = new TaskInstanceRecordStateDO(); taskInstanceRecordStateDO.setTaskInstanceId(jobEvent.getJobInstanceId()); Message message = MessageBuilder.withPayload(jobEvent) - .setHeader("TaskInstanceRecordStateDO", taskInstanceRecordStateDO).build(); + .setHeader("TaskInstanceRecordStateDO", JsonUtils.toJson(taskInstanceRecordStateDO)) + .build(); streamBridge.send(taskOutput, message); log.info("Dispatch job event successfully, event: {}", jobEvent); } From 1853189c4e7c607f243e42c29c987caa52fa8eaa Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Thu, 23 May 2024 12:01:12 +0800 Subject: [PATCH 07/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ... => JobExecuteContextThreadLocalRepo.java} | 14 +-- .../TaskInstanceRecordStateCache.java | 76 ---------------- .../common/context/JobInstanceContext.java | 46 ++++++++++ .../context/PropagatedJobExecuteContext.java | 48 ++++++++++ .../JobExecuteContextInterceptor.java | 79 ++++++++++++++++ ...ecuteContextMessageChannelInterceptor.java | 90 +++++++++++++++++++ .../JobInstanceAccessInterceptor.java | 59 ++++++++++++ .../impl/TaskInstanceIdDynamicCondition.java | 10 ++- .../event/TaskExecuteMQEventDispatcher.java | 10 +-- .../service/impl/TaskInstanceServiceImpl.java | 8 ++ 10 files changed, 347 insertions(+), 93 deletions(-) rename src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/{TaskInstanceRecordStateThreadLocalRepo.java => JobExecuteContextThreadLocalRepo.java} (78%) delete mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/TaskInstanceRecordStateCache.java create mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobInstanceContext.java create mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/PropagatedJobExecuteContext.java create mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextInterceptor.java create mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java create mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobInstanceAccessInterceptor.java diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/TaskInstanceRecordStateThreadLocalRepo.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobExecuteContextThreadLocalRepo.java similarity index 78% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/TaskInstanceRecordStateThreadLocalRepo.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobExecuteContextThreadLocalRepo.java index 641066ce0b..6bd5bf6887 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/TaskInstanceRecordStateThreadLocalRepo.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobExecuteContextThreadLocalRepo.java @@ -24,24 +24,24 @@ package com.tencent.bk.job.execute.colddata; -import com.tencent.bk.job.execute.model.TaskInstanceRecordStateDO; +import com.tencent.bk.job.execute.common.context.PropagatedJobExecuteContext; /** - * 作业托管（ThreadLocal方式) + * 作业执行上下文托管（ThreadLocal方式) */ -public class TaskInstanceRecordStateThreadLocalRepo { +public class JobExecuteContextThreadLocalRepo { - public static final ThreadLocal HOLDER = new ThreadLocal<>(); + public static final ThreadLocal HOLDER = new ThreadLocal<>(); - public static void set(TaskInstanceRecordStateDO taskInstanceRecordStateDO) { - HOLDER.set(taskInstanceRecordStateDO); + public static void set(PropagatedJobExecuteContext context) { + HOLDER.set(context); } public static void unset() { HOLDER.remove(); } - public static TaskInstanceRecordStateDO get() { + public static PropagatedJobExecuteContext get() { return HOLDER.get(); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/TaskInstanceRecordStateCache.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/TaskInstanceRecordStateCache.java deleted file mode 100644 index bd1db06613..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/TaskInstanceRecordStateCache.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.colddata; - -import com.tencent.bk.job.common.redis.BaseRedisCache; -import com.tencent.bk.job.execute.model.TaskInstanceRecordStateDO; -import io.micrometer.core.instrument.MeterRegistry; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.data.redis.core.RedisTemplate; -import org.springframework.stereotype.Component; - -import java.util.concurrent.TimeUnit; - -/** - * 作业实例记录状态缓存 - */ -@Slf4j -@Component -public class TaskInstanceRecordStateCache extends BaseRedisCache { - - private final RedisTemplate redisTemplate; - - // 1小时过期 - private static final int EXPIRE_HOURS = 1; - - @Autowired - public TaskInstanceRecordStateCache(@Qualifier("jsonRedisTemplate") RedisTemplate redisTemplate, - MeterRegistry meterRegistry) { - super(meterRegistry, "TaskInstanceRecordStateCache"); - this.redisTemplate = redisTemplate; - } - - /** - * 更新 - * - * @param recordStateDO 更新数据 - */ - public void addOrUpdate(TaskInstanceRecordStateDO recordStateDO) { - String key = buildKey(recordStateDO.getTaskInstanceId()); - redisTemplate.opsForValue().set(key, recordStateDO, EXPIRE_HOURS, TimeUnit.HOURS); - } - - public TaskInstanceRecordStateDO get(Long taskInstanceId) { - String key = buildKey(taskInstanceId); - Object cacheObj = redisTemplate.opsForValue().get(key); - return cacheObj != null ? (TaskInstanceRecordStateDO) cacheObj : null; - } - - private String buildKey(Long taskInstanceId) { - return "job:taskInstance:state:" + taskInstanceId; - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobInstanceContext.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobInstanceContext.java new file mode 100644 index 0000000000..01e525a98e --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobInstanceContext.java @@ -0,0 +1,46 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.common.context; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.constant.ArchiveFlag; +import lombok.Data; + +/** + * 可传播的Job作业实例上下文，用于在异步线程、跨进程(http/mq 通信等)场景下传递、共享作业实例数据 + */ +@Data +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public class JobInstanceContext { + + private Long taskInstanceId; + + /** + * 数据归档状态 + * + * @see ArchiveFlag + */ + private int archiveFlag = ArchiveFlag.HOT.getValue(); +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/PropagatedJobExecuteContext.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/PropagatedJobExecuteContext.java new file mode 100644 index 0000000000..d6f7a44db9 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/PropagatedJobExecuteContext.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.common.context; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import lombok.Data; + +/** + * 作业执行上下文。该上下文是可传播的，用于在异步线程、跨进程(http/mq 通信等)场景下传递、共享执行上下文 + */ +@Data +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public class PropagatedJobExecuteContext { + + public static final String KEY = "JobExecuteContext"; + + /** + * 业务 + */ + private AppResourceScope resourceScope; + + private String username; + + private JobInstanceContext jobInstanceContext; +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextInterceptor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextInterceptor.java new file mode 100644 index 0000000000..b647a706c2 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextInterceptor.java @@ -0,0 +1,79 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.common.interceptor; + +import com.tencent.bk.job.common.annotation.JobInterceptor; +import com.tencent.bk.job.common.constant.InterceptorOrder; +import com.tencent.bk.job.common.context.JobContext; +import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; +import com.tencent.bk.job.execute.common.context.PropagatedJobExecuteContext; +import lombok.extern.slf4j.Slf4j; +import org.springframework.lang.NonNull; +import org.springframework.web.servlet.AsyncHandlerInterceptor; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * JobContext 初始化 + */ +@Slf4j +@JobInterceptor(order = InterceptorOrder.Init.HIGHEST + 1, pathPatterns = "/**") +public class JobExecuteContextInterceptor implements AsyncHandlerInterceptor { + + @Override + public boolean preHandle(@NonNull HttpServletRequest request, + @NonNull HttpServletResponse response, + @NonNull Object handler) { + if (!shouldFilter(request)) { + return true; + } + + JobContext jobContext = JobContextUtil.getContext(); + if (jobContext != null) { + PropagatedJobExecuteContext jobExecuteContext = new PropagatedJobExecuteContext(); + jobExecuteContext.setResourceScope(jobContext.getAppResourceScope()); + jobExecuteContext.setUsername(jobContext.getUsername()); + JobExecuteContextThreadLocalRepo.set(jobExecuteContext); + } + + return true; + } + + private boolean shouldFilter(HttpServletRequest request) { + String uri = request.getRequestURI(); + // 只拦截web/service/esb的API请求 + return uri.startsWith("/web/") || uri.startsWith("/service/") || uri.startsWith("/esb/"); + } + + @Override + public void afterCompletion(@NonNull HttpServletRequest request, + @NonNull HttpServletResponse response, + @NonNull Object handler, + Exception ex) { + JobExecuteContextThreadLocalRepo.unset(); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java new file mode 100644 index 0000000000..748e9b66c8 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java @@ -0,0 +1,90 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.common.interceptor; + +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; +import com.tencent.bk.job.execute.common.context.PropagatedJobExecuteContext; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.support.ChannelInterceptor; +import org.springframework.stereotype.Component; + +@Component +@Slf4j +public class JobExecuteContextMessageChannelInterceptor implements ChannelInterceptor { + + @Override + public Message preSend(Message message, MessageChannel channel) { + log.info("preSend"); + PropagatedJobExecuteContext context = JobExecuteContextThreadLocalRepo.get(); + if (context != null) { + log.info("setJobExecuteContextMessageHeader"); + message.getHeaders().put(PropagatedJobExecuteContext.KEY, JsonUtils.toJson(context)); + } + return ChannelInterceptor.super.preSend(message, channel); + } + + @Override + public void postSend(Message message, MessageChannel channel, boolean sent) { + log.info("postSend"); + ChannelInterceptor.super.postSend(message, channel, sent); + } + + @Override + public void afterSendCompletion(Message message, MessageChannel channel, boolean sent, Exception ex) { + log.info("afterSendCompletion"); + ChannelInterceptor.super.afterSendCompletion(message, channel, sent, ex); + } + + @Override + public boolean preReceive(MessageChannel channel) { + log.info("preReceive"); + return ChannelInterceptor.super.preReceive(channel); + } + + @Override + public Message postReceive(Message message, MessageChannel channel) { + log.info("postReceive"); + MessageHeaders headers = message.getHeaders(); + String jobExecuteContextJson = (String) headers.get(PropagatedJobExecuteContext.KEY); + if (StringUtils.isNotEmpty(jobExecuteContextJson)) { + log.info("setJobExecuteContextThreadLocalRepo"); + JobExecuteContextThreadLocalRepo.set(JsonUtils.fromJson(jobExecuteContextJson, + PropagatedJobExecuteContext.class)); + } + return ChannelInterceptor.super.postReceive(message, channel); + } + + @Override + public void afterReceiveCompletion(Message message, MessageChannel channel, Exception ex) { + log.info("afterReceiveCompletion"); + JobExecuteContextThreadLocalRepo.unset(); + ChannelInterceptor.super.afterReceiveCompletion(message, channel, ex); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobInstanceAccessInterceptor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobInstanceAccessInterceptor.java new file mode 100644 index 0000000000..3157b7e458 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobInstanceAccessInterceptor.java @@ -0,0 +1,59 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.common.interceptor; + +import com.tencent.bk.job.common.annotation.JobInterceptor; +import com.tencent.bk.job.common.constant.InterceptorOrder; +import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * Uri权限控制拦截 + */ +@Slf4j +@Component +@JobInterceptor(pathPatterns = {"/web/**", "/esb/**", "/service/**"}, + order = InterceptorOrder.Completion.LOWEST) +public class JobInstanceAccessInterceptor extends HandlerInterceptorAdapter { + + + public JobInstanceAccessInterceptor() { + + } + + @Override + public void afterCompletion(HttpServletRequest request, + HttpServletResponse response, + Object handler, + Exception ex) { + log.info("Unset JobInstanceContextThreadLocalRepo afterCompletion"); + JobExecuteContextThreadLocalRepo.unset(); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java index 194e435d66..63ed9eee16 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java @@ -30,6 +30,8 @@ import com.tencent.bk.job.common.util.feature.FeatureIdConstants; import com.tencent.bk.job.common.util.feature.FeatureToggle; import com.tencent.bk.job.common.util.feature.ToggleStrategyContextParams; +import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; +import com.tencent.bk.job.execute.common.context.PropagatedJobExecuteContext; import lombok.extern.slf4j.Slf4j; import org.jooq.Condition; import org.jooq.impl.DSL; @@ -41,7 +43,13 @@ public class TaskInstanceIdDynamicCondition { public static Condition build(Long taskInstanceId, Function taskInstanceIdConditionBuilder) { - ResourceScope resourceScope = JobContextUtil.getAppResourceScope(); + PropagatedJobExecuteContext jobExecuteContext = JobExecuteContextThreadLocalRepo.get(); + if (jobExecuteContext == null) { + log.warn("TaskInstanceIdDynamicCondition : Empty JobExecuteContext!"); + // 为了不影响兼容性，忽略错误 + return DSL.trueCondition(); + } + ResourceScope resourceScope = jobExecuteContext.getResourceScope(); if (resourceScope == null) { log.warn("TaskInstanceIdDynamicCondition : Empty resource scope!"); // 为了不影响兼容性，忽略错误 diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/TaskExecuteMQEventDispatcher.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/TaskExecuteMQEventDispatcher.java index 39c572d447..88d0cfcdc7 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/TaskExecuteMQEventDispatcher.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/TaskExecuteMQEventDispatcher.java @@ -26,13 +26,10 @@ import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.execute.engine.model.JobCallbackDTO; -import com.tencent.bk.job.execute.model.TaskInstanceRecordStateDO; import com.tencent.bk.job.execute.model.TaskNotifyDTO; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cloud.stream.function.StreamBridge; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.MessageBuilder; import org.springframework.stereotype.Component; /** @@ -60,12 +57,7 @@ public TaskExecuteMQEventDispatcher(StreamBridge streamBridge) { public void dispatchJobEvent(JobEvent jobEvent) { log.info("Begin to dispatch job event, event: {}", jobEvent); String taskOutput = "task-out-0"; - TaskInstanceRecordStateDO taskInstanceRecordStateDO = new TaskInstanceRecordStateDO(); - taskInstanceRecordStateDO.setTaskInstanceId(jobEvent.getJobInstanceId()); - Message message = MessageBuilder.withPayload(jobEvent) - .setHeader("TaskInstanceRecordStateDO", JsonUtils.toJson(taskInstanceRecordStateDO)) - .build(); - streamBridge.send(taskOutput, message); + streamBridge.send(taskOutput, jobEvent); log.info("Dispatch job event successfully, event: {}", jobEvent); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceServiceImpl.java index e97f4f30ea..35d7fc7115 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceServiceImpl.java @@ -35,7 +35,10 @@ import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.auth.ExecuteAuthService; +import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.common.context.JobInstanceContext; +import com.tencent.bk.job.execute.common.context.PropagatedJobExecuteContext; import com.tencent.bk.job.execute.dao.TaskInstanceDAO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.service.ApplicationService; @@ -112,6 +115,11 @@ public TaskInstanceDTO getTaskInstance(String username, long appId, long taskIns TaskInstanceDTO taskInstance = getTaskInstance(taskInstanceId); checkTaskInstanceExist(appId, taskInstanceId, taskInstance); auditAndAuthViewTaskInstance(username, taskInstance); + PropagatedJobExecuteContext jobExecuteContext = JobExecuteContextThreadLocalRepo.get(); + if (jobExecuteContext != null) { + JobInstanceContext jobInstanceContext = new JobInstanceContext(); + jobInstanceContext.setTaskInstanceId(taskInstanceId); + } return taskInstance; } From c5dae1cf2cfd4cab26ca72a1189c52b012002ac8 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Thu, 23 May 2024 12:12:53 +0800 Subject: [PATCH 08/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../interceptor/JobExecuteContextMessageChannelInterceptor.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java index 748e9b66c8..9ae05c650f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java @@ -29,6 +29,7 @@ import com.tencent.bk.job.execute.common.context.PropagatedJobExecuteContext; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.springframework.integration.config.GlobalChannelInterceptor; import org.springframework.messaging.Message; import org.springframework.messaging.MessageChannel; import org.springframework.messaging.MessageHeaders; @@ -37,6 +38,7 @@ @Component @Slf4j +@GlobalChannelInterceptor public class JobExecuteContextMessageChannelInterceptor implements ChannelInterceptor { @Override From 22be3d32e3423927e47c566cb07a08c31f4f247e Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Thu, 23 May 2024 12:21:39 +0800 Subject: [PATCH 09/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../execute/colddata/JobExecuteContextThreadLocalRepo.java | 4 ++++ .../common/interceptor/JobExecuteContextInterceptor.java | 2 ++ 2 files changed, 6 insertions(+) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobExecuteContextThreadLocalRepo.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobExecuteContextThreadLocalRepo.java index 6bd5bf6887..8f49b8f927 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobExecuteContextThreadLocalRepo.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobExecuteContextThreadLocalRepo.java @@ -25,15 +25,18 @@ package com.tencent.bk.job.execute.colddata; import com.tencent.bk.job.execute.common.context.PropagatedJobExecuteContext; +import lombok.extern.slf4j.Slf4j; /** * 作业执行上下文托管（ThreadLocal方式) */ +@Slf4j public class JobExecuteContextThreadLocalRepo { public static final ThreadLocal HOLDER = new ThreadLocal<>(); public static void set(PropagatedJobExecuteContext context) { + log.info("SetJobExecuteContextThreadLocalRepo"); HOLDER.set(context); } @@ -42,6 +45,7 @@ public static void unset() { } public static PropagatedJobExecuteContext get() { + log.info("GetJobExecuteContextThreadLocalRepo"); return HOLDER.get(); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextInterceptor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextInterceptor.java index b647a706c2..50fcbdf8b6 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextInterceptor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextInterceptor.java @@ -32,6 +32,7 @@ import com.tencent.bk.job.execute.common.context.PropagatedJobExecuteContext; import lombok.extern.slf4j.Slf4j; import org.springframework.lang.NonNull; +import org.springframework.stereotype.Component; import org.springframework.web.servlet.AsyncHandlerInterceptor; import javax.servlet.http.HttpServletRequest; @@ -41,6 +42,7 @@ * JobContext 初始化 */ @Slf4j +@Component @JobInterceptor(order = InterceptorOrder.Init.HIGHEST + 1, pathPatterns = "/**") public class JobExecuteContextInterceptor implements AsyncHandlerInterceptor { From 1de8d9d4e35d2a62aafa83c3947a042990d6babd Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Thu, 23 May 2024 14:30:05 +0800 Subject: [PATCH 10/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../JobExecuteContextMessageChannelInterceptor.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java index 9ae05c650f..fdfbc700a4 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java @@ -34,6 +34,7 @@ import org.springframework.messaging.MessageChannel; import org.springframework.messaging.MessageHeaders; import org.springframework.messaging.support.ChannelInterceptor; +import org.springframework.messaging.support.MessageBuilder; import org.springframework.stereotype.Component; @Component @@ -45,11 +46,15 @@ public class JobExecuteContextMessageChannelInterceptor implements ChannelInterc public Message preSend(Message message, MessageChannel channel) { log.info("preSend"); PropagatedJobExecuteContext context = JobExecuteContextThreadLocalRepo.get(); - if (context != null) { - log.info("setJobExecuteContextMessageHeader"); - message.getHeaders().put(PropagatedJobExecuteContext.KEY, JsonUtils.toJson(context)); + if (context == null) { + return ChannelInterceptor.super.preSend(message, channel); } - return ChannelInterceptor.super.preSend(message, channel); + log.info("setJobExecuteContextMessageHeader"); + Message newMessage = + MessageBuilder.fromMessage(message) + .setHeader(PropagatedJobExecuteContext.KEY, JsonUtils.toJson(context)) + .build(); + return ChannelInterceptor.super.preSend(newMessage, channel); } @Override From 227a9f595ed23605fb432febc105290ed7be2143 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Thu, 23 May 2024 15:45:33 +0800 Subject: [PATCH 11/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...ecuteContextMessageChannelInterceptor.java | 39 ++++++++++++++----- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java index fdfbc700a4..d9d48e6b54 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java @@ -32,47 +32,48 @@ import org.springframework.integration.config.GlobalChannelInterceptor; import org.springframework.messaging.Message; import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.MessageHandler; import org.springframework.messaging.MessageHeaders; -import org.springframework.messaging.support.ChannelInterceptor; +import org.springframework.messaging.support.ExecutorChannelInterceptor; import org.springframework.messaging.support.MessageBuilder; import org.springframework.stereotype.Component; @Component @Slf4j @GlobalChannelInterceptor -public class JobExecuteContextMessageChannelInterceptor implements ChannelInterceptor { +public class JobExecuteContextMessageChannelInterceptor implements ExecutorChannelInterceptor { @Override public Message preSend(Message message, MessageChannel channel) { log.info("preSend"); PropagatedJobExecuteContext context = JobExecuteContextThreadLocalRepo.get(); if (context == null) { - return ChannelInterceptor.super.preSend(message, channel); + return ExecutorChannelInterceptor.super.preSend(message, channel); } log.info("setJobExecuteContextMessageHeader"); Message newMessage = MessageBuilder.fromMessage(message) .setHeader(PropagatedJobExecuteContext.KEY, JsonUtils.toJson(context)) .build(); - return ChannelInterceptor.super.preSend(newMessage, channel); + return ExecutorChannelInterceptor.super.preSend(newMessage, channel); } @Override public void postSend(Message message, MessageChannel channel, boolean sent) { log.info("postSend"); - ChannelInterceptor.super.postSend(message, channel, sent); + ExecutorChannelInterceptor.super.postSend(message, channel, sent); } @Override public void afterSendCompletion(Message message, MessageChannel channel, boolean sent, Exception ex) { log.info("afterSendCompletion"); - ChannelInterceptor.super.afterSendCompletion(message, channel, sent, ex); + ExecutorChannelInterceptor.super.afterSendCompletion(message, channel, sent, ex); } @Override public boolean preReceive(MessageChannel channel) { log.info("preReceive"); - return ChannelInterceptor.super.preReceive(channel); + return ExecutorChannelInterceptor.super.preReceive(channel); } @Override @@ -85,13 +86,33 @@ public Message postReceive(Message message, MessageChannel channel) { JobExecuteContextThreadLocalRepo.set(JsonUtils.fromJson(jobExecuteContextJson, PropagatedJobExecuteContext.class)); } - return ChannelInterceptor.super.postReceive(message, channel); + return ExecutorChannelInterceptor.super.postReceive(message, channel); } @Override public void afterReceiveCompletion(Message message, MessageChannel channel, Exception ex) { log.info("afterReceiveCompletion"); JobExecuteContextThreadLocalRepo.unset(); - ChannelInterceptor.super.afterReceiveCompletion(message, channel, ex); + ExecutorChannelInterceptor.super.afterReceiveCompletion(message, channel, ex); + } + + @Override + public Message beforeHandle(Message message, MessageChannel channel, MessageHandler handler) { + log.info("beforeHandle"); + MessageHeaders headers = message.getHeaders(); + String jobExecuteContextJson = (String) headers.get(PropagatedJobExecuteContext.KEY); + if (StringUtils.isNotEmpty(jobExecuteContextJson)) { + log.info("setJobExecuteContextThreadLocalRepo"); + JobExecuteContextThreadLocalRepo.set(JsonUtils.fromJson(jobExecuteContextJson, + PropagatedJobExecuteContext.class)); + } + return ExecutorChannelInterceptor.super.beforeHandle(message, channel, handler); + } + + @Override + public void afterMessageHandled(Message message, MessageChannel channel, MessageHandler handler, Exception ex) { + log.info("afterMessageHandled"); + JobExecuteContextThreadLocalRepo.unset(); + ExecutorChannelInterceptor.super.afterMessageHandled(message, channel, handler, ex); } } From 266f9910c3ed9d6acbc74eb9e2acd33effab9800 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Thu, 23 May 2024 17:22:04 +0800 Subject: [PATCH 12/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../config/JobFunctionConfiguration.java | 21 ++---- .../listener/BaseJobExecuteMqListener.java | 66 +++++++++++++++++++ .../engine/listener/GseTaskListener.java | 10 ++- .../execute/engine/listener/JobListener.java | 16 +++-- .../listener/ResultHandleResumeListener.java | 7 +- .../execute/engine/listener/StepListener.java | 10 ++- 6 files changed, 101 insertions(+), 29 deletions(-) create mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/BaseJobExecuteMqListener.java diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobFunctionConfiguration.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobFunctionConfiguration.java index 81c1e57f4d..eb8857bcb1 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobFunctionConfiguration.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobFunctionConfiguration.java @@ -24,7 +24,6 @@ package com.tencent.bk.job.execute.config; -import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.execute.engine.listener.CallbackListener; import com.tencent.bk.job.execute.engine.listener.GseTaskListener; import com.tencent.bk.job.execute.engine.listener.JobListener; @@ -36,7 +35,6 @@ import com.tencent.bk.job.execute.engine.listener.event.ResultHandleTaskResumeEvent; import com.tencent.bk.job.execute.engine.listener.event.StepEvent; import com.tencent.bk.job.execute.engine.model.JobCallbackDTO; -import com.tencent.bk.job.execute.model.TaskInstanceRecordStateDO; import com.tencent.bk.job.execute.model.TaskNotifyDTO; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -58,31 +56,26 @@ public class JobFunctionConfiguration { public Consumer> handleJobEvent(@Autowired JobListener jobListener) { log.info("Init handleJobEvent consumer"); - return message -> { - TaskInstanceRecordStateDO taskInstanceRecordStateDO = JsonUtils.fromJson( - message.getHeaders().get("TaskInstanceRecordStateDO", String.class), - TaskInstanceRecordStateDO.class); - jobListener.handleEvent(message.getPayload(), taskInstanceRecordStateDO); - }; + return jobListener::onEvent; } @Bean - public Consumer handleStepEvent(@Autowired StepListener stepListener) { + public Consumer> handleStepEvent(@Autowired StepListener stepListener) { log.info("Init handleStepEvent consumer"); - return stepListener::handleEvent; + return stepListener::onEvent; } @Bean - public Consumer handleGseTaskEvent(@Autowired GseTaskListener gseTaskListener) { + public Consumer> handleGseTaskEvent(@Autowired GseTaskListener gseTaskListener) { log.info("Init handleGseTaskEvent consumer"); - return gseTaskListener::handleEvent; + return gseTaskListener::onEvent; } @Bean - public Consumer handleResultHandleResumeEvent( + public Consumer> handleResultHandleResumeEvent( @Autowired ResultHandleResumeListener resultHandleResumeListener) { log.info("Init handleResultHandleResumeEvent consumer"); - return resultHandleResumeListener::handleEvent; + return resultHandleResumeListener::onEvent; } @Bean diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/BaseJobExecuteMqListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/BaseJobExecuteMqListener.java new file mode 100644 index 0000000000..10567c8a02 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/BaseJobExecuteMqListener.java @@ -0,0 +1,66 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.listener; + +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; +import com.tencent.bk.job.execute.common.context.PropagatedJobExecuteContext; +import com.tencent.bk.job.execute.engine.listener.event.Event; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; + +@Slf4j +public abstract class BaseJobExecuteMqListener { + + + public final void onEvent(Message message) { + beforeHandleMessage(message); + try { + handleEvent(message); + } finally { + afterHandle(message); + } + } + + private void beforeHandleMessage(Message message) { + log.info("beforeHandleMessage"); + MessageHeaders headers = message.getHeaders(); + String jobExecuteContextJson = (String) headers.get(PropagatedJobExecuteContext.KEY); + if (StringUtils.isNotEmpty(jobExecuteContextJson)) { + log.info("setJobExecuteContextThreadLocalRepo"); + JobExecuteContextThreadLocalRepo.set(JsonUtils.fromJson(jobExecuteContextJson, + PropagatedJobExecuteContext.class)); + } + } + + private void afterHandle(Message message) { + log.info("afterHandleMessage"); + JobExecuteContextThreadLocalRepo.unset(); + } + + protected abstract void handleEvent(Message message); +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseTaskListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseTaskListener.java index ded0d99b5f..7bb81056fc 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseTaskListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseTaskListener.java @@ -28,12 +28,14 @@ import com.tencent.bk.job.execute.common.exception.MessageHandlerUnavailableException; import com.tencent.bk.job.execute.engine.consts.GseTaskActionEnum; import com.tencent.bk.job.execute.engine.executor.GseTaskManager; +import com.tencent.bk.job.execute.engine.listener.event.Event; import com.tencent.bk.job.execute.engine.listener.event.GseTaskEvent; import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.monitor.metrics.GseTasksExceptionCounter; import com.tencent.bk.job.execute.service.GseTaskService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.messaging.Message; import org.springframework.stereotype.Component; /** @@ -41,7 +43,7 @@ */ @Component @Slf4j -public class GseTaskListener { +public class GseTaskListener extends BaseJobExecuteMqListener { private final GseTaskManager gseTaskManager; private final GseTaskService gseTaskService; private final GseTasksExceptionCounter gseTasksExceptionCounter; @@ -58,9 +60,11 @@ public GseTaskListener(GseTaskManager gseTaskManager, /** * 处理GSE任务相关的事件 * - * @param gseTaskEvent GSE任务事件 + * @param message 消息 */ - public void handleEvent(GseTaskEvent gseTaskEvent) { + @Override + public void handleEvent(Message message) { + GseTaskEvent gseTaskEvent = (GseTaskEvent) message.getPayload(); log.info("Handle gse task event: {}, duration: {}ms", gseTaskEvent, gseTaskEvent.duration()); GseTaskDTO gseTask = gseTaskService.getGseTask(gseTaskEvent.getJobInstanceId(), gseTaskEvent.getGseTaskId()); String requestId = gseTaskEvent.getRequestId(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java index 59b726cd63..3ec929ccba 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java @@ -30,6 +30,7 @@ import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.common.util.TaskCostCalculator; import com.tencent.bk.job.execute.engine.consts.JobActionEnum; +import com.tencent.bk.job.execute.engine.listener.event.Event; import com.tencent.bk.job.execute.engine.listener.event.JobEvent; import com.tencent.bk.job.execute.engine.listener.event.StepEvent; import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; @@ -37,7 +38,6 @@ import com.tencent.bk.job.execute.model.RollingConfigDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; -import com.tencent.bk.job.execute.model.TaskInstanceRecordStateDO; import com.tencent.bk.job.execute.model.db.RollingConfigDetailDO; import com.tencent.bk.job.execute.service.NotifyService; import com.tencent.bk.job.execute.service.RollingConfigService; @@ -47,6 +47,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.messaging.Message; import org.springframework.stereotype.Component; import java.util.Collection; @@ -57,7 +58,7 @@ */ @Component("jobEventListener") @Slf4j -public class JobListener { +public class JobListener extends BaseJobExecuteMqListener { private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; private final StatisticsService statisticsService; @@ -85,11 +86,12 @@ public JobListener(TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, /** * 处理作业执行相关的事件 * - * @param jobEvent 作业执行相关的事件 + * @param message 消息 */ - public void handleEvent(JobEvent jobEvent, TaskInstanceRecordStateDO taskInstanceRecordStateDO) { - log.info("Handle job event, event: {}, duration: {}ms, taskInstanceRecordStateDO: {}", - jobEvent, jobEvent.duration(), taskInstanceRecordStateDO); + @Override + public void handleEvent(Message message) { + JobEvent jobEvent = (JobEvent) message.getPayload(); + log.info("Handle job event, event: {}, duration: {}ms", jobEvent, jobEvent.duration()); long jobInstanceId = jobEvent.getJobInstanceId(); JobActionEnum action = JobActionEnum.valueOf(jobEvent.getAction()); try { @@ -319,7 +321,7 @@ private void startStep(StepInstanceBaseDTO stepInstance) { stepInstance.getId(), RunStatusEnum.ROLLING_WAITING.getValue()); taskExecuteMQEventDispatcher.dispatchStepEvent( StepEvent.startStep(stepInstance.getTaskInstanceId(), stepInstance.getId(), - stepInstance.getBatch() + 1)); + stepInstance.getBatch() + 1)); } else { stepInstanceService.updateStepStatus(stepInstance.getTaskInstanceId(), stepInstance.getId(), RunStatusEnum.BLANK.getValue()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ResultHandleResumeListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ResultHandleResumeListener.java index 23a2f5c1a0..b1721ce0c6 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ResultHandleResumeListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ResultHandleResumeListener.java @@ -30,6 +30,7 @@ import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.config.FileDistributeConfig; import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; +import com.tencent.bk.job.execute.engine.listener.event.Event; import com.tencent.bk.job.execute.engine.listener.event.ResultHandleTaskResumeEvent; import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; import com.tencent.bk.job.execute.engine.model.FileDest; @@ -56,6 +57,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.messaging.Message; import org.springframework.stereotype.Component; import java.util.HashMap; @@ -69,7 +71,7 @@ */ @Component @Slf4j -public class ResultHandleResumeListener { +public class ResultHandleResumeListener extends BaseJobExecuteMqListener { private final TaskInstanceService taskInstanceService; private final ResultHandleManager resultHandleManager; @@ -132,7 +134,8 @@ public ResultHandleResumeListener(TaskInstanceService taskInstanceService, /** * 恢复被中断的作业结果处理任务 */ - public void handleEvent(ResultHandleTaskResumeEvent event) { + public void handleEvent(Message message) { + ResultHandleTaskResumeEvent event = (ResultHandleTaskResumeEvent) message.getPayload(); log.info("Receive gse task result handle task resume event: {}, duration: {}ms", event, event.duration()); GseTaskDTO gseTask = gseTaskService.getGseTask(event.getJobInstanceId(), event.getGseTaskId()); long stepInstanceId = gseTask.getStepInstanceId(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepListener.java index bb9203efdd..1d488f29a6 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepListener.java @@ -25,11 +25,13 @@ package com.tencent.bk.job.execute.engine.listener; import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; +import com.tencent.bk.job.execute.engine.listener.event.Event; import com.tencent.bk.job.execute.engine.listener.event.StepEvent; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.service.StepInstanceService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.messaging.Message; import org.springframework.stereotype.Component; /** @@ -37,7 +39,7 @@ */ @Component @Slf4j -public class StepListener { +public class StepListener extends BaseJobExecuteMqListener { private final StepInstanceService stepInstanceService; private final GseStepEventHandler gseStepEventHandler; private final ConfirmStepEventHandler confirmStepEventHandler; @@ -54,9 +56,11 @@ public StepListener(StepInstanceService stepInstanceService, /** * 处理步骤执行相关的事件 * - * @param stepEvent 步骤执行相关的事件 + * @param message 消息 */ - public void handleEvent(StepEvent stepEvent) { + @Override + public void handleEvent(Message message) { + StepEvent stepEvent = (StepEvent) message.getPayload(); log.info("Handle step event: {}, duration: {}ms", stepEvent, stepEvent.duration()); long stepInstanceId = stepEvent.getStepInstanceId(); try { From 1d6c056c15e05e559634956befeb05e6af4450dc Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Thu, 23 May 2024 17:37:00 +0800 Subject: [PATCH 13/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../JobExecuteContextThreadLocalRepo.java | 4 +- .../JobExecuteContextInterceptor.java | 1 + ...ecuteContextMessageChannelInterceptor.java | 51 +------------------ 3 files changed, 4 insertions(+), 52 deletions(-) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobExecuteContextThreadLocalRepo.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobExecuteContextThreadLocalRepo.java index 8f49b8f927..097c336960 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobExecuteContextThreadLocalRepo.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobExecuteContextThreadLocalRepo.java @@ -36,16 +36,16 @@ public class JobExecuteContextThreadLocalRepo { public static final ThreadLocal HOLDER = new ThreadLocal<>(); public static void set(PropagatedJobExecuteContext context) { - log.info("SetJobExecuteContextThreadLocalRepo"); + log.info("SetJobExecuteContextThreadLocalRepo, context: {}", context); HOLDER.set(context); } public static void unset() { + log.info("RemoveJobExecuteContext"); HOLDER.remove(); } public static PropagatedJobExecuteContext get() { - log.info("GetJobExecuteContextThreadLocalRepo"); return HOLDER.get(); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextInterceptor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextInterceptor.java index 50fcbdf8b6..06bca1c231 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextInterceptor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextInterceptor.java @@ -56,6 +56,7 @@ public boolean preHandle(@NonNull HttpServletRequest request, JobContext jobContext = JobContextUtil.getContext(); if (jobContext != null) { + log.info("JobExecuteContextInterceptor -> Set JobExecuteContext"); PropagatedJobExecuteContext jobExecuteContext = new PropagatedJobExecuteContext(); jobExecuteContext.setResourceScope(jobContext.getAppResourceScope()); jobExecuteContext.setUsername(jobContext.getUsername()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java index d9d48e6b54..797531afc9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java @@ -28,12 +28,9 @@ import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; import com.tencent.bk.job.execute.common.context.PropagatedJobExecuteContext; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.springframework.integration.config.GlobalChannelInterceptor; import org.springframework.messaging.Message; import org.springframework.messaging.MessageChannel; -import org.springframework.messaging.MessageHandler; -import org.springframework.messaging.MessageHeaders; import org.springframework.messaging.support.ExecutorChannelInterceptor; import org.springframework.messaging.support.MessageBuilder; import org.springframework.stereotype.Component; @@ -50,7 +47,7 @@ public Message preSend(Message message, MessageChannel channel) { if (context == null) { return ExecutorChannelInterceptor.super.preSend(message, channel); } - log.info("setJobExecuteContextMessageHeader"); + log.info("setJobExecuteContextMessageHeader, context: {}", context); Message newMessage = MessageBuilder.fromMessage(message) .setHeader(PropagatedJobExecuteContext.KEY, JsonUtils.toJson(context)) @@ -69,50 +66,4 @@ public void afterSendCompletion(Message message, MessageChannel channel, bool log.info("afterSendCompletion"); ExecutorChannelInterceptor.super.afterSendCompletion(message, channel, sent, ex); } - - @Override - public boolean preReceive(MessageChannel channel) { - log.info("preReceive"); - return ExecutorChannelInterceptor.super.preReceive(channel); - } - - @Override - public Message postReceive(Message message, MessageChannel channel) { - log.info("postReceive"); - MessageHeaders headers = message.getHeaders(); - String jobExecuteContextJson = (String) headers.get(PropagatedJobExecuteContext.KEY); - if (StringUtils.isNotEmpty(jobExecuteContextJson)) { - log.info("setJobExecuteContextThreadLocalRepo"); - JobExecuteContextThreadLocalRepo.set(JsonUtils.fromJson(jobExecuteContextJson, - PropagatedJobExecuteContext.class)); - } - return ExecutorChannelInterceptor.super.postReceive(message, channel); - } - - @Override - public void afterReceiveCompletion(Message message, MessageChannel channel, Exception ex) { - log.info("afterReceiveCompletion"); - JobExecuteContextThreadLocalRepo.unset(); - ExecutorChannelInterceptor.super.afterReceiveCompletion(message, channel, ex); - } - - @Override - public Message beforeHandle(Message message, MessageChannel channel, MessageHandler handler) { - log.info("beforeHandle"); - MessageHeaders headers = message.getHeaders(); - String jobExecuteContextJson = (String) headers.get(PropagatedJobExecuteContext.KEY); - if (StringUtils.isNotEmpty(jobExecuteContextJson)) { - log.info("setJobExecuteContextThreadLocalRepo"); - JobExecuteContextThreadLocalRepo.set(JsonUtils.fromJson(jobExecuteContextJson, - PropagatedJobExecuteContext.class)); - } - return ExecutorChannelInterceptor.super.beforeHandle(message, channel, handler); - } - - @Override - public void afterMessageHandled(Message message, MessageChannel channel, MessageHandler handler, Exception ex) { - log.info("afterMessageHandled"); - JobExecuteContextThreadLocalRepo.unset(); - ExecutorChannelInterceptor.super.afterMessageHandled(message, channel, handler, ex); - } } From 3ef71bd1937e7156d9487dcd082be4061afd9ef0 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Thu, 23 May 2024 17:48:18 +0800 Subject: [PATCH 14/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../JobExecuteContextInterceptor.java | 2 +- .../JobInstanceAccessInterceptor.java | 59 ------------------- 2 files changed, 1 insertion(+), 60 deletions(-) delete mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobInstanceAccessInterceptor.java diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextInterceptor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextInterceptor.java index 06bca1c231..25b77b70ae 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextInterceptor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextInterceptor.java @@ -43,7 +43,7 @@ */ @Slf4j @Component -@JobInterceptor(order = InterceptorOrder.Init.HIGHEST + 1, pathPatterns = "/**") +@JobInterceptor(order = InterceptorOrder.Init.LOWEST, pathPatterns = "/**") public class JobExecuteContextInterceptor implements AsyncHandlerInterceptor { @Override diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobInstanceAccessInterceptor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobInstanceAccessInterceptor.java deleted file mode 100644 index 3157b7e458..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobInstanceAccessInterceptor.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.common.interceptor; - -import com.tencent.bk.job.common.annotation.JobInterceptor; -import com.tencent.bk.job.common.constant.InterceptorOrder; -import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -/** - * Uri权限控制拦截 - */ -@Slf4j -@Component -@JobInterceptor(pathPatterns = {"/web/**", "/esb/**", "/service/**"}, - order = InterceptorOrder.Completion.LOWEST) -public class JobInstanceAccessInterceptor extends HandlerInterceptorAdapter { - - - public JobInstanceAccessInterceptor() { - - } - - @Override - public void afterCompletion(HttpServletRequest request, - HttpServletResponse response, - Object handler, - Exception ex) { - log.info("Unset JobInstanceContextThreadLocalRepo afterCompletion"); - JobExecuteContextThreadLocalRepo.unset(); - } -} From a4d4711c964ba0475130acf42057418cb613e35e Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Fri, 24 May 2024 10:10:06 +0800 Subject: [PATCH 15/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/AppScopeMappingService.java | 2 ++ .../JobExecuteContextThreadLocalRepo.java | 8 ++++---- ...uteContext.java => JobExecuteContext.java} | 4 ++-- .../JobExecuteContextInterceptor.java | 4 ++-- ...ecuteContextMessageChannelInterceptor.java | 6 +++--- .../impl/TaskInstanceIdDynamicCondition.java | 4 ++-- .../listener/BaseJobExecuteMqListener.java | 6 +++--- .../prepare/FilePrepareServiceImpl.java | 3 ++- .../result/AbstractResultHandleTask.java | 1 - .../engine/result/ResultHandleManager.java | 3 ++- .../ScheduledContinuousResultHandleTask.java | 19 ++++++++++++++++--- .../service/impl/TaskInstanceServiceImpl.java | 4 ++-- ...tractLocalCacheAppScopeMappingService.java | 6 ++++++ 13 files changed, 46 insertions(+), 24 deletions(-) rename src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/{PropagatedJobExecuteContext.java => JobExecuteContext.java} (90%) diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/service/AppScopeMappingService.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/service/AppScopeMappingService.java index aeb7eaa1c5..b9d0dff93e 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/service/AppScopeMappingService.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/service/AppScopeMappingService.java @@ -63,6 +63,8 @@ public interface AppScopeMappingService { */ AppResourceScope getAppResourceScope(Long appId, String scopeType, String scopeId); + AppResourceScope getAppResourceScope(Long appId); + /** * 根据scopeType&scopeId获取AppResourceScope * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobExecuteContextThreadLocalRepo.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobExecuteContextThreadLocalRepo.java index 097c336960..7259575d5a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobExecuteContextThreadLocalRepo.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobExecuteContextThreadLocalRepo.java @@ -24,7 +24,7 @@ package com.tencent.bk.job.execute.colddata; -import com.tencent.bk.job.execute.common.context.PropagatedJobExecuteContext; +import com.tencent.bk.job.execute.common.context.JobExecuteContext; import lombok.extern.slf4j.Slf4j; /** @@ -33,9 +33,9 @@ @Slf4j public class JobExecuteContextThreadLocalRepo { - public static final ThreadLocal HOLDER = new ThreadLocal<>(); + public static final ThreadLocal HOLDER = new ThreadLocal<>(); - public static void set(PropagatedJobExecuteContext context) { + public static void set(JobExecuteContext context) { log.info("SetJobExecuteContextThreadLocalRepo, context: {}", context); HOLDER.set(context); } @@ -45,7 +45,7 @@ public static void unset() { HOLDER.remove(); } - public static PropagatedJobExecuteContext get() { + public static JobExecuteContext get() { return HOLDER.get(); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/PropagatedJobExecuteContext.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobExecuteContext.java similarity index 90% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/PropagatedJobExecuteContext.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobExecuteContext.java index d6f7a44db9..87050f734b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/PropagatedJobExecuteContext.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobExecuteContext.java @@ -29,11 +29,11 @@ import lombok.Data; /** - * 作业执行上下文。该上下文是可传播的，用于在异步线程、跨进程(http/mq 通信等)场景下传递、共享执行上下文 + * 作业执行上下文 */ @Data @JsonInclude(JsonInclude.Include.NON_EMPTY) -public class PropagatedJobExecuteContext { +public class JobExecuteContext { public static final String KEY = "JobExecuteContext"; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextInterceptor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextInterceptor.java index 25b77b70ae..68a4015e04 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextInterceptor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextInterceptor.java @@ -29,7 +29,7 @@ import com.tencent.bk.job.common.context.JobContext; import com.tencent.bk.job.common.util.JobContextUtil; import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; -import com.tencent.bk.job.execute.common.context.PropagatedJobExecuteContext; +import com.tencent.bk.job.execute.common.context.JobExecuteContext; import lombok.extern.slf4j.Slf4j; import org.springframework.lang.NonNull; import org.springframework.stereotype.Component; @@ -57,7 +57,7 @@ public boolean preHandle(@NonNull HttpServletRequest request, JobContext jobContext = JobContextUtil.getContext(); if (jobContext != null) { log.info("JobExecuteContextInterceptor -> Set JobExecuteContext"); - PropagatedJobExecuteContext jobExecuteContext = new PropagatedJobExecuteContext(); + JobExecuteContext jobExecuteContext = new JobExecuteContext(); jobExecuteContext.setResourceScope(jobContext.getAppResourceScope()); jobExecuteContext.setUsername(jobContext.getUsername()); JobExecuteContextThreadLocalRepo.set(jobExecuteContext); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java index 797531afc9..817a02107f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java @@ -26,7 +26,7 @@ import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; -import com.tencent.bk.job.execute.common.context.PropagatedJobExecuteContext; +import com.tencent.bk.job.execute.common.context.JobExecuteContext; import lombok.extern.slf4j.Slf4j; import org.springframework.integration.config.GlobalChannelInterceptor; import org.springframework.messaging.Message; @@ -43,14 +43,14 @@ public class JobExecuteContextMessageChannelInterceptor implements ExecutorChann @Override public Message preSend(Message message, MessageChannel channel) { log.info("preSend"); - PropagatedJobExecuteContext context = JobExecuteContextThreadLocalRepo.get(); + JobExecuteContext context = JobExecuteContextThreadLocalRepo.get(); if (context == null) { return ExecutorChannelInterceptor.super.preSend(message, channel); } log.info("setJobExecuteContextMessageHeader, context: {}", context); Message newMessage = MessageBuilder.fromMessage(message) - .setHeader(PropagatedJobExecuteContext.KEY, JsonUtils.toJson(context)) + .setHeader(JobExecuteContext.KEY, JsonUtils.toJson(context)) .build(); return ExecutorChannelInterceptor.super.preSend(newMessage, channel); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java index 63ed9eee16..610faad960 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java @@ -31,7 +31,7 @@ import com.tencent.bk.job.common.util.feature.FeatureToggle; import com.tencent.bk.job.common.util.feature.ToggleStrategyContextParams; import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; -import com.tencent.bk.job.execute.common.context.PropagatedJobExecuteContext; +import com.tencent.bk.job.execute.common.context.JobExecuteContext; import lombok.extern.slf4j.Slf4j; import org.jooq.Condition; import org.jooq.impl.DSL; @@ -43,7 +43,7 @@ public class TaskInstanceIdDynamicCondition { public static Condition build(Long taskInstanceId, Function taskInstanceIdConditionBuilder) { - PropagatedJobExecuteContext jobExecuteContext = JobExecuteContextThreadLocalRepo.get(); + JobExecuteContext jobExecuteContext = JobExecuteContextThreadLocalRepo.get(); if (jobExecuteContext == null) { log.warn("TaskInstanceIdDynamicCondition : Empty JobExecuteContext!"); // 为了不影响兼容性，忽略错误 diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/BaseJobExecuteMqListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/BaseJobExecuteMqListener.java index 10567c8a02..ef0a69a3bc 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/BaseJobExecuteMqListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/BaseJobExecuteMqListener.java @@ -26,7 +26,7 @@ import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; -import com.tencent.bk.job.execute.common.context.PropagatedJobExecuteContext; +import com.tencent.bk.job.execute.common.context.JobExecuteContext; import com.tencent.bk.job.execute.engine.listener.event.Event; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -49,11 +49,11 @@ public final void onEvent(Message message) { private void beforeHandleMessage(Message message) { log.info("beforeHandleMessage"); MessageHeaders headers = message.getHeaders(); - String jobExecuteContextJson = (String) headers.get(PropagatedJobExecuteContext.KEY); + String jobExecuteContextJson = (String) headers.get(JobExecuteContext.KEY); if (StringUtils.isNotEmpty(jobExecuteContextJson)) { log.info("setJobExecuteContextThreadLocalRepo"); JobExecuteContextThreadLocalRepo.set(JsonUtils.fromJson(jobExecuteContextJson, - PropagatedJobExecuteContext.class)); + JobExecuteContext.class)); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareServiceImpl.java index a5c3fd7e72..b95f21f21e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareServiceImpl.java @@ -153,7 +153,8 @@ public void prepareFileForGseTask(StepInstanceDTO stepInstance) { new FilePrepareControlTask( this, taskInstanceService, - taskExecuteMQEventDispatcher, stepInstance, + taskExecuteMQEventDispatcher, + stepInstance, latch, resultList, filePrepareTaskResultHandler, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/AbstractResultHandleTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/AbstractResultHandleTask.java index a7841ab103..7314d85a5a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/AbstractResultHandleTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/AbstractResultHandleTask.java @@ -210,7 +210,6 @@ public abstract class AbstractResultHandleTask implements ContinuousScheduled */ protected String gseTaskInfo; - protected AbstractResultHandleTask(TaskInstanceService taskInstanceService, GseTaskService gseTaskService, LogService logService, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ResultHandleManager.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ResultHandleManager.java index 475c8c9c6d..7f993a5b9d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ResultHandleManager.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ResultHandleManager.java @@ -24,6 +24,7 @@ package com.tencent.bk.job.execute.engine.result; +import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; import com.tencent.bk.job.execute.common.exception.MessageHandlerUnavailableException; import com.tencent.bk.job.execute.common.ha.DestroyOrder; import com.tencent.bk.job.execute.config.JobExecuteConfig; @@ -157,7 +158,7 @@ public void handleDeliveredTask(ContinuousScheduledTask task) { log.info("Handle delivered task: {}", task); ScheduledContinuousResultHandleTask scheduleTask = new ScheduledContinuousResultHandleTask(resultHandleTaskSampler, tracer, task, this, - resultHandleTaskKeepaliveManager, resultHandleLimiter); + resultHandleTaskKeepaliveManager, resultHandleLimiter, JobExecuteContextThreadLocalRepo.get()); synchronized (lifecycleMonitor) { if (!isActive()) { log.warn("ResultHandleManager is not active, reject! task: {}", task); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScheduledContinuousResultHandleTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScheduledContinuousResultHandleTask.java index 94f62b8be2..68ae732112 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScheduledContinuousResultHandleTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScheduledContinuousResultHandleTask.java @@ -24,6 +24,8 @@ package com.tencent.bk.job.execute.engine.result; +import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; +import com.tencent.bk.job.execute.common.context.JobExecuteContext; import com.tencent.bk.job.execute.engine.result.ha.ResultHandleLimiter; import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; import com.tencent.bk.job.execute.monitor.ExecuteMetricNames; @@ -70,6 +72,11 @@ public class ScheduledContinuousResultHandleTask extends DelayedTask { */ private final Span parent; + /** + * 作业执行上下文信息 + */ + private final JobExecuteContext jobExecuteContext; + /** * ScheduledContinuousQueuedTask Constructor * @@ -78,12 +85,15 @@ public class ScheduledContinuousResultHandleTask extends DelayedTask { * @param task 任务 * @param resultHandleManager resultHandleManager * @param resultHandleLimiter 限流 + * @param jobExecuteContext 作业执行上下文 */ public ScheduledContinuousResultHandleTask(ResultHandleTaskSampler sampler, - Tracer tracer, ContinuousScheduledTask task, + Tracer tracer, + ContinuousScheduledTask task, ResultHandleManager resultHandleManager, ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, - ResultHandleLimiter resultHandleLimiter) { + ResultHandleLimiter resultHandleLimiter, + JobExecuteContext jobExecuteContext) { this.sampler = sampler; this.tracer = tracer; this.parent = tracer.currentSpan(); @@ -93,6 +103,7 @@ public ScheduledContinuousResultHandleTask(ResultHandleTaskSampler sampler, this.tasksQueue = resultHandleManager.getTasksQueue(); this.resultHandleTaskKeepaliveManager = resultHandleTaskKeepaliveManager; this.resultHandleLimiter = resultHandleLimiter; + this.jobExecuteContext = jobExecuteContext; } private Span getChildSpan() { @@ -103,11 +114,13 @@ private Span getChildSpan() { public void execute() { Span span = getChildSpan(); try (Tracer.SpanInScope ignored = this.tracer.withSpan(span.start())) { + JobExecuteContextThreadLocalRepo.set(this.jobExecuteContext); doExecute(); } catch (Exception e) { span.error(e); throw e; } finally { + JobExecuteContextThreadLocalRepo.unset(); span.end(); } } @@ -162,7 +175,7 @@ public void doExecute() { if (isExecutable) { long end = System.nanoTime(); sampler.getMeterRegistry().timer(ExecuteMetricNames.RESULT_HANDLE_TASK_SCHEDULE_PREFIX, - "task_type", task.getTaskType(), "status", status) + "task_type", task.getTaskType(), "status", status) .record(end - start, TimeUnit.NANOSECONDS); } if (!isReScheduled) { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceServiceImpl.java index 35d7fc7115..1c77d43938 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceServiceImpl.java @@ -37,8 +37,8 @@ import com.tencent.bk.job.execute.auth.ExecuteAuthService; import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.common.context.JobExecuteContext; import com.tencent.bk.job.execute.common.context.JobInstanceContext; -import com.tencent.bk.job.execute.common.context.PropagatedJobExecuteContext; import com.tencent.bk.job.execute.dao.TaskInstanceDAO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.service.ApplicationService; @@ -115,7 +115,7 @@ public TaskInstanceDTO getTaskInstance(String username, long appId, long taskIns TaskInstanceDTO taskInstance = getTaskInstance(taskInstanceId); checkTaskInstanceExist(appId, taskInstanceId, taskInstance); auditAndAuthViewTaskInstance(username, taskInstance); - PropagatedJobExecuteContext jobExecuteContext = JobExecuteContextThreadLocalRepo.get(); + JobExecuteContext jobExecuteContext = JobExecuteContextThreadLocalRepo.get(); if (jobExecuteContext != null) { JobInstanceContext jobInstanceContext = new JobInstanceContext(); jobInstanceContext.setTaskInstanceId(taskInstanceId); diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/AbstractLocalCacheAppScopeMappingService.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/AbstractLocalCacheAppScopeMappingService.java index 233ea4b7d9..36547bb319 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/AbstractLocalCacheAppScopeMappingService.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/AbstractLocalCacheAppScopeMappingService.java @@ -135,6 +135,12 @@ public AppResourceScope getAppResourceScope(Long appId, String scopeType, String } } + @Override + public AppResourceScope getAppResourceScope(Long appId) { + ResourceScope resourceScope = getScopeByAppId(appId); + return new AppResourceScope(appId, resourceScope); + } + @Override public AppResourceScope getAppResourceScope(String scopeType, String scopeId) { return new AppResourceScope(scopeType, scopeId, getAppIdByScope(scopeType, scopeId)); From 109bf51a087e57c438a34f203251a66578a97384 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Fri, 24 May 2024 10:52:53 +0800 Subject: [PATCH 16/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/context/JobInstanceContext.java | 6 ++ .../execute/service/TaskExecuteService.java | 8 +- .../service/impl/TaskExecuteServiceImpl.java | 91 ++++++++++++++----- 3 files changed, 79 insertions(+), 26 deletions(-) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobInstanceContext.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobInstanceContext.java index 01e525a98e..36fecd8ce3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobInstanceContext.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobInstanceContext.java @@ -27,12 +27,14 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.tencent.bk.job.common.constant.ArchiveFlag; import lombok.Data; +import lombok.NoArgsConstructor; /** * 可传播的Job作业实例上下文，用于在异步线程、跨进程(http/mq 通信等)场景下传递、共享作业实例数据 */ @Data @JsonInclude(JsonInclude.Include.NON_EMPTY) +@NoArgsConstructor public class JobInstanceContext { private Long taskInstanceId; @@ -43,4 +45,8 @@ public class JobInstanceContext { * @see ArchiveFlag */ private int archiveFlag = ArchiveFlag.HOT.getValue(); + + public JobInstanceContext(Long taskInstanceId) { + this.taskInstanceId = taskInstanceId; + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskExecuteService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskExecuteService.java index 468efb4d4e..21b8e937ef 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskExecuteService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskExecuteService.java @@ -77,7 +77,9 @@ public interface TaskExecuteService { * @param operator 操作者 * @param executeVariableValues 全局变量 */ - TaskInstanceDTO createTaskInstanceForRedo(Long appId, Long taskInstanceId, String operator, + TaskInstanceDTO createTaskInstanceForRedo(Long appId, + Long taskInstanceId, + String operator, List executeVariableValues) ; @@ -109,7 +111,9 @@ TaskInstanceDTO createTaskInstanceForRedo(Long appId, Long taskInstanceId, Strin * @param taskInstanceId 作业实例 ID * @param operation 操作类型 */ - void doTaskOperation(Long appId, String operator, long taskInstanceId, + void doTaskOperation(Long appId, + String operator, + long taskInstanceId, TaskOperationEnum operation); /** diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java index 66387ad101..33058b9b36 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java @@ -52,10 +52,13 @@ import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.execute.audit.ExecuteJobAuditEventBuilder; import com.tencent.bk.job.execute.auth.ExecuteAuthService; +import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; +import com.tencent.bk.job.execute.common.context.JobExecuteContext; +import com.tencent.bk.job.execute.common.context.JobInstanceContext; import com.tencent.bk.job.execute.common.converter.StepTypeExecuteTypeConverter; import com.tencent.bk.job.execute.config.JobExecuteConfig; import com.tencent.bk.job.execute.constants.ScriptSourceEnum; @@ -307,6 +310,10 @@ private void saveTaskInstance(StopWatch watch, watch.start("saveInstance"); Long taskInstanceId = taskInstanceService.addTaskInstance(taskInstance); taskInstance.setId(taskInstanceId); + + // 添加作业执行上下文，用于全局共享、传播上下文信息 + addJobInstanceContext(taskInstance); + stepInstance.setTaskInstanceId(taskInstanceId); stepInstance.setStepNum(1); stepInstance.setStepOrder(1); @@ -336,6 +343,15 @@ private void saveTaskInstance(StopWatch watch, watch.stop(); } + private void addJobInstanceContext(TaskInstanceDTO taskInstance) { + JobExecuteContext jobExecuteContext = JobExecuteContextThreadLocalRepo.get(); + if (jobExecuteContext != null) { + JobInstanceContext jobInstanceContext = new JobInstanceContext(taskInstance.getId()); + jobExecuteContext.setJobInstanceContext(jobInstanceContext); + JobExecuteContextThreadLocalRepo.set(jobExecuteContext); + } + } + private void auditFastJobExecute(TaskInstanceDTO taskInstance) { addFastJobExecuteAuditInstance(taskInstance); addJobInstanceInfoToExtendData(taskInstance); @@ -1692,22 +1708,21 @@ private ExecuteTargetDTO convertToServersDTO(ServiceTaskTargetDTO taskTarget) { } @Override - public Integer doStepOperation(Long appId, String operator, - StepOperationDTO stepOperation) throws ServiceException { + public Integer doStepOperation(Long appId, + String operator, + StepOperationDTO stepOperation) { long stepInstanceId = stepOperation.getStepInstanceId(); StepOperationEnum operation = stepOperation.getOperation(); + log.info("Operate step, appId:{}, stepInstanceId:{}, operator:{}, operation:{}", appId, stepInstanceId, operator, operation.getValue()); - StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail( - stepOperation.getTaskInstanceId(),stepInstanceId); - if (stepInstance == null) { - log.warn("Step instance {} is not exist", stepInstanceId); - throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); - } - if (!stepInstance.getAppId().equals(appId)) { - log.warn("Step instance {} is not in app:{}", stepInstance, appId); - throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); - } + + TaskInstanceDTO taskInstance = queryTaskInstanceAndCheckExist(appId, stepOperation.getTaskInstanceId()); + addJobInstanceContext(taskInstance); + + StepInstanceDTO stepInstance = queryStepInstanceAndCheckExist( + appId, stepOperation.getTaskInstanceId(), stepInstanceId); + int executeCount = stepInstance.getExecuteCount(); switch (operation) { case CONFIRM_CONTINUE: @@ -1746,6 +1761,29 @@ public Integer doStepOperation(Long appId, String operator, return executeCount; } + private TaskInstanceDTO queryTaskInstanceAndCheckExist(long appId, long taskInstanceId) { + TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(taskInstanceId); + if (taskInstance == null || !taskInstance.getAppId().equals(appId)) { + log.warn("Task instance is not exist, appId:{}, taskInstanceId:{}", appId, taskInstance); + throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); + } + return taskInstance; + } + + private StepInstanceDTO queryStepInstanceAndCheckExist(long appId, long taskInstanceId, long stepInstanceId) { + StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail( + taskInstanceId, stepInstanceId); + if (stepInstance == null) { + log.warn("Step instance {} is not exist", stepInstanceId); + throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); + } + if (!stepInstance.getAppId().equals(appId)) { + log.warn("Step instance {} is not in app:{}", stepInstance, appId); + throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); + } + return stepInstance; + } + private void continueRolling(StepInstanceDTO stepInstance) { // 只有“等待用户”的滚动步骤可以继续滚动 if (stepInstance.getStatus() != RunStatusEnum.WAITING_USER) { @@ -1919,7 +1957,8 @@ private void retryStepFail(StepInstanceDTO stepInstance, String operator) { taskInstanceService.updateTaskStatus(stepInstance.getTaskInstanceId(), RunStatusEnum.RUNNING.getValue()); stepInstanceService.addStepInstanceExecuteCount( stepInstance.getTaskInstanceId(), stepInstance.getId()); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.retryStepFail(stepInstance.getTaskInstanceId(), stepInstance.getId())); + taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.retryStepFail(stepInstance.getTaskInstanceId(), + stepInstance.getId())); OperationLogDTO operationLog = buildCommonStepOperationLog(stepInstance, operator, UserOperationEnum.RETRY_STEP_FAIL); taskOperationLogService.saveOperationLog(operationLog); @@ -2012,29 +2051,33 @@ private OperationLogDTO buildTaskOperationLog(TaskInstanceDTO taskInstance, Stri } @Override - public void terminateJob(String username, Long appId, Long taskInstanceId) throws ServiceException { - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(taskInstanceId); - if (taskInstance == null || !taskInstance.getAppId().equals(appId)) { - log.warn("Task instance is not exist, appId:{}, taskInstanceId:{}", appId, taskInstance); - throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); - } + public void terminateJob(String username, Long appId, Long taskInstanceId) { + TaskInstanceDTO taskInstance = queryTaskInstanceAndCheckExist(appId, taskInstanceId); + terminateJob(username, taskInstance); + } + + private void terminateJob(String operator, TaskInstanceDTO taskInstance) { + addJobInstanceContext(taskInstance); if (RunStatusEnum.RUNNING != taskInstance.getStatus() && RunStatusEnum.WAITING_USER != taskInstance.getStatus()) { log.warn("TaskInstance:{} status is not running/waiting, should not terminate it!", taskInstance.getId()); throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); } - taskExecuteMQEventDispatcher.dispatchJobEvent(JobEvent.stopJob(taskInstanceId)); - OperationLogDTO operationLog = buildTaskOperationLog(taskInstance, username, UserOperationEnum.TERMINATE_JOB); + taskExecuteMQEventDispatcher.dispatchJobEvent(JobEvent.stopJob(taskInstance.getId())); + OperationLogDTO operationLog = buildTaskOperationLog(taskInstance, operator, UserOperationEnum.TERMINATE_JOB); taskOperationLogService.saveOperationLog(operationLog); } @Override - public void doTaskOperation(Long appId, String operator, long taskInstanceId, - TaskOperationEnum operation) throws ServiceException { + public void doTaskOperation(Long appId, + String operator, + long taskInstanceId, + TaskOperationEnum operation) { log.info("Operate task instance, appId:{}, taskInstanceId:{}, operator:{}, operation:{}", appId, taskInstanceId, operator, operation.getValue()); + TaskInstanceDTO taskInstance = queryTaskInstanceAndCheckExist(appId, taskInstanceId); if (operation == TaskOperationEnum.TERMINATE_JOB) { - terminateJob(operator, appId, taskInstanceId); + terminateJob(operator, taskInstance); } else { log.warn("Undefined task operation!"); } From 30c2248f75bd4058d2c1871ca684c12ad6b8e9f6 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Fri, 24 May 2024 11:26:58 +0800 Subject: [PATCH 17/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../config/JobFunctionConfiguration.java | 4 +-- .../impl/TaskInstanceIdDynamicCondition.java | 14 +++++--- ...MqListener.java => BaseJobMqListener.java} | 12 +++---- .../engine/listener/CallbackListener.java | 5 ++- .../engine/listener/GseTaskListener.java | 6 ++-- .../execute/engine/listener/JobListener.java | 6 ++-- .../engine/listener/NotifyMsgListener.java | 5 ++- .../listener/ResultHandleResumeListener.java | 6 ++-- .../execute/engine/listener/StepListener.java | 6 ++-- .../execute/engine/listener/event/Event.java | 2 +- .../engine/listener/event/JobMessage.java | 32 +++++++++++++++++++ .../execute/engine/model/JobCallbackDTO.java | 3 +- .../bk/job/execute/model/TaskNotifyDTO.java | 3 +- 13 files changed, 75 insertions(+), 29 deletions(-) rename src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/{BaseJobExecuteMqListener.java => BaseJobMqListener.java} (86%) create mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/JobMessage.java diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobFunctionConfiguration.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobFunctionConfiguration.java index eb8857bcb1..91f4ca05fe 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobFunctionConfiguration.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobFunctionConfiguration.java @@ -79,13 +79,13 @@ public Consumer> handleResultHandleResumeEv } @Bean - public Consumer handleNotifyMsg(@Autowired NotifyMsgListener notifyMsgListener) { + public Consumer> handleNotifyMsg(@Autowired NotifyMsgListener notifyMsgListener) { log.info("Init handleNotifyMsg consumer"); return notifyMsgListener::handleMessage; } @Bean - public Consumer handleCallbackMsg(@Autowired CallbackListener callbackListener) { + public Consumer> handleCallbackMsg(@Autowired CallbackListener callbackListener) { log.info("Init handleCallbackMsg consumer"); return callbackListener::handleMessage; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java index 610faad960..acde5610c4 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java @@ -38,6 +38,14 @@ import java.util.function.Function; +/** + * task_instance_id DAO 层查询条件动态构造。 + * 背景：由于分库分表改造，原有的一些表(比如step_instance_script)中需要加入 task_instance_id 字段作为分片键。从数据的唯一性来说， + * 查询的时候传入 task_instance_id 条件是多余的，涉及的这些表的唯一业务主键都是(step_instance_id+execute_count+其它字段)。 + *

+ * 考虑到 db 变更过程中表中 task_instance_id 可能为 0 的情况（历史数据）， + * 所以，需要通过业务 ID 、task_instance_id 的值动态去构造查询条件，兼容历史数据的查询 + */ @Slf4j public class TaskInstanceIdDynamicCondition { @@ -46,7 +54,7 @@ public static Condition build(Long taskInstanceId, JobExecuteContext jobExecuteContext = JobExecuteContextThreadLocalRepo.get(); if (jobExecuteContext == null) { log.warn("TaskInstanceIdDynamicCondition : Empty JobExecuteContext!"); - // 为了不影响兼容性，忽略错误 + // JobExecuteContext 正常应该不会为 null 。为了不影响，忽略错误,直接返回 TRUE Condition return DSL.trueCondition(); } ResourceScope resourceScope = jobExecuteContext.getResourceScope(); @@ -61,9 +69,7 @@ public static Condition build(Long taskInstanceId, .addContextParam(ToggleStrategyContextParams.CTX_PARAM_RESOURCE_SCOPE, JobContextUtil.getAppResourceScope()))) { if (taskInstanceId == null || taskInstanceId <= 0L) { - log.warn("TaskInstanceIdDynamicCondition : Invalid taskInstanceId for building query condition. " + - "taskInstanceId : {}", - taskInstanceId); + log.warn("TaskInstanceIdDynamicCondition : Invalid taskInstanceId {}", taskInstanceId); // 为了不影响兼容性，忽略错误 return DSL.trueCondition(); } else { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/BaseJobExecuteMqListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/BaseJobMqListener.java similarity index 86% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/BaseJobExecuteMqListener.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/BaseJobMqListener.java index ef0a69a3bc..06e10dcff9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/BaseJobExecuteMqListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/BaseJobMqListener.java @@ -27,17 +27,17 @@ import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; import com.tencent.bk.job.execute.common.context.JobExecuteContext; -import com.tencent.bk.job.execute.engine.listener.event.Event; +import com.tencent.bk.job.execute.engine.listener.event.JobMessage; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.messaging.Message; import org.springframework.messaging.MessageHeaders; @Slf4j -public abstract class BaseJobExecuteMqListener { +public abstract class BaseJobMqListener { - public final void onEvent(Message message) { + public final void onEvent(Message message) { beforeHandleMessage(message); try { handleEvent(message); @@ -46,7 +46,7 @@ public final void onEvent(Message message) { } } - private void beforeHandleMessage(Message message) { + private void beforeHandleMessage(Message message) { log.info("beforeHandleMessage"); MessageHeaders headers = message.getHeaders(); String jobExecuteContextJson = (String) headers.get(JobExecuteContext.KEY); @@ -57,10 +57,10 @@ private void beforeHandleMessage(Message message) { } } - private void afterHandle(Message message) { + private void afterHandle(Message message) { log.info("afterHandleMessage"); JobExecuteContextThreadLocalRepo.unset(); } - protected abstract void handleEvent(Message message); + protected abstract void handleEvent(Message message); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/CallbackListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/CallbackListener.java index c9a8cc5c26..3028354589 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/CallbackListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/CallbackListener.java @@ -26,8 +26,10 @@ import com.tencent.bk.job.common.util.http.HttpConPoolUtil; import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.execute.engine.listener.event.JobMessage; import com.tencent.bk.job.execute.engine.model.JobCallbackDTO; import lombok.extern.slf4j.Slf4j; +import org.springframework.messaging.Message; import org.springframework.stereotype.Component; import java.net.MalformedURLException; @@ -43,7 +45,8 @@ public class CallbackListener { /** * 处理回调请求 */ - public void handleMessage(JobCallbackDTO callbackDTO) { + public void handleMessage(Message message) { + JobCallbackDTO callbackDTO = (JobCallbackDTO) message.getPayload(); long taskInstanceId = callbackDTO.getId(); try { log.info("Handle callback, taskInstanceId: {}, msg: {}", taskInstanceId, callbackDTO); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseTaskListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseTaskListener.java index 7bb81056fc..aa8dba3c3e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseTaskListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseTaskListener.java @@ -28,8 +28,8 @@ import com.tencent.bk.job.execute.common.exception.MessageHandlerUnavailableException; import com.tencent.bk.job.execute.engine.consts.GseTaskActionEnum; import com.tencent.bk.job.execute.engine.executor.GseTaskManager; -import com.tencent.bk.job.execute.engine.listener.event.Event; import com.tencent.bk.job.execute.engine.listener.event.GseTaskEvent; +import com.tencent.bk.job.execute.engine.listener.event.JobMessage; import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.monitor.metrics.GseTasksExceptionCounter; import com.tencent.bk.job.execute.service.GseTaskService; @@ -43,7 +43,7 @@ */ @Component @Slf4j -public class GseTaskListener extends BaseJobExecuteMqListener { +public class GseTaskListener extends BaseJobMqListener { private final GseTaskManager gseTaskManager; private final GseTaskService gseTaskService; private final GseTasksExceptionCounter gseTasksExceptionCounter; @@ -63,7 +63,7 @@ public GseTaskListener(GseTaskManager gseTaskManager, * @param message 消息 */ @Override - public void handleEvent(Message message) { + public void handleEvent(Message message) { GseTaskEvent gseTaskEvent = (GseTaskEvent) message.getPayload(); log.info("Handle gse task event: {}, duration: {}ms", gseTaskEvent, gseTaskEvent.duration()); GseTaskDTO gseTask = gseTaskService.getGseTask(gseTaskEvent.getJobInstanceId(), gseTaskEvent.getGseTaskId()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java index 3ec929ccba..4004ad5cf8 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java @@ -30,8 +30,8 @@ import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.common.util.TaskCostCalculator; import com.tencent.bk.job.execute.engine.consts.JobActionEnum; -import com.tencent.bk.job.execute.engine.listener.event.Event; import com.tencent.bk.job.execute.engine.listener.event.JobEvent; +import com.tencent.bk.job.execute.engine.listener.event.JobMessage; import com.tencent.bk.job.execute.engine.listener.event.StepEvent; import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; import com.tencent.bk.job.execute.engine.model.JobCallbackDTO; @@ -58,7 +58,7 @@ */ @Component("jobEventListener") @Slf4j -public class JobListener extends BaseJobExecuteMqListener { +public class JobListener extends BaseJobMqListener { private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; private final StatisticsService statisticsService; @@ -89,7 +89,7 @@ public JobListener(TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, * @param message 消息 */ @Override - public void handleEvent(Message message) { + public void handleEvent(Message message) { JobEvent jobEvent = (JobEvent) message.getPayload(); log.info("Handle job event, event: {}, duration: {}ms", jobEvent, jobEvent.duration()); long jobInstanceId = jobEvent.getJobInstanceId(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/NotifyMsgListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/NotifyMsgListener.java index 33e3b8fb80..4348dd9535 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/NotifyMsgListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/NotifyMsgListener.java @@ -25,11 +25,13 @@ package com.tencent.bk.job.execute.engine.listener; import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.execute.engine.listener.event.JobMessage; import com.tencent.bk.job.execute.model.TaskNotifyDTO; import com.tencent.bk.job.execute.service.NotifyService; import com.tencent.bk.job.manage.api.common.constants.notify.ExecuteStatusEnum; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.messaging.Message; import org.springframework.stereotype.Component; /** @@ -45,7 +47,8 @@ public NotifyMsgListener(NotifyService notifyService) { this.notifyService = notifyService; } - public void handleMessage(TaskNotifyDTO taskNotifyDTO) { + public void handleMessage(Message message) { + TaskNotifyDTO taskNotifyDTO = (TaskNotifyDTO) message.getPayload(); log.info("Begin to send msg:{}", JsonUtils.toJson(taskNotifyDTO)); ExecuteStatusEnum executeStatus = ExecuteStatusEnum.get(taskNotifyDTO.getResourceExecuteStatus()); if (executeStatus == null) { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ResultHandleResumeListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ResultHandleResumeListener.java index b1721ce0c6..c6d32a2553 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ResultHandleResumeListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ResultHandleResumeListener.java @@ -30,7 +30,7 @@ import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.config.FileDistributeConfig; import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; -import com.tencent.bk.job.execute.engine.listener.event.Event; +import com.tencent.bk.job.execute.engine.listener.event.JobMessage; import com.tencent.bk.job.execute.engine.listener.event.ResultHandleTaskResumeEvent; import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; import com.tencent.bk.job.execute.engine.model.FileDest; @@ -71,7 +71,7 @@ */ @Component @Slf4j -public class ResultHandleResumeListener extends BaseJobExecuteMqListener { +public class ResultHandleResumeListener extends BaseJobMqListener { private final TaskInstanceService taskInstanceService; private final ResultHandleManager resultHandleManager; @@ -134,7 +134,7 @@ public ResultHandleResumeListener(TaskInstanceService taskInstanceService, /** * 恢复被中断的作业结果处理任务 */ - public void handleEvent(Message message) { + public void handleEvent(Message message) { ResultHandleTaskResumeEvent event = (ResultHandleTaskResumeEvent) message.getPayload(); log.info("Receive gse task result handle task resume event: {}, duration: {}ms", event, event.duration()); GseTaskDTO gseTask = gseTaskService.getGseTask(event.getJobInstanceId(), event.getGseTaskId()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepListener.java index 1d488f29a6..f6184eae46 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepListener.java @@ -25,7 +25,7 @@ package com.tencent.bk.job.execute.engine.listener; import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; -import com.tencent.bk.job.execute.engine.listener.event.Event; +import com.tencent.bk.job.execute.engine.listener.event.JobMessage; import com.tencent.bk.job.execute.engine.listener.event.StepEvent; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.service.StepInstanceService; @@ -39,7 +39,7 @@ */ @Component @Slf4j -public class StepListener extends BaseJobExecuteMqListener { +public class StepListener extends BaseJobMqListener { private final StepInstanceService stepInstanceService; private final GseStepEventHandler gseStepEventHandler; private final ConfirmStepEventHandler confirmStepEventHandler; @@ -59,7 +59,7 @@ public StepListener(StepInstanceService stepInstanceService, * @param message 消息 */ @Override - public void handleEvent(Message message) { + public void handleEvent(Message message) { StepEvent stepEvent = (StepEvent) message.getPayload(); log.info("Handle step event: {}, duration: {}ms", stepEvent, stepEvent.duration()); long stepInstanceId = stepEvent.getStepInstanceId(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/Event.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/Event.java index 16a00664ec..01ddd4a4a6 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/Event.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/Event.java @@ -35,7 +35,7 @@ */ @Data @JsonInclude(JsonInclude.Include.NON_NULL) -public class Event { +public class Event implements JobMessage { /** * 事件源 */ diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/JobMessage.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/JobMessage.java new file mode 100644 index 0000000000..883f1932c6 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/JobMessage.java @@ -0,0 +1,32 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.listener.event; + +/** + * MQ Job消息接口 + */ +public interface JobMessage { + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/JobCallbackDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/JobCallbackDTO.java index b29e87aa2b..af50019d3c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/JobCallbackDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/JobCallbackDTO.java @@ -26,6 +26,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.execute.engine.listener.event.JobMessage; import lombok.Getter; import lombok.Setter; import lombok.ToString; @@ -36,7 +37,7 @@ @Getter @ToString @JsonInclude(JsonInclude.Include.NON_NULL) -public class JobCallbackDTO { +public class JobCallbackDTO implements JobMessage { @JsonProperty("job_instance_id") private long id; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskNotifyDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskNotifyDTO.java index c15f08fb50..ea05301184 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskNotifyDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskNotifyDTO.java @@ -26,6 +26,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; +import com.tencent.bk.job.execute.engine.listener.event.JobMessage; import lombok.Getter; import lombok.Setter; @@ -35,7 +36,7 @@ @Getter @Setter @JsonInclude(JsonInclude.Include.NON_EMPTY) -public class TaskNotifyDTO { +public class TaskNotifyDTO implements JobMessage { /** * 操作者 */ From 4b7a3843120298f35fbf52be9af7fe9d08addceb Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Fri, 24 May 2024 11:44:37 +0800 Subject: [PATCH 18/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ... => JobExecuteContextHttpInterceptor.java} | 2 +- ...ExecuteContextPropagateMqInterceptor.java} | 20 ++++--------------- .../config/JobFunctionConfiguration.java | 4 ++-- .../engine/listener/CallbackListener.java | 5 +++-- .../engine/listener/NotifyMsgListener.java | 5 +++-- 5 files changed, 13 insertions(+), 23 deletions(-) rename src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/{JobExecuteContextInterceptor.java => JobExecuteContextHttpInterceptor.java} (97%) rename src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/{JobExecuteContextMessageChannelInterceptor.java => JobExecuteContextPropagateMqInterceptor.java} (76%) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextInterceptor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextHttpInterceptor.java similarity index 97% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextInterceptor.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextHttpInterceptor.java index 68a4015e04..474dab4517 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextInterceptor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextHttpInterceptor.java @@ -44,7 +44,7 @@ @Slf4j @Component @JobInterceptor(order = InterceptorOrder.Init.LOWEST, pathPatterns = "/**") -public class JobExecuteContextInterceptor implements AsyncHandlerInterceptor { +public class JobExecuteContextHttpInterceptor implements AsyncHandlerInterceptor { @Override public boolean preHandle(@NonNull HttpServletRequest request, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextPropagateMqInterceptor.java similarity index 76% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextPropagateMqInterceptor.java index 817a02107f..23e830fdac 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextMessageChannelInterceptor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextPropagateMqInterceptor.java @@ -31,39 +31,27 @@ import org.springframework.integration.config.GlobalChannelInterceptor; import org.springframework.messaging.Message; import org.springframework.messaging.MessageChannel; -import org.springframework.messaging.support.ExecutorChannelInterceptor; +import org.springframework.messaging.support.ChannelInterceptor; import org.springframework.messaging.support.MessageBuilder; import org.springframework.stereotype.Component; @Component @Slf4j @GlobalChannelInterceptor -public class JobExecuteContextMessageChannelInterceptor implements ExecutorChannelInterceptor { +public class JobExecuteContextPropagateMqInterceptor implements ChannelInterceptor { @Override public Message preSend(Message message, MessageChannel channel) { log.info("preSend"); JobExecuteContext context = JobExecuteContextThreadLocalRepo.get(); if (context == null) { - return ExecutorChannelInterceptor.super.preSend(message, channel); + return ChannelInterceptor.super.preSend(message, channel); } log.info("setJobExecuteContextMessageHeader, context: {}", context); Message newMessage = MessageBuilder.fromMessage(message) .setHeader(JobExecuteContext.KEY, JsonUtils.toJson(context)) .build(); - return ExecutorChannelInterceptor.super.preSend(newMessage, channel); - } - - @Override - public void postSend(Message message, MessageChannel channel, boolean sent) { - log.info("postSend"); - ExecutorChannelInterceptor.super.postSend(message, channel, sent); - } - - @Override - public void afterSendCompletion(Message message, MessageChannel channel, boolean sent, Exception ex) { - log.info("afterSendCompletion"); - ExecutorChannelInterceptor.super.afterSendCompletion(message, channel, sent, ex); + return ChannelInterceptor.super.preSend(newMessage, channel); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobFunctionConfiguration.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobFunctionConfiguration.java index 91f4ca05fe..259695814f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobFunctionConfiguration.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobFunctionConfiguration.java @@ -81,13 +81,13 @@ public Consumer> handleResultHandleResumeEv @Bean public Consumer> handleNotifyMsg(@Autowired NotifyMsgListener notifyMsgListener) { log.info("Init handleNotifyMsg consumer"); - return notifyMsgListener::handleMessage; + return notifyMsgListener::onEvent; } @Bean public Consumer> handleCallbackMsg(@Autowired CallbackListener callbackListener) { log.info("Init handleCallbackMsg consumer"); - return callbackListener::handleMessage; + return callbackListener::onEvent; } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/CallbackListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/CallbackListener.java index 3028354589..f9ee84299c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/CallbackListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/CallbackListener.java @@ -40,12 +40,13 @@ */ @Component @Slf4j -public class CallbackListener { +public class CallbackListener extends BaseJobMqListener { /** * 处理回调请求 */ - public void handleMessage(Message message) { + @Override + public void handleEvent(Message message) { JobCallbackDTO callbackDTO = (JobCallbackDTO) message.getPayload(); long taskInstanceId = callbackDTO.getId(); try { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/NotifyMsgListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/NotifyMsgListener.java index 4348dd9535..6a0568d835 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/NotifyMsgListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/NotifyMsgListener.java @@ -39,7 +39,7 @@ */ @Component @Slf4j -public class NotifyMsgListener { +public class NotifyMsgListener extends BaseJobMqListener { private final NotifyService notifyService; @Autowired @@ -47,7 +47,8 @@ public NotifyMsgListener(NotifyService notifyService) { this.notifyService = notifyService; } - public void handleMessage(Message message) { + @Override + public void handleEvent(Message message) { TaskNotifyDTO taskNotifyDTO = (TaskNotifyDTO) message.getPayload(); log.info("Begin to send msg:{}", JsonUtils.toJson(taskNotifyDTO)); ExecuteStatusEnum executeStatus = ExecuteStatusEnum.get(taskNotifyDTO.getResourceExecuteStatus()); From 8db2f6b554227488e465e889140d7f06406c60f6 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Fri, 24 May 2024 14:27:31 +0800 Subject: [PATCH 19/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/web/WebExecuteTaskResource.java | 2 +- .../api/web/WebTaskInstanceResource.java | 71 +++++++++++++------ ...sbGetStepInstanceDetailV3ResourceImpl.java | 2 +- .../web/impl/WebSearchToolsResourceImpl.java | 2 +- .../WebTaskExecutionResultResourceImpl.java | 23 ++---- .../web/impl/WebTaskInstanceResourceImpl.java | 14 +++- .../execute/service/StepInstanceService.java | 2 +- .../service/impl/StepInstanceServiceImpl.java | 2 +- 8 files changed, 73 insertions(+), 45 deletions(-) diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebExecuteTaskResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebExecuteTaskResource.java index c702bb5610..2cc0e1510d 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebExecuteTaskResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebExecuteTaskResource.java @@ -140,7 +140,7 @@ Response fastPushFile( ); @ApiOperation(value = "执行作业步骤操作", produces = "application/json") - @PostMapping(value = {"/do-step-operation/taskInstanceId/{taskInstanceId}/stepInstanceId/{stepInstanceId}"}) + @PostMapping(value = {"/taskInstance/{taskInstanceId}/stepInstance/{stepInstanceId}/stepOperation"}) Response doStepOperation( @ApiParam("用户名，网关自动传入") @RequestHeader("username") diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskInstanceResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskInstanceResource.java index e20490a731..ed881adab7 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskInstanceResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskInstanceResource.java @@ -24,7 +24,9 @@ package com.tencent.bk.job.execute.api.web; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; import com.tencent.bk.job.common.annotation.WebAPI; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.execute.model.web.vo.ExecuteStepVO; @@ -53,24 +55,51 @@ @RestController @WebAPI public interface WebTaskInstanceResource { + @Deprecated + @CompatibleImplementation(name = "dao_add_task_instance_id", deprecatedVersion = "3.10.x", + type = CompatibleType.DEPLOY, explain = "发布完成后可以删除") @ApiOperation(value = "获取作业步骤实例详情", produces = "application/json") @GetMapping(value = {"/scope/{scopeType}/{scopeId}/task-instance/step_instance/{stepInstanceId}"}) Response getStepInstanceDetail( @ApiParam("用户名，网关自动传入") @RequestHeader("username") - String username, + String username, @ApiIgnore @RequestAttribute(value = "appResourceScope") - AppResourceScope appResourceScope, + AppResourceScope appResourceScope, @ApiParam(value = "资源范围类型", required = true) @PathVariable(value = "scopeType") - String scopeType, + String scopeType, @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") - String scopeId, + String scopeId, @ApiParam(value = "步骤实例ID", name = "stepInstanceId", required = true) @PathVariable("stepInstanceId") - Long stepInstanceId + Long stepInstanceId + ); + + @ApiOperation(value = "获取作业步骤实例详情", produces = "application/json") + @GetMapping(value = { + "/scope/{scopeType}/{scopeId}/taskInstance/{taskInstanceId}/stepInstance/{stepInstanceId}/detail"}) + Response getStepInstanceDetailV2( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "作业实例ID", name = "taskInstanceId", required = true) + @PathVariable("taskInstanceId") + Long taskInstanceId, + @ApiParam(value = "步骤实例ID", name = "stepInstanceId", required = true) + @PathVariable("stepInstanceId") + Long stepInstanceId ); @ApiOperation(value = "获取作业实例全局参数", produces = "application/json") @@ -78,19 +107,19 @@ Response getStepInstanceDetail( Response> getTaskInstanceVariables( @ApiParam("用户名，网关自动传入") @RequestHeader("username") - String username, + String username, @ApiIgnore @RequestAttribute(value = "appResourceScope") - AppResourceScope appResourceScope, + AppResourceScope appResourceScope, @ApiParam(value = "资源范围类型", required = true) @PathVariable(value = "scopeType") - String scopeType, + String scopeType, @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") - String scopeId, + String scopeId, @ApiParam(value = "作业实例ID", name = "taskInstanceId", required = true) @PathVariable("taskInstanceId") - Long taskInstanceId + Long taskInstanceId ); @ApiOperation(value = "获取作业操作日志", produces = "application/json") @@ -98,19 +127,19 @@ Response> getTaskInstanceVariables( Response> getTaskInstanceOperationLog( @ApiParam("用户名，网关自动传入") @RequestHeader("username") - String username, + String username, @ApiIgnore @RequestAttribute(value = "appResourceScope") - AppResourceScope appResourceScope, + AppResourceScope appResourceScope, @ApiParam(value = "资源范围类型", required = true) @PathVariable(value = "scopeType") - String scopeType, + String scopeType, @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") - String scopeId, + String scopeId, @ApiParam(value = "作业实例ID", name = "taskInstanceId", required = true) @PathVariable("taskInstanceId") - Long taskInstanceId + Long taskInstanceId ); @ApiOperation(value = "获取作业实例详情,包括步骤列表和全局变量列表", produces = "application/json") @@ -118,19 +147,19 @@ Response> getTaskInstanceOperationLog( Response getTaskInstanceDetail( @ApiParam("用户名，网关自动传入") @RequestHeader("username") - String username, + String username, @ApiIgnore @RequestAttribute(value = "appResourceScope") - AppResourceScope appResourceScope, + AppResourceScope appResourceScope, @ApiParam(value = "资源范围类型", required = true) @PathVariable(value = "scopeType") - String scopeType, + String scopeType, @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") - String scopeId, + String scopeId, @ApiParam(value = "作业实例ID", name = "taskInstanceId", required = true) @PathVariable("taskInstanceId") - Long taskInstanceId + Long taskInstanceId ); @ApiOperation(value = "获取作业实例基本信息", produces = "application/json") @@ -140,6 +169,6 @@ Response getTaskInstanceBasic( @RequestHeader("username") String username, @ApiParam(value = "作业实例ID", name = "taskInstanceId", required = true) @PathVariable("taskInstanceId") - Long taskInstanceId + Long taskInstanceId ); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceDetailV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceDetailV3ResourceImpl.java index 29dbbc4e22..0ffcd6bc2b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceDetailV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceDetailV3ResourceImpl.java @@ -80,7 +80,7 @@ public EsbResp getStepInstanceDetail(String username, Long stepInstanceId) { long appId = appScopeMappingService.getAppIdByScope(scopeType, scopeId); - StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail(appId, stepInstanceId); + StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail(taskInstanceId, stepInstanceId); taskInstanceAccessProcessor.processBeforeAccess(username, appId, stepInstance.getTaskInstanceId()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebSearchToolsResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebSearchToolsResourceImpl.java index d4e5cc8993..7407d465fe 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebSearchToolsResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebSearchToolsResourceImpl.java @@ -92,7 +92,7 @@ public Response getTaskLink(String username, @Override public Response> getTaskLinkByStepId(String username, Long stepInstanceId) { - StepInstanceBaseDTO stepInstanceBase = stepInstanceService.getBaseStepInstance(stepInstanceId); + StepInstanceBaseDTO stepInstanceBase = stepInstanceService.getBaseStepInstanceById(stepInstanceId); if (stepInstanceBase == null) { String errorMsg = "not found StepInstance by " + stepInstanceId; log.warn(errorMsg); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java index 776e22c027..293cc36fc0 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java @@ -677,8 +677,7 @@ public Response getScriptLogContentByHost(String username, Integer executeCount, Long hostId, Integer batch) { - StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance( - appResourceScope.getAppId(), stepInstanceId); + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstanceById(stepInstanceId); auditAndAuthViewStepInstance(username, appResourceScope, stepInstance); ScriptExecuteObjectLogContent scriptExecuteObjectLogContent = @@ -702,7 +701,7 @@ public Response> getStepVariableByHost(String username, Long hostId, String ip) { StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail( - appResourceScope.getAppId(), stepInstanceId); + null, stepInstanceId); if (!stepInstance.isScriptStep() || stepInstance.getScriptType() != ScriptTypeEnum.SHELL) { return Response.buildSuccessResp(Collections.emptyList()); } @@ -943,16 +942,6 @@ private void auditAndAuthViewStepInstance(String username, appResourceScope.getAppId(), stepInstance.getTaskInstanceId()); } - private void auditAndAuthViewStepInstance(String username, - AppResourceScope appResourceScope, - Long stepInstanceId) { - StepInstanceBaseDTO stepInstance = - stepInstanceService.getBaseStepInstance(appResourceScope.getAppId(), stepInstanceId); - taskInstanceAccessProcessor.processBeforeAccess(username, - appResourceScope.getAppId(), stepInstance.getTaskInstanceId()); - } - - @Override @AuditEntry(actionId = ActionId.VIEW_HISTORY) public Response> getHostsByResultType(String username, @@ -1078,7 +1067,7 @@ public Response getScriptLogContentByExecuteObject(Str Integer executeCount, Integer batch) { StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance( - appResourceScope.getAppId(), stepInstanceId); + appResourceScope.getAppId(), taskInstanceId, stepInstanceId); auditAndAuthViewStepInstance(username, appResourceScope, stepInstance); int actualExecuteCount = computeActualExecuteCount(stepInstance, executeCount); @@ -1114,7 +1103,7 @@ public Response getFileLogContentByExecuteObject(String Integer batch, Integer mode) { StepInstanceDTO stepInstance = - stepInstanceService.getStepInstanceDetail(appResourceScope.getAppId(), stepInstanceId); + stepInstanceService.getStepInstanceDetail(taskInstanceId, stepInstanceId); auditAndAuthViewStepInstance(username, appResourceScope, stepInstance); ExecuteObjectFileLogVO result = new ExecuteObjectFileLogVO(); @@ -1179,7 +1168,7 @@ public Response> getFileLogContentByFileTaskIds Integer batch, List taskIds) { StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance( - appResourceScope.getAppId(), stepInstanceId); + taskInstanceId, stepInstanceId); auditAndAuthViewStepInstance(username, appResourceScope, stepInstance); int actualExecuteCount = computeActualExecuteCount(stepInstance, executeCount); @@ -1204,7 +1193,7 @@ public Response> getStepVariableByExecuteObject(String u Integer executeObjectType, Long executeObjectResourceId) { StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail( - appResourceScope.getAppId(), stepInstanceId); + taskInstanceId, stepInstanceId); if (!stepInstance.isScriptStep() || stepInstance.getScriptType() != ScriptTypeEnum.SHELL) { return Response.buildSuccessResp(Collections.emptyList()); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java index 503feb44cf..c2aac679c7 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java @@ -121,8 +121,18 @@ public Response getStepInstanceDetail(String username, String scopeId, Long stepInstanceId) { - StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail( - appResourceScope.getAppId(), stepInstanceId); + return getStepInstanceDetailV2(username, appResourceScope, scopeType, scopeId, null, stepInstanceId); + } + + @Override + @AuditEntry(actionId = ActionId.VIEW_HISTORY) + public Response getStepInstanceDetailV2(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + Long taskInstanceId, + Long stepInstanceId) { + StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail(taskInstanceId, stepInstanceId); taskInstanceAccessProcessor.processBeforeAccess(username, appResourceScope.getAppId(), stepInstance.getTaskInstanceId()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceService.java index 14e0676314..f0db33876f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceService.java @@ -116,7 +116,7 @@ Map computeStepExecuteObjects(StepInstanceBaseDTO stepInst * @param stepInstanceId 步骤实例ID * @return 步骤基本信息 */ - StepInstanceBaseDTO getBaseStepInstance(long stepInstanceId); + StepInstanceBaseDTO getBaseStepInstanceById(long stepInstanceId); /** * 获取步骤基本信息 diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java index 33af9df86c..c611ebc8a7 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java @@ -142,7 +142,7 @@ public StepInstanceBaseDTO getBaseStepInstance(Long taskInstanceId, long stepIns } @Override - public StepInstanceBaseDTO getBaseStepInstance(long stepInstanceId) { + public StepInstanceBaseDTO getBaseStepInstanceById(long stepInstanceId) { return stepInstanceDAO.getStepInstanceBase(stepInstanceId); } From 6b4e41f9e433040734ed7438186cbd57d44bcf67 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Fri, 24 May 2024 15:32:02 +0800 Subject: [PATCH 20/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/web/WebExecuteTaskResource.java | 30 +++++- .../web/WebTaskExecutionResultResource.java | 4 +- ...nstanceVariableDAOImplIntegrationTest.java | 2 +- .../web/impl/WebExecuteTaskResourceImpl.java | 14 ++- .../WebTaskExecutionResultResourceImpl.java | 75 +++++++-------- .../colddata/JobInstanceRecordQuery.java | 37 -------- .../JobInstanceRecordQueryAspect.java | 91 ------------------- .../common/context/JobExecuteContext.java | 10 +- .../JobExecuteContextThreadLocalRepo.java | 11 ++- .../JobExecuteContextHttpInterceptor.java | 8 +- ...bExecuteContextPropagateMqInterceptor.java | 10 +- .../execute/dao/StepInstanceVariableDAO.java | 5 +- .../dao/impl/StepInstanceVariableDAOImpl.java | 12 ++- .../impl/TaskInstanceIdDynamicCondition.java | 14 +-- .../engine/listener/BaseJobMqListener.java | 2 +- .../engine/result/ResultHandleManager.java | 2 +- .../ScheduledContinuousResultHandleTask.java | 2 +- .../job/execute/model/StepOperationDTO.java | 2 +- .../execute/service/TaskExecuteService.java | 2 - .../service/impl/TaskExecuteServiceImpl.java | 2 +- .../service/impl/TaskInstanceServiceImpl.java | 2 +- 21 files changed, 130 insertions(+), 207 deletions(-) delete mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobInstanceRecordQuery.java delete mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobInstanceRecordQueryAspect.java rename src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/{colddata => common/context}/JobExecuteContextThreadLocalRepo.java (86%) diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebExecuteTaskResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebExecuteTaskResource.java index 2cc0e1510d..c0a65cc4d2 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebExecuteTaskResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebExecuteTaskResource.java @@ -24,7 +24,9 @@ package com.tencent.bk.job.execute.api.web; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; import com.tencent.bk.job.common.annotation.WebAPI; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.execute.model.web.request.RedoTaskRequest; @@ -140,8 +142,34 @@ Response fastPushFile( ); @ApiOperation(value = "执行作业步骤操作", produces = "application/json") - @PostMapping(value = {"/taskInstance/{taskInstanceId}/stepInstance/{stepInstanceId}/stepOperation"}) + @PostMapping(value = {"/do-step-operation/stepInstanceId/{stepInstanceId}"}) + @Deprecated + @CompatibleImplementation(name = "dao_add_task_instance_id", deprecatedVersion = "3.10.x", + type = CompatibleType.DEPLOY, explain = "发布完成后可以删除") Response doStepOperation( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "步骤实例ID", required = true, example = "1") + @PathVariable("stepInstanceId") + Long stepInstanceId, + @ApiParam(value = "步骤实例操作请求报文", name = "operation", required = true) + @RequestBody + WebStepOperation operation + ); + + @ApiOperation(value = "执行作业步骤操作", produces = "application/json") + @PostMapping(value = {"/taskInstance/{taskInstanceId}/stepInstance/{stepInstanceId}/operate"}) + Response doStepOperationV2( @ApiParam("用户名，网关自动传入") @RequestHeader("username") String username, diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java index 868f7f4129..6f5bb76489 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java @@ -521,7 +521,7 @@ Response> getExecuteObjectsByResultType( @ApiOperation(value = "获取步骤执行历史", produces = "application/json") @GetMapping(value = {"/step-execution-history/{stepInstanceId}"}) @Deprecated - @CompatibleImplementation(name = "add_task_instance_id", deprecatedVersion = "3.10.x", type = CompatibleType.DEPLOY, + @CompatibleImplementation(name = "dao_add_task_instance_id", deprecatedVersion = "3.10.x", type = CompatibleType.DEPLOY, explain = "发布完成后可以删除") Response> listStepExecutionHistory( @ApiParam("用户名，网关自动传入") @@ -560,7 +560,7 @@ Response> listStepExecutionHistoryV2( @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") String scopeId, - @ApiParam(value = "作业实例ID", name = "stepInstanceId", required = true) + @ApiParam(value = "作业实例ID", name = "taskInstanceId", required = true) @PathVariable("taskInstanceId") Long taskInstanceId, @ApiParam(value = "步骤实例ID", name = "stepInstanceId", required = true) diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceVariableDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceVariableDAOImplIntegrationTest.java index 97e10a3d8f..dcce271fcd 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceVariableDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceVariableDAOImplIntegrationTest.java @@ -81,7 +81,7 @@ void testSaveVariableValues() { stepInstanceVariableDAO.saveVariableValues(variableValues); StepInstanceVariableValuesDTO actual = stepInstanceVariableDAO - .getStepVariableValues(200L, 1, VariableValueTypeEnum.OUTPUT); + .getStepVariableValues(100L, 200L, 1, VariableValueTypeEnum.OUTPUT); assertThat(actual.getStepInstanceId()).isEqualTo(200L); assertThat(actual.getExecuteCount()).isEqualTo(1); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java index 970b2d80b7..bc3f6fefeb 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java @@ -491,7 +491,6 @@ private StepInstanceDTO buildFastFileStepInstance(String userName, Long appId, W } - private List convertFileSource(List fileSources) { if (fileSources == null) { return null; @@ -531,9 +530,20 @@ public Response doStepOperation(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, - Long taskInstanceId, Long stepInstanceId, WebStepOperation operation) { + return doStepOperationV2(username, appResourceScope, scopeType, scopeId, + null, stepInstanceId, operation); + } + + @Override + public Response doStepOperationV2(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + Long taskInstanceId, + Long stepInstanceId, + WebStepOperation operation) { StepOperationEnum stepOperationEnum = StepOperationEnum.getStepOperation(operation.getOperationCode()); if (stepOperationEnum == null) { throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java index 293cc36fc0..3c986ecc26 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java @@ -148,44 +148,44 @@ public class WebTaskExecutionResultResourceImpl implements WebTaskExecutionResul private final LoadingCache> roleCache = CacheBuilder.newBuilder() .maximumSize(10).expireAfterWrite(10, TimeUnit.MINUTES). - build(new CacheLoader>() { - @Override - public Map load(String lang) { - InternalResponse> resp = notifyResource.getNotifyRoles(lang); - log.info("Get notify roles, resp={}", resp); - if (!resp.isSuccess() || resp.getData() == null) { - return new HashMap<>(); - } else { - List appRoles = resp.getData(); - Map codeNameMap = new HashMap<>(); - if (appRoles != null) { - appRoles.forEach(role -> codeNameMap.put(role.getCode(), role.getName())); - } - return codeNameMap; + build(new CacheLoader>() { + @Override + public Map load(String lang) { + InternalResponse> resp = notifyResource.getNotifyRoles(lang); + log.info("Get notify roles, resp={}", resp); + if (!resp.isSuccess() || resp.getData() == null) { + return new HashMap<>(); + } else { + List appRoles = resp.getData(); + Map codeNameMap = new HashMap<>(); + if (appRoles != null) { + appRoles.forEach(role -> codeNameMap.put(role.getCode(), role.getName())); } + return codeNameMap; } } - ); + } + ); private final LoadingCache> channelCache = CacheBuilder.newBuilder() .maximumSize(10).expireAfterWrite(10, TimeUnit.MINUTES). - build(new CacheLoader>() { - @Override - public Map load(String lang) { - InternalResponse> resp = notifyResource.getNotifyChannels(lang); - log.info("Get notify channels, resp={}", resp); - if (!resp.isSuccess() || resp.getData() == null) { - return new HashMap<>(); - } else { - List channels = resp.getData(); - Map typeNameMap = new HashMap<>(); - if (channels != null) { - channels.forEach(channel -> typeNameMap.put(channel.getType(), channel.getName())); - } - return typeNameMap; + build(new CacheLoader>() { + @Override + public Map load(String lang) { + InternalResponse> resp = notifyResource.getNotifyChannels(lang); + log.info("Get notify channels, resp={}", resp); + if (!resp.isSuccess() || resp.getData() == null) { + return new HashMap<>(); + } else { + List channels = resp.getData(); + Map typeNameMap = new HashMap<>(); + if (channels != null) { + channels.forEach(channel -> typeNameMap.put(channel.getType(), channel.getName())); } + return typeNameMap; } } - ); + } + ); @Autowired public WebTaskExecutionResultResourceImpl(TaskResultService taskResultService, @@ -984,17 +984,7 @@ public Response> listStepExecutionHistory(String use Long stepInstanceId, Integer batch) { - List stepExecutionRecords = taskResultService.listStepExecutionHistory(username, - appResourceScope.getAppId(), null, stepInstanceId, batch); - - return Response.buildSuccessResp(stepExecutionRecords.stream().map(stepExecutionRecord -> { - StepExecutionRecordVO vo = new StepExecutionRecordVO(); - vo.setStepInstanceId(stepInstanceId); - vo.setRetryCount(stepExecutionRecord.getRetryCount()); - vo.setExecuteCount(stepExecutionRecord.getRetryCount()); - vo.setCreateTime(stepExecutionRecord.getCreateTime()); - return vo; - }).collect(Collectors.toList())); + return listStepExecutionHistoryV2(username, appResourceScope, scopeType, scopeId, null, stepInstanceId, batch); } @Override @@ -1003,7 +993,8 @@ public Response> listStepExecutionHistoryV2(String u String scopeType, String scopeId, Long taskInstanceId, - Long stepInstanceId, Integer batch) { + Long stepInstanceId, + Integer batch) { List stepExecutionRecords = taskResultService.listStepExecutionHistory(username, appResourceScope.getAppId(), taskInstanceId, stepInstanceId, batch); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobInstanceRecordQuery.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobInstanceRecordQuery.java deleted file mode 100644 index 2ce32842b8..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobInstanceRecordQuery.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.colddata; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Inherited; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@Target({ElementType.TYPE, ElementType.METHOD}) -@Retention(RetentionPolicy.RUNTIME) -@Inherited -public @interface JobInstanceRecordQuery { -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobInstanceRecordQueryAspect.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobInstanceRecordQueryAspect.java deleted file mode 100644 index ab2caf2fcd..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobInstanceRecordQueryAspect.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.colddata; - -import lombok.extern.slf4j.Slf4j; - - -//@Aspect -@Slf4j -public class JobInstanceRecordQueryAspect { -// private final MeterRegistry registry; -// -// public JobInstanceRecordQueryAspect(MeterRegistry registry) { -// log.info("Init JobInstanceRecordQueryAspect"); -// this.registry = registry; -// } -// -// @Around("within(com.tencent.bk.job..*) && execution (@com.tencent.bk.job.execute.colddata.JobInstanceRecordQuery * *.*(..))") -// public Object process(ProceedingJoinPoint pjp) throws Throwable { -// RequestAttributes requestAttributes = RequestContextHolder.getRequestAttributes(); -// HttpServletRequest request = null; -// if (requestAttributes != null) { -// request = ((ServletRequestAttributes)requestAttributes).getRequest(); -// } -// if (request == null) { -// return pjp.proceed(); -// } -// -// JobContextUtil.set -// -// String appCode = request.getHeader(JobCommonHeaders.APP_CODE); -// Tags tags = Tags.of("app_code", StringUtils.isNotBlank(appCode) ? appCode : "None"); -// -// Method method = ((MethodSignature) pjp.getSignature()).getMethod(); -// JobInstanceRecordQuery timed = method.getAnnotation(JobInstanceRecordQuery.class); -// final String metricName = timed.value(); -// return processWithTimer(pjp, timed, metricName, tags); -// } -// -// private Object processWithTimer(ProceedingJoinPoint pjp, JobInstanceRecordQuery timed, String metricName, -// Tags tags) throws Throwable { -// -// Timer.Sample sample = Timer.start(registry); -// Exception exception = null; -// try { -// return pjp.proceed(); -// } catch (Exception ex) { -// exception = ex; -// throw ex; -// } finally { -// tags.and(exception(exception)); -// record(timed, metricName, sample, tags); -// } -// } -// -// private void record(JobInstanceRecordQuery timed, String metricName, Timer.Sample sample, Tags tags) { -// try { -// sample.stop(Timer.builder(metricName) -// .description(timed.description().isEmpty() ? null : timed.description()) -// .tags(tags) -// .tags(timed.extraTags()) -// .publishPercentileHistogram(timed.histogram()) -// .publishPercentiles(timed.percentiles().length == 0 ? null : timed.percentiles()) -// .register(registry)); -// } catch (Exception e) { -// // ignoring on purpose -// } -// } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobExecuteContext.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobExecuteContext.java index 87050f734b..104c850400 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobExecuteContext.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobExecuteContext.java @@ -29,7 +29,7 @@ import lombok.Data; /** - * 作业执行上下文 + * 作业执行上下文信息，用于在全局共享、异步传播(跨线程、跨 mq等场景) */ @Data @JsonInclude(JsonInclude.Include.NON_EMPTY) @@ -38,11 +38,17 @@ public class JobExecuteContext { public static final String KEY = "JobExecuteContext"; /** - * 业务 + * 业务信息 */ private AppResourceScope resourceScope; + /** + * 用户账号 + */ private String username; + /** + * 作业实例 + */ private JobInstanceContext jobInstanceContext; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobExecuteContextThreadLocalRepo.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobExecuteContextThreadLocalRepo.java similarity index 86% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobExecuteContextThreadLocalRepo.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobExecuteContextThreadLocalRepo.java index 7259575d5a..e2e0e42297 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/colddata/JobExecuteContextThreadLocalRepo.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobExecuteContextThreadLocalRepo.java @@ -22,9 +22,8 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.execute.colddata; +package com.tencent.bk.job.execute.common.context; -import com.tencent.bk.job.execute.common.context.JobExecuteContext; import lombok.extern.slf4j.Slf4j; /** @@ -36,12 +35,16 @@ public class JobExecuteContextThreadLocalRepo { public static final ThreadLocal HOLDER = new ThreadLocal<>(); public static void set(JobExecuteContext context) { - log.info("SetJobExecuteContextThreadLocalRepo, context: {}", context); + if (log.isDebugEnabled()) { + log.info("SetJobExecuteContextThreadLocalRepo, context: {}", context); + } HOLDER.set(context); } public static void unset() { - log.info("RemoveJobExecuteContext"); + if (log.isDebugEnabled()) { + log.debug("RemoveFromJobExecuteContextThreadLocalRepo"); + } HOLDER.remove(); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextHttpInterceptor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextHttpInterceptor.java index 474dab4517..b399788c5c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextHttpInterceptor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextHttpInterceptor.java @@ -28,8 +28,8 @@ import com.tencent.bk.job.common.constant.InterceptorOrder; import com.tencent.bk.job.common.context.JobContext; import com.tencent.bk.job.common.util.JobContextUtil; -import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; import com.tencent.bk.job.execute.common.context.JobExecuteContext; +import com.tencent.bk.job.execute.common.context.JobExecuteContextThreadLocalRepo; import lombok.extern.slf4j.Slf4j; import org.springframework.lang.NonNull; import org.springframework.stereotype.Component; @@ -39,7 +39,7 @@ import javax.servlet.http.HttpServletResponse; /** - * JobContext 初始化 + * 在请求处理之前，初始话作业执行上下文(JobExecuteContext）；在请求结束之前，删除 JobExecuteContext */ @Slf4j @Component @@ -56,11 +56,13 @@ public boolean preHandle(@NonNull HttpServletRequest request, JobContext jobContext = JobContextUtil.getContext(); if (jobContext != null) { - log.info("JobExecuteContextInterceptor -> Set JobExecuteContext"); JobExecuteContext jobExecuteContext = new JobExecuteContext(); jobExecuteContext.setResourceScope(jobContext.getAppResourceScope()); jobExecuteContext.setUsername(jobContext.getUsername()); JobExecuteContextThreadLocalRepo.set(jobExecuteContext); + if (log.isDebugEnabled()) { + log.debug("JobExecuteContextInterceptor -> Set JobExecuteContext : {}", jobExecuteContext); + } } return true; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextPropagateMqInterceptor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextPropagateMqInterceptor.java index 23e830fdac..44f5fe57e2 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextPropagateMqInterceptor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextPropagateMqInterceptor.java @@ -25,8 +25,8 @@ package com.tencent.bk.job.execute.common.interceptor; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; import com.tencent.bk.job.execute.common.context.JobExecuteContext; +import com.tencent.bk.job.execute.common.context.JobExecuteContextThreadLocalRepo; import lombok.extern.slf4j.Slf4j; import org.springframework.integration.config.GlobalChannelInterceptor; import org.springframework.messaging.Message; @@ -35,6 +35,9 @@ import org.springframework.messaging.support.MessageBuilder; import org.springframework.stereotype.Component; +/** + * Mq 消息全局拦截 - JobExecuteContext 上下文传递 + */ @Component @Slf4j @GlobalChannelInterceptor @@ -42,12 +45,13 @@ public class JobExecuteContextPropagateMqInterceptor implements ChannelIntercept @Override public Message preSend(Message message, MessageChannel channel) { - log.info("preSend"); JobExecuteContext context = JobExecuteContextThreadLocalRepo.get(); if (context == null) { return ChannelInterceptor.super.preSend(message, channel); } - log.info("setJobExecuteContextMessageHeader, context: {}", context); + if (log.isDebugEnabled()) { + log.debug("SetJobExecuteContextMessageHeader, context: {}", context); + } Message newMessage = MessageBuilder.fromMessage(message) .setHeader(JobExecuteContext.KEY, JsonUtils.toJson(context)) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceVariableDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceVariableDAO.java index a72a3d1132..b3084eb86b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceVariableDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceVariableDAO.java @@ -43,12 +43,15 @@ public interface StepInstanceVariableDAO { /** * 获取步骤变量值 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param variableValueType 参数值类型 * @return 步骤变量值 */ - StepInstanceVariableValuesDTO getStepVariableValues(long stepInstanceId, int executeCount, + StepInstanceVariableValuesDTO getStepVariableValues(Long taskInstanceId, + long stepInstanceId, + int executeCount, VariableValueTypeEnum variableValueType); /** diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceVariableDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceVariableDAOImpl.java index 5d36f36170..70f83927de 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceVariableDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceVariableDAOImpl.java @@ -45,8 +45,8 @@ @Repository public class StepInstanceVariableDAOImpl implements StepInstanceVariableDAO { private static final StepInstanceVariable TABLE = StepInstanceVariable.STEP_INSTANCE_VARIABLE; - private DSLContext ctx; - private TableField[] FIELDS = {TABLE.TASK_INSTANCE_ID, TABLE.STEP_INSTANCE_ID, + private final DSLContext ctx; + private final TableField[] FIELDS = {TABLE.TASK_INSTANCE_ID, TABLE.STEP_INSTANCE_ID, TABLE.EXECUTE_COUNT, TABLE.TYPE, TABLE.PARAM_VALUES}; @@ -65,11 +65,15 @@ public void saveVariableValues(StepInstanceVariableValuesDTO variableValues) { } @Override - public StepInstanceVariableValuesDTO getStepVariableValues(long stepInstanceId, int executeCount, + public StepInstanceVariableValuesDTO getStepVariableValues(Long taskInstanceId, + long stepInstanceId, + int executeCount, VariableValueTypeEnum variableValueType) { Record record = ctx.select(FIELDS) .from(TABLE) - .where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) + .where(TaskInstanceIdDynamicCondition.build(taskInstanceId, + TABLE.TASK_INSTANCE_ID::eq)) + .and(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) .and(TABLE.EXECUTE_COUNT.eq(executeCount)) .and(TABLE.TYPE.eq(JooqDataTypeUtil.toByte(variableValueType.getValue()))) .limit(1) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java index acde5610c4..de31fef402 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java @@ -30,8 +30,8 @@ import com.tencent.bk.job.common.util.feature.FeatureIdConstants; import com.tencent.bk.job.common.util.feature.FeatureToggle; import com.tencent.bk.job.common.util.feature.ToggleStrategyContextParams; -import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; import com.tencent.bk.job.execute.common.context.JobExecuteContext; +import com.tencent.bk.job.execute.common.context.JobExecuteContextThreadLocalRepo; import lombok.extern.slf4j.Slf4j; import org.jooq.Condition; import org.jooq.impl.DSL; @@ -53,14 +53,16 @@ public static Condition build(Long taskInstanceId, Function taskInstanceIdConditionBuilder) { JobExecuteContext jobExecuteContext = JobExecuteContextThreadLocalRepo.get(); if (jobExecuteContext == null) { - log.warn("TaskInstanceIdDynamicCondition : Empty JobExecuteContext!"); - // JobExecuteContext 正常应该不会为 null 。为了不影响，忽略错误,直接返回 TRUE Condition + log.info("TaskInstanceIdDynamicCondition : Empty JobExecuteContext!"); + // JobExecuteContext 正常应该不会为 null 。为了不影响请求正常处理，忽略错误,直接返回 TRUE Condition + // (不会影响 DAO 查询，task_instance_id 仅作为分片功能实用，实际业务数据关系并不强依赖 task_instance_id) return DSL.trueCondition(); } ResourceScope resourceScope = jobExecuteContext.getResourceScope(); if (resourceScope == null) { - log.warn("TaskInstanceIdDynamicCondition : Empty resource scope!"); - // 为了不影响兼容性，忽略错误 + log.info("TaskInstanceIdDynamicCondition : Empty resource scope!"); + // 无法根据业务决定是否使用 task_instance_id 作为查询条件。为了不影响请求正常处理,直接返回 TRUE Condition + // (不会影响 DAO 查询，task_instance_id 仅作为分片功能实用，实际业务数据关系并不强依赖 task_instance_id) return DSL.trueCondition(); } if (FeatureToggle.checkFeature( @@ -69,7 +71,7 @@ public static Condition build(Long taskInstanceId, .addContextParam(ToggleStrategyContextParams.CTX_PARAM_RESOURCE_SCOPE, JobContextUtil.getAppResourceScope()))) { if (taskInstanceId == null || taskInstanceId <= 0L) { - log.warn("TaskInstanceIdDynamicCondition : Invalid taskInstanceId {}", taskInstanceId); + log.info("TaskInstanceIdDynamicCondition : Invalid taskInstanceId {}", taskInstanceId); // 为了不影响兼容性，忽略错误 return DSL.trueCondition(); } else { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/BaseJobMqListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/BaseJobMqListener.java index 06e10dcff9..aedf83078c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/BaseJobMqListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/BaseJobMqListener.java @@ -25,8 +25,8 @@ package com.tencent.bk.job.execute.engine.listener; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; import com.tencent.bk.job.execute.common.context.JobExecuteContext; +import com.tencent.bk.job.execute.common.context.JobExecuteContextThreadLocalRepo; import com.tencent.bk.job.execute.engine.listener.event.JobMessage; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ResultHandleManager.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ResultHandleManager.java index 7f993a5b9d..8bd5778d78 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ResultHandleManager.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ResultHandleManager.java @@ -24,7 +24,7 @@ package com.tencent.bk.job.execute.engine.result; -import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; +import com.tencent.bk.job.execute.common.context.JobExecuteContextThreadLocalRepo; import com.tencent.bk.job.execute.common.exception.MessageHandlerUnavailableException; import com.tencent.bk.job.execute.common.ha.DestroyOrder; import com.tencent.bk.job.execute.config.JobExecuteConfig; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScheduledContinuousResultHandleTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScheduledContinuousResultHandleTask.java index 68ae732112..e11f496a65 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScheduledContinuousResultHandleTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScheduledContinuousResultHandleTask.java @@ -24,8 +24,8 @@ package com.tencent.bk.job.execute.engine.result; -import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; import com.tencent.bk.job.execute.common.context.JobExecuteContext; +import com.tencent.bk.job.execute.common.context.JobExecuteContextThreadLocalRepo; import com.tencent.bk.job.execute.engine.result.ha.ResultHandleLimiter; import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; import com.tencent.bk.job.execute.monitor.ExecuteMetricNames; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepOperationDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepOperationDTO.java index e354bae898..5a39a34056 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepOperationDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepOperationDTO.java @@ -34,7 +34,7 @@ public class StepOperationDTO { /** * 步骤实例ID */ - private long taskInstanceId; + private Long taskInstanceId; /** * 步骤实例ID */ diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskExecuteService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskExecuteService.java index 21b8e937ef..134d9376da 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskExecuteService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskExecuteService.java @@ -24,7 +24,6 @@ package com.tencent.bk.job.execute.service; -import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.execute.constants.TaskOperationEnum; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; import com.tencent.bk.job.execute.model.FastTaskDTO; @@ -90,7 +89,6 @@ TaskInstanceDTO createTaskInstanceForRedo(Long appId, * @param operator 操作者 * @param stepOperation 步骤操作 * @return 执行次数 - * @throws ServiceException */ Integer doStepOperation(Long appId, String operator, StepOperationDTO stepOperation); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java index 33058b9b36..5a816eb50d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java @@ -52,12 +52,12 @@ import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.execute.audit.ExecuteJobAuditEventBuilder; import com.tencent.bk.job.execute.auth.ExecuteAuthService; -import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; import com.tencent.bk.job.execute.common.context.JobExecuteContext; +import com.tencent.bk.job.execute.common.context.JobExecuteContextThreadLocalRepo; import com.tencent.bk.job.execute.common.context.JobInstanceContext; import com.tencent.bk.job.execute.common.converter.StepTypeExecuteTypeConverter; import com.tencent.bk.job.execute.config.JobExecuteConfig; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceServiceImpl.java index 1c77d43938..38c3934d23 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceServiceImpl.java @@ -35,9 +35,9 @@ import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.auth.ExecuteAuthService; -import com.tencent.bk.job.execute.colddata.JobExecuteContextThreadLocalRepo; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.common.context.JobExecuteContext; +import com.tencent.bk.job.execute.common.context.JobExecuteContextThreadLocalRepo; import com.tencent.bk.job.execute.common.context.JobInstanceContext; import com.tencent.bk.job.execute.dao.TaskInstanceDAO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; From 46be8299ee03d8beb671e0dafcb3598ec148541f Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Thu, 15 Aug 2024 14:36:48 +0800 Subject: [PATCH 21/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../FileAgentTaskDAOImplIntegrationTest.java | 27 ++- ...ScriptAgentTaskDAOImplIntegrationTest.java | 30 ++- .../resources/init_file_agent_task_data.sql | 17 +- .../resources/init_script_agent_task_data.sql | 21 +- .../bk/job/execute/dao/FileAgentTaskDAO.java | 63 ++++-- .../job/execute/dao/ScriptAgentTaskDAO.java | 37 ++- .../dao/impl/FileAgentTaskDAOImpl.java | 112 ++++++---- .../dao/impl/ScriptAgentTaskDAOImpl.java | 120 ++++++---- .../FileExecuteObjectTaskServiceImpl.java | 45 ++-- .../ScriptExecuteObjectTaskServiceImpl.java | 45 +++- ...ob_execute_20240520-1000_V3.10.0_mysql.sql | 211 ++++++++++++++---- 11 files changed, 508 insertions(+), 220 deletions(-) diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImplIntegrationTest.java index 8b67ea9823..ce0f9aeec7 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImplIntegrationTest.java @@ -59,12 +59,15 @@ public class FileAgentTaskDAOImplIntegrationTest { @DisplayName("根据主机ID获取Agent任务") public void testGetAgentTaskByHostId() { long hostId = 101L; + long taskInstanceId = 1L; long stepInstanceId = 1L; int executeCount = 0; int batch = 1; FileTaskModeEnum mode = FileTaskModeEnum.UPLOAD; - ExecuteObjectTask agentTask = fileAgentTaskDAO.getAgentTaskByHostId(stepInstanceId, executeCount, batch, mode, hostId); + ExecuteObjectTask agentTask = fileAgentTaskDAO.getAgentTaskByHostId(taskInstanceId, + stepInstanceId, executeCount, batch, mode, hostId); + assertThat(agentTask.getTaskInstanceId()).isEqualTo(taskInstanceId); assertThat(agentTask.getStepInstanceId()).isEqualTo(stepInstanceId); assertThat(agentTask.getExecuteCount()).isEqualTo(executeCount); assertThat(agentTask.getBatch()).isEqualTo(batch); @@ -84,6 +87,7 @@ public void testGetAgentTaskByHostId() { public void testBatchSaveAgentTasks() { List agentTaskList = new ArrayList<>(); ExecuteObjectTask agentTask1 = new ExecuteObjectTask(); + agentTask1.setTaskInstanceId(100L); agentTask1.setStepInstanceId(100L); agentTask1.setExecuteCount(1); agentTask1.setActualExecuteCount(1); @@ -100,6 +104,7 @@ public void testBatchSaveAgentTasks() { agentTaskList.add(agentTask1); ExecuteObjectTask agentTask2 = new ExecuteObjectTask(); + agentTask2.setTaskInstanceId(100L); agentTask2.setStepInstanceId(100L); agentTask2.setExecuteCount(1); agentTask2.setActualExecuteCount(1); @@ -118,8 +123,9 @@ public void testBatchSaveAgentTasks() { fileAgentTaskDAO.batchSaveAgentTasks(agentTaskList); - ExecuteObjectTask agentTask1Return = fileAgentTaskDAO.getAgentTaskByHostId(100L, 1, 1, + ExecuteObjectTask agentTask1Return = fileAgentTaskDAO.getAgentTaskByHostId(100L, 100L, 1, 1, FileTaskModeEnum.UPLOAD, 101L); + assertThat(agentTask1Return.getTaskInstanceId()).isEqualTo(100L); assertThat(agentTask1Return.getStepInstanceId()).isEqualTo(100L); assertThat(agentTask1Return.getExecuteCount()).isEqualTo(1L); assertThat(agentTask1Return.getActualExecuteCount()).isEqualTo(1L); @@ -135,8 +141,9 @@ public void testBatchSaveAgentTasks() { assertThat(agentTask1Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.AGENT_ERROR); - ExecuteObjectTask agentTask2Return = fileAgentTaskDAO.getAgentTaskByHostId(100L, 1, 1, + ExecuteObjectTask agentTask2Return = fileAgentTaskDAO.getAgentTaskByHostId(100L, 100L, 1, 1, FileTaskModeEnum.DOWNLOAD, 102L); + assertThat(agentTask2Return.getTaskInstanceId()).isEqualTo(100L); assertThat(agentTask2Return.getStepInstanceId()).isEqualTo(100L); assertThat(agentTask2Return.getExecuteCount()).isEqualTo(1L); assertThat(agentTask2Return.getActualExecuteCount()).isEqualTo(1L); @@ -157,6 +164,7 @@ public void testBatchSaveAgentTasks() { public void testBatchUpdateAgentTasks() { List agentTaskList = new ArrayList<>(); ExecuteObjectTask agentTask1 = new ExecuteObjectTask(); + agentTask1.setTaskInstanceId(1L); agentTask1.setStepInstanceId(1L); agentTask1.setExecuteCount(0); agentTask1.setBatch(2); @@ -172,6 +180,7 @@ public void testBatchUpdateAgentTasks() { agentTaskList.add(agentTask1); ExecuteObjectTask agentTask2 = new ExecuteObjectTask(); + agentTask2.setTaskInstanceId(1L); agentTask2.setStepInstanceId(1L); agentTask2.setExecuteCount(0); agentTask2.setBatch(2); @@ -189,8 +198,9 @@ public void testBatchUpdateAgentTasks() { fileAgentTaskDAO.batchUpdateAgentTasks(agentTaskList); - ExecuteObjectTask agentTask1Return = fileAgentTaskDAO.getAgentTaskByHostId(1L, 0, 2, + ExecuteObjectTask agentTask1Return = fileAgentTaskDAO.getAgentTaskByHostId(1L, 1L, 0, 2, FileTaskModeEnum.UPLOAD, 101L); + assertThat(agentTask1Return.getTaskInstanceId()).isEqualTo(1L); assertThat(agentTask1Return.getStepInstanceId()).isEqualTo(1L); assertThat(agentTask1Return.getExecuteCount()).isEqualTo(0L); assertThat(agentTask1Return.getBatch()).isEqualTo(2); @@ -204,8 +214,9 @@ public void testBatchUpdateAgentTasks() { assertThat(agentTask1Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.AGENT_ERROR); - ExecuteObjectTask agentTask2Return = fileAgentTaskDAO.getAgentTaskByHostId(1L, 0, 2, + ExecuteObjectTask agentTask2Return = fileAgentTaskDAO.getAgentTaskByHostId(1L, 1L, 0, 2, FileTaskModeEnum.DOWNLOAD, 103L); + assertThat(agentTask2Return.getTaskInstanceId()).isEqualTo(1L); assertThat(agentTask2Return.getStepInstanceId()).isEqualTo(1L); assertThat(agentTask2Return.getExecuteCount()).isEqualTo(0L); assertThat(agentTask2Return.getBatch()).isEqualTo(2); @@ -222,7 +233,7 @@ public void testBatchUpdateAgentTasks() { @Test @DisplayName("Agent任务结果分组") public void listResultGroups() { - List resultGroups = fileAgentTaskDAO.listResultGroups(1L, 0, null); + List resultGroups = fileAgentTaskDAO.listResultGroups(1L, 1L, 0, null); assertThat(resultGroups.size()).isEqualTo(2); assertThat(resultGroups).extracting("status").containsOnly(9, 11); @@ -235,7 +246,7 @@ public void listResultGroups() { } // 根据滚动执行批次查询 - resultGroups = fileAgentTaskDAO.listResultGroups(1L, 0, 2); + resultGroups = fileAgentTaskDAO.listResultGroups(1L, 1L, 0, 2); assertThat(resultGroups.size()).isEqualTo(2); assertThat(resultGroups).extracting("status").containsOnly(9, 11); @@ -250,7 +261,7 @@ public void listResultGroups() { @Test public void testListAgentTaskByResultGroup() { - List agentTasks = fileAgentTaskDAO.listAgentTaskByResultGroup(1L, 0, 2, 9); + List agentTasks = fileAgentTaskDAO.listAgentTaskByResultGroup(1L, 1L, 0, 2, 9); assertThat(agentTasks.size()).isEqualTo(1); assertThat(agentTasks.get(0).getStepInstanceId()).isEqualTo(1L); assertThat(agentTasks.get(0).getExecuteCount()).isEqualTo(0); diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImplIntegrationTest.java index 83430b39c6..33d4b9b0cb 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImplIntegrationTest.java @@ -58,12 +58,14 @@ public class ScriptAgentTaskDAOImplIntegrationTest { @DisplayName("根据主机获取Agent任务") public void testGetAgentTaskByHostId() { long hostId = 101L; + long taskInstanceId = 1L; long stepInstanceId = 1L; int executeCount = 0; int batch = 1; - ExecuteObjectTask agentTask = scriptAgentTaskDAO.getAgentTaskByHostId(stepInstanceId, executeCount, batch, - hostId); + ExecuteObjectTask agentTask = scriptAgentTaskDAO.getAgentTaskByHostId(taskInstanceId, + stepInstanceId, executeCount, batch, hostId); + assertThat(agentTask.getTaskInstanceId()).isEqualTo(taskInstanceId); assertThat(agentTask.getStepInstanceId()).isEqualTo(stepInstanceId); assertThat(agentTask.getExecuteCount()).isEqualTo(executeCount); assertThat(agentTask.getBatch()).isEqualTo(batch); @@ -87,6 +89,7 @@ public void testGetAgentTaskByHostId() { public void testBatchSaveAgentTasks() { List agentTaskList = new ArrayList<>(); ExecuteObjectTask agentTask1 = new ExecuteObjectTask(); + agentTask1.setTaskInstanceId(100L); agentTask1.setStepInstanceId(100L); agentTask1.setExecuteCount(1); agentTask1.setActualExecuteCount(1); @@ -104,6 +107,7 @@ public void testBatchSaveAgentTasks() { agentTaskList.add(agentTask1); ExecuteObjectTask agentTask2 = new ExecuteObjectTask(); + agentTask2.setTaskInstanceId(100L); agentTask2.setStepInstanceId(100L); agentTask2.setExecuteCount(1); agentTask2.setActualExecuteCount(1); @@ -124,7 +128,8 @@ public void testBatchSaveAgentTasks() { scriptAgentTaskDAO.batchSaveAgentTasks(agentTaskList); - ExecuteObjectTask agentTask1Return = scriptAgentTaskDAO.getAgentTaskByHostId(100L, 1, 1, 101L); + ExecuteObjectTask agentTask1Return = scriptAgentTaskDAO.getAgentTaskByHostId(100L, 100L, 1, 1, 101L); + assertThat(agentTask1Return.getTaskInstanceId()).isEqualTo(100L); assertThat(agentTask1Return.getStepInstanceId()).isEqualTo(100L); assertThat(agentTask1Return.getExecuteCount()).isEqualTo(1L); assertThat(agentTask1Return.getActualExecuteCount()).isEqualTo(1L); @@ -141,7 +146,8 @@ public void testBatchSaveAgentTasks() { assertThat(agentTask1Return.getExitCode()).isEqualTo(1); - ExecuteObjectTask agentTask2Return = scriptAgentTaskDAO.getAgentTaskByHostId(100L, 1, 1, 102L); + ExecuteObjectTask agentTask2Return = scriptAgentTaskDAO.getAgentTaskByHostId(100L, 100L, 1, 1, 102L); + assertThat(agentTask2Return.getTaskInstanceId()).isEqualTo(100L); assertThat(agentTask2Return.getStepInstanceId()).isEqualTo(100L); assertThat(agentTask2Return.getExecuteCount()).isEqualTo(1L); assertThat(agentTask2Return.getActualExecuteCount()).isEqualTo(1L); @@ -163,6 +169,7 @@ public void testBatchSaveAgentTasks() { public void testBatchUpdateAgentTasks() { List agentTaskList = new ArrayList<>(); ExecuteObjectTask agentTask1 = new ExecuteObjectTask(); + agentTask1.setTaskInstanceId(1L); agentTask1.setStepInstanceId(1L); agentTask1.setExecuteCount(0); agentTask1.setBatch(3); @@ -179,6 +186,7 @@ public void testBatchUpdateAgentTasks() { agentTaskList.add(agentTask1); ExecuteObjectTask agentTask2 = new ExecuteObjectTask(); + agentTask2.setTaskInstanceId(1L); agentTask2.setStepInstanceId(1L); agentTask2.setExecuteCount(0); agentTask2.setBatch(3); @@ -198,7 +206,8 @@ public void testBatchUpdateAgentTasks() { scriptAgentTaskDAO.batchUpdateAgentTasks(agentTaskList); - ExecuteObjectTask agentTask1Return = scriptAgentTaskDAO.getAgentTaskByHostId(1L, 0, 3, 103L); + ExecuteObjectTask agentTask1Return = scriptAgentTaskDAO.getAgentTaskByHostId(1L, 1L, 0, 3, 103L); + assertThat(agentTask1Return.getTaskInstanceId()).isEqualTo(1L); assertThat(agentTask1Return.getStepInstanceId()).isEqualTo(1L); assertThat(agentTask1Return.getExecuteCount()).isEqualTo(0L); assertThat(agentTask1Return.getBatch()).isEqualTo(3); @@ -214,7 +223,8 @@ public void testBatchUpdateAgentTasks() { assertThat(agentTask1Return.getExitCode()).isEqualTo(1); - ExecuteObjectTask agentTask2Return = scriptAgentTaskDAO.getAgentTaskByHostId(1L, 0, 3, 104L); + ExecuteObjectTask agentTask2Return = scriptAgentTaskDAO.getAgentTaskByHostId(1L, 1L, 0, 3, 104L); + assertThat(agentTask2Return.getTaskInstanceId()).isEqualTo(1L); assertThat(agentTask2Return.getStepInstanceId()).isEqualTo(1L); assertThat(agentTask2Return.getExecuteCount()).isEqualTo(0L); assertThat(agentTask2Return.getBatch()).isEqualTo(3); @@ -232,14 +242,14 @@ public void testBatchUpdateAgentTasks() { @Test public void testGetSuccessIpCount() { - Integer count = scriptAgentTaskDAO.getSuccessAgentTaskCount(1L, 0); + Integer count = scriptAgentTaskDAO.getSuccessAgentTaskCount(1L, 1L, 0); assertThat(count).isEqualTo(4); } @Test @DisplayName("Agent任务结果分组") public void listResultGroups() { - List resultGroups = scriptAgentTaskDAO.listResultGroups(1L, 0, null); + List resultGroups = scriptAgentTaskDAO.listResultGroups(1L, 1L, 0, null); assertThat(resultGroups.size()).isEqualTo(2); assertThat(resultGroups).extracting("status").containsOnly(9, 11); @@ -253,7 +263,7 @@ public void listResultGroups() { } // 根据滚动执行批次查询 - resultGroups = scriptAgentTaskDAO.listResultGroups(1L, 0, 3); + resultGroups = scriptAgentTaskDAO.listResultGroups(1L, 1L, 0, 3); assertThat(resultGroups.size()).isEqualTo(2); assertThat(resultGroups).extracting("status").containsOnly(9, 11); @@ -269,7 +279,7 @@ public void listResultGroups() { @Test public void testListAgentTaskByResultGroup() { - List agentTasks = scriptAgentTaskDAO.listAgentTaskByResultGroup(1L, 0, 1, 9, "succ"); + List agentTasks = scriptAgentTaskDAO.listAgentTaskByResultGroup(1L, 1L, 0, 1, 9, "succ"); assertThat(agentTasks.size()).isEqualTo(1); assertThat(agentTasks.get(0).getStepInstanceId()).isEqualTo(1L); assertThat(agentTasks.get(0).getExecuteCount()).isEqualTo(0); diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_file_agent_task_data.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_file_agent_task_data.sql index ab4d5a8a5d..f2ce90448c 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/init_file_agent_task_data.sql +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_file_agent_task_data.sql @@ -25,7 +25,8 @@ TRUNCATE TABLE job_execute.gse_file_agent_task; INSERT INTO job_execute.gse_file_agent_task - (step_instance_id, + (task_instance_id, + step_instance_id, execute_count, batch, mode, @@ -38,10 +39,10 @@ INSERT INTO job_execute.gse_file_agent_task total_time, error_code) VALUES - (1,0,1,0,101,'0:127.0.0.1',1,9,1565767148000,1565767149000,1000,0), - (1,0,1,1,102,'0:127.0.0.2',2,9,1565767148000,1565767149000,1000,0), - (1,0,2,0,101,'0:127.0.0.1',3,9,1565767150000,1565767151000,1000,0), - (1,0,2,1,103,'0:127.0.0.3',3,9,1565767150000,1565767151000,1000,0), - (1,0,2,1,104,'0:127.0.0.4',3,11,1565767150000,1565767151000,1000,99), - (2,0,0,0,101,'0:127.0.0.1',4,9,1565767148000,1565767149000,1000,0), - (2,0,0,1,102,'0:127.0.0.2',4,9,1565767148000,1565767149000,1000,0); + (1,1,0,1,0,101,'0:127.0.0.1',1,9,1565767148000,1565767149000,1000,0), + (1,1,0,1,1,102,'0:127.0.0.2',2,9,1565767148000,1565767149000,1000,0), + (1,1,0,2,0,101,'0:127.0.0.1',3,9,1565767150000,1565767151000,1000,0), + (1,1,0,2,1,103,'0:127.0.0.3',3,9,1565767150000,1565767151000,1000,0), + (1,1,0,2,1,104,'0:127.0.0.4',3,11,1565767150000,1565767151000,1000,99), + (2,2,0,0,0,101,'0:127.0.0.1',4,9,1565767148000,1565767149000,1000,0), + (2,2,0,0,1,102,'0:127.0.0.2',4,9,1565767148000,1565767149000,1000,0); diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_script_agent_task_data.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_script_agent_task_data.sql index f8d2fc25d3..4082e32c0a 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/init_script_agent_task_data.sql +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_script_agent_task_data.sql @@ -25,7 +25,8 @@ TRUNCATE TABLE job_execute.gse_script_agent_task; INSERT INTO job_execute.gse_script_agent_task - (step_instance_id, + (task_instance_id, + step_instance_id, execute_count, batch, host_id, @@ -40,12 +41,12 @@ INSERT INTO job_execute.gse_script_agent_task tag, log_offset) VALUES - (1,0,1,101,'0:127.0.0.1',1,9,1565767148000,1565767149000,1316,0,0,'succ',0), - (1,0,2,102,'0:127.0.0.2',2,9,1565767148000,1565767149000,1211,0,0,'succ',0), - (1,0,3,103,'0:127.0.0.3',3,9,1565767148000,1565767149000,1211,0,0,'succ',0), - (1,0,3,104,'0:127.0.0.4',3,9,1565767148000,1565767149000,1211,0,0,'succ',0), - (1,0,3,105,'0:127.0.0.5',3,11,1565767148000,1565767149000,1211,0,0,'fail',0), - (2,0,0,101,'0:127.0.0.1',4,9,1565767148000,1565767209000,1211,0,0,'succ',0), - (2,1,0,101,'0:127.0.0.1',4,9,1565766610000,1565767211000,1215,0,0,'succ',0), - (3,0,0,101,'0:127.0.0.1',5,9,1565766610000,1565767211000,1215,0,0,'succ',0), - (3,0,0,102,'0:127.0.0.2',5,9,1565766610000,1565767211000,1215,0,0,'succ',0); + (1,1,0,1,101,'0:127.0.0.1',1,9,1565767148000,1565767149000,1316,0,0,'succ',0), + (1,1,0,2,102,'0:127.0.0.2',2,9,1565767148000,1565767149000,1211,0,0,'succ',0), + (1,1,0,3,103,'0:127.0.0.3',3,9,1565767148000,1565767149000,1211,0,0,'succ',0), + (1,1,0,3,104,'0:127.0.0.4',3,9,1565767148000,1565767149000,1211,0,0,'succ',0), + (1,1,0,3,105,'0:127.0.0.5',3,11,1565767148000,1565767149000,1211,0,0,'fail',0), + (2,2,0,0,101,'0:127.0.0.1',4,9,1565767148000,1565767209000,1211,0,0,'succ',0), + (2,2,1,0,101,'0:127.0.0.1',4,9,1565766610000,1565767211000,1215,0,0,'succ',0), + (3,3,0,0,101,'0:127.0.0.1',5,9,1565766610000,1565767211000,1215,0,0,'succ',0), + (3,3,0,0,102,'0:127.0.0.2',5,9,1565766610000,1565767211000,1215,0,0,'succ',0); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java index b705508699..fb10c96499 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java @@ -57,39 +57,47 @@ public interface FileAgentTaskDAO { /** * 获取步骤成功执行的Agent任务数量 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @return 步骤成功执行的Agent任务数量 */ - int getSuccessAgentTaskCount(long stepInstanceId, int executeCount); + int getSuccessAgentTaskCount(Long taskInstanceId, long stepInstanceId, int executeCount); /** * 查询执行结果分组 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 * @return 执行结果分组 */ - List listResultGroups(long stepInstanceId, int executeCount, Integer batch); + List listResultGroups(Long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer batch); /** * 根据执行结果查询Agent任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 * @param status 任务状态 * @return Agent任务 */ - List listAgentTaskByResultGroup(Long stepInstanceId, - Integer executeCount, - Integer batch, - Integer status); + List listAgentTaskByResultGroup(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status); /** * 根据执行结果查询Agent任务(排序、限制返回数量) * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 @@ -99,39 +107,44 @@ List listAgentTaskByResultGroup(Long stepInstanceId, * @param order 排序方式 * @return Agent任务 */ - List listAgentTaskByResultGroup(Long stepInstanceId, - Integer executeCount, - Integer batch, - Integer status, - Integer limit, - String orderField, - Order order); + List listAgentTaskByResultGroup(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status, + Integer limit, + String orderField, + Order order); /** * 获取agent任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；传入null或者0将忽略该参数 * @param fileTaskMode 文件分发任务模式;传入null表示忽略该过滤条件 * @return agent任务 */ - List listAgentTasks(Long stepInstanceId, - Integer executeCount, - Integer batch, - FileTaskModeEnum fileTaskMode); + List listAgentTasks(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + FileTaskModeEnum fileTaskMode); /** * 根据GSE任务ID获取agent任务 * - * @param gseTaskId GSE任务ID + * @param taskInstanceId 作业实例 ID + * @param gseTaskId GSE任务ID * @return agent任务 */ - List listAgentTasksByGseTaskId(Long gseTaskId); + List listAgentTasksByGseTaskId(Long taskInstanceId, Long gseTaskId); /** * 根据hostId查询Agent任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；传入null或者0将忽略该参数 @@ -139,20 +152,26 @@ List listAgentTasks(Long stepInstanceId, * @param hostId 主机ID * @return Agent任务 */ - ExecuteObjectTask getAgentTaskByHostId(Long stepInstanceId, Integer executeCount, Integer batch, - FileTaskModeEnum mode, long hostId); + ExecuteObjectTask getAgentTaskByHostId(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + FileTaskModeEnum mode, + long hostId); /** * 批量更新AgentTask的字段 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 条件 - 步骤实例ID * @param executeCount 条件 - 重试次数 * @param batch 条件 - 滚动执行批次；传入null将忽略该条件 * @param actualExecuteCount 值 - Agent任务实际执行的步骤重试次数；如果传入null，则不更新 * @param gseTaskId 值 - Agent任务对应的GSE_TASK_ID；如果传入null，则不更新 */ - void updateAgentTaskFields(long stepInstanceId, + void updateAgentTaskFields(Long taskInstanceId, + long stepInstanceId, int executeCount, Integer batch, Integer actualExecuteCount, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java index 9c04f220e8..8611622398 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java @@ -56,25 +56,31 @@ public interface ScriptAgentTaskDAO { /** * 获取步骤成功执行的Agent任务数量 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @return 步骤成功执行的Agent任务数量 */ - int getSuccessAgentTaskCount(long stepInstanceId, int executeCount); + int getSuccessAgentTaskCount(Long taskInstanceId, long stepInstanceId, int executeCount); /** * 查询执行结果分组 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 * @return 执行结果分组 */ - List listResultGroups(long stepInstanceId, int executeCount, Integer batch); + List listResultGroups(Long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer batch); /** * 根据执行结果查询Agent任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 @@ -82,7 +88,8 @@ public interface ScriptAgentTaskDAO { * @param tag 用户自定义分组标签 * @return Agent任务 */ - List listAgentTaskByResultGroup(Long stepInstanceId, + List listAgentTaskByResultGroup(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, Integer status, @@ -91,6 +98,7 @@ List listAgentTaskByResultGroup(Long stepInstanceId, /** * 根据执行结果查询Agent任务(排序、限制返回数量) * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 @@ -101,7 +109,8 @@ List listAgentTaskByResultGroup(Long stepInstanceId, * @param order 排序方式 * @return Agent任务 */ - List listAgentTaskByResultGroup(Long stepInstanceId, + List listAgentTaskByResultGroup(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, Integer status, @@ -113,44 +122,54 @@ List listAgentTaskByResultGroup(Long stepInstanceId, /** * 获取Agent任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；传入null或者0将忽略该参数 * @return Agent任务信息 */ - List listAgentTasks(Long stepInstanceId, + List listAgentTasks(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch); /** * 根据GSE任务ID获取Agent任务 * - * @param gseTaskId GSE任务ID + * @param taskInstanceId 作业实例 ID + * @param gseTaskId GSE任务ID * @return Agent任务 */ - List listAgentTasksByGseTaskId(Long gseTaskId); + List listAgentTasksByGseTaskId(Long taskInstanceId, Long gseTaskId); /** * 根据hostId查询Agent任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；传入null或者0将忽略该参数 * @param hostId 主机ID * @return Agent任务 */ - ExecuteObjectTask getAgentTaskByHostId(Long stepInstanceId, Integer executeCount, Integer batch, long hostId); + ExecuteObjectTask getAgentTaskByHostId(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + long hostId); /** * 部分更新AgentTask的字段 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 条件 - 步骤实例ID * @param executeCount 条件 - 重试次数 * @param batch 条件 - 滚动执行批次；传入null将忽略该条件 * @param actualExecuteCount 值 - Agent任务实际执行的步骤重试次数；如果传入null，则不更新 * @param gseTaskId 值 - Agent任务对应的GSE_TASK_ID；如果传入null，则不更新 */ - void updateAgentTaskFields(long stepInstanceId, + void updateAgentTaskFields(Long taskInstanceId, + long stepInstanceId, int executeCount, Integer batch, Integer actualExecuteCount, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java index fc8f0fe766..d6777478d6 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java @@ -67,6 +67,7 @@ public class FileAgentTaskDAOImpl implements FileAgentTaskDAO { private static final GseFileAgentTask T_GSE_FILE_AGENT_TASK = GseFileAgentTask.GSE_FILE_AGENT_TASK; private static final TableField[] ALL_FIELDS = { + T_GSE_FILE_AGENT_TASK.TASK_INSTANCE_ID, T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID, T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT, T_GSE_FILE_AGENT_TASK.ACTUAL_EXECUTE_COUNT, @@ -91,26 +92,28 @@ public FileAgentTaskDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext CTX @Override public void batchSaveAgentTasks(Collection agentTasks) { - String sql = "insert into gse_file_agent_task (step_instance_id, execute_count, actual_execute_count, batch," - + "mode, host_id, agent_id ,gse_task_id,status, start_time, end_time, total_time, error_code)" - + " values (?,?,?,?,?,?,?,?,?,?,?,?,?)"; - Object[][] params = new Object[agentTasks.size()][13]; + String sql = "insert into gse_file_agent_task (task_instance_id, step_instance_id, execute_count, " + + "actual_execute_count, batch, mode, host_id, agent_id ,gse_task_id,status, start_time, " + + "end_time, total_time, error_code) " + + "values (?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + Object[][] params = new Object[agentTasks.size()][14]; int batchCount = 0; for (ExecuteObjectTask agentTask : agentTasks) { - Object[] param = new Object[13]; - param[0] = agentTask.getStepInstanceId(); - param[1] = agentTask.getExecuteCount(); - param[2] = agentTask.getActualExecuteCount(); - param[3] = agentTask.getBatch(); - param[4] = agentTask.getFileTaskMode().getValue(); - param[5] = agentTask.getHostId(); - param[6] = agentTask.getAgentId() == null ? "" : agentTask.getAgentId(); - param[7] = agentTask.getGseTaskId(); - param[8] = agentTask.getStatus().getValue(); - param[9] = agentTask.getStartTime(); - param[10] = agentTask.getEndTime(); - param[11] = agentTask.getTotalTime(); - param[12] = agentTask.getErrorCode(); + Object[] param = new Object[14]; + param[0] = agentTask.getTaskInstanceId(); + param[1] = agentTask.getStepInstanceId(); + param[2] = agentTask.getExecuteCount(); + param[3] = agentTask.getActualExecuteCount(); + param[4] = agentTask.getBatch(); + param[5] = agentTask.getFileTaskMode().getValue(); + param[6] = agentTask.getHostId(); + param[7] = agentTask.getAgentId() == null ? "" : agentTask.getAgentId(); + param[8] = agentTask.getGseTaskId(); + param[9] = agentTask.getStatus().getValue(); + param[10] = agentTask.getStartTime(); + param[11] = agentTask.getEndTime(); + param[12] = agentTask.getTotalTime(); + param[13] = agentTask.getErrorCode(); params[batchCount++] = param; } CTX.batch(sql, params).execute(); @@ -123,29 +126,31 @@ public void batchUpdateAgentTasks(Collection agentTasks) { } String sql = "update gse_file_agent_task set gse_task_id = ?, status = ?, start_time = ?, end_time = ?" + ", total_time = ?, error_code = ?" - + " where step_instance_id = ? and execute_count = ? and batch = ? and mode = ? and host_id = ?"; - Object[][] params = new Object[agentTasks.size()][11]; + + " where task_instance_id = ? and step_instance_id = ? and execute_count = ?" + + " and batch = ? and mode = ? and host_id = ?"; + Object[][] params = new Object[agentTasks.size()][12]; int batchCount = 0; for (ExecuteObjectTask agentTask : agentTasks) { - Object[] param = new Object[11]; + Object[] param = new Object[12]; param[0] = agentTask.getGseTaskId(); param[1] = agentTask.getStatus().getValue(); param[2] = agentTask.getStartTime(); param[3] = agentTask.getEndTime(); param[4] = agentTask.getTotalTime(); param[5] = agentTask.getErrorCode(); - param[6] = agentTask.getStepInstanceId(); - param[7] = agentTask.getExecuteCount(); - param[8] = agentTask.getBatch(); - param[9] = agentTask.getFileTaskMode().getValue(); - param[10] = agentTask.getHostId(); + param[6] = agentTask.getTaskInstanceId(); + param[7] = agentTask.getStepInstanceId(); + param[8] = agentTask.getExecuteCount(); + param[9] = agentTask.getBatch(); + param[10] = agentTask.getFileTaskMode().getValue(); + param[11] = agentTask.getHostId(); params[batchCount++] = param; } CTX.batch(sql, params).execute(); } @Override - public int getSuccessAgentTaskCount(long stepInstanceId, int executeCount) { + public int getSuccessAgentTaskCount(Long taskInstanceId, long stepInstanceId, int executeCount) { Integer count = CTX.selectCount() .from(T_GSE_FILE_AGENT_TASK) .where(T_GSE_FILE_AGENT_TASK.STATUS.in(ExecuteObjectTaskStatusEnum.LAST_SUCCESS.getValue(), @@ -153,18 +158,30 @@ public int getSuccessAgentTaskCount(long stepInstanceId, int executeCount) { .and(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)) .and(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .fetchOne(0, Integer.class); return count == null ? 0 : count; } + private Condition buildTaskInstanceIdQueryCondition(Long taskInstanceId) { + return TaskInstanceIdDynamicCondition.build( + taskInstanceId, + T_GSE_FILE_AGENT_TASK.TASK_INSTANCE_ID::eq + ); + } + @Override - public List listResultGroups(long stepInstanceId, int executeCount, Integer batch) { + public List listResultGroups(Long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer batch) { SelectConditionStep selectConditionStep = CTX.select(T_GSE_FILE_AGENT_TASK.STATUS, count().as("ip_count")) .from(T_GSE_FILE_AGENT_TASK) .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)) - .and(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())); + .and(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); if (batch != null && batch > 0) { selectConditionStep.and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch.shortValue())); } @@ -186,7 +203,8 @@ public List listResultGroups(long stepInstanceId, int execut } @Override - public List listAgentTaskByResultGroup(Long stepInstanceId, + public List listAgentTaskByResultGroup(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, Integer status) { @@ -195,7 +213,8 @@ public List listAgentTaskByResultGroup(Long stepInstanceId, .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) .and(T_GSE_FILE_AGENT_TASK.STATUS.eq(status)) - .and(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())); + .and(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); if (batch != null && batch > 0) { selectConditionStep.and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch.shortValue())); } @@ -209,7 +228,8 @@ public List listAgentTaskByResultGroup(Long stepInstanceId, } @Override - public List listAgentTaskByResultGroup(Long stepInstanceId, + public List listAgentTaskByResultGroup(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, Integer status, @@ -221,6 +241,7 @@ public List listAgentTaskByResultGroup(Long stepInstanceId, conditions.add(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())); conditions.add(T_GSE_FILE_AGENT_TASK.STATUS.eq(status)); conditions.add(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())); + conditions.add(buildTaskInstanceIdQueryCondition(taskInstanceId)); SelectConditionStep select = CTX.select(ALL_FIELDS) .from(T_GSE_FILE_AGENT_TASK) @@ -276,14 +297,16 @@ private OrderField buildOrderField(String field, Order order) { } @Override - public List listAgentTasks(Long stepInstanceId, + public List listAgentTasks(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, FileTaskModeEnum fileTaskMode) { SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) .from(T_GSE_FILE_AGENT_TASK) .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())); + .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); if (batch != null && batch > 0) { selectConditionStep.and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch.shortValue())); } @@ -306,6 +329,7 @@ private ExecuteObjectTask extract(Record record) { return null; } ExecuteObjectTask agentTask = new ExecuteObjectTask(); + agentTask.setTaskInstanceId(record.get(T_GSE_FILE_AGENT_TASK.TASK_INSTANCE_ID)); agentTask.setStepInstanceId(record.get(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID)); agentTask.setExecuteCount(record.get(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT)); Short actualExecuteCount = record.get(T_GSE_FILE_AGENT_TASK.ACTUAL_EXECUTE_COUNT); @@ -324,7 +348,7 @@ private ExecuteObjectTask extract(Record record) { } @Override - public List listAgentTasksByGseTaskId(Long gseTaskId) { + public List listAgentTasksByGseTaskId(Long taskInstanceId, Long gseTaskId) { if (gseTaskId == null || gseTaskId <= 0) { return Collections.emptyList(); } @@ -334,6 +358,7 @@ public List listAgentTasksByGseTaskId(Long gseTaskId) { Result result = CTX.select(ALL_FIELDS) .from(T_GSE_FILE_AGENT_TASK) .where(T_GSE_FILE_AGENT_TASK.GSE_TASK_ID.eq(gseTaskId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .fetch(); if (result.size() > 0) { result.forEach(record -> agentTaskList.add(extract(record))); @@ -342,15 +367,20 @@ public List listAgentTasksByGseTaskId(Long gseTaskId) { } @Override - public ExecuteObjectTask getAgentTaskByHostId(Long stepInstanceId, Integer executeCount, Integer batch, - FileTaskModeEnum mode, long hostId) { + public ExecuteObjectTask getAgentTaskByHostId(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + FileTaskModeEnum mode, + long hostId) { SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) .from(T_GSE_FILE_AGENT_TASK) .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) .and(T_GSE_FILE_AGENT_TASK.MODE.eq(mode.getValue().byteValue())) - .and(T_GSE_FILE_AGENT_TASK.HOST_ID.eq(hostId)); + .and(T_GSE_FILE_AGENT_TASK.HOST_ID.eq(hostId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); if (batch != null && batch > 0) { // 滚动执行批次，传入null或者0将忽略该参数 selectConditionStep.and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch.shortValue())); @@ -361,7 +391,8 @@ public ExecuteObjectTask getAgentTaskByHostId(Long stepInstanceId, Integer execu } @Override - public void updateAgentTaskFields(long stepInstanceId, + public void updateAgentTaskFields(Long taskInstanceId, + long stepInstanceId, int executeCount, Integer batch, Integer actualExecuteCount, @@ -388,7 +419,8 @@ public void updateAgentTaskFields(long stepInstanceId, UpdateConditionStep updateConditionStep = updateSetMoreStep .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)); + .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); if (batch != null) { updateConditionStep.and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch.shortValue())); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java index 755a6a1d30..fc2fe11c86 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java @@ -67,6 +67,7 @@ public class ScriptAgentTaskDAOImpl implements ScriptAgentTaskDAO { private static final GseScriptAgentTask T_GSE_SCRIPT_AGENT_TASK = GseScriptAgentTask.GSE_SCRIPT_AGENT_TASK; private static final TableField[] ALL_FIELDS = { + T_GSE_SCRIPT_AGENT_TASK.TASK_INSTANCE_ID, T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID, T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT, T_GSE_SCRIPT_AGENT_TASK.ACTUAL_EXECUTE_COUNT, @@ -93,29 +94,30 @@ public ScriptAgentTaskDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext C @Override public void batchSaveAgentTasks(Collection agentTasks) { - String sql = "insert into gse_script_agent_task (step_instance_id, execute_count, actual_execute_count, batch," - + " host_id, agent_id, gse_task_id, status, start_time, end_time, total_time, error_code, exit_code, tag," - + " log_offset)" - + " values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; - Object[][] params = new Object[agentTasks.size()][15]; + String sql = "insert into gse_script_agent_task (task_instance_id, step_instance_id, execute_count, " + + "actual_execute_count, batch, host_id, agent_id, gse_task_id, status, start_time, end_time, " + + "total_time, error_code, exit_code, tag, log_offset) " + + "values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + Object[][] params = new Object[agentTasks.size()][16]; int batchCount = 0; for (ExecuteObjectTask agentTask : agentTasks) { - Object[] param = new Object[15]; - param[0] = agentTask.getStepInstanceId(); - param[1] = agentTask.getExecuteCount(); - param[2] = agentTask.getActualExecuteCount(); - param[3] = agentTask.getBatch(); - param[4] = agentTask.getHostId(); - param[5] = agentTask.getAgentId() == null ? "" : agentTask.getAgentId(); - param[6] = agentTask.getGseTaskId(); - param[7] = agentTask.getStatus().getValue(); - param[8] = agentTask.getStartTime(); - param[9] = agentTask.getEndTime(); - param[10] = agentTask.getTotalTime(); - param[11] = agentTask.getErrorCode(); - param[12] = agentTask.getExitCode(); - param[13] = StringUtils.truncate(agentTask.getTag(), JobConstants.RESULT_GROUP_TAG_MAX_LENGTH); - param[14] = agentTask.getScriptLogOffset(); + Object[] param = new Object[16]; + param[0] = agentTask.getTaskInstanceId(); + param[1] = agentTask.getStepInstanceId(); + param[2] = agentTask.getExecuteCount(); + param[3] = agentTask.getActualExecuteCount(); + param[4] = agentTask.getBatch(); + param[5] = agentTask.getHostId(); + param[6] = agentTask.getAgentId() == null ? "" : agentTask.getAgentId(); + param[7] = agentTask.getGseTaskId(); + param[8] = agentTask.getStatus().getValue(); + param[9] = agentTask.getStartTime(); + param[10] = agentTask.getEndTime(); + param[11] = agentTask.getTotalTime(); + param[12] = agentTask.getErrorCode(); + param[13] = agentTask.getExitCode(); + param[14] = StringUtils.truncate(agentTask.getTag(), JobConstants.RESULT_GROUP_TAG_MAX_LENGTH); + param[15] = agentTask.getScriptLogOffset(); params[batchCount++] = param; } CTX.batch(sql, params).execute(); @@ -128,11 +130,12 @@ public void batchUpdateAgentTasks(Collection agentTasks) { } String sql = "update gse_script_agent_task set gse_task_id = ?, status = ?, start_time = ?, end_time = ?" + ", total_time = ?, error_code = ?, exit_code = ?, tag = ?, log_offset = ?" - + " where step_instance_id = ? and execute_count = ? and batch = ? and host_id = ?"; - Object[][] params = new Object[agentTasks.size()][13]; + + " where task_instance_id = ? and step_instance_id = ? and execute_count = ?" + + " and batch = ? and host_id = ?"; + Object[][] params = new Object[agentTasks.size()][14]; int batchCount = 0; for (ExecuteObjectTask agentTask : agentTasks) { - Object[] param = new Object[13]; + Object[] param = new Object[14]; param[0] = agentTask.getGseTaskId(); param[1] = agentTask.getStatus().getValue(); param[2] = agentTask.getStartTime(); @@ -142,34 +145,47 @@ public void batchUpdateAgentTasks(Collection agentTasks) { param[6] = agentTask.getExitCode(); param[7] = StringUtils.truncate(agentTask.getTag(), JobConstants.RESULT_GROUP_TAG_MAX_LENGTH); param[8] = agentTask.getScriptLogOffset(); - param[9] = agentTask.getStepInstanceId(); - param[10] = agentTask.getExecuteCount(); - param[11] = agentTask.getBatch(); - param[12] = agentTask.getHostId(); + param[9] = agentTask.getTaskInstanceId(); + param[10] = agentTask.getStepInstanceId(); + param[11] = agentTask.getExecuteCount(); + param[12] = agentTask.getBatch(); + param[13] = agentTask.getHostId(); params[batchCount++] = param; } CTX.batch(sql, params).execute(); } @Override - public int getSuccessAgentTaskCount(long stepInstanceId, int executeCount) { + public int getSuccessAgentTaskCount(Long taskInstanceId, long stepInstanceId, int executeCount) { Integer count = CTX.selectCount() .from(T_GSE_SCRIPT_AGENT_TASK) .where(T_GSE_SCRIPT_AGENT_TASK.STATUS.in(ExecuteObjectTaskStatusEnum.LAST_SUCCESS.getValue(), ExecuteObjectTaskStatusEnum.SUCCESS.getValue())) .and(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .fetchOne(0, Integer.class); return count == null ? 0 : count; } + private Condition buildTaskInstanceIdQueryCondition(Long taskInstanceId) { + return TaskInstanceIdDynamicCondition.build( + taskInstanceId, + T_GSE_SCRIPT_AGENT_TASK.TASK_INSTANCE_ID::eq + ); + } + @Override - public List listResultGroups(long stepInstanceId, int executeCount, Integer batch) { + public List listResultGroups(Long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer batch) { SelectConditionStep selectConditionStep = CTX.select(T_GSE_SCRIPT_AGENT_TASK.STATUS, T_GSE_SCRIPT_AGENT_TASK.TAG, count().as("ip_count")) .from(T_GSE_SCRIPT_AGENT_TASK) .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)); + .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); if (batch != null && batch > 0) { selectConditionStep.and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch.shortValue())); } @@ -191,7 +207,8 @@ public List listResultGroups(long stepInstanceId, int execut } @Override - public List listAgentTaskByResultGroup(Long stepInstanceId, + public List listAgentTaskByResultGroup(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, Integer status, @@ -201,21 +218,23 @@ public List listAgentTaskByResultGroup(Long stepInstanceId, .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) .and(T_GSE_SCRIPT_AGENT_TASK.STATUS.eq(status)) - .and(T_GSE_SCRIPT_AGENT_TASK.TAG.eq(tag == null ? "" : tag)); + .and(T_GSE_SCRIPT_AGENT_TASK.TAG.eq(tag == null ? "" : tag)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); if (batch != null && batch > 0) { selectConditionStep.and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch.shortValue())); } Result result = selectConditionStep.fetch(); List agentTasks = new ArrayList<>(); - if (result.size() > 0) { + if (!result.isEmpty()) { result.forEach(record -> agentTasks.add(extract(record))); } return agentTasks; } @Override - public List listAgentTaskByResultGroup(Long stepInstanceId, + public List listAgentTaskByResultGroup(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, Integer status, @@ -228,6 +247,7 @@ public List listAgentTaskByResultGroup(Long stepInstanceId, conditions.add(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())); conditions.add(T_GSE_SCRIPT_AGENT_TASK.STATUS.eq(status)); conditions.add(T_GSE_SCRIPT_AGENT_TASK.TAG.eq(tag == null ? "" : tag)); + conditions.add(buildTaskInstanceIdQueryCondition(taskInstanceId)); SelectConditionStep select = CTX.select(ALL_FIELDS) .from(T_GSE_SCRIPT_AGENT_TASK) @@ -262,7 +282,7 @@ public List listAgentTaskByResultGroup(Long stepInstanceId, result = select.fetch(); } - if (result.size() > 0) { + if (!result.isEmpty()) { result.into(record -> agentTasks.add(extract(record))); } return agentTasks; @@ -289,19 +309,21 @@ private OrderField buildOrderField(String field, Order order) { } @Override - public List listAgentTasks(Long stepInstanceId, + public List listAgentTasks(Long taskInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch) { SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) .from(T_GSE_SCRIPT_AGENT_TASK) .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())); + .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); if (batch != null && batch > 0) { selectConditionStep.and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch.shortValue())); } Result result = selectConditionStep.fetch(); List agentTaskList = new ArrayList<>(); - if (result.size() != 0) { + if (!result.isEmpty()) { result.map(record -> { agentTaskList.add(extract(record)); return null; @@ -315,6 +337,7 @@ private ExecuteObjectTask extract(Record record) { return null; } ExecuteObjectTask agentTask = new ExecuteObjectTask(); + agentTask.setTaskInstanceId(record.get(T_GSE_SCRIPT_AGENT_TASK.TASK_INSTANCE_ID)); agentTask.setStepInstanceId(record.get(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID)); agentTask.setExecuteCount(record.get(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT)); Short actualExecuteCount = record.get(T_GSE_SCRIPT_AGENT_TASK.ACTUAL_EXECUTE_COUNT); @@ -335,7 +358,7 @@ private ExecuteObjectTask extract(Record record) { } @Override - public List listAgentTasksByGseTaskId(Long gseTaskId) { + public List listAgentTasksByGseTaskId(Long taskInstanceId, Long gseTaskId) { if (gseTaskId == null || gseTaskId <= 0) { return Collections.emptyList(); } @@ -345,22 +368,27 @@ public List listAgentTasksByGseTaskId(Long gseTaskId) { Result result = CTX.select(ALL_FIELDS) .from(T_GSE_SCRIPT_AGENT_TASK) .where(T_GSE_SCRIPT_AGENT_TASK.GSE_TASK_ID.eq(gseTaskId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .fetch(); - if (result.size() > 0) { + if (!result.isEmpty()) { result.forEach(record -> agentTaskList.add(extract(record))); } return agentTaskList; } @Override - public ExecuteObjectTask getAgentTaskByHostId(Long stepInstanceId, Integer executeCount, Integer batch, + public ExecuteObjectTask getAgentTaskByHostId(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, long hostId) { SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) .from(T_GSE_SCRIPT_AGENT_TASK) .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) - .and(T_GSE_SCRIPT_AGENT_TASK.HOST_ID.eq(hostId)); + .and(T_GSE_SCRIPT_AGENT_TASK.HOST_ID.eq(hostId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); if (batch != null && batch > 0) { // 滚动执行批次，传入null或者0将忽略该参数 selectConditionStep.and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch.shortValue())); @@ -372,7 +400,8 @@ public ExecuteObjectTask getAgentTaskByHostId(Long stepInstanceId, Integer execu @Override - public void updateAgentTaskFields(long stepInstanceId, + public void updateAgentTaskFields(Long taskInstanceId, + long stepInstanceId, int executeCount, Integer batch, Integer actualExecuteCount, @@ -399,7 +428,8 @@ public void updateAgentTaskFields(long stepInstanceId, UpdateConditionStep updateConditionStep = updateSetMoreStep .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)); + .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); if (batch != null) { updateConditionStep.and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch.shortValue())); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileExecuteObjectTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileExecuteObjectTaskServiceImpl.java index afed5e2c06..43a92e5cce 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileExecuteObjectTaskServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileExecuteObjectTaskServiceImpl.java @@ -11,7 +11,6 @@ import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.service.FileExecuteObjectTaskService; import com.tencent.bk.job.execute.service.StepInstanceService; -import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -31,17 +30,14 @@ public class FileExecuteObjectTaskServiceImpl private final FileExecuteObjectTaskDAO fileExecuteObjectTaskDAO; private final FileAgentTaskDAO fileAgentTaskDAO; - private final TaskInstanceService taskInstanceService; @Autowired public FileExecuteObjectTaskServiceImpl(StepInstanceService stepInstanceService, FileExecuteObjectTaskDAO fileExecuteObjectTaskDAO, - FileAgentTaskDAO fileAgentTaskDAO, - TaskInstanceService taskInstanceService) { + FileAgentTaskDAO fileAgentTaskDAO) { super(stepInstanceService); this.fileAgentTaskDAO = fileAgentTaskDAO; this.fileExecuteObjectTaskDAO = fileExecuteObjectTaskDAO; - this.taskInstanceService = taskInstanceService; } @Override @@ -75,7 +71,7 @@ public int getSuccessTaskCount(Long taskInstanceId, long stepInstanceId, int exe if (isStepInstanceRecordExist(taskInstanceId, stepInstanceId)) { return fileExecuteObjectTaskDAO.getSuccessTaskCount(taskInstanceId, stepInstanceId, executeCount); } else { - return fileAgentTaskDAO.getSuccessAgentTaskCount(stepInstanceId, executeCount); + return fileAgentTaskDAO.getSuccessAgentTaskCount(taskInstanceId, stepInstanceId, executeCount); } } @@ -92,7 +88,8 @@ public List listTasksByGseTaskId(StepInstanceBaseDTO stepInst stepInstance.getTaskInstanceId(), gseTaskId); } else { // 兼容老版本数据 - executeObjectTasks = fileAgentTaskDAO.listAgentTasksByGseTaskId(gseTaskId); + executeObjectTasks = fileAgentTaskDAO.listAgentTasksByGseTaskId( + stepInstance.getTaskInstanceId(), gseTaskId); } fillExecuteObjectForExecuteObjectTasks(stepInstance, executeObjectTasks); @@ -120,8 +117,8 @@ public ExecuteObjectTask getTaskByExecuteObjectCompositeKey(StepInstanceBaseDTO } else { // 兼容老版本不使用执行对象的场景(仅支持主机） Long hostId = executeObject.getHost().getHostId(); - executeObjectTask = fileAgentTaskDAO.getAgentTaskByHostId(stepInstanceId, executeCount, - batch, fileTaskMode, hostId); + executeObjectTask = fileAgentTaskDAO.getAgentTaskByHostId(stepInstance.getTaskInstanceId(), + stepInstanceId, executeCount, batch, fileTaskMode, hostId); } executeObjectTask.setExecuteObject(executeObject); return executeObjectTask; @@ -157,7 +154,8 @@ public List listResultGroups(StepInstanceBaseDTO stepInstanc stepInstance.getTaskInstanceId(), stepInstanceId, executeCount, batch); } else { // 兼容历史数据 - resultGroups = fileAgentTaskDAO.listResultGroups(stepInstanceId, executeCount, batch); + resultGroups = fileAgentTaskDAO.listResultGroups(stepInstance.getTaskInstanceId(), + stepInstanceId, executeCount, batch); } return resultGroups; } @@ -176,7 +174,7 @@ public List listTaskByResultGroup(StepInstanceBaseDTO stepIns } else { // 兼容历史数据 executeObjectTasks = fileAgentTaskDAO.listAgentTaskByResultGroup( - stepInstance.getId(), executeCount, batch, status); + stepInstance.getTaskInstanceId(), stepInstance.getId(), executeCount, batch, status); } fillExecuteObjectForExecuteObjectTasks(stepInstance, executeObjectTasks); @@ -208,8 +206,16 @@ public List listTaskByResultGroup(StepInstanceBaseDTO stepIns ); } else { // 兼容历史数据 - executeObjectTasks = fileAgentTaskDAO.listAgentTaskByResultGroup(stepInstance.getId(), executeCount, - batch, status, limit, orderField, order); + executeObjectTasks = fileAgentTaskDAO.listAgentTaskByResultGroup( + stepInstance.getTaskInstanceId(), + stepInstance.getId(), + executeCount, + batch, + status, + limit, + orderField, + order + ); } fillExecuteObjectForExecuteObjectTasks(stepInstance, executeObjectTasks); @@ -237,8 +243,14 @@ public void updateTaskFields(StepInstanceBaseDTO stepInstance, ); } else { // 兼容老版本方式 - fileAgentTaskDAO.updateAgentTaskFields(stepInstance.getId(), executeCount, batch, - actualExecuteCount, gseTaskId); + fileAgentTaskDAO.updateAgentTaskFields( + stepInstance.getTaskInstanceId(), + stepInstance.getId(), + executeCount, + batch, + actualExecuteCount, + gseTaskId + ); } } @@ -254,7 +266,8 @@ public List listTasks(StepInstanceBaseDTO stepInstance, stepInstance.getTaskInstanceId(), stepInstanceId, executeCount, batch, fileTaskMode); } else { // 兼容老版本数据 - executeObjectTasks = fileAgentTaskDAO.listAgentTasks(stepInstanceId, executeCount, batch, fileTaskMode); + executeObjectTasks = fileAgentTaskDAO.listAgentTasks(stepInstance.getTaskInstanceId(), + stepInstanceId, executeCount, batch, fileTaskMode); } fillExecuteObjectForExecuteObjectTasks(stepInstance, executeObjectTasks); return executeObjectTasks; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptExecuteObjectTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptExecuteObjectTaskServiceImpl.java index 1d00daf72e..1f1ec1f01b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptExecuteObjectTaskServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptExecuteObjectTaskServiceImpl.java @@ -69,7 +69,7 @@ public int getSuccessTaskCount(Long taskInstanceId, long stepInstanceId, int exe if (isStepInstanceRecordExist(taskInstanceId, stepInstanceId)) { return scriptExecuteObjectTaskDAO.getSuccessTaskCount(taskInstanceId, stepInstanceId, executeCount); } else { - return scriptAgentTaskDAO.getSuccessAgentTaskCount(stepInstanceId, executeCount); + return scriptAgentTaskDAO.getSuccessAgentTaskCount(taskInstanceId, stepInstanceId, executeCount); } } @@ -82,7 +82,8 @@ public List listTasks(StepInstanceBaseDTO stepInstance, Integ stepInstanceId, executeCount, batch); } else { // 兼容老版本数据 - executeObjectTasks = scriptAgentTaskDAO.listAgentTasks(stepInstanceId, executeCount, batch); + executeObjectTasks = scriptAgentTaskDAO.listAgentTasks(stepInstance.getTaskInstanceId(), + stepInstanceId, executeCount, batch); } fillExecuteObjectForExecuteObjectTasks(stepInstance, executeObjectTasks); return executeObjectTasks; @@ -97,7 +98,8 @@ public List listTasksByGseTaskId(StepInstanceBaseDTO stepInst stepInstance.getTaskInstanceId(), gseTaskId); } else { // 兼容老版本数据 - executeObjectTasks = scriptAgentTaskDAO.listAgentTasksByGseTaskId(gseTaskId); + executeObjectTasks = scriptAgentTaskDAO.listAgentTasksByGseTaskId( + stepInstance.getTaskInstanceId(), gseTaskId); } fillExecuteObjectForExecuteObjectTasks(stepInstance, executeObjectTasks); return executeObjectTasks; @@ -126,8 +128,13 @@ public ExecuteObjectTask getTaskByExecuteObjectCompositeKey(StepInstanceBaseDTO } else { // 兼容老版本不使用执行对象的场景(仅支持主机） Long hostId = executeObject.getHost().getHostId(); - executeObjectTask = scriptAgentTaskDAO.getAgentTaskByHostId(stepInstanceId, executeCount, - batch, hostId); + executeObjectTask = scriptAgentTaskDAO.getAgentTaskByHostId( + stepInstance.getTaskInstanceId(), + stepInstanceId, + executeCount, + batch, + hostId + ); } fillExecuteObjectForExecuteObjectTask(stepInstance, executeObjectTask); return executeObjectTask; @@ -165,7 +172,8 @@ public List listResultGroups(StepInstanceBaseDTO stepInstanc ); } else { // 兼容历史数据 - resultGroups = scriptAgentTaskDAO.listResultGroups(stepInstanceId, executeCount, batch); + resultGroups = scriptAgentTaskDAO.listResultGroups(stepInstance.getTaskInstanceId(), + stepInstanceId, executeCount, batch); } return resultGroups; } @@ -190,7 +198,13 @@ public List listTaskByResultGroup(StepInstanceBaseDTO stepIns } else { // 兼容历史数据 executeObjectTasks = scriptAgentTaskDAO.listAgentTaskByResultGroup( - stepInstance.getId(), executeCount, batch, status, tag); + stepInstance.getTaskInstanceId(), + stepInstance.getId(), + executeCount, + batch, + status, + tag + ); } fillExecuteObjectForExecuteObjectTasks(stepInstance, executeObjectTasks); @@ -222,8 +236,17 @@ public List listTaskByResultGroup(StepInstanceBaseDTO stepIns ); } else { // 兼容历史数据 - executeObjectTasks = scriptAgentTaskDAO.listAgentTaskByResultGroup(stepInstance.getId(), executeCount, - batch, status, tag, limit, orderField, order); + executeObjectTasks = scriptAgentTaskDAO.listAgentTaskByResultGroup( + stepInstance.getTaskInstanceId(), + stepInstance.getId(), + executeCount, + batch, + status, + tag, + limit, + orderField, + order + ); } fillExecuteObjectForExecuteObjectTasks(stepInstance, executeObjectTasks); @@ -245,8 +268,8 @@ public void updateTaskFields(StepInstanceBaseDTO stepInstance, executeCount, batch, actualExecuteCount, gseTaskId); } else { // 兼容老版本方式 - scriptAgentTaskDAO.updateAgentTaskFields(stepInstance.getId(), executeCount, batch, - actualExecuteCount, gseTaskId); + scriptAgentTaskDAO.updateAgentTaskFields(stepInstance.getTaskInstanceId(), stepInstance.getId(), + executeCount, batch, actualExecuteCount, gseTaskId); } } } diff --git a/support-files/sql/job-execute/0026_job_execute_20240520-1000_V3.10.0_mysql.sql b/support-files/sql/job-execute/0026_job_execute_20240520-1000_V3.10.0_mysql.sql index 340aa91f3e..539c972eb3 100644 --- a/support-files/sql/job-execute/0026_job_execute_20240520-1000_V3.10.0_mysql.sql +++ b/support-files/sql/job-execute/0026_job_execute_20240520-1000_V3.10.0_mysql.sql @@ -13,6 +13,39 @@ BEGIN SET AUTOCOMMIT = 0; SELECT DATABASE() INTO db; + -- task_instance + IF NOT EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'task_instance' + AND INDEX_NAME = 'idx_create_time_id') THEN + ALTER TABLE task_instance ADD INDEX `idx_create_time_id` (`create_time`,`id`); + END IF; + IF EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'task_instance' + AND INDEX_NAME = 'idx_create_time') THEN + ALTER TABLE task_instance DROP INDEX `idx_create_time`; + END IF; + + -- step_instance + IF NOT EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'step_instance' + AND INDEX_NAME = 'idx_task_instance_id_id') THEN + ALTER TABLE step_instance ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); + END IF; + IF EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'step_instance' + AND INDEX_NAME = 'idx_task_instance_id') THEN + ALTER TABLE step_instance DROP INDEX `idx_task_instance_id`; + END IF; + + -- step_instance_script IF NOT EXISTS(SELECT 1 FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = db @@ -24,10 +57,11 @@ BEGIN FROM information_schema.statistics WHERE TABLE_SCHEMA = db AND TABLE_NAME = 'step_instance_script' - AND INDEX_NAME = 'idx_task_instance_id') THEN - ALTER TABLE step_instance_script ADD INDEX `idx_task_instance_id` (`task_instance_id`); + AND INDEX_NAME = 'idx_task_instance_id_step_instance_id') THEN + ALTER TABLE step_instance_script ADD INDEX `idx_task_instance_id_step_instance_id` (`task_instance_id`,`step_instance_id`); END IF; + -- step_instance_file IF NOT EXISTS(SELECT 1 FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = db @@ -39,10 +73,11 @@ BEGIN FROM information_schema.statistics WHERE TABLE_SCHEMA = db AND TABLE_NAME = 'step_instance_file' - AND INDEX_NAME = 'idx_task_instance_id') THEN - ALTER TABLE step_instance_file ADD INDEX `idx_task_instance_id` (`task_instance_id`); + AND INDEX_NAME = 'idx_task_instance_id_step_instance_id') THEN + ALTER TABLE step_instance_file ADD INDEX `idx_task_instance_id_step_instance_id` (`task_instance_id`,`step_instance_id`); END IF; + -- step_instance_confirm IF NOT EXISTS(SELECT 1 FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = db @@ -54,10 +89,11 @@ BEGIN FROM information_schema.statistics WHERE TABLE_SCHEMA = db AND TABLE_NAME = 'step_instance_confirm' - AND INDEX_NAME = 'idx_task_instance_id') THEN - ALTER TABLE step_instance_confirm ADD INDEX `idx_task_instance_id` (`task_instance_id`); + AND INDEX_NAME = 'idx_task_instance_id_step_instance_id') THEN + ALTER TABLE step_instance_confirm ADD INDEX `idx_task_instance_id_step_instance_id` (`task_instance_id`,`step_instance_id`); END IF; + -- gse_task IF NOT EXISTS(SELECT 1 FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = db @@ -69,40 +105,75 @@ BEGIN FROM information_schema.statistics WHERE TABLE_SCHEMA = db AND TABLE_NAME = 'gse_task' + AND INDEX_NAME = 'idx_task_instance_id_id') THEN + ALTER TABLE gse_task ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); + END IF; + + -- gse_script_agent_task + IF NOT EXISTS(SELECT 1 + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'gse_script_agent_task' + AND COLUMN_NAME = 'task_instance_id') THEN + ALTER TABLE gse_script_agent_task ADD COLUMN `task_instance_id` bigint(20) NOT NULL DEFAULT '0'; + END IF; + IF NOT EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'gse_script_agent_task' + AND INDEX_NAME = 'idx_task_instance_id_id') THEN + ALTER TABLE gse_script_agent_task ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); + END IF; + + -- gse_file_agent_task + IF NOT EXISTS(SELECT 1 + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'gse_file_agent_task' + AND COLUMN_NAME = 'task_instance_id') THEN + ALTER TABLE gse_file_agent_task ADD COLUMN `task_instance_id` bigint(20) NOT NULL DEFAULT '0'; + END IF; + IF NOT EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'gse_file_agent_task' + AND INDEX_NAME = 'idx_task_instance_id_id') THEN + ALTER TABLE gse_file_agent_task ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); + END IF; + + -- gse_script_execute_obj_task + IF NOT EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'gse_script_execute_obj_task' + AND INDEX_NAME = 'idx_task_instance_id_id') THEN + ALTER TABLE gse_script_execute_obj_task ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); + END IF; + IF EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'gse_script_execute_obj_task' + AND INDEX_NAME = 'idx_task_instance_id') THEN + ALTER TABLE gse_script_execute_obj_task DROP INDEX `idx_task_instance_id`; + END IF; + + -- gse_file_execute_obj_task + IF NOT EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'gse_file_execute_obj_task' + AND INDEX_NAME = 'idx_task_instance_id_id') THEN + ALTER TABLE gse_file_execute_obj_task ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); + END IF; + IF EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'gse_file_execute_obj_task' AND INDEX_NAME = 'idx_task_instance_id') THEN - ALTER TABLE gse_task ADD INDEX `idx_task_instance_id` (`task_instance_id`); - END IF; - - -- IF NOT EXISTS(SELECT 1 - -- FROM information_schema.COLUMNS - -- WHERE TABLE_SCHEMA = db - -- AND TABLE_NAME = 'gse_script_agent_task' - -- AND COLUMN_NAME = 'task_instance_id') THEN - -- ALTER TABLE gse_script_agent_task ADD COLUMN `task_instance_id` bigint(20) NOT NULL DEFAULT '0'; - -- END IF; - -- IF NOT EXISTS(SELECT 1 - -- FROM information_schema.statistics - -- WHERE TABLE_SCHEMA = db - -- AND TABLE_NAME = 'gse_script_agent_task' - -- AND INDEX_NAME = 'idx_task_instance_id') THEN - -- ALTER TABLE gse_script_agent_task ADD INDEX `idx_task_instance_id` (`task_instance_id`); - -- END IF; - - -- IF NOT EXISTS(SELECT 1 - -- FROM information_schema.COLUMNS - -- WHERE TABLE_SCHEMA = db - -- AND TABLE_NAME = 'gse_file_agent_task' - -- AND COLUMN_NAME = 'task_instance_id') THEN - -- ALTER TABLE gse_file_agent_task ADD COLUMN `task_instance_id` bigint(20) NOT NULL DEFAULT '0'; - -- END IF; - -- IF NOT EXISTS(SELECT 1 - -- FROM information_schema.statistics - -- WHERE TABLE_SCHEMA = db - -- AND TABLE_NAME = 'gse_file_agent_task' - -- AND INDEX_NAME = 'idx_task_instance_id') THEN - -- ALTER TABLE gse_file_agent_task ADD INDEX `idx_task_instance_id` (`task_instance_id`); - -- END IF; + ALTER TABLE gse_file_execute_obj_task DROP INDEX `idx_task_instance_id`; + END IF; + -- file_source_task_log IF NOT EXISTS(SELECT 1 FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = db @@ -114,10 +185,20 @@ BEGIN FROM information_schema.statistics WHERE TABLE_SCHEMA = db AND TABLE_NAME = 'file_source_task_log' - AND INDEX_NAME = 'idx_task_instance_id') THEN - ALTER TABLE file_source_task_log ADD INDEX `idx_task_instance_id` (`task_instance_id`); + AND INDEX_NAME = 'idx_task_instance_id_id') THEN + ALTER TABLE file_source_task_log ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); END IF; + -- rolling_config + IF NOT EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'operation_log' + AND INDEX_NAME = 'idx_task_instance_id_id') THEN + ALTER TABLE rolling_config ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); + END IF; + + -- step_instance_rolling_task IF NOT EXISTS(SELECT 1 FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = db @@ -129,8 +210,56 @@ BEGIN FROM information_schema.statistics WHERE TABLE_SCHEMA = db AND TABLE_NAME = 'step_instance_rolling_task' + AND INDEX_NAME = 'idx_task_instance_id_id') THEN + ALTER TABLE step_instance_rolling_task ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); + END IF; + + -- operation_log + IF NOT EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'operation_log' + AND INDEX_NAME = 'idx_task_instance_id_id') THEN + ALTER TABLE operation_log ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); + END IF; + IF EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'operation_log' + AND INDEX_NAME = 'idx_task_instance_id') THEN + ALTER TABLE operation_log DROP INDEX `idx_task_instance_id`; + END IF; + + -- task_instance_variable + IF NOT EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'task_instance_variable' + AND INDEX_NAME = 'idx_task_instance_id_id') THEN + ALTER TABLE task_instance_variable ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); + END IF; + IF EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'task_instance_variable' + AND INDEX_NAME = 'idx_task_instance_id') THEN + ALTER TABLE task_instance_variable DROP INDEX `idx_task_instance_id`; + END IF; + + -- step_instance_variable + IF NOT EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'step_instance_variable' + AND INDEX_NAME = 'idx_task_instance_id_id') THEN + ALTER TABLE step_instance_variable ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); + END IF; + IF EXISTS(SELECT 1 + FROM information_schema.statistics + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'step_instance_variable' AND INDEX_NAME = 'idx_task_instance_id') THEN - ALTER TABLE step_instance_rolling_task ADD INDEX `idx_task_instance_id` (`task_instance_id`); + ALTER TABLE step_instance_variable DROP INDEX `idx_task_instance_id`; END IF; COMMIT; From 6d1513023fc73b66d0958f6d66f070ed9fdffb45 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Sat, 16 Nov 2024 15:08:34 +0800 Subject: [PATCH 22/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/build.gradle | 6 + .../common-mysql-sharding/build.gradle | 42 +++ .../leaf_gen_jooq.gradle | 78 +++++ .../sharding/mysql/JooqLeafIdAllocator.java | 119 +++++++ .../common/sharding/mysql/SegmentIdKeys.java | 45 +++ .../mysql/config/LeafAutoConfiguration.java | 105 ++++++ .../sharding/mysql/config/LeafProperties.java | 41 +++ .../main/resources/META-INF/spring.factories | 2 + .../common/mysql/dynamic/id/IdGenType.java | 68 ++++ .../service-job-backup/build.gradle | 1 + .../src/test/resources/application-test.yml | 6 + .../service-job-execute/build.gradle | 1 + .../execute/config/IdGenConfiguration.java | 106 ++++++ .../AbstractPropBasedDynamicComponent.java | 326 ++++++++++++++++++ .../dao/common/AutoIncrementIdGen.java | 82 +++++ .../bk/job/execute/dao/common/IdGen.java | 52 +++ .../common/PropBasedDynamicDataSource.java | 310 ++--------------- .../dao/common/PropBasedDynamicIdGen.java | 124 +++++++ .../job/execute/dao/common/SegmentIdGen.java | 85 +++++ .../third/ThirdFilePrepareService.java | 14 +- .../prepare/third/ThirdFilePrepareTask.java | 12 +- .../service/FileSourceTaskLogService.java | 43 +++ .../execute/service/RollingConfigService.java | 2 + .../FileExecuteObjectTaskServiceImpl.java | 7 +- .../impl/FileSourceTaskLogServiceImpl.java | 82 +++++ .../service/impl/GseTaskServiceImpl.java | 6 +- .../impl/RollingConfigServiceImpl.java | 14 +- .../ScriptExecuteObjectTaskServiceImpl.java | 7 +- .../StepInstanceRollingTaskServiceImpl.java | 6 +- .../service/impl/StepInstanceServiceImpl.java | 6 +- .../StepInstanceVariableValueServiceImpl.java | 7 +- .../service/impl/TaskInstanceServiceImpl.java | 8 +- .../impl/TaskInstanceVariableServiceImpl.java | 6 +- .../impl/TaskOperationLogServiceImpl.java | 6 +- .../service/impl/TaskResultServiceImpl.java | 10 +- .../templates/job-execute/configmap.yaml | 21 ++ .../kubernetes/charts/bk-job/values.yaml | 9 + ...b_execute_20241115-1000_V3.11.2_mysql.sql} | 174 +++++++++- .../0001_job_leaf_20241115-1000_V3.11.2.sql | 26 ++ 39 files changed, 1745 insertions(+), 320 deletions(-) create mode 100644 src/backend/commons/common-mysql-sharding/build.gradle create mode 100644 src/backend/commons/common-mysql-sharding/leaf_gen_jooq.gradle create mode 100644 src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/JooqLeafIdAllocator.java create mode 100644 src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/SegmentIdKeys.java create mode 100644 src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/LeafAutoConfiguration.java create mode 100644 src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/LeafProperties.java create mode 100644 src/backend/commons/common-mysql-sharding/src/main/resources/META-INF/spring.factories create mode 100644 src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/id/IdGenType.java create mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/IdGenConfiguration.java create mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AbstractPropBasedDynamicComponent.java create mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AutoIncrementIdGen.java create mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/IdGen.java create mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/PropBasedDynamicIdGen.java create mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/SegmentIdGen.java create mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileSourceTaskLogService.java create mode 100644 src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileSourceTaskLogServiceImpl.java rename support-files/sql/job-execute/{0026_job_execute_20240520-1000_V3.10.0_mysql.sql => 0026_job_execute_20241115-1000_V3.11.2_mysql.sql} (71%) create mode 100644 support-files/sql/job-leaf/0001_job_leaf_20241115-1000_V3.11.2.sql diff --git a/src/backend/build.gradle b/src/backend/build.gradle index d97e094082..453bb86898 100644 --- a/src/backend/build.gradle +++ b/src/backend/build.gradle @@ -152,6 +152,9 @@ ext { set('mockitoVersion', "4.0.0") set('embeddedRedisVersion', "0.6") set('openai4jVersion', "0.18.0") + set('shardingJdbcVersion', "5.4.1") + set('snakeyamlVersion', "1.33") + set('bkDevOpsLeafVersion', "1.0.2-RELEASE") } group "com.tencent.bk.job" @@ -348,6 +351,9 @@ subprojects { dependency "org.mockito:mockito-inline:$mockitoVersion" dependency "com.github.kstyrc:embedded-redis:$embeddedRedisVersion" dependency "dev.ai4j:openai4j:$openai4jVersion" + dependency "org.apache.shardingsphere:shardingsphere-jdbc-core:$shardingJdbcVersion" + dependency "org.yaml:snakeyaml:$snakeyamlVersion" + dependency "com.tencent.devops.leaf:leaf-boot-starter:$bkDevOpsLeafVersion" } } dependencies { diff --git a/src/backend/commons/common-mysql-sharding/build.gradle b/src/backend/commons/common-mysql-sharding/build.gradle new file mode 100644 index 0000000000..f30db15c16 --- /dev/null +++ b/src/backend/commons/common-mysql-sharding/build.gradle @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +dependencies { + api project(":commons:common") + implementation 'io.micrometer:micrometer-registry-prometheus' + implementation("org.springframework.boot:spring-boot-autoconfigure") + implementation 'org.springframework:spring-web' + implementation 'org.aspectj:aspectjrt' + implementation 'org.aspectj:aspectjweaver' + api 'org.apache.shardingsphere:shardingsphere-jdbc-core' + api 'org.yaml:snakeyaml' + api 'com.tencent.devops.leaf:leaf-boot-starter' + api 'org.jooq:jooq' + api 'org.springframework.boot:spring-boot-starter-jdbc' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' + testImplementation 'org.junit.jupiter:junit-jupiter' + testImplementation 'org.springframework.boot:spring-boot-starter-test' +} +apply from: "./leaf_gen_jooq.gradle" diff --git a/src/backend/commons/common-mysql-sharding/leaf_gen_jooq.gradle b/src/backend/commons/common-mysql-sharding/leaf_gen_jooq.gradle new file mode 100644 index 0000000000..d79c2dbbe1 --- /dev/null +++ b/src/backend/commons/common-mysql-sharding/leaf_gen_jooq.gradle @@ -0,0 +1,78 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +apply plugin: 'nu.studer.jooq' + +dependencies { + api "org.jooq:jooq" + jooqRuntime "mysql:mysql-connector-java" +} + +def databaseName = "job_leaf" +def targetPackageName = "com.tencent.bk.job.common.sharding.mysql.jooq.model" + +jooq { + genenrate(sourceSets.main) { + + jdbc { + driver = 'com.mysql.cj.jdbc.Driver' + def mysqlURL = System.getProperty("mysqlURL") + def mysqlUser = System.getProperty("mysqlUser") + def mysqlPasswd = System.getProperty("mysqlPasswd") + + if (mysqlURL == null) { + // gradle.properties 中的 Jooq DB 配置 + mysqlURL = project.findProperty("DB_HOST") + mysqlUser = project.findProperty("DB_USERNAME") + mysqlPasswd = project.findProperty("DB_PASSWORD") + } + + println("mysqlURL=" + mysqlURL) + println("mysqlUser=" + mysqlUser) + url = "jdbc:mysql://${mysqlURL}/${databaseName}?useSSL=false&serverTimezone=UTC" + user = mysqlUser + password = mysqlPasswd + } + + generator { + name = 'org.jooq.codegen.DefaultGenerator' + database { + name = 'org.jooq.meta.mysql.MySQLDatabase' + inputSchema = "${databaseName}" + } + + generate { + relations = false + deprecated = false + fluentSetters = true + generatedAnnotation = false + javaTimeTypes = true + } + + target { + packageName = "${targetPackageName}" + encoding = 'UTF-8' + } + } + } +} diff --git a/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/JooqLeafIdAllocator.java b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/JooqLeafIdAllocator.java new file mode 100644 index 0000000000..33124851f3 --- /dev/null +++ b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/JooqLeafIdAllocator.java @@ -0,0 +1,119 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.sharding.mysql; + +import com.tencent.bk.job.common.sharding.mysql.jooq.model.tables.TLeafAlloc; +import com.tencent.bk.job.common.sharding.mysql.jooq.model.tables.records.TLeafAllocRecord; +import com.tencent.bk.job.common.util.date.DateUtils; +import com.tencent.devops.leaf.segment.dao.IDAllocDao; +import com.tencent.devops.leaf.segment.model.LeafAlloc; +import org.jooq.DSLContext; +import org.jooq.Record1; +import org.jooq.Result; +import org.jooq.impl.DSL; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; + +/** + * 基于 JOOQ+美团 Leaf分布式 ID 组件，生成分布式 ID + */ +public class JooqLeafIdAllocator implements IDAllocDao { + private final DSLContext dslContext; + + private static final TLeafAlloc T_LEAF_ALLOC = TLeafAlloc.T_LEAF_ALLOC; + + + public JooqLeafIdAllocator(DSLContext dslContext) { + this.dslContext = dslContext; + } + + @Override + public List getAllLeafAllocs() { + Result leafAllocRecords = dslContext.selectFrom(T_LEAF_ALLOC).fetch(); + List leafAllocs = new ArrayList<>(); + leafAllocRecords.forEach(leafAllocRecord -> { + LeafAlloc leafAlloc = generateLeafAlloc(leafAllocRecord); + leafAllocs.add(leafAlloc); + }); + return leafAllocs; + } + + private LeafAlloc generateLeafAlloc(TLeafAllocRecord leafAllocRecord) { + LeafAlloc leafAlloc = new LeafAlloc(); + leafAlloc.setKey(leafAllocRecord.getBizTag()); + leafAlloc.setMaxId(leafAllocRecord.getMaxId()); + leafAlloc.setStep(leafAllocRecord.getStep()); + leafAlloc.setUpdateTime(DateUtils.defaultLocalDateTime(leafAllocRecord.getUpdateTime())); + return leafAlloc; + } + + @Override + public LeafAlloc updateMaxIdAndGetLeafAlloc(String tag) { + TLeafAllocRecord leafAllocRecord; + leafAllocRecord = dslContext.transactionResult(t -> { + DSLContext context = DSL.using(t); + context.update(T_LEAF_ALLOC) + .set(T_LEAF_ALLOC.MAX_ID, T_LEAF_ALLOC.MAX_ID.add(T_LEAF_ALLOC.STEP)) + .set(T_LEAF_ALLOC.UPDATE_TIME, LocalDateTime.now()) + .where(T_LEAF_ALLOC.BIZ_TAG.eq(tag)) + .execute(); + return context.selectFrom(T_LEAF_ALLOC) + .where(T_LEAF_ALLOC.BIZ_TAG + .eq(tag)) + .fetchOne(); + }); + return generateLeafAlloc(leafAllocRecord); + } + + @Override + public LeafAlloc updateMaxIdByCustomStepAndGetLeafAlloc(LeafAlloc leafAlloc) { + TLeafAllocRecord leafAllocRecord; + leafAllocRecord = dslContext.transactionResult(t -> { + DSLContext context = DSL.using(t); + context.update(T_LEAF_ALLOC) + .set(T_LEAF_ALLOC.MAX_ID, T_LEAF_ALLOC.MAX_ID.add(leafAlloc.getStep())) + .set(T_LEAF_ALLOC.UPDATE_TIME, LocalDateTime.now()) + .where(T_LEAF_ALLOC.BIZ_TAG.eq(leafAlloc.getKey())) + .execute(); + return context.selectFrom(T_LEAF_ALLOC) + .where(T_LEAF_ALLOC.BIZ_TAG.eq(leafAlloc.getKey())) + .fetchOne(); + }); + return generateLeafAlloc(leafAllocRecord); + } + + @Override + public List getAllTags() { + Result> tagRecords = dslContext + .select(T_LEAF_ALLOC.BIZ_TAG) + .from(T_LEAF_ALLOC) + .fetch(); + List tags = new ArrayList<>(); + tagRecords.forEach(record -> tags.add(record.value1())); + return tags; + } +} diff --git a/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/SegmentIdKeys.java b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/SegmentIdKeys.java new file mode 100644 index 0000000000..a84862b8e8 --- /dev/null +++ b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/SegmentIdKeys.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.sharding.mysql; + +/** + * 分布式 ID KEY + */ +public class SegmentIdKeys { + + public static final String KEY_JOB_EXECUTE_TASK_INSTANCE = "job_execute.task_instance"; + public static final String KEY_JOB_EXECUTE_STEP_INSTANCE = "job_execute.step_instance"; + public static final String KEY_JOB_EXECUTE_GSE_TASK = "job_execute.gse_task"; + public static final String KEY_JOB_EXECUTE_OPERATION_LOG = "job_execute.operation_log"; + public static final String KEY_JOB_EXECUTE_FILE_SOURCE_TASK_LOG = "job_execute.file_source_task_log"; + public static final String KEY_JOB_EXECUTE_GSE_FILE_EXECUTE_OBJ_TASK = "job_execute.gse_file_execute_obj_task"; + public static final String KEY_JOB_EXECUTE_GSE_SCRIPT_EXECUTE_OBJ_TASK = + "job_execute.gse_script_execute_obj_task"; + public static final String KEY_JOB_EXECUTE_ROLLING_CONFIG = "job_execute.rolling_config"; + public static final String KEY_JOB_EXECUTE_STEP_INSTANCE_ROLLING_TASK = + "job_execute.step_instance_rolling_task"; + public static final String KEY_JOB_EXECUTE_STEP_INSTANCE_VARIABLE = "job_execute.step_instance_variable"; + public static final String KEY_JOB_EXECUTE_TASK_INSTANCE_VARIABLE = "job_execute.task_instance_variable"; +} diff --git a/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/LeafAutoConfiguration.java b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/LeafAutoConfiguration.java new file mode 100644 index 0000000000..01ff46324f --- /dev/null +++ b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/LeafAutoConfiguration.java @@ -0,0 +1,105 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.sharding.mysql.config; + +import com.tencent.bk.job.common.sharding.mysql.JooqLeafIdAllocator; +import org.jooq.ConnectionProvider; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.jooq.impl.DataSourceConnectionProvider; +import org.jooq.impl.DefaultConfiguration; +import org.jooq.impl.DefaultDSLContext; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.jdbc.DataSourceBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.DependsOn; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.jdbc.datasource.TransactionAwareDataSourceProxy; + +import javax.sql.DataSource; + +@Configuration(value = "leafDbConfig") +@ConditionalOnProperty(value = "leaf.enabled", havingValue = "true") +@EnableConfigurationProperties({LeafProperties.class}) +public class LeafAutoConfiguration { + @Qualifier("leaf-data-source") + @Bean(name = "leaf-data-source") + @ConfigurationProperties(prefix = "spring.datasource.leaf") + public DataSource dataSource() { + return DataSourceBuilder.create().build(); + } + + @Qualifier("leafTransactionManager") + @Bean(name = "leafTransactionManager") + @DependsOn("leaf-data-source") + public DataSourceTransactionManager transactionManager( + @Qualifier("leaf-data-source") DataSource dataSource) { + return new DataSourceTransactionManager(dataSource); + } + + @Qualifier("leaf-jdbc-template") + @Bean(name = "leaf-jdbc-template") + public JdbcTemplate jdbcTemplate( + @Qualifier("leaf-data-source") DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + @Qualifier("leaf-dsl-context") + @Bean(name = "leaf-dsl-context") + public DSLContext dslContext(@Qualifier("leaf-jooq-conf") org.jooq.Configuration configuration) { + return new DefaultDSLContext(configuration); + } + + @Qualifier("leaf-jooq-conf") + @Bean(name = "leaf-jooq-conf") + public org.jooq.Configuration + jooqConf(@Qualifier("leaf-conn-provider") ConnectionProvider connectionProvider) { + return new DefaultConfiguration().derive(connectionProvider).derive(SQLDialect.MYSQL); + } + + @Qualifier("leaf-conn-provider") + @Bean(name = "leaf-conn-provider") + public ConnectionProvider connectionProvider( + @Qualifier("leafTransactionAwareDataSource") DataSource dataSource) { + return new DataSourceConnectionProvider(dataSource); + } + + @Qualifier("leafTransactionAwareDataSource") + @Bean(name = "leafTransactionAwareDataSource") + public TransactionAwareDataSourceProxy + transactionAwareDataSourceProxy(@Qualifier("leaf-data-source") DataSource dataSource) { + return new TransactionAwareDataSourceProxy(dataSource); + } + + @Bean("jooqLeafIdAllocator") + public JooqLeafIdAllocator jooqLeafIdAllocator(@Qualifier("leaf-dsl-context") DSLContext dslContext) { + return new JooqLeafIdAllocator(dslContext); + } +} diff --git a/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/LeafProperties.java b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/LeafProperties.java new file mode 100644 index 0000000000..f31d088f56 --- /dev/null +++ b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/LeafProperties.java @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.sharding.mysql.config; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties(prefix = "leaf") +@Getter +@Setter +@ToString +public class LeafProperties { + /** + * 是否启用leaf + */ + private boolean enabled; +} diff --git a/src/backend/commons/common-mysql-sharding/src/main/resources/META-INF/spring.factories b/src/backend/commons/common-mysql-sharding/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..4f8aad8350 --- /dev/null +++ b/src/backend/commons/common-mysql-sharding/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bk.job.common.sharding.mysql.config.LeafAutoConfiguration diff --git a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/id/IdGenType.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/id/IdGenType.java new file mode 100644 index 0000000000..e36123c524 --- /dev/null +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/id/IdGenType.java @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.mysql.dynamic.id; + +import lombok.Getter; + +@Getter +public enum IdGenType { + AUTO_INCREMENT(IdGenType.Constants.AUTO_INCREMENT), + SEGMENT(IdGenType.Constants.SEGMENT); + + public static class Constants { + public final static String AUTO_INCREMENT = "auto_increment"; + public final static String SEGMENT = "segment"; + } + + private final String type; + + IdGenType(String type) { + this.type = type; + } + + public static IdGenType valOf(String type) { + if (type == null) { + return null; + } + for (IdGenType value : values()) { + if (value.getType().equals(type)) { + return value; + } + } + throw new IllegalArgumentException("No IdGenType constant: " + type); + } + + public static boolean checkValid(String type) { + if (type == null) { + return false; + } + for (IdGenType value : values()) { + if (value.getType().equals(type)) { + return true; + } + } + return false; + } +} diff --git a/src/backend/job-backup/service-job-backup/build.gradle b/src/backend/job-backup/service-job-backup/build.gradle index aea946b2a7..54bad99001 100644 --- a/src/backend/job-backup/service-job-backup/build.gradle +++ b/src/backend/job-backup/service-job-backup/build.gradle @@ -35,6 +35,7 @@ dependencies { api project(":commons:common") api project(":commons:common-crypto") api project(":commons:common-mysql") + api project(":commons:common-mysql-sharding") implementation("org.springframework.boot:spring-boot-starter-web") implementation("org.springframework.boot:spring-boot-starter-jdbc") diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/application-test.yml b/src/backend/job-execute/boot-job-execute/src/test/resources/application-test.yml index bd59958fae..01cd5f8e32 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/application-test.yml +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/application-test.yml @@ -48,3 +48,9 @@ esb: mysql: standalone: enabled : true +leaf: + enabled: false +idGen: + type: auto_increment + migration: + enabled: false diff --git a/src/backend/job-execute/service-job-execute/build.gradle b/src/backend/job-execute/service-job-execute/build.gradle index 8233a9b0e9..b19dff218c 100644 --- a/src/backend/job-execute/service-job-execute/build.gradle +++ b/src/backend/job-execute/service-job-execute/build.gradle @@ -37,6 +37,7 @@ dependencies { api project(':commons:common-redis') api project(":commons:artifactory-sdk") api project(":commons:common-mysql") + api project(":commons:common-mysql-sharding") api("org.springframework.boot:spring-boot-starter-web") api("org.springframework.boot:spring-boot-starter-jdbc") api "org.springframework.boot:spring-boot-starter-jooq" diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/IdGenConfiguration.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/IdGenConfiguration.java new file mode 100644 index 0000000000..3e9dd4d7c9 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/IdGenConfiguration.java @@ -0,0 +1,106 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.config; + +import com.tencent.bk.job.common.mysql.dynamic.id.IdGenType; +import com.tencent.bk.job.common.util.toggle.prop.PropToggleStore; +import com.tencent.bk.job.execute.dao.common.AutoIncrementIdGen; +import com.tencent.bk.job.execute.dao.common.PropBasedDynamicIdGen; +import com.tencent.bk.job.execute.dao.common.SegmentIdGen; +import com.tencent.devops.leaf.service.SegmentService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.AllNestedConditions; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.core.RedisTemplate; + +@Configuration(value = "jobExecuteIdGenConfiguration") +@Slf4j +public class IdGenConfiguration { + + @Conditional(AutoIncrementIdGenCondition.class) + @Bean + public AutoIncrementIdGen autoIncrementIdGen() { + log.info("Init AutoIncrementIdGen"); + return new AutoIncrementIdGen(); + } + + static class AutoIncrementIdGenCondition extends AllNestedConditions { + public AutoIncrementIdGenCondition() { + super(ConfigurationPhase.REGISTER_BEAN); + } + + @ConditionalOnProperty(value = "idGen.type", havingValue = IdGenType.Constants.AUTO_INCREMENT) + static class IdGenTypeCondition { + + } + + @ConditionalOnProperty(value = "idGen.migration.enabled", havingValue = "false", matchIfMissing = true) + static class MigrationDisable { + + } + } + + @Conditional(SegmentIdGenCondition.class) + @Bean + public SegmentIdGen segmentIdGen(SegmentService segmentService) { + log.info("Init SegmentIdGen"); + return new SegmentIdGen(segmentService); + } + + static class SegmentIdGenCondition extends AllNestedConditions { + public SegmentIdGenCondition() { + super(ConfigurationPhase.REGISTER_BEAN); + } + + @ConditionalOnProperty(value = "idGen.type", havingValue = IdGenType.Constants.SEGMENT) + static class IdGenTypeCondition { + + } + + @ConditionalOnProperty(value = "idGen.migration.enabled", havingValue = "false", matchIfMissing = true) + static class MigrationDisable { + + } + } + + @ConditionalOnProperty(value = "idGen.migration.enabled", havingValue = "true") + @Bean + public PropBasedDynamicIdGen propBasedDynamicIdGen( + SegmentService segmentService, + @Qualifier("jsonRedisTemplate") RedisTemplate redisTemplate, + PropToggleStore propToggleStore + ) { + log.info("Init PropBasedDynamicIdGen"); + AutoIncrementIdGen autoIncrementIdGen = new AutoIncrementIdGen(); + SegmentIdGen segmentIdGen = new SegmentIdGen(segmentService); + return new PropBasedDynamicIdGen(autoIncrementIdGen, segmentIdGen, redisTemplate, propToggleStore); + } + + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AbstractPropBasedDynamicComponent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AbstractPropBasedDynamicComponent.java new file mode 100644 index 0000000000..9bf634ef7e --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AbstractPropBasedDynamicComponent.java @@ -0,0 +1,326 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.common; + +import com.tencent.bk.job.common.mysql.dynamic.ds.MigrationStatus; +import com.tencent.bk.job.common.redis.util.LockUtils; +import com.tencent.bk.job.common.util.ThreadUtils; +import com.tencent.bk.job.common.util.ip.IpUtils; +import com.tencent.bk.job.common.util.toggle.prop.PropChangeEventListener; +import com.tencent.bk.job.common.util.toggle.prop.PropToggle; +import com.tencent.bk.job.common.util.toggle.prop.PropToggleStore; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.data.redis.core.RedisTemplate; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static com.tencent.bk.job.common.mysql.dynamic.ds.MigrationStatus.FAIL; +import static com.tencent.bk.job.common.mysql.dynamic.ds.MigrationStatus.MIGRATED; +import static com.tencent.bk.job.common.mysql.dynamic.ds.MigrationStatus.MIGRATING; +import static com.tencent.bk.job.common.mysql.dynamic.ds.MigrationStatus.NOT_START; +import static com.tencent.bk.job.common.mysql.dynamic.ds.MigrationStatus.PREPARING; + +/** + * 基于属性动态控制的组件 + */ +@Slf4j +public abstract class AbstractPropBasedDynamicComponent implements PropChangeEventListener { + + private Map candidateComponents; + + private volatile MigrationStatus status = NOT_START; + + private final Object lock = new Object(); + + private final RedisTemplate redisTemplate; + + private volatile C current; + + private final PropToggleStore propToggleStore; + + private volatile boolean isInitial = false; + + /** + * 动态属性名称 - 迁移目标 + */ + private final String migrateTargetPropName; + /** + * 动态属性名称 - 参与迁移的微服务实例数量 + */ + private static final String PROP_NAME_SERVICE_INSTANCE_COUNT = "job_execute_service_instance_count"; + + /** + * 迁移使用的分布式锁 Redis KEY + */ + private final String migrationLockRedisKey; + /** + * 整体迁移状态 Redis KEY + */ + private final String globalMigrationStatusRedisKey; + /** + * 服务实例迁移状态 Redis KEY + */ + private final String serviceInstanceMigrationStatusRedisKey; + /** + * 服务节点 IP + */ + private final String serviceNodeIp; + + private static final String REDIS_KEY_PREFIX = "job:comp:mig:"; + + public AbstractPropBasedDynamicComponent( + RedisTemplate redisTemplate, + PropToggleStore propToggleStore, + String migrateComponentName) { + this.redisTemplate = redisTemplate; + this.propToggleStore = propToggleStore; + + this.serviceNodeIp = IpUtils.getFirstMachineIP(); + this.migrateTargetPropName = migrateComponentName; + this.migrationLockRedisKey = REDIS_KEY_PREFIX + migrateComponentName; + this.globalMigrationStatusRedisKey = REDIS_KEY_PREFIX + "global:status:" + migrateComponentName; + this.serviceInstanceMigrationStatusRedisKey = REDIS_KEY_PREFIX + ":svc:status:" + migrateComponentName; + + // 注册监听属性变化 + this.propToggleStore.addPropChangeEventListener(migrateTargetPropName, this); + } + + protected void initCandidateComponents(Map candidateComponents) { + this.candidateComponents = candidateComponents; + } + + /** + * 获取当前组件 + * 如果组件处于迁移状态中，会阻塞当前线程直到切换完成 + */ + public C getCurrent(boolean blockWhenMigration) { + checkInit(); + if (status == MigrationStatus.MIGRATING && blockWhenMigration) { + // 如果组件正在迁移中，并且 blockWhenMigration = true, 需要等待迁移完成；当前线程阻塞 + try { + synchronized (lock) { + log.info("Component is migrating, block until migrated"); + lock.wait(); + log.info("Continue after component migrated"); + } + } catch (InterruptedException e) { + log.error("Get current component error", e); + } + } + return current; + } + + private void checkInit() { + if (!isInitial) { + synchronized (this) { + if (!isInitial) { + // 从配置文件读取组件名称，并初始化 + String propValue = propToggleStore.getPropToggle(migrateTargetPropName).getDefaultValue(); + log.info("Init default component by prop: {}:{}", migrateTargetPropName, propValue); + this.current = getComponentByProp(this.candidateComponents, propValue); + if (this.current == null) { + log.error("No match component for {}", propValue); + throw new IllegalArgumentException("Unsupported component define by prop: " + propValue); + } + log.info("Use {} as default component", current.getClass()); + this.isInitial = true; + } + } + } + } + + + @Override + public void handlePropChangeEvent(String propName, PropToggle currentValue) { + log.info("Handle prop change event, propName: {}, value: {}", propName, currentValue.getDefaultValue()); + C targetComponent = getComponentByProp(this.candidateComponents, currentValue.getDefaultValue()); + if (targetComponent == null) { + log.warn("No match component for prop : {}", currentValue.getDefaultValue()); + return; + } + if (targetComponent == current) { + // 迁移目标不变，无需处理 + log.info("Component is not changed, skip migration!"); + return; + } + + migrateComponent(targetComponent); + } + + private void migrateComponent(C targetComponent) { + long startTime = System.currentTimeMillis(); + log.info("Migrate component from {} to {} start...", current.getClass(), targetComponent.getClass()); + boolean success = false; + try { + PropToggle migrateServiceInstanceCountProp = + propToggleStore.getPropToggle(PROP_NAME_SERVICE_INSTANCE_COUNT); + if (migrateServiceInstanceCountProp == null) { + log.error("Prop [" + PROP_NAME_SERVICE_INSTANCE_COUNT + "] is required. Skip migration"); + return; + } + int migrateServiceInstanceCount = Integer.parseInt(migrateServiceInstanceCountProp.getDefaultValue()); + if (status == NOT_START) { + synchronized (this) { + if (status == NOT_START) { + status = PREPARING; + initServiceInstanceMigrationStatus(); + // 为了在同一时间(接近）在多个微服务实例切换到新的组件，需要判断处于 PREPARING 状态的服务实例是否符合预期数量 + updateServiceInstanceMigrationStatus(PREPARING); + while (true) { + if (System.currentTimeMillis() - startTime > 60000L) { + // 超过一分钟，放弃本次迁移 + log.info("Prepare migration cost 1min, terminate migration"); + updateServiceInstanceMigrationStatus(MigrationStatus.FAIL); + return; + } + long prepareServiceInstanceCount = + redisTemplate.opsForHash().size(serviceInstanceMigrationStatusRedisKey); + if (prepareServiceInstanceCount >= migrateServiceInstanceCount) { + log.info("All service node are ready, actualReadyNodeCount: {}, expected: {}", + prepareServiceInstanceCount, migrateServiceInstanceCount); + // 所有服务实例都确认收到组件迁移事件，准备启动迁移 + updateGlobalMigrationStatus(MIGRATING); + updateServiceInstanceMigrationStatus(MIGRATING); + // 等待 5s，等待当前组件正在执行的操作完成（比如 db 读写请求) + log.info("Wait 5s before migration"); + ThreadUtils.sleep(5000L); + break; + } else { + log.info("Wait all service instance ready, actual: {}, expected: {}", + prepareServiceInstanceCount, migrateServiceInstanceCount); + ThreadUtils.sleep(100L); + } + } + this.current = targetComponent; + updateServiceInstanceMigrationStatus(MIGRATED); + success = true; + } else { + log.warn("Unexpected migration status {}, skip migration", status); + } + } + } else { + log.warn("Unexpected migration status {}, skip migration", status); + } + } catch (Throwable e) { + log.error("Migrate component error", e); + } finally { + status = NOT_START; + clearAfterMigrated(); + synchronized (lock) { + lock.notifyAll(); + } + log.info("Migrate component done, isSuccess : {}", success); + } + } + + private void clearAfterMigrated() { + try { + List allServiceInstanceMigStatus = + redisTemplate.opsForHash().values(serviceInstanceMigrationStatusRedisKey); + List serviceInstancesMigStatusList = + castList(allServiceInstanceMigStatus, k -> MigrationStatus.valOf((Integer) k)); + if (serviceInstancesMigStatusList.stream().allMatch(status -> status == MIGRATED || status == FAIL)) { + // 所有服务实例都完成了组件迁移，开始清理 + log.info("All migration done. Delete all migration temporary data"); + redisTemplate.delete(serviceInstanceMigrationStatusRedisKey); + redisTemplate.delete(globalMigrationStatusRedisKey); + } else { + log.info("Some service instance node not yet complete migration"); + } + } catch (Throwable e) { + log.error("Clear migration caught exception", e); + } + } + + private void updateServiceInstanceMigrationStatus(MigrationStatus migrationStatus) { + log.info("Update service instance migration status {}", migrationStatus); + redisTemplate.opsForHash().put(serviceInstanceMigrationStatusRedisKey, + serviceNodeIp, migrationStatus.getStatus()); + this.status = migrationStatus; + } + + private void updateGlobalMigrationStatus(MigrationStatus migrationStatus) { + log.info("Update global migration status {}", migrationStatus); + redisTemplate.opsForValue().set(globalMigrationStatusRedisKey, + migrationStatus.getStatus()); + } + + private List castList(List source, Function mapping) { + if (CollectionUtils.isEmpty(source)) { + return Collections.emptyList(); + } + List list = new ArrayList<>(source.size()); + list.addAll(source.stream().map(mapping).collect(Collectors.toList())); + return list; + } + + private void initServiceInstanceMigrationStatus() { + MigrationStatus migrationStatus = getGlobalMigrationStatus(); + if (migrationStatus == null) { + boolean locked = false; + try { + locked = lockForUpdate(); + if (locked) { + // 二次确认，保证获取锁期间数据没有发生改变 + migrationStatus = getGlobalMigrationStatus(); + if (migrationStatus == null) { + // 初始化迁移状态 + log.info("Init migration status"); + redisTemplate.opsForValue().set(globalMigrationStatusRedisKey, PREPARING.getStatus()); + redisTemplate.delete(serviceInstanceMigrationStatusRedisKey); + } + } + } finally { + if (locked) { + unlock(); + } + } + } + } + + private MigrationStatus getGlobalMigrationStatus() { + Object migrationStatus = redisTemplate.opsForValue().get(globalMigrationStatusRedisKey); + if (migrationStatus == null) { + return null; + } + return MigrationStatus.valOf((Integer) migrationStatus); + } + + private boolean lockForUpdate() { + return LockUtils.lock(migrationLockRedisKey, serviceNodeIp, 60000L, 60); + } + + private boolean unlock() { + return LockUtils.releaseDistributedLock(migrationLockRedisKey, serviceNodeIp); + } + + protected abstract C getComponentByProp(Map candidateComponents, String propValue); +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AutoIncrementIdGen.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AutoIncrementIdGen.java new file mode 100644 index 0000000000..235a3cef19 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AutoIncrementIdGen.java @@ -0,0 +1,82 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.common; + +public class AutoIncrementIdGen implements IdGen { + @Override + public Long genTaskInstanceId() { + return null; + } + + @Override + public Long genStepInstanceId() { + return null; + } + + @Override + public Long genGseTaskId() { + return null; + } + + @Override + public Long genOperationLogId() { + return null; + } + + @Override + public Long genFileSourceTaskLogId() { + return null; + } + + @Override + public Long genGseFileExecuteObjTaskId() { + return null; + } + + @Override + public Long genGseScriptExecuteObjTaskId() { + return null; + } + + @Override + public Long genRollingConfigId() { + return null; + } + + @Override + public Long genStepInstanceRollingTaskId() { + return null; + } + + @Override + public Long genStepInstanceVariableId() { + return null; + } + + @Override + public Long genTaskInstanceVariableId() { + return null; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/IdGen.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/IdGen.java new file mode 100644 index 0000000000..cd542b9431 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/IdGen.java @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.common; + +/** + * ID 生成器 + */ +public interface IdGen { + Long genTaskInstanceId(); + + Long genStepInstanceId(); + + Long genGseTaskId(); + + Long genOperationLogId(); + + Long genFileSourceTaskLogId(); + + Long genGseFileExecuteObjTaskId(); + + Long genGseScriptExecuteObjTaskId(); + + Long genRollingConfigId(); + + Long genStepInstanceRollingTaskId(); + + Long genStepInstanceVariableId(); + + Long genTaskInstanceVariableId(); +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/PropBasedDynamicDataSource.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/PropBasedDynamicDataSource.java index 07531e2f27..d3c8df2ab9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/PropBasedDynamicDataSource.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/PropBasedDynamicDataSource.java @@ -27,102 +27,38 @@ import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; import com.tencent.bk.job.common.mysql.dynamic.ds.DataSourceMode; import com.tencent.bk.job.common.mysql.dynamic.ds.DbOperationEnum; -import com.tencent.bk.job.common.mysql.dynamic.ds.MigrationStatus; import com.tencent.bk.job.common.mysql.dynamic.ds.MySQLOperation; import com.tencent.bk.job.common.mysql.dynamic.ds.StandaloneDSLContextProvider; import com.tencent.bk.job.common.mysql.dynamic.ds.VerticalShardingDSLContextProvider; -import com.tencent.bk.job.common.redis.util.LockUtils; -import com.tencent.bk.job.common.util.ThreadUtils; -import com.tencent.bk.job.common.util.ip.IpUtils; -import com.tencent.bk.job.common.util.toggle.prop.PropChangeEventListener; -import com.tencent.bk.job.common.util.toggle.prop.PropToggle; import com.tencent.bk.job.common.util.toggle.prop.PropToggleStore; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; import org.springframework.data.redis.core.RedisTemplate; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static com.tencent.bk.job.common.mysql.dynamic.ds.MigrationStatus.FAIL; -import static com.tencent.bk.job.common.mysql.dynamic.ds.MigrationStatus.MIGRATED; -import static com.tencent.bk.job.common.mysql.dynamic.ds.MigrationStatus.MIGRATING; -import static com.tencent.bk.job.common.mysql.dynamic.ds.MigrationStatus.NOT_START; -import static com.tencent.bk.job.common.mysql.dynamic.ds.MigrationStatus.PREPARING; +import java.util.HashMap; +import java.util.Map; /** * 基于属性动态控制的数据源，可以根据属性值切换到不同的数据源 */ @Slf4j -public class PropBasedDynamicDataSource implements PropChangeEventListener { - - private final StandaloneDSLContextProvider standaloneDSLContextProvider; - - private final VerticalShardingDSLContextProvider verticalShardingDSLContextProvider; - - private volatile MigrationStatus status = NOT_START; - - private final Object lock = new Object(); - - private final RedisTemplate redisTemplate; - - /** - * 当前数据源模式 - */ - private volatile DataSourceMode currentDataSourceMode; - - /** - * 当前 DSLContextProvider - */ - private volatile DSLContextProvider currentContextProvider; - - private final PropToggleStore propToggleStore; - - private volatile boolean isInitial = false; - - /** - * 动态属性名称 - 迁移目标数据源类型 - */ - private static final String PROP_NAME_MIGRATE_TARGET_DATASOURCE_MODE = - "job_execute_mysql_migration_target_datasource_mode"; - /** - * 动态属性名称 - 参与 db 迁移的微服务实例数量 - */ - private static final String PROP_NAME_SERVICE_INSTANCE_COUNT = "job_execute_mysql_migration_service_instance_count"; - - /** - * 迁移使用的分布式锁 Redis KEY - */ - private static final String DB_MIGRATION_LOCK_KEY = "job:execute:mysql:migration"; - /** - * 整体迁移状态 Redis KEY - */ - private static final String REDIS_KEY_GLOBAL_MIGRATION_STATUS = "job:execute:mysql:migration:global:status"; - /** - * 服务实例迁移状态 Redis KEY - */ - private static final String REDIS_KEY_SERVICE_INSTANCE_MIGRATION_STATUS = - "job:execute:mysql:migration:service:instance:status"; - /** - * 服务节点 IP - */ - private final String serviceNodeIp; +public class PropBasedDynamicDataSource extends AbstractPropBasedDynamicComponent { public PropBasedDynamicDataSource( StandaloneDSLContextProvider standaloneDSLContextProvider, VerticalShardingDSLContextProvider verticalShardingDSLContextProvider, RedisTemplate redisTemplate, PropToggleStore propToggleStore) { - this.standaloneDSLContextProvider = standaloneDSLContextProvider; - this.verticalShardingDSLContextProvider = verticalShardingDSLContextProvider; - this.redisTemplate = redisTemplate; - this.propToggleStore = propToggleStore; - this.serviceNodeIp = IpUtils.getFirstMachineIP(); - // 注册监听属性变化 - this.propToggleStore.addPropChangeEventListener(PROP_NAME_MIGRATE_TARGET_DATASOURCE_MODE, this); + super(redisTemplate, propToggleStore, "execute_mysql"); + + Map candidateDSLContextProviders = new HashMap<>(); + if (standaloneDSLContextProvider != null) { + candidateDSLContextProviders.put(DataSourceMode.Constants.STANDALONE, standaloneDSLContextProvider); + } + if (verticalShardingDSLContextProvider != null) { + candidateDSLContextProviders.put(DataSourceMode.Constants.VERTICAL_SHARDING, + verticalShardingDSLContextProvider); + } + super.initCandidateComponents(candidateDSLContextProviders); } /** @@ -132,224 +68,20 @@ public PropBasedDynamicDataSource( * @return DSLContextProvider */ public DSLContextProvider getCurrent(MySQLOperation op) { - checkInit(); - // 迁移时阻塞写请求，读请求不影响 if (op.op() == DbOperationEnum.WRITE) { - if (status == MigrationStatus.MIGRATING) { - // 如果数据源正在迁移中，需要等待迁移完成；当前线程阻塞 - try { - synchronized (lock) { - log.info("Datasource is migrating, block until migrated"); - lock.wait(); - log.info("Continue after datasource migrated"); - } - } catch (InterruptedException e) { - log.error("Get current DSLContext error", e); - } - } - } - return currentContextProvider; - } - - private void checkInit() { - if (!isInitial) { - synchronized (this) { - if (!isInitial) { - // 从配置文件读取数据源模式，初始化当前数据源 - DataSourceMode dataSourceMode = DataSourceMode.valOf( - propToggleStore.getPropToggle(PROP_NAME_MIGRATE_TARGET_DATASOURCE_MODE).getDefaultValue()); - log.info("Init default DSLContextProvider, dataSourceMode: {}", dataSourceMode); - this.currentDataSourceMode = dataSourceMode; - this.currentContextProvider = chooseDefaultDSLContextProviderByMode(dataSourceMode); - log.info("Use {} as default DSLContextProvider", currentContextProvider.getClass()); - this.isInitial = true; - } - } - } - } - - private DSLContextProvider chooseDefaultDSLContextProviderByMode(DataSourceMode mode) { - if (mode == DataSourceMode.STANDALONE) { - if (standaloneDSLContextProvider == null) { - throw new IllegalStateException("StandaloneDSLContextProvider not exist"); - } - return standaloneDSLContextProvider; - } else if (mode == DataSourceMode.VERTICAL_SHARDING) { - if (verticalShardingDSLContextProvider == null) { - throw new IllegalStateException("VerticalShardingDSLContextProvider not exist"); - } - return verticalShardingDSLContextProvider; + return getCurrent(true); } else { - log.error("Unsupported DataSourceMode"); - throw new IllegalArgumentException("Unsupported DataSourceMode"); + return getCurrent(false); } } @Override - public void handlePropChangeEvent(String propName, PropToggle currentValue) { - log.info("Handle prop change event, propName: {}, value: {}", propName, currentValue != null ? - currentValue.getDefaultValue() : null); - String targetDataSourceModeValue = currentValue.getDefaultValue(); - if (!DataSourceMode.checkValid(targetDataSourceModeValue)) { - log.error("Invalid target datasource mode : {}, skip migration", targetDataSourceModeValue); - return; - } - DataSourceMode targetDataSourceMode = DataSourceMode.valOf(targetDataSourceModeValue); - if (targetDataSourceMode == currentDataSourceMode) { - // 迁移目标不变，无需处理 - log.info("DataSourceMode is not changed, skip migration!"); - return; - } - - migrateDataSource(targetDataSourceMode); - } - - private void migrateDataSource(DataSourceMode targetDataSourceMode) { - long startTime = System.currentTimeMillis(); - DSLContextProvider targetDSLContextProvider = chooseDefaultDSLContextProviderByMode(targetDataSourceMode); - log.info("Migrate datasource from {} to {} start...", currentDataSourceMode, targetDataSourceMode); - boolean success = false; - try { - PropToggle migrateServiceInstanceCountProp = - propToggleStore.getPropToggle(PROP_NAME_SERVICE_INSTANCE_COUNT); - if (migrateServiceInstanceCountProp == null) { - log.error("Prop [" + PROP_NAME_SERVICE_INSTANCE_COUNT + "] is required. Skip migration"); - return; - } - int migrateServiceInstanceCount = Integer.parseInt(migrateServiceInstanceCountProp.getDefaultValue()); - if (status == NOT_START) { - synchronized (this) { - if (status == NOT_START) { - status = PREPARING; - initServiceInstanceMigrationStatus(); - // 为了在同一时间(接近）在多个微服务实例切换到新的数据源，需要判断处于 PREPARING 状态的服务实例是否符合预期数量 - updateServiceInstanceMigrationStatus(PREPARING); - while (true) { - if (System.currentTimeMillis() - startTime > 60000L) { - // 超过一分钟，放弃本次迁移 - log.info("Prepare migration cost 1min, terminate migration"); - updateServiceInstanceMigrationStatus(MigrationStatus.FAIL); - return; - } - long prepareServiceInstanceCount = - redisTemplate.opsForHash().size(REDIS_KEY_SERVICE_INSTANCE_MIGRATION_STATUS); - if (prepareServiceInstanceCount >= migrateServiceInstanceCount) { - log.info("All service node are ready, actualReadyNodeCount: {}, expected: {}", - prepareServiceInstanceCount, migrateServiceInstanceCount); - // 所有服务实例都确认收到DB数据源迁移事件，准备启动迁移 - updateGlobalMigrationStatus(MIGRATING); - updateServiceInstanceMigrationStatus(MIGRATING); - // 等待 5s，等待当前DB数据源正在执行的读写请求都完成 - log.info("Wait 5s before migration"); - ThreadUtils.sleep(5000L); - break; - } else { - log.info("Wait all service instance ready, actual: {}, expected: {}", - prepareServiceInstanceCount, migrateServiceInstanceCount); - ThreadUtils.sleep(100L); - } - } - this.currentDataSourceMode = targetDataSourceMode; - this.currentContextProvider = targetDSLContextProvider; - updateServiceInstanceMigrationStatus(MIGRATED); - success = true; - } else { - log.warn("Unexpected migration status {}, skip migration", status); - } - } - } else { - log.warn("Unexpected migration status {}, skip migration", status); - } - } catch (Throwable e) { - log.error("Migrate datasource error", e); - } finally { - status = NOT_START; - clearAfterMigrated(); - synchronized (lock) { - lock.notifyAll(); - } - log.info("Migrate datasource finish, isSuccess : {}", success); - } - } - - private void clearAfterMigrated() { - try { - List allServiceInstanceMigStatus = - redisTemplate.opsForHash().values(REDIS_KEY_SERVICE_INSTANCE_MIGRATION_STATUS); - List serviceInstancesMigStatusList = - castList(allServiceInstanceMigStatus, k -> MigrationStatus.valOf((Integer) k)); - if (serviceInstancesMigStatusList.stream().allMatch(status -> status == MIGRATED || status == FAIL)) { - // 所有服务实例都完成了 db 迁移，开始清理 - log.info("All migration done. Delete all migration temporary data"); - redisTemplate.delete(REDIS_KEY_SERVICE_INSTANCE_MIGRATION_STATUS); - redisTemplate.delete(REDIS_KEY_GLOBAL_MIGRATION_STATUS); - } else { - log.info("Some service instance node not yet complete migration"); - } - } catch (Throwable e) { - log.error("Clear migration caught exception", e); - } - } - - private void updateServiceInstanceMigrationStatus(MigrationStatus migrationStatus) { - log.info("Update service instance migration status {}", migrationStatus); - redisTemplate.opsForHash().put(REDIS_KEY_SERVICE_INSTANCE_MIGRATION_STATUS, - serviceNodeIp, migrationStatus.getStatus()); - this.status = migrationStatus; - } - - private void updateGlobalMigrationStatus(MigrationStatus migrationStatus) { - log.info("Update global migration status {}", migrationStatus); - redisTemplate.opsForValue().set(REDIS_KEY_GLOBAL_MIGRATION_STATUS, - migrationStatus.getStatus()); - } - - private List castList(List source, Function mapping) { - if (CollectionUtils.isEmpty(source)) { - return Collections.emptyList(); - } - List list = new ArrayList<>(source.size()); - list.addAll(source.stream().map(mapping).collect(Collectors.toList())); - return list; - } - - private void initServiceInstanceMigrationStatus() { - MigrationStatus migrationStatus = getGlobalMigrationStatus(); - if (migrationStatus == null) { - boolean locked = false; - try { - locked = lockForUpdate(); - if (locked) { - // 二次确认，保证获取锁期间数据没有发生改变 - migrationStatus = getGlobalMigrationStatus(); - if (migrationStatus == null) { - // 初始化迁移状态 - log.info("Init migration status"); - redisTemplate.opsForValue().set(REDIS_KEY_GLOBAL_MIGRATION_STATUS, PREPARING.getStatus()); - redisTemplate.delete(REDIS_KEY_SERVICE_INSTANCE_MIGRATION_STATUS); - } - } - } finally { - if (locked) { - unlock(); - } - } - } - } - - private MigrationStatus getGlobalMigrationStatus() { - Object migrationStatus = redisTemplate.opsForValue().get(REDIS_KEY_GLOBAL_MIGRATION_STATUS); - if (migrationStatus == null) { + protected DSLContextProvider getComponentByProp(Map candidateComponents, + String propValue) { + if (!DataSourceMode.checkValid(propValue)) { + log.error("Invalid target datasource mode : {}, skip migration", propValue); return null; } - return MigrationStatus.valOf((Integer) migrationStatus); - } - - private boolean lockForUpdate() { - return LockUtils.lock(DB_MIGRATION_LOCK_KEY, serviceNodeIp, 60000L, 60); - } - - private boolean unlock() { - return LockUtils.releaseDistributedLock(DB_MIGRATION_LOCK_KEY, serviceNodeIp); + return candidateComponents.get(propValue); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/PropBasedDynamicIdGen.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/PropBasedDynamicIdGen.java new file mode 100644 index 0000000000..8ea22a6bc4 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/PropBasedDynamicIdGen.java @@ -0,0 +1,124 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.common; + +import com.tencent.bk.job.common.mysql.dynamic.id.IdGenType; +import com.tencent.bk.job.common.util.toggle.prop.PropToggleStore; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.RedisTemplate; + +import java.util.HashMap; +import java.util.Map; + +/** + * 基于属性动态控制的ID生成器 + */ +@Slf4j +public class PropBasedDynamicIdGen extends AbstractPropBasedDynamicComponent implements IdGen { + + public PropBasedDynamicIdGen( + AutoIncrementIdGen autoIncrementIdGenerator, + SegmentIdGen segmentIdGenerator, + RedisTemplate redisTemplate, + PropToggleStore propToggleStore) { + super(redisTemplate, propToggleStore, "execute_id_gen"); + Map candidateIdGens = new HashMap<>(); + if (autoIncrementIdGenerator != null) { + candidateIdGens.put(IdGenType.Constants.AUTO_INCREMENT, autoIncrementIdGenerator); + } + if (segmentIdGenerator != null) { + candidateIdGens.put(IdGenType.Constants.SEGMENT, segmentIdGenerator); + } + super.initCandidateComponents(candidateIdGens); + } + + private IdGen currentIdGen() { + return getCurrent(true); + } + + @Override + protected IdGen getComponentByProp(Map candidateComponents, String propValue) { + if (!IdGenType.checkValid(propValue)) { + log.error("Invalid target IdGenType : {}, skip migration", propValue); + return null; + } + return candidateComponents.get(propValue); + } + + @Override + public Long genTaskInstanceId() { + return currentIdGen().genTaskInstanceId(); + } + + @Override + public Long genStepInstanceId() { + return currentIdGen().genStepInstanceId(); + } + + @Override + public Long genGseTaskId() { + return currentIdGen().genGseTaskId(); + } + + @Override + public Long genOperationLogId() { + return currentIdGen().genOperationLogId(); + } + + @Override + public Long genFileSourceTaskLogId() { + return currentIdGen().genFileSourceTaskLogId(); + } + + @Override + public Long genGseFileExecuteObjTaskId() { + return currentIdGen().genGseFileExecuteObjTaskId(); + } + + @Override + public Long genGseScriptExecuteObjTaskId() { + return currentIdGen().genGseScriptExecuteObjTaskId(); + } + + @Override + public Long genRollingConfigId() { + return currentIdGen().genRollingConfigId(); + } + + @Override + public Long genStepInstanceRollingTaskId() { + return currentIdGen().genStepInstanceRollingTaskId(); + } + + @Override + public Long genStepInstanceVariableId() { + return currentIdGen().genStepInstanceVariableId(); + } + + @Override + public Long genTaskInstanceVariableId() { + return currentIdGen().genTaskInstanceVariableId(); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/SegmentIdGen.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/SegmentIdGen.java new file mode 100644 index 0000000000..291661406c --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/SegmentIdGen.java @@ -0,0 +1,85 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.common; + +import com.tencent.bk.job.common.sharding.mysql.SegmentIdKeys; +import com.tencent.devops.leaf.service.SegmentService; + +public class SegmentIdGen implements IdGen { + + private final SegmentService segmentService; + + public SegmentIdGen(SegmentService segmentServices) { + this.segmentService = segmentServices; + } + + private Long gen(String key) { + return segmentService.getId(key).getId(); + } + + public Long genTaskInstanceId() { + return gen(SegmentIdKeys.KEY_JOB_EXECUTE_TASK_INSTANCE); + } + + public Long genStepInstanceId() { + return gen(SegmentIdKeys.KEY_JOB_EXECUTE_STEP_INSTANCE); + } + + public Long genGseTaskId() { + return gen(SegmentIdKeys.KEY_JOB_EXECUTE_GSE_TASK); + } + + public Long genOperationLogId() { + return gen(SegmentIdKeys.KEY_JOB_EXECUTE_OPERATION_LOG); + } + + public Long genFileSourceTaskLogId() { + return gen(SegmentIdKeys.KEY_JOB_EXECUTE_FILE_SOURCE_TASK_LOG); + } + + public Long genGseFileExecuteObjTaskId() { + return gen(SegmentIdKeys.KEY_JOB_EXECUTE_GSE_FILE_EXECUTE_OBJ_TASK); + } + + public Long genGseScriptExecuteObjTaskId() { + return gen(SegmentIdKeys.KEY_JOB_EXECUTE_GSE_SCRIPT_EXECUTE_OBJ_TASK); + } + + public Long genRollingConfigId() { + return gen(SegmentIdKeys.KEY_JOB_EXECUTE_ROLLING_CONFIG); + } + + public Long genStepInstanceRollingTaskId() { + return gen(SegmentIdKeys.KEY_JOB_EXECUTE_STEP_INSTANCE_ROLLING_TASK); + } + + public Long genStepInstanceVariableId() { + return gen(SegmentIdKeys.KEY_JOB_EXECUTE_STEP_INSTANCE_VARIABLE); + } + + public Long genTaskInstanceVariableId() { + return gen(SegmentIdKeys.KEY_JOB_EXECUTE_TASK_INSTANCE_VARIABLE); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareService.java index 69fb5b846b..1e9db39052 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareService.java @@ -28,7 +28,6 @@ import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.file.PathUtil; -import com.tencent.bk.job.execute.dao.FileSourceTaskLogDAO; import com.tencent.bk.job.execute.engine.listener.event.GseTaskEvent; import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; import com.tencent.bk.job.execute.engine.prepare.JobTaskContext; @@ -40,6 +39,7 @@ import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.service.AccountService; +import com.tencent.bk.job.execute.service.FileSourceTaskLogService; import com.tencent.bk.job.execute.service.LogService; import com.tencent.bk.job.execute.service.StepInstanceService; import com.tencent.bk.job.file_gateway.api.inner.ServiceFileSourceTaskResource; @@ -74,7 +74,7 @@ public class ThirdFilePrepareService { private final ResultHandleManager resultHandleManager; private final ServiceFileSourceTaskResource fileSourceTaskResource; private final StepInstanceService stepInstanceService; - private final FileSourceTaskLogDAO fileSourceTaskLogDAO; + private final FileSourceTaskLogService fileSourceTaskLogService; private final AccountService accountService; private final FileWorkerHostService fileWorkerHostService; private final LogService logService; @@ -86,7 +86,7 @@ public class ThirdFilePrepareService { public ThirdFilePrepareService(ResultHandleManager resultHandleManager, ServiceFileSourceTaskResource fileSourceTaskResource, StepInstanceService stepInstanceService, - FileSourceTaskLogDAO fileSourceTaskLogDAO, + FileSourceTaskLogService fileSourceTaskLogService, AccountService accountService, FileWorkerHostService fileWorkerHostService, LogService logService, @@ -94,7 +94,7 @@ public ThirdFilePrepareService(ResultHandleManager resultHandleManager, this.resultHandleManager = resultHandleManager; this.fileSourceTaskResource = fileSourceTaskResource; this.stepInstanceService = stepInstanceService; - this.fileSourceTaskLogDAO = fileSourceTaskLogDAO; + this.fileSourceTaskLogService = fileSourceTaskLogService; this.accountService = accountService; this.fileWorkerHostService = fileWorkerHostService; this.logService = logService; @@ -255,7 +255,7 @@ private void insertOrUpdateFileSourceTaskLog(FileSourceTaskLogDTO fileSourceTask boolean shouldRetry; do { try { - int insertedNum = fileSourceTaskLogDAO.insertFileSourceTaskLog(fileSourceTaskLogDTO); + int insertedNum = fileSourceTaskLogService.addFileSourceTaskLog(fileSourceTaskLogDTO); log.info("{} fileSourceTaskLog inserted", insertedNum); return; } catch (DataAccessException e) { @@ -269,7 +269,7 @@ private void insertOrUpdateFileSourceTaskLog(FileSourceTaskLogDTO fileSourceTask } } } while (shouldRetry); - int updatedNum = fileSourceTaskLogDAO.updateFileSourceTaskLogByStepInstance(fileSourceTaskLogDTO); + int updatedNum = fileSourceTaskLogService.updateFileSourceTaskLog(fileSourceTaskLogDTO); log.info("{} fileSourceTaskLog updated", updatedNum); } @@ -397,7 +397,7 @@ private ThirdFilePrepareTask asyncWatchThirdFilePulling( new RecordableThirdFilePrepareTaskResultHandler(stepInstance, resultHandler)); batchResultHandleTask.initDependentService( fileSourceTaskResource, stepInstanceService, accountService, - fileWorkerHostService, logService, taskExecuteMQEventDispatcher, fileSourceTaskLogDAO + fileWorkerHostService, logService, taskExecuteMQEventDispatcher, fileSourceTaskLogService ); resultHandleManager.handleDeliveredTask(batchResultHandleTask); return batchResultHandleTask; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareTask.java index 927bf532df..1ed9f84eec 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareTask.java @@ -31,7 +31,6 @@ import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.execute.common.constants.FileDistStatusEnum; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.dao.FileSourceTaskLogDAO; import com.tencent.bk.job.execute.engine.listener.event.EventSource; import com.tencent.bk.job.execute.engine.listener.event.JobEvent; import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; @@ -48,6 +47,7 @@ import com.tencent.bk.job.execute.model.FileSourceTaskLogDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.service.AccountService; +import com.tencent.bk.job.execute.service.FileSourceTaskLogService; import com.tencent.bk.job.execute.service.LogService; import com.tencent.bk.job.execute.service.StepInstanceService; import com.tencent.bk.job.file_gateway.api.inner.ServiceFileSourceTaskResource; @@ -95,7 +95,7 @@ public class ThirdFilePrepareTask implements ContinuousScheduledTask, JobTaskCon private FileWorkerHostService fileWorkerHostService; private LogService logService; private TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; - private FileSourceTaskLogDAO fileSourceTaskLogDAO; + private FileSourceTaskLogService fileSourceTaskLogService; private final ThirdFilePrepareTaskResultHandler resultHandler; private StepInstanceService stepInstanceService; private int pullTimes = 0; @@ -129,14 +129,14 @@ public void initDependentService( FileWorkerHostService fileWorkerHostService, LogService logService, TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, - FileSourceTaskLogDAO fileSourceTaskLogDAO + FileSourceTaskLogService fileSourceTaskLogService ) { this.fileSourceTaskResource = fileSourceTaskResource; this.accountService = accountService; this.fileWorkerHostService = fileWorkerHostService; this.logService = logService; this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; - this.fileSourceTaskLogDAO = fileSourceTaskLogDAO; + this.fileSourceTaskLogService = fileSourceTaskLogService; this.stepInstanceService = stepInstanceService; } @@ -289,13 +289,13 @@ private void handleFileSourceTaskResult( } private void updateBatchTaskTimeStatistics() { - FileSourceTaskLogDTO fileSourceTaskLogDTO = fileSourceTaskLogDAO.getFileSourceTaskLogByBatchTaskId( + FileSourceTaskLogDTO fileSourceTaskLogDTO = fileSourceTaskLogService.getFileSourceTaskLogByBatchTaskId( stepInstance.getTaskInstanceId(), batchTaskId); if (fileSourceTaskLogDTO == null) { return; } Long endTime = System.currentTimeMillis(); - fileSourceTaskLogDAO.updateTimeConsumingByBatchTaskId( + fileSourceTaskLogService.updateTimeConsumingByBatchTaskId( stepInstance.getTaskInstanceId(), batchTaskId, null, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileSourceTaskLogService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileSourceTaskLogService.java new file mode 100644 index 0000000000..1654d87a7d --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileSourceTaskLogService.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.service; + +import com.tencent.bk.job.execute.model.FileSourceTaskLogDTO; + +public interface FileSourceTaskLogService { + int addFileSourceTaskLog(FileSourceTaskLogDTO fileSourceTaskLog); + + int updateFileSourceTaskLog(FileSourceTaskLogDTO fileSourceTaskLog); + + FileSourceTaskLogDTO getFileSourceTaskLog(Long taskInstanceId, long stepInstanceId, int executeCount); + + FileSourceTaskLogDTO getFileSourceTaskLogByBatchTaskId(Long taskInstanceId, String fileSourceBatchTaskId); + + int updateTimeConsumingByBatchTaskId(Long taskInstanceId, + String fileSourceBatchTaskId, + Long startTime, + Long endTime, + Long totalTime); +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/RollingConfigService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/RollingConfigService.java index 6dfc87296c..027528175f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/RollingConfigService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/RollingConfigService.java @@ -60,4 +60,6 @@ public interface RollingConfigService { * @return boolean true启用，false未启用 */ boolean isTaskRollingEnabled(long taskInstanceId); + + long addRollingConfig(RollingConfigDTO rollingConfig); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileExecuteObjectTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileExecuteObjectTaskServiceImpl.java index 43a92e5cce..25a1b42d89 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileExecuteObjectTaskServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileExecuteObjectTaskServiceImpl.java @@ -3,6 +3,7 @@ import com.tencent.bk.job.common.constant.Order; import com.tencent.bk.job.execute.dao.FileAgentTaskDAO; import com.tencent.bk.job.execute.dao.FileExecuteObjectTaskDAO; +import com.tencent.bk.job.execute.dao.common.IdGen; import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; import com.tencent.bk.job.execute.model.ExecuteObjectTask; @@ -30,14 +31,17 @@ public class FileExecuteObjectTaskServiceImpl private final FileExecuteObjectTaskDAO fileExecuteObjectTaskDAO; private final FileAgentTaskDAO fileAgentTaskDAO; + private final IdGen idGen; @Autowired public FileExecuteObjectTaskServiceImpl(StepInstanceService stepInstanceService, FileExecuteObjectTaskDAO fileExecuteObjectTaskDAO, - FileAgentTaskDAO fileAgentTaskDAO) { + FileAgentTaskDAO fileAgentTaskDAO, + IdGen idGen) { super(stepInstanceService); this.fileAgentTaskDAO = fileAgentTaskDAO; this.fileExecuteObjectTaskDAO = fileExecuteObjectTaskDAO; + this.idGen = idGen; } @Override @@ -45,6 +49,7 @@ public void batchSaveTasks(Collection tasks) { if (CollectionUtils.isEmpty(tasks)) { return; } + tasks.forEach(task -> task.setId(idGen.genGseFileExecuteObjTaskId())); if (isSaveTasksUsingExecuteObjectMode(tasks)) { fileExecuteObjectTaskDAO.batchSaveTasks(tasks); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileSourceTaskLogServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileSourceTaskLogServiceImpl.java new file mode 100644 index 0000000000..3d43d1fc60 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileSourceTaskLogServiceImpl.java @@ -0,0 +1,82 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.service.impl; + +import com.tencent.bk.job.execute.dao.FileSourceTaskLogDAO; +import com.tencent.bk.job.execute.dao.common.IdGen; +import com.tencent.bk.job.execute.model.FileSourceTaskLogDTO; +import com.tencent.bk.job.execute.service.FileSourceTaskLogService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +@Service +@Slf4j +public class FileSourceTaskLogServiceImpl implements FileSourceTaskLogService { + + private final FileSourceTaskLogDAO fileSourceTaskLogDAO; + + private final IdGen idGen; + + public FileSourceTaskLogServiceImpl(FileSourceTaskLogDAO fileSourceTaskLogDAO, IdGen idGen) { + this.fileSourceTaskLogDAO = fileSourceTaskLogDAO; + this.idGen = idGen; + } + + @Override + public int addFileSourceTaskLog(FileSourceTaskLogDTO fileSourceTaskLog) { + fileSourceTaskLog.setId(idGen.genFileSourceTaskLogId()); + return fileSourceTaskLogDAO.insertFileSourceTaskLog(fileSourceTaskLog); + } + + @Override + public int updateFileSourceTaskLog(FileSourceTaskLogDTO fileSourceTaskLog) { + return fileSourceTaskLogDAO.updateFileSourceTaskLogByStepInstance(fileSourceTaskLog); + } + + @Override + public FileSourceTaskLogDTO getFileSourceTaskLog(Long taskInstanceId, long stepInstanceId, int executeCount) { + return fileSourceTaskLogDAO.getFileSourceTaskLog(taskInstanceId, stepInstanceId, executeCount); + } + + @Override + public FileSourceTaskLogDTO getFileSourceTaskLogByBatchTaskId(Long taskInstanceId, String fileSourceBatchTaskId) { + return fileSourceTaskLogDAO.getFileSourceTaskLogByBatchTaskId(taskInstanceId, fileSourceBatchTaskId); + } + + @Override + public int updateTimeConsumingByBatchTaskId(Long taskInstanceId, + String fileSourceBatchTaskId, + Long startTime, + Long endTime, + Long totalTime) { + return fileSourceTaskLogDAO.updateTimeConsumingByBatchTaskId( + taskInstanceId, + fileSourceBatchTaskId, + startTime, + endTime, + totalTime + ); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/GseTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/GseTaskServiceImpl.java index 97e05078f3..cdff344f3a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/GseTaskServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/GseTaskServiceImpl.java @@ -25,6 +25,7 @@ package com.tencent.bk.job.execute.service.impl; import com.tencent.bk.job.execute.dao.GseTaskDAO; +import com.tencent.bk.job.execute.dao.common.IdGen; import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.GseTaskSimpleDTO; import com.tencent.bk.job.execute.service.GseTaskService; @@ -39,14 +40,17 @@ public class GseTaskServiceImpl implements GseTaskService { private final GseTaskDAO gseTaskDAO; + private final IdGen idGen; @Autowired - public GseTaskServiceImpl(GseTaskDAO gseTaskDAO) { + public GseTaskServiceImpl(GseTaskDAO gseTaskDAO, IdGen idGen) { this.gseTaskDAO = gseTaskDAO; + this.idGen = idGen; } @Override public Long saveGseTask(GseTaskDTO gseTask) { + gseTask.setId(idGen.genGseTaskId()); return gseTaskDAO.saveGseTask(gseTask); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/RollingConfigServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/RollingConfigServiceImpl.java index 8fabee9e7c..dfe5679450 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/RollingConfigServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/RollingConfigServiceImpl.java @@ -28,6 +28,7 @@ import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.execute.dao.RollingConfigDAO; +import com.tencent.bk.job.execute.dao.common.IdGen; import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.engine.rolling.RollingBatchExecuteObjectsResolver; import com.tencent.bk.job.execute.engine.rolling.RollingExecuteObjectBatch; @@ -55,10 +56,13 @@ public class RollingConfigServiceImpl implements RollingConfigService { private final RollingConfigDAO rollingConfigDAO; + private final IdGen idGen; + @Autowired - public RollingConfigServiceImpl(RollingConfigDAO rollingConfigDAO) { + public RollingConfigServiceImpl(RollingConfigDAO rollingConfigDAO, IdGen idGen) { this.rollingConfigDAO = rollingConfigDAO; + this.idGen = idGen; } @Override @@ -124,7 +128,7 @@ public RollingConfigDTO saveRollingConfigForFastJob(FastTaskDTO fastTask) { stepRollingConfigs.put(stepInstance.getId(), new StepRollingConfigDO(true)); rollingConfigDetailDO.setStepRollingConfigs(stepRollingConfigs); - Long rollingConfigId = rollingConfigDAO.saveRollingConfig(taskInstanceRollingConfig); + Long rollingConfigId = addRollingConfig(taskInstanceRollingConfig); taskInstanceRollingConfig.setId(rollingConfigId); return taskInstanceRollingConfig; } @@ -138,4 +142,10 @@ public RollingConfigDTO getRollingConfig(long rollingConfigId) { public boolean isTaskRollingEnabled(long taskInstanceId) { return rollingConfigDAO.existsRollingConfig(taskInstanceId); } + + @Override + public long addRollingConfig(RollingConfigDTO rollingConfig) { + rollingConfig.setId(idGen.genRollingConfigId()); + return rollingConfigDAO.saveRollingConfig(rollingConfig); + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptExecuteObjectTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptExecuteObjectTaskServiceImpl.java index 1f1ec1f01b..97894d1c97 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptExecuteObjectTaskServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptExecuteObjectTaskServiceImpl.java @@ -3,6 +3,7 @@ import com.tencent.bk.job.common.constant.Order; import com.tencent.bk.job.execute.dao.ScriptAgentTaskDAO; import com.tencent.bk.job.execute.dao.ScriptExecuteObjectTaskDAO; +import com.tencent.bk.job.execute.dao.common.IdGen; import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; import com.tencent.bk.job.execute.model.ExecuteObjectTask; @@ -29,14 +30,17 @@ public class ScriptExecuteObjectTaskServiceImpl private final ScriptExecuteObjectTaskDAO scriptExecuteObjectTaskDAO; private final ScriptAgentTaskDAO scriptAgentTaskDAO; + private final IdGen idGen; @Autowired public ScriptExecuteObjectTaskServiceImpl(StepInstanceService stepInstanceService, ScriptExecuteObjectTaskDAO scriptExecuteObjectTaskDAO, - ScriptAgentTaskDAO scriptAgentTaskDAO) { + ScriptAgentTaskDAO scriptAgentTaskDAO, + IdGen idGen) { super(stepInstanceService); this.scriptExecuteObjectTaskDAO = scriptExecuteObjectTaskDAO; this.scriptAgentTaskDAO = scriptAgentTaskDAO; + this.idGen = idGen; } @Override @@ -45,6 +49,7 @@ public void batchSaveTasks(Collection tasks) { return; } + tasks.forEach(task -> task.setId(idGen.genGseScriptExecuteObjTaskId())); if (isSaveTasksUsingExecuteObjectMode(tasks)) { scriptExecuteObjectTaskDAO.batchSaveTasks(tasks); } else { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceRollingTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceRollingTaskServiceImpl.java index d08d670e63..2cc3679653 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceRollingTaskServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceRollingTaskServiceImpl.java @@ -26,6 +26,7 @@ import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.dao.StepInstanceRollingTaskDAO; +import com.tencent.bk.job.execute.dao.common.IdGen; import com.tencent.bk.job.execute.model.StepInstanceRollingTaskDTO; import com.tencent.bk.job.execute.service.StepInstanceRollingTaskService; import lombok.extern.slf4j.Slf4j; @@ -44,11 +45,13 @@ public class StepInstanceRollingTaskServiceImpl implements StepInstanceRollingTaskService { private final StepInstanceRollingTaskDAO stepInstanceRollingTaskDAO; + private final IdGen idGen; @Autowired public StepInstanceRollingTaskServiceImpl( - StepInstanceRollingTaskDAO stepInstanceRollingTaskDAO) { + StepInstanceRollingTaskDAO stepInstanceRollingTaskDAO, IdGen idGen) { this.stepInstanceRollingTaskDAO = stepInstanceRollingTaskDAO; + this.idGen = idGen; } @Override @@ -104,6 +107,7 @@ public List listRollingTasksByBatch(Long taskInstanc @Override public long saveRollingTask(StepInstanceRollingTaskDTO rollingTask) { + rollingTask.setId(idGen.genStepInstanceRollingTaskId()); return stepInstanceRollingTaskDAO.saveRollingTask(rollingTask); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java index c611ebc8a7..7cdf0e778c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java @@ -31,6 +31,7 @@ import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; import com.tencent.bk.job.execute.dao.StepInstanceDAO; +import com.tencent.bk.job.execute.dao.common.IdGen; import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; import com.tencent.bk.job.execute.model.FileSourceDTO; @@ -65,10 +66,12 @@ public class StepInstanceServiceImpl implements StepInstanceService { private final StepInstanceDAO stepInstanceDAO; + private final IdGen idGen; @Autowired - public StepInstanceServiceImpl(StepInstanceDAO stepInstanceDAO) { + public StepInstanceServiceImpl(StepInstanceDAO stepInstanceDAO, IdGen idGen) { this.stepInstanceDAO = stepInstanceDAO; + this.idGen = idGen; } @Override @@ -89,6 +92,7 @@ public StepInstanceBaseDTO getNextStepInstance(Long taskInstanceId, @Override public long addStepInstance(StepInstanceDTO stepInstance) { + stepInstance.setId(idGen.genStepInstanceId()); long stepInstanceId = stepInstanceDAO.addStepInstanceBase(stepInstance); if (stepInstanceId > 0) { stepInstance.setId(stepInstanceId); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceVariableValueServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceVariableValueServiceImpl.java index 78dbf0006c..e07dd8f839 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceVariableValueServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceVariableValueServiceImpl.java @@ -28,6 +28,7 @@ import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.ip.IpUtils; import com.tencent.bk.job.execute.dao.StepInstanceVariableDAO; +import com.tencent.bk.job.execute.dao.common.IdGen; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; import com.tencent.bk.job.execute.engine.model.TaskVariablesAnalyzeResult; import com.tencent.bk.job.execute.model.HostVariableValuesDTO; @@ -57,18 +58,22 @@ public class StepInstanceVariableValueServiceImpl implements StepInstanceVariabl private final StepInstanceVariableDAO stepInstanceVariableDAO; private final StepInstanceService stepInstanceService; private final TaskInstanceVariableService taskInstanceVariableService; + private final IdGen idGen; @Autowired public StepInstanceVariableValueServiceImpl(StepInstanceVariableDAO stepInstanceVariableDAO, StepInstanceService stepInstanceService, - TaskInstanceVariableService taskInstanceVariableService) { + TaskInstanceVariableService taskInstanceVariableService, + IdGen idGen) { this.stepInstanceVariableDAO = stepInstanceVariableDAO; this.stepInstanceService = stepInstanceService; this.taskInstanceVariableService = taskInstanceVariableService; + this.idGen = idGen; } @Override public void saveVariableValues(StepInstanceVariableValuesDTO variableValues) { + variableValues.setId(idGen.genStepInstanceVariableId()); stepInstanceVariableDAO.saveVariableValues(variableValues); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceServiceImpl.java index 38c3934d23..b530c82f65 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceServiceImpl.java @@ -40,6 +40,7 @@ import com.tencent.bk.job.execute.common.context.JobExecuteContextThreadLocalRepo; import com.tencent.bk.job.execute.common.context.JobInstanceContext; import com.tencent.bk.job.execute.dao.TaskInstanceDAO; +import com.tencent.bk.job.execute.dao.common.IdGen; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.service.ApplicationService; import com.tencent.bk.job.execute.service.StepInstanceService; @@ -62,21 +63,26 @@ public class TaskInstanceServiceImpl implements TaskInstanceService { private final ExecuteAuthService executeAuthService; private final StepInstanceService stepInstanceService; + private final IdGen idGen; + @Autowired public TaskInstanceServiceImpl(ApplicationService applicationService, TaskInstanceDAO taskInstanceDAO, TaskInstanceVariableService taskInstanceVariableService, ExecuteAuthService executeAuthService, - StepInstanceService stepInstanceService) { + StepInstanceService stepInstanceService, + IdGen idGen) { this.applicationService = applicationService; this.stepInstanceService = stepInstanceService; this.taskInstanceDAO = taskInstanceDAO; this.taskInstanceVariableService = taskInstanceVariableService; this.executeAuthService = executeAuthService; + this.idGen = idGen; } @Override public long addTaskInstance(TaskInstanceDTO taskInstance) { + taskInstance.setId(idGen.genTaskInstanceId()); return taskInstanceDAO.addTaskInstance(taskInstance); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceVariableServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceVariableServiceImpl.java index 471a8014ef..138e6db955 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceVariableServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceVariableServiceImpl.java @@ -27,6 +27,7 @@ import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.execute.dao.TaskInstanceVariableDAO; +import com.tencent.bk.job.execute.dao.common.IdGen; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; import com.tencent.bk.job.execute.model.ExecuteTargetDTO; import lombok.extern.slf4j.Slf4j; @@ -39,10 +40,12 @@ @Slf4j public class TaskInstanceVariableServiceImpl implements com.tencent.bk.job.execute.service.TaskInstanceVariableService { private final TaskInstanceVariableDAO taskInstanceVariableDAO; + private final IdGen idGen; @Autowired - public TaskInstanceVariableServiceImpl(TaskInstanceVariableDAO taskInstanceVariableDAO) { + public TaskInstanceVariableServiceImpl(TaskInstanceVariableDAO taskInstanceVariableDAO, IdGen idGen) { this.taskInstanceVariableDAO = taskInstanceVariableDAO; + this.idGen = idGen; } @Override @@ -80,6 +83,7 @@ public void saveTaskInstanceVariables(List taskVarList) { && taskVariable.getExecuteTarget() != null) { taskVariable.setValue(JsonUtils.toJson(taskVariable.getExecuteTarget())); } + taskVariable.setId(idGen.genTaskInstanceVariableId()); } taskInstanceVariableDAO.saveTaskInstanceVariables(taskVarList); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskOperationLogServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskOperationLogServiceImpl.java index cf5600410d..b167b20905 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskOperationLogServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskOperationLogServiceImpl.java @@ -25,6 +25,7 @@ package com.tencent.bk.job.execute.service.impl; import com.tencent.bk.job.execute.dao.OperationLogDAO; +import com.tencent.bk.job.execute.dao.common.IdGen; import com.tencent.bk.job.execute.model.OperationLogDTO; import com.tencent.bk.job.execute.service.TaskOperationLogService; import org.springframework.beans.factory.annotation.Autowired; @@ -35,14 +36,17 @@ @Service public class TaskOperationLogServiceImpl implements TaskOperationLogService { private final OperationLogDAO operationLogDAO; + private final IdGen idGen; @Autowired - public TaskOperationLogServiceImpl(OperationLogDAO operationLogDAO) { + public TaskOperationLogServiceImpl(OperationLogDAO operationLogDAO, IdGen idGen) { this.operationLogDAO = operationLogDAO; + this.idGen = idGen; } @Override public long saveOperationLog(OperationLogDTO operationLog) { + operationLog.setId(idGen.genOperationLogId()); return operationLogDAO.saveOperationLog(operationLog); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java index 67123192a5..09ae65f21f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java @@ -38,7 +38,6 @@ import com.tencent.bk.job.execute.common.converter.StepTypeExecuteTypeConverter; import com.tencent.bk.job.execute.common.util.TaskCostCalculator; import com.tencent.bk.job.execute.constants.UserOperationEnum; -import com.tencent.bk.job.execute.dao.FileSourceTaskLogDAO; import com.tencent.bk.job.execute.dao.StepInstanceDAO; import com.tencent.bk.job.execute.dao.TaskInstanceDAO; import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; @@ -64,6 +63,7 @@ import com.tencent.bk.job.execute.model.inner.CronTaskExecuteResult; import com.tencent.bk.job.execute.model.inner.ServiceCronTaskExecuteResultStatistics; import com.tencent.bk.job.execute.service.FileExecuteObjectTaskService; +import com.tencent.bk.job.execute.service.FileSourceTaskLogService; import com.tencent.bk.job.execute.service.LogService; import com.tencent.bk.job.execute.service.RollingConfigService; import com.tencent.bk.job.execute.service.ScriptExecuteObjectTaskService; @@ -102,7 +102,7 @@ public class TaskResultServiceImpl implements TaskResultService { private final TaskInstanceDAO taskInstanceDAO; private final StepInstanceDAO stepInstanceDAO; private final TaskInstanceService taskInstanceService; - private final FileSourceTaskLogDAO fileSourceTaskLogDAO; + private final FileSourceTaskLogService fileSourceTaskLogService; private final ScriptExecuteObjectTaskService scriptExecuteObjectTaskService; private final FileExecuteObjectTaskService fileExecuteObjectTaskService; private final LogService logService; @@ -116,7 +116,7 @@ public class TaskResultServiceImpl implements TaskResultService { public TaskResultServiceImpl(TaskInstanceDAO taskInstanceDAO, StepInstanceDAO stepInstanceDAO, TaskInstanceService taskInstanceService, - FileSourceTaskLogDAO fileSourceTaskLogDAO, + FileSourceTaskLogService fileSourceTaskLogService, ScriptExecuteObjectTaskService scriptExecuteObjectTaskService, FileExecuteObjectTaskService fileExecuteObjectTaskService, LogService logService, @@ -128,7 +128,7 @@ public TaskResultServiceImpl(TaskInstanceDAO taskInstanceDAO, this.taskInstanceDAO = taskInstanceDAO; this.stepInstanceDAO = stepInstanceDAO; this.taskInstanceService = taskInstanceService; - this.fileSourceTaskLogDAO = fileSourceTaskLogDAO; + this.fileSourceTaskLogService = fileSourceTaskLogService; this.scriptExecuteObjectTaskService = scriptExecuteObjectTaskService; this.fileExecuteObjectTaskService = fileExecuteObjectTaskService; this.logService = logService; @@ -460,7 +460,7 @@ public StepExecutionDetailDTO getStepExecutionResult(String username, if (stepInstance.isFileStep()) { watch.start("involveFileSourceTaskLog"); FileSourceTaskLogDTO fileSourceTaskLog = - fileSourceTaskLogDAO.getFileSourceTaskLog( + fileSourceTaskLogService.getFileSourceTaskLog( stepInstance.getTaskInstanceId(), stepInstance.getId(), queryExecuteCount diff --git a/support-files/kubernetes/charts/bk-job/templates/job-execute/configmap.yaml b/support-files/kubernetes/charts/bk-job/templates/job-execute/configmap.yaml index 2689ad88c2..ca81f9f521 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-execute/configmap.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-execute/configmap.yaml @@ -229,4 +229,25 @@ data: enabled: {{ .Values.executeConfig.mysql.verticalSharding.enabled }} migration: enabled: {{ .Values.executeConfig.mysql.migration.enabled }} + idGen: + type: {{ .Values.executeConfig.idGen.type }} + migration: + enabled: {{ .Values.executeConfig.idGen.migration.enabled }} + {{ if .Values.executeConfig.leaf.enabled }} + # 分布式 ID （leaf 组件配置) + leaf: + segment: + allocStrategyDaoBeanName: jooqLeafIdAllocator + enable: true + url: {{ include "job.jdbcMysqlScheme" . }}://{{- include "job.mariadb.host" . }}:{{- include "job.mariadb.port" . }}/job_leaf{{ include "job.mariadb.connection.properties" . }} + username: {{ include "job.mariadb.username" . }} + {{ if .Values.externalMariaDB.existingPasswordSecret }} + password: {{ .Values.externalMariaDB.existingPasswordKey | default "mariadb-password" | printf "${%s}" }} + {{- else -}} + password: ${mariadb-password} + {{- end }} + snowflake: + enable: false + name: job-leaf + {{- end }} {{- end }} diff --git a/support-files/kubernetes/charts/bk-job/values.yaml b/support-files/kubernetes/charts/bk-job/values.yaml index f8b2a029a0..b27febd89a 100644 --- a/support-files/kubernetes/charts/bk-job/values.yaml +++ b/support-files/kubernetes/charts/bk-job/values.yaml @@ -1023,6 +1023,15 @@ executeConfig: # jdbcUrl: jdbc:otel:mysql://db_host_c:3306/job_execute # username: "job" # password: "job" + # 分布式ID组件 leaf 配置 + leaf: + enabled: false + idGen: + migration: + enabled: false + # id生成器配置。auto_increment: 自增长，segment: leaf segment id + type: auto_increment + ## job-crontab定时任务配置 crontabConfig: # 是否部署定时任务微服务，默认部署 diff --git a/support-files/sql/job-execute/0026_job_execute_20240520-1000_V3.10.0_mysql.sql b/support-files/sql/job-execute/0026_job_execute_20241115-1000_V3.11.2_mysql.sql similarity index 71% rename from support-files/sql/job-execute/0026_job_execute_20240520-1000_V3.10.0_mysql.sql rename to support-files/sql/job-execute/0026_job_execute_20241115-1000_V3.11.2_mysql.sql index 539c972eb3..6f62d29889 100644 --- a/support-files/sql/job-execute/0026_job_execute_20240520-1000_V3.10.0_mysql.sql +++ b/support-files/sql/job-execute/0026_job_execute_20241115-1000_V3.11.2_mysql.sql @@ -2,6 +2,7 @@ USE job_execute; SET NAMES utf8mb4; +-- 更新 schema DROP PROCEDURE IF EXISTS job_schema_update; DELIMITER @@ -29,6 +30,15 @@ BEGIN ALTER TABLE task_instance DROP INDEX `idx_create_time`; END IF; + -- task_instance_host + IF NOT EXISTS(SELECT 1 + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'task_instance_host' + AND COLUMN_NAME = 'app_id') THEN + ALTER TABLE task_instance_host ADD COLUMN `app_id` bigint(20) NOT NULL DEFAULT '0'; + END IF; + -- step_instance IF NOT EXISTS(SELECT 1 FROM information_schema.statistics @@ -140,7 +150,6 @@ BEGIN AND INDEX_NAME = 'idx_task_instance_id_id') THEN ALTER TABLE gse_file_agent_task ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); END IF; - -- gse_script_execute_obj_task IF NOT EXISTS(SELECT 1 FROM information_schema.statistics @@ -269,4 +278,165 @@ COMMIT; CALL job_schema_update(); -DROP PROCEDURE IF EXISTS job_schema_update; \ No newline at end of file +DROP PROCEDURE IF EXISTS job_schema_update; + + +-- 更新 task_instance_id 列 +DROP PROCEDURE IF EXISTS job_add_task_instance_id; + +DELIMITER + +CREATE PROCEDURE job_add_task_instance_id() +BEGIN + + DECLARE minId BIGINT; + DECLARE maxId BIGINT; + DECLARE fromId BIGINT; + DECLARE endId BIGINT; + + SET AUTOCOMMIT = 0; + + SELECT MIN(id), MAX(id) INTO minId, maxId FROM step_instance; + + SET fromId = minId; + + WHILE fromId <= maxId DO + SET endId = fromId + 999; + + UPDATE file_source_task_log t1 + INNER JOIN ( + SELECT id, task_instance_id + FROM step_instance + WHERE id BETWEEN fromId AND endId) + AS tmp ON t1.step_instance_id = tmp.id + SET t1.task_instance_id = tmp.task_instance_id + WHERE t1.task_instance_id = 0; + + UPDATE gse_file_agent_task t1 + INNER JOIN ( + SELECT id, task_instance_id + FROM step_instance + WHERE id BETWEEN fromId AND endId) + AS tmp ON t1.step_instance_id = tmp.id + SET t1.task_instance_id = tmp.task_instance_id + WHERE t1.task_instance_id = 0; + + UPDATE gse_script_agent_task t1 + INNER JOIN ( + SELECT id, task_instance_id + FROM step_instance + WHERE id BETWEEN fromId AND endId) + AS tmp ON t1.step_instance_id = tmp.id + SET t1.task_instance_id = tmp.task_instance_id + WHERE t1.task_instance_id = 0; + + UPDATE gse_task t1 + INNER JOIN ( + SELECT id, task_instance_id + FROM step_instance + WHERE id BETWEEN fromId AND endId) + AS tmp ON t1.step_instance_id = tmp.id + SET t1.task_instance_id = tmp.task_instance_id + WHERE t1.task_instance_id = 0; + + UPDATE step_instance_confirm t1 + INNER JOIN ( + SELECT id, task_instance_id + FROM step_instance + WHERE id BETWEEN fromId AND endId) + AS tmp ON t1.step_instance_id = tmp.id + SET t1.task_instance_id = tmp.task_instance_id + WHERE t1.task_instance_id = 0; + + UPDATE step_instance_script t1 + INNER JOIN ( + SELECT id, task_instance_id + FROM step_instance + WHERE id BETWEEN fromId AND endId) + AS tmp ON t1.step_instance_id = tmp.id + SET t1.task_instance_id = tmp.task_instance_id + WHERE t1.task_instance_id = 0; + + UPDATE step_instance_file t1 + INNER JOIN ( + SELECT id, task_instance_id + FROM step_instance + WHERE id BETWEEN fromId AND endId) + AS tmp ON t1.step_instance_id = tmp.id + SET t1.task_instance_id = tmp.task_instance_id + WHERE t1.task_instance_id = 0; + + UPDATE step_instance_rolling_task t1 + INNER JOIN ( + SELECT id, task_instance_id + FROM step_instance + WHERE id BETWEEN fromId AND endId) + AS tmp ON t1.step_instance_id = tmp.id + SET t1.task_instance_id = tmp.task_instance_id + WHERE t1.task_instance_id = 0; + + UPDATE step_instance_variable t1 + INNER JOIN ( + SELECT id, task_instance_id + FROM step_instance + WHERE id BETWEEN fromId AND endId) + AS tmp ON t1.step_instance_id = tmp.id + SET t1.task_instance_id = tmp.task_instance_id + WHERE t1.task_instance_id = 0; + + COMMIT; + + SET fromId = endId + 1; + END WHILE; + +END +DELIMITER ; +COMMIT; + +CALL job_add_task_instance_id(); +DROP PROCEDURE IF EXISTS job_add_task_instance_id; + + +-- 更新 task_instance_host 表字段数据 +DROP PROCEDURE IF EXISTS job_update_task_instance_host_data; + +DELIMITER + +CREATE PROCEDURE job_update_task_instance_host_data() +BEGIN + + DECLARE minId BIGINT; + DECLARE maxId BIGINT; + DECLARE fromId BIGINT; + DECLARE endId BIGINT; + + SET AUTOCOMMIT = 0; + + SELECT MIN(id), MAX(id) INTO minId, maxId FROM task_instance; + + SET fromId = minId; + + WHILE fromId <= maxId DO + SET endId = fromId + 999; + + UPDATE task_instance_host t1 + INNER JOIN ( + SELECT id,app_id + FROM task_instance + WHERE id BETWEEN fromId AND endId) + AS tmp ON t1.task_instance_id = tmp.id + SET t1.app_id = tmp.app_id + WHERE t1.app_id = 0; + + COMMIT; + + SET fromId = endId + 1; + END WHILE; + +END +DELIMITER ; +COMMIT; + +CALL job_update_task_instance_host_data(); +DROP PROCEDURE IF EXISTS job_update_task_instance_host_data; + diff --git a/support-files/sql/job-leaf/0001_job_leaf_20241115-1000_V3.11.2.sql b/support-files/sql/job-leaf/0001_job_leaf_20241115-1000_V3.11.2.sql new file mode 100644 index 0000000000..b37471b57b --- /dev/null +++ b/support-files/sql/job-leaf/0001_job_leaf_20241115-1000_V3.11.2.sql @@ -0,0 +1,26 @@ +SET NAMES utf8mb4; +CREATE DATABASE IF NOT EXISTS job_leaf DEFAULT CHARACTER SET utf8mb4; +USE job_leaf; + + +CREATE TABLE IF NOT EXISTS `t_leaf_alloc` ( + `biz_tag` varchar(128) NOT NULL DEFAULT '', + `max_id` bigint(20) NOT NULL DEFAULT '1', + `step` int(11) NOT NULL, + `description` varchar(256) DEFAULT NULL, + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`biz_tag`) +) ENGINE=InnoDB; + + +insert ignore into t_leaf_alloc(biz_tag, max_id, step, description) values('job_execute.task_instance', 1, 2000, 'job_execute.task_instance'); +insert ignore into t_leaf_alloc(biz_tag, max_id, step, description) values('job_execute.step_instance', 1, 2000, 'job_execute.step_instance'); +insert ignore into t_leaf_alloc(biz_tag, max_id, step, description) values('job_execute.gse_task', 1, 2000, 'job_execute.gse_task'); +insert ignore into t_leaf_alloc(biz_tag, max_id, step, description) values('job_execute.operation_log', 1, 2000, 'job_execute.operation_log'); +insert ignore into t_leaf_alloc(biz_tag, max_id, step, description) values('job_execute.file_source_task_log', 1, 2000, 'job_execute.file_source_task_log'); +insert ignore into t_leaf_alloc(biz_tag, max_id, step, description) values('job_execute.gse_file_execute_obj_task', 1, 2000, 'job_execute.gse_file_execute_obj_task'); +insert ignore into t_leaf_alloc(biz_tag, max_id, step, description) values('job_execute.gse_script_execute_obj_task', 1, 2000, 'job_execute.gse_script_execute_obj_task'); +insert ignore into t_leaf_alloc(biz_tag, max_id, step, description) values('job_execute.rolling_config', 1, 2000, 'job_execute.rolling_config'); +insert ignore into t_leaf_alloc(biz_tag, max_id, step, description) values('job_execute.step_instance_rolling_task', 1, 2000, 'job_execute.step_instance_rolling_task'); +insert ignore into t_leaf_alloc(biz_tag, max_id, step, description) values('job_execute.step_instance_variable', 1, 2000, 'job_execute.step_instance_variable'); +insert ignore into t_leaf_alloc(biz_tag, max_id, step, description) values('job_execute.task_instance_variable', 1, 2000, 'job_execute.task_instance_variable'); \ No newline at end of file From 74c83322c25f7f0ae7de4bd53e6f707a727551c6 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Sun, 17 Nov 2024 18:07:29 +0800 Subject: [PATCH 23/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mysql/ShardingConfigParseException.java | 37 + .../ShardingDatasourceAutoConfiguration.java | 307 +++++ .../mysql/config/ShardingProperties.java | 244 ++++ .../main/resources/META-INF/spring.factories | 3 +- .../impl/ServiceArchiveResourceImpl.java | 30 +- .../job/backup/archive/AbstractArchivist.java | 518 --------- .../AbstractJobInstanceArchiveTask.java | 268 +++++ .../archive/AbstractTableArchivist.java | 93 ++ .../backup/archive/ArchiveSummaryHolder.java | 8 +- .../archive/ArchiveTablePropsStorage.java | 93 ++ .../job/backup/archive/ArchiveTaskLock.java | 53 +- .../archive/ArchiveTaskPrioritySorter.java | 125 ++ .../backup/archive/ArchiveTaskService.java | 67 ++ .../job/backup/archive/ArchiveTaskWorker.java | 50 + .../archive/JobExecuteArchiveManage.java | 1024 ++++++++--------- .../JobInstanceAppDataArchiveTask.java | 84 ++ .../archive/JobInstanceArchiveCronJobs.java | 69 ++ .../archive/JobInstanceArchiveTask.java | 43 + .../JobInstanceArchiveTaskGenerator.java | 187 +++ .../JobInstanceArchiveTaskScheduleLock.java | 78 ++ .../JobInstanceArchiveTaskScheduler.java | 219 ++++ .../archive/JobInstanceHotArchivist.java | 96 ++ .../JobInstanceMainDataArchiveTask.java | 123 ++ .../archive/JobInstanceSubTableArchiver.java | 47 + .../{ => archive}/dao/ArchiveProgressDAO.java | 2 +- .../backup/archive/dao/ArchiveTaskDAO.java | 56 + .../{ => archive}/dao/ExecuteArchiveDAO.java | 2 +- .../backup/archive/dao/ExecuteRecordDAO.java | 81 ++ .../archive/dao/JobInstanceColdDAO.java | 44 + .../archive/dao/JobInstanceHotRecordDAO.java | 82 ++ .../bk/job/backup/archive/dao/RecordDAO.java | 66 ++ .../impl/AbstractJobInstanceHotRecordDAO.java | 114 ++ .../AbstractJobInstanceRecordResultSet.java | 108 ++ .../dao/impl/ArchiveProgressDAOImpl.java | 4 +- .../archive/dao/impl/ArchiveTaskDAOImpl.java | 188 +++ .../dao/impl/ExecuteArchiveDAOImpl.java | 4 +- .../dao/impl/FileSourceTaskLogRecordDAO.java | 88 ++ .../dao/impl/GseFileAgentTaskRecordDAO.java | 86 ++ .../impl/GseFileExecuteObjTaskRecordDAO.java | 88 ++ .../dao/impl/GseScriptAgentTaskRecordDAO.java | 91 ++ .../GseScriptExecuteObjTaskRecordDAO.java | 86 ++ .../archive/dao/impl/GseTaskRecordDAO.java | 83 ++ .../dao/impl/JobInstanceColdDAOImpl.java | 149 +++ .../JobInstanceRecordResultSetFactory.java | 101 ++ .../dao/impl/OperationLogRecordDAO.java | 87 ++ .../archive/dao/impl/RecordResultSet.java | 46 + .../impl/RollingConfigRecordDAO.java} | 77 +- .../impl/StepInstanceConfirmRecordDAO.java | 88 ++ .../dao/impl/StepInstanceFileRecordDAO.java | 87 ++ .../dao/impl/StepInstanceRecordDAO.java | 87 ++ .../StepInstanceRollingTaskRecordDAO.java | 86 ++ .../dao/impl/StepInstanceScriptRecordDAO.java | 87 ++ .../impl/StepInstanceVariableRecordDAO.java | 85 ++ .../dao/impl/TaskInstanceHostRecordDAO.java | 83 ++ .../dao/impl/TaskInstanceRecordDAO.java | 119 ++ .../impl/TaskInstanceVariableRecordDAO.java | 86 ++ .../AbstractJobInstanceSubTableArchiver.java | 77 ++ .../impl/FileSourceTaskLogArchiver.java | 44 + .../impl/FileSourceTaskLogArchivist.java | 63 - .../impl/GseFileAgentTaskArchiver.java | 44 + .../impl/GseFileAgentTaskArchivist.java | 61 - .../impl/GseFileExecuteObjTaskArchiver.java | 44 + .../impl/GseFileExecuteObjTaskArchivist.java | 61 - .../impl/GseScriptAgentTaskArchiver.java | 44 + .../impl/GseScriptAgentTaskArchivist.java | 61 - .../impl/GseScriptExecuteObjTaskArchiver.java | 44 + .../GseScriptExecuteObjTaskArchivist.java | 61 - .../backup/archive/impl/GseTaskArchiver.java | 44 + .../archive/impl/OperationLogArchiver.java | 44 + .../archive/impl/OperationLogArchivist.java | 61 - .../archive/impl/RollingConfigArchiver.java | 44 + .../archive/impl/StepInstanceArchiver.java | 44 + .../archive/impl/StepInstanceArchivist.java | 61 - .../impl/StepInstanceConfirmArchiver.java | 44 + .../impl/StepInstanceConfirmArchivist.java | 61 - ...ist.java => StepInstanceFileArchiver.java} | 44 +- .../impl/StepInstanceFileArchivist.java | 61 - .../impl/StepInstanceRollingTaskArchiver.java | 44 + .../StepInstanceRollingTaskArchivist.java | 61 - .../impl/StepInstanceScriptArchiver.java | 44 + .../impl/StepInstanceScriptArchivist.java | 61 - .../impl/StepInstanceVariableArchiver.java | 44 + .../impl/StepInstanceVariableArchivist.java | 61 - .../archive/impl/TaskInstanceArchiver.java | 44 + .../archive/impl/TaskInstanceArchivist.java | 61 - .../impl/TaskInstanceHostArchiver.java | 44 + .../impl/TaskInstanceHostArchivist.java | 66 -- .../impl/TaskInstanceVariableArchiver.java | 44 + .../impl/TaskInstanceVariableArchivist.java | 61 - .../archive/model/ArchiveProgressDTO.java | 45 + .../archive/model/ArchiveTableProps.java | 49 + .../archive/model/ArchiveTaskSummary.java | 69 ++ .../backup/archive/model/BackupResult.java | 54 + .../job/backup/archive/model/DbDataNode.java | 95 ++ .../model/JobInstanceArchiveTaskInfo.java | 94 ++ .../model/TimeAndIdBasedArchiveProcess.java | 64 ++ .../backup/config/ArchiveConfiguration.java | 304 ++++- ...Properties.java => ArchiveProperties.java} | 54 +- .../config/ExecuteBackupDbConfiguration.java | 6 +- .../constant/ArchiveTaskStatusEnum.java | 69 ++ .../backup/constant/ArchiveTaskTypeEnum.java | 54 + .../backup/constant/DbDataNodeTypeEnum.java | 57 + .../bk/job/backup/dao/ExecuteRecordDAO.java | 57 - .../dao/impl/AbstractExecuteRecordDAO.java | 95 -- .../dao/impl/FileSourceTaskLogRecordDAO.java | 46 - .../dao/impl/GseFileAgentTaskRecordDAO.java | 48 - .../impl/GseFileExecuteObjTaskRecordDAO.java | 48 - .../dao/impl/GseScriptAgentTaskRecordDAO.java | 47 - .../GseScriptExecuteObjTaskRecordDAO.java | 47 - .../job/backup/dao/impl/GseTaskRecordDAO.java | 43 - .../dao/impl/OperationLogRecordDAO.java | 29 - .../dao/impl/RollingConfigRecordDAO.java | 29 - .../impl/StepInstanceConfirmRecordDAO.java | 30 - .../dao/impl/StepInstanceFileRecordDAO.java | 29 - .../dao/impl/StepInstanceRecordDAO.java | 53 - .../StepInstanceRollingTaskRecordDAO.java | 46 - .../dao/impl/StepInstanceScriptRecordDAO.java | 29 - .../impl/StepInstanceVariableRecordDAO.java | 45 - .../dao/impl/TaskInstanceHostRecordDAO.java | 26 - .../dao/impl/TaskInstanceRecordDAO.java | 49 - .../impl/TaskInstanceVariableRecordDAO.java | 29 - .../impl/ArchiveProgressServiceImpl.java | 2 +- .../templates/job-execute/configmap.yaml | 17 + .../kubernetes/charts/bk-job/values.yaml | 3 + 124 files changed, 7081 insertions(+), 2934 deletions(-) create mode 100644 src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/ShardingConfigParseException.java create mode 100644 src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/ShardingDatasourceAutoConfiguration.java create mode 100644 src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/ShardingProperties.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractArchivist.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractTableArchivist.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTablePropsStorage.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskPrioritySorter.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskService.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskWorker.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceAppDataArchiveTask.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveCronJobs.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTask.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduleLock.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceHotArchivist.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceSubTableArchiver.java rename src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/{ => archive}/dao/ArchiveProgressDAO.java (97%) create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ArchiveTaskDAO.java rename src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/{ => archive}/dao/ExecuteArchiveDAO.java (97%) create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ExecuteRecordDAO.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/JobInstanceColdDAO.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/JobInstanceHotRecordDAO.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/RecordDAO.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceHotRecordDAO.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceRecordResultSet.java rename src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/{ => archive}/dao/impl/ArchiveProgressDAOImpl.java (97%) create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java rename src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/{ => archive}/dao/impl/ExecuteArchiveDAOImpl.java (98%) create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/FileSourceTaskLogRecordDAO.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileAgentTaskRecordDAO.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileExecuteObjTaskRecordDAO.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptAgentTaskRecordDAO.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptExecuteObjTaskRecordDAO.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseTaskRecordDAO.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/JobInstanceColdDAOImpl.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/JobInstanceRecordResultSetFactory.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/OperationLogRecordDAO.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/RecordResultSet.java rename src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/{impl/RollingConfigArchivist.java => dao/impl/RollingConfigRecordDAO.java} (50%) create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceConfirmRecordDAO.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceFileRecordDAO.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRecordDAO.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRollingTaskRecordDAO.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceScriptRecordDAO.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceVariableRecordDAO.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceHostRecordDAO.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceRecordDAO.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceVariableRecordDAO.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/AbstractJobInstanceSubTableArchiver.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/FileSourceTaskLogArchiver.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/FileSourceTaskLogArchivist.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileAgentTaskArchiver.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileAgentTaskArchivist.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileExecuteObjTaskArchiver.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileExecuteObjTaskArchivist.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptAgentTaskArchiver.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptAgentTaskArchivist.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptExecuteObjTaskArchiver.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptExecuteObjTaskArchivist.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskArchiver.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/OperationLogArchiver.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/OperationLogArchivist.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/RollingConfigArchiver.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceArchiver.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceArchivist.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceConfirmArchiver.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceConfirmArchivist.java rename src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/{GseTaskArchivist.java => StepInstanceFileArchiver.java} (50%) delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceFileArchivist.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceRollingTaskArchiver.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceRollingTaskArchivist.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceScriptArchiver.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceScriptArchivist.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceVariableArchiver.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceVariableArchivist.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceArchiver.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceArchivist.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceHostArchiver.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceHostArchivist.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceVariableArchiver.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceVariableArchivist.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveProgressDTO.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTableProps.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTaskSummary.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/BackupResult.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DbDataNode.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/JobInstanceArchiveTaskInfo.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/TimeAndIdBasedArchiveProcess.java rename src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/{ArchiveDBProperties.java => ArchiveProperties.java} (75%) create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/ArchiveTaskStatusEnum.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/ArchiveTaskTypeEnum.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/DbDataNodeTypeEnum.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ExecuteRecordDAO.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/AbstractExecuteRecordDAO.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/FileSourceTaskLogRecordDAO.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseFileAgentTaskRecordDAO.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseFileExecuteObjTaskRecordDAO.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseScriptAgentTaskRecordDAO.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseScriptExecuteObjTaskRecordDAO.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseTaskRecordDAO.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/OperationLogRecordDAO.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/RollingConfigRecordDAO.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceConfirmRecordDAO.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceFileRecordDAO.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceRecordDAO.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceRollingTaskRecordDAO.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceScriptRecordDAO.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceVariableRecordDAO.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceHostRecordDAO.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceRecordDAO.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceVariableRecordDAO.java diff --git a/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/ShardingConfigParseException.java b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/ShardingConfigParseException.java new file mode 100644 index 0000000000..d36650b6b7 --- /dev/null +++ b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/ShardingConfigParseException.java @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.sharding.mysql; + +import com.tencent.bk.job.common.exception.InternalException; + +/** + * 分片配置解析异常 + */ +public class ShardingConfigParseException extends InternalException { + + public ShardingConfigParseException(String message) { + super(message); + } +} diff --git a/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/ShardingDatasourceAutoConfiguration.java b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/ShardingDatasourceAutoConfiguration.java new file mode 100644 index 0000000000..87f12a4335 --- /dev/null +++ b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/ShardingDatasourceAutoConfiguration.java @@ -0,0 +1,307 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.sharding.mysql.config; + +import com.tencent.bk.job.common.sharding.mysql.ShardingConfigParseException; +import com.zaxxer.hikari.HikariDataSource; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; +import org.apache.shardingsphere.driver.api.ShardingSphereDataSourceFactory; +import org.apache.shardingsphere.infra.config.algorithm.AlgorithmConfiguration; +import org.apache.shardingsphere.infra.config.mode.ModeConfiguration; +import org.apache.shardingsphere.infra.config.rule.RuleConfiguration; +import org.apache.shardingsphere.mode.repository.standalone.StandalonePersistRepositoryConfiguration; +import org.apache.shardingsphere.sharding.api.config.ShardingRuleConfiguration; +import org.apache.shardingsphere.sharding.api.config.rule.ShardingTableReferenceRuleConfiguration; +import org.apache.shardingsphere.sharding.api.config.rule.ShardingTableRuleConfiguration; +import org.apache.shardingsphere.sharding.api.config.strategy.keygen.KeyGenerateStrategyConfiguration; +import org.apache.shardingsphere.sharding.api.config.strategy.sharding.HintShardingStrategyConfiguration; +import org.apache.shardingsphere.sharding.api.config.strategy.sharding.ShardingStrategyConfiguration; +import org.apache.shardingsphere.sharding.api.config.strategy.sharding.StandardShardingStrategyConfiguration; +import org.jooq.ConnectionProvider; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.jooq.impl.DataSourceConnectionProvider; +import org.jooq.impl.DefaultConfiguration; +import org.jooq.impl.DefaultDSLContext; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.DependsOn; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.jdbc.datasource.TransactionAwareDataSourceProxy; + +import javax.sql.DataSource; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; + +@Configuration(proxyBeanMethods = false) +@EnableConfigurationProperties({ShardingProperties.class}) +@ConditionalOnProperty(value = "sharding.enabled", havingValue = "true") +@Slf4j +public class ShardingDatasourceAutoConfiguration { + + @Bean("jobShardingDataSource") + public DataSource shardingDataSource(ShardingProperties shardingProperties) throws SQLException { + log.info("Init sharding datasource start ..."); + ShardingProperties.ShardingsphereProperties shardingsphereProperties = shardingProperties.getShardingsphere(); + // 指定逻辑 Database 名称 + String databaseName = shardingsphereProperties.getDatabaseName(); + // 构建运行模式 + ModeConfiguration modeConfig = createModeConfiguration(); + // 构建真实数据源 + Map dataSourceMap = createDataSources(shardingsphereProperties.getDataSources()); + // 构建具体规则 + List ruleConfigs = new ArrayList<>(); + ruleConfigs.add(createShardingRuleConfiguration(shardingsphereProperties.getShardingRule())); + // 构建系统级属性配置 + Properties globalProps = createShardingGlobalProps(shardingsphereProperties.getProps()); + DataSource dataSource = ShardingSphereDataSourceFactory.createDataSource(databaseName, modeConfig, + dataSourceMap, ruleConfigs, globalProps); + log.info("Init sharding datasource successfully"); + return dataSource; + } + + private ModeConfiguration createModeConfiguration() { + log.info("Load sharding mode, type: Standalone"); + return new ModeConfiguration("Standalone", + new StandalonePersistRepositoryConfiguration("JDBC", new Properties())); + } + + private Map createDataSources( + Map dataSourcePropMap) { + log.info("Init datasourceMap start ..."); + + Map dataSourceMap = new HashMap<>(); + + dataSourcePropMap.forEach((name, dataSourceProp) -> { + log.info("Create sharding datasource, name : {}, dataSourceProp: {}", name, dataSourceProp); + HikariDataSource dataSource = new HikariDataSource(); + dataSource.setDriverClassName(dataSourceProp.getDriverClassName()); + dataSource.setJdbcUrl(dataSourceProp.getJdbcUrl()); + dataSource.setUsername(dataSourceProp.getUsername()); + dataSource.setPassword(dataSourceProp.getPassword()); + dataSourceMap.put(name, dataSource); + }); + log.info("Init datasourceMap successfully"); + return dataSourceMap; + } + + private ShardingRuleConfiguration createShardingRuleConfiguration( + ShardingProperties.ShardingsphereProperties.ShardingRule shardingRuleProps) { + log.info("Init sharding rule configuration start ..."); + + ShardingRuleConfiguration shardingRuleConfiguration = new ShardingRuleConfiguration(); + + // 分片表规则列表 + shardingRuleConfiguration.getTables() + .addAll(createShardingTableRuleConfigurations(shardingRuleProps.getTables())); + + // 绑定表规则列表 + if (CollectionUtils.isNotEmpty(shardingRuleProps.getBindingTables())) { + shardingRuleProps.getBindingTables().forEach(bindingTables -> { + String ruleName = "binding_rule_" + + Arrays.stream(bindingTables.split(",")).map(String::trim).collect(Collectors.joining("_")); + log.info("Create binding table group, ruleName: {}, bindingTables: {}", ruleName, bindingTables); + shardingRuleConfiguration.getBindingTableGroups().add( + new ShardingTableReferenceRuleConfiguration(ruleName, bindingTables)); + }); + } + + // 默认分库策略 + if (shardingRuleProps.getDefaultDatabaseStrategy() != null) { + log.info("Create default database sharding strategy, strategy: {}", + shardingRuleProps.getDefaultDatabaseStrategy()); + + shardingRuleConfiguration.setDefaultDatabaseShardingStrategy( + createShardingStrategyConfiguration(shardingRuleProps.getDefaultDatabaseStrategy())); + } + + // 默认分表策略 + if (shardingRuleProps.getDefaultTableStrategy() != null) { + log.info("Create default table sharding strategy, strategy: {}", + shardingRuleProps.getDefaultTableStrategy()); + shardingRuleConfiguration.setDefaultTableShardingStrategy( + createShardingStrategyConfiguration(shardingRuleProps.getDefaultTableStrategy())); + } + + // 分片算法配置 + if (MapUtils.isNotEmpty(shardingRuleProps.getShardingAlgorithms())) { + shardingRuleProps.getShardingAlgorithms().forEach((name, algorithm) -> { + log.info("Create sharding algorithms, name: {}, type: {}, props: {}", + name, algorithm.getType(), algorithm.getProps()); + Properties props = toProperties(algorithm.getProps()); + shardingRuleConfiguration.getShardingAlgorithms() + .put(name, new AlgorithmConfiguration(algorithm.getType(), props)); + }); + } + + // 自增列生成算法配置 + if (MapUtils.isNotEmpty(shardingRuleProps.getKeyGenerators())) { + shardingRuleProps.getKeyGenerators().forEach((name, generator) -> { + log.info("Create key generator, name: {}, type: {}", name, generator.getType()); + Properties props = toProperties(generator.getProps()); + shardingRuleConfiguration.getKeyGenerators() + .put(name, new AlgorithmConfiguration(generator.getType(), props)); + }); + } + + log.info("Init sharding rule configuration successfully"); + + return shardingRuleConfiguration; + } + + private Properties toProperties(Map propsMap) { + Properties props = new Properties(); + props.putAll(propsMap); + return props; + } + + + private List createShardingTableRuleConfigurations( + Map tables) { + + List configurations = new ArrayList<>(tables.size()); + + tables.forEach((tableName, tableShardingRule) -> + configurations.add(createShardingTableRuleConfiguration(tableName, tableShardingRule))); + + return configurations; + } + + private ShardingTableRuleConfiguration createShardingTableRuleConfiguration( + String tableName, + ShardingProperties.ShardingsphereProperties.TableShardingRule tableShardingRule) { + log.info("[{}] Create sharding table rule configuration, tableShardingRule: {}", + tableName, tableShardingRule); + + ShardingTableRuleConfiguration configuration = new ShardingTableRuleConfiguration( + tableName, tableShardingRule.getActualDataNodes()); + if (tableShardingRule.getKeyGenerateStrategy() != null) { + log.info("[{}] Create key generate strategy configuration, column: {}, keyGeneratorName: {}", + tableName, + tableShardingRule.getKeyGenerateStrategy().getColumn(), + tableShardingRule.getKeyGenerateStrategy().getKeyGeneratorName()); + configuration.setKeyGenerateStrategy( + new KeyGenerateStrategyConfiguration( + tableShardingRule.getKeyGenerateStrategy().getColumn(), + tableShardingRule.getKeyGenerateStrategy().getKeyGeneratorName() + ) + ); + } + if (tableShardingRule.getDatabaseStrategy() != null) { + log.info("[{}] Create database sharding strategy, strategy: {}", + tableName, tableShardingRule.getDatabaseStrategy()); + configuration.setDatabaseShardingStrategy( + createShardingStrategyConfiguration(tableShardingRule.getDatabaseStrategy())); + } + if (tableShardingRule.getTableStrategy() != null) { + log.info("[{}] Create table sharding strategy, strategy: {}", + tableName, tableShardingRule.getTableStrategy()); + configuration.setTableShardingStrategy( + createShardingStrategyConfiguration(tableShardingRule.getTableStrategy())); + } + + return configuration; + } + + private ShardingStrategyConfiguration createShardingStrategyConfiguration( + ShardingProperties.ShardingsphereProperties.ShardingStrategy shardingStrategyProps) { + String shardingStrategyType = shardingStrategyProps.getType().trim().toLowerCase(); + switch (shardingStrategyType) { + case "standard": + return new StandardShardingStrategyConfiguration( + shardingStrategyProps.getShardingColumn(), + shardingStrategyProps.getShardingAlgorithmName() + ); + case "hint": + return new HintShardingStrategyConfiguration( + shardingStrategyProps.getShardingAlgorithmName() + ); + default: + throw new ShardingConfigParseException("Not support sharding algorithm"); + } + } + + + private Properties createShardingGlobalProps(Map globalPropsMap) { + // 配置 shardingsphere 系统级配置 + if (globalPropsMap == null || globalPropsMap.isEmpty()) { + return null; + } + return toProperties(globalPropsMap); + } + + @Qualifier("jobShardingTransactionManager") + @Bean(name = "jobShardingTransactionManager") + @DependsOn("jobShardingDataSource") + public DataSourceTransactionManager transactionManager( + @Qualifier("jobShardingDataSource") DataSource dataSource) { + return new DataSourceTransactionManager(dataSource); + } + + @Qualifier("job-sharding-jdbc-template") + @Bean(name = "job-sharding-jdbc-template") + public JdbcTemplate jdbcTemplate(@Qualifier("jobShardingDataSource") DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + @Qualifier("job-sharding-dsl-context") + @Bean(name = "job-sharding-dsl-context") + public DSLContext dslContext(@Qualifier("job-sharding-jooq-conf") org.jooq.Configuration configuration) { + return new DefaultDSLContext(configuration); + } + + @Qualifier("job-sharding-jooq-conf") + @Bean(name = "job-sharding-jooq-conf") + public org.jooq.Configuration jooqConf( + @Qualifier("job-sharding-conn-provider") ConnectionProvider connectionProvider) { + return new DefaultConfiguration().derive(connectionProvider).derive(SQLDialect.MYSQL); + } + + @Qualifier("job-sharding-conn-provider") + @Bean(name = "job-sharding-conn-provider") + public ConnectionProvider connectionProvider( + @Qualifier("jobShardingTransactionAwareDataSource") DataSource dataSource) { + return new DataSourceConnectionProvider(dataSource); + } + + @Qualifier("jobShardingTransactionAwareDataSource") + @Bean(name = "jobShardingTransactionAwareDataSource") + public TransactionAwareDataSourceProxy + transactionAwareDataSourceProxy(@Qualifier("jobShardingDataSource") DataSource dataSource) { + return new TransactionAwareDataSourceProxy(dataSource); + } +} diff --git a/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/ShardingProperties.java b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/ShardingProperties.java new file mode 100644 index 0000000000..f0573bbbab --- /dev/null +++ b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/ShardingProperties.java @@ -0,0 +1,244 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.sharding.mysql.config; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.springframework.boot.context.properties.ConfigurationProperties; + +import java.util.List; +import java.util.Map; +import java.util.StringJoiner; + +@ConfigurationProperties(prefix = "sharding") +@Getter +@Setter +@ToString +public class ShardingProperties { + /** + * 是否启用分库分表 + */ + private boolean enabled; + /** + * 数据库实例数量 + */ + private Integer dbNodeCount; + /** + * 每个数据库实例上的表分片数量 + */ + private Integer tableNodeCount; + + /** + * 单 db -> 分库分表迁移配置 + */ + private MigrationProperties migration; + + /** + * ShardingSphere 分片组件配置 + */ + private ShardingsphereProperties shardingsphere; + + @Getter + @Setter + @ToString + public static class MigrationProperties { + private boolean enabled; + } + + @Getter + @Setter + @ToString + public static class ShardingsphereProperties { + /** + * 分片数据库名 + */ + private String databaseName; + /** + * 数据源集群。 key:集群名称；value:集群配置 + */ + private Map dataSources; + /** + * 分片规则 + */ + private ShardingRule shardingRule; + /** + * 系统级属性配置 + */ + private Map props; + + @Getter + @Setter + public static class DataSource { + /** + * 数据库驱动类名，以数据库连接池自身配置为准 + */ + private String driverClassName; + /** + * 数据库 URL 连接，以数据库连接池自身配置为准 + */ + private String jdbcUrl; + private String username; + private String password; + + @Override + public String toString() { + return new StringJoiner(", ", DataSource.class.getSimpleName() + "[", "]") + .add("driverClassName='" + driverClassName + "'") + .add("jdbcUrl='" + jdbcUrl + "'") + .add("username='" + username + "'") + .add("password='******'") + .toString(); + } + } + + + /** + * 数据分片规则配置 + */ + @Getter + @Setter + @ToString + public static class ShardingRule { + /** + * 表对应的分片规则。 key: 表名称;value: 分片规则 + */ + private Map tables; + /** + * 分片算法定义 + */ + private Map shardingAlgorithms; + /** + * 分布式序列算法配置 + */ + private Map keyGenerators; + /** + * 默认数据库分片策略 + */ + private ShardingStrategy defaultDatabaseStrategy; + /** + * 默认表分片策略 + */ + private ShardingStrategy defaultTableStrategy; + /** + * 默认分片列名称 + */ + private String defaultShardingColumn; + /** + * 默认的分布式序列策略 + */ + private String defaultKeyGenerateStrategy; + + private List bindingTables; + } + + + @Getter + @Setter + @ToString + public static class TableShardingRule { + /** + * 由数据源名 + 表名组成 + */ + private String actualDataNodes; + /** + * 分库策略 + */ + private ShardingStrategy databaseStrategy; + /** + * 分表策略，同分库策略 + */ + private ShardingStrategy tableStrategy; + /** + * 分布式序列策略 + */ + private KeyGenerateStrategy keyGenerateStrategy; + } + + /** + * 分片计算策略 + */ + @Getter + @Setter + @ToString + public static class ShardingStrategy { + private String type; + private String shardingColumn; + private String shardingAlgorithmName; + } + + /** + * 分布式序列策略 + */ + @Getter + @Setter + @ToString + public static class KeyGenerateStrategy { + /** + * 自增列名称，缺省表示不使用自增主键生成器 + */ + private String column; + /** + * 分布式序列算法名称 + */ + private String keyGeneratorName; + } + + /** + * 分布式序列生成配置 + */ + @Getter + @Setter + @ToString + public static class KeyGenerator { + /** + * 分布式序列算法类型 + */ + private String type; + /** + * 分布式序列算法属性配置 + */ + private Map props; + } + + /** + * 分片算法配置 + */ + @Getter + @Setter + @ToString + public static class ShardingAlgorithm { + /** + * 分片算法类型 + */ + private String type; + /** + * 分片算法属性配置 + */ + private Map props; + } + + } +} diff --git a/src/backend/commons/common-mysql-sharding/src/main/resources/META-INF/spring.factories b/src/backend/commons/common-mysql-sharding/src/main/resources/META-INF/spring.factories index 4f8aad8350..f020f2ef71 100644 --- a/src/backend/commons/common-mysql-sharding/src/main/resources/META-INF/spring.factories +++ b/src/backend/commons/common-mysql-sharding/src/main/resources/META-INF/spring.factories @@ -1,2 +1,3 @@ org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ -com.tencent.bk.job.common.sharding.mysql.config.LeafAutoConfiguration +com.tencent.bk.job.common.sharding.mysql.config.LeafAutoConfiguration,\ +com.tencent.bk.job.common.sharding.mysql.config.ShardingDatasourceAutoConfiguration diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/inner/impl/ServiceArchiveResourceImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/inner/impl/ServiceArchiveResourceImpl.java index da10f3cc1f..a332b6bad9 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/inner/impl/ServiceArchiveResourceImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/inner/impl/ServiceArchiveResourceImpl.java @@ -26,7 +26,7 @@ import com.tencent.bk.job.backup.api.inner.ServiceArchiveResource; import com.tencent.bk.job.backup.archive.JobExecuteArchiveManage; -import com.tencent.bk.job.backup.config.ArchiveDBProperties; +import com.tencent.bk.job.backup.config.ArchiveProperties; import com.tencent.bk.job.backup.constant.ArchiveModeEnum; import com.tencent.bk.job.backup.model.inner.ServiceArchiveDBRequest; import com.tencent.bk.job.common.model.InternalResponse; @@ -53,24 +53,24 @@ public ServiceArchiveResourceImpl(@Autowired(required = false) JobExecuteArchive @Override public InternalResponse archive(ServiceArchiveDBRequest request) { log.info("Begin archive db, request: {}", request); - ArchiveDBProperties archiveDBProperties = new ArchiveDBProperties(); + ArchiveProperties archiveProperties = new ArchiveProperties(); if (StringUtils.isNotEmpty(request.getMode())) { - archiveDBProperties.setMode(request.getMode()); + archiveProperties.setMode(request.getMode()); } else { - archiveDBProperties.setMode(ArchiveModeEnum.BACKUP_THEN_DELETE.getMode()); + archiveProperties.setMode(ArchiveModeEnum.BACKUP_THEN_DELETE.getMode()); } - archiveDBProperties.setEnabled(true); - archiveDBProperties.setKeepDays(request.getKeepDays()); - archiveDBProperties.setBatchInsertRowSize(request.getBatchInsertRowSize()); - archiveDBProperties.setDeleteRowLimit(request.getDeleteRowLimit()); - archiveDBProperties.setReadIdStepSize(request.getReadIdStepSize()); - archiveDBProperties.setReadRowLimit(request.getReadRowLimit()); + archiveProperties.setEnabled(true); + archiveProperties.setKeepDays(request.getKeepDays()); + archiveProperties.setBatchInsertRowSize(request.getBatchInsertRowSize()); + archiveProperties.setDeleteRowLimit(request.getDeleteRowLimit()); + archiveProperties.setReadIdStepSize(request.getReadIdStepSize()); + archiveProperties.setReadRowLimit(request.getReadRowLimit()); if (request.getTableConfigs() != null) { - Map tableConfigMap = new HashMap<>(); + Map tableConfigMap = new HashMap<>(); request.getTableConfigs().forEach((table, config) -> { - ArchiveDBProperties.TableConfig tableConfig = new ArchiveDBProperties.TableConfig(); + ArchiveProperties.TableConfig tableConfig = new ArchiveProperties.TableConfig(); tableConfig.setBatchInsertRowSize(config.getBatchInsertRowSize()); tableConfig.setDeleteRowLimit(config.getDeleteRowLimit()); tableConfig.setReadIdStepSize(config.getReadIdStepSize()); @@ -78,10 +78,10 @@ public InternalResponse archive(ServiceArchiveDBRequest request) { tableConfigMap.put(table, tableConfig); }); - archiveDBProperties.setTableConfigs(tableConfigMap); + archiveProperties.setTableConfigs(tableConfigMap); } - jobExecuteArchiveManage.archive(archiveDBProperties); - return InternalResponse.buildSuccessResp(archiveDBProperties); + jobExecuteArchiveManage.archive(archiveProperties); + return InternalResponse.buildSuccessResp(archiveProperties); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractArchivist.java deleted file mode 100644 index 5ced19a6f8..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractArchivist.java +++ /dev/null @@ -1,518 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive; - -import com.tencent.bk.job.backup.config.ArchiveDBProperties; -import com.tencent.bk.job.backup.constant.ArchiveModeEnum; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.ExecuteRecordDAO; -import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; -import com.tencent.bk.job.backup.model.dto.ArchiveProgressDTO; -import com.tencent.bk.job.backup.model.dto.ArchiveSummary; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.jooq.TableRecord; -import org.slf4j.helpers.MessageFormatter; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.function.Function; - -/** - * 表归档基础实现 - * - * @param 表记录 - */ -@Data -@Slf4j -public abstract class AbstractArchivist> { - protected ExecuteRecordDAO executeRecordDAO; - protected ExecuteArchiveDAO executeArchiveDAO; - protected ArchiveProgressService archiveProgressService; - private ArchiveDBProperties archiveDBProperties; - /** - * 需要归档的记录的最大 ID (include) - */ - private Long maxNeedArchiveId; - /** - * 需要归档的记录的最小 ID (include) - */ - private Long minNeedArchiveId; - /** - * 表中已存在的记录的最小 ID - */ - private Long minExistedRecordArchiveId; - /** - * 上次归档任务已备份的最后一条数据的ID - */ - private Long lastBackupId; - /** - * 上次归档任务已删除的最后一条数据的ID - */ - private Long lastDeleteId; - - private CountDownLatch countDownLatch; - /** - * 读取DB 步长 - */ - protected int readIdStepSize; - /** - * 每批次从 db 表中读取的记录数量 - */ - protected int readRowLimit; - /** - * 写入归档数据，单批次最小行数 - */ - protected int batchInsertRowSize; - /** - * 删除数据ID增加步长 - */ - protected int deleteIdStepSize = 10_000; - /** - * 每次执行删除的最大行数 - */ - protected int deleteLimitRowCount; - protected String tableName; - private ArchiveSummary archiveSummary; - private boolean isAcquireLock; - - private ArchiveTaskLock archiveTaskLock; - - private final ArchiveErrorTaskCounter archiveErrorTaskCounter; - - public AbstractArchivist(ExecuteRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService, - ArchiveDBProperties archiveDBProperties, - ArchiveTaskLock archiveTaskLock, - Long maxNeedArchiveId, - CountDownLatch countDownLatch, - ArchiveErrorTaskCounter archiveErrorTaskCounter) { - this.executeRecordDAO = executeRecordDAO; - this.executeArchiveDAO = executeArchiveDAO; - this.archiveProgressService = archiveProgressService; - this.archiveDBProperties = archiveDBProperties; - this.tableName = executeRecordDAO.getTable().getName().toLowerCase(); - this.readIdStepSize = computeValuePreferTableConfig(archiveDBProperties.getReadIdStepSize(), - ArchiveDBProperties.TableConfig::getReadIdStepSize); - this.batchInsertRowSize = computeValuePreferTableConfig(archiveDBProperties.getBatchInsertRowSize(), - ArchiveDBProperties.TableConfig::getBatchInsertRowSize); - this.readRowLimit = computeValuePreferTableConfig(archiveDBProperties.getReadRowLimit(), - ArchiveDBProperties.TableConfig::getReadRowLimit); - this.deleteLimitRowCount = computeValuePreferTableConfig(archiveDBProperties.getDeleteRowLimit(), - ArchiveDBProperties.TableConfig::getDeleteRowLimit); - this.maxNeedArchiveId = maxNeedArchiveId; - this.countDownLatch = countDownLatch; - this.archiveSummary = new ArchiveSummary(this.tableName); - this.archiveTaskLock = archiveTaskLock; - this.archiveErrorTaskCounter = archiveErrorTaskCounter; - } - - /** - * 计算归档参数的值 - * - * @param defaultValue 默认值 - * @param tableValueProvider DB 表作用域下的参数值提供者 - */ - private V computeValuePreferTableConfig( - V defaultValue, - Function tableValueProvider - ) { - - V value = defaultValue; - if (archiveDBProperties.getTableConfigs() != null - && archiveDBProperties.getTableConfigs().containsKey(tableName)) { - ArchiveDBProperties.TableConfig tableConfig = archiveDBProperties.getTableConfigs().get(tableName); - V tableValue = tableValueProvider.apply(tableConfig); - if (tableValue != null) { - value = tableValue; - } - } - return value; - } - - public void archive() { - try { - if (!acquireLock()) { - archiveSummary.setSkip(!isAcquireLock); - return; - } - - if (!archiveDBProperties.isEnabled()) { - archiveSummary.setSkip(true); - log.info("[{}] Archive is disabled, skip archive", tableName); - return; - } - archiveSummary.setEnabled(true); - - log.info("[{}] Start archive", tableName); - - initArchiveIdSettings(); - - log.info("[{}] Archive record config, readIdStepSize: {}, readRowLimit: {}, batchInsertRowSize: {}, " + - "deleteLimitRowCount: {}", - tableName, readIdStepSize, readRowLimit, batchInsertRowSize, deleteLimitRowCount); - - if (minExistedRecordArchiveId == null) { - // min 查询返回 null，说明是空表，无需归档 - archiveSummary.setSkip(true); - archiveSummary.setSuccess(true); - archiveSummary.setMessage("Empty table, do not need archive"); - log.info("[{}] Empty table, do not need archive!", tableName); - return; - } - - if (maxNeedArchiveId < this.minNeedArchiveId) { - log.info("[{}] MinNeedArchiveId {} is greater than maxNeedArchiveId {}, skip archive table!", - tableName, minNeedArchiveId, maxNeedArchiveId); - archiveSummary.setSkip(true); - archiveSummary.setSuccess(true); - archiveSummary.setMessage("MinNeedArchiveId is greater than maxNeedArchiveId, skip archive table"); - return; - } - - // 归档 - backupAndDelete(); - - } catch (Throwable e) { - String msg = MessageFormatter.format( - "Error while archiving {}", - tableName - ).getMessage(); - log.error(msg, e); - archiveErrorTaskCounter.increment(); - archiveSummary.setMessage(e.getMessage()); - } finally { - archiveSummary.setArchiveMode(archiveDBProperties.getMode()); - storeArchiveSummary(); - if (this.isAcquireLock) { - archiveTaskLock.unlock(tableName); - } - countDownLatch.countDown(); - } - } - - private void backupAndDelete() throws IOException { - boolean backupEnabled = isBackupEnable(archiveDBProperties); - boolean deleteEnabled = isDeleteEnable(archiveDBProperties); - long backupRows = 0; - long readRows = 0; - long deleteRows = 0; - long startTime = System.currentTimeMillis(); - long start = this.minNeedArchiveId - 1; - long stop = start; - boolean success = true; - long backupReadRecordCost = 0; - long backupWriteRecordCost = 0; - long deleteCost = 0; - log.info("[{}] Start backup and delete process, backupEnabled: {}, deleteEnabled: {}", tableName, - backupEnabled, deleteEnabled); - try { - while (maxNeedArchiveId > start) { - // start < id <= stop - stop = Math.min(maxNeedArchiveId, start + readIdStepSize); - - log.info("[{}] LoopArchive, current: [{}-{}]", tableName, start, stop); - - BackupResult backupResult = null; - if (backupEnabled) { - backupResult = backupRecords(start, stop); - readRows += backupResult.getReadRows(); - backupRows += backupResult.getBackupRows(); - backupReadRecordCost += backupResult.getReadCost(); - backupWriteRecordCost += backupResult.getWriteCost(); - } - - if (deleteEnabled) { - long deleteStartTime = System.currentTimeMillis(); - if (backupResult != null) { - if (backupResult.getReadRows() > 0) { - // 降低 delete 执行次数：备份过程中读取的数据行数大于 0，才会执行 delete 操作 - deleteRows += delete(start, stop); - } - } else { - deleteRows += delete(start, stop); - } - deleteCost += (System.currentTimeMillis() - deleteStartTime); - } - - start = stop; - } - } catch (Throwable e) { - success = false; - throw e; - } finally { - long archiveCost = System.currentTimeMillis() - startTime; - log.info( - "Archive {} finished, result: {}, minNeedArchiveId: {}, maxNeedArchiveId: {}, readRows: {}, " + - "backupRows: {}, deleteRows: {}, cost: {}ms", - tableName, - success ? "success" : "fail", - minNeedArchiveId, - maxNeedArchiveId, - readRows, - backupRows, - deleteRows, - archiveCost - ); - setArchiveSummary( - minNeedArchiveId, - maxNeedArchiveId, - readRows, - backupRows, - deleteRows, - backupEnabled ? stop : null, - deleteEnabled ? stop : null, - archiveCost, - success, - backupReadRecordCost, - backupWriteRecordCost, - deleteCost - ); - } - } - - protected boolean isBackupEnable(ArchiveDBProperties archiveDBProperties) { - ArchiveModeEnum archiveMode = ArchiveModeEnum.valOf(archiveDBProperties.getMode()); - return archiveDBProperties.isEnabled() - && (ArchiveModeEnum.BACKUP_THEN_DELETE == archiveMode || ArchiveModeEnum.BACKUP_ONLY == archiveMode); - } - - protected boolean isDeleteEnable(ArchiveDBProperties archiveDBProperties) { - ArchiveModeEnum archiveMode = ArchiveModeEnum.valOf(archiveDBProperties.getMode()); - return archiveDBProperties.isEnabled() - && (ArchiveModeEnum.BACKUP_THEN_DELETE == archiveMode || ArchiveModeEnum.DELETE_ONLY == archiveMode); - } - - private boolean acquireLock() { - this.isAcquireLock = archiveTaskLock.lock(tableName); - if (!isAcquireLock) { - log.info("[{}] Acquire lock fail", tableName); - } - return isAcquireLock; - } - - /** - * 对一个批次数据进行备份 - * - * @param start 数据起始记录ID - * @param stop 数据终止记录ID - * @return 备份结果 - */ - private BackupResult backupRecords(long start, long stop) throws IOException { - if (lastBackupId >= stop) { - // 说明数据已经备份过，跳过 - log.info("[{}] Record is already backup, skip. lastBackId: {}", tableName, lastBackupId); - return new BackupResult(0L, 0L, 0L, 0L); - } - long startId = start; - if (lastBackupId > start) { - // 从上次备份结束的 ID 位置开始 - startId = lastBackupId; - } - - List recordList = new ArrayList<>(readRowLimit); - long backupRows = 0; - long readRows = 0; - long readCost = 0; - long writeCost = 0; - long offset = 0L; - List records; - do { - // 选取start= batchInsertRowSize) { - long writeStartTime = System.currentTimeMillis(); - int insertRows = insertAndReset(recordList); - writeCost += (System.currentTimeMillis() - writeStartTime); - backupRows += insertRows; - } - offset += readRowLimit; - - } while (records.size() == readRowLimit); - - if (CollectionUtils.isNotEmpty(recordList)) { - // 处理没有达到批量插入阈值的最后一个批次的数据 - int insertRows = insertAndReset(recordList); - backupRows += insertRows; - } - - updateBackupProgress(stop); - - return new BackupResult(readRows, backupRows, readCost, writeCost); - } - - @Data - private static class BackupResult { - /** - * 读取的记录数量 - */ - private long readRows; - /** - * 备份成功的记录数量 - */ - private long backupRows; - /** - * 读取耗时 - */ - private long readCost; - /** - * 备份写入耗时 - */ - private long writeCost; - - public BackupResult(long readRows, long backupRows, long readCost, long writeCost) { - this.readRows = readRows; - this.backupRows = backupRows; - this.readCost = readCost; - this.writeCost = writeCost; - } - } - - private void setArchiveSummary(Long minNeedArchiveId, - Long maxNeedArchiveId, - long readRows, - long backupRows, - long deleteRows, - Long lastBackupId, - Long lastDeleteId, - long archiveCost, - boolean success, - long backupReadCost, - long backupWriteCost, - long deleteCost) { - archiveSummary.setArchiveIdStart(minNeedArchiveId); - archiveSummary.setArchiveIdEnd(maxNeedArchiveId); - archiveSummary.setNeedArchiveRecordSize(readRows); - archiveSummary.setBackupRecordSize(backupRows); - archiveSummary.setDeleteRecordSize(deleteRows); - archiveSummary.setLastBackupId(lastBackupId); - archiveSummary.setLastDeletedId(lastDeleteId); - archiveSummary.setArchiveCost(archiveCost); - archiveSummary.setSuccess(success); - archiveSummary.setBackupReadCost(backupReadCost); - archiveSummary.setBackupWriteCost(backupWriteCost); - archiveSummary.setDeleteCost(deleteCost); - } - - /** - * 初始化本次归档数据的ID设置 - */ - private void initArchiveIdSettings() { - ArchiveProgressDTO archiveProgress = archiveProgressService.queryArchiveProgress(tableName); - this.lastBackupId = (archiveProgress == null || archiveProgress.getLastBackupId() == null) ? - 0 : archiveProgress.getLastBackupId(); - this.lastDeleteId = (archiveProgress == null || archiveProgress.getLastDeletedId() == null) ? - 0 : archiveProgress.getLastDeletedId(); - this.minExistedRecordArchiveId = executeRecordDAO.getMinArchiveId(); - this.minNeedArchiveId = minExistedRecordArchiveId; - log.info("[{}] Init archive id settings, lastBackupId: {}, lastDeleteId: {}, minExistedRecordArchiveId: {}," - + " minNeedArchiveId: {}, maxNeedArchiveId: {}", - tableName, lastBackupId, lastDeleteId, minExistedRecordArchiveId, minNeedArchiveId, maxNeedArchiveId); - } - - private void updateBackupProgress(long archiveId) { - ArchiveProgressDTO archiveProgress = new ArchiveProgressDTO(); - archiveProgress.setTableName(tableName); - archiveProgress.setLastBackupTime(System.currentTimeMillis()); - archiveProgress.setLastBackupId(archiveId); - archiveProgressService.saveArchiveProgress(archiveProgress); - } - - private void updateDeleteProgress(long archiveId) { - ArchiveProgressDTO archiveProgress = new ArchiveProgressDTO(); - archiveProgress.setTableName(tableName); - archiveProgress.setLastDeleteTime(System.currentTimeMillis()); - archiveProgress.setLastDeletedId(archiveId); - archiveProgressService.saveDeleteProgress(archiveProgress); - } - - /** - * 对一个批次数据进行删除 - * - * @param start 数据起始记录ID - * @param stop 数据终止记录ID - */ - private int delete(long start, long stop) { - long startTime = System.currentTimeMillis(); - - int deleteCount = deleteRecord(start, stop); - updateDeleteProgress(stop); - log.info("Delete {}, start: {}, stop: {}, delete rows: {}, cost: {}ms", tableName, - start, stop, deleteCount, System.currentTimeMillis() - startTime); - return deleteCount; - } - - private int insertAndReset(List recordList) throws IOException { - if (CollectionUtils.isEmpty(recordList)) { - return 0; - } - long startTime = System.currentTimeMillis(); - int recordSize = recordList.size(); - int insertRows = batchInsert(recordList); - recordList.clear(); - if (insertRows != recordSize) { - throw new ArchiveException(String.format("Insert rows not expected, expect: %s, actual: %s", - recordSize, insertRows)); - } - long costTime = System.currentTimeMillis() - startTime; - log.info("Batch insert {}, maxBatchSize: {}, insert rows: {}, cost: {}", - tableName, batchInsertRowSize, insertRows, costTime); - return insertRows; - } - - private void storeArchiveSummary() { - ArchiveSummaryHolder.getInstance().addArchiveSummary(this.archiveSummary); - } - - private List listRecord(Long start, Long stop, Long offset, Long limit) { - return executeRecordDAO.listRecords(start, stop, offset, limit); - } - - private int batchInsert(List recordList) throws IOException { - return executeArchiveDAO.batchInsert(recordList, batchInsertRowSize); - } - - private int deleteRecord(Long start, Long stop) { - return executeRecordDAO.deleteRecords(start, stop, deleteLimitRowCount); - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java new file mode 100644 index 0000000000..7582103260 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java @@ -0,0 +1,268 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.JobInstanceHotRecordDAO; +import com.tencent.bk.job.backup.archive.model.ArchiveTaskSummary; +import com.tencent.bk.job.backup.archive.model.DbDataNode; +import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; +import com.tencent.bk.job.backup.archive.model.TimeAndIdBasedArchiveProcess; +import com.tencent.bk.job.backup.config.ArchiveProperties; +import com.tencent.bk.job.backup.constant.ArchiveModeEnum; +import com.tencent.bk.job.backup.constant.ArchiveTaskStatusEnum; +import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; +import com.tencent.bk.job.common.util.json.JsonUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.jooq.TableRecord; +import org.slf4j.helpers.MessageFormatter; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * 作业实例数据归档任务基础实现 + * + * @param 表记录 + */ +@Slf4j +public abstract class AbstractJobInstanceArchiveTask> implements JobInstanceArchiveTask { + protected JobInstanceHotRecordDAO jobInstanceHotRecordDAO; + protected JobInstanceColdDAO jobInstanceColdDAO; + protected final ArchiveProperties archiveProperties; + private final ArchiveTaskLock archiveTaskLock; + protected final ArchiveErrorTaskCounter archiveErrorTaskCounter; + protected final ArchiveTaskService archiveTaskService; + protected final ArchiveTablePropsStorage archiveTablePropsStorage; + + protected String taskId; + protected DbDataNode dbDataNode; + protected JobInstanceArchiveTaskInfo archiveTask; + + /** + * 归档进度 + */ + private TimeAndIdBasedArchiveProcess progress; + + private final ArchiveTaskSummary archiveTaskSummary; + + private boolean isAcquireLock; + + + public AbstractJobInstanceArchiveTask(JobInstanceHotRecordDAO jobInstanceHotRecordDAO, + JobInstanceColdDAO jobInstanceColdDAO, + ArchiveProperties archiveProperties, + ArchiveTaskLock archiveTaskLock, + ArchiveErrorTaskCounter archiveErrorTaskCounter, + JobInstanceArchiveTaskInfo archiveTask, + ArchiveTaskService archiveTaskService, + ArchiveTablePropsStorage archiveTablePropsStorage) { + this.jobInstanceHotRecordDAO = jobInstanceHotRecordDAO; + this.jobInstanceColdDAO = jobInstanceColdDAO; + this.archiveProperties = archiveProperties; + this.archiveTaskLock = archiveTaskLock; + this.archiveErrorTaskCounter = archiveErrorTaskCounter; + this.archiveTask = archiveTask; + this.archiveTaskService = archiveTaskService; + this.archiveTablePropsStorage = archiveTablePropsStorage; + this.progress = archiveTask.getProcess(); + this.taskId = buildTaskId(archiveTask); + this.archiveTaskSummary = new ArchiveTaskSummary(archiveTask, archiveProperties.getMode()); + } + + private String buildTaskId(JobInstanceArchiveTaskInfo archiveTask) { + return archiveTask.getTaskType() + ":" + archiveTask.getDbDataNode().toDataNodeId() + + ":" + archiveTask.getDay() + ":" + archiveTask.getHour(); + } + + + @Override + public void execute() { + archive(); + } + + @Override + public void stop() { + + } + + private void archive() { + try { + if (!acquireLock()) { + archiveTaskSummary.setSkip(!isAcquireLock); + return; + } + + log.info("[{}] Start archive task", taskId); + // 归档 + backupAndDelete(); + } catch (Throwable e) { + String msg = MessageFormatter.format( + "Error while execute archive task : {}", + taskId + ).getMessage(); + log.error(msg, e); + archiveTaskSummary.setMessage(e.getMessage()); + + archiveErrorTaskCounter.increment(); + + // 更新归档任务状态 + updateArchiveProgress(ArchiveTaskStatusEnum.FAIL, null); + } finally { + if (this.isAcquireLock) { + archiveTaskLock.unlock(taskId); + } + log.info( + "[{}] Archive finished, result: {}", + taskId, + JsonUtils.toJson(archiveTaskSummary) + ); + } + } + + private void backupAndDelete() { + boolean backupEnabled = isBackupEnable(); + boolean deleteEnabled = isDeleteEnable(); + + int readLimit = 1000; + long archivedJobInstanceCount = 0; + + long startTime = System.currentTimeMillis(); + log.info("[{}] Archive task mode: {}, backupEnabled: {}, deleteEnabled: {}", + taskId, archiveProperties.getMode(), backupEnabled, deleteEnabled); + try { + List jobInstanceRecords; + do { + if (progress != null) { + jobInstanceRecords = readSortedJobInstanceFromHotDB(progress.getTimestamp(), + archiveTask.getToTimestamp(), progress.getId(), readLimit); + } else { + jobInstanceRecords = readSortedJobInstanceFromHotDB(archiveTask.getFromTimestamp(), + archiveTask.getToTimestamp(), null, readLimit); + } + + if (CollectionUtils.isEmpty(jobInstanceRecords)) { + updateArchiveProgress(ArchiveTaskStatusEnum.SUCCESS, null); + return; + } + archivedJobInstanceCount++; + + List jobInstanceIds = + jobInstanceRecords.stream().map(this::extractJobInstanceId).collect(Collectors.toList()); + + // 写入数据到冷 db + if (backupEnabled) { + backupJobInstanceToColdDb(jobInstanceRecords); + } + // 从热 db 删除数据 + if (deleteEnabled) { + deleteJobInstanceHotData(jobInstanceIds); + } + + // 更新归档进度 + T lastRecord = jobInstanceRecords.get(jobInstanceRecords.size() - 1); + Long lastTimestamp = extractJobInstanceCreateTime(lastRecord); + Long lastJobInstanceId = extractJobInstanceId(lastRecord); + progress = new TimeAndIdBasedArchiveProcess(lastTimestamp, lastJobInstanceId); + boolean isFinished = jobInstanceRecords.size() < readLimit; + updateArchiveProgress(isFinished ? ArchiveTaskStatusEnum.SUCCESS : ArchiveTaskStatusEnum.RUNNING, + progress); + } while (jobInstanceRecords.size() == readLimit); + } finally { + long archiveCost = System.currentTimeMillis() - startTime; + archiveTaskSummary.setArchivedRecordSize(archivedJobInstanceCount); + archiveTaskSummary.setArchiveCost(archiveCost); + } + } + + /** + * 备份作业实例数据到冷存储 + * + * @param jobInstances 作业实例列表 + */ + protected abstract void backupJobInstanceToColdDb(List jobInstances); + + /** + * 删除作业实例热数据 + * + * @param jobInstanceIds 作业实例 ID 列表 + */ + protected abstract void deleteJobInstanceHotData(List jobInstanceIds); + + protected boolean isBackupEnable() { + ArchiveModeEnum archiveMode = ArchiveModeEnum.valOf(archiveProperties.getMode()); + return archiveProperties.isEnabled() + && (ArchiveModeEnum.BACKUP_THEN_DELETE == archiveMode || ArchiveModeEnum.BACKUP_ONLY == archiveMode); + } + + protected boolean isDeleteEnable() { + ArchiveModeEnum archiveMode = ArchiveModeEnum.valOf(archiveProperties.getMode()); + return archiveProperties.isEnabled() + && (ArchiveModeEnum.BACKUP_THEN_DELETE == archiveMode || ArchiveModeEnum.DELETE_ONLY == archiveMode); + } + + private boolean acquireLock() { + this.isAcquireLock = archiveTaskLock.lock(taskId); + if (!isAcquireLock) { + log.info("[{}] Acquire lock fail", taskId); + } + return isAcquireLock; + } + + private void updateArchiveProgress(ArchiveTaskStatusEnum taskStatus, TimeAndIdBasedArchiveProcess progress) { + archiveTask.setStatus(taskStatus); + archiveTask.setProcess(progress); + archiveTaskService.updateTask(archiveTask); + } + + /** + * 从热 db 读取作业实例熟悉，按照时间+ID 的顺序排序 + * + * @param fromTimestamp 时间范围-起始-作业实例创建时间（include) + * @param endTimestamp 时间范围-起始-作业实例创建时间（exclude) + * @param fromJobInstanceId 作业实例 ID-起始 (exclude) + * @param limit 读取记录最大数量 + * @return 作业实例记录 + */ + protected abstract List readSortedJobInstanceFromHotDB(Long fromTimestamp, + Long endTimestamp, + Long fromJobInstanceId, + int limit); + + /** + * 从作业实例记录中提取作业实例 ID + * + * @param record 作业实例记录 + */ + protected abstract Long extractJobInstanceId(T record); + + /** + * 从作业实例记录中提取作业实例创建时间 + * + * @param record 作业实例记录 + */ + protected abstract Long extractJobInstanceCreateTime(T record); +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractTableArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractTableArchivist.java new file mode 100644 index 0000000000..b21e7ad4b1 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractTableArchivist.java @@ -0,0 +1,93 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.JobInstanceHotRecordDAO; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.jooq.TableRecord; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; + +@Slf4j +public abstract class AbstractTableArchivist> { + + protected String tableName; + protected JobInstanceHotRecordDAO jobInstanceHotRecordDAO; + protected JobInstanceColdDAO jobInstanceColdDAO; + protected int batchInsertRowSize; + protected int deleteLimitRowCount; + + public AbstractTableArchivist(String tableName, + JobInstanceHotRecordDAO jobInstanceHotRecordDAO, + JobInstanceColdDAO jobInstanceColdDAO, + int batchInsertRowSize, + int deleteLimitRowCount) { + this.tableName = tableName; + this.jobInstanceHotRecordDAO = jobInstanceHotRecordDAO; + this.jobInstanceColdDAO = jobInstanceColdDAO; + this.batchInsertRowSize = batchInsertRowSize; + this.deleteLimitRowCount = deleteLimitRowCount; + } + + void listRecord(Collection jobInstanceIds); + + void listRecord(Collection jobInstanceIds); + + void deleteRecord(Collection jobInstanceIds); + + void saveCold(); + + protected int insertAndReset(List recordList) throws IOException { + if (CollectionUtils.isEmpty(recordList)) { + return 0; + } + long startTime = System.currentTimeMillis(); + int recordSize = recordList.size(); + int insertRows = batchInsert(recordList); + recordList.clear(); + if (insertRows != recordSize) { + throw new ArchiveException(String.format("Insert rows not expected, expect: %s, actual: %s", + recordSize, insertRows)); + } + long costTime = System.currentTimeMillis() - startTime; + log.info("Batch insert {}, maxBatchSize: {}, insert rows: {}, cost: {}", + tableName, batchInsertRowSize, insertRows, costTime); + return insertRows; + } + + private int batchInsert(List recordList) throws IOException { + return jobInstanceColdDAO.batchInsert(recordList, batchInsertRowSize); + } + + private int deleteRecord(List jobInstanceIds) { + return jobInstanceHotRecordDAO.deleteRecords(start, stop, deleteLimitRowCount); + } + + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveSummaryHolder.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveSummaryHolder.java index dbb563bb7d..53501ee4ae 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveSummaryHolder.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveSummaryHolder.java @@ -24,7 +24,7 @@ package com.tencent.bk.job.backup.archive; -import com.tencent.bk.job.backup.model.dto.ArchiveSummary; +import com.tencent.bk.job.backup.archive.model.ArchiveTaskSummary; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.common.util.json.JsonUtils; import lombok.extern.slf4j.Slf4j; @@ -35,7 +35,7 @@ @Slf4j public class ArchiveSummaryHolder { - private Map summaryMap = new ConcurrentHashMap<>(); + private Map summaryMap = new ConcurrentHashMap<>(); private Long endTimeInMills; private ArchiveSummaryHolder() { @@ -50,12 +50,12 @@ public void init(Long endTimeInMills) { this.endTimeInMills = endTimeInMills; } - public void addArchiveSummary(ArchiveSummary summary) { + public void addArchiveSummary(ArchiveTaskSummary summary) { if (summary == null) { return; } summary.setArchiveEndDate(DateUtils.formatUnixTimestamp(endTimeInMills, ChronoUnit.MILLIS)); - summaryMap.put(summary.getTableName(), summary); + summaryMap.put(summary.getTaskId(), summary); } public void print() { diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTablePropsStorage.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTablePropsStorage.java new file mode 100644 index 0000000000..b3fd1f4a68 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTablePropsStorage.java @@ -0,0 +1,93 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.model.ArchiveTableProps; +import com.tencent.bk.job.backup.config.ArchiveProperties; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.Map; + +@Component +public class ArchiveTablePropsStorage { + private final ArchiveProperties archiveProperties; + private final Map tablePropsMap = new HashMap<>(); + + + @Autowired + public ArchiveTablePropsStorage(ArchiveProperties archiveProperties) { + this.archiveProperties = archiveProperties; + if (archiveProperties.getTableConfigs() != null && !archiveProperties.getTableConfigs().isEmpty()) { + archiveProperties.getTableConfigs().forEach( + (tableName, tableConfig) -> + tablePropsMap.put(tableName, new ArchiveTableProps( + tableConfig.getReadRowLimit(), + tableConfig.getBatchInsertRowSize(), + tableConfig.getDeleteRowLimit() + ))); + } + } + + /** + * @param tableName 表名 + *

+ * 从 热 DB 表中读取归档数据，每次读取的记录数量限制 + */ + public int getReadRowLimit(String tableName) { + ArchiveTableProps props = tablePropsMap.get(tableName); + if (props == null || props.getReadRowLimit() == null) { + return archiveProperties.getReadRowLimit(); + } + return props.getReadRowLimit(); + } + + /** + * @param tableName 表名 + *

+ * 写入归档数据到冷 DB，单批次最小行数 + */ + public int getBatchInsertRowSize(String tableName) { + ArchiveTableProps props = tablePropsMap.get(tableName); + if (props == null || props.getBatchInsertRowSize() == null) { + return archiveProperties.getBatchInsertRowSize(); + } + return props.getBatchInsertRowSize(); + } + + /** + * @param tableName 表名 + *

+ * 从热 DB 删除数据，每次删除的最大行数 + */ + public int getDeleteLimitRowCount(String tableName) { + ArchiveTableProps props = tablePropsMap.get(tableName); + if (props == null || props.getDeleteLimitRowCount() == null) { + return archiveProperties.getDeleteRowLimit(); + } + return props.getDeleteLimitRowCount(); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskLock.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskLock.java index a8640499ad..9455c466a2 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskLock.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskLock.java @@ -35,9 +35,12 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +/** + * 归档任务执行分布式锁 + */ @Slf4j public class ArchiveTaskLock { - private final String ARCHIVE_LOCK_KEY_PREFIX = "JOB_EXECUTE_LOG_ARCHIVE_LOCK"; + private final String ARCHIVE_LOCK_KEY_PREFIX = "archive:task:execute"; /** * 归档任务锁时间 1h */ @@ -49,12 +52,12 @@ public class ArchiveTaskLock { private volatile long lastAcquireLockTimeMS = 0L; /** - * Key: tableName ; Value: 分布式锁请求 requestId + * Key: taskId ; Value: 分布式锁请求 requestId */ private final Map locks = new ConcurrentHashMap<>(); /** * 分布式锁保持，避免超时失效的心跳线程 - * Key: tableName ; Value: 心跳线程 + * Key: taskId ; Value: 心跳线程 */ private final Map lockKeepThreads = new ConcurrentHashMap<>(); @@ -64,25 +67,25 @@ public ArchiveTaskLock(StringRedisTemplate redisTemplate) { this.redisTemplate = redisTemplate; } - public synchronized boolean lock(String tableName) { + public synchronized boolean lock(String taskId) { while (System.currentTimeMillis() - lastAcquireLockTimeMS < MIN_ACQUIRE_LOCK_INTERVAL_MS) { ThreadUtils.sleep(10L); } String lockRequestId = UUID.randomUUID().toString(); - String archiveLockKey = ARCHIVE_LOCK_KEY_PREFIX + "_" + tableName; + String archiveLockKey = ARCHIVE_LOCK_KEY_PREFIX + "_" + taskId; if (!LockUtils.tryGetDistributedLock(archiveLockKey, lockRequestId, LOCK_TIME)) { - log.info("Acquire archive task lock failed! tableName: {}", tableName); + log.info("Acquire archive task lock failed! taskId: {}", taskId); return false; } else { - log.info("Acquire archive task lock successfully! tableName: {}", tableName); + log.info("Acquire archive task lock successfully! taskId: {}", taskId); this.lastAcquireLockTimeMS = System.currentTimeMillis(); - this.locks.put(tableName, lockRequestId); - startRedisKeyHeartBeatThread(tableName, archiveLockKey, lockRequestId); + this.locks.put(taskId, lockRequestId); + startRedisKeyHeartBeatThread(taskId, archiveLockKey, lockRequestId); return true; } } - private void startRedisKeyHeartBeatThread(String tableName, + private void startRedisKeyHeartBeatThread(String taskId, String archiveLockKey, String lockRequestId) { // 开一个心跳子线程，维持锁状态不会因为超时失效 @@ -94,37 +97,37 @@ private void startRedisKeyHeartBeatThread(String tableName, LOCK_TIME, 30 * 60 * 1000L ); - redisKeyHeartBeatThread.setName("[ArchiveTask-" + tableName + "]-redisKeyHeartBeatThread"); - lockKeepThreads.put(tableName, redisKeyHeartBeatThread); + redisKeyHeartBeatThread.setName("[ArchiveTask-" + taskId + "]-redisKeyHeartBeatThread"); + lockKeepThreads.put(taskId, redisKeyHeartBeatThread); - log.info("Start redis key heart beat thread for ArchiveTask:{}", tableName); + log.info("Start redis key heart beat thread for ArchiveTask:{}", taskId); redisKeyHeartBeatThread.start(); } - private void stopRedisKeyHeartBeatThread(String tableName) { - RedisKeyHeartBeatThread heartBeatThread = lockKeepThreads.get(tableName); + private void stopRedisKeyHeartBeatThread(String taskId) { + RedisKeyHeartBeatThread heartBeatThread = lockKeepThreads.get(taskId); if (heartBeatThread == null) { - log.error("RedisKeyHeartBeatThread for table {} not exist", tableName); + log.error("RedisKeyHeartBeatThread for table {} not exist", taskId); return; } - log.info("Stop redis key heart beat thread for ArchiveTask:{}", tableName); + log.info("Stop redis key heart beat thread for ArchiveTask:{}", taskId); heartBeatThread.stopAtOnce(); - lockKeepThreads.remove(tableName); + lockKeepThreads.remove(taskId); } - public synchronized void unlock(String tableName, String lockRequestId) { + public synchronized void unlock(String taskId, String lockRequestId) { // 先停止分布式锁维持线程 - stopRedisKeyHeartBeatThread(tableName); + stopRedisKeyHeartBeatThread(taskId); - String archiveLockKey = ARCHIVE_LOCK_KEY_PREFIX + "_" + tableName; + String archiveLockKey = ARCHIVE_LOCK_KEY_PREFIX + "_" + taskId; LockUtils.releaseDistributedLock(archiveLockKey, lockRequestId); - this.locks.remove(tableName); + this.locks.remove(taskId); } - public synchronized void unlock(String tableName) { - String lockRequestId = this.locks.get(tableName); + public synchronized void unlock(String taskId) { + String lockRequestId = this.locks.get(taskId); if (StringUtils.isNotEmpty(lockRequestId)) { - unlock(tableName, lockRequestId); + unlock(taskId, lockRequestId); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskPrioritySorter.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskPrioritySorter.java new file mode 100644 index 0000000000..b3c349d802 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskPrioritySorter.java @@ -0,0 +1,125 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class ArchiveTaskPrioritySorter { + + public static void sort(List runningTasks, + Integer dbNodeCount, + List candidateTasks) { + if (CollectionUtils.isEmpty(candidateTasks)) { + return; + } + List priorityDbIndexList = + computeDbPriorityByRunningTasks(runningTasks, dbNodeCount); + sortArchiveTask(candidateTasks, priorityDbIndexList); + } + + private static void sortArchiveTask(List candidateTasks, + List priorityDbIndexList) { + candidateTasks.sort((task1, task2) -> { + int task1DbPriority = priorityDbIndexList.indexOf(task1.getDbDataNode().getDbIndex()); + int task2DbPriority = priorityDbIndexList.indexOf(task2.getDbDataNode().getDbIndex()); + if (task1DbPriority < task2DbPriority) { + return -1; + } else if (task1DbPriority > task2DbPriority) { + return 1; + } else { + int day1 = task1.getDay(); + int day2 = task2.getDay(); + if (day1 < day2) { + return -1; + } else if (day1 > day2) { + return 1; + } else { + int hour1 = task1.getHour(); + int hour2 = task2.getHour(); + return Integer.compare(hour1, hour2); + } + } + }); + } + + /** + * 根据正在运行中的任务在 DB 实例的分布情况，计算下一个归档任务对应 DB 的优先级 + * + * @param runningTasks 正在运行中的归档任务 + * @param dbNodeCount 归档 DB 实例数量 + * @return 优先级队列。列表的值为 dbIndex, 按照优先级从高->低排序 + */ + private static List computeDbPriorityByRunningTasks(List runningTasks, + int dbNodeCount) { + Map dbTaskCountMap = computeDbTaskCount(runningTasks, dbNodeCount); + + return dbTaskCountMap.entrySet() + .stream() + .sorted(Map.Entry.comparingByValue()) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + } + + + /** + * 计算归档任务在 db 实例的分布情况 + * + * @param runningTasks 正在运行中的归档任务 + * @param dbNodeCount 归档 DB 实例数量 + * @return 任务分布情况。 key-db序号;value-正在运行中的归档任务数量 + */ + private static Map computeDbTaskCount(List runningTasks, + int dbNodeCount) { + Map dbTaskCountMap = new HashMap<>(); + if (CollectionUtils.isEmpty(runningTasks)) { + return dbTaskCountMap; + } + + runningTasks.forEach(task -> { + Integer dbIndex = task.getDbDataNode().getDbIndex(); + dbTaskCountMap.compute(dbIndex, (k, v) -> { + if (v == null) { + v = 1; + } else { + v++; + } + return v; + }); + }); + + for (int dbIndex = 0; dbIndex < dbNodeCount; dbIndex++) { + dbTaskCountMap.putIfAbsent(dbIndex, 0); + } + return dbTaskCountMap; + } + + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskService.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskService.java new file mode 100644 index 0000000000..b541428b03 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskService.java @@ -0,0 +1,67 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.dao.ArchiveTaskDAO; +import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; +import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +public class ArchiveTaskService { + + private final ArchiveTaskDAO archiveTaskDAO; + + public ArchiveTaskService(ArchiveTaskDAO archiveTaskDAO) { + this.archiveTaskDAO = archiveTaskDAO; + } + + /** + * 获取最新的归档任务 + * + * @param taskType 归档任务类型 + */ + public JobInstanceArchiveTaskInfo getLatestArchiveTask(ArchiveTaskTypeEnum taskType) { + return archiveTaskDAO.getLatestArchiveTask(taskType); + } + + public void saveArchiveTask(JobInstanceArchiveTaskInfo jobInstanceArchiveTaskInfo) { + archiveTaskDAO.saveArchiveTask(jobInstanceArchiveTaskInfo); + } + + public List listRunningTasks(ArchiveTaskTypeEnum taskType) { + return archiveTaskDAO.listRunningTasks(taskType); + } + + public List listScheduleTasks(ArchiveTaskTypeEnum taskType, int limit) { + return archiveTaskDAO.listScheduleTasks(taskType, limit); + } + + public void updateTask(JobInstanceArchiveTaskInfo archiveTask) { + archiveTaskDAO.updateTask(archiveTask); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskWorker.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskWorker.java new file mode 100644 index 0000000000..903309f53b --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskWorker.java @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ArchiveTaskWorker extends Thread { + + private JobInstanceArchiveTask archiveTask; + + public ArchiveTaskWorker(JobInstanceArchiveTask archiveTask) { + this.setName("ArchiveWorker"); + this.archiveTask = archiveTask; + } + + @Override + public void run() { + try { + log.info("Archive task begin"); + archiveTask.execute(); + log.info("Archive task finished"); + } catch (Throwable e) { + log.warn("Thread interrupted!"); + } + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobExecuteArchiveManage.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobExecuteArchiveManage.java index 738d205de4..ed1a7867d6 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobExecuteArchiveManage.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobExecuteArchiveManage.java @@ -24,549 +24,519 @@ package com.tencent.bk.job.backup.archive; -import com.tencent.bk.job.backup.archive.impl.FileSourceTaskLogArchivist; -import com.tencent.bk.job.backup.archive.impl.GseFileAgentTaskArchivist; -import com.tencent.bk.job.backup.archive.impl.GseFileExecuteObjTaskArchivist; -import com.tencent.bk.job.backup.archive.impl.GseScriptAgentTaskArchivist; -import com.tencent.bk.job.backup.archive.impl.GseScriptExecuteObjTaskArchivist; -import com.tencent.bk.job.backup.archive.impl.GseTaskArchivist; -import com.tencent.bk.job.backup.archive.impl.OperationLogArchivist; -import com.tencent.bk.job.backup.archive.impl.RollingConfigArchivist; -import com.tencent.bk.job.backup.archive.impl.StepInstanceArchivist; -import com.tencent.bk.job.backup.archive.impl.StepInstanceConfirmArchivist; -import com.tencent.bk.job.backup.archive.impl.StepInstanceFileArchivist; -import com.tencent.bk.job.backup.archive.impl.StepInstanceRollingTaskArchivist; -import com.tencent.bk.job.backup.archive.impl.StepInstanceScriptArchivist; -import com.tencent.bk.job.backup.archive.impl.StepInstanceVariableArchivist; -import com.tencent.bk.job.backup.archive.impl.TaskInstanceArchivist; -import com.tencent.bk.job.backup.archive.impl.TaskInstanceHostArchivist; -import com.tencent.bk.job.backup.archive.impl.TaskInstanceVariableArchivist; -import com.tencent.bk.job.backup.config.ArchiveDBProperties; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.ExecuteRecordDAO; -import com.tencent.bk.job.backup.dao.impl.FileSourceTaskLogRecordDAO; -import com.tencent.bk.job.backup.dao.impl.GseFileAgentTaskRecordDAO; -import com.tencent.bk.job.backup.dao.impl.GseFileExecuteObjTaskRecordDAO; -import com.tencent.bk.job.backup.dao.impl.GseScriptAgentTaskRecordDAO; -import com.tencent.bk.job.backup.dao.impl.GseScriptExecuteObjTaskRecordDAO; -import com.tencent.bk.job.backup.dao.impl.GseTaskRecordDAO; -import com.tencent.bk.job.backup.dao.impl.OperationLogRecordDAO; -import com.tencent.bk.job.backup.dao.impl.RollingConfigRecordDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceConfirmRecordDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceFileRecordDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceRecordDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceRollingTaskRecordDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceScriptRecordDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceVariableRecordDAO; -import com.tencent.bk.job.backup.dao.impl.TaskInstanceHostRecordDAO; -import com.tencent.bk.job.backup.dao.impl.TaskInstanceRecordDAO; -import com.tencent.bk.job.backup.dao.impl.TaskInstanceVariableRecordDAO; -import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; -import com.tencent.bk.job.backup.model.dto.ArchiveProgressDTO; -import com.tencent.bk.job.backup.service.ArchiveProgressService; import lombok.extern.slf4j.Slf4j; -import org.joda.time.DateTime; -import org.slf4j.helpers.MessageFormatter; import org.springframework.context.SmartLifecycle; -import org.springframework.scheduling.annotation.Scheduled; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; @Slf4j public class JobExecuteArchiveManage implements SmartLifecycle { +// +// private final ArchiveDBProperties archiveDBProperties; +// private final ExecutorService archiveExecutor; +// private final ArchiveProgressService archiveProgressService; +// private final TaskInstanceRecordDAO taskInstanceRecordDAO; +// private final StepInstanceRecordDAO stepInstanceRecordDAO; +// private final StepInstanceScriptRecordDAO stepInstanceScriptRecordDAO; +// private final StepInstanceFileRecordDAO stepInstanceFileRecordDAO; +// private final StepInstanceConfirmRecordDAO stepInstanceConfirmRecordDAO; +// private final StepInstanceVariableRecordDAO stepInstanceVariableRecordDAO; +// private final TaskInstanceVariableRecordDAO taskInstanceVariableRecordDAO; +// private final OperationLogRecordDAO operationLogRecordDAO; +// private final FileSourceTaskLogRecordDAO fileSourceTaskLogRecordDAO; +// private final GseTaskRecordDAO gseTaskRecordDAO; +// private final GseScriptAgentTaskRecordDAO gseScriptAgentTaskRecordDAO; +// private final GseFileAgentTaskRecordDAO gseFileAgentTaskRecordDAO; +// private final GseScriptExecuteObjTaskRecordDAO gseScriptExecuteObjTaskRecordDAO; +// private final GseFileExecuteObjTaskRecordDAO gseFileExecuteObjTaskRecordDAO; +// private final StepInstanceRollingTaskRecordDAO stepInstanceRollingTaskRecordDAO; +// private final RollingConfigRecordDAO rollingConfigRecordDAO; +// private final TaskInstanceHostRecordDAO taskInstanceHostRecordDAO; +// private final JobInstanceColdDAO jobInstanceColdDAO; +// private final ArchiveTaskLock archiveTaskLock; +// private final ArchiveErrorTaskCounter archiveErrorTaskCounter; +// +// +// /** +// * whether this component is currently running(Spring Lifecycle isRunning method) +// */ +// private volatile boolean running = false; +// +// public JobExecuteArchiveManage(TaskInstanceRecordDAO taskInstanceRecordDAO, +// StepInstanceRecordDAO stepInstanceRecordDAO, +// StepInstanceScriptRecordDAO stepInstanceScriptRecordDAO, +// StepInstanceFileRecordDAO stepInstanceFileRecordDAO, +// StepInstanceConfirmRecordDAO stepInstanceConfirmRecordDAO, +// StepInstanceVariableRecordDAO stepInstanceVariableRecordDAO, +// TaskInstanceVariableRecordDAO taskInstanceVariableRecordDAO, +// OperationLogRecordDAO operationLogRecordDAO, +// FileSourceTaskLogRecordDAO fileSourceTaskLogRecordDAO, +// GseTaskRecordDAO gseTaskRecordDAO, +// GseScriptAgentTaskRecordDAO gseScriptAgentTaskRecordDAO, +// GseFileAgentTaskRecordDAO gseFileAgentTaskRecordDAO, +// GseScriptExecuteObjTaskRecordDAO gseScriptExecuteObjTaskRecordDAO, +// GseFileExecuteObjTaskRecordDAO gseFileExecuteObjTaskRecordDAO, +// StepInstanceRollingTaskRecordDAO stepInstanceRollingTaskRecordDAO, +// RollingConfigRecordDAO rollingConfigRecordDAO, +// TaskInstanceHostRecordDAO taskInstanceHostRecordDAO, +// JobInstanceColdDAO jobInstanceColdDAO, +// ArchiveProgressService archiveProgressService, +// ArchiveDBProperties archiveDBProperties, +// ExecutorService archiveExecutor, +// ArchiveTaskLock archiveTaskLock, +// ArchiveErrorTaskCounter archiveErrorTaskCounter) { +// log.info("Init JobExecuteArchiveManage! archiveConfig: {}", archiveDBProperties); +// this.archiveDBProperties = archiveDBProperties; +// this.archiveProgressService = archiveProgressService; +// this.archiveExecutor = archiveExecutor; +// this.taskInstanceRecordDAO = taskInstanceRecordDAO; +// this.stepInstanceRecordDAO = stepInstanceRecordDAO; +// this.stepInstanceScriptRecordDAO = stepInstanceScriptRecordDAO; +// this.stepInstanceFileRecordDAO = stepInstanceFileRecordDAO; +// this.stepInstanceConfirmRecordDAO = stepInstanceConfirmRecordDAO; +// this.stepInstanceVariableRecordDAO = stepInstanceVariableRecordDAO; +// this.taskInstanceVariableRecordDAO = taskInstanceVariableRecordDAO; +// this.operationLogRecordDAO = operationLogRecordDAO; +// this.fileSourceTaskLogRecordDAO = fileSourceTaskLogRecordDAO; +// this.gseTaskRecordDAO = gseTaskRecordDAO; +// this.gseScriptAgentTaskRecordDAO = gseScriptAgentTaskRecordDAO; +// this.gseFileAgentTaskRecordDAO = gseFileAgentTaskRecordDAO; +// this.gseScriptExecuteObjTaskRecordDAO = gseScriptExecuteObjTaskRecordDAO; +// this.gseFileExecuteObjTaskRecordDAO = gseFileExecuteObjTaskRecordDAO; +// this.stepInstanceRollingTaskRecordDAO = stepInstanceRollingTaskRecordDAO; +// this.rollingConfigRecordDAO = rollingConfigRecordDAO; +// this.taskInstanceHostRecordDAO = taskInstanceHostRecordDAO; +// this.jobInstanceColdDAO = jobInstanceColdDAO; +// this.archiveTaskLock = archiveTaskLock; +// this.archiveErrorTaskCounter = archiveErrorTaskCounter; +// } +// +// @Scheduled(cron = "${job.backup.archive.execute.cron:0 0 4 * * *}") +// public void cronArchive() { +// archive(archiveDBProperties); +// } +// +// public void archive(ArchiveDBProperties archiveDBProperties) { +// try { +// new ArchiveThread(archiveDBProperties).start(); +// } catch (Throwable e) { +// log.error("Error while archive job_execute data", e); +// } +// } +// +// @Override +// public void start() { +// this.running = true; +// } +// +// @Override +// public void stop() { +// log.info("Stop JobExecuteArchiveManage!"); +// archiveTaskLock.unlockAll(); +// log.info("Release all archive locks when stop!"); +// this.running = false; +// } +// +// @Override +// public boolean isRunning() { +// return this.running; +// } +// +// +// class ArchiveThread extends Thread { +// private final ArchiveDBProperties archiveDBProperties; +// +// +// ArchiveThread(ArchiveDBProperties archiveDBProperties) { +// this.archiveDBProperties = archiveDBProperties; +// this.setName("Data-Archive-Thread"); +// } +// +// @Override +// public void run() { +//// try { +//// log.info("Job Execute archive task begin, archiveConfig: {}", archiveDBProperties); +//// if (archiveDBProperties.isEnabled()) { +//// doArchive(getEndTime(archiveDBProperties.getKeepDays())); +//// } else { +//// log.info("Archive tasks are disabled, skip archive"); +//// } +//// log.info("Job Execute archive task finished"); +//// } catch (InterruptedException e) { +//// log.warn("Thread interrupted!"); +//// } +// } +// +// private Long getEndTime(int archiveDays) { +// DateTime now = DateTime.now(); +// // 置为前一天天 24:00:00 +// long todayMaxMills = now.minusMillis(now.getMillisOfDay()).getMillis(); +// +// //减掉当前xx天后 +// long archiveMills = archiveDays * 24 * 3600 * 1000L; +// return todayMaxMills - archiveMills; +// } +// +// +// private void doArchive(Long endTime) throws InterruptedException { +// try { +// log.info("Start job execute archive before {}", endTime); +// +// long maxNeedArchiveTaskInstanceId = computeMaxNeedArchiveTaskInstanceId(endTime); +// long maxNeedArchiveStepInstanceId = +// computeMaxNeedArchiveStepInstanceId(maxNeedArchiveTaskInstanceId); +// +// log.info("Compute archive instance id range, maxNeedArchiveTaskInstanceId: {}, " + +// "maxNeedArchiveStepInstanceId: {}", maxNeedArchiveTaskInstanceId, +// maxNeedArchiveStepInstanceId); +// +// ArchiveSummaryHolder.getInstance().init(endTime); +// archive(maxNeedArchiveTaskInstanceId, maxNeedArchiveStepInstanceId); +// ArchiveSummaryHolder.getInstance().print(); +// +// log.info("Job execute archive before {} success", endTime); +// } catch (InterruptedException e) { +// throw e; +// } catch (Throwable e) { +// String msg = MessageFormatter.format( +// "Error while do archive!|{}", +// endTime +// ).getMessage(); +// log.error(msg, e); +// } +// } +// +// public long computeMaxNeedArchiveTaskInstanceId(Long endTime) { +// long lastArchivedId = getLastArchiveId(taskInstanceRecordDAO); +// long maxId = taskInstanceRecordDAO.getMaxId(endTime); +// return Math.max(lastArchivedId, maxId); +// } +// +// public long computeMaxNeedArchiveStepInstanceId(Long taskInstanceId) { +// long lastArchivedId = getLastArchiveId(stepInstanceRecordDAO); +// long maxId = stepInstanceRecordDAO.getMaxId(taskInstanceId); +// return Math.max(lastArchivedId, maxId); +// } +// +// private long getLastArchiveId(JobInstanceHotRecordDAO jobInstanceHotRecordDAO) { +// String tableName = jobInstanceHotRecordDAO.getTable().getName().toLowerCase(); +// ArchiveProgressDTO archiveProgress = +// archiveProgressService.queryArchiveProgress(tableName); +// return archiveProgress != null ? archiveProgress.getLastBackupId() : 0L; +// } +// +// private void archive(long maxNeedArchiveTaskInstanceId, long maxNeedArchiveStepInstanceId) +// throws InterruptedException { +// CountDownLatch countDownLatch = new CountDownLatch(17); +// log.info("Submitting archive task..."); +// +// // task_instance +// addTaskInstanceArchiveTask(maxNeedArchiveTaskInstanceId, countDownLatch); +// // step_instance +// addStepInstanceArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); +// // step_instance_confirm +// addStepInstanceConfirmArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); +// // step_instance_file +// addStepInstanceFileArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); +// // step_instance_script +// addStepInstanceScriptArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); +// // file_source_task_log +// addFileSourceTaskLogArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); +// // task_instance_variable +// addTaskInstanceVariableArchiveTask(maxNeedArchiveTaskInstanceId, countDownLatch); +// // step_instance_variable +// addStepInstanceVariableArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); +// // operation_log +// addOperationLogArchiveTask(maxNeedArchiveTaskInstanceId, countDownLatch); +// // gse_task +// addGseTaskArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); +// // gse_script_agent_task +// addGseScriptAgentTaskArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); +// // gse_file_agent_task +// addGseFileAgentTaskArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); +// // step_instance_rolling_task +// addStepInstanceRollingTaskArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); +// // rolling_config +// addRollingConfigArchiveTask(maxNeedArchiveTaskInstanceId, countDownLatch); +// // task_instance_host +// addTaskInstanceHostArchiveTask(maxNeedArchiveTaskInstanceId, countDownLatch); +// // gse_script_execute_obj_task +// addGseScriptExecuteObjTaskArchiveTask(maxNeedArchiveTaskInstanceId, countDownLatch); +// // gse_file_execute_obj_task +// addGseFileExecuteObjTaskArchiveTask(maxNeedArchiveTaskInstanceId, countDownLatch); +// +// log.info("Archive task submitted. Waiting for complete..."); +// countDownLatch.await(); +// +// log.info("Archive task execute completed."); +// } +// +// private void addTaskInstanceArchiveTask(Long maxNeedArchiveTaskInstanceId, +// CountDownLatch countDownLatch) { +// archiveExecutor.execute(() -> +// new TaskInstanceArchivist( +// taskInstanceRecordDAO, +// jobInstanceColdDAO, +// archiveProgressService, +// archiveDBProperties, +// archiveTaskLock, +// maxNeedArchiveTaskInstanceId, +// countDownLatch, +// archiveErrorTaskCounter +// ).archive()); +// } +// +// private void addStepInstanceArchiveTask(Long maxNeedArchiveStepInstanceId, +// CountDownLatch countDownLatch) { +// archiveExecutor.execute(() -> +// new StepInstanceArchivist( +// stepInstanceRecordDAO, +// jobInstanceColdDAO, +// archiveProgressService, +// archiveDBProperties, +// archiveTaskLock, +// maxNeedArchiveStepInstanceId, +// countDownLatch, +// archiveErrorTaskCounter) +// .archive()); +// } +// +// private void addStepInstanceConfirmArchiveTask(Long maxNeedArchiveStepInstanceId, +// CountDownLatch countDownLatch) { +// archiveExecutor.execute(() -> +// new StepInstanceConfirmArchivist( +// stepInstanceConfirmRecordDAO, +// jobInstanceColdDAO, +// archiveProgressService, +// archiveDBProperties, +// archiveTaskLock, +// maxNeedArchiveStepInstanceId, +// countDownLatch, +// archiveErrorTaskCounter) +// .archive()); +// } +// +// private void addStepInstanceFileArchiveTask(Long maxNeedArchiveStepInstanceId, +// CountDownLatch countDownLatch) { +// archiveExecutor.execute(() -> +// new StepInstanceFileArchivist( +// stepInstanceFileRecordDAO, +// jobInstanceColdDAO, +// archiveProgressService, +// archiveDBProperties, +// archiveTaskLock, +// maxNeedArchiveStepInstanceId, +// countDownLatch, +// archiveErrorTaskCounter) +// .archive()); +// } +// +// private void addStepInstanceScriptArchiveTask(Long maxNeedArchiveStepInstanceId, +// CountDownLatch countDownLatch) { +// archiveExecutor.execute(() -> +// new StepInstanceScriptArchivist( +// stepInstanceScriptRecordDAO, +// jobInstanceColdDAO, +// archiveProgressService, +// archiveDBProperties, +// archiveTaskLock, +// maxNeedArchiveStepInstanceId, +// countDownLatch, +// archiveErrorTaskCounter) +// .archive()); +// } +// +// private void addFileSourceTaskLogArchiveTask(Long maxNeedArchiveStepInstanceId, +// CountDownLatch countDownLatch) { +// archiveExecutor.execute(() -> +// new FileSourceTaskLogArchivist( +// fileSourceTaskLogRecordDAO, +// jobInstanceColdDAO, +// archiveProgressService, +// archiveDBProperties, +// archiveTaskLock, +// maxNeedArchiveStepInstanceId, +// countDownLatch, +// archiveErrorTaskCounter) +// .archive()); +// } +// +// private void addTaskInstanceVariableArchiveTask(Long maxNeedArchiveTaskInstanceId, +// CountDownLatch countDownLatch) { +// archiveExecutor.execute(() -> +// new TaskInstanceVariableArchivist( +// taskInstanceVariableRecordDAO, +// jobInstanceColdDAO, +// archiveProgressService, +// archiveDBProperties, +// archiveTaskLock, +// maxNeedArchiveTaskInstanceId, +// countDownLatch, +// archiveErrorTaskCounter +// ).archive()); +// } +// +// private void addStepInstanceVariableArchiveTask(Long maxNeedArchiveStepInstanceId, +// CountDownLatch countDownLatch) { +// archiveExecutor.execute(() -> +// new StepInstanceVariableArchivist( +// stepInstanceVariableRecordDAO, +// jobInstanceColdDAO, +// archiveProgressService, +// archiveDBProperties, +// archiveTaskLock, +// maxNeedArchiveStepInstanceId, +// countDownLatch, +// archiveErrorTaskCounter) +// .archive()); +// } +// +// private void addOperationLogArchiveTask(Long maxNeedArchiveTaskInstanceId, +// CountDownLatch countDownLatch) { +// archiveExecutor.execute(() -> +// new OperationLogArchivist( +// operationLogRecordDAO, +// jobInstanceColdDAO, +// archiveProgressService, +// archiveDBProperties, +// archiveTaskLock, +// maxNeedArchiveTaskInstanceId, +// countDownLatch, +// archiveErrorTaskCounter +// ).archive()); +// } +// +// private void addGseTaskArchiveTask(Long maxNeedArchiveStepInstanceId, CountDownLatch countDownLatch) { +// archiveExecutor.execute(() -> +// new GseTaskArchivist( +// gseTaskRecordDAO, +// jobInstanceColdDAO, +// archiveProgressService, +// archiveDBProperties, +// archiveTaskLock, +// maxNeedArchiveStepInstanceId, +// countDownLatch, +// archiveErrorTaskCounter) +// .archive()); +// } +// +// private void addGseScriptAgentTaskArchiveTask(Long maxNeedArchiveStepInstanceId, +// CountDownLatch countDownLatch) { +// archiveExecutor.execute(() -> +// new GseScriptAgentTaskArchivist( +// gseScriptAgentTaskRecordDAO, +// jobInstanceColdDAO, +// archiveProgressService, +// archiveDBProperties, +// archiveTaskLock, +// maxNeedArchiveStepInstanceId, +// countDownLatch, +// archiveErrorTaskCounter) +// .archive()); +// } +// +// private void addGseFileAgentTaskArchiveTask(Long maxNeedArchiveStepInstanceId, +// CountDownLatch countDownLatch) { +// archiveExecutor.execute(() -> +// new GseFileAgentTaskArchivist( +// gseFileAgentTaskRecordDAO, +// jobInstanceColdDAO, +// archiveProgressService, +// archiveDBProperties, +// archiveTaskLock, +// maxNeedArchiveStepInstanceId, +// countDownLatch, +// archiveErrorTaskCounter) +// .archive()); +// } +// +// private void addStepInstanceRollingTaskArchiveTask(Long maxNeedArchiveStepInstanceId, +// CountDownLatch countDownLatch) { +// archiveExecutor.execute(() -> +// new StepInstanceRollingTaskArchivist( +// stepInstanceRollingTaskRecordDAO, +// jobInstanceColdDAO, +// archiveProgressService, +// archiveDBProperties, +// archiveTaskLock, +// maxNeedArchiveStepInstanceId, +// countDownLatch, +// archiveErrorTaskCounter) +// .archive()); +// } +// +// private void addGseScriptExecuteObjTaskArchiveTask(Long maxNeedArchiveTaskInstanceId, +// CountDownLatch countDownLatch) { +// archiveExecutor.execute(() -> +// new GseScriptExecuteObjTaskArchivist( +// gseScriptExecuteObjTaskRecordDAO, +// jobInstanceColdDAO, +// archiveProgressService, +// archiveDBProperties, +// archiveTaskLock, +// maxNeedArchiveTaskInstanceId, +// countDownLatch, +// archiveErrorTaskCounter) +// .archive()); +// } +// +// private void addGseFileExecuteObjTaskArchiveTask(Long maxNeedArchiveTaskInstanceId, +// CountDownLatch countDownLatch) { +// archiveExecutor.execute(() -> +// new GseFileExecuteObjTaskArchivist( +// gseFileExecuteObjTaskRecordDAO, +// jobInstanceColdDAO, +// archiveProgressService, +// archiveDBProperties, +// archiveTaskLock, +// maxNeedArchiveTaskInstanceId, +// countDownLatch, +// archiveErrorTaskCounter) +// .archive()); +// } +// +// private void addRollingConfigArchiveTask(Long maxNeedArchiveTaskInstanceId, +// CountDownLatch countDownLatch) { +// archiveExecutor.execute(() -> +// new RollingConfigArchivist( +// rollingConfigRecordDAO, +// jobInstanceColdDAO, +// archiveProgressService, +// archiveDBProperties, +// archiveTaskLock, +// maxNeedArchiveTaskInstanceId, +// countDownLatch, +// archiveErrorTaskCounter +// ).archive()); +// } +// +// private void addTaskInstanceHostArchiveTask(Long maxNeedArchiveTaskInstanceId, +// CountDownLatch countDownLatch) { +// archiveExecutor.execute(() -> +// new TaskInstanceHostArchivist( +// taskInstanceHostRecordDAO, +// jobInstanceColdDAO, +// archiveProgressService, +// archiveDBProperties, +// archiveTaskLock, +// maxNeedArchiveTaskInstanceId, +// countDownLatch, +// archiveErrorTaskCounter +// ).archive()); +// } +// } - private final ArchiveDBProperties archiveDBProperties; - private final ExecutorService archiveExecutor; - private final ArchiveProgressService archiveProgressService; - private final TaskInstanceRecordDAO taskInstanceRecordDAO; - private final StepInstanceRecordDAO stepInstanceRecordDAO; - private final StepInstanceScriptRecordDAO stepInstanceScriptRecordDAO; - private final StepInstanceFileRecordDAO stepInstanceFileRecordDAO; - private final StepInstanceConfirmRecordDAO stepInstanceConfirmRecordDAO; - private final StepInstanceVariableRecordDAO stepInstanceVariableRecordDAO; - private final TaskInstanceVariableRecordDAO taskInstanceVariableRecordDAO; - private final OperationLogRecordDAO operationLogRecordDAO; - private final FileSourceTaskLogRecordDAO fileSourceTaskLogRecordDAO; - private final GseTaskRecordDAO gseTaskRecordDAO; - private final GseScriptAgentTaskRecordDAO gseScriptAgentTaskRecordDAO; - private final GseFileAgentTaskRecordDAO gseFileAgentTaskRecordDAO; - private final GseScriptExecuteObjTaskRecordDAO gseScriptExecuteObjTaskRecordDAO; - private final GseFileExecuteObjTaskRecordDAO gseFileExecuteObjTaskRecordDAO; - private final StepInstanceRollingTaskRecordDAO stepInstanceRollingTaskRecordDAO; - private final RollingConfigRecordDAO rollingConfigRecordDAO; - private final TaskInstanceHostRecordDAO taskInstanceHostRecordDAO; - private final ExecuteArchiveDAO executeArchiveDAO; - private final ArchiveTaskLock archiveTaskLock; - private final ArchiveErrorTaskCounter archiveErrorTaskCounter; - - - /** - * whether this component is currently running(Spring Lifecycle isRunning method) - */ - private volatile boolean running = false; - - public JobExecuteArchiveManage(TaskInstanceRecordDAO taskInstanceRecordDAO, - StepInstanceRecordDAO stepInstanceRecordDAO, - StepInstanceScriptRecordDAO stepInstanceScriptRecordDAO, - StepInstanceFileRecordDAO stepInstanceFileRecordDAO, - StepInstanceConfirmRecordDAO stepInstanceConfirmRecordDAO, - StepInstanceVariableRecordDAO stepInstanceVariableRecordDAO, - TaskInstanceVariableRecordDAO taskInstanceVariableRecordDAO, - OperationLogRecordDAO operationLogRecordDAO, - FileSourceTaskLogRecordDAO fileSourceTaskLogRecordDAO, - GseTaskRecordDAO gseTaskRecordDAO, - GseScriptAgentTaskRecordDAO gseScriptAgentTaskRecordDAO, - GseFileAgentTaskRecordDAO gseFileAgentTaskRecordDAO, - GseScriptExecuteObjTaskRecordDAO gseScriptExecuteObjTaskRecordDAO, - GseFileExecuteObjTaskRecordDAO gseFileExecuteObjTaskRecordDAO, - StepInstanceRollingTaskRecordDAO stepInstanceRollingTaskRecordDAO, - RollingConfigRecordDAO rollingConfigRecordDAO, - TaskInstanceHostRecordDAO taskInstanceHostRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService, - ArchiveDBProperties archiveDBProperties, - ExecutorService archiveExecutor, - ArchiveTaskLock archiveTaskLock, - ArchiveErrorTaskCounter archiveErrorTaskCounter) { - log.info("Init JobExecuteArchiveManage! archiveConfig: {}", archiveDBProperties); - this.archiveDBProperties = archiveDBProperties; - this.archiveProgressService = archiveProgressService; - this.archiveExecutor = archiveExecutor; - this.taskInstanceRecordDAO = taskInstanceRecordDAO; - this.stepInstanceRecordDAO = stepInstanceRecordDAO; - this.stepInstanceScriptRecordDAO = stepInstanceScriptRecordDAO; - this.stepInstanceFileRecordDAO = stepInstanceFileRecordDAO; - this.stepInstanceConfirmRecordDAO = stepInstanceConfirmRecordDAO; - this.stepInstanceVariableRecordDAO = stepInstanceVariableRecordDAO; - this.taskInstanceVariableRecordDAO = taskInstanceVariableRecordDAO; - this.operationLogRecordDAO = operationLogRecordDAO; - this.fileSourceTaskLogRecordDAO = fileSourceTaskLogRecordDAO; - this.gseTaskRecordDAO = gseTaskRecordDAO; - this.gseScriptAgentTaskRecordDAO = gseScriptAgentTaskRecordDAO; - this.gseFileAgentTaskRecordDAO = gseFileAgentTaskRecordDAO; - this.gseScriptExecuteObjTaskRecordDAO = gseScriptExecuteObjTaskRecordDAO; - this.gseFileExecuteObjTaskRecordDAO = gseFileExecuteObjTaskRecordDAO; - this.stepInstanceRollingTaskRecordDAO = stepInstanceRollingTaskRecordDAO; - this.rollingConfigRecordDAO = rollingConfigRecordDAO; - this.taskInstanceHostRecordDAO = taskInstanceHostRecordDAO; - this.executeArchiveDAO = executeArchiveDAO; - this.archiveTaskLock = archiveTaskLock; - this.archiveErrorTaskCounter = archiveErrorTaskCounter; - } - - @Scheduled(cron = "${job.backup.archive.execute.cron:0 0 4 * * *}") - public void cronArchive() { - archive(archiveDBProperties); - } - - public void archive(ArchiveDBProperties archiveDBProperties) { - try { - new ArchiveThread(archiveDBProperties).start(); - } catch (Throwable e) { - log.error("Error while archive job_execute data", e); - } - } @Override public void start() { - this.running = true; + } @Override public void stop() { - log.info("Stop JobExecuteArchiveManage!"); - archiveTaskLock.unlockAll(); - log.info("Release all archive locks when stop!"); - this.running = false; + } @Override public boolean isRunning() { - return this.running; - } - - - class ArchiveThread extends Thread { - private final ArchiveDBProperties archiveDBProperties; - - - ArchiveThread(ArchiveDBProperties archiveDBProperties) { - this.archiveDBProperties = archiveDBProperties; - this.setName("Data-Archive-Thread"); - } - - @Override - public void run() { - try { - log.info("Job Execute archive task begin, archiveConfig: {}", archiveDBProperties); - if (archiveDBProperties.isEnabled()) { - doArchive(getEndTime(archiveDBProperties.getKeepDays())); - } else { - log.info("Archive tasks are disabled, skip archive"); - } - log.info("Job Execute archive task finished"); - } catch (InterruptedException e) { - log.warn("Thread interrupted!"); - } - } - - private Long getEndTime(int archiveDays) { - DateTime now = DateTime.now(); - // 置为前一天天 24:00:00 - long todayMaxMills = now.minusMillis(now.getMillisOfDay()).getMillis(); - - //减掉当前xx天后 - long archiveMills = archiveDays * 24 * 3600 * 1000L; - return todayMaxMills - archiveMills; - } - - - private void doArchive(Long endTime) throws InterruptedException { - try { - log.info("Start job execute archive before {}", endTime); - - long maxNeedArchiveTaskInstanceId = computeMaxNeedArchiveTaskInstanceId(endTime); - long maxNeedArchiveStepInstanceId = - computeMaxNeedArchiveStepInstanceId(maxNeedArchiveTaskInstanceId); - - log.info("Compute archive instance id range, maxNeedArchiveTaskInstanceId: {}, " + - "maxNeedArchiveStepInstanceId: {}", maxNeedArchiveTaskInstanceId, - maxNeedArchiveStepInstanceId); - - ArchiveSummaryHolder.getInstance().init(endTime); - archive(maxNeedArchiveTaskInstanceId, maxNeedArchiveStepInstanceId); - ArchiveSummaryHolder.getInstance().print(); - - log.info("Job execute archive before {} success", endTime); - } catch (InterruptedException e) { - throw e; - } catch (Throwable e) { - String msg = MessageFormatter.format( - "Error while do archive!|{}", - endTime - ).getMessage(); - log.error(msg, e); - } - } - - public long computeMaxNeedArchiveTaskInstanceId(Long endTime) { - long lastArchivedId = getLastArchiveId(taskInstanceRecordDAO); - long maxId = taskInstanceRecordDAO.getMaxId(endTime); - return Math.max(lastArchivedId, maxId); - } - - public long computeMaxNeedArchiveStepInstanceId(Long taskInstanceId) { - long lastArchivedId = getLastArchiveId(stepInstanceRecordDAO); - long maxId = stepInstanceRecordDAO.getMaxId(taskInstanceId); - return Math.max(lastArchivedId, maxId); - } - - private long getLastArchiveId(ExecuteRecordDAO executeRecordDAO) { - String tableName = executeRecordDAO.getTable().getName().toLowerCase(); - ArchiveProgressDTO archiveProgress = - archiveProgressService.queryArchiveProgress(tableName); - return archiveProgress != null ? archiveProgress.getLastBackupId() : 0L; - } - - private void archive(long maxNeedArchiveTaskInstanceId, long maxNeedArchiveStepInstanceId) - throws InterruptedException { - CountDownLatch countDownLatch = new CountDownLatch(17); - log.info("Submitting archive task..."); - - // task_instance - addTaskInstanceArchiveTask(maxNeedArchiveTaskInstanceId, countDownLatch); - // step_instance - addStepInstanceArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); - // step_instance_confirm - addStepInstanceConfirmArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); - // step_instance_file - addStepInstanceFileArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); - // step_instance_script - addStepInstanceScriptArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); - // file_source_task_log - addFileSourceTaskLogArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); - // task_instance_variable - addTaskInstanceVariableArchiveTask(maxNeedArchiveTaskInstanceId, countDownLatch); - // step_instance_variable - addStepInstanceVariableArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); - // operation_log - addOperationLogArchiveTask(maxNeedArchiveTaskInstanceId, countDownLatch); - // gse_task - addGseTaskArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); - // gse_script_agent_task - addGseScriptAgentTaskArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); - // gse_file_agent_task - addGseFileAgentTaskArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); - // step_instance_rolling_task - addStepInstanceRollingTaskArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); - // rolling_config - addRollingConfigArchiveTask(maxNeedArchiveTaskInstanceId, countDownLatch); - // task_instance_host - addTaskInstanceHostArchiveTask(maxNeedArchiveTaskInstanceId, countDownLatch); - // gse_script_execute_obj_task - addGseScriptExecuteObjTaskArchiveTask(maxNeedArchiveTaskInstanceId, countDownLatch); - // gse_file_execute_obj_task - addGseFileExecuteObjTaskArchiveTask(maxNeedArchiveTaskInstanceId, countDownLatch); - - log.info("Archive task submitted. Waiting for complete..."); - countDownLatch.await(); - - log.info("Archive task execute completed."); - } - - private void addTaskInstanceArchiveTask(Long maxNeedArchiveTaskInstanceId, - CountDownLatch countDownLatch) { - archiveExecutor.execute(() -> - new TaskInstanceArchivist( - taskInstanceRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveTaskInstanceId, - countDownLatch, - archiveErrorTaskCounter - ).archive()); - } - - private void addStepInstanceArchiveTask(Long maxNeedArchiveStepInstanceId, - CountDownLatch countDownLatch) { - archiveExecutor.execute(() -> - new StepInstanceArchivist( - stepInstanceRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveStepInstanceId, - countDownLatch, - archiveErrorTaskCounter) - .archive()); - } - - private void addStepInstanceConfirmArchiveTask(Long maxNeedArchiveStepInstanceId, - CountDownLatch countDownLatch) { - archiveExecutor.execute(() -> - new StepInstanceConfirmArchivist( - stepInstanceConfirmRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveStepInstanceId, - countDownLatch, - archiveErrorTaskCounter) - .archive()); - } - - private void addStepInstanceFileArchiveTask(Long maxNeedArchiveStepInstanceId, - CountDownLatch countDownLatch) { - archiveExecutor.execute(() -> - new StepInstanceFileArchivist( - stepInstanceFileRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveStepInstanceId, - countDownLatch, - archiveErrorTaskCounter) - .archive()); - } - - private void addStepInstanceScriptArchiveTask(Long maxNeedArchiveStepInstanceId, - CountDownLatch countDownLatch) { - archiveExecutor.execute(() -> - new StepInstanceScriptArchivist( - stepInstanceScriptRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveStepInstanceId, - countDownLatch, - archiveErrorTaskCounter) - .archive()); - } - - private void addFileSourceTaskLogArchiveTask(Long maxNeedArchiveStepInstanceId, - CountDownLatch countDownLatch) { - archiveExecutor.execute(() -> - new FileSourceTaskLogArchivist( - fileSourceTaskLogRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveStepInstanceId, - countDownLatch, - archiveErrorTaskCounter) - .archive()); - } - - private void addTaskInstanceVariableArchiveTask(Long maxNeedArchiveTaskInstanceId, - CountDownLatch countDownLatch) { - archiveExecutor.execute(() -> - new TaskInstanceVariableArchivist( - taskInstanceVariableRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveTaskInstanceId, - countDownLatch, - archiveErrorTaskCounter - ).archive()); - } - - private void addStepInstanceVariableArchiveTask(Long maxNeedArchiveStepInstanceId, - CountDownLatch countDownLatch) { - archiveExecutor.execute(() -> - new StepInstanceVariableArchivist( - stepInstanceVariableRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveStepInstanceId, - countDownLatch, - archiveErrorTaskCounter) - .archive()); - } - - private void addOperationLogArchiveTask(Long maxNeedArchiveTaskInstanceId, - CountDownLatch countDownLatch) { - archiveExecutor.execute(() -> - new OperationLogArchivist( - operationLogRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveTaskInstanceId, - countDownLatch, - archiveErrorTaskCounter - ).archive()); - } - - private void addGseTaskArchiveTask(Long maxNeedArchiveStepInstanceId, CountDownLatch countDownLatch) { - archiveExecutor.execute(() -> - new GseTaskArchivist( - gseTaskRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveStepInstanceId, - countDownLatch, - archiveErrorTaskCounter) - .archive()); - } - - private void addGseScriptAgentTaskArchiveTask(Long maxNeedArchiveStepInstanceId, - CountDownLatch countDownLatch) { - archiveExecutor.execute(() -> - new GseScriptAgentTaskArchivist( - gseScriptAgentTaskRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveStepInstanceId, - countDownLatch, - archiveErrorTaskCounter) - .archive()); - } - - private void addGseFileAgentTaskArchiveTask(Long maxNeedArchiveStepInstanceId, - CountDownLatch countDownLatch) { - archiveExecutor.execute(() -> - new GseFileAgentTaskArchivist( - gseFileAgentTaskRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveStepInstanceId, - countDownLatch, - archiveErrorTaskCounter) - .archive()); - } - - private void addStepInstanceRollingTaskArchiveTask(Long maxNeedArchiveStepInstanceId, - CountDownLatch countDownLatch) { - archiveExecutor.execute(() -> - new StepInstanceRollingTaskArchivist( - stepInstanceRollingTaskRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveStepInstanceId, - countDownLatch, - archiveErrorTaskCounter) - .archive()); - } - - private void addGseScriptExecuteObjTaskArchiveTask(Long maxNeedArchiveTaskInstanceId, - CountDownLatch countDownLatch) { - archiveExecutor.execute(() -> - new GseScriptExecuteObjTaskArchivist( - gseScriptExecuteObjTaskRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveTaskInstanceId, - countDownLatch, - archiveErrorTaskCounter) - .archive()); - } - - private void addGseFileExecuteObjTaskArchiveTask(Long maxNeedArchiveTaskInstanceId, - CountDownLatch countDownLatch) { - archiveExecutor.execute(() -> - new GseFileExecuteObjTaskArchivist( - gseFileExecuteObjTaskRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveTaskInstanceId, - countDownLatch, - archiveErrorTaskCounter) - .archive()); - } - - private void addRollingConfigArchiveTask(Long maxNeedArchiveTaskInstanceId, - CountDownLatch countDownLatch) { - archiveExecutor.execute(() -> - new RollingConfigArchivist( - rollingConfigRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveTaskInstanceId, - countDownLatch, - archiveErrorTaskCounter - ).archive()); - } - - private void addTaskInstanceHostArchiveTask(Long maxNeedArchiveTaskInstanceId, - CountDownLatch countDownLatch) { - archiveExecutor.execute(() -> - new TaskInstanceHostArchivist( - taskInstanceHostRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveTaskInstanceId, - countDownLatch, - archiveErrorTaskCounter - ).archive()); - } + return false; } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceAppDataArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceAppDataArchiveTask.java new file mode 100644 index 0000000000..7f6bae488c --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceAppDataArchiveTask.java @@ -0,0 +1,84 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.JobInstanceHotRecordDAO; +import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; +import com.tencent.bk.job.backup.config.ArchiveProperties; +import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; +import com.tencent.bk.job.execute.model.tables.records.TaskInstanceRecord; + +import java.util.List; + +/** + * 作业实例数据归档-分库分表场景下，冗余的基于业务 ID 进行分库分表的数据 + */ +public class JobInstanceAppDataArchiveTask extends AbstractJobInstanceArchiveTask { + + public JobInstanceAppDataArchiveTask(JobInstanceHotRecordDAO jobInstanceHotRecordDAO, + JobInstanceColdDAO jobInstanceColdDAO, + ArchiveProperties archiveDbProperties, + ArchiveTaskLock archiveTaskLock, + ArchiveErrorTaskCounter archiveErrorTaskCounter, + JobInstanceArchiveTaskInfo archiveTask, + ArchiveTaskService archiveTaskService) { + super(jobInstanceHotRecordDAO, + jobInstanceColdDAO, + archiveDbProperties, + archiveTaskLock, + archiveErrorTaskCounter, + archiveTask, + archiveTaskService); + } + + @Override + protected void backupJobInstanceToColdDb(List jobInstances, List jobInstanceIds) { + + } + + @Override + protected void deleteJobInstanceHotData(List jobInstanceIds) { + + } + + @Override + protected List readSortedJobInstanceFromHotDB(Long fromTimestamp, + Long endTimestamp, + Long fromJobInstanceId, + int limit) { + return null; + } + + @Override + protected Long extractJobInstanceId(TaskInstanceRecord record) { + return null; + } + + @Override + protected Long extractJobInstanceCreateTime(TaskInstanceRecord record) { + return null; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveCronJobs.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveCronJobs.java new file mode 100644 index 0000000000..bbcae195c4 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveCronJobs.java @@ -0,0 +1,69 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +/** + * 归档定时任务 + */ +@Component +@EnableScheduling +@Slf4j +public class JobInstanceArchiveCronJobs { + + private final JobInstanceArchiveTaskGenerator jobInstanceArchiveTaskGenerator; + + private final JobInstanceArchiveTaskScheduler jobInstanceArchiveTaskScheduler; + + public JobInstanceArchiveCronJobs(JobInstanceArchiveTaskGenerator jobInstanceArchiveTaskGenerator, + JobInstanceArchiveTaskScheduler jobInstanceArchiveTaskScheduler) { + this.jobInstanceArchiveTaskGenerator = jobInstanceArchiveTaskGenerator; + this.jobInstanceArchiveTaskScheduler = jobInstanceArchiveTaskScheduler; + } + + /** + * 定时创建归档任务 + */ + @Scheduled(cron = "${job.backup.archive.execute.cron:0,6,12,18 0 0 * * *}") + public void generateArchiveTask() { + log.info("Generate archive task start..."); + jobInstanceArchiveTaskGenerator.generate(); + log.info("Generate archive task done"); + } + + /** + * 定时调度并执行归档任务 + */ + @Scheduled(cron = "${job.backup.archive.execute.cron:1,7,13,19 0 0 * * *}") + public void scheduleAndExecuteArchiveTask() { + log.info("Schedule and execute archive task start..."); + jobInstanceArchiveTaskScheduler.schedule(); + log.info("Schedule and execute archive task done"); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTask.java new file mode 100644 index 0000000000..a2a8973fd7 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTask.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +/** + * 作业执行实例归档任务 + */ +public interface JobInstanceArchiveTask { + + /** + * 执行任务 + */ + void execute(); + + + /** + * 归档任务被终止时候调用 + */ + void stop(); + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java new file mode 100644 index 0000000000..6413940a4f --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java @@ -0,0 +1,187 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceRecordDAO; +import com.tencent.bk.job.backup.archive.model.DbDataNode; +import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; +import com.tencent.bk.job.backup.config.ArchiveProperties; +import com.tencent.bk.job.backup.constant.ArchiveTaskStatusEnum; +import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; +import com.tencent.bk.job.backup.constant.DbDataNodeTypeEnum; +import com.tencent.bk.job.common.mysql.JobTransactional; +import com.tencent.bk.job.common.mysql.dynamic.ds.DataSourceMode; +import com.tencent.bk.job.common.util.date.DateUtils; +import com.tencent.bk.job.common.util.json.JsonUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.joda.time.DateTime; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; + +/** + * 生成作业实例归档任务 + */ +@Slf4j +public class JobInstanceArchiveTaskGenerator { + + private final ArchiveTaskService archiveTaskService; + + private final TaskInstanceRecordDAO taskInstanceRecordDAO; + + private final ArchiveProperties archiveProperties; + + + public JobInstanceArchiveTaskGenerator(ArchiveTaskService archiveTaskService, + TaskInstanceRecordDAO taskInstanceRecordDAO, + ArchiveProperties archiveProperties) { + + this.archiveTaskService = archiveTaskService; + this.taskInstanceRecordDAO = taskInstanceRecordDAO; + this.archiveProperties = archiveProperties; + } + + @JobTransactional(transactionManager = "jobBackupTransactionManager") + public void generate() { + List archiveTaskList = new ArrayList<>(); + + LocalDateTime startDateTime = computeArchiveStartDateTime(); + LocalDateTime endDateTime = unixTimestampToUtcLocalDateTime( + getEndTime(archiveProperties.getKeepDays())); + while (startDateTime.isBefore(endDateTime)) { + if (isHorizontalShardingEnabled()) { + // 作业实例数据归档任务 + archiveTaskList.addAll(buildArchiveTasksForShardingDataNodes(ArchiveTaskTypeEnum.JOB_INSTANCE, + startDateTime, archiveProperties.getTasks().getTaskInstance().getShardingDataNodes())); + // 作业实例按业务冗余数据归档 + archiveTaskList.addAll(buildArchiveTasksForShardingDataNodes(ArchiveTaskTypeEnum.JOB_INSTANCE_APP, + startDateTime, archiveProperties.getTasks().getTaskInstance().getShardingDataNodes())); + } else { + DbDataNode dbDataNode = new DbDataNode(DbDataNodeTypeEnum.STANDALONE, null, null, null); + archiveTaskList.add(buildArchiveTask(ArchiveTaskTypeEnum.JOB_INSTANCE, startDateTime, dbDataNode)); + } + + startDateTime = startDateTime.plusHours(1L); + } + + if (CollectionUtils.isNotEmpty(archiveTaskList)) { + archiveTaskList.forEach(archiveTaskService::saveArchiveTask); + log.info("Add archive tasks : {}", JsonUtils.toJson(archiveTaskList)); + } + } + + private List buildArchiveTasksForShardingDataNodes( + ArchiveTaskTypeEnum archiveTaskType, + LocalDateTime startDateTime, + List shardingDataNodes) { + List tasks = new ArrayList<>(); + + shardingDataNodes.forEach(dataNode -> { + int dbNodeCount = dataNode.getDbCount(); + int tableNodeCount = dataNode.getTableCount(); + String dataSource = dataNode.getDataSourceName(); + for (int dbNodeIndex = 0; dbNodeIndex < dbNodeCount; dbNodeIndex++) { + for (int tableNodeIndex = 0; tableNodeIndex < tableNodeCount; tableNodeIndex++) { + DbDataNode dbDataNode = new DbDataNode(DbDataNodeTypeEnum.SHARDING, dataSource, + dbNodeIndex, tableNodeIndex); + // 作业实例数据归档任务 + tasks.add(buildArchiveTask(archiveTaskType, startDateTime, dbDataNode)); + } + } + }); + + return tasks; + } + + private JobInstanceArchiveTaskInfo buildArchiveTask(ArchiveTaskTypeEnum archiveTaskType, + LocalDateTime startDateTime, + DbDataNode dbDataNode) { + JobInstanceArchiveTaskInfo archiveTask = new JobInstanceArchiveTaskInfo(); + int day = computeDay(startDateTime); + int hour = computeHour(startDateTime); + archiveTask.setDay(day); + archiveTask.setHour(hour); + archiveTask.setFromTimestamp(1000 * startDateTime.toEpochSecond(ZoneOffset.UTC)); + archiveTask.setToTimestamp(1000L * startDateTime.plusHours(1L).toEpochSecond(ZoneOffset.UTC)); + archiveTask.setTaskType(archiveTaskType); + archiveTask.setDbDataNode(dbDataNode); + archiveTask.setStatus(ArchiveTaskStatusEnum.PENDING); + return archiveTask; + } + + + private LocalDateTime computeArchiveStartDateTime() { + LocalDateTime startDateTime; + JobInstanceArchiveTaskInfo latestArchiveTask = + archiveTaskService.getLatestArchiveTask(ArchiveTaskTypeEnum.JOB_INSTANCE); + if (latestArchiveTask == null) { + log.info("Latest archive task is empty, try compute from task_instance table record"); + Long minJobCreateTime = taskInstanceRecordDAO.getMinJobCreateTime(); + startDateTime = toHourlyRoundDown(unixTimestampToUtcLocalDateTime(minJobCreateTime)); + } else { + startDateTime = unixTimestampToUtcLocalDateTime(latestArchiveTask.getToTimestamp()); + } + + return startDateTime; + } + + private boolean isHorizontalShardingEnabled() { + return archiveProperties.getTasks().getTaskInstance().getDataSourceMode() + .equals(DataSourceMode.Constants.HORIZONTAL_SHARDING); + } + + private int computeDay(LocalDateTime dateTime) { + return Integer.parseInt(DateUtils.formatLocalDateTime(dateTime, "%Y%m%d")); + } + + private int computeHour(LocalDateTime dateTime) { + return dateTime.getHour(); + } + + private Long getEndTime(int archiveDays) { + DateTime now = DateTime.now(); + // 置为前一天天 24:00:00 + long todayMaxMills = now.minusMillis(now.getMillisOfDay()).getMillis(); + + //减掉当前xx天后 + long archiveMills = archiveDays * 24 * 3600 * 1000L; + return todayMaxMills - archiveMills; + } + + private LocalDateTime unixTimestampToUtcLocalDateTime(long unixTimestamp) { + // 创建一个 Instant 对象，表示从 1970-01-01T00:00:00Z 开始的指定秒数 + Instant instant = Instant.ofEpochSecond(unixTimestamp); + // 将 Instant 对象转换为 UTC 时区的 LocalDateTime 对象 + return LocalDateTime.ofInstant(instant, ZoneOffset.UTC); + } + + private LocalDateTime toHourlyRoundDown(LocalDateTime localDateTime) { + return localDateTime.withMinute(0).withSecond(0).withNano(0); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduleLock.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduleLock.java new file mode 100644 index 0000000000..e1aa2d41e7 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduleLock.java @@ -0,0 +1,78 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.common.redis.util.LockUtils; +import com.tencent.bk.job.common.util.ThreadUtils; +import lombok.extern.slf4j.Slf4j; + +import java.util.UUID; + +/** + * 归档任务调度分布式锁 + */ +@Slf4j +public class JobInstanceArchiveTaskScheduleLock { + private final String LOCK_KEY = "job:instance:archive:task:schedule"; + /** + * 锁时间（60s) + */ + private final Long LOCK_TIME = 60 * 1000L; + /** + * 最小获取锁间隔时间 + */ + private final long MIN_ACQUIRE_LOCK_INTERVAL_MS = 1000L; + + private volatile long lastAcquireLockTimeMS = 0L; + + private volatile String lockRequestId = null; + + public JobInstanceArchiveTaskScheduleLock() { + } + + public synchronized boolean lock() { + while (System.currentTimeMillis() - lastAcquireLockTimeMS < MIN_ACQUIRE_LOCK_INTERVAL_MS) { + // 为了保证在分布式系统中多个节点都能均匀获取到任务，需要最小获取锁间隔时间，让其他服务节点优先获取任务锁 + ThreadUtils.sleep(100L); + } + String lockRequestId = UUID.randomUUID().toString(); + if (!LockUtils.tryGetDistributedLock(LOCK_KEY, lockRequestId, LOCK_TIME)) { + log.info("Acquire job instance archive task schedule lock failed!"); + return false; + } else { + log.info("Acquire job instance archive task schedule lock successfully!"); + this.lastAcquireLockTimeMS = System.currentTimeMillis(); + this.lockRequestId = lockRequestId; + return true; + } + } + + + public synchronized void unlock() { + LockUtils.releaseDistributedLock(LOCK_KEY, lockRequestId); + this.lockRequestId = null; + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java new file mode 100644 index 0000000000..f4aab44821 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java @@ -0,0 +1,219 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceRecordDAO; +import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; +import com.tencent.bk.job.backup.config.ArchiveProperties; +import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; +import com.tencent.bk.job.common.sharding.mysql.config.ShardingProperties; +import com.tencent.bk.job.common.util.ThreadUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.context.SmartLifecycle; +import org.springframework.core.task.SimpleAsyncTaskExecutor; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * 作业执行历史归档任务调度 + */ +@Slf4j +public class JobInstanceArchiveTaskScheduler extends SmartLifecycle { + + private final ArchiveTaskTypeEnum archiveTaskType = ArchiveTaskTypeEnum.JOB_INSTANCE; + + private final ArchiveTaskService archiveTaskService; + + private final TaskInstanceRecordDAO taskInstanceRecordDAO; + + private final ArchiveProperties archiveProperties; + + private final ShardingProperties shardingProperties; + + private final JobInstanceArchiveTaskScheduleLock jobInstanceArchiveTaskScheduleLock; + + private final Set workers = new HashSet<>(); + + /** + * 异步归档任务执行器 + */ + private final Executor taskExecutor = new SimpleAsyncTaskExecutor("JobInstanceArchiveTaskExecutor"); + + private final AtomicInteger runningTasksCount = new AtomicInteger(0); + + private final Object lifecycleMonitor = new Object(); + + /** + * 作业执行历史归档任务调度组件是否处于活动状态 + */ + private volatile boolean active; + + + public JobInstanceArchiveTaskScheduler(ArchiveTaskService archiveTaskService, + TaskInstanceRecordDAO taskInstanceRecordDAO, + ArchiveProperties archiveProperties, + ShardingProperties shardingProperties, + JobInstanceArchiveTaskScheduleLock jobInstanceArchiveTaskScheduleLock) { + this.archiveTaskService = archiveTaskService; + this.taskInstanceRecordDAO = taskInstanceRecordDAO; + this.archiveProperties = archiveProperties; + this.shardingProperties = shardingProperties; + this.jobInstanceArchiveTaskScheduleLock = jobInstanceArchiveTaskScheduleLock; + } + + public void schedule() { + try { + if (isRunning) { + log.info("JobInstanceArchiveTaskScheduler is running, skip"); + return; + } else { + this.isRunning = true; + } + + while (true) { + // 获取归档任务调度锁 + boolean locked = jobInstanceArchiveTaskScheduleLock.lock(); + if (!locked) { + return; + } + if (runningTasksCount.get() >= archiveProperties.getConcurrent()) { + // 休眠一分钟，等待并行任务减少 + ThreadUtils.sleep(1000 * 60L); + return; + } + + List runningTasks = + archiveTaskService.listRunningTasks(ArchiveTaskTypeEnum.JOB_INSTANCE); + List needScheduleTasks = + archiveTaskService.listScheduleTasks(ArchiveTaskTypeEnum.JOB_INSTANCE, 100); + if (CollectionUtils.isEmpty(needScheduleTasks)) { + // 所有任务都已经被调度完成，退出本次任务调度 + return; + } + ArchiveTaskPrioritySorter.sort(runningTasks, shardingProperties.getDbNodeCount(), needScheduleTasks); + startArchiveTask(needScheduleTasks.get(0)); + } + } finally { + this.isRunning = false; + jobInstanceArchiveTaskScheduleLock.unlock(); + } + + } + + private void startArchiveTask(JobInstanceArchiveTaskInfo archiveTaskInfo) { + log.info("Start JobInstanceArchiveTask : {}", archiveTaskInfo); + runningTasksCount.incrementAndGet(); + JobInstanceMainDataArchiveTask archiveTask = new JobInstanceMainDataArchiveTask(); + ArchiveTaskWorker worker = new ArchiveTaskWorker(archiveTask); + workers.add(worker); + worker.start(); + } + + /** + * Spring Bean 生命周期管理-启动 + */ + @Override + public void start() { + synchronized (lifecycleMonitor) { + this.active = true; + } + } + + /** + * Spring Bean 生命周期管理-停止 + */ + @Override + public void stop() { + log.info("JobInstanceArchiveTaskScheduler stopping."); + synchronized (this.lifecycleMonitor) { + if (!isActive()) { + log.info("Shutdown ignored - JobInstanceArchiveTaskScheduler is not active already"); + return; + } + this.active = false; + } + stopTasksGraceful(); + log.info("JobInstanceArchiveTaskScheduler stop successfully!"); + } + + private void stopTasksGraceful() { + log.info("Stop archive tasks graceful - start"); + long start = System.currentTimeMillis(); + StopTaskCounter stopTaskCounter = null; + synchronized (lifecycleMonitor) { + if (!this.scheduledTasks.isEmpty()) { + log.info("Stop archive tasks, size: {}, tasks: {}", scheduledTasks.size(), scheduledTasks); + stopTaskCounter = StopTaskCounter.getInstance(); + stopTaskCounter.initCounter(scheduledTasks.keySet()); + } + for (ScheduledContinuousResultHandleTask task : scheduledTasks.values()) { + shutdownExecutor.execute(new StopTask(task, tracer)); + } + } + try { + if (stopTaskCounter != null) { + stopTaskCounter.waitingForAllTasksDone(); + } + } catch (Throwable e) { + log.error("Stop tasks caught exception", e); + } + long end = System.currentTimeMillis(); + log.info("Stop tasks graceful - end, cost: {}", end - start); + } + + @Override + public boolean isRunning() { + synchronized (this.lifecycleMonitor) { + return (this.active); + } + } + + boolean isActive() { + synchronized (this.lifecycleMonitor) { + return this.active; + } + } + + /** + * 消费者是否处于运行状态 + * + * @param worker 消费者 + * @return 是否运行 + */ + private boolean isWorkerActive(TaskWorker worker) { + boolean workerActive; + synchronized (this.workersMonitor) { + workerActive = this.workers.contains(worker); + } + return workerActive && this.isActive(); + } + + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceHotArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceHotArchivist.java new file mode 100644 index 0000000000..70ee4df5a2 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceHotArchivist.java @@ -0,0 +1,96 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import org.apache.commons.collections4.CollectionUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class JobInstanceHotArchivist { + /** + * 对一个批次数据进行备份 + * + * @param start 数据起始记录ID + * @param stop 数据终止记录ID + * @return 备份结果 + */ + private AbstractJobInstanceArchiveTask.BackupResult backupRecords(List jobInstanceIds) throws IOException { + if (lastBackupId >= stop) { + // 说明数据已经备份过，跳过 + log.info("[{}] Record is already backup, skip. lastBackId: {}", tableName, lastBackupId); + return new AbstractJobInstanceArchiveTask.BackupResult(0L, 0L, 0L, 0L); + } + long startId = start; + if (lastBackupId > start) { + // 从上次备份结束的 ID 位置开始 + startId = lastBackupId; + } + + List recordList = new ArrayList<>(readRowLimit); + long backupRows = 0; + long readRows = 0; + long readCost = 0; + long writeCost = 0; + long offset = 0L; + List records; + do { + // 选取start= batchInsertRowSize) { + long writeStartTime = System.currentTimeMillis(); + int insertRows = insertAndReset(recordList); + writeCost += (System.currentTimeMillis() - writeStartTime); + backupRows += insertRows; + } + offset += readRowLimit; + + } while (records.size() == readRowLimit); + + if (CollectionUtils.isNotEmpty(recordList)) { + // 处理没有达到批量插入阈值的最后一个批次的数据 + int insertRows = insertAndReset(recordList); + backupRows += insertRows; + } + + updateArchiveProgress(stop); + + return new AbstractJobInstanceArchiveTask.BackupResult(readRows, backupRows, readCost, writeCost); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java new file mode 100644 index 0000000000..98878b9bc1 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java @@ -0,0 +1,123 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.JobInstanceHotRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceRecordDAO; +import com.tencent.bk.job.backup.archive.impl.GseScriptExecuteObjTaskArchiver; +import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; +import com.tencent.bk.job.backup.config.ArchiveProperties; +import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; +import com.tencent.bk.job.execute.model.tables.TaskInstance; +import com.tencent.bk.job.execute.model.tables.records.TaskInstanceRecord; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * 作业实例数据归档 + **/ +@Slf4j +public class JobInstanceMainDataArchiveTask extends AbstractJobInstanceArchiveTask { + + private final TaskInstanceRecordDAO taskInstanceRecordDAO; + + private final List subTableArchivers = new ArrayList<>(); + + public JobInstanceMainDataArchiveTask(TaskInstanceRecordDAO taskInstanceRecordDAO, + GseScriptExecuteObjTaskArchiver gseScriptExecuteObjTaskArchiver, + JobInstanceHotRecordDAO jobInstanceHotRecordDAO, + JobInstanceColdDAO jobInstanceColdDAO, + ArchiveProperties archiveProperties, + ArchiveTaskLock archiveTaskLock, + ArchiveErrorTaskCounter archiveErrorTaskCounter, + JobInstanceArchiveTaskInfo archiveTask, + ArchiveTaskService archiveTaskService, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceHotRecordDAO, + jobInstanceColdDAO, + archiveProperties, + archiveTaskLock, + archiveErrorTaskCounter, + archiveTask, + archiveTaskService, + archiveTablePropsStorage + ); + this.taskInstanceRecordDAO = taskInstanceRecordDAO; + + this.subTableArchivers.add(gseScriptExecuteObjTaskArchiver); + } + + + @Override + protected void backupJobInstanceToColdDb(List jobInstanceRecords) { + List jobInstanceIds = + jobInstanceRecords.stream().map(this::extractJobInstanceId).collect(Collectors.toList()); + // 备份主表数据 + jobInstanceColdDAO.batchInsert(jobInstanceRecords, 1000); + // 备份子表数据 + subTableArchivers.forEach(tableArchiver -> { + tableArchiver.backupRecords(jobInstanceIds); + }); + } + + @Override + protected void deleteJobInstanceHotData(List jobInstanceIds) { + // 先删除子表数据 + subTableArchivers.forEach(tableArchiver -> { + tableArchiver.deleteRecords(jobInstanceIds); + }); + // 删除主表数据 + jobInstanceHotRecordDAO.deleteRecords(jobInstanceIds, + archiveTablePropsStorage.getDeleteLimitRowCount(TaskInstance.TASK_INSTANCE.getName())); + } + + @Override + protected List readSortedJobInstanceFromHotDB(Long fromTimestamp, + Long endTimestamp, + Long fromJobInstanceId, + int limit) { + return taskInstanceRecordDAO.readSortedJobInstanceFromHotDB( + fromTimestamp, + endTimestamp, + fromJobInstanceId, + limit + ); + } + + @Override + protected Long extractJobInstanceId(TaskInstanceRecord record) { + return record.get(TaskInstance.TASK_INSTANCE.ID); + } + + @Override + protected Long extractJobInstanceCreateTime(TaskInstanceRecord record) { + return record.get(TaskInstance.TASK_INSTANCE.CREATE_TIME); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceSubTableArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceSubTableArchiver.java new file mode 100644 index 0000000000..ad61b13c77 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceSubTableArchiver.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import java.util.List; + +/** + * 作业实例子表归档 + */ +public interface JobInstanceSubTableArchiver { + + /** + * 备份作业实例数据到冷存储 + * + * @param jobInstanceIds 作业实例 ID 列表 + */ + void backupRecords(List jobInstanceIds); + + /** + * 删除作业实例热数据 + * + * @param jobInstanceIds 作业实例 ID 列表 + */ + void deleteRecords(List jobInstanceIds); +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ArchiveProgressDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ArchiveProgressDAO.java similarity index 97% rename from src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ArchiveProgressDAO.java rename to src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ArchiveProgressDAO.java index 625309b8d4..995ce38dbe 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ArchiveProgressDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ArchiveProgressDAO.java @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.backup.dao; +package com.tencent.bk.job.backup.archive.dao; import com.tencent.bk.job.backup.model.dto.ArchiveProgressDTO; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ArchiveTaskDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ArchiveTaskDAO.java new file mode 100644 index 0000000000..d0072417c2 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ArchiveTaskDAO.java @@ -0,0 +1,56 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao; + +import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; +import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; + +import java.util.List; + +/** + * 归档任务 DAO + */ +public interface ArchiveTaskDAO { + + /** + * 获取最新的归档任务 + * + * @param taskType 归档任务类型 + */ + JobInstanceArchiveTaskInfo getLatestArchiveTask(ArchiveTaskTypeEnum taskType); + + void saveArchiveTask(JobInstanceArchiveTaskInfo jobInstanceArchiveTaskInfo); + + List listRunningTasks(ArchiveTaskTypeEnum taskType); + + List listScheduleTasks(ArchiveTaskTypeEnum taskType, int limit); + + /** + * 更新归档任务 + * + * @param archiveTask 归档任务 + */ + void updateTask(JobInstanceArchiveTaskInfo archiveTask); +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ExecuteArchiveDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ExecuteArchiveDAO.java similarity index 97% rename from src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ExecuteArchiveDAO.java rename to src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ExecuteArchiveDAO.java index 60fcd5cf09..7326ce4336 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ExecuteArchiveDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ExecuteArchiveDAO.java @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.backup.dao; +package com.tencent.bk.job.backup.archive.dao; import org.jooq.TableRecord; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ExecuteRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ExecuteRecordDAO.java new file mode 100644 index 0000000000..7251fdf5eb --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ExecuteRecordDAO.java @@ -0,0 +1,81 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao; + +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.TableField; + +import java.util.List; + +/** + * job-execute 微服务的表数据DAO + * + * @param 表记录 + */ +public interface ExecuteRecordDAO { + + /** + * 获取表 + * + * @return 表 + */ + Table getTable(); + + /** + * 获取用于查询归档记录的ID字段 + * + * @return ID字段 + */ + TableField getArchiveIdField(); + + /** + * 根据起始/结束ID获取表记录 + * + * @param start 起始ID(exclude) + * @param end 结束ID(include) + * @param offset 记录偏移数 + * @param limit 获取的记录数量 + * @return 表记录 + */ + List listRecords(Long start, Long end, Long offset, Long limit); + + /** + * 根据起始/结束ID删除表记录 + * + * @param start 起始ID(exclude) + * @param end 结束ID(include) + * @param maxLimitedDeleteRows 批量删除每批次limit + * @return 删除的记录数量 + */ + int deleteRecords(Long start, Long end, long maxLimitedDeleteRows); + + /** + * 获取表中最小归档ID + * + * @return id值。如果表中没有数据，那么返回 null + */ + Long getMinArchiveId(); +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/JobInstanceColdDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/JobInstanceColdDAO.java new file mode 100644 index 0000000000..2dc7195f2c --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/JobInstanceColdDAO.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao; + +import com.tencent.bk.job.backup.archive.ArchiveException; +import org.jooq.TableRecord; + +import java.util.List; + +/** + * 作业实例存储冷数据 DAO + */ +public interface JobInstanceColdDAO { + /** + * 批量写入数据到冷 db + * + * @param recordList 记录列表 + * @param bulkSize db 批量插入的最大记录数量 + * @return 写入的数据行数 + */ + Integer batchInsert(List> recordList, int bulkSize) throws ArchiveException; +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/JobInstanceHotRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/JobInstanceHotRecordDAO.java new file mode 100644 index 0000000000..104bcfe11f --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/JobInstanceHotRecordDAO.java @@ -0,0 +1,82 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao; + +import com.tencent.bk.job.backup.archive.dao.impl.RecordResultSet; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.TableField; + +import java.util.Collection; +import java.util.List; + +/** + * 作业实例热数据 DAO + * + * @param 表记录 + */ +public interface JobInstanceHotRecordDAO { + + /** + * 获取表 + * + * @return 表 + */ + Table getTable(); + + /** + * 获取表中作业实例 ID 对应的字段 + * + * @return ID字段 + */ + TableField getJobInstanceIdField(); + + /** + * 根据作业实例 ID 列表获取表记录 + * + * @param jobInstanceIds 作业实例 ID 列表 + * @param readRowLimit 获取的记录数量 + * @return 表记录 + */ + List listRecords(Collection jobInstanceIds, Long readRowLimit); + + /** + * ResultSet 方式查询 + * + * @param jobInstanceIds 作业实例 ID 列表 + * @param readRowLimit 每次查询获取的记录数量上限 + * @return ResultSet + */ + RecordResultSet executeQuery(Collection jobInstanceIds, long readRowLimit); + + /** + * 根据起始/结束ID删除表记录 + * + * @param jobInstanceIds 作业实例 ID 列表 + * @param maxLimitedDeleteRows 批量删除每批次limit + * @return 删除的记录数量 + */ + int deleteRecords(Collection jobInstanceIds, long maxLimitedDeleteRows); +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/RecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/RecordDAO.java new file mode 100644 index 0000000000..fccefb6b42 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/RecordDAO.java @@ -0,0 +1,66 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao; + +import org.jooq.Condition; +import org.jooq.Record; +import org.jooq.Table; + +import java.util.Collection; +import java.util.List; + +/** + * DB 表操作 DAO + * + * @param 表记录 + */ +public interface RecordDAO { + + /** + * 获取表 + * + * @return 表 + */ + Table getTable(); + + + /** + * 根据条件查询表记录 + * + * @param conditions 查询条件 + * @param limit 获取的记录数量 + * @return 表记录 + */ + List listRecords(List conditions, Long readRowLimit); + + /** + * 根据起始/结束ID删除表记录 + * + * @param jobInstanceIds 作业实例 ID 列表 + * @param deleteRowLimit 批量删除每批次limit + * @return 删除的记录数量 + */ + int deleteRecords(Collection jobInstanceIds, long deleteRowLimit); +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceHotRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceHotRecordDAO.java new file mode 100644 index 0000000000..793e369c06 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceHotRecordDAO.java @@ -0,0 +1,114 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.backup.archive.dao.JobInstanceHotRecordDAO; +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import org.apache.commons.collections4.CollectionUtils; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.OrderField; +import org.jooq.Record; +import org.jooq.Result; +import org.jooq.SelectConditionStep; +import org.jooq.Table; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +public abstract class AbstractJobInstanceHotRecordDAO implements JobInstanceHotRecordDAO { + + protected final DSLContextProvider dslContextProvider; + + protected final String tableName; + + public AbstractJobInstanceHotRecordDAO(DSLContextProvider dslContextProvider, String tableName) { + this.dslContextProvider = dslContextProvider; + this.tableName = tableName; + } + + @Override + public List listRecords(Collection jobInstanceIds, Long readRowLimit) { + return query(getTable(), buildBasicConditions(jobInstanceIds), readRowLimit); + } + + protected List listRecordsByConditions(List conditions, Long readRowLimit) { + return query(getTable(), conditions, readRowLimit); + } + + @Override + public int deleteRecords(Collection jobInstanceIds, long maxLimitedDeleteRows) { + return deleteWithLimit(getTable(), buildBasicConditions(jobInstanceIds), maxLimitedDeleteRows); + } + + public List buildBasicConditions(Collection jobInstanceIds) { + List conditions = new ArrayList<>(); + conditions.add(getJobInstanceIdField().in(jobInstanceIds)); + return conditions; + } + + private int deleteWithLimit(Table table, List conditions, long maxLimitedDeleteRows) { + int totalDeleteRows = 0; + while (true) { + int deletedRows = dsl() + .delete(table) + .where(conditions) + .limit(maxLimitedDeleteRows) + .execute(); + totalDeleteRows += deletedRows; + if (deletedRows < maxLimitedDeleteRows) { + break; + } + } + return totalDeleteRows; + } + + protected List query(Table table, + List conditions, + Long readRowLimit) { + SelectConditionStep selectConditionStep = dsl() + .select() + .from(table) + .where(conditions); + + Result result; + if (CollectionUtils.isNotEmpty(getListRecordsOrderFields())) { + result = selectConditionStep.orderBy(getListRecordsOrderFields()).limit(0, readRowLimit).fetch(); + } else { + result = selectConditionStep.limit(0, readRowLimit).fetch(); + } + return result.into(getTable()); + + } + + public abstract Table getTable(); + + protected abstract Collection> getListRecordsOrderFields(); + + protected DSLContext dsl() { + return this.dslContextProvider.get(tableName); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceRecordResultSet.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceRecordResultSet.java new file mode 100644 index 0000000000..f0b052ada4 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceRecordResultSet.java @@ -0,0 +1,108 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import org.apache.commons.collections4.CollectionUtils; +import org.jooq.Condition; +import org.jooq.Record; + +import java.util.Collection; +import java.util.List; + +/** + * 作业实例数据查询结果集 + * + * @param + */ +abstract class AbstractJobInstanceRecordResultSet implements RecordResultSet { + private final AbstractJobInstanceHotRecordDAO jobInstanceHotRecordDAO; + private final Collection jobInstanceIds; + private final Long readRowLimit; + /** + * 当前查询的记录列表 + */ + private List records; + /** + * 当前查询的最后一条记录 + */ + private T lastRecord; + private boolean hasNext = true; + + + public AbstractJobInstanceRecordResultSet(AbstractJobInstanceHotRecordDAO jobInstanceHotRecordDAO, + Collection jobInstanceIds, + Long readRowLimit) { + this.jobInstanceHotRecordDAO = jobInstanceHotRecordDAO; + this.jobInstanceIds = jobInstanceIds; + this.readRowLimit = readRowLimit; + } + + @Override + public boolean hasNext() { + if (!hasNext) { + return false; + } + if (lastRecord == null) { + // 首次查询 + records = jobInstanceHotRecordDAO.listRecords(jobInstanceIds, readRowLimit); + } else { + // 非首次查询，需要加入数据查询偏移条件 + List offsetConditions = buildOffsetQueryConditions(lastRecord); + if (CollectionUtils.isEmpty(offsetConditions)) { + // 如果构造的数据查询偏移条件为空，不再继续查询 + hasNext = false; + records = null; + lastRecord = null; + return false; + } + List conditions = jobInstanceHotRecordDAO.buildBasicConditions(jobInstanceIds); + conditions.addAll(offsetConditions); + records = jobInstanceHotRecordDAO.listRecordsByConditions(conditions, readRowLimit); + } + + if (CollectionUtils.isEmpty(records)) { + hasNext = false; + lastRecord = null; + } else { + lastRecord = records.get(records.size() - 1); + // 如果查询记录数量小于 readRowLimit 或者指定全量查询(readRowLimit == null 表示) + hasNext = readRowLimit != null && records.size() >= readRowLimit; + } + return hasNext; + } + + /** + * 构造数据查询偏移条件 + * + * @param lastRecord 当前最后一条记录 + * @return 数据查询偏移条件 + */ + protected abstract List buildOffsetQueryConditions(T lastRecord); + + @Override + public List getRecords() { + return records; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/ArchiveProgressDAOImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveProgressDAOImpl.java similarity index 97% rename from src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/ArchiveProgressDAOImpl.java rename to src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveProgressDAOImpl.java index c82c31f17e..ff8124fa8d 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/ArchiveProgressDAOImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveProgressDAOImpl.java @@ -22,9 +22,9 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.backup.dao.impl; +package com.tencent.bk.job.backup.archive.dao.impl; -import com.tencent.bk.job.backup.dao.ArchiveProgressDAO; +import com.tencent.bk.job.backup.archive.dao.ArchiveProgressDAO; import com.tencent.bk.job.backup.model.dto.ArchiveProgressDTO; import com.tencent.bk.job.backup.model.tables.ArchiveProgress; import org.jooq.DSLContext; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java new file mode 100644 index 0000000000..1ee04d5ae1 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java @@ -0,0 +1,188 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.backup.archive.dao.ArchiveTaskDAO; +import com.tencent.bk.job.backup.archive.model.DbDataNode; +import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; +import com.tencent.bk.job.backup.archive.model.TimeAndIdBasedArchiveProcess; +import com.tencent.bk.job.backup.constant.ArchiveTaskStatusEnum; +import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; +import com.tencent.bk.job.backup.model.tables.ArchiveTask; +import com.tencent.bk.job.backup.model.tables.records.ArchiveTaskRecord; +import com.tencent.bk.job.common.mysql.jooq.JooqDataTypeUtil; +import org.jooq.DSLContext; +import org.jooq.Record; +import org.jooq.Result; +import org.jooq.TableField; +import org.jooq.UpdateSetMoreStep; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Repository; + +import java.util.ArrayList; +import java.util.List; + +@Repository +public class ArchiveTaskDAOImpl implements ArchiveTaskDAO { + + private final DSLContext ctx; + private static final ArchiveTask T = ArchiveTask.ARCHIVE_TASK; + + private static final TableField[] ALL_FIELDS = { + T.TASK_TYPE, + T.DATA_NODE, + T.DAY, + T.HOUR, + T.FROM_TIMESTAMP, + T.TO_TIMESTAMP, + T.PROCESS, + T.STATUS, + T.CREATE_TIME, + T.LAST_UPDATE_TIME + }; + + @Autowired + public ArchiveTaskDAOImpl(@Qualifier("job-backup-dsl-context") DSLContext ctx) { + this.ctx = ctx; + } + + @Override + public JobInstanceArchiveTaskInfo getLatestArchiveTask(ArchiveTaskTypeEnum taskType) { + Record record = ctx.select(ALL_FIELDS) + .from(T) + .where(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(taskType.getType()))) + .orderBy(T.DAY.desc(), T.HOUR.desc()) + .fetchOne(); + + return extract(record); + } + + private JobInstanceArchiveTaskInfo extract(Record record) { + if (record == null) { + return null; + } + JobInstanceArchiveTaskInfo archiveTask = new JobInstanceArchiveTaskInfo(); + archiveTask.setTaskType(ArchiveTaskTypeEnum.valOf(record.get(T.TASK_TYPE))); + archiveTask.setDbDataNode(DbDataNode.fromDataNodeId(record.get(T.DATA_NODE))); + archiveTask.setDay(record.get(T.DAY)); + archiveTask.setHour(JooqDataTypeUtil.toInteger(record.get(T.HOUR))); + archiveTask.setFromTimestamp(record.get(T.FROM_TIMESTAMP)); + archiveTask.setToTimestamp(record.get(T.TO_TIMESTAMP)); + archiveTask.setProcess(TimeAndIdBasedArchiveProcess.fromPersistentProcess(record.get(T.PROCESS))); + archiveTask.setStatus(ArchiveTaskStatusEnum.valOf(record.get(T.STATUS))); + archiveTask.setCreateTime(record.get(T.CREATE_TIME)); + archiveTask.setLastUpdateTime(record.get(T.LAST_UPDATE_TIME)); + return archiveTask; + } + + @Override + public void saveArchiveTask(JobInstanceArchiveTaskInfo jobInstanceArchiveTaskInfo) { + long createTime = System.currentTimeMillis(); + ctx.insertInto( + T, + T.TASK_TYPE, + T.DATA_NODE, + T.DAY, + T.HOUR, + T.FROM_TIMESTAMP, + T.TO_TIMESTAMP, + T.PROCESS, + T.STATUS, + T.CREATE_TIME, + T.LAST_UPDATE_TIME) + .values( + JooqDataTypeUtil.toByte(jobInstanceArchiveTaskInfo.getTaskType().getType()), + jobInstanceArchiveTaskInfo.getDbDataNode().toDataNodeId(), + jobInstanceArchiveTaskInfo.getDay(), + JooqDataTypeUtil.toByte(jobInstanceArchiveTaskInfo.getHour()), + jobInstanceArchiveTaskInfo.getFromTimestamp(), + jobInstanceArchiveTaskInfo.getToTimestamp(), + jobInstanceArchiveTaskInfo.getProcess() != null ? + jobInstanceArchiveTaskInfo.getProcess().toPersistentProcess() : null, + JooqDataTypeUtil.toByte(jobInstanceArchiveTaskInfo.getStatus().getStatus()), + createTime, + createTime + ) + .execute(); + } + + @Override + public List listRunningTasks(ArchiveTaskTypeEnum taskType) { + Result result = ctx.select(ALL_FIELDS) + .from(T) + .where(T.STATUS.eq(JooqDataTypeUtil.toByte(ArchiveTaskStatusEnum.RUNNING.getStatus()))) + .and(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(taskType.getType()))) + .fetch(); + + List tasks = new ArrayList<>(result.size()); + result.forEach(record -> tasks.add(extract(record))); + return tasks; + } + + @Override + public List listScheduleTasks(ArchiveTaskTypeEnum taskType, int limit) { + Result result = ctx.select(ALL_FIELDS) + .from(T) + .where(T.STATUS.in( + JooqDataTypeUtil.toByte(ArchiveTaskStatusEnum.PENDING.getStatus()), + JooqDataTypeUtil.toByte(ArchiveTaskStatusEnum.SUSPENDED.getStatus()), + JooqDataTypeUtil.toByte(ArchiveTaskStatusEnum.FAIL.getStatus()))) + .and(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(taskType.getType()))) + .orderBy(T.DAY.desc(), T.HOUR.desc()) + .limit(limit) + .fetch(); + + List tasks = new ArrayList<>(result.size()); + result.forEach(record -> tasks.add(extract(record))); + return tasks; + } + + @Override + public void updateTask(JobInstanceArchiveTaskInfo archiveTask) { + if (archiveTask.getStatus() == null && archiveTask.getProcess() == null) { + // 无需更新 + return; + } + UpdateSetMoreStep updateSetMoreStep; + updateSetMoreStep = ctx + .update(T) + .set(T.LAST_UPDATE_TIME, System.currentTimeMillis()); + if (archiveTask.getStatus() != null) { + updateSetMoreStep + .set(T.STATUS, JooqDataTypeUtil.toByte(archiveTask.getStatus().getStatus())); + } + if (archiveTask.getProcess() != null) { + updateSetMoreStep + .set(T.PROCESS, archiveTask.getProcess().toPersistentProcess()); + } + updateSetMoreStep + .where(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(archiveTask.getTaskType().getType()))) + .and(T.DATA_NODE.eq(archiveTask.getDbDataNode().toDataNodeId())) + .and(T.DAY.eq(archiveTask.getDay())) + .and(T.HOUR.eq(archiveTask.getHour().byteValue())) + .execute(); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/ExecuteArchiveDAOImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ExecuteArchiveDAOImpl.java similarity index 98% rename from src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/ExecuteArchiveDAOImpl.java rename to src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ExecuteArchiveDAOImpl.java index 33ceaea286..0421d13285 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/ExecuteArchiveDAOImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ExecuteArchiveDAOImpl.java @@ -22,9 +22,9 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.backup.dao.impl; +package com.tencent.bk.job.backup.archive.dao.impl; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; +import com.tencent.bk.job.backup.archive.dao.ExecuteArchiveDAO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.jooq.DSLContext; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/FileSourceTaskLogRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/FileSourceTaskLogRecordDAO.java new file mode 100644 index 0000000000..993a698155 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/FileSourceTaskLogRecordDAO.java @@ -0,0 +1,88 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + + +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.FileSourceTaskLog; +import com.tencent.bk.job.execute.model.tables.records.FileSourceTaskLogRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.Table; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * file_source_task_log DAO + */ +public class FileSourceTaskLogRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final FileSourceTaskLog TABLE = FileSourceTaskLog.FILE_SOURCE_TASK_LOG; + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(FileSourceTaskLog.FILE_SOURCE_TASK_LOG.TASK_INSTANCE_ID.asc()); + ORDER_FIELDS.add(FileSourceTaskLog.FILE_SOURCE_TASK_LOG.ID.asc()); + } + + public FileSourceTaskLogRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE.getName()); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.TASK_INSTANCE_ID.ge(lastRecord.getTaskInstanceId())); + conditions.add(TABLE.ID.gt(lastRecord.getId())); + return conditions; + } + ); + } + + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileAgentTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileAgentTaskRecordDAO.java new file mode 100644 index 0000000000..91cc76c771 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileAgentTaskRecordDAO.java @@ -0,0 +1,86 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.GseFileAgentTask; +import com.tencent.bk.job.execute.model.tables.records.GseFileAgentTaskRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.Table; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * gse_file_agent_task DAO + */ +public class GseFileAgentTaskRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final GseFileAgentTask TABLE = GseFileAgentTask.GSE_FILE_AGENT_TASK; + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(GseFileAgentTask.GSE_FILE_AGENT_TASK.TASK_INSTANCE_ID.asc()); + ORDER_FIELDS.add(GseFileAgentTask.GSE_FILE_AGENT_TASK.ID.asc()); + } + + public GseFileAgentTaskRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE.getName()); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getArchiveIdField() { + return TABLE.STEP_INSTANCE_ID; + } + + @Override + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.TASK_INSTANCE_ID.ge(lastRecord.getTaskInstanceId())); + conditions.add(TABLE.ID.gt(lastRecord.getId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileExecuteObjTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileExecuteObjTaskRecordDAO.java new file mode 100644 index 0000000000..ce860a61c0 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileExecuteObjTaskRecordDAO.java @@ -0,0 +1,88 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.GseFileExecuteObjTask; +import com.tencent.bk.job.execute.model.tables.records.GseFileExecuteObjTaskRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.Table; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * gse_file_execute_obj_task DAO + */ +public class GseFileExecuteObjTaskRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final GseFileExecuteObjTask TABLE = GseFileExecuteObjTask.GSE_FILE_EXECUTE_OBJ_TASK; + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(GseFileExecuteObjTask.GSE_FILE_EXECUTE_OBJ_TASK.STEP_INSTANCE_ID.asc()); + ORDER_FIELDS.add(GseFileExecuteObjTask.GSE_FILE_EXECUTE_OBJ_TASK.EXECUTE_COUNT.asc()); + ORDER_FIELDS.add(GseFileExecuteObjTask.GSE_FILE_EXECUTE_OBJ_TASK.BATCH.asc()); + ORDER_FIELDS.add(GseFileExecuteObjTask.GSE_FILE_EXECUTE_OBJ_TASK.MODE.asc()); + ORDER_FIELDS.add(GseFileExecuteObjTask.GSE_FILE_EXECUTE_OBJ_TASK.EXECUTE_OBJ_ID.asc()); + } + + public GseFileExecuteObjTaskRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE.getName()); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.TASK_INSTANCE_ID.ge(lastRecord.getTaskInstanceId())); + conditions.add(TABLE.ID.gt(lastRecord.getId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptAgentTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptAgentTaskRecordDAO.java new file mode 100644 index 0000000000..1f19b87a07 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptAgentTaskRecordDAO.java @@ -0,0 +1,91 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.GseScriptAgentTask; +import com.tencent.bk.job.execute.model.tables.records.GseScriptAgentTaskRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.Table; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * gse_script_agent_task DAO + */ +public class GseScriptAgentTaskRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final GseScriptAgentTask TABLE = GseScriptAgentTask.GSE_SCRIPT_AGENT_TASK; + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(GseScriptAgentTask.GSE_SCRIPT_AGENT_TASK.TASK_INSTANCE_ID.asc()); + ORDER_FIELDS.add(GseScriptAgentTask.GSE_SCRIPT_AGENT_TASK.ID.asc()); + } + + public GseScriptAgentTaskRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE.getName()); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getArchiveIdField() { + return TABLE.STEP_INSTANCE_ID; + } + + @Override + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.TASK_INSTANCE_ID.ge(lastRecord.getTaskInstanceId())); + conditions.add(TABLE.ID.gt(lastRecord.getId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptExecuteObjTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptExecuteObjTaskRecordDAO.java new file mode 100644 index 0000000000..9d4edf0135 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptExecuteObjTaskRecordDAO.java @@ -0,0 +1,86 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.GseScriptExecuteObjTask; +import com.tencent.bk.job.execute.model.tables.records.GseScriptExecuteObjTaskRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.Table; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * gse_script_execute_obj_task DAO + */ +public class GseScriptExecuteObjTaskRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final GseScriptExecuteObjTask TABLE = GseScriptExecuteObjTask.GSE_SCRIPT_EXECUTE_OBJ_TASK; + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(GseScriptExecuteObjTask.GSE_SCRIPT_EXECUTE_OBJ_TASK.TASK_INSTANCE_ID.asc()); + ORDER_FIELDS.add(GseScriptExecuteObjTask.GSE_SCRIPT_EXECUTE_OBJ_TASK.ID.asc()); + } + + public GseScriptExecuteObjTaskRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE.getName()); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + @Override + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.TASK_INSTANCE_ID.ge(lastRecord.getTaskInstanceId())); + conditions.add(TABLE.ID.gt(lastRecord.getId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseTaskRecordDAO.java new file mode 100644 index 0000000000..80f7240dcf --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseTaskRecordDAO.java @@ -0,0 +1,83 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.GseTask; +import com.tencent.bk.job.execute.model.tables.records.GseTaskRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.Table; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +public class GseTaskRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final GseTask TABLE = GseTask.GSE_TASK; + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(GseTask.GSE_TASK.TASK_INSTANCE_ID.asc()); + ORDER_FIELDS.add(GseTask.GSE_TASK.ID.asc()); + } + + public GseTaskRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE.getName()); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.TASK_INSTANCE_ID.ge(lastRecord.getTaskInstanceId())); + conditions.add(TABLE.ID.gt(lastRecord.getId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/JobInstanceColdDAOImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/JobInstanceColdDAOImpl.java new file mode 100644 index 0000000000..8435033bb2 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/JobInstanceColdDAOImpl.java @@ -0,0 +1,149 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.backup.archive.ArchiveException; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.jooq.DSLContext; +import org.jooq.Loader; +import org.jooq.LoaderError; +import org.jooq.TableRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.List; + + +@Slf4j +public class JobInstanceColdDAOImpl implements JobInstanceColdDAO { + + private static final Logger ARCHIVE_FAILED_LOGGER = LoggerFactory.getLogger("ArchiveFailedLogger"); + + private final DSLContext context; + + public JobInstanceColdDAOImpl(DSLContext coldDbDslContext) { + log.info("Init JobInstanceColdDAOImpl."); + this.context = coldDbDslContext; + } + + @Override + public Integer batchInsert(List> recordList, int bulkSize) throws ArchiveException { + long start = System.currentTimeMillis(); + int successInsertedRecords = 0; + String table = recordList.get(0).getTable().getName(); + boolean success = true; + try { + Loader loader = + context.loadInto(recordList.get(0).getTable()) + // 由于这里是批量写入，jooq 不允许使用 onDuplicateKeyIgnore/onDuplicateKeyUpdate. + // 否则会报错"Cannot apply bulk loading with onDuplicateKey flags" + // 所以这里暂时使用 onDuplicateKeyError 错误处理方式，等后续流程进一步判断是否是主键冲突错误 + // issue 参考：https://github.com/jOOQ/jOOQ/issues/12740 + .onDuplicateKeyError() + .bulkAfter(bulkSize) + .loadRecords(recordList) + .fieldsCorresponding() + .execute(); + successInsertedRecords = loader.stored(); + String bulkInsertResult = successInsertedRecords == recordList.size() ? "success" : "fail"; + log.info( + "InsertBulk: Load {} data|result|{}|executed|{}|processed|{}|stored|{}|ignored|{}|errors|{}", + table, + bulkInsertResult, + loader.executed(), + loader.processed(), + loader.stored(), + loader.ignored(), + loader.errors().size() + ); + if (CollectionUtils.isNotEmpty(loader.errors())) { + for (LoaderError error : loader.errors()) { + ARCHIVE_FAILED_LOGGER.error("Error while load {} data, exception: {}， error row: {}", table, + error.exception().getMessage(), error.row()); + } + if (hasDuplicateError(loader.errors())) { + // 如果存在主键冲突的数据，尝试每一条记录单独插入，就可以使用 onDuplicateKeyIgnore 错误处理方式 + successInsertedRecords = insertSingle(recordList); + } + } + } catch (IOException e) { + String errorMsg = String.format("Error while loading %s data!", table); + log.error(errorMsg, e); + success = false; + throw new ArchiveException(errorMsg, e); + } finally { + log.info("Load data to {} done! success: {}, total: {}, inserted: {}, cost: {}ms", table, success, + recordList.size(), successInsertedRecords, System.currentTimeMillis() - start); + } + + return successInsertedRecords; + } + + private boolean hasDuplicateError(List errors) { + // 通过 mysql 执行的错误消息判断是否是由于数据唯一性冲突引起的 + return errors.stream().anyMatch( + error -> error.exception().getMessage() != null + && error.exception().getMessage().contains("Duplicate entry")); + } + + private int insertSingle(List> recordList) throws IOException { + int successInsertedRecords; + String table = recordList.get(0).getTable().getName(); + try { + Loader loader = + context.loadInto(recordList.get(0).getTable()) + .onDuplicateKeyIgnore() + .loadRecords(recordList) + .fieldsCorresponding() + .execute(); + successInsertedRecords = loader.stored() + loader.ignored(); + String bulkInsertResult = successInsertedRecords == recordList.size() ? "success" : "fail"; + log.info( + "InsertSingle: Load {} data|result|{}|executed|{}|processed|{}|stored|{}|ignored|{}|errors|{}", + table, + bulkInsertResult, + loader.executed(), + loader.processed(), + loader.stored(), + loader.ignored(), + loader.errors().size() + ); + if (CollectionUtils.isNotEmpty(loader.errors())) { + for (LoaderError error : loader.errors()) { + ARCHIVE_FAILED_LOGGER.error("InsertSingle: Error while load {} data, exception: {}, error row: {}", + table, error.row(), error.exception().getMessage()); + } + } + } catch (IOException e) { + String errorMsg = String.format("Error while loading %s data!", table); + log.error(errorMsg, e); + throw e; + } + return successInsertedRecords; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/JobInstanceRecordResultSetFactory.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/JobInstanceRecordResultSetFactory.java new file mode 100644 index 0000000000..a45160f50e --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/JobInstanceRecordResultSetFactory.java @@ -0,0 +1,101 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import org.jooq.Condition; +import org.jooq.Record; + +import java.util.Collection; +import java.util.List; +import java.util.function.Function; + +public class JobInstanceRecordResultSetFactory { + + /** + * 创建一个多次分批查询 + * + * @param jobInstanceHotRecordDAO dao + * @param jobInstanceIds 作业实例 ID 列表 + * @param readRowLimit 单次查询最大读取行数限制 + * @param offsetConditionBuilder 查询偏移条件构造 + * @param 记录 + * @return ResultSet + */ + public static DefaultJobInstanceRecordResultSet createMultiQueryResultSet( + AbstractJobInstanceHotRecordDAO jobInstanceHotRecordDAO, + Collection jobInstanceIds, + Long readRowLimit, + Function> offsetConditionBuilder) { + return new DefaultJobInstanceRecordResultSet<>( + jobInstanceHotRecordDAO, + jobInstanceIds, + readRowLimit, + offsetConditionBuilder + ); + } + + /** + * 创建一个单次全量查询 + * + * @param jobInstanceHotRecordDAO dao + * @param jobInstanceIds 作业实例 ID 列表 + * @param readRowLimit 单次查询最大读取行数限制 + * @param 记录 + * @return ResultSet + */ + public static DefaultJobInstanceRecordResultSet createOneQueryResultSet( + AbstractJobInstanceHotRecordDAO jobInstanceHotRecordDAO, + Collection jobInstanceIds, + Long readRowLimit) { + return new DefaultJobInstanceRecordResultSet<>( + jobInstanceHotRecordDAO, + jobInstanceIds, + readRowLimit, + lastRecord -> null + ); + } + + public static class DefaultJobInstanceRecordResultSet + extends AbstractJobInstanceRecordResultSet { + + Function> offsetConditionBuilder; + + public DefaultJobInstanceRecordResultSet( + AbstractJobInstanceHotRecordDAO jobInstanceHotRecordDAO, + Collection jobInstanceIds, + Long readRowLimit, + Function> offsetConditionBuilder) { + super(jobInstanceHotRecordDAO, jobInstanceIds, readRowLimit); + this.offsetConditionBuilder = offsetConditionBuilder; + } + + @Override + protected List buildOffsetQueryConditions(T lastRecord) { + return offsetConditionBuilder.apply(lastRecord); + } + } + + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/OperationLogRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/OperationLogRecordDAO.java new file mode 100644 index 0000000000..29b53805a1 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/OperationLogRecordDAO.java @@ -0,0 +1,87 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.OperationLog; +import com.tencent.bk.job.execute.model.tables.records.OperationLogRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.Table; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * operation_log DAO + */ +public class OperationLogRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final OperationLog TABLE = OperationLog.OPERATION_LOG; + + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(OperationLog.OPERATION_LOG.TASK_INSTANCE_ID.asc()); + ORDER_FIELDS.add(OperationLog.OPERATION_LOG.ID.asc()); + } + + public OperationLogRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE.getName()); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.TASK_INSTANCE_ID.ge(lastRecord.getTaskInstanceId())); + conditions.add(TABLE.ID.gt(lastRecord.getId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/RecordResultSet.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/RecordResultSet.java new file mode 100644 index 0000000000..c030b749a9 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/RecordResultSet.java @@ -0,0 +1,46 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import org.jooq.Record; + +import java.util.List; + +/** + * 表查询结果 + * + * @param + */ +public interface RecordResultSet { + /** + * 是否还需要继续查询 + */ + boolean hasNext(); + + /** + * 获取本次查询的表记录 + */ + List getRecords(); +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/RollingConfigArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/RollingConfigRecordDAO.java similarity index 50% rename from src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/RollingConfigArchivist.java rename to src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/RollingConfigRecordDAO.java index 7d6563dee6..20ecb80ebc 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/RollingConfigArchivist.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/RollingConfigRecordDAO.java @@ -22,40 +22,57 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.archive.ArchiveTaskLock; -import com.tencent.bk.job.backup.config.ArchiveDBProperties; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.RollingConfigRecordDAO; -import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; -import com.tencent.bk.job.backup.service.ArchiveProgressService; +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.RollingConfig; import com.tencent.bk.job.execute.model.tables.records.RollingConfigRecord; +import org.jooq.OrderField; +import org.jooq.Table; +import org.jooq.TableField; -import java.util.concurrent.CountDownLatch; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; /** - * rolling_config 表归档 + * rolling_config DAO */ -public class RollingConfigArchivist extends AbstractArchivist { - - public RollingConfigArchivist(RollingConfigRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService, - ArchiveDBProperties archiveDBProperties, - ArchiveTaskLock archiveTaskLock, - Long maxNeedArchiveId, - CountDownLatch countDownLatch, - ArchiveErrorTaskCounter archiveErrorTaskCounter) { - super(executeRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveId, - countDownLatch, - archiveErrorTaskCounter); - this.deleteIdStepSize = 100_000; +public class RollingConfigRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final RollingConfig TABLE = RollingConfig.ROLLING_CONFIG; + + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(RollingConfig.ROLLING_CONFIG.TASK_INSTANCE_ID.asc()); + } + + public RollingConfigRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE.getName()); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createOneQueryResultSet( + this, + jobInstanceIds, + readRowLimit + ); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceConfirmRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceConfirmRecordDAO.java new file mode 100644 index 0000000000..07934ce9c9 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceConfirmRecordDAO.java @@ -0,0 +1,88 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.StepInstanceConfirm; +import com.tencent.bk.job.execute.model.tables.records.StepInstanceConfirmRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.Table; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * step_instance_confirm DAO + */ +public class StepInstanceConfirmRecordDAO extends AbstractJobInstanceHotRecordDAO { + + + private static final StepInstanceConfirm TABLE = StepInstanceConfirm.STEP_INSTANCE_CONFIRM; + + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(StepInstanceConfirm.STEP_INSTANCE_CONFIRM.TASK_INSTANCE_ID.asc()); + ORDER_FIELDS.add(StepInstanceConfirm.STEP_INSTANCE_CONFIRM.STEP_INSTANCE_ID.asc()); + } + + + public StepInstanceConfirmRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE.getName()); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.TASK_INSTANCE_ID.ge(lastRecord.getTaskInstanceId())); + conditions.add(TABLE.STEP_INSTANCE_ID.gt(lastRecord.getStepInstanceId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceFileRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceFileRecordDAO.java new file mode 100644 index 0000000000..e10b712068 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceFileRecordDAO.java @@ -0,0 +1,87 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.StepInstanceFile; +import com.tencent.bk.job.execute.model.tables.records.StepInstanceFileRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.Table; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * step_instance_file DAO + */ +public class StepInstanceFileRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final StepInstanceFile TABLE = StepInstanceFile.STEP_INSTANCE_FILE; + + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(StepInstanceFile.STEP_INSTANCE_FILE.TASK_INSTANCE_ID.asc()); + ORDER_FIELDS.add(StepInstanceFile.STEP_INSTANCE_FILE.STEP_INSTANCE_ID.asc()); + } + + + public StepInstanceFileRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE.getName()); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.TASK_INSTANCE_ID.ge(lastRecord.getTaskInstanceId())); + conditions.add(TABLE.STEP_INSTANCE_ID.gt(lastRecord.getStepInstanceId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRecordDAO.java new file mode 100644 index 0000000000..14c995fdf6 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRecordDAO.java @@ -0,0 +1,87 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.StepInstance; +import com.tencent.bk.job.execute.model.tables.records.StepInstanceRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.Table; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * step_instance DAO + */ +public class StepInstanceRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final StepInstance TABLE = StepInstance.STEP_INSTANCE; + + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(StepInstance.STEP_INSTANCE.TASK_INSTANCE_ID.asc()); + ORDER_FIELDS.add(StepInstance.STEP_INSTANCE.ID.asc()); + } + + public StepInstanceRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE.getName()); + } + + @Override + public Table getTable() { + return TABLE; + } + + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.TASK_INSTANCE_ID.ge(lastRecord.getTaskInstanceId())); + conditions.add(TABLE.ID.gt(lastRecord.getId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRollingTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRollingTaskRecordDAO.java new file mode 100644 index 0000000000..d80b2188a4 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRollingTaskRecordDAO.java @@ -0,0 +1,86 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.StepInstanceRollingTask; +import com.tencent.bk.job.execute.model.tables.records.StepInstanceRollingTaskRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.Table; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * step_instance_rolling_task DAO + */ +public class StepInstanceRollingTaskRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final StepInstanceRollingTask TABLE = StepInstanceRollingTask.STEP_INSTANCE_ROLLING_TASK; + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(StepInstanceRollingTask.STEP_INSTANCE_ROLLING_TASK.TASK_INSTANCE_ID.asc()); + ORDER_FIELDS.add(StepInstanceRollingTask.STEP_INSTANCE_ROLLING_TASK.ID.asc()); + } + + public StepInstanceRollingTaskRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE.getName()); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.TASK_INSTANCE_ID.ge(lastRecord.getTaskInstanceId())); + conditions.add(TABLE.ID.gt(lastRecord.getId())); + return conditions; + } + ); + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceScriptRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceScriptRecordDAO.java new file mode 100644 index 0000000000..2b41c56695 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceScriptRecordDAO.java @@ -0,0 +1,87 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.StepInstanceScript; +import com.tencent.bk.job.execute.model.tables.records.StepInstanceScriptRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.Table; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * step_instance_script DAO + */ +public class StepInstanceScriptRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final StepInstanceScript TABLE = StepInstanceScript.STEP_INSTANCE_SCRIPT; + + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(StepInstanceScript.STEP_INSTANCE_SCRIPT.TASK_INSTANCE_ID.asc()); + ORDER_FIELDS.add(StepInstanceScript.STEP_INSTANCE_SCRIPT.STEP_INSTANCE_ID.asc()); + } + + + public StepInstanceScriptRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE.getName()); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.TASK_INSTANCE_ID.ge(lastRecord.getTaskInstanceId())); + conditions.add(TABLE.STEP_INSTANCE_ID.gt(lastRecord.getStepInstanceId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceVariableRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceVariableRecordDAO.java new file mode 100644 index 0000000000..74f207cbf4 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceVariableRecordDAO.java @@ -0,0 +1,85 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.StepInstanceVariable; +import com.tencent.bk.job.execute.model.tables.records.StepInstanceVariableRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.Table; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * step_instance_variable DAO + */ +public class StepInstanceVariableRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final StepInstanceVariable TABLE = StepInstanceVariable.STEP_INSTANCE_VARIABLE; + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(StepInstanceVariable.STEP_INSTANCE_VARIABLE.TASK_INSTANCE_ID.asc()); + ORDER_FIELDS.add(StepInstanceVariable.STEP_INSTANCE_VARIABLE.ID.asc()); + } + + public StepInstanceVariableRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE.getName()); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.TASK_INSTANCE_ID.ge(lastRecord.getTaskInstanceId())); + conditions.add(TABLE.ID.gt(lastRecord.getId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceHostRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceHostRecordDAO.java new file mode 100644 index 0000000000..6b746187da --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceHostRecordDAO.java @@ -0,0 +1,83 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.TaskInstanceHost; +import com.tencent.bk.job.execute.model.tables.records.TaskInstanceHostRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.Table; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +public class TaskInstanceHostRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final TaskInstanceHost TABLE = TaskInstanceHost.TASK_INSTANCE_HOST; + + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(TaskInstanceHost.TASK_INSTANCE_HOST.TASK_INSTANCE_ID.asc()); + ORDER_FIELDS.add(TaskInstanceHost.TASK_INSTANCE_HOST.HOST_ID.asc()); + } + + public TaskInstanceHostRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE.getName()); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.TASK_INSTANCE_ID.ge(lastRecord.getTaskInstanceId())); + conditions.add(TABLE.HOST_ID.gt(lastRecord.getHostId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceRecordDAO.java new file mode 100644 index 0000000000..388499f460 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceRecordDAO.java @@ -0,0 +1,119 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.TaskInstance; +import com.tencent.bk.job.execute.model.tables.records.TaskInstanceRecord; +import org.jooq.OrderField; +import org.jooq.Record; +import org.jooq.Record1; +import org.jooq.Result; +import org.jooq.SelectConditionStep; +import org.jooq.Table; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import static org.jooq.impl.DSL.min; + +/** + * task_instance DAO + */ +public class TaskInstanceRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final TaskInstance TABLE = TaskInstance.TASK_INSTANCE; + + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(TaskInstance.TASK_INSTANCE.ID.asc()); + } + + + public TaskInstanceRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE.getName()); + } + + @Override + public Table getTable() { + return TABLE; + } + + public Long getMinJobCreateTime() { + Record1 record = + dsl().select(min(TABLE.CREATE_TIME)) + .from(TABLE) + .fetchOne(); + if (record != null) { + Long minJobCreateTime = (Long) record.get(0); + if (minJobCreateTime != null) { + return minJobCreateTime; + } + } + return Long.MAX_VALUE; + } + + public List readSortedJobInstanceFromHotDB(Long fromTimestamp, + Long endTimestamp, + Long fromJobInstanceId, + int limit) { + SelectConditionStep selectConditionStep = + dsl().select() + .from(TABLE) + .where(TABLE.CREATE_TIME.greaterOrEqual(fromTimestamp)) + .and(TABLE.CREATE_TIME.lessThan(endTimestamp)); + if (fromJobInstanceId != null) { + selectConditionStep.and(TABLE.ID.greaterThan(fromJobInstanceId)); + } + Result result = selectConditionStep.orderBy(TABLE.CREATE_TIME.asc(), TABLE.ID.asc()) + .limit(limit) + .fetch(); + return result.into(TABLE); + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.ID; + } + + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createOneQueryResultSet( + this, + jobInstanceIds, + readRowLimit + ); + } + + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceVariableRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceVariableRecordDAO.java new file mode 100644 index 0000000000..41553e3b5a --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceVariableRecordDAO.java @@ -0,0 +1,86 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.TaskInstanceVariable; +import com.tencent.bk.job.execute.model.tables.records.TaskInstanceVariableRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.Table; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * task_instance_variable DAO + */ +public class TaskInstanceVariableRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final TaskInstanceVariable TABLE = TaskInstanceVariable.TASK_INSTANCE_VARIABLE; + + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(TaskInstanceVariable.TASK_INSTANCE_VARIABLE.TASK_INSTANCE_ID.asc()); + ORDER_FIELDS.add(TaskInstanceVariable.TASK_INSTANCE_VARIABLE.ID.asc()); + } + + public TaskInstanceVariableRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE.getName()); + } + + @Override + public Table getTable() { + return TABLE; + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.TASK_INSTANCE_ID.ge(lastRecord.getTaskInstanceId())); + conditions.add(TABLE.ID.gt(lastRecord.getId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/AbstractJobInstanceSubTableArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/AbstractJobInstanceSubTableArchiver.java new file mode 100644 index 0000000000..8b014c75b7 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/AbstractJobInstanceSubTableArchiver.java @@ -0,0 +1,77 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.JobInstanceSubTableArchiver; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.AbstractJobInstanceHotRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.RecordResultSet; +import org.apache.commons.collections4.CollectionUtils; +import org.jooq.TableRecord; + +import java.util.List; + +public class AbstractJobInstanceSubTableArchiver implements JobInstanceSubTableArchiver { + + private final JobInstanceColdDAO jobInstanceColdDAO; + + private final AbstractJobInstanceHotRecordDAO> jobInstanceHotRecordDAO; + + private final ArchiveTablePropsStorage archiveTablePropsStorage; + + protected String tableName; + + public AbstractJobInstanceSubTableArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + AbstractJobInstanceHotRecordDAO> jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + this.jobInstanceColdDAO = jobInstanceColdDAO; + this.jobInstanceHotRecordDAO = jobInstanceHotRecordDAO; + this.archiveTablePropsStorage = archiveTablePropsStorage; + this.tableName = jobInstanceHotRecordDAO.getTable().getName(); + } + + @Override + public void backupRecords(List jobInstanceIds) { + RecordResultSet> recordResultSet = + jobInstanceHotRecordDAO.executeQuery(jobInstanceIds, + archiveTablePropsStorage.getReadRowLimit(tableName)); + while (recordResultSet.hasNext()) { + List> records = recordResultSet.getRecords(); + if (CollectionUtils.isNotEmpty(records)) { + jobInstanceColdDAO.batchInsert(records, + archiveTablePropsStorage.getBatchInsertRowSize(tableName)); + } + } + } + + @Override + public void deleteRecords(List jobInstanceIds) { + jobInstanceHotRecordDAO.deleteRecords(jobInstanceIds, + archiveTablePropsStorage.getDeleteLimitRowCount(tableName)); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/FileSourceTaskLogArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/FileSourceTaskLogArchiver.java new file mode 100644 index 0000000000..e7f9a37396 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/FileSourceTaskLogArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.FileSourceTaskLogRecordDAO; + + +public class FileSourceTaskLogArchiver extends AbstractJobInstanceSubTableArchiver { + + public FileSourceTaskLogArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + FileSourceTaskLogRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/FileSourceTaskLogArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/FileSourceTaskLogArchivist.java deleted file mode 100644 index ae86f24fe5..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/FileSourceTaskLogArchivist.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.archive.ArchiveTaskLock; -import com.tencent.bk.job.backup.config.ArchiveDBProperties; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.FileSourceTaskLogRecordDAO; -import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import com.tencent.bk.job.execute.model.tables.records.FileSourceTaskLogRecord; - -import java.util.concurrent.CountDownLatch; - -/** - * file_source_task_log 表归档 - */ -public class FileSourceTaskLogArchivist extends AbstractArchivist { - - public FileSourceTaskLogArchivist(FileSourceTaskLogRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService, - ArchiveDBProperties archiveDBProperties, - ArchiveTaskLock archiveTaskLock, - Long maxNeedArchiveId, - CountDownLatch countDownLatch, - ArchiveErrorTaskCounter archiveErrorTaskCounter) { - super(executeRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveId, - countDownLatch, - archiveErrorTaskCounter); - this.deleteIdStepSize = 1_000; - } - - -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileAgentTaskArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileAgentTaskArchiver.java new file mode 100644 index 0000000000..048828b976 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileAgentTaskArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.GseFileAgentTaskRecordDAO; + + +public class GseFileAgentTaskArchiver extends AbstractJobInstanceSubTableArchiver { + + public GseFileAgentTaskArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + GseFileAgentTaskRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileAgentTaskArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileAgentTaskArchivist.java deleted file mode 100644 index a31fdcd6e3..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileAgentTaskArchivist.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.archive.ArchiveTaskLock; -import com.tencent.bk.job.backup.config.ArchiveDBProperties; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.GseFileAgentTaskRecordDAO; -import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import com.tencent.bk.job.execute.model.tables.records.GseFileAgentTaskRecord; - -import java.util.concurrent.CountDownLatch; - -/** - * gse_file_agent_task 表归档 - */ -public class GseFileAgentTaskArchivist extends AbstractArchivist { - - public GseFileAgentTaskArchivist(GseFileAgentTaskRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService, - ArchiveDBProperties archiveDBProperties, - ArchiveTaskLock archiveTaskLock, - Long maxNeedArchiveId, - CountDownLatch countDownLatch, - ArchiveErrorTaskCounter archiveErrorTaskCounter) { - super(executeRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveId, - countDownLatch, - archiveErrorTaskCounter); - this.deleteIdStepSize = 1_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileExecuteObjTaskArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileExecuteObjTaskArchiver.java new file mode 100644 index 0000000000..16d7711ec4 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileExecuteObjTaskArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.GseFileExecuteObjTaskRecordDAO; + + +public class GseFileExecuteObjTaskArchiver extends AbstractJobInstanceSubTableArchiver { + + public GseFileExecuteObjTaskArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + GseFileExecuteObjTaskRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileExecuteObjTaskArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileExecuteObjTaskArchivist.java deleted file mode 100644 index a3e6e257fe..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileExecuteObjTaskArchivist.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.archive.ArchiveTaskLock; -import com.tencent.bk.job.backup.config.ArchiveDBProperties; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.GseFileExecuteObjTaskRecordDAO; -import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import com.tencent.bk.job.execute.model.tables.records.GseFileExecuteObjTaskRecord; - -import java.util.concurrent.CountDownLatch; - -/** - * gse_file_execute_obj_task 表归档 - */ -public class GseFileExecuteObjTaskArchivist extends AbstractArchivist { - - public GseFileExecuteObjTaskArchivist(GseFileExecuteObjTaskRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService, - ArchiveDBProperties archiveDBProperties, - ArchiveTaskLock archiveTaskLock, - Long maxNeedArchiveId, - CountDownLatch countDownLatch, - ArchiveErrorTaskCounter archiveErrorTaskCounter) { - super(executeRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveId, - countDownLatch, - archiveErrorTaskCounter); - this.deleteIdStepSize = 1_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptAgentTaskArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptAgentTaskArchiver.java new file mode 100644 index 0000000000..91570179af --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptAgentTaskArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.GseScriptAgentTaskRecordDAO; + + +public class GseScriptAgentTaskArchiver extends AbstractJobInstanceSubTableArchiver { + + public GseScriptAgentTaskArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + GseScriptAgentTaskRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptAgentTaskArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptAgentTaskArchivist.java deleted file mode 100644 index 646ab699b2..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptAgentTaskArchivist.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.archive.ArchiveTaskLock; -import com.tencent.bk.job.backup.config.ArchiveDBProperties; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.GseScriptAgentTaskRecordDAO; -import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import com.tencent.bk.job.execute.model.tables.records.GseScriptAgentTaskRecord; - -import java.util.concurrent.CountDownLatch; - -/** - * gse_script_agent_task 表归档 - */ -public class GseScriptAgentTaskArchivist extends AbstractArchivist { - - public GseScriptAgentTaskArchivist(GseScriptAgentTaskRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService, - ArchiveDBProperties archiveDBProperties, - ArchiveTaskLock archiveTaskLock, - Long maxNeedArchiveId, - CountDownLatch countDownLatch, - ArchiveErrorTaskCounter archiveErrorTaskCounter) { - super(executeRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveId, - countDownLatch, - archiveErrorTaskCounter); - this.deleteIdStepSize = 1_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptExecuteObjTaskArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptExecuteObjTaskArchiver.java new file mode 100644 index 0000000000..6ea443d6c0 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptExecuteObjTaskArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.GseScriptExecuteObjTaskRecordDAO; + + +public class GseScriptExecuteObjTaskArchiver extends AbstractJobInstanceSubTableArchiver { + + public GseScriptExecuteObjTaskArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + GseScriptExecuteObjTaskRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptExecuteObjTaskArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptExecuteObjTaskArchivist.java deleted file mode 100644 index 5dfd706744..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptExecuteObjTaskArchivist.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.archive.ArchiveTaskLock; -import com.tencent.bk.job.backup.config.ArchiveDBProperties; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.GseScriptExecuteObjTaskRecordDAO; -import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import com.tencent.bk.job.execute.model.tables.records.GseScriptExecuteObjTaskRecord; - -import java.util.concurrent.CountDownLatch; - -/** - * gse_script_execute_obj_task 表归档 - */ -public class GseScriptExecuteObjTaskArchivist extends AbstractArchivist { - - public GseScriptExecuteObjTaskArchivist(GseScriptExecuteObjTaskRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService, - ArchiveDBProperties archiveDBProperties, - ArchiveTaskLock archiveTaskLock, - Long maxNeedArchiveId, - CountDownLatch countDownLatch, - ArchiveErrorTaskCounter archiveErrorTaskCounter) { - super(executeRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveId, - countDownLatch, - archiveErrorTaskCounter); - this.deleteIdStepSize = 1_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskArchiver.java new file mode 100644 index 0000000000..39893aa59c --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.GseTaskRecordDAO; + + +public class GseTaskArchiver extends AbstractJobInstanceSubTableArchiver { + + public GseTaskArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + GseTaskRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/OperationLogArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/OperationLogArchiver.java new file mode 100644 index 0000000000..bec8563891 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/OperationLogArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.OperationLogRecordDAO; + + +public class OperationLogArchiver extends AbstractJobInstanceSubTableArchiver { + + public OperationLogArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + OperationLogRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/OperationLogArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/OperationLogArchivist.java deleted file mode 100644 index 643670fb3c..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/OperationLogArchivist.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.archive.ArchiveTaskLock; -import com.tencent.bk.job.backup.config.ArchiveDBProperties; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.OperationLogRecordDAO; -import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import com.tencent.bk.job.execute.model.tables.records.OperationLogRecord; - -import java.util.concurrent.CountDownLatch; - -/** - * operation_log 表归档 - */ -public class OperationLogArchivist extends AbstractArchivist { - - public OperationLogArchivist(OperationLogRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService, - ArchiveDBProperties archiveDBProperties, - ArchiveTaskLock archiveTaskLock, - Long maxNeedArchiveId, - CountDownLatch countDownLatch, - ArchiveErrorTaskCounter archiveErrorTaskCounter) { - super(executeRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveId, - countDownLatch, - archiveErrorTaskCounter); - this.deleteIdStepSize = 10_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/RollingConfigArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/RollingConfigArchiver.java new file mode 100644 index 0000000000..d441fd611c --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/RollingConfigArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.RollingConfigRecordDAO; + + +public class RollingConfigArchiver extends AbstractJobInstanceSubTableArchiver { + + public RollingConfigArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + RollingConfigRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceArchiver.java new file mode 100644 index 0000000000..5a02645343 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceRecordDAO; + + +public class StepInstanceArchiver extends AbstractJobInstanceSubTableArchiver { + + public StepInstanceArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + StepInstanceRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceArchivist.java deleted file mode 100644 index 91f511edde..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceArchivist.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.archive.ArchiveTaskLock; -import com.tencent.bk.job.backup.config.ArchiveDBProperties; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceRecordDAO; -import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import com.tencent.bk.job.execute.model.tables.records.StepInstanceRecord; - -import java.util.concurrent.CountDownLatch; - -/** - * step_instance 表归档 - */ -public class StepInstanceArchivist extends AbstractArchivist { - - public StepInstanceArchivist(StepInstanceRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService, - ArchiveDBProperties archiveDBProperties, - ArchiveTaskLock archiveTaskLock, - Long maxNeedArchiveId, - CountDownLatch countDownLatch, - ArchiveErrorTaskCounter archiveErrorTaskCounter) { - super(executeRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveId, - countDownLatch, - archiveErrorTaskCounter); - this.deleteIdStepSize = 10_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceConfirmArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceConfirmArchiver.java new file mode 100644 index 0000000000..7ba38e2ec9 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceConfirmArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceConfirmRecordDAO; + + +public class StepInstanceConfirmArchiver extends AbstractJobInstanceSubTableArchiver { + + public StepInstanceConfirmArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + StepInstanceConfirmRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceConfirmArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceConfirmArchivist.java deleted file mode 100644 index 00178bb6e9..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceConfirmArchivist.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.archive.ArchiveTaskLock; -import com.tencent.bk.job.backup.config.ArchiveDBProperties; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceConfirmRecordDAO; -import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import com.tencent.bk.job.execute.model.tables.records.StepInstanceConfirmRecord; - -import java.util.concurrent.CountDownLatch; - -/** - * step_instance_confirm 表归档 - */ -public class StepInstanceConfirmArchivist extends AbstractArchivist { - - public StepInstanceConfirmArchivist(StepInstanceConfirmRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService, - ArchiveDBProperties archiveDBProperties, - ArchiveTaskLock archiveTaskLock, - Long maxNeedArchiveId, - CountDownLatch countDownLatch, - ArchiveErrorTaskCounter archiveErrorTaskCounter) { - super(executeRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveId, - countDownLatch, - archiveErrorTaskCounter); - this.deleteIdStepSize = 100_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceFileArchiver.java similarity index 50% rename from src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskArchivist.java rename to src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceFileArchiver.java index bf70d4cc34..626dafc35d 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskArchivist.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceFileArchiver.java @@ -24,38 +24,22 @@ package com.tencent.bk.job.backup.archive.impl; -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.archive.ArchiveTaskLock; -import com.tencent.bk.job.backup.config.ArchiveDBProperties; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.GseTaskRecordDAO; -import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import com.tencent.bk.job.execute.model.tables.records.GseTaskRecord; +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceFileRecordDAO; -import java.util.concurrent.CountDownLatch; -/** - * gse_task 表归档 - */ -public class GseTaskArchivist extends AbstractArchivist { +public class StepInstanceFileArchiver extends AbstractJobInstanceSubTableArchiver { - public GseTaskArchivist(GseTaskRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService, - ArchiveDBProperties archiveDBProperties, - ArchiveTaskLock archiveTaskLock, - Long maxNeedArchiveId, - CountDownLatch countDownLatch, - ArchiveErrorTaskCounter archiveErrorTaskCounter) { - super(executeRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveId, - countDownLatch, - archiveErrorTaskCounter); - this.deleteIdStepSize = 10_000; + public StepInstanceFileArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + StepInstanceFileRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); } + } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceFileArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceFileArchivist.java deleted file mode 100644 index 647eee998a..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceFileArchivist.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.archive.ArchiveTaskLock; -import com.tencent.bk.job.backup.config.ArchiveDBProperties; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceFileRecordDAO; -import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import com.tencent.bk.job.execute.model.tables.records.StepInstanceFileRecord; - -import java.util.concurrent.CountDownLatch; - -/** - * step_instance_file 表归档 - */ -public class StepInstanceFileArchivist extends AbstractArchivist { - - public StepInstanceFileArchivist(StepInstanceFileRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService, - ArchiveDBProperties archiveDBProperties, - ArchiveTaskLock archiveTaskLock, - Long maxNeedArchiveId, - CountDownLatch countDownLatch, - ArchiveErrorTaskCounter archiveErrorTaskCounter) { - super(executeRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveId, - countDownLatch, - archiveErrorTaskCounter); - this.deleteIdStepSize = 10_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceRollingTaskArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceRollingTaskArchiver.java new file mode 100644 index 0000000000..21ae208d43 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceRollingTaskArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceRollingTaskRecordDAO; + + +public class StepInstanceRollingTaskArchiver extends AbstractJobInstanceSubTableArchiver { + + public StepInstanceRollingTaskArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + StepInstanceRollingTaskRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceRollingTaskArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceRollingTaskArchivist.java deleted file mode 100644 index 1896c7fdbe..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceRollingTaskArchivist.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.archive.ArchiveTaskLock; -import com.tencent.bk.job.backup.config.ArchiveDBProperties; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceRollingTaskRecordDAO; -import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import com.tencent.bk.job.execute.model.tables.records.StepInstanceRollingTaskRecord; - -import java.util.concurrent.CountDownLatch; - -/** - * step_instance_rolling_task 表归档 - */ -public class StepInstanceRollingTaskArchivist extends AbstractArchivist { - - public StepInstanceRollingTaskArchivist(StepInstanceRollingTaskRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService, - ArchiveDBProperties archiveDBProperties, - ArchiveTaskLock archiveTaskLock, - Long maxNeedArchiveId, - CountDownLatch countDownLatch, - ArchiveErrorTaskCounter archiveErrorTaskCounter) { - super(executeRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveId, - countDownLatch, - archiveErrorTaskCounter); - this.deleteIdStepSize = 100_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceScriptArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceScriptArchiver.java new file mode 100644 index 0000000000..5dc5f72e2c --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceScriptArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceScriptRecordDAO; + + +public class StepInstanceScriptArchiver extends AbstractJobInstanceSubTableArchiver { + + public StepInstanceScriptArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + StepInstanceScriptRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceScriptArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceScriptArchivist.java deleted file mode 100644 index c7c0cd66c3..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceScriptArchivist.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.archive.ArchiveTaskLock; -import com.tencent.bk.job.backup.config.ArchiveDBProperties; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceScriptRecordDAO; -import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import com.tencent.bk.job.execute.model.tables.records.StepInstanceScriptRecord; - -import java.util.concurrent.CountDownLatch; - -/** - * step_instance_script 表归档 - */ -public class StepInstanceScriptArchivist extends AbstractArchivist { - - public StepInstanceScriptArchivist(StepInstanceScriptRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService, - ArchiveDBProperties archiveDBProperties, - ArchiveTaskLock archiveTaskLock, - Long maxNeedArchiveId, - CountDownLatch countDownLatch, - ArchiveErrorTaskCounter archiveErrorTaskCounter) { - super(executeRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveId, - countDownLatch, - archiveErrorTaskCounter); - this.deleteIdStepSize = 10_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceVariableArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceVariableArchiver.java new file mode 100644 index 0000000000..b391fa5962 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceVariableArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceVariableRecordDAO; + + +public class StepInstanceVariableArchiver extends AbstractJobInstanceSubTableArchiver { + + public StepInstanceVariableArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + StepInstanceVariableRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceVariableArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceVariableArchivist.java deleted file mode 100644 index 6847e58629..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceVariableArchivist.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.archive.ArchiveTaskLock; -import com.tencent.bk.job.backup.config.ArchiveDBProperties; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceVariableRecordDAO; -import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import com.tencent.bk.job.execute.model.tables.records.StepInstanceVariableRecord; - -import java.util.concurrent.CountDownLatch; - -/** - * step_instance_variable 表归档 - */ -public class StepInstanceVariableArchivist extends AbstractArchivist { - - public StepInstanceVariableArchivist(StepInstanceVariableRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService, - ArchiveDBProperties archiveDBProperties, - ArchiveTaskLock archiveTaskLock, - Long maxNeedArchiveId, - CountDownLatch countDownLatch, - ArchiveErrorTaskCounter archiveErrorTaskCounter) { - super(executeRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveId, - countDownLatch, - archiveErrorTaskCounter); - this.deleteIdStepSize = 100_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceArchiver.java new file mode 100644 index 0000000000..6ffff81e9a --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceRecordDAO; + + +public class TaskInstanceArchiver extends AbstractJobInstanceSubTableArchiver { + + public TaskInstanceArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + TaskInstanceRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceArchivist.java deleted file mode 100644 index 7506b28606..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceArchivist.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.archive.ArchiveTaskLock; -import com.tencent.bk.job.backup.config.ArchiveDBProperties; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.TaskInstanceRecordDAO; -import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import com.tencent.bk.job.execute.model.tables.records.TaskInstanceRecord; - -import java.util.concurrent.CountDownLatch; - -/** - * task_instance 表归档 - */ -public class TaskInstanceArchivist extends AbstractArchivist { - - public TaskInstanceArchivist(TaskInstanceRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService, - ArchiveDBProperties archiveDBProperties, - ArchiveTaskLock archiveTaskLock, - Long maxNeedArchiveId, - CountDownLatch countDownLatch, - ArchiveErrorTaskCounter archiveErrorTaskCounter) { - super(executeRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveId, - countDownLatch, - archiveErrorTaskCounter); - this.deleteIdStepSize = 10_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceHostArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceHostArchiver.java new file mode 100644 index 0000000000..15d02ac59b --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceHostArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceHostRecordDAO; + + +public class TaskInstanceHostArchiver extends AbstractJobInstanceSubTableArchiver { + + public TaskInstanceHostArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + TaskInstanceHostRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceHostArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceHostArchivist.java deleted file mode 100644 index 76174eae1e..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceHostArchivist.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.archive.ArchiveTaskLock; -import com.tencent.bk.job.backup.config.ArchiveDBProperties; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.TaskInstanceHostRecordDAO; -import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import com.tencent.bk.job.execute.model.tables.records.TaskInstanceHostRecord; - -import java.util.concurrent.CountDownLatch; - -/** - * task_instance_host 表归档 - */ -public class TaskInstanceHostArchivist extends AbstractArchivist { - - public TaskInstanceHostArchivist(TaskInstanceHostRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService, - ArchiveDBProperties archiveDBProperties, - ArchiveTaskLock archiveTaskLock, - Long maxNeedArchiveId, - CountDownLatch countDownLatch, - ArchiveErrorTaskCounter archiveErrorTaskCounter) { - super(executeRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveId, - countDownLatch, - archiveErrorTaskCounter); - this.deleteIdStepSize = 10_000; - } - - protected boolean isBackupEnable(ArchiveDBProperties archiveDBProperties) { - // task_instance_host 属于检索表，用于根据主机ip查询作业执行历史。该表数据较大，归档慢，且归档之后冷数据并无价值，无需归档 - return false; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceVariableArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceVariableArchiver.java new file mode 100644 index 0000000000..9479187e1f --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceVariableArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceVariableRecordDAO; + + +public class TaskInstanceVariableArchiver extends AbstractJobInstanceSubTableArchiver { + + public TaskInstanceVariableArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + TaskInstanceVariableRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceVariableArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceVariableArchivist.java deleted file mode 100644 index dc9cbf8a6c..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceVariableArchivist.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.archive.ArchiveTaskLock; -import com.tencent.bk.job.backup.config.ArchiveDBProperties; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.TaskInstanceVariableRecordDAO; -import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import com.tencent.bk.job.execute.model.tables.records.TaskInstanceVariableRecord; - -import java.util.concurrent.CountDownLatch; - -/** - * task_instance_variable 表归档 - */ -public class TaskInstanceVariableArchivist extends AbstractArchivist { - - public TaskInstanceVariableArchivist(TaskInstanceVariableRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService, - ArchiveDBProperties archiveDBProperties, - ArchiveTaskLock archiveTaskLock, - Long maxNeedArchiveId, - CountDownLatch countDownLatch, - ArchiveErrorTaskCounter archiveErrorTaskCounter) { - super(executeRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveDBProperties, - archiveTaskLock, - maxNeedArchiveId, - countDownLatch, - archiveErrorTaskCounter); - this.deleteIdStepSize = 100_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveProgressDTO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveProgressDTO.java new file mode 100644 index 0000000000..c05924bc36 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveProgressDTO.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.model; + +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class ArchiveProgressDTO { + private String tableName; + /** + * 最后备份ID + */ + private Long lastBackupId; + + /** + * 最后删除ID + */ + private Long lastDeletedId; + private Long lastBackupTime; + private Long lastDeleteTime; +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTableProps.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTableProps.java new file mode 100644 index 0000000000..32f779f346 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTableProps.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.model; + +import lombok.Data; + +@Data +public class ArchiveTableProps { + /** + * 从 热 DB 表中读取归档数据，每次读取的记录数量限制 + */ + private Integer readRowLimit; + /** + * 写入归档数据到冷 DB，单批次最小行数 + */ + private Integer batchInsertRowSize; + /** + * 从热 DB 删除数据，每次删除的最大行数 + */ + private Integer deleteLimitRowCount; + + public ArchiveTableProps(Integer readRowLimit, Integer batchInsertRowSize, Integer deleteLimitRowCount) { + this.readRowLimit = readRowLimit; + this.batchInsertRowSize = batchInsertRowSize; + this.deleteLimitRowCount = deleteLimitRowCount; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTaskSummary.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTaskSummary.java new file mode 100644 index 0000000000..64bf0988ac --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTaskSummary.java @@ -0,0 +1,69 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@JsonInclude(JsonInclude.Include.NON_EMPTY) +@NoArgsConstructor +public class ArchiveTaskSummary { + /** + * 归档任务 + */ + private JobInstanceArchiveTaskInfo archiveTask; + + /** + * 归档模式 + */ + private String archiveMode; + + /** + * 归档总耗时（单位毫秒) + */ + private Long archiveCost; + + /** + * 归档的记录数量 + */ + private Long archivedRecordSize; + + /** + * 归档详细说明信息 + */ + private String message; + + /** + * 任务是否被跳过 + */ + private boolean skip; + + public ArchiveTaskSummary(JobInstanceArchiveTaskInfo archiveTask, String archiveMode) { + this.archiveTask = archiveTask; + this.archiveMode = archiveMode; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/BackupResult.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/BackupResult.java new file mode 100644 index 0000000000..ff7146fdc3 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/BackupResult.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.model; + +import lombok.Data; + +@Data +public class BackupResult { + /** + * 读取的记录数量 + */ + private long readRows; + /** + * 备份成功的记录数量 + */ + private long backupRows; + /** + * 读取耗时 + */ + private long readCost; + /** + * 备份写入耗时 + */ + private long writeCost; + + public BackupResult(long readRows, long backupRows, long readCost, long writeCost) { + this.readRows = readRows; + this.backupRows = backupRows; + this.readCost = readCost; + this.writeCost = writeCost; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DbDataNode.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DbDataNode.java new file mode 100644 index 0000000000..432ac28c68 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DbDataNode.java @@ -0,0 +1,95 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.model; + +import com.tencent.bk.job.backup.constant.DbDataNodeTypeEnum; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * db 数据节点信息 + */ +@Data +@NoArgsConstructor +public class DbDataNode { + /** + * db 数据节点类型 + */ + private DbDataNodeTypeEnum type; + /** + * 分库分表数据源前缀 + */ + private String dataSource; + /** + * 分库分表 db 实例位置索引，从 0 开始 + */ + private Integer dbIndex; + + /** + * 分库分表表位置索引，从 0 开始 + */ + private Integer tableIndex; + + public DbDataNode(DbDataNodeTypeEnum type, String dataSource, Integer dbIndex, Integer tableIndex) { + this.type = type; + this.dataSource = dataSource; + this.dbIndex = dbIndex; + this.tableIndex = tableIndex; + } + + public String toDataNodeId() { + switch (type) { + case STANDALONE: + return type + ":" + "ds_standalone"; + case SHARDING: + return type + ":" + dataSource + ":" + dbIndex + ":" + tableIndex; + default: + throw new IllegalArgumentException("Invalid DbDataNodeTypeEnum"); + } + } + + public static DbDataNode fromDataNodeId(String dataNodeId) { + String[] dataNodeParts = dataNodeId.split(":"); + DbDataNodeTypeEnum dbDataNodeType = DbDataNodeTypeEnum.valOf(Integer.parseInt(dataNodeParts[0])); + switch (dbDataNodeType) { + case STANDALONE: + return new DbDataNode(dbDataNodeType, null, null, null); + case SHARDING: + return new DbDataNode( + dbDataNodeType, + dataNodeParts[1], + Integer.parseInt(dataNodeParts[2]), + Integer.parseInt(dataNodeParts[3]) + ); + default: + throw new IllegalArgumentException("Invalid DbDataNodeId"); + } + } + + @Override + public DbDataNode clone() { + return new DbDataNode(type, dbIndex, tableIndex); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/JobInstanceArchiveTaskInfo.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/JobInstanceArchiveTaskInfo.java new file mode 100644 index 0000000000..8112c2d52d --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/JobInstanceArchiveTaskInfo.java @@ -0,0 +1,94 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.model; + +import com.tencent.bk.job.backup.constant.ArchiveTaskStatusEnum; +import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; +import lombok.Data; +import lombok.ToString; + +/** + * 作业实例归档任务信息 + */ +@Data +@ToString +public class JobInstanceArchiveTaskInfo { + /** + * 归档任务类型 + */ + private ArchiveTaskTypeEnum taskType; + /** + * 归档数据节点信息 + */ + private DbDataNode dbDataNode; + + /** + * 归档任务所在天.比如 20240806 + */ + private Integer day; + /** + * 归档任务所在小时。 1-24 + */ + private Integer hour; + /** + * 归档任务时间范围-from timestamp + */ + private Long fromTimestamp; + /** + * 归档任务时间范围-to timestamp + */ + private Long toTimestamp; + /** + * 归档进度 + */ + private TimeAndIdBasedArchiveProcess process; + /** + * 归档任务状态 + */ + private ArchiveTaskStatusEnum status; + /** + * 归档任务创建时间 + */ + private Long createTime; + /** + * 归档任务最后更新时间 + */ + private Long lastUpdateTime; + + public JobInstanceArchiveTaskInfo clone() { + JobInstanceArchiveTaskInfo archiveTask = new JobInstanceArchiveTaskInfo(); + archiveTask.setTaskType(taskType); + archiveTask.setDbDataNode(dbDataNode.clone()); + archiveTask.setDay(day); + archiveTask.setHour(hour); + archiveTask.setFromTimestamp(fromTimestamp); + archiveTask.setToTimestamp(toTimestamp); + archiveTask.setProcess(process.clone()); + archiveTask.setStatus(status); + archiveTask.setCreateTime(createTime); + archiveTask.setLastUpdateTime(lastUpdateTime); + return archiveTask; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/TimeAndIdBasedArchiveProcess.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/TimeAndIdBasedArchiveProcess.java new file mode 100644 index 0000000000..c26f393e28 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/TimeAndIdBasedArchiveProcess.java @@ -0,0 +1,64 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.model; + +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; + +/** + * 基于数据创建时间（primary)+id(secondary) 的归档进度 + */ +@Data +@NoArgsConstructor +public class TimeAndIdBasedArchiveProcess { + private Long timestamp; + private Long id; + + public TimeAndIdBasedArchiveProcess(Long timestamp, Long id) { + this.timestamp = timestamp; + this.id = id; + } + + @Override + public TimeAndIdBasedArchiveProcess clone() { + return new TimeAndIdBasedArchiveProcess(timestamp, id); + } + + public String toPersistentProcess() { + return timestamp + ":" + id; + } + + public static TimeAndIdBasedArchiveProcess fromPersistentProcess(String process) { + if (StringUtils.isEmpty(process)) { + return null; + } + String[] processParts = process.split(":"); + return new TimeAndIdBasedArchiveProcess( + Long.parseLong(processParts[0]), + Long.parseLong(processParts[1]) + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java index a150ba657d..6a44c2bc16 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java @@ -24,27 +24,49 @@ package com.tencent.bk.job.backup.config; +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; import com.tencent.bk.job.backup.archive.ArchiveTaskLock; +import com.tencent.bk.job.backup.archive.ArchiveTaskService; import com.tencent.bk.job.backup.archive.JobExecuteArchiveManage; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.ExecuteArchiveDAOImpl; -import com.tencent.bk.job.backup.dao.impl.FileSourceTaskLogRecordDAO; -import com.tencent.bk.job.backup.dao.impl.GseFileAgentTaskRecordDAO; -import com.tencent.bk.job.backup.dao.impl.GseFileExecuteObjTaskRecordDAO; -import com.tencent.bk.job.backup.dao.impl.GseScriptAgentTaskRecordDAO; -import com.tencent.bk.job.backup.dao.impl.GseScriptExecuteObjTaskRecordDAO; -import com.tencent.bk.job.backup.dao.impl.GseTaskRecordDAO; -import com.tencent.bk.job.backup.dao.impl.OperationLogRecordDAO; -import com.tencent.bk.job.backup.dao.impl.RollingConfigRecordDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceConfirmRecordDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceFileRecordDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceRecordDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceRollingTaskRecordDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceScriptRecordDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceVariableRecordDAO; -import com.tencent.bk.job.backup.dao.impl.TaskInstanceHostRecordDAO; -import com.tencent.bk.job.backup.dao.impl.TaskInstanceRecordDAO; -import com.tencent.bk.job.backup.dao.impl.TaskInstanceVariableRecordDAO; +import com.tencent.bk.job.backup.archive.JobInstanceArchiveTaskGenerator; +import com.tencent.bk.job.backup.archive.JobInstanceArchiveTaskScheduler; +import com.tencent.bk.job.backup.archive.dao.ExecuteArchiveDAO; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.FileSourceTaskLogRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.GseFileAgentTaskRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.GseFileExecuteObjTaskRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.GseScriptAgentTaskRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.GseScriptExecuteObjTaskRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.GseTaskRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.JobInstanceColdDAOImpl; +import com.tencent.bk.job.backup.archive.dao.impl.OperationLogRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.RollingConfigRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceConfirmRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceFileRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceRollingTaskRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceScriptRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceVariableRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceHostRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceVariableRecordDAO; +import com.tencent.bk.job.backup.archive.impl.FileSourceTaskLogArchiver; +import com.tencent.bk.job.backup.archive.impl.GseFileAgentTaskArchiver; +import com.tencent.bk.job.backup.archive.impl.GseFileExecuteObjTaskArchiver; +import com.tencent.bk.job.backup.archive.impl.GseScriptAgentTaskArchiver; +import com.tencent.bk.job.backup.archive.impl.GseScriptExecuteObjTaskArchiver; +import com.tencent.bk.job.backup.archive.impl.GseTaskArchiver; +import com.tencent.bk.job.backup.archive.impl.OperationLogArchiver; +import com.tencent.bk.job.backup.archive.impl.RollingConfigArchiver; +import com.tencent.bk.job.backup.archive.impl.StepInstanceArchiver; +import com.tencent.bk.job.backup.archive.impl.StepInstanceConfirmArchiver; +import com.tencent.bk.job.backup.archive.impl.StepInstanceFileArchiver; +import com.tencent.bk.job.backup.archive.impl.StepInstanceRollingTaskArchiver; +import com.tencent.bk.job.backup.archive.impl.StepInstanceScriptArchiver; +import com.tencent.bk.job.backup.archive.impl.StepInstanceVariableArchiver; +import com.tencent.bk.job.backup.archive.impl.TaskInstanceArchiver; +import com.tencent.bk.job.backup.archive.impl.TaskInstanceHostArchiver; +import com.tencent.bk.job.backup.archive.impl.TaskInstanceVariableArchiver; import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; import com.tencent.bk.job.backup.service.ArchiveProgressService; import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; @@ -69,7 +91,7 @@ @Configuration @EnableScheduling @Slf4j -@EnableConfigurationProperties(ArchiveDBProperties.class) +@EnableConfigurationProperties(ArchiveProperties.class) @Import({ExecuteDbConfiguration.class, ExecuteBackupDbConfiguration.class}) public class ArchiveConfiguration { @@ -87,6 +109,18 @@ public TaskInstanceRecordDAO taskInstanceRecordDAO( return new TaskInstanceRecordDAO(dslContextProvider); } + @Bean + public TaskInstanceArchiver taskInstanceArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + TaskInstanceRecordDAO taskInstanceRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new TaskInstanceArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + taskInstanceRecordDAO, + archiveTablePropsStorage); + } + @Bean(name = "stepInstanceRecordDAO") public StepInstanceRecordDAO stepInstanceRecordDAO( @Qualifier("job-execute-dsl-context-provider") DSLContextProvider dslContextProvider) { @@ -94,6 +128,18 @@ public StepInstanceRecordDAO stepInstanceRecordDAO( return new StepInstanceRecordDAO(dslContextProvider); } + @Bean + public StepInstanceArchiver stepInstanceArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + StepInstanceRecordDAO stepInstanceRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new StepInstanceArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + stepInstanceRecordDAO, + archiveTablePropsStorage); + } + @Bean(name = "stepInstanceScriptRecordDAO") public StepInstanceScriptRecordDAO stepInstanceScriptRecordDAO( @Qualifier("job-execute-dsl-context-provider") DSLContextProvider dslContextProvider) { @@ -101,6 +147,18 @@ public StepInstanceScriptRecordDAO stepInstanceScriptRecordDAO( return new StepInstanceScriptRecordDAO(dslContextProvider); } + @Bean + public StepInstanceScriptArchiver stepInstanceScriptArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + StepInstanceScriptRecordDAO stepInstanceScriptRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new StepInstanceScriptArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + stepInstanceScriptRecordDAO, + archiveTablePropsStorage); + } + @Bean(name = "stepInstanceFileRecordDAO") public StepInstanceFileRecordDAO stepInstanceFileRecordDAO( @Qualifier("job-execute-dsl-context-provider") DSLContextProvider dslContextProvider) { @@ -108,6 +166,18 @@ public StepInstanceFileRecordDAO stepInstanceFileRecordDAO( return new StepInstanceFileRecordDAO(dslContextProvider); } + @Bean + public StepInstanceFileArchiver stepInstanceFileArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + StepInstanceFileRecordDAO stepInstanceFileRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new StepInstanceFileArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + stepInstanceFileRecordDAO, + archiveTablePropsStorage); + } + @Bean(name = "stepInstanceConfirmRecordDAO") public StepInstanceConfirmRecordDAO stepInstanceConfirmRecordDAO( @Qualifier("job-execute-dsl-context-provider") DSLContextProvider dslContextProvider) { @@ -115,6 +185,18 @@ public StepInstanceConfirmRecordDAO stepInstanceConfirmRecordDAO( return new StepInstanceConfirmRecordDAO(dslContextProvider); } + @Bean + public StepInstanceConfirmArchiver stepInstanceConfirmArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + StepInstanceConfirmRecordDAO stepInstanceConfirmRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new StepInstanceConfirmArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + stepInstanceConfirmRecordDAO, + archiveTablePropsStorage); + } + @Bean(name = "stepInstanceVariableRecordDAO") public StepInstanceVariableRecordDAO stepInstanceVariableRecordDAO( @Qualifier("job-execute-dsl-context-provider") DSLContextProvider dslContextProvider) { @@ -122,6 +204,18 @@ public StepInstanceVariableRecordDAO stepInstanceVariableRecordDAO( return new StepInstanceVariableRecordDAO(dslContextProvider); } + @Bean + public StepInstanceVariableArchiver stepInstanceVariableArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + StepInstanceVariableRecordDAO stepInstanceVariableRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new StepInstanceVariableArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + stepInstanceVariableRecordDAO, + archiveTablePropsStorage); + } + @Bean(name = "taskInstanceVariableRecordDAO") public TaskInstanceVariableRecordDAO taskInstanceVariableRecordDAO( @Qualifier("job-execute-dsl-context-provider") DSLContextProvider dslContextProvider) { @@ -129,6 +223,18 @@ public TaskInstanceVariableRecordDAO taskInstanceVariableRecordDAO( return new TaskInstanceVariableRecordDAO(dslContextProvider); } + @Bean + public TaskInstanceVariableArchiver taskInstanceVariableArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + TaskInstanceVariableRecordDAO taskInstanceVariableRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new TaskInstanceVariableArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + taskInstanceVariableRecordDAO, + archiveTablePropsStorage); + } + @Bean(name = "operationLogRecordDAO") public OperationLogRecordDAO operationLogRecordDAO( @Qualifier("job-execute-dsl-context-provider") DSLContextProvider dslContextProvider) { @@ -136,6 +242,18 @@ public OperationLogRecordDAO operationLogRecordDAO( return new OperationLogRecordDAO(dslContextProvider); } + @Bean + public OperationLogArchiver taskInstanceVariableArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + OperationLogRecordDAO operationLogRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new OperationLogArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + operationLogRecordDAO, + archiveTablePropsStorage); + } + @Bean(name = "fileSourceTaskLogRecordDAO") public FileSourceTaskLogRecordDAO fileSourceTaskLogRecordDAO( @Qualifier("job-execute-dsl-context-provider") DSLContextProvider dslContextProvider) { @@ -143,6 +261,18 @@ public FileSourceTaskLogRecordDAO fileSourceTaskLogRecordDAO( return new FileSourceTaskLogRecordDAO(dslContextProvider); } + @Bean + public FileSourceTaskLogArchiver taskInstanceVariableArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + FileSourceTaskLogRecordDAO fileSourceTaskLogRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new FileSourceTaskLogArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + fileSourceTaskLogRecordDAO, + archiveTablePropsStorage); + } + @Bean(name = "gseTaskRecordDAO") public GseTaskRecordDAO gseTaskRecordDAO( @Qualifier("job-execute-dsl-context-provider") DSLContextProvider dslContextProvider) { @@ -150,6 +280,18 @@ public GseTaskRecordDAO gseTaskRecordDAO( return new GseTaskRecordDAO(dslContextProvider); } + @Bean + public GseTaskArchiver gseTaskArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + GseTaskRecordDAO gseTaskRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new GseTaskArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + gseTaskRecordDAO, + archiveTablePropsStorage); + } + @Bean(name = "gseScriptAgentTaskRecordDAO") public GseScriptAgentTaskRecordDAO gseScriptAgentTaskRecordDAO( @Qualifier("job-execute-dsl-context-provider") DSLContextProvider dslContextProvider) { @@ -157,6 +299,18 @@ public GseScriptAgentTaskRecordDAO gseScriptAgentTaskRecordDAO( return new GseScriptAgentTaskRecordDAO(dslContextProvider); } + @Bean + public GseScriptAgentTaskArchiver gseScriptAgentTaskArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + GseScriptAgentTaskRecordDAO gseScriptAgentTaskRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new GseScriptAgentTaskArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + gseScriptAgentTaskRecordDAO, + archiveTablePropsStorage); + } + @Bean(name = "gseFileAgentTaskRecordDAO") public GseFileAgentTaskRecordDAO gseFileAgentTaskRecordDAO( @Qualifier("job-execute-dsl-context-provider") DSLContextProvider dslContextProvider) { @@ -164,6 +318,18 @@ public GseFileAgentTaskRecordDAO gseFileAgentTaskRecordDAO( return new GseFileAgentTaskRecordDAO(dslContextProvider); } + @Bean + public GseFileAgentTaskArchiver gseFileAgentTaskArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + GseFileAgentTaskRecordDAO gseFileAgentTaskRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new GseFileAgentTaskArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + gseFileAgentTaskRecordDAO, + archiveTablePropsStorage); + } + @Bean(name = "gseScriptExecuteObjTaskRecordDAO") public GseScriptExecuteObjTaskRecordDAO gseScriptExecuteObjTaskRecordDAO( @Qualifier("job-execute-dsl-context-provider") DSLContextProvider dslContextProvider) { @@ -171,6 +337,18 @@ public GseScriptExecuteObjTaskRecordDAO gseScriptExecuteObjTaskRecordDAO( return new GseScriptExecuteObjTaskRecordDAO(dslContextProvider); } + @Bean + public GseScriptExecuteObjTaskArchiver gseScriptExecuteObjTaskArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + GseScriptExecuteObjTaskRecordDAO gseScriptExecuteObjTaskRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new GseScriptExecuteObjTaskArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + gseScriptExecuteObjTaskRecordDAO, + archiveTablePropsStorage); + } + @Bean(name = "gseFileExecuteObjTaskRecordDAO") public GseFileExecuteObjTaskRecordDAO gseFileExecuteObjTaskRecordDAO( @Qualifier("job-execute-dsl-context-provider") DSLContextProvider dslContextProvider) { @@ -178,6 +356,18 @@ public GseFileExecuteObjTaskRecordDAO gseFileExecuteObjTaskRecordDAO( return new GseFileExecuteObjTaskRecordDAO(dslContextProvider); } + @Bean + public GseFileExecuteObjTaskArchiver gseScriptExecuteObjTaskArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + GseFileExecuteObjTaskRecordDAO gseFileExecuteObjTaskRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new GseFileExecuteObjTaskArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + gseFileExecuteObjTaskRecordDAO, + archiveTablePropsStorage); + } + @Bean(name = "stepInstanceRollingTaskRecordDAO") public StepInstanceRollingTaskRecordDAO stepInstanceRollingTaskRecordDAO( @Qualifier("job-execute-dsl-context-provider") DSLContextProvider dslContextProvider) { @@ -185,6 +375,18 @@ public StepInstanceRollingTaskRecordDAO stepInstanceRollingTaskRecordDAO( return new StepInstanceRollingTaskRecordDAO(dslContextProvider); } + @Bean + public StepInstanceRollingTaskArchiver stepInstanceRollingTaskArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + StepInstanceRollingTaskRecordDAO stepInstanceRollingTaskRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new StepInstanceRollingTaskArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + stepInstanceRollingTaskRecordDAO, + archiveTablePropsStorage); + } + @Bean(name = "rollingConfigRecordDAO") public RollingConfigRecordDAO rollingConfigRecordDAO( @Qualifier("job-execute-dsl-context-provider") DSLContextProvider dslContextProvider) { @@ -192,6 +394,18 @@ public RollingConfigRecordDAO rollingConfigRecordDAO( return new RollingConfigRecordDAO(dslContextProvider); } + @Bean + public RollingConfigArchiver rollingConfigArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + RollingConfigRecordDAO rollingConfigRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new RollingConfigArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + rollingConfigRecordDAO, + archiveTablePropsStorage); + } + @Bean(name = "taskInstanceHostRecordDAO") public TaskInstanceHostRecordDAO taskInstanceHostRecordDAO( @Qualifier("job-execute-dsl-context-provider") DSLContextProvider dslContextProvider) { @@ -199,18 +413,31 @@ public TaskInstanceHostRecordDAO taskInstanceHostRecordDAO( return new TaskInstanceHostRecordDAO(dslContextProvider); } + @Bean + public TaskInstanceHostArchiver taskInstanceHostArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + TaskInstanceHostRecordDAO taskInstanceHostRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new TaskInstanceHostArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + taskInstanceHostRecordDAO, + archiveTablePropsStorage); + } + } /** - * job-execute 归档数据备份 DB 配置 + * job-execute 归档冷 DB 配置 */ @Configuration - @Conditional(ExecuteBackupDbConfiguration.JobExecuteBackupDbInitCondition.class) + @Conditional(ExecuteBackupDbConfiguration.JobExecuteColdDbInitCondition.class) public static class ExecuteBackupDAOConfig { @Bean(name = "execute-archive-dao") - public ExecuteArchiveDAO executeArchiveDAO(@Qualifier("job-execute-archive-dsl-context") DSLContext context) { + public JobInstanceColdDAOImpl jobInstanceColdDAO( + @Qualifier("job-execute-archive-dsl-context") DSLContext context) { log.info("Init ExecuteArchiveDAO"); - return new ExecuteArchiveDAOImpl(context); + return new JobInstanceColdDAOImpl(context); } } @@ -222,6 +449,31 @@ public ArchiveTaskLock archiveTaskLock(StringRedisTemplate redisTemplate) { } + @Bean + @ConditionalOnExpression("${job.backup.archive.execute.enabled:false}") + public JobInstanceArchiveTaskGenerator jobInstanceArchiveTaskGenerator(ArchiveTaskService archiveTaskService, + TaskInstanceRecordDAO taskInstanceRecordDAO, + ArchiveProperties archiveProperties) { + return new JobInstanceArchiveTaskGenerator( + archiveTaskService, + taskInstanceRecordDAO, + archiveProperties + ); + } + + @Bean + @ConditionalOnExpression("${job.backup.archive.execute.enabled:false}") + public JobInstanceArchiveTaskScheduler jobInstanceArchiveTaskScheduler(ArchiveTaskService archiveTaskService, + TaskInstanceRecordDAO taskInstanceRecordDAO, + ArchiveProperties archiveProperties) { + return new JobInstanceArchiveTaskScheduler( + archiveTaskService, + taskInstanceRecordDAO, + archiveProperties, + + ); + } + @Bean @ConditionalOnExpression("${job.backup.archive.execute.enabled:false}") public JobExecuteArchiveManage jobExecuteArchiveManage( @@ -245,7 +497,7 @@ public JobExecuteArchiveManage jobExecuteArchiveManage( ObjectProvider executeArchiveDAOObjectProvider, ArchiveProgressService archiveProgressService, @Qualifier("archiveExecutor") ExecutorService archiveExecutor, - ArchiveDBProperties archiveDBProperties, + ArchiveProperties archiveProperties, ArchiveTaskLock archiveTaskLock, ArchiveErrorTaskCounter archiveErrorTaskCounter) { @@ -270,7 +522,7 @@ public JobExecuteArchiveManage jobExecuteArchiveManage( taskInstanceHostRecordDAOObjectProvider.getIfAvailable(), executeArchiveDAOObjectProvider.getIfAvailable(), archiveProgressService, - archiveDBProperties, + archiveProperties, archiveExecutor, archiveTaskLock, archiveErrorTaskCounter); diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveDBProperties.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveProperties.java similarity index 75% rename from src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveDBProperties.java rename to src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveProperties.java index 39243ac35f..e28322072a 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveDBProperties.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveProperties.java @@ -31,13 +31,14 @@ import lombok.ToString; import org.springframework.boot.context.properties.ConfigurationProperties; +import java.util.List; import java.util.Map; @Getter @Setter @ToString @ConfigurationProperties(prefix = "job.backup.archive.execute") -public class ArchiveDBProperties { +public class ArchiveProperties { /** * 是否启用 DB 归档 */ @@ -55,6 +56,11 @@ public class ArchiveDBProperties { */ private String cron; + /** + * 归档任务并行执行数量 + */ + private Integer concurrent = 10; + /** * DB数据保留天数 */ @@ -82,6 +88,11 @@ public class ArchiveDBProperties { private Map tableConfigs; + /** + * 归档任务配置 + */ + private ArchiveTasksConfig tasks; + @Data public static class TableConfig { /** @@ -105,5 +116,46 @@ public static class TableConfig { private Integer readRowLimit; } + @Data + public static class ArchiveTasksConfig { + /** + * 作业实例数据归档任务配置 + */ + private ArchiveTaskConfig taskInstance; + } + + @Data + public static class ArchiveTaskConfig { + /** + * 数据源模式 + * + * @see com.tencent.bk.job.common.mysql.dynamic.ds.DataSourceMode + */ + private String dataSourceMode; + + /** + * 要归档的分库分表数据节点配置 + */ + private List shardingDataNodes; + + } + + + @Data + public static class ShardingDataNode { + /** + * 数据源名称 + */ + private String dataSourceName; + /** + * 分库数量 + */ + private Integer dbCount; + /** + * 分表数量 + */ + private Integer tableCount; + } + } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteBackupDbConfiguration.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteBackupDbConfiguration.java index 65e5c0a052..147627517f 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteBackupDbConfiguration.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteBackupDbConfiguration.java @@ -44,7 +44,7 @@ import javax.sql.DataSource; @Configuration(value = "executeBackupDbConfiguration") -@Conditional(ExecuteBackupDbConfiguration.JobExecuteBackupDbInitCondition.class) +@Conditional(ExecuteBackupDbConfiguration.JobExecuteColdDbInitCondition.class) public class ExecuteBackupDbConfiguration { @Qualifier("job-execute-archive-source") @@ -75,8 +75,8 @@ public ConnectionProvider executeArchiveConnectionProvider( return new DataSourceConnectionProvider(dataSource); } - static class JobExecuteBackupDbInitCondition extends AllNestedConditions { - public JobExecuteBackupDbInitCondition() { + static class JobExecuteColdDbInitCondition extends AllNestedConditions { + public JobExecuteColdDbInitCondition() { super(ConfigurationPhase.PARSE_CONFIGURATION); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/ArchiveTaskStatusEnum.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/ArchiveTaskStatusEnum.java new file mode 100644 index 0000000000..da20a137ae --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/ArchiveTaskStatusEnum.java @@ -0,0 +1,69 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.constant; + +import lombok.Getter; + +/** + * 归档任务状态 + */ +@Getter +public enum ArchiveTaskStatusEnum { + /** + * 待执行 + */ + PENDING(0), + /** + * 暂停 + */ + SUSPENDED(1), + /** + * 运行中 + */ + RUNNING(2), + /** + * 失败 + */ + FAIL(3), + /** + * 成功 + */ + SUCCESS(4); + + private final int status; + + ArchiveTaskStatusEnum(int status) { + this.status = status; + } + + public static ArchiveTaskStatusEnum valOf(int status) { + for (ArchiveTaskStatusEnum taskStatus : values()) { + if (taskStatus.getStatus() == status) { + return taskStatus; + } + } + throw new IllegalArgumentException("No ArchiveTaskStatusEnum constant: " + status); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/ArchiveTaskTypeEnum.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/ArchiveTaskTypeEnum.java new file mode 100644 index 0000000000..e3d84d3fd8 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/ArchiveTaskTypeEnum.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.constant; + +import lombok.Getter; + +@Getter +public enum ArchiveTaskTypeEnum { + /** + * 作业实例数据归档 + */ + JOB_INSTANCE(1), + /** + * 作业实例按业务冗余数据归档 + */ + JOB_INSTANCE_APP(2); + + private final int type; + + ArchiveTaskTypeEnum(int type) { + this.type = type; + } + + public static ArchiveTaskTypeEnum valOf(int type) { + for (ArchiveTaskTypeEnum taskType : values()) { + if (taskType.getType() == type) { + return taskType; + } + } + throw new IllegalArgumentException("No ArchiveTaskTypeEnum constant: " + type); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/DbDataNodeTypeEnum.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/DbDataNodeTypeEnum.java new file mode 100644 index 0000000000..3edf2663e0 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/DbDataNodeTypeEnum.java @@ -0,0 +1,57 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.constant; + +import lombok.Getter; + +/** + * DB 数据节点类型 + */ +@Getter +public enum DbDataNodeTypeEnum { + /** + * 单节点 + */ + STANDALONE(0), + /** + * 分库分表节点 + */ + SHARDING(1); + + private final int value; + + DbDataNodeTypeEnum(int value) { + this.value = value; + } + + public static DbDataNodeTypeEnum valOf(int type) { + for (DbDataNodeTypeEnum dataNodeType : values()) { + if (dataNodeType.getValue() == type) { + return dataNodeType; + } + } + throw new IllegalArgumentException("No DbDataNodeTypeEnum constant: " + type); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ExecuteRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ExecuteRecordDAO.java deleted file mode 100644 index 0271acf7a8..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ExecuteRecordDAO.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.tencent.bk.job.backup.dao; - -import org.jooq.Record; -import org.jooq.Table; -import org.jooq.TableField; - -import java.util.List; - -/** - * job-execute 微服务的表数据DAO - * - * @param 表记录 - */ -public interface ExecuteRecordDAO { - - /** - * 获取表 - * - * @return 表 - */ - Table getTable(); - - /** - * 获取用于查询归档记录的ID字段 - * - * @return ID字段 - */ - TableField getArchiveIdField(); - - /** - * 根据起始/结束ID获取表记录 - * - * @param start 起始ID(exclude) - * @param end 结束ID(include) - * @param offset 记录偏移数 - * @param limit 获取的记录数量 - * @return 表记录 - */ - List listRecords(Long start, Long end, Long offset, Long limit); - - /** - * 根据起始/结束ID删除表记录 - * - * @param start 起始ID(exclude) - * @param end 结束ID(include) - * @param maxLimitedDeleteRows 批量删除每批次limit - * @return 删除的记录数量 - */ - int deleteRecords(Long start, Long end, long maxLimitedDeleteRows); - - /** - * 获取表中最小归档ID - * - * @return id值。如果表中没有数据，那么返回 null - */ - Long getMinArchiveId(); -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/AbstractExecuteRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/AbstractExecuteRecordDAO.java deleted file mode 100644 index 5c13ee6972..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/AbstractExecuteRecordDAO.java +++ /dev/null @@ -1,95 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.backup.dao.ExecuteRecordDAO; -import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; -import org.apache.commons.collections4.CollectionUtils; -import org.jooq.Condition; -import org.jooq.DSLContext; -import org.jooq.OrderField; -import org.jooq.Record; -import org.jooq.Record1; -import org.jooq.Result; -import org.jooq.SelectConditionStep; -import org.jooq.Table; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -import static org.jooq.impl.DSL.min; - -public abstract class AbstractExecuteRecordDAO implements ExecuteRecordDAO { - - protected final DSLContextProvider dslContextProvider; - - protected final String tableName; - - public AbstractExecuteRecordDAO(DSLContextProvider dslContextProvider, String tableName) { - this.dslContextProvider = dslContextProvider; - this.tableName = tableName; - } - - @Override - public List listRecords(Long start, Long end, Long offset, Long limit) { - Result result = query(getTable(), buildConditions(start, end), offset, limit); - return result.into(getTable()); - } - - @Override - public int deleteRecords(Long start, Long end, long maxLimitedDeleteRows) { - return deleteWithLimit(getTable(), buildConditions(start, end), maxLimitedDeleteRows); - } - - private List buildConditions(Long start, Long end) { - List conditions = new ArrayList<>(); - conditions.add(getArchiveIdField().greaterThan(start)); - conditions.add(getArchiveIdField().lessOrEqual(end)); - return conditions; - } - - private int deleteWithLimit(Table table, List conditions, long maxLimitedDeleteRows) { - int totalDeleteRows = 0; - while (true) { - int deletedRows = dsl().delete(table).where(conditions).limit(maxLimitedDeleteRows).execute(); - totalDeleteRows += deletedRows; - if (deletedRows < maxLimitedDeleteRows) { - break; - } - } - return totalDeleteRows; - } - - private Result query(Table table, - List conditions, - Long offset, - Long limit) { - SelectConditionStep selectConditionStep = dsl().select() - .from(table) - .where(conditions); - - if (CollectionUtils.isNotEmpty(getListRecordsOrderFields())) { - return selectConditionStep.orderBy(getListRecordsOrderFields()).limit(offset, limit).fetch(); - } else { - return selectConditionStep.limit(offset, limit).fetch(); - } - } - - @Override - public Long getMinArchiveId() { - Record1 firstArchiveIdRecord = dsl().select(min(getArchiveIdField())).from(getTable()).fetchOne(); - if (firstArchiveIdRecord != null && firstArchiveIdRecord.get(0) != null) { - return (Long) firstArchiveIdRecord.get(0); - } - return null; - } - - protected DSLContext dsl() { - return this.dslContextProvider.get(tableName); - } - - public abstract Table getTable(); - - protected Collection> getListRecordsOrderFields() { - return null; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/FileSourceTaskLogRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/FileSourceTaskLogRecordDAO.java deleted file mode 100644 index 0cf829b36a..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/FileSourceTaskLogRecordDAO.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - - -import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; -import com.tencent.bk.job.execute.model.tables.FileSourceTaskLog; -import com.tencent.bk.job.execute.model.tables.records.FileSourceTaskLogRecord; -import org.jooq.OrderField; -import org.jooq.Table; -import org.jooq.TableField; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -/** - * file_source_task_log DAO - */ -public class FileSourceTaskLogRecordDAO extends AbstractExecuteRecordDAO { - - private static final FileSourceTaskLog TABLE = FileSourceTaskLog.FILE_SOURCE_TASK_LOG; - private static final List> ORDER_FIELDS = new ArrayList<>(); - - static { - ORDER_FIELDS.add(FileSourceTaskLog.FILE_SOURCE_TASK_LOG.STEP_INSTANCE_ID.asc()); - ORDER_FIELDS.add(FileSourceTaskLog.FILE_SOURCE_TASK_LOG.EXECUTE_COUNT.asc()); - } - - public FileSourceTaskLogRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.STEP_INSTANCE_ID; - } - - @Override - protected Collection> getListRecordsOrderFields() { - return ORDER_FIELDS; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseFileAgentTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseFileAgentTaskRecordDAO.java deleted file mode 100644 index 62499517f4..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseFileAgentTaskRecordDAO.java +++ /dev/null @@ -1,48 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; -import com.tencent.bk.job.execute.model.tables.GseFileAgentTask; -import com.tencent.bk.job.execute.model.tables.records.GseFileAgentTaskRecord; -import org.jooq.OrderField; -import org.jooq.Table; -import org.jooq.TableField; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -/** - * gse_file_agent_task DAO - */ -public class GseFileAgentTaskRecordDAO extends AbstractExecuteRecordDAO { - - private static final GseFileAgentTask TABLE = GseFileAgentTask.GSE_FILE_AGENT_TASK; - private static final List> ORDER_FIELDS = new ArrayList<>(); - - static { - ORDER_FIELDS.add(GseFileAgentTask.GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.asc()); - ORDER_FIELDS.add(GseFileAgentTask.GSE_FILE_AGENT_TASK.EXECUTE_COUNT.asc()); - ORDER_FIELDS.add(GseFileAgentTask.GSE_FILE_AGENT_TASK.BATCH.asc()); - ORDER_FIELDS.add(GseFileAgentTask.GSE_FILE_AGENT_TASK.MODE.asc()); - ORDER_FIELDS.add(GseFileAgentTask.GSE_FILE_AGENT_TASK.HOST_ID.asc()); - } - - public GseFileAgentTaskRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.STEP_INSTANCE_ID; - } - - @Override - protected Collection> getListRecordsOrderFields() { - return ORDER_FIELDS; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseFileExecuteObjTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseFileExecuteObjTaskRecordDAO.java deleted file mode 100644 index cefaea8064..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseFileExecuteObjTaskRecordDAO.java +++ /dev/null @@ -1,48 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; -import com.tencent.bk.job.execute.model.tables.GseFileExecuteObjTask; -import com.tencent.bk.job.execute.model.tables.records.GseFileExecuteObjTaskRecord; -import org.jooq.OrderField; -import org.jooq.Table; -import org.jooq.TableField; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -/** - * gse_file_execute_obj_task DAO - */ -public class GseFileExecuteObjTaskRecordDAO extends AbstractExecuteRecordDAO { - - private static final GseFileExecuteObjTask TABLE = GseFileExecuteObjTask.GSE_FILE_EXECUTE_OBJ_TASK; - private static final List> ORDER_FIELDS = new ArrayList<>(); - - static { - ORDER_FIELDS.add(GseFileExecuteObjTask.GSE_FILE_EXECUTE_OBJ_TASK.STEP_INSTANCE_ID.asc()); - ORDER_FIELDS.add(GseFileExecuteObjTask.GSE_FILE_EXECUTE_OBJ_TASK.EXECUTE_COUNT.asc()); - ORDER_FIELDS.add(GseFileExecuteObjTask.GSE_FILE_EXECUTE_OBJ_TASK.BATCH.asc()); - ORDER_FIELDS.add(GseFileExecuteObjTask.GSE_FILE_EXECUTE_OBJ_TASK.MODE.asc()); - ORDER_FIELDS.add(GseFileExecuteObjTask.GSE_FILE_EXECUTE_OBJ_TASK.EXECUTE_OBJ_ID.asc()); - } - - public GseFileExecuteObjTaskRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.TASK_INSTANCE_ID; - } - - @Override - protected Collection> getListRecordsOrderFields() { - return ORDER_FIELDS; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseScriptAgentTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseScriptAgentTaskRecordDAO.java deleted file mode 100644 index da73556aee..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseScriptAgentTaskRecordDAO.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; -import com.tencent.bk.job.execute.model.tables.GseScriptAgentTask; -import com.tencent.bk.job.execute.model.tables.records.GseScriptAgentTaskRecord; -import org.jooq.OrderField; -import org.jooq.Table; -import org.jooq.TableField; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -/** - * gse_script_agent_task DAO - */ -public class GseScriptAgentTaskRecordDAO extends AbstractExecuteRecordDAO { - - private static final GseScriptAgentTask TABLE = GseScriptAgentTask.GSE_SCRIPT_AGENT_TASK; - private static final List> ORDER_FIELDS = new ArrayList<>(); - - static { - ORDER_FIELDS.add(GseScriptAgentTask.GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.asc()); - ORDER_FIELDS.add(GseScriptAgentTask.GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.asc()); - ORDER_FIELDS.add(GseScriptAgentTask.GSE_SCRIPT_AGENT_TASK.BATCH.asc()); - ORDER_FIELDS.add(GseScriptAgentTask.GSE_SCRIPT_AGENT_TASK.HOST_ID.asc()); - } - - public GseScriptAgentTaskRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.STEP_INSTANCE_ID; - } - - @Override - protected Collection> getListRecordsOrderFields() { - return ORDER_FIELDS; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseScriptExecuteObjTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseScriptExecuteObjTaskRecordDAO.java deleted file mode 100644 index 834b57d797..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseScriptExecuteObjTaskRecordDAO.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; -import com.tencent.bk.job.execute.model.tables.GseScriptExecuteObjTask; -import com.tencent.bk.job.execute.model.tables.records.GseScriptExecuteObjTaskRecord; -import org.jooq.OrderField; -import org.jooq.Table; -import org.jooq.TableField; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -/** - * gse_script_execute_obj_task DAO - */ -public class GseScriptExecuteObjTaskRecordDAO extends AbstractExecuteRecordDAO { - - private static final GseScriptExecuteObjTask TABLE = GseScriptExecuteObjTask.GSE_SCRIPT_EXECUTE_OBJ_TASK; - private static final List> ORDER_FIELDS = new ArrayList<>(); - - static { - ORDER_FIELDS.add(GseScriptExecuteObjTask.GSE_SCRIPT_EXECUTE_OBJ_TASK.STEP_INSTANCE_ID.asc()); - ORDER_FIELDS.add(GseScriptExecuteObjTask.GSE_SCRIPT_EXECUTE_OBJ_TASK.EXECUTE_COUNT.asc()); - ORDER_FIELDS.add(GseScriptExecuteObjTask.GSE_SCRIPT_EXECUTE_OBJ_TASK.BATCH.asc()); - ORDER_FIELDS.add(GseScriptExecuteObjTask.GSE_SCRIPT_EXECUTE_OBJ_TASK.EXECUTE_OBJ_ID.asc()); - } - - public GseScriptExecuteObjTaskRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.TASK_INSTANCE_ID; - } - - @Override - protected Collection> getListRecordsOrderFields() { - return ORDER_FIELDS; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseTaskRecordDAO.java deleted file mode 100644 index b4c52b5a07..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseTaskRecordDAO.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; -import com.tencent.bk.job.execute.model.tables.GseTask; -import com.tencent.bk.job.execute.model.tables.records.GseTaskRecord; -import org.jooq.OrderField; -import org.jooq.Table; -import org.jooq.TableField; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -public class GseTaskRecordDAO extends AbstractExecuteRecordDAO { - - private static final GseTask TABLE = GseTask.GSE_TASK; - private static final List> ORDER_FIELDS = new ArrayList<>(); - - static { - ORDER_FIELDS.add(GseTask.GSE_TASK.STEP_INSTANCE_ID.asc()); - ORDER_FIELDS.add(GseTask.GSE_TASK.EXECUTE_COUNT.asc()); - ORDER_FIELDS.add(GseTask.GSE_TASK.BATCH.asc()); - } - - public GseTaskRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.STEP_INSTANCE_ID; - } - - @Override - protected Collection> getListRecordsOrderFields() { - return ORDER_FIELDS; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/OperationLogRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/OperationLogRecordDAO.java deleted file mode 100644 index b89ab356e7..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/OperationLogRecordDAO.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; -import com.tencent.bk.job.execute.model.tables.OperationLog; -import com.tencent.bk.job.execute.model.tables.records.OperationLogRecord; -import org.jooq.Table; -import org.jooq.TableField; - -/** - * operation_log DAO - */ -public class OperationLogRecordDAO extends AbstractExecuteRecordDAO { - - private static final OperationLog TABLE = OperationLog.OPERATION_LOG; - - public OperationLogRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.TASK_INSTANCE_ID; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/RollingConfigRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/RollingConfigRecordDAO.java deleted file mode 100644 index 7ecebad24d..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/RollingConfigRecordDAO.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; -import com.tencent.bk.job.execute.model.tables.RollingConfig; -import com.tencent.bk.job.execute.model.tables.records.RollingConfigRecord; -import org.jooq.Table; -import org.jooq.TableField; - -/** - * rolling_config DAO - */ -public class RollingConfigRecordDAO extends AbstractExecuteRecordDAO { - - private static final RollingConfig TABLE = RollingConfig.ROLLING_CONFIG; - - public RollingConfigRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.TASK_INSTANCE_ID; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceConfirmRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceConfirmRecordDAO.java deleted file mode 100644 index 11d7390a35..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceConfirmRecordDAO.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; -import com.tencent.bk.job.execute.model.tables.StepInstanceConfirm; -import com.tencent.bk.job.execute.model.tables.records.StepInstanceConfirmRecord; -import org.jooq.Table; -import org.jooq.TableField; - -/** - * step_instance_confirm DAO - */ -public class StepInstanceConfirmRecordDAO extends AbstractExecuteRecordDAO { - - - private static final StepInstanceConfirm TABLE = StepInstanceConfirm.STEP_INSTANCE_CONFIRM; - - public StepInstanceConfirmRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.STEP_INSTANCE_ID; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceFileRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceFileRecordDAO.java deleted file mode 100644 index 884758e778..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceFileRecordDAO.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; -import com.tencent.bk.job.execute.model.tables.StepInstanceFile; -import com.tencent.bk.job.execute.model.tables.records.StepInstanceFileRecord; -import org.jooq.Table; -import org.jooq.TableField; - -/** - * step_instance_file DAO - */ -public class StepInstanceFileRecordDAO extends AbstractExecuteRecordDAO { - - private static final StepInstanceFile TABLE = StepInstanceFile.STEP_INSTANCE_FILE; - - public StepInstanceFileRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.STEP_INSTANCE_ID; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceRecordDAO.java deleted file mode 100644 index 72c918af65..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceRecordDAO.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; -import com.tencent.bk.job.execute.model.tables.StepInstance; -import com.tencent.bk.job.execute.model.tables.records.StepInstanceRecord; -import org.jooq.Record1; -import org.jooq.Table; -import org.jooq.TableField; - -import static org.jooq.impl.DSL.max; - -/** - * step_instance DAO - */ -public class StepInstanceRecordDAO extends AbstractExecuteRecordDAO { - - private static final StepInstance TABLE = StepInstance.STEP_INSTANCE; - - public StepInstanceRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); - } - - @Override - public Table getTable() { - return TABLE; - } - - /** - * 获取作业实例ID范围内的步骤实例ID最大值 - * - * @param taskInstanceId 作业实例ID - * @return 步骤实例ID 最大值 - */ - public Long getMaxId(Long taskInstanceId) { - Record1 record = - dsl().select(max(TABLE.ID)) - .from(TABLE) - .where(TABLE.TASK_INSTANCE_ID.lessOrEqual(taskInstanceId)) - .fetchOne(); - if (record != null) { - Long maxId = (Long) record.get(0); - if (maxId != null) { - return maxId; - } - } - return 0L; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.ID; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceRollingTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceRollingTaskRecordDAO.java deleted file mode 100644 index 52d3ee8f2b..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceRollingTaskRecordDAO.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; -import com.tencent.bk.job.execute.model.tables.StepInstanceRollingTask; -import com.tencent.bk.job.execute.model.tables.records.StepInstanceRollingTaskRecord; -import org.jooq.OrderField; -import org.jooq.Table; -import org.jooq.TableField; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -/** - * step_instance_rolling_task DAO - */ -public class StepInstanceRollingTaskRecordDAO extends AbstractExecuteRecordDAO { - - private static final StepInstanceRollingTask TABLE = StepInstanceRollingTask.STEP_INSTANCE_ROLLING_TASK; - private static final List> ORDER_FIELDS = new ArrayList<>(); - - static { - ORDER_FIELDS.add(StepInstanceRollingTask.STEP_INSTANCE_ROLLING_TASK.STEP_INSTANCE_ID.asc()); - ORDER_FIELDS.add(StepInstanceRollingTask.STEP_INSTANCE_ROLLING_TASK.EXECUTE_COUNT.asc()); - ORDER_FIELDS.add(StepInstanceRollingTask.STEP_INSTANCE_ROLLING_TASK.BATCH.asc()); - } - - public StepInstanceRollingTaskRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.STEP_INSTANCE_ID; - } - - @Override - protected Collection> getListRecordsOrderFields() { - return ORDER_FIELDS; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceScriptRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceScriptRecordDAO.java deleted file mode 100644 index e2800028a9..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceScriptRecordDAO.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; -import com.tencent.bk.job.execute.model.tables.StepInstanceScript; -import com.tencent.bk.job.execute.model.tables.records.StepInstanceScriptRecord; -import org.jooq.Table; -import org.jooq.TableField; - -/** - * step_instance_script DAO - */ -public class StepInstanceScriptRecordDAO extends AbstractExecuteRecordDAO { - - private static final StepInstanceScript TABLE = StepInstanceScript.STEP_INSTANCE_SCRIPT; - - public StepInstanceScriptRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.STEP_INSTANCE_ID; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceVariableRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceVariableRecordDAO.java deleted file mode 100644 index b3a0694761..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceVariableRecordDAO.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; -import com.tencent.bk.job.execute.model.tables.StepInstanceVariable; -import com.tencent.bk.job.execute.model.tables.records.StepInstanceVariableRecord; -import org.jooq.OrderField; -import org.jooq.Table; -import org.jooq.TableField; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -/** - * step_instance_variable DAO - */ -public class StepInstanceVariableRecordDAO extends AbstractExecuteRecordDAO { - - private static final StepInstanceVariable TABLE = StepInstanceVariable.STEP_INSTANCE_VARIABLE; - private static final List> ORDER_FIELDS = new ArrayList<>(); - - static { - ORDER_FIELDS.add(StepInstanceVariable.STEP_INSTANCE_VARIABLE.STEP_INSTANCE_ID.asc()); - ORDER_FIELDS.add(StepInstanceVariable.STEP_INSTANCE_VARIABLE.EXECUTE_COUNT.asc()); - } - - public StepInstanceVariableRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.STEP_INSTANCE_ID; - } - - @Override - protected Collection> getListRecordsOrderFields() { - return ORDER_FIELDS; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceHostRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceHostRecordDAO.java deleted file mode 100644 index 3dd0d24f79..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceHostRecordDAO.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; -import com.tencent.bk.job.execute.model.tables.TaskInstanceHost; -import com.tencent.bk.job.execute.model.tables.records.TaskInstanceHostRecord; -import org.jooq.Table; -import org.jooq.TableField; - -public class TaskInstanceHostRecordDAO extends AbstractExecuteRecordDAO { - - private static final TaskInstanceHost TABLE = TaskInstanceHost.TASK_INSTANCE_HOST; - - public TaskInstanceHostRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.TASK_INSTANCE_ID; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceRecordDAO.java deleted file mode 100644 index eb83c8d719..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceRecordDAO.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; -import com.tencent.bk.job.execute.model.tables.TaskInstance; -import com.tencent.bk.job.execute.model.tables.records.TaskInstanceRecord; -import org.jooq.Record1; -import org.jooq.Table; -import org.jooq.TableField; - -import static org.jooq.impl.DSL.max; - -/** - * task_instance DAO - */ -public class TaskInstanceRecordDAO extends AbstractExecuteRecordDAO { - - private static final TaskInstance TABLE = TaskInstance.TASK_INSTANCE; - - public TaskInstanceRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.ID; - } - - public Long getMaxId(Long endTime) { - Record1 record = - dsl().select(max(TABLE.ID)) - .from(TABLE) - .where(TABLE.CREATE_TIME.lessOrEqual(endTime)) - .fetchOne(); - if (record != null) { - Long maxId = (Long) record.get(0); - if (maxId != null) { - return maxId; - } - } - return 0L; - } - - -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceVariableRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceVariableRecordDAO.java deleted file mode 100644 index d2c07730e3..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceVariableRecordDAO.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; -import com.tencent.bk.job.execute.model.tables.TaskInstanceVariable; -import com.tencent.bk.job.execute.model.tables.records.TaskInstanceVariableRecord; -import org.jooq.Table; -import org.jooq.TableField; - -/** - * task_instance_variable DAO - */ -public class TaskInstanceVariableRecordDAO extends AbstractExecuteRecordDAO { - - private static final TaskInstanceVariable TABLE = TaskInstanceVariable.TASK_INSTANCE_VARIABLE; - - public TaskInstanceVariableRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.TASK_INSTANCE_ID; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/ArchiveProgressServiceImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/ArchiveProgressServiceImpl.java index a57deaf192..e3f998beac 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/ArchiveProgressServiceImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/ArchiveProgressServiceImpl.java @@ -24,7 +24,7 @@ package com.tencent.bk.job.backup.service.impl; -import com.tencent.bk.job.backup.dao.ArchiveProgressDAO; +import com.tencent.bk.job.backup.archive.dao.ArchiveProgressDAO; import com.tencent.bk.job.backup.model.dto.ArchiveProgressDTO; import com.tencent.bk.job.backup.service.ArchiveProgressService; import lombok.extern.slf4j.Slf4j; diff --git a/support-files/kubernetes/charts/bk-job/templates/job-execute/configmap.yaml b/support-files/kubernetes/charts/bk-job/templates/job-execute/configmap.yaml index ca81f9f521..e780988df4 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-execute/configmap.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-execute/configmap.yaml @@ -193,6 +193,23 @@ data: {{- end }} {{- end }} {{- end }} + {{ if .Values.executeConfig.leaf.enabled }} + leaf: + driver-class-name: {{ include "job.jdbcMysqlDriverClass" . }} + type: com.zaxxer.hikari.HikariDataSource + jdbc-url: {{ include "job.jdbcMysqlScheme" . }}://{{- include "job.mariadb.host" . }}:{{- include "job.mariadb.port" . }}/job_leaf{{ include "job.mariadb.connection.properties" . }} + username: {{ include "job.mariadb.username" . }} + {{ if .Values.externalMariaDB.existingPasswordSecret }} + password: {{ .Values.externalMariaDB.existingPasswordKey | default "mariadb-password" | printf "${%s}" }} + {{- else -}} + password: ${mariadb-password} + {{- end }} + maximum-pool-size: 50 + minimum-idle: 10 + idle-timeout: 600000 + poolName: "job-leaf" + validationTimeout: 5000 + {{- end }} redis: {{- include "job.redis.config" . | indent 8 }} database: 0 diff --git a/support-files/kubernetes/charts/bk-job/values.yaml b/support-files/kubernetes/charts/bk-job/values.yaml index b27febd89a..478086d881 100644 --- a/support-files/kubernetes/charts/bk-job/values.yaml +++ b/support-files/kubernetes/charts/bk-job/values.yaml @@ -1235,6 +1235,9 @@ backupConfig: batchInsertRowSize: 1000 # 每次执行删除的最大行数。为了减少 MySQL 事务时长，DB 性能受限时可适当降低该值 deleteRowLimit: 1000 + tasks: + job_instance: + image: registry: hub.bktencent.com repository: blueking/job-backup From 6d9a6c24b2de9b765903cb96c120ed7e754d8bdf Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Sun, 17 Nov 2024 18:07:35 +0800 Subject: [PATCH 24/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...job_backup_20241115-1000_V3.11.2_mysql.sql | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 support-files/sql/job-backup/0005_job_backup_20241115-1000_V3.11.2_mysql.sql diff --git a/support-files/sql/job-backup/0005_job_backup_20241115-1000_V3.11.2_mysql.sql b/support-files/sql/job-backup/0005_job_backup_20241115-1000_V3.11.2_mysql.sql new file mode 100644 index 0000000000..0130618ced --- /dev/null +++ b/support-files/sql/job-backup/0005_job_backup_20241115-1000_V3.11.2_mysql.sql @@ -0,0 +1,22 @@ +USE `job_backup`; + +SET NAMES utf8mb4; + +CREATE TABLE IF NOT EXISTS `archive_task` ( + `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `id` BIGINT(20) NOT NULL AUTO_INCREMENT, + `task_type` TINYINT(2), + `data_node` VARCHAR(128) NOT NULL, + `day` INT(8), + `hour` TINYINT(2), + `from_timestamp` BIGINT(20) NOT NULL, + `to_timestamp` BIGINT(20) NOT NULL, + `process` VARCHAR(256), + `status` TINYINT(2) NOT NULL DEFAULT '0', + `create_time` BIGINT(20) NOT NULL, + `last_update_time` BIGINT(20) NOT NULL, + PRIMARY KEY (`id`) USING BTREE, + UNIQUE KEY `uk_type_node_day_hour` (`task_type`,`data_node`,`day`,`hour`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + From 5abf5644b21adc7e135627fb46bd954edfe97320 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Sun, 17 Nov 2024 21:45:04 +0800 Subject: [PATCH 25/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../AbstractJobInstanceArchiveTask.java | 37 +++-- .../job/backup/archive/ArchiveTaskWorker.java | 2 +- .../archive/JobInstanceArchiveTask.java | 2 + .../JobInstanceArchiveTaskScheduler.java | 150 +++++++++++++----- .../JobInstanceMainDataArchiveTask.java | 18 +-- .../archive/JobInstanceSubTableArchivers.java | 88 ++++++++++ .../job/backup/archive/StopTaskCounter.java | 114 +++++++++++++ .../job/backup/archive/model/DbDataNode.java | 4 +- .../model/JobInstanceArchiveTaskInfo.java | 4 + .../backup/config/ArchiveConfiguration.java | 104 +++++------- ...n.java => ExecuteColdDbConfiguration.java} | 11 +- ...on.java => ExecuteHotDbConfiguration.java} | 9 +- 12 files changed, 397 insertions(+), 146 deletions(-) create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceSubTableArchivers.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/StopTaskCounter.java rename src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/{ExecuteBackupDbConfiguration.java => ExecuteColdDbConfiguration.java} (95%) rename src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/{ExecuteDbConfiguration.java => ExecuteHotDbConfiguration.java} (98%) diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java index 7582103260..c3e3f2fb49 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java @@ -25,9 +25,7 @@ package com.tencent.bk.job.backup.archive; import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; -import com.tencent.bk.job.backup.archive.dao.JobInstanceHotRecordDAO; import com.tencent.bk.job.backup.archive.model.ArchiveTaskSummary; -import com.tencent.bk.job.backup.archive.model.DbDataNode; import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; import com.tencent.bk.job.backup.archive.model.TimeAndIdBasedArchiveProcess; import com.tencent.bk.job.backup.config.ArchiveProperties; @@ -50,7 +48,6 @@ */ @Slf4j public abstract class AbstractJobInstanceArchiveTask> implements JobInstanceArchiveTask { - protected JobInstanceHotRecordDAO jobInstanceHotRecordDAO; protected JobInstanceColdDAO jobInstanceColdDAO; protected final ArchiveProperties archiveProperties; private final ArchiveTaskLock archiveTaskLock; @@ -59,8 +56,7 @@ public abstract class AbstractJobInstanceArchiveTask> i protected final ArchiveTablePropsStorage archiveTablePropsStorage; protected String taskId; - protected DbDataNode dbDataNode; - protected JobInstanceArchiveTaskInfo archiveTask; + protected JobInstanceArchiveTaskInfo archiveTaskInfo; /** * 归档进度 @@ -72,25 +68,23 @@ public abstract class AbstractJobInstanceArchiveTask> i private boolean isAcquireLock; - public AbstractJobInstanceArchiveTask(JobInstanceHotRecordDAO jobInstanceHotRecordDAO, - JobInstanceColdDAO jobInstanceColdDAO, + public AbstractJobInstanceArchiveTask(JobInstanceColdDAO jobInstanceColdDAO, ArchiveProperties archiveProperties, ArchiveTaskLock archiveTaskLock, ArchiveErrorTaskCounter archiveErrorTaskCounter, - JobInstanceArchiveTaskInfo archiveTask, + JobInstanceArchiveTaskInfo archiveTaskInfo, ArchiveTaskService archiveTaskService, ArchiveTablePropsStorage archiveTablePropsStorage) { - this.jobInstanceHotRecordDAO = jobInstanceHotRecordDAO; this.jobInstanceColdDAO = jobInstanceColdDAO; this.archiveProperties = archiveProperties; this.archiveTaskLock = archiveTaskLock; this.archiveErrorTaskCounter = archiveErrorTaskCounter; - this.archiveTask = archiveTask; + this.archiveTaskInfo = archiveTaskInfo; this.archiveTaskService = archiveTaskService; this.archiveTablePropsStorage = archiveTablePropsStorage; - this.progress = archiveTask.getProcess(); - this.taskId = buildTaskId(archiveTask); - this.archiveTaskSummary = new ArchiveTaskSummary(archiveTask, archiveProperties.getMode()); + this.progress = archiveTaskInfo.getProcess(); + this.taskId = archiveTaskInfo.buildTaskUniqueId(); + this.archiveTaskSummary = new ArchiveTaskSummary(archiveTaskInfo, archiveProperties.getMode()); } private String buildTaskId(JobInstanceArchiveTaskInfo archiveTask) { @@ -158,10 +152,10 @@ private void backupAndDelete() { do { if (progress != null) { jobInstanceRecords = readSortedJobInstanceFromHotDB(progress.getTimestamp(), - archiveTask.getToTimestamp(), progress.getId(), readLimit); + archiveTaskInfo.getToTimestamp(), progress.getId(), readLimit); } else { - jobInstanceRecords = readSortedJobInstanceFromHotDB(archiveTask.getFromTimestamp(), - archiveTask.getToTimestamp(), null, readLimit); + jobInstanceRecords = readSortedJobInstanceFromHotDB(archiveTaskInfo.getFromTimestamp(), + archiveTaskInfo.getToTimestamp(), null, readLimit); } if (CollectionUtils.isEmpty(jobInstanceRecords)) { @@ -233,9 +227,14 @@ private boolean acquireLock() { } private void updateArchiveProgress(ArchiveTaskStatusEnum taskStatus, TimeAndIdBasedArchiveProcess progress) { - archiveTask.setStatus(taskStatus); - archiveTask.setProcess(progress); - archiveTaskService.updateTask(archiveTask); + archiveTaskInfo.setStatus(taskStatus); + archiveTaskInfo.setProcess(progress); + archiveTaskService.updateTask(archiveTaskInfo); + } + + @Override + public String getTaskId() { + return this.taskId; } /** diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskWorker.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskWorker.java index 903309f53b..68db01c14a 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskWorker.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskWorker.java @@ -29,7 +29,7 @@ @Slf4j public class ArchiveTaskWorker extends Thread { - private JobInstanceArchiveTask archiveTask; + private final JobInstanceArchiveTask archiveTask; public ArchiveTaskWorker(JobInstanceArchiveTask archiveTask) { this.setName("ArchiveWorker"); diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTask.java index a2a8973fd7..e99513471b 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTask.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTask.java @@ -40,4 +40,6 @@ public interface JobInstanceArchiveTask { */ void stop(); + String getTaskId(); + } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java index f4aab44821..7bf58efe48 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java @@ -24,30 +24,32 @@ package com.tencent.bk.job.backup.archive; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceRecordDAO; import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; import com.tencent.bk.job.backup.config.ArchiveProperties; import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; +import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; import com.tencent.bk.job.common.sharding.mysql.config.ShardingProperties; import com.tencent.bk.job.common.util.ThreadUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.springframework.context.SmartLifecycle; -import org.springframework.core.task.SimpleAsyncTaskExecutor; -import java.util.HashSet; import java.util.List; -import java.util.Set; -import java.util.concurrent.Executor; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; /** * 作业执行历史归档任务调度 */ @Slf4j -public class JobInstanceArchiveTaskScheduler extends SmartLifecycle { - - private final ArchiveTaskTypeEnum archiveTaskType = ArchiveTaskTypeEnum.JOB_INSTANCE; +public class JobInstanceArchiveTaskScheduler implements SmartLifecycle { private final ArchiveTaskService archiveTaskService; @@ -59,12 +61,12 @@ public class JobInstanceArchiveTaskScheduler extends SmartLifecycle { private final JobInstanceArchiveTaskScheduleLock jobInstanceArchiveTaskScheduleLock; - private final Set workers = new HashSet<>(); + private final JobInstanceSubTableArchivers jobInstanceSubTableArchivers; - /** - * 异步归档任务执行器 - */ - private final Executor taskExecutor = new SimpleAsyncTaskExecutor("JobInstanceArchiveTaskExecutor"); + private final JobInstanceColdDAO jobInstanceColdDAO; + private final ArchiveTaskLock archiveTaskLock; + private final ArchiveErrorTaskCounter archiveErrorTaskCounter; + private final ArchiveTablePropsStorage archiveTablePropsStorage; private final AtomicInteger runningTasksCount = new AtomicInteger(0); @@ -74,54 +76,90 @@ public class JobInstanceArchiveTaskScheduler extends SmartLifecycle { * 作业执行历史归档任务调度组件是否处于活动状态 */ private volatile boolean active; + /** + * 组件是否正在运行(用于 Spring Lifecycle isRunning 判断) + */ + private volatile boolean running = false; + /** + * 是否正在进行任务调度中 + */ + private volatile boolean scheduling = false; + + private final ExecutorService shutdownExecutor = new ThreadPoolExecutor(1, 20, 120L, + TimeUnit.SECONDS, new LinkedBlockingQueue<>()); + + /** + * 调度的所有的任务 + */ + private final Map scheduledTasks = new ConcurrentHashMap<>(); public JobInstanceArchiveTaskScheduler(ArchiveTaskService archiveTaskService, TaskInstanceRecordDAO taskInstanceRecordDAO, ArchiveProperties archiveProperties, ShardingProperties shardingProperties, - JobInstanceArchiveTaskScheduleLock jobInstanceArchiveTaskScheduleLock) { + JobInstanceArchiveTaskScheduleLock jobInstanceArchiveTaskScheduleLock, + JobInstanceSubTableArchivers jobInstanceSubTableArchivers, + JobInstanceColdDAO jobInstanceColdDAO, + ArchiveTaskLock archiveTaskLock, + ArchiveErrorTaskCounter archiveErrorTaskCounter, + ArchiveTablePropsStorage archiveTablePropsStorage) { this.archiveTaskService = archiveTaskService; this.taskInstanceRecordDAO = taskInstanceRecordDAO; this.archiveProperties = archiveProperties; this.shardingProperties = shardingProperties; this.jobInstanceArchiveTaskScheduleLock = jobInstanceArchiveTaskScheduleLock; + this.jobInstanceSubTableArchivers = jobInstanceSubTableArchivers; + this.jobInstanceColdDAO = jobInstanceColdDAO; + this.archiveTaskLock = archiveTaskLock; + this.archiveErrorTaskCounter = archiveErrorTaskCounter; + this.archiveTablePropsStorage = archiveTablePropsStorage; } public void schedule() { try { - if (isRunning) { - log.info("JobInstanceArchiveTaskScheduler is running, skip"); + if (isScheduling()) { + log.info("Scheduler is already working"); + return; + } + if (!isActive()) { + log.info("JobInstanceArchiveTaskScheduler is not active, skip"); return; - } else { - this.isRunning = true; } while (true) { // 获取归档任务调度锁 boolean locked = jobInstanceArchiveTaskScheduleLock.lock(); if (!locked) { - return; + log.info("Get lock fail, wait 1s"); + ThreadUtils.sleep(1000L); + continue; } if (runningTasksCount.get() >= archiveProperties.getConcurrent()) { // 休眠一分钟，等待并行任务减少 + log.info("Running archive task count exceed concurrent limit : {}, wait 60s", + archiveProperties.getConcurrent()); ThreadUtils.sleep(1000 * 60L); - return; + continue; } - - List runningTasks = - archiveTaskService.listRunningTasks(ArchiveTaskTypeEnum.JOB_INSTANCE); + // 获取待调度的任务列表 List needScheduleTasks = archiveTaskService.listScheduleTasks(ArchiveTaskTypeEnum.JOB_INSTANCE, 100); if (CollectionUtils.isEmpty(needScheduleTasks)) { // 所有任务都已经被调度完成，退出本次任务调度 return; } + + // 获取正在执行中的任务列表 + List runningTasks = + archiveTaskService.listRunningTasks(ArchiveTaskTypeEnum.JOB_INSTANCE); + // 对待调度的任务进行优先级排序，保证同一个 db 上的归档任务数量尽可能均衡，避免出现db 热点 ArchiveTaskPrioritySorter.sort(runningTasks, shardingProperties.getDbNodeCount(), needScheduleTasks); + startArchiveTask(needScheduleTasks.get(0)); } } finally { - this.isRunning = false; + this.scheduling = false; jobInstanceArchiveTaskScheduleLock.unlock(); } @@ -129,10 +167,26 @@ public void schedule() { private void startArchiveTask(JobInstanceArchiveTaskInfo archiveTaskInfo) { log.info("Start JobInstanceArchiveTask : {}", archiveTaskInfo); + JobInstanceMainDataArchiveTask archiveTask = new JobInstanceMainDataArchiveTask( + taskInstanceRecordDAO, + jobInstanceSubTableArchivers, + jobInstanceColdDAO, + archiveProperties, + archiveTaskLock, + archiveErrorTaskCounter, + archiveTaskInfo, + archiveTaskService, + archiveTablePropsStorage + ); + synchronized (lifecycleMonitor) { + if (!isActive()) { + log.info("JobInstanceArchiveTaskScheduler is not active, skip"); + return; + } + scheduledTasks.put(archiveTask.getTaskId(), archiveTask); + } runningTasksCount.incrementAndGet(); - JobInstanceMainDataArchiveTask archiveTask = new JobInstanceMainDataArchiveTask(); ArchiveTaskWorker worker = new ArchiveTaskWorker(archiveTask); - workers.add(worker); worker.start(); } @@ -141,8 +195,12 @@ private void startArchiveTask(JobInstanceArchiveTaskInfo archiveTaskInfo) { */ @Override public void start() { + if (isRunning()) { + return; + } synchronized (lifecycleMonitor) { this.active = true; + this.running = true; } } @@ -158,6 +216,7 @@ public void stop() { return; } this.active = false; + this.running = false; } stopTasksGraceful(); log.info("JobInstanceArchiveTaskScheduler stop successfully!"); @@ -173,8 +232,8 @@ private void stopTasksGraceful() { stopTaskCounter = StopTaskCounter.getInstance(); stopTaskCounter.initCounter(scheduledTasks.keySet()); } - for (ScheduledContinuousResultHandleTask task : scheduledTasks.values()) { - shutdownExecutor.execute(new StopTask(task, tracer)); + for (JobInstanceArchiveTask task : scheduledTasks.values()) { + shutdownExecutor.execute(new StopTask(task)); } } try { @@ -188,32 +247,41 @@ private void stopTasksGraceful() { log.info("Stop tasks graceful - end, cost: {}", end - start); } + private static final class StopTask implements Runnable { + private final JobInstanceArchiveTask task; + + StopTask(JobInstanceArchiveTask task) { + this.task = task; + } + + @Override + public void run() { + try { + task.stop(); + } catch (Throwable e) { + String errorMsg = "Stop archive task caught exception, task: " + task; + log.warn(errorMsg, e); + } + } + } + @Override public boolean isRunning() { synchronized (this.lifecycleMonitor) { - return (this.active); + return (this.running); } } - boolean isActive() { + private boolean isActive() { synchronized (this.lifecycleMonitor) { return this.active; } } - /** - * 消费者是否处于运行状态 - * - * @param worker 消费者 - * @return 是否运行 - */ - private boolean isWorkerActive(TaskWorker worker) { - boolean workerActive; - synchronized (this.workersMonitor) { - workerActive = this.workers.contains(worker); + private boolean isScheduling() { + synchronized (this.lifecycleMonitor) { + return this.scheduling; } - return workerActive && this.isActive(); } - } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java index 98878b9bc1..b3dc084f8a 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java @@ -25,9 +25,7 @@ package com.tencent.bk.job.backup.archive; import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; -import com.tencent.bk.job.backup.archive.dao.JobInstanceHotRecordDAO; import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceRecordDAO; -import com.tencent.bk.job.backup.archive.impl.GseScriptExecuteObjTaskArchiver; import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; import com.tencent.bk.job.backup.config.ArchiveProperties; import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; @@ -35,7 +33,6 @@ import com.tencent.bk.job.execute.model.tables.records.TaskInstanceRecord; import lombok.extern.slf4j.Slf4j; -import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -47,11 +44,10 @@ public class JobInstanceMainDataArchiveTask extends AbstractJobInstanceArchiveTa private final TaskInstanceRecordDAO taskInstanceRecordDAO; - private final List subTableArchivers = new ArrayList<>(); + private final JobInstanceSubTableArchivers jobInstanceSubTableArchivers; public JobInstanceMainDataArchiveTask(TaskInstanceRecordDAO taskInstanceRecordDAO, - GseScriptExecuteObjTaskArchiver gseScriptExecuteObjTaskArchiver, - JobInstanceHotRecordDAO jobInstanceHotRecordDAO, + JobInstanceSubTableArchivers jobInstanceSubTableArchivers, JobInstanceColdDAO jobInstanceColdDAO, ArchiveProperties archiveProperties, ArchiveTaskLock archiveTaskLock, @@ -60,7 +56,6 @@ public JobInstanceMainDataArchiveTask(TaskInstanceRecordDAO taskInstanceRecordDA ArchiveTaskService archiveTaskService, ArchiveTablePropsStorage archiveTablePropsStorage) { super( - jobInstanceHotRecordDAO, jobInstanceColdDAO, archiveProperties, archiveTaskLock, @@ -70,8 +65,7 @@ public JobInstanceMainDataArchiveTask(TaskInstanceRecordDAO taskInstanceRecordDA archiveTablePropsStorage ); this.taskInstanceRecordDAO = taskInstanceRecordDAO; - - this.subTableArchivers.add(gseScriptExecuteObjTaskArchiver); + this.jobInstanceSubTableArchivers = jobInstanceSubTableArchivers; } @@ -82,7 +76,7 @@ protected void backupJobInstanceToColdDb(List jobInstanceRec // 备份主表数据 jobInstanceColdDAO.batchInsert(jobInstanceRecords, 1000); // 备份子表数据 - subTableArchivers.forEach(tableArchiver -> { + jobInstanceSubTableArchivers.getAll().forEach(tableArchiver -> { tableArchiver.backupRecords(jobInstanceIds); }); } @@ -90,11 +84,11 @@ protected void backupJobInstanceToColdDb(List jobInstanceRec @Override protected void deleteJobInstanceHotData(List jobInstanceIds) { // 先删除子表数据 - subTableArchivers.forEach(tableArchiver -> { + jobInstanceSubTableArchivers.getAll().forEach(tableArchiver -> { tableArchiver.deleteRecords(jobInstanceIds); }); // 删除主表数据 - jobInstanceHotRecordDAO.deleteRecords(jobInstanceIds, + taskInstanceRecordDAO.deleteRecords(jobInstanceIds, archiveTablePropsStorage.getDeleteLimitRowCount(TaskInstance.TASK_INSTANCE.getName())); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceSubTableArchivers.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceSubTableArchivers.java new file mode 100644 index 0000000000..a55467e5f1 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceSubTableArchivers.java @@ -0,0 +1,88 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.impl.FileSourceTaskLogArchiver; +import com.tencent.bk.job.backup.archive.impl.GseFileAgentTaskArchiver; +import com.tencent.bk.job.backup.archive.impl.GseFileExecuteObjTaskArchiver; +import com.tencent.bk.job.backup.archive.impl.GseScriptAgentTaskArchiver; +import com.tencent.bk.job.backup.archive.impl.GseScriptExecuteObjTaskArchiver; +import com.tencent.bk.job.backup.archive.impl.GseTaskArchiver; +import com.tencent.bk.job.backup.archive.impl.OperationLogArchiver; +import com.tencent.bk.job.backup.archive.impl.RollingConfigArchiver; +import com.tencent.bk.job.backup.archive.impl.StepInstanceArchiver; +import com.tencent.bk.job.backup.archive.impl.StepInstanceConfirmArchiver; +import com.tencent.bk.job.backup.archive.impl.StepInstanceFileArchiver; +import com.tencent.bk.job.backup.archive.impl.StepInstanceRollingTaskArchiver; +import com.tencent.bk.job.backup.archive.impl.StepInstanceScriptArchiver; +import com.tencent.bk.job.backup.archive.impl.StepInstanceVariableArchiver; +import com.tencent.bk.job.backup.archive.impl.TaskInstanceHostArchiver; +import com.tencent.bk.job.backup.archive.impl.TaskInstanceVariableArchiver; + +import java.util.ArrayList; +import java.util.List; + +public class JobInstanceSubTableArchivers { + + private final List subTableArchivers = new ArrayList<>(); + + public JobInstanceSubTableArchivers(FileSourceTaskLogArchiver fileSourceTaskLogArchiver, + GseFileAgentTaskArchiver gseFileAgentTaskArchiver, + GseFileExecuteObjTaskArchiver gseFileExecuteObjTaskArchiver, + GseScriptAgentTaskArchiver gseScriptAgentTaskArchiver, + GseScriptExecuteObjTaskArchiver gseScriptExecuteObjTaskArchiver, + GseTaskArchiver gseTaskArchiver, + OperationLogArchiver operationLogArchiver, + RollingConfigArchiver rollingConfigArchiver, + StepInstanceArchiver stepInstanceArchiver, + StepInstanceConfirmArchiver stepInstanceConfirmArchiver, + StepInstanceFileArchiver stepInstanceFileArchiver, + StepInstanceScriptArchiver stepInstanceScriptArchiver, + StepInstanceRollingTaskArchiver stepInstanceRollingTaskArchiver, + StepInstanceVariableArchiver stepInstanceVariableArchiver, + TaskInstanceHostArchiver taskInstanceHostArchiver, + TaskInstanceVariableArchiver taskInstanceVariableArchiver) { + this.subTableArchivers.add(fileSourceTaskLogArchiver); + this.subTableArchivers.add(gseFileAgentTaskArchiver); + this.subTableArchivers.add(gseFileExecuteObjTaskArchiver); + this.subTableArchivers.add(gseScriptAgentTaskArchiver); + this.subTableArchivers.add(gseScriptExecuteObjTaskArchiver); + this.subTableArchivers.add(gseTaskArchiver); + this.subTableArchivers.add(operationLogArchiver); + this.subTableArchivers.add(rollingConfigArchiver); + this.subTableArchivers.add(stepInstanceArchiver); + this.subTableArchivers.add(stepInstanceConfirmArchiver); + this.subTableArchivers.add(stepInstanceFileArchiver); + this.subTableArchivers.add(stepInstanceScriptArchiver); + this.subTableArchivers.add(stepInstanceRollingTaskArchiver); + this.subTableArchivers.add(stepInstanceVariableArchiver); + this.subTableArchivers.add(taskInstanceHostArchiver); + this.subTableArchivers.add(taskInstanceVariableArchiver); + } + + public List getAll() { + return this.subTableArchivers; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/StopTaskCounter.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/StopTaskCounter.java new file mode 100644 index 0000000000..200dff8ab9 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/StopTaskCounter.java @@ -0,0 +1,114 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.common.util.ThreadUtils; +import lombok.extern.slf4j.Slf4j; + +import javax.annotation.PreDestroy; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.CountDownLatch; + +/** + * 停止任务计数 + */ +@Slf4j +public class StopTaskCounter { + private final Object taskMonitor = new Object(); + private CountDownLatch latch; + private final Set taskIds = new HashSet<>(); + private volatile boolean monitorInitial = false; + private volatile boolean isAllTaskDone = false; + + private StopTaskCounter() { + } + + public static StopTaskCounter getInstance() { + return Inner.instance; + } + + void initCounter(Set taskIds) { + this.taskIds.addAll(taskIds); + this.latch = new CountDownLatch(taskIds.size()); + startMonitor(); + } + + public void decrement(String taskId) { + synchronized (taskMonitor) { + if (this.latch != null) { + if (taskIds.remove(taskId)) { + this.latch.countDown(); + log.info("Task is stopped, remove from counter! taskId: {}", taskId); + } else { + log.warn("Unexpected stopped taskId: {}", taskId); + } + } + } + } + + void waitingForAllTasksDone() { + try { + log.info("Waiting for all tasks done! total: {}", latch.getCount()); + this.latch.await(); + this.isAllTaskDone = true; + log.info("All tasks are stopped!"); + } catch (InterruptedException e) { + log.warn("Stop task counter wait interrupted", e); + } + } + + private synchronized void startMonitor() { + if (!monitorInitial) { + log.info("Start StopTaskCounter monitor ..."); + Thread monitorThread = new Thread("StopTaskCounterMonitor") { + @Override + public void run() { + while (!isAllTaskDone) { + log.info("Waiting for tasks stopped! taskSize: {}, tasks: {}", taskIds.size(), taskIds); + ThreadUtils.sleep(2000L); + } + } + }; + monitorThread.start(); + monitorInitial = true; + } + } + + @PreDestroy + private void destroy() { + if (isAllTaskDone) { + log.info("All task are stopped!"); + } else { + log.warn("Tasks are not completely stopped! remain task size: {}, remain tasks: {}", taskIds.size(), + taskIds); + } + } + + private static class Inner { + private static final StopTaskCounter instance = new StopTaskCounter(); + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DbDataNode.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DbDataNode.java index 432ac28c68..665569695b 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DbDataNode.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DbDataNode.java @@ -39,7 +39,7 @@ public class DbDataNode { */ private DbDataNodeTypeEnum type; /** - * 分库分表数据源前缀 + * 分库分表数据源 */ private String dataSource; /** @@ -90,6 +90,6 @@ public static DbDataNode fromDataNodeId(String dataNodeId) { @Override public DbDataNode clone() { - return new DbDataNode(type, dbIndex, tableIndex); + return new DbDataNode(type, dataSource, dbIndex, tableIndex); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/JobInstanceArchiveTaskInfo.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/JobInstanceArchiveTaskInfo.java index 8112c2d52d..0b3d694717 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/JobInstanceArchiveTaskInfo.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/JobInstanceArchiveTaskInfo.java @@ -91,4 +91,8 @@ public JobInstanceArchiveTaskInfo clone() { archiveTask.setLastUpdateTime(lastUpdateTime); return archiveTask; } + + public String buildTaskUniqueId() { + return taskType.getType() + ":" + day + ":" + hour + ":" + dbDataNode.toDataNodeId(); + } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java index 6a44c2bc16..992984d4fa 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java @@ -27,10 +27,9 @@ import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; import com.tencent.bk.job.backup.archive.ArchiveTaskLock; import com.tencent.bk.job.backup.archive.ArchiveTaskService; -import com.tencent.bk.job.backup.archive.JobExecuteArchiveManage; import com.tencent.bk.job.backup.archive.JobInstanceArchiveTaskGenerator; import com.tencent.bk.job.backup.archive.JobInstanceArchiveTaskScheduler; -import com.tencent.bk.job.backup.archive.dao.ExecuteArchiveDAO; +import com.tencent.bk.job.backup.archive.JobInstanceSubTableArchivers; import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; import com.tencent.bk.job.backup.archive.dao.impl.FileSourceTaskLogRecordDAO; import com.tencent.bk.job.backup.archive.dao.impl.GseFileAgentTaskRecordDAO; @@ -67,8 +66,6 @@ import com.tencent.bk.job.backup.archive.impl.TaskInstanceArchiver; import com.tencent.bk.job.backup.archive.impl.TaskInstanceHostArchiver; import com.tencent.bk.job.backup.archive.impl.TaskInstanceVariableArchiver; -import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; -import com.tencent.bk.job.backup.service.ArchiveProgressService; import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; @@ -83,8 +80,6 @@ import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.scheduling.annotation.EnableScheduling; -import java.util.concurrent.ExecutorService; - /** * job-execute 模块数据归档配置 */ @@ -92,7 +87,7 @@ @EnableScheduling @Slf4j @EnableConfigurationProperties(ArchiveProperties.class) -@Import({ExecuteDbConfiguration.class, ExecuteBackupDbConfiguration.class}) +@Import({ExecuteHotDbConfiguration.class, ExecuteColdDbConfiguration.class}) public class ArchiveConfiguration { /** @@ -425,13 +420,50 @@ public TaskInstanceHostArchiver taskInstanceHostArchiver( archiveTablePropsStorage); } + @Bean + public JobInstanceSubTableArchivers jobInstanceSubTableArchivers( + FileSourceTaskLogArchiver fileSourceTaskLogArchiver, + GseFileAgentTaskArchiver gseFileAgentTaskArchiver, + GseFileExecuteObjTaskArchiver gseFileExecuteObjTaskArchiver, + GseScriptAgentTaskArchiver gseScriptAgentTaskArchiver, + GseScriptExecuteObjTaskArchiver gseScriptExecuteObjTaskArchiver, + GseTaskArchiver gseTaskArchiver, + OperationLogArchiver operationLogArchiver, + RollingConfigArchiver rollingConfigArchiver, + StepInstanceArchiver stepInstanceArchiver, + StepInstanceConfirmArchiver stepInstanceConfirmArchiver, + StepInstanceFileArchiver stepInstanceFileArchiver, + StepInstanceScriptArchiver stepInstanceScriptArchiver, + StepInstanceRollingTaskArchiver stepInstanceRollingTaskArchiver, + StepInstanceVariableArchiver stepInstanceVariableArchiver, + TaskInstanceHostArchiver taskInstanceHostArchiver, + TaskInstanceVariableArchiver taskInstanceVariableArchiver) { + return new JobInstanceSubTableArchivers( + fileSourceTaskLogArchiver, + gseFileAgentTaskArchiver, + gseFileExecuteObjTaskArchiver, + gseScriptAgentTaskArchiver, + gseScriptExecuteObjTaskArchiver, + gseTaskArchiver, + operationLogArchiver, + rollingConfigArchiver, + stepInstanceArchiver, + stepInstanceConfirmArchiver, + stepInstanceFileArchiver, + stepInstanceScriptArchiver, + stepInstanceRollingTaskArchiver, + stepInstanceVariableArchiver, + taskInstanceHostArchiver, + taskInstanceVariableArchiver); + } + } /** * job-execute 归档冷 DB 配置 */ @Configuration - @Conditional(ExecuteBackupDbConfiguration.JobExecuteColdDbInitCondition.class) + @Conditional(ExecuteColdDbConfiguration.JobExecuteColdDbInitCondition.class) public static class ExecuteBackupDAOConfig { @Bean(name = "execute-archive-dao") public JobInstanceColdDAOImpl jobInstanceColdDAO( @@ -471,60 +503,6 @@ public JobInstanceArchiveTaskScheduler jobInstanceArchiveTaskScheduler(ArchiveTa taskInstanceRecordDAO, archiveProperties, - ); - } - - @Bean - @ConditionalOnExpression("${job.backup.archive.execute.enabled:false}") - public JobExecuteArchiveManage jobExecuteArchiveManage( - ObjectProvider taskInstanceRecordDAOObjectProvider, - ObjectProvider stepInstanceRecordDAOObjectProvider, - ObjectProvider stepInstanceScriptRecordDAOObjectProvider, - ObjectProvider stepInstanceFileRecordDAOObjectProvider, - ObjectProvider stepInstanceConfirmRecordDAOObjectProvider, - ObjectProvider stepInstanceVariableRecordDAOObjectProvider, - ObjectProvider taskInstanceVariableRecordDAOObjectProvider, - ObjectProvider operationLogRecordDAOObjectProvider, - ObjectProvider fileSourceTaskLogRecordDAOObjectProvider, - ObjectProvider gseTaskRecordDAOObjectProvider, - ObjectProvider gseScriptAgentTaskRecordDAOObjectProvider, - ObjectProvider gseFileAgentTaskRecordDAOObjectProvider, - ObjectProvider gseScriptExecuteObjTaskRecordDAOObjectProvider, - ObjectProvider gseFileExecuteObjTaskRecordDAOObjectProvider, - ObjectProvider stepInstanceRollingTaskRecordDAOObjectProvider, - ObjectProvider rollingConfigRecordDAOObjectProvider, - ObjectProvider taskInstanceHostRecordDAOObjectProvider, - ObjectProvider executeArchiveDAOObjectProvider, - ArchiveProgressService archiveProgressService, - @Qualifier("archiveExecutor") ExecutorService archiveExecutor, - ArchiveProperties archiveProperties, - ArchiveTaskLock archiveTaskLock, - ArchiveErrorTaskCounter archiveErrorTaskCounter) { - - log.info("Init JobExecuteArchiveManage"); - return new JobExecuteArchiveManage( - taskInstanceRecordDAOObjectProvider.getIfAvailable(), - stepInstanceRecordDAOObjectProvider.getIfAvailable(), - stepInstanceScriptRecordDAOObjectProvider.getIfAvailable(), - stepInstanceFileRecordDAOObjectProvider.getIfAvailable(), - stepInstanceConfirmRecordDAOObjectProvider.getIfAvailable(), - stepInstanceVariableRecordDAOObjectProvider.getIfAvailable(), - taskInstanceVariableRecordDAOObjectProvider.getIfAvailable(), - operationLogRecordDAOObjectProvider.getIfAvailable(), - fileSourceTaskLogRecordDAOObjectProvider.getIfAvailable(), - gseTaskRecordDAOObjectProvider.getIfAvailable(), - gseScriptAgentTaskRecordDAOObjectProvider.getIfAvailable(), - gseFileAgentTaskRecordDAOObjectProvider.getIfAvailable(), - gseScriptExecuteObjTaskRecordDAOObjectProvider.getIfAvailable(), - gseFileExecuteObjTaskRecordDAOObjectProvider.getIfAvailable(), - stepInstanceRollingTaskRecordDAOObjectProvider.getIfAvailable(), - rollingConfigRecordDAOObjectProvider.getIfAvailable(), - taskInstanceHostRecordDAOObjectProvider.getIfAvailable(), - executeArchiveDAOObjectProvider.getIfAvailable(), - archiveProgressService, - archiveProperties, - archiveExecutor, - archiveTaskLock, - archiveErrorTaskCounter); + ); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteBackupDbConfiguration.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteColdDbConfiguration.java similarity index 95% rename from src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteBackupDbConfiguration.java rename to src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteColdDbConfiguration.java index 147627517f..ec06946b6c 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteBackupDbConfiguration.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteColdDbConfiguration.java @@ -43,9 +43,12 @@ import javax.sql.DataSource; -@Configuration(value = "executeBackupDbConfiguration") -@Conditional(ExecuteBackupDbConfiguration.JobExecuteColdDbInitCondition.class) -public class ExecuteBackupDbConfiguration { +/** + * 归档-冷 DB 配置 + */ +@Configuration(value = "executeColdDbConfiguration") +@Conditional(ExecuteColdDbConfiguration.JobExecuteColdDbInitCondition.class) +public class ExecuteColdDbConfiguration { @Qualifier("job-execute-archive-source") @Bean(name = "job-execute-archive-source") @@ -110,6 +113,4 @@ static class ArchiveModeBackupOnlyCondition { } - - } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteDbConfiguration.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteHotDbConfiguration.java similarity index 98% rename from src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteDbConfiguration.java rename to src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteHotDbConfiguration.java index 5fa0b6428a..c383d60980 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteDbConfiguration.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteHotDbConfiguration.java @@ -51,10 +51,13 @@ import javax.sql.DataSource; -@Configuration("executeDbConfiguration") -@Conditional(ExecuteDbConfiguration.ExecuteDbInitCondition.class) +/** + * 归档-job-execute 热 DB 配置 + */ +@Configuration("executeHotDbConfiguration") +@Conditional(ExecuteHotDbConfiguration.ExecuteDbInitCondition.class) @Slf4j -public class ExecuteDbConfiguration { +public class ExecuteHotDbConfiguration { @ConditionalOnProperty(value = "job.backup.archive.execute.mariadb.dataSourceMode", havingValue = DataSourceMode.Constants.STANDALONE, matchIfMissing = false) From b2330a9952d39812055b2b07daa00f162d651375 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Mon, 18 Nov 2024 17:35:34 +0800 Subject: [PATCH 26/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mysql/config/LeafAutoConfiguration.java | 3 + .../ShardingDatasourceAutoConfiguration.java | 3 + .../mysql/config/ShardingProperties.java | 3 + .../api/inner/ServiceArchiveResource.java | 43 -- .../impl/ServiceArchiveResourceImpl.java | 87 --- .../AbstractJobInstanceArchiveTask.java | 49 +- .../archive/AbstractTableArchivist.java | 93 --- .../ArchiveDbNodePriorityEvaluator.java | 120 ++++ .../backup/archive/ArchiveSummaryHolder.java | 68 --- .../archive/ArchiveTablePropsStorage.java | 15 +- .../archive/ArchiveTaskPrioritySorter.java | 125 ---- .../job/backup/archive/ArchiveTaskWorker.java | 7 +- .../archive/JobExecuteArchiveManage.java | 542 ------------------ .../JobInstanceAppDataArchiveTask.java | 84 --- .../JobInstanceArchiveTaskGenerator.java | 4 + .../JobInstanceArchiveTaskScheduler.java | 41 +- .../archive/JobInstanceHotArchivist.java | 96 ---- .../JobInstanceMainDataArchiveTask.java | 1 + .../backup/archive/dao/ArchiveTaskDAO.java | 11 +- .../archive/dao/JobInstanceHotRecordDAO.java | 2 +- ...uteVerticalShardingDSLContextProvider.java | 2 +- .../impl/AbstractJobInstanceHotRecordDAO.java | 2 +- .../archive/dao/impl/ArchiveTaskDAOImpl.java | 36 +- .../dao/impl/ExecuteArchiveDAOImpl.java | 148 ----- .../dao/impl/FileSourceTaskLogRecordDAO.java | 2 + .../dao/impl/GseFileAgentTaskRecordDAO.java | 6 +- .../impl/GseFileExecuteObjTaskRecordDAO.java | 4 +- .../dao/impl/GseScriptAgentTaskRecordDAO.java | 13 +- .../GseScriptExecuteObjTaskRecordDAO.java | 2 + .../archive/dao/impl/GseTaskRecordDAO.java | 4 +- .../dao/impl/OperationLogRecordDAO.java | 2 + .../dao/impl/RollingConfigRecordDAO.java | 2 + .../impl/StepInstanceConfirmRecordDAO.java | 2 + .../dao/impl/StepInstanceFileRecordDAO.java | 4 +- .../dao/impl/StepInstanceRecordDAO.java | 4 +- .../StepInstanceRollingTaskRecordDAO.java | 4 +- .../dao/impl/StepInstanceScriptRecordDAO.java | 2 + .../impl/StepInstanceVariableRecordDAO.java | 2 + .../dao/impl/TaskInstanceHostRecordDAO.java | 4 +- .../dao/impl/TaskInstanceRecordDAO.java | 2 + .../impl/TaskInstanceVariableRecordDAO.java | 2 + .../AbstractJobInstanceRecordResultSet.java | 3 +- .../JobInstanceRecordResultSetFactory.java | 3 +- .../{impl => resultset}/RecordResultSet.java | 2 +- .../AbstractJobInstanceSubTableArchiver.java | 2 +- .../job/backup/archive/model/DbDataNode.java | 25 +- .../{ => service}/ArchiveTaskService.java | 17 +- .../backup/config/ArchiveConfiguration.java | 27 +- .../config/ExecuteHotDbConfiguration.java | 2 +- .../ArchiveDbNodePriorityEvaluatorTest.java | 99 ++++ ...job_backup_20241115-1000_V3.11.2_mysql.sql | 4 +- 51 files changed, 467 insertions(+), 1363 deletions(-) delete mode 100644 src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/api/inner/ServiceArchiveResource.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/inner/impl/ServiceArchiveResourceImpl.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractTableArchivist.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluator.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveSummaryHolder.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskPrioritySorter.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobExecuteArchiveManage.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceAppDataArchiveTask.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceHotArchivist.java rename src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/{ => dao/ds}/JobExecuteVerticalShardingDSLContextProvider.java (98%) delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ExecuteArchiveDAOImpl.java rename src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/{impl => resultset}/AbstractJobInstanceRecordResultSet.java (96%) rename src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/{impl => resultset}/JobInstanceRecordResultSetFactory.java (96%) rename src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/{impl => resultset}/RecordResultSet.java (96%) rename src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/{ => service}/ArchiveTaskService.java (81%) create mode 100644 src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluatorTest.java diff --git a/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/LeafAutoConfiguration.java b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/LeafAutoConfiguration.java index 01ff46324f..009833682b 100644 --- a/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/LeafAutoConfiguration.java +++ b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/LeafAutoConfiguration.java @@ -45,6 +45,9 @@ import javax.sql.DataSource; +/** + * 美团 leaf 分布式 ID 组件自动装配 + */ @Configuration(value = "leafDbConfig") @ConditionalOnProperty(value = "leaf.enabled", havingValue = "true") @EnableConfigurationProperties({LeafProperties.class}) diff --git a/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/ShardingDatasourceAutoConfiguration.java b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/ShardingDatasourceAutoConfiguration.java index 87f12a4335..8bc5d7556c 100644 --- a/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/ShardingDatasourceAutoConfiguration.java +++ b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/ShardingDatasourceAutoConfiguration.java @@ -67,6 +67,9 @@ import java.util.Properties; import java.util.stream.Collectors; +/** + * 分库分表组件 ShardingSphere 配置 + */ @Configuration(proxyBeanMethods = false) @EnableConfigurationProperties({ShardingProperties.class}) @ConditionalOnProperty(value = "sharding.enabled", havingValue = "true") diff --git a/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/ShardingProperties.java b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/ShardingProperties.java index f0573bbbab..e52d2f8b87 100644 --- a/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/ShardingProperties.java +++ b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/ShardingProperties.java @@ -33,6 +33,9 @@ import java.util.Map; import java.util.StringJoiner; +/** + * 分库分表配置 + */ @ConfigurationProperties(prefix = "sharding") @Getter @Setter diff --git a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/api/inner/ServiceArchiveResource.java b/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/api/inner/ServiceArchiveResource.java deleted file mode 100644 index 354c108115..0000000000 --- a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/api/inner/ServiceArchiveResource.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.api.inner; - -import com.tencent.bk.job.backup.model.inner.ServiceArchiveDBRequest; -import com.tencent.bk.job.common.annotation.InternalAPI; -import com.tencent.bk.job.common.model.InternalResponse; -import io.swagger.annotations.Api; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -@Api(tags = {"DB_Archive"}) -@RequestMapping("/service/archiveDB") -@RestController -@InternalAPI -public interface ServiceArchiveResource { - @PostMapping - InternalResponse archive(@RequestBody ServiceArchiveDBRequest request); -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/inner/impl/ServiceArchiveResourceImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/inner/impl/ServiceArchiveResourceImpl.java deleted file mode 100644 index a332b6bad9..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/inner/impl/ServiceArchiveResourceImpl.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.api.inner.impl; - -import com.tencent.bk.job.backup.api.inner.ServiceArchiveResource; -import com.tencent.bk.job.backup.archive.JobExecuteArchiveManage; -import com.tencent.bk.job.backup.config.ArchiveProperties; -import com.tencent.bk.job.backup.constant.ArchiveModeEnum; -import com.tencent.bk.job.backup.model.inner.ServiceArchiveDBRequest; -import com.tencent.bk.job.common.model.InternalResponse; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Profile; -import org.springframework.web.bind.annotation.RestController; - -import java.util.HashMap; -import java.util.Map; - -@RestController -@Slf4j -@Profile("dev") -public class ServiceArchiveResourceImpl implements ServiceArchiveResource { - - private final JobExecuteArchiveManage jobExecuteArchiveManage; - - public ServiceArchiveResourceImpl(@Autowired(required = false) JobExecuteArchiveManage jobExecuteArchiveManage) { - this.jobExecuteArchiveManage = jobExecuteArchiveManage; - } - - @Override - public InternalResponse archive(ServiceArchiveDBRequest request) { - log.info("Begin archive db, request: {}", request); - ArchiveProperties archiveProperties = new ArchiveProperties(); - if (StringUtils.isNotEmpty(request.getMode())) { - archiveProperties.setMode(request.getMode()); - } else { - archiveProperties.setMode(ArchiveModeEnum.BACKUP_THEN_DELETE.getMode()); - } - archiveProperties.setEnabled(true); - archiveProperties.setKeepDays(request.getKeepDays()); - archiveProperties.setBatchInsertRowSize(request.getBatchInsertRowSize()); - archiveProperties.setDeleteRowLimit(request.getDeleteRowLimit()); - archiveProperties.setReadIdStepSize(request.getReadIdStepSize()); - archiveProperties.setReadRowLimit(request.getReadRowLimit()); - - if (request.getTableConfigs() != null) { - Map tableConfigMap = new HashMap<>(); - - request.getTableConfigs().forEach((table, config) -> { - ArchiveProperties.TableConfig tableConfig = new ArchiveProperties.TableConfig(); - tableConfig.setBatchInsertRowSize(config.getBatchInsertRowSize()); - tableConfig.setDeleteRowLimit(config.getDeleteRowLimit()); - tableConfig.setReadIdStepSize(config.getReadIdStepSize()); - tableConfig.setReadRowLimit(config.getReadRowLimit()); - tableConfigMap.put(table, tableConfig); - }); - - archiveProperties.setTableConfigs(tableConfigMap); - } - - jobExecuteArchiveManage.archive(archiveProperties); - return InternalResponse.buildSuccessResp(archiveProperties); - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java index c3e3f2fb49..1b3396fd57 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java @@ -28,6 +28,7 @@ import com.tencent.bk.job.backup.archive.model.ArchiveTaskSummary; import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; import com.tencent.bk.job.backup.archive.model.TimeAndIdBasedArchiveProcess; +import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; import com.tencent.bk.job.backup.config.ArchiveProperties; import com.tencent.bk.job.backup.constant.ArchiveModeEnum; import com.tencent.bk.job.backup.constant.ArchiveTaskStatusEnum; @@ -66,6 +67,18 @@ public abstract class AbstractJobInstanceArchiveTask> i private final ArchiveTaskSummary archiveTaskSummary; private boolean isAcquireLock; + /** + * 任务是否已停止 + */ + protected volatile boolean isStopped = false; + /** + * 任务终止标识 + */ + protected volatile boolean stopFlag = false; + /** + * 同步锁 + */ + private final Object stopMonitor = new Object(); public AbstractJobInstanceArchiveTask(JobInstanceColdDAO jobInstanceColdDAO, @@ -87,12 +100,6 @@ public AbstractJobInstanceArchiveTask(JobInstanceColdDAO jobInstanceColdDAO, this.archiveTaskSummary = new ArchiveTaskSummary(archiveTaskInfo, archiveProperties.getMode()); } - private String buildTaskId(JobInstanceArchiveTaskInfo archiveTask) { - return archiveTask.getTaskType() + ":" + archiveTask.getDbDataNode().toDataNodeId() - + ":" + archiveTask.getDay() + ":" + archiveTask.getHour(); - } - - @Override public void execute() { archive(); @@ -100,9 +107,30 @@ public void execute() { @Override public void stop() { + synchronized (stopMonitor) { + this.stopFlag = true; + } + } + private boolean checkStopFlag() { + synchronized (stopMonitor) { + return stopFlag; + } } + private void stopTask() { + synchronized (stopMonitor) { + if (!isStopped) { + isStopped = true; + StopTaskCounter.getInstance().decrement(taskId); + log.info("Stop archive task successfully, taskId: {}", taskId); + } else { + log.info("Archive task is stopped, taskId: {}", taskId); + } + } + } + + private void archive() { try { if (!acquireLock()) { @@ -111,6 +139,8 @@ private void archive() { } log.info("[{}] Start archive task", taskId); + archiveTaskInfo.setStatus(ArchiveTaskStatusEnum.RUNNING); + archiveTaskService.updateTask(archiveTaskInfo); // 归档 backupAndDelete(); } catch (Throwable e) { @@ -134,6 +164,9 @@ private void archive() { taskId, JsonUtils.toJson(archiveTaskSummary) ); + if (checkStopFlag()) { + stopTask(); + } } } @@ -150,6 +183,10 @@ private void backupAndDelete() { try { List jobInstanceRecords; do { + // 检查任务终止标识 + if (checkStopFlag()) { + stopTask(); + } if (progress != null) { jobInstanceRecords = readSortedJobInstanceFromHotDB(progress.getTimestamp(), archiveTaskInfo.getToTimestamp(), progress.getId(), readLimit); diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractTableArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractTableArchivist.java deleted file mode 100644 index b21e7ad4b1..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractTableArchivist.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive; - -import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; -import com.tencent.bk.job.backup.archive.dao.JobInstanceHotRecordDAO; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.jooq.TableRecord; - -import java.io.IOException; -import java.util.Collection; -import java.util.List; - -@Slf4j -public abstract class AbstractTableArchivist> { - - protected String tableName; - protected JobInstanceHotRecordDAO jobInstanceHotRecordDAO; - protected JobInstanceColdDAO jobInstanceColdDAO; - protected int batchInsertRowSize; - protected int deleteLimitRowCount; - - public AbstractTableArchivist(String tableName, - JobInstanceHotRecordDAO jobInstanceHotRecordDAO, - JobInstanceColdDAO jobInstanceColdDAO, - int batchInsertRowSize, - int deleteLimitRowCount) { - this.tableName = tableName; - this.jobInstanceHotRecordDAO = jobInstanceHotRecordDAO; - this.jobInstanceColdDAO = jobInstanceColdDAO; - this.batchInsertRowSize = batchInsertRowSize; - this.deleteLimitRowCount = deleteLimitRowCount; - } - - void listRecord(Collection jobInstanceIds); - - void listRecord(Collection jobInstanceIds); - - void deleteRecord(Collection jobInstanceIds); - - void saveCold(); - - protected int insertAndReset(List recordList) throws IOException { - if (CollectionUtils.isEmpty(recordList)) { - return 0; - } - long startTime = System.currentTimeMillis(); - int recordSize = recordList.size(); - int insertRows = batchInsert(recordList); - recordList.clear(); - if (insertRows != recordSize) { - throw new ArchiveException(String.format("Insert rows not expected, expect: %s, actual: %s", - recordSize, insertRows)); - } - long costTime = System.currentTimeMillis() - startTime; - log.info("Batch insert {}, maxBatchSize: {}, insert rows: {}, cost: {}", - tableName, batchInsertRowSize, insertRows, costTime); - return insertRows; - } - - private int batchInsert(List recordList) throws IOException { - return jobInstanceColdDAO.batchInsert(recordList, batchInsertRowSize); - } - - private int deleteRecord(List jobInstanceIds) { - return jobInstanceHotRecordDAO.deleteRecords(start, stop, deleteLimitRowCount); - } - - -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluator.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluator.java new file mode 100644 index 0000000000..4723f27555 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluator.java @@ -0,0 +1,120 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; +import lombok.Data; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * 归档任务调度,DB 节点优先级计算，保证归档任务均衡分布在每个 db 节点上 + */ +public class ArchiveDbNodePriorityEvaluator { + + /** + * 归档任务调度优先级计算 + * + * @param runningTasks 正在执行中的归档任务 + * @param scheduleTaskCountGroupByDb 待调度归档任务计数（根据 db) + * @return DbNodeTasksInfo + */ + public static DbNodeTasksInfo evaluatePriorityDbNode(List runningTasks, + Map scheduleTaskCountGroupByDb) { + Map runningTaskCountGroupByDb = groupRunningTaskCountByDb(runningTasks); + + List dbNodeTasksInfos = new ArrayList<>(); + scheduleTaskCountGroupByDb.keySet().forEach(dbNodeId -> + dbNodeTasksInfos.add(new DbNodeTasksInfo( + dbNodeId, + runningTaskCountGroupByDb.get(dbNodeId), + scheduleTaskCountGroupByDb.get(dbNodeId) + ))); + // 优先级排序 + sortDbNodeTaskCounters(dbNodeTasksInfos); + return dbNodeTasksInfos.get(0); + } + + @Data + public static class DbNodeTasksInfo { + private String dbNodeId; + private int runningTaskCount; + private int scheduleTaskCount; + + public DbNodeTasksInfo(String dbNodeId, Integer runningTaskCount, Integer scheduleTaskCount) { + this.dbNodeId = dbNodeId; + this.runningTaskCount = runningTaskCount == null ? 0 : runningTaskCount; + this.scheduleTaskCount = scheduleTaskCount == null ? 0 : scheduleTaskCount; + } + } + + private static Map groupRunningTaskCountByDb(List runningTasks) { + Map dbTaskCountMap = new HashMap<>(); + if (CollectionUtils.isEmpty(runningTasks)) { + return dbTaskCountMap; + } + + runningTasks.forEach(task -> { + String dbNodeId = task.getDbDataNode().toDbNodeId(); + dbTaskCountMap.compute(dbNodeId, (k, v) -> { + if (v == null) { + v = 1; + } else { + v++; + } + return v; + }); + }); + return dbTaskCountMap; + } + + private static void sortDbNodeTaskCounters(List dbNodeTasksInfos) { + dbNodeTasksInfos.sort((node1, node2) -> { + // 先比较每个 db 上正在运行的任务数量，任务数量越少优先级越高 + int node1RunningTaskCount = node1.getRunningTaskCount(); + int node2RunningTaskCount = node2.getRunningTaskCount(); + if (node1RunningTaskCount < node2RunningTaskCount) { + return -1; + } else if (node1RunningTaskCount > node2RunningTaskCount) { + return 1; + } else { + // 再比较每个 db 上的待调度任务的数量，待调度任务数量越多优先级越高 + int node1ScheduleTaskCount = node1.getScheduleTaskCount(); + int node2ScheduleTaskCount = node2.getScheduleTaskCount(); + if (node1ScheduleTaskCount < node2ScheduleTaskCount) { + return 1; + } else if (node1ScheduleTaskCount > node2ScheduleTaskCount) { + return -1; + } else { + return 0; + } + } + }); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveSummaryHolder.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveSummaryHolder.java deleted file mode 100644 index 53501ee4ae..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveSummaryHolder.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive; - -import com.tencent.bk.job.backup.archive.model.ArchiveTaskSummary; -import com.tencent.bk.job.common.util.date.DateUtils; -import com.tencent.bk.job.common.util.json.JsonUtils; -import lombok.extern.slf4j.Slf4j; - -import java.time.temporal.ChronoUnit; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -@Slf4j -public class ArchiveSummaryHolder { - private Map summaryMap = new ConcurrentHashMap<>(); - private Long endTimeInMills; - - private ArchiveSummaryHolder() { - } - - public static ArchiveSummaryHolder getInstance() { - return Inner.instance; - } - - public void init(Long endTimeInMills) { - this.summaryMap.clear(); - this.endTimeInMills = endTimeInMills; - } - - public void addArchiveSummary(ArchiveTaskSummary summary) { - if (summary == null) { - return; - } - summary.setArchiveEndDate(DateUtils.formatUnixTimestamp(endTimeInMills, ChronoUnit.MILLIS)); - summaryMap.put(summary.getTaskId(), summary); - } - - public void print() { - log.info("Archive summary : {}", JsonUtils.toJson(summaryMap.values())); - } - - private static class Inner { - private static final ArchiveSummaryHolder instance = new ArchiveSummaryHolder(); - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTablePropsStorage.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTablePropsStorage.java index b3fd1f4a68..0c13885fe1 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTablePropsStorage.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTablePropsStorage.java @@ -32,6 +32,9 @@ import java.util.HashMap; import java.util.Map; +/** + * 归档配置存储 + */ @Component public class ArchiveTablePropsStorage { private final ArchiveProperties archiveProperties; @@ -53,9 +56,9 @@ public ArchiveTablePropsStorage(ArchiveProperties archiveProperties) { } /** + * 从 热 DB 表中读取归档数据，每次读取的记录数量限制 + * * @param tableName 表名 - *

- * 从 热 DB 表中读取归档数据，每次读取的记录数量限制 */ public int getReadRowLimit(String tableName) { ArchiveTableProps props = tablePropsMap.get(tableName); @@ -66,9 +69,9 @@ public int getReadRowLimit(String tableName) { } /** + * 写入归档数据到冷 DB，单批次最小行数 + * * @param tableName 表名 - *

- * 写入归档数据到冷 DB，单批次最小行数 */ public int getBatchInsertRowSize(String tableName) { ArchiveTableProps props = tablePropsMap.get(tableName); @@ -79,9 +82,9 @@ public int getBatchInsertRowSize(String tableName) { } /** + * 从热 DB 删除数据，每次删除的最大行数 + * * @param tableName 表名 - *

- * 从热 DB 删除数据，每次删除的最大行数 */ public int getDeleteLimitRowCount(String tableName) { ArchiveTableProps props = tablePropsMap.get(tableName); diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskPrioritySorter.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskPrioritySorter.java deleted file mode 100644 index b3c349d802..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskPrioritySorter.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive; - -import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; -import org.apache.commons.collections4.CollectionUtils; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class ArchiveTaskPrioritySorter { - - public static void sort(List runningTasks, - Integer dbNodeCount, - List candidateTasks) { - if (CollectionUtils.isEmpty(candidateTasks)) { - return; - } - List priorityDbIndexList = - computeDbPriorityByRunningTasks(runningTasks, dbNodeCount); - sortArchiveTask(candidateTasks, priorityDbIndexList); - } - - private static void sortArchiveTask(List candidateTasks, - List priorityDbIndexList) { - candidateTasks.sort((task1, task2) -> { - int task1DbPriority = priorityDbIndexList.indexOf(task1.getDbDataNode().getDbIndex()); - int task2DbPriority = priorityDbIndexList.indexOf(task2.getDbDataNode().getDbIndex()); - if (task1DbPriority < task2DbPriority) { - return -1; - } else if (task1DbPriority > task2DbPriority) { - return 1; - } else { - int day1 = task1.getDay(); - int day2 = task2.getDay(); - if (day1 < day2) { - return -1; - } else if (day1 > day2) { - return 1; - } else { - int hour1 = task1.getHour(); - int hour2 = task2.getHour(); - return Integer.compare(hour1, hour2); - } - } - }); - } - - /** - * 根据正在运行中的任务在 DB 实例的分布情况，计算下一个归档任务对应 DB 的优先级 - * - * @param runningTasks 正在运行中的归档任务 - * @param dbNodeCount 归档 DB 实例数量 - * @return 优先级队列。列表的值为 dbIndex, 按照优先级从高->低排序 - */ - private static List computeDbPriorityByRunningTasks(List runningTasks, - int dbNodeCount) { - Map dbTaskCountMap = computeDbTaskCount(runningTasks, dbNodeCount); - - return dbTaskCountMap.entrySet() - .stream() - .sorted(Map.Entry.comparingByValue()) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - } - - - /** - * 计算归档任务在 db 实例的分布情况 - * - * @param runningTasks 正在运行中的归档任务 - * @param dbNodeCount 归档 DB 实例数量 - * @return 任务分布情况。 key-db序号;value-正在运行中的归档任务数量 - */ - private static Map computeDbTaskCount(List runningTasks, - int dbNodeCount) { - Map dbTaskCountMap = new HashMap<>(); - if (CollectionUtils.isEmpty(runningTasks)) { - return dbTaskCountMap; - } - - runningTasks.forEach(task -> { - Integer dbIndex = task.getDbDataNode().getDbIndex(); - dbTaskCountMap.compute(dbIndex, (k, v) -> { - if (v == null) { - v = 1; - } else { - v++; - } - return v; - }); - }); - - for (int dbIndex = 0; dbIndex < dbNodeCount; dbIndex++) { - dbTaskCountMap.putIfAbsent(dbIndex, 0); - } - return dbTaskCountMap; - } - - -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskWorker.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskWorker.java index 68db01c14a..261ee743c9 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskWorker.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskWorker.java @@ -26,22 +26,23 @@ import lombok.extern.slf4j.Slf4j; +/** + * 归档任务执行线程 + */ @Slf4j public class ArchiveTaskWorker extends Thread { private final JobInstanceArchiveTask archiveTask; public ArchiveTaskWorker(JobInstanceArchiveTask archiveTask) { - this.setName("ArchiveWorker"); + this.setName("ArchiveWorker-" + archiveTask.getTaskId()); this.archiveTask = archiveTask; } @Override public void run() { try { - log.info("Archive task begin"); archiveTask.execute(); - log.info("Archive task finished"); } catch (Throwable e) { log.warn("Thread interrupted!"); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobExecuteArchiveManage.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobExecuteArchiveManage.java deleted file mode 100644 index ed1a7867d6..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobExecuteArchiveManage.java +++ /dev/null @@ -1,542 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.context.SmartLifecycle; - -@Slf4j -public class JobExecuteArchiveManage implements SmartLifecycle { -// -// private final ArchiveDBProperties archiveDBProperties; -// private final ExecutorService archiveExecutor; -// private final ArchiveProgressService archiveProgressService; -// private final TaskInstanceRecordDAO taskInstanceRecordDAO; -// private final StepInstanceRecordDAO stepInstanceRecordDAO; -// private final StepInstanceScriptRecordDAO stepInstanceScriptRecordDAO; -// private final StepInstanceFileRecordDAO stepInstanceFileRecordDAO; -// private final StepInstanceConfirmRecordDAO stepInstanceConfirmRecordDAO; -// private final StepInstanceVariableRecordDAO stepInstanceVariableRecordDAO; -// private final TaskInstanceVariableRecordDAO taskInstanceVariableRecordDAO; -// private final OperationLogRecordDAO operationLogRecordDAO; -// private final FileSourceTaskLogRecordDAO fileSourceTaskLogRecordDAO; -// private final GseTaskRecordDAO gseTaskRecordDAO; -// private final GseScriptAgentTaskRecordDAO gseScriptAgentTaskRecordDAO; -// private final GseFileAgentTaskRecordDAO gseFileAgentTaskRecordDAO; -// private final GseScriptExecuteObjTaskRecordDAO gseScriptExecuteObjTaskRecordDAO; -// private final GseFileExecuteObjTaskRecordDAO gseFileExecuteObjTaskRecordDAO; -// private final StepInstanceRollingTaskRecordDAO stepInstanceRollingTaskRecordDAO; -// private final RollingConfigRecordDAO rollingConfigRecordDAO; -// private final TaskInstanceHostRecordDAO taskInstanceHostRecordDAO; -// private final JobInstanceColdDAO jobInstanceColdDAO; -// private final ArchiveTaskLock archiveTaskLock; -// private final ArchiveErrorTaskCounter archiveErrorTaskCounter; -// -// -// /** -// * whether this component is currently running(Spring Lifecycle isRunning method) -// */ -// private volatile boolean running = false; -// -// public JobExecuteArchiveManage(TaskInstanceRecordDAO taskInstanceRecordDAO, -// StepInstanceRecordDAO stepInstanceRecordDAO, -// StepInstanceScriptRecordDAO stepInstanceScriptRecordDAO, -// StepInstanceFileRecordDAO stepInstanceFileRecordDAO, -// StepInstanceConfirmRecordDAO stepInstanceConfirmRecordDAO, -// StepInstanceVariableRecordDAO stepInstanceVariableRecordDAO, -// TaskInstanceVariableRecordDAO taskInstanceVariableRecordDAO, -// OperationLogRecordDAO operationLogRecordDAO, -// FileSourceTaskLogRecordDAO fileSourceTaskLogRecordDAO, -// GseTaskRecordDAO gseTaskRecordDAO, -// GseScriptAgentTaskRecordDAO gseScriptAgentTaskRecordDAO, -// GseFileAgentTaskRecordDAO gseFileAgentTaskRecordDAO, -// GseScriptExecuteObjTaskRecordDAO gseScriptExecuteObjTaskRecordDAO, -// GseFileExecuteObjTaskRecordDAO gseFileExecuteObjTaskRecordDAO, -// StepInstanceRollingTaskRecordDAO stepInstanceRollingTaskRecordDAO, -// RollingConfigRecordDAO rollingConfigRecordDAO, -// TaskInstanceHostRecordDAO taskInstanceHostRecordDAO, -// JobInstanceColdDAO jobInstanceColdDAO, -// ArchiveProgressService archiveProgressService, -// ArchiveDBProperties archiveDBProperties, -// ExecutorService archiveExecutor, -// ArchiveTaskLock archiveTaskLock, -// ArchiveErrorTaskCounter archiveErrorTaskCounter) { -// log.info("Init JobExecuteArchiveManage! archiveConfig: {}", archiveDBProperties); -// this.archiveDBProperties = archiveDBProperties; -// this.archiveProgressService = archiveProgressService; -// this.archiveExecutor = archiveExecutor; -// this.taskInstanceRecordDAO = taskInstanceRecordDAO; -// this.stepInstanceRecordDAO = stepInstanceRecordDAO; -// this.stepInstanceScriptRecordDAO = stepInstanceScriptRecordDAO; -// this.stepInstanceFileRecordDAO = stepInstanceFileRecordDAO; -// this.stepInstanceConfirmRecordDAO = stepInstanceConfirmRecordDAO; -// this.stepInstanceVariableRecordDAO = stepInstanceVariableRecordDAO; -// this.taskInstanceVariableRecordDAO = taskInstanceVariableRecordDAO; -// this.operationLogRecordDAO = operationLogRecordDAO; -// this.fileSourceTaskLogRecordDAO = fileSourceTaskLogRecordDAO; -// this.gseTaskRecordDAO = gseTaskRecordDAO; -// this.gseScriptAgentTaskRecordDAO = gseScriptAgentTaskRecordDAO; -// this.gseFileAgentTaskRecordDAO = gseFileAgentTaskRecordDAO; -// this.gseScriptExecuteObjTaskRecordDAO = gseScriptExecuteObjTaskRecordDAO; -// this.gseFileExecuteObjTaskRecordDAO = gseFileExecuteObjTaskRecordDAO; -// this.stepInstanceRollingTaskRecordDAO = stepInstanceRollingTaskRecordDAO; -// this.rollingConfigRecordDAO = rollingConfigRecordDAO; -// this.taskInstanceHostRecordDAO = taskInstanceHostRecordDAO; -// this.jobInstanceColdDAO = jobInstanceColdDAO; -// this.archiveTaskLock = archiveTaskLock; -// this.archiveErrorTaskCounter = archiveErrorTaskCounter; -// } -// -// @Scheduled(cron = "${job.backup.archive.execute.cron:0 0 4 * * *}") -// public void cronArchive() { -// archive(archiveDBProperties); -// } -// -// public void archive(ArchiveDBProperties archiveDBProperties) { -// try { -// new ArchiveThread(archiveDBProperties).start(); -// } catch (Throwable e) { -// log.error("Error while archive job_execute data", e); -// } -// } -// -// @Override -// public void start() { -// this.running = true; -// } -// -// @Override -// public void stop() { -// log.info("Stop JobExecuteArchiveManage!"); -// archiveTaskLock.unlockAll(); -// log.info("Release all archive locks when stop!"); -// this.running = false; -// } -// -// @Override -// public boolean isRunning() { -// return this.running; -// } -// -// -// class ArchiveThread extends Thread { -// private final ArchiveDBProperties archiveDBProperties; -// -// -// ArchiveThread(ArchiveDBProperties archiveDBProperties) { -// this.archiveDBProperties = archiveDBProperties; -// this.setName("Data-Archive-Thread"); -// } -// -// @Override -// public void run() { -//// try { -//// log.info("Job Execute archive task begin, archiveConfig: {}", archiveDBProperties); -//// if (archiveDBProperties.isEnabled()) { -//// doArchive(getEndTime(archiveDBProperties.getKeepDays())); -//// } else { -//// log.info("Archive tasks are disabled, skip archive"); -//// } -//// log.info("Job Execute archive task finished"); -//// } catch (InterruptedException e) { -//// log.warn("Thread interrupted!"); -//// } -// } -// -// private Long getEndTime(int archiveDays) { -// DateTime now = DateTime.now(); -// // 置为前一天天 24:00:00 -// long todayMaxMills = now.minusMillis(now.getMillisOfDay()).getMillis(); -// -// //减掉当前xx天后 -// long archiveMills = archiveDays * 24 * 3600 * 1000L; -// return todayMaxMills - archiveMills; -// } -// -// -// private void doArchive(Long endTime) throws InterruptedException { -// try { -// log.info("Start job execute archive before {}", endTime); -// -// long maxNeedArchiveTaskInstanceId = computeMaxNeedArchiveTaskInstanceId(endTime); -// long maxNeedArchiveStepInstanceId = -// computeMaxNeedArchiveStepInstanceId(maxNeedArchiveTaskInstanceId); -// -// log.info("Compute archive instance id range, maxNeedArchiveTaskInstanceId: {}, " + -// "maxNeedArchiveStepInstanceId: {}", maxNeedArchiveTaskInstanceId, -// maxNeedArchiveStepInstanceId); -// -// ArchiveSummaryHolder.getInstance().init(endTime); -// archive(maxNeedArchiveTaskInstanceId, maxNeedArchiveStepInstanceId); -// ArchiveSummaryHolder.getInstance().print(); -// -// log.info("Job execute archive before {} success", endTime); -// } catch (InterruptedException e) { -// throw e; -// } catch (Throwable e) { -// String msg = MessageFormatter.format( -// "Error while do archive!|{}", -// endTime -// ).getMessage(); -// log.error(msg, e); -// } -// } -// -// public long computeMaxNeedArchiveTaskInstanceId(Long endTime) { -// long lastArchivedId = getLastArchiveId(taskInstanceRecordDAO); -// long maxId = taskInstanceRecordDAO.getMaxId(endTime); -// return Math.max(lastArchivedId, maxId); -// } -// -// public long computeMaxNeedArchiveStepInstanceId(Long taskInstanceId) { -// long lastArchivedId = getLastArchiveId(stepInstanceRecordDAO); -// long maxId = stepInstanceRecordDAO.getMaxId(taskInstanceId); -// return Math.max(lastArchivedId, maxId); -// } -// -// private long getLastArchiveId(JobInstanceHotRecordDAO jobInstanceHotRecordDAO) { -// String tableName = jobInstanceHotRecordDAO.getTable().getName().toLowerCase(); -// ArchiveProgressDTO archiveProgress = -// archiveProgressService.queryArchiveProgress(tableName); -// return archiveProgress != null ? archiveProgress.getLastBackupId() : 0L; -// } -// -// private void archive(long maxNeedArchiveTaskInstanceId, long maxNeedArchiveStepInstanceId) -// throws InterruptedException { -// CountDownLatch countDownLatch = new CountDownLatch(17); -// log.info("Submitting archive task..."); -// -// // task_instance -// addTaskInstanceArchiveTask(maxNeedArchiveTaskInstanceId, countDownLatch); -// // step_instance -// addStepInstanceArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); -// // step_instance_confirm -// addStepInstanceConfirmArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); -// // step_instance_file -// addStepInstanceFileArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); -// // step_instance_script -// addStepInstanceScriptArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); -// // file_source_task_log -// addFileSourceTaskLogArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); -// // task_instance_variable -// addTaskInstanceVariableArchiveTask(maxNeedArchiveTaskInstanceId, countDownLatch); -// // step_instance_variable -// addStepInstanceVariableArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); -// // operation_log -// addOperationLogArchiveTask(maxNeedArchiveTaskInstanceId, countDownLatch); -// // gse_task -// addGseTaskArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); -// // gse_script_agent_task -// addGseScriptAgentTaskArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); -// // gse_file_agent_task -// addGseFileAgentTaskArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); -// // step_instance_rolling_task -// addStepInstanceRollingTaskArchiveTask(maxNeedArchiveStepInstanceId, countDownLatch); -// // rolling_config -// addRollingConfigArchiveTask(maxNeedArchiveTaskInstanceId, countDownLatch); -// // task_instance_host -// addTaskInstanceHostArchiveTask(maxNeedArchiveTaskInstanceId, countDownLatch); -// // gse_script_execute_obj_task -// addGseScriptExecuteObjTaskArchiveTask(maxNeedArchiveTaskInstanceId, countDownLatch); -// // gse_file_execute_obj_task -// addGseFileExecuteObjTaskArchiveTask(maxNeedArchiveTaskInstanceId, countDownLatch); -// -// log.info("Archive task submitted. Waiting for complete..."); -// countDownLatch.await(); -// -// log.info("Archive task execute completed."); -// } -// -// private void addTaskInstanceArchiveTask(Long maxNeedArchiveTaskInstanceId, -// CountDownLatch countDownLatch) { -// archiveExecutor.execute(() -> -// new TaskInstanceArchivist( -// taskInstanceRecordDAO, -// jobInstanceColdDAO, -// archiveProgressService, -// archiveDBProperties, -// archiveTaskLock, -// maxNeedArchiveTaskInstanceId, -// countDownLatch, -// archiveErrorTaskCounter -// ).archive()); -// } -// -// private void addStepInstanceArchiveTask(Long maxNeedArchiveStepInstanceId, -// CountDownLatch countDownLatch) { -// archiveExecutor.execute(() -> -// new StepInstanceArchivist( -// stepInstanceRecordDAO, -// jobInstanceColdDAO, -// archiveProgressService, -// archiveDBProperties, -// archiveTaskLock, -// maxNeedArchiveStepInstanceId, -// countDownLatch, -// archiveErrorTaskCounter) -// .archive()); -// } -// -// private void addStepInstanceConfirmArchiveTask(Long maxNeedArchiveStepInstanceId, -// CountDownLatch countDownLatch) { -// archiveExecutor.execute(() -> -// new StepInstanceConfirmArchivist( -// stepInstanceConfirmRecordDAO, -// jobInstanceColdDAO, -// archiveProgressService, -// archiveDBProperties, -// archiveTaskLock, -// maxNeedArchiveStepInstanceId, -// countDownLatch, -// archiveErrorTaskCounter) -// .archive()); -// } -// -// private void addStepInstanceFileArchiveTask(Long maxNeedArchiveStepInstanceId, -// CountDownLatch countDownLatch) { -// archiveExecutor.execute(() -> -// new StepInstanceFileArchivist( -// stepInstanceFileRecordDAO, -// jobInstanceColdDAO, -// archiveProgressService, -// archiveDBProperties, -// archiveTaskLock, -// maxNeedArchiveStepInstanceId, -// countDownLatch, -// archiveErrorTaskCounter) -// .archive()); -// } -// -// private void addStepInstanceScriptArchiveTask(Long maxNeedArchiveStepInstanceId, -// CountDownLatch countDownLatch) { -// archiveExecutor.execute(() -> -// new StepInstanceScriptArchivist( -// stepInstanceScriptRecordDAO, -// jobInstanceColdDAO, -// archiveProgressService, -// archiveDBProperties, -// archiveTaskLock, -// maxNeedArchiveStepInstanceId, -// countDownLatch, -// archiveErrorTaskCounter) -// .archive()); -// } -// -// private void addFileSourceTaskLogArchiveTask(Long maxNeedArchiveStepInstanceId, -// CountDownLatch countDownLatch) { -// archiveExecutor.execute(() -> -// new FileSourceTaskLogArchivist( -// fileSourceTaskLogRecordDAO, -// jobInstanceColdDAO, -// archiveProgressService, -// archiveDBProperties, -// archiveTaskLock, -// maxNeedArchiveStepInstanceId, -// countDownLatch, -// archiveErrorTaskCounter) -// .archive()); -// } -// -// private void addTaskInstanceVariableArchiveTask(Long maxNeedArchiveTaskInstanceId, -// CountDownLatch countDownLatch) { -// archiveExecutor.execute(() -> -// new TaskInstanceVariableArchivist( -// taskInstanceVariableRecordDAO, -// jobInstanceColdDAO, -// archiveProgressService, -// archiveDBProperties, -// archiveTaskLock, -// maxNeedArchiveTaskInstanceId, -// countDownLatch, -// archiveErrorTaskCounter -// ).archive()); -// } -// -// private void addStepInstanceVariableArchiveTask(Long maxNeedArchiveStepInstanceId, -// CountDownLatch countDownLatch) { -// archiveExecutor.execute(() -> -// new StepInstanceVariableArchivist( -// stepInstanceVariableRecordDAO, -// jobInstanceColdDAO, -// archiveProgressService, -// archiveDBProperties, -// archiveTaskLock, -// maxNeedArchiveStepInstanceId, -// countDownLatch, -// archiveErrorTaskCounter) -// .archive()); -// } -// -// private void addOperationLogArchiveTask(Long maxNeedArchiveTaskInstanceId, -// CountDownLatch countDownLatch) { -// archiveExecutor.execute(() -> -// new OperationLogArchivist( -// operationLogRecordDAO, -// jobInstanceColdDAO, -// archiveProgressService, -// archiveDBProperties, -// archiveTaskLock, -// maxNeedArchiveTaskInstanceId, -// countDownLatch, -// archiveErrorTaskCounter -// ).archive()); -// } -// -// private void addGseTaskArchiveTask(Long maxNeedArchiveStepInstanceId, CountDownLatch countDownLatch) { -// archiveExecutor.execute(() -> -// new GseTaskArchivist( -// gseTaskRecordDAO, -// jobInstanceColdDAO, -// archiveProgressService, -// archiveDBProperties, -// archiveTaskLock, -// maxNeedArchiveStepInstanceId, -// countDownLatch, -// archiveErrorTaskCounter) -// .archive()); -// } -// -// private void addGseScriptAgentTaskArchiveTask(Long maxNeedArchiveStepInstanceId, -// CountDownLatch countDownLatch) { -// archiveExecutor.execute(() -> -// new GseScriptAgentTaskArchivist( -// gseScriptAgentTaskRecordDAO, -// jobInstanceColdDAO, -// archiveProgressService, -// archiveDBProperties, -// archiveTaskLock, -// maxNeedArchiveStepInstanceId, -// countDownLatch, -// archiveErrorTaskCounter) -// .archive()); -// } -// -// private void addGseFileAgentTaskArchiveTask(Long maxNeedArchiveStepInstanceId, -// CountDownLatch countDownLatch) { -// archiveExecutor.execute(() -> -// new GseFileAgentTaskArchivist( -// gseFileAgentTaskRecordDAO, -// jobInstanceColdDAO, -// archiveProgressService, -// archiveDBProperties, -// archiveTaskLock, -// maxNeedArchiveStepInstanceId, -// countDownLatch, -// archiveErrorTaskCounter) -// .archive()); -// } -// -// private void addStepInstanceRollingTaskArchiveTask(Long maxNeedArchiveStepInstanceId, -// CountDownLatch countDownLatch) { -// archiveExecutor.execute(() -> -// new StepInstanceRollingTaskArchivist( -// stepInstanceRollingTaskRecordDAO, -// jobInstanceColdDAO, -// archiveProgressService, -// archiveDBProperties, -// archiveTaskLock, -// maxNeedArchiveStepInstanceId, -// countDownLatch, -// archiveErrorTaskCounter) -// .archive()); -// } -// -// private void addGseScriptExecuteObjTaskArchiveTask(Long maxNeedArchiveTaskInstanceId, -// CountDownLatch countDownLatch) { -// archiveExecutor.execute(() -> -// new GseScriptExecuteObjTaskArchivist( -// gseScriptExecuteObjTaskRecordDAO, -// jobInstanceColdDAO, -// archiveProgressService, -// archiveDBProperties, -// archiveTaskLock, -// maxNeedArchiveTaskInstanceId, -// countDownLatch, -// archiveErrorTaskCounter) -// .archive()); -// } -// -// private void addGseFileExecuteObjTaskArchiveTask(Long maxNeedArchiveTaskInstanceId, -// CountDownLatch countDownLatch) { -// archiveExecutor.execute(() -> -// new GseFileExecuteObjTaskArchivist( -// gseFileExecuteObjTaskRecordDAO, -// jobInstanceColdDAO, -// archiveProgressService, -// archiveDBProperties, -// archiveTaskLock, -// maxNeedArchiveTaskInstanceId, -// countDownLatch, -// archiveErrorTaskCounter) -// .archive()); -// } -// -// private void addRollingConfigArchiveTask(Long maxNeedArchiveTaskInstanceId, -// CountDownLatch countDownLatch) { -// archiveExecutor.execute(() -> -// new RollingConfigArchivist( -// rollingConfigRecordDAO, -// jobInstanceColdDAO, -// archiveProgressService, -// archiveDBProperties, -// archiveTaskLock, -// maxNeedArchiveTaskInstanceId, -// countDownLatch, -// archiveErrorTaskCounter -// ).archive()); -// } -// -// private void addTaskInstanceHostArchiveTask(Long maxNeedArchiveTaskInstanceId, -// CountDownLatch countDownLatch) { -// archiveExecutor.execute(() -> -// new TaskInstanceHostArchivist( -// taskInstanceHostRecordDAO, -// jobInstanceColdDAO, -// archiveProgressService, -// archiveDBProperties, -// archiveTaskLock, -// maxNeedArchiveTaskInstanceId, -// countDownLatch, -// archiveErrorTaskCounter -// ).archive()); -// } -// } - - - @Override - public void start() { - - } - - @Override - public void stop() { - - } - - @Override - public boolean isRunning() { - return false; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceAppDataArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceAppDataArchiveTask.java deleted file mode 100644 index 7f6bae488c..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceAppDataArchiveTask.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive; - -import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; -import com.tencent.bk.job.backup.archive.dao.JobInstanceHotRecordDAO; -import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; -import com.tencent.bk.job.backup.config.ArchiveProperties; -import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; -import com.tencent.bk.job.execute.model.tables.records.TaskInstanceRecord; - -import java.util.List; - -/** - * 作业实例数据归档-分库分表场景下，冗余的基于业务 ID 进行分库分表的数据 - */ -public class JobInstanceAppDataArchiveTask extends AbstractJobInstanceArchiveTask { - - public JobInstanceAppDataArchiveTask(JobInstanceHotRecordDAO jobInstanceHotRecordDAO, - JobInstanceColdDAO jobInstanceColdDAO, - ArchiveProperties archiveDbProperties, - ArchiveTaskLock archiveTaskLock, - ArchiveErrorTaskCounter archiveErrorTaskCounter, - JobInstanceArchiveTaskInfo archiveTask, - ArchiveTaskService archiveTaskService) { - super(jobInstanceHotRecordDAO, - jobInstanceColdDAO, - archiveDbProperties, - archiveTaskLock, - archiveErrorTaskCounter, - archiveTask, - archiveTaskService); - } - - @Override - protected void backupJobInstanceToColdDb(List jobInstances, List jobInstanceIds) { - - } - - @Override - protected void deleteJobInstanceHotData(List jobInstanceIds) { - - } - - @Override - protected List readSortedJobInstanceFromHotDB(Long fromTimestamp, - Long endTimestamp, - Long fromJobInstanceId, - int limit) { - return null; - } - - @Override - protected Long extractJobInstanceId(TaskInstanceRecord record) { - return null; - } - - @Override - protected Long extractJobInstanceCreateTime(TaskInstanceRecord record) { - return null; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java index 6413940a4f..33e0898d4f 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java @@ -27,6 +27,7 @@ import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceRecordDAO; import com.tencent.bk.job.backup.archive.model.DbDataNode; import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; +import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; import com.tencent.bk.job.backup.config.ArchiveProperties; import com.tencent.bk.job.backup.constant.ArchiveTaskStatusEnum; import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; @@ -75,6 +76,7 @@ public void generate() { LocalDateTime endDateTime = unixTimestampToUtcLocalDateTime( getEndTime(archiveProperties.getKeepDays())); while (startDateTime.isBefore(endDateTime)) { + // 水平分库分表 if (isHorizontalShardingEnabled()) { // 作业实例数据归档任务 archiveTaskList.addAll(buildArchiveTasksForShardingDataNodes(ArchiveTaskTypeEnum.JOB_INSTANCE, @@ -83,6 +85,7 @@ public void generate() { archiveTaskList.addAll(buildArchiveTasksForShardingDataNodes(ArchiveTaskTypeEnum.JOB_INSTANCE_APP, startDateTime, archiveProperties.getTasks().getTaskInstance().getShardingDataNodes())); } else { + // 单db DbDataNode dbDataNode = new DbDataNode(DbDataNodeTypeEnum.STANDALONE, null, null, null); archiveTaskList.add(buildArchiveTask(ArchiveTaskTypeEnum.JOB_INSTANCE, startDateTime, dbDataNode)); } @@ -102,6 +105,7 @@ private List buildArchiveTasksForShardingDataNodes( List shardingDataNodes) { List tasks = new ArrayList<>(); + // 任务：dataNode + day + hour shardingDataNodes.forEach(dataNode -> { int dbNodeCount = dataNode.getDbCount(); int tableNodeCount = dataNode.getTableCount(); diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java index 7bf58efe48..1559f76552 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java @@ -27,13 +27,12 @@ import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceRecordDAO; import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; +import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; import com.tencent.bk.job.backup.config.ArchiveProperties; import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; -import com.tencent.bk.job.common.sharding.mysql.config.ShardingProperties; import com.tencent.bk.job.common.util.ThreadUtils; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; import org.springframework.context.SmartLifecycle; import java.util.List; @@ -57,8 +56,6 @@ public class JobInstanceArchiveTaskScheduler implements SmartLifecycle { private final ArchiveProperties archiveProperties; - private final ShardingProperties shardingProperties; - private final JobInstanceArchiveTaskScheduleLock jobInstanceArchiveTaskScheduleLock; private final JobInstanceSubTableArchivers jobInstanceSubTableArchivers; @@ -97,7 +94,6 @@ public class JobInstanceArchiveTaskScheduler implements SmartLifecycle { public JobInstanceArchiveTaskScheduler(ArchiveTaskService archiveTaskService, TaskInstanceRecordDAO taskInstanceRecordDAO, ArchiveProperties archiveProperties, - ShardingProperties shardingProperties, JobInstanceArchiveTaskScheduleLock jobInstanceArchiveTaskScheduleLock, JobInstanceSubTableArchivers jobInstanceSubTableArchivers, JobInstanceColdDAO jobInstanceColdDAO, @@ -107,7 +103,6 @@ public JobInstanceArchiveTaskScheduler(ArchiveTaskService archiveTaskService, this.archiveTaskService = archiveTaskService; this.taskInstanceRecordDAO = taskInstanceRecordDAO; this.archiveProperties = archiveProperties; - this.shardingProperties = shardingProperties; this.jobInstanceArchiveTaskScheduleLock = jobInstanceArchiveTaskScheduleLock; this.jobInstanceSubTableArchivers = jobInstanceSubTableArchivers; this.jobInstanceColdDAO = jobInstanceColdDAO; @@ -135,28 +130,38 @@ public void schedule() { ThreadUtils.sleep(1000L); continue; } - if (runningTasksCount.get() >= archiveProperties.getConcurrent()) { - // 休眠一分钟，等待并行任务减少 - log.info("Running archive task count exceed concurrent limit : {}, wait 60s", - archiveProperties.getConcurrent()); - ThreadUtils.sleep(1000 * 60L); - continue; - } + // 获取待调度的任务列表 - List needScheduleTasks = - archiveTaskService.listScheduleTasks(ArchiveTaskTypeEnum.JOB_INSTANCE, 100); - if (CollectionUtils.isEmpty(needScheduleTasks)) { + Map scheduleTasksGroupByDb = + archiveTaskService.countScheduleTasksGroupByDb(ArchiveTaskTypeEnum.JOB_INSTANCE); + if (scheduleTasksGroupByDb.isEmpty()) { // 所有任务都已经被调度完成，退出本次任务调度 + log.info("All archive task is done!"); return; } // 获取正在执行中的任务列表 List runningTasks = archiveTaskService.listRunningTasks(ArchiveTaskTypeEnum.JOB_INSTANCE); + // 对待调度的任务进行优先级排序，保证同一个 db 上的归档任务数量尽可能均衡，避免出现db 热点 - ArchiveTaskPrioritySorter.sort(runningTasks, shardingProperties.getDbNodeCount(), needScheduleTasks); + ArchiveDbNodePriorityEvaluator.DbNodeTasksInfo highestPriorityDbNodeTasksInfo = + ArchiveDbNodePriorityEvaluator.evaluatePriorityDbNode(runningTasks, scheduleTasksGroupByDb); + if (highestPriorityDbNodeTasksInfo.getRunningTaskCount() >= archiveProperties.getConcurrent()) { + // 休眠一分钟，等待并行任务减少 + log.info("Running archive task count exceed concurrent limit : {}, wait 60s", + archiveProperties.getConcurrent()); + ThreadUtils.sleep(1000 * 60L); + continue; + } + + // 获取优先级最高的归档任务 + String dbNodeId = highestPriorityDbNodeTasksInfo.getDbNodeId(); + JobInstanceArchiveTaskInfo archiveTaskInfo = + archiveTaskService.getFirstScheduleArchiveTaskByDb(ArchiveTaskTypeEnum.JOB_INSTANCE, dbNodeId); - startArchiveTask(needScheduleTasks.get(0)); + // 启动任务 + startArchiveTask(archiveTaskInfo); } } finally { this.scheduling = false; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceHotArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceHotArchivist.java deleted file mode 100644 index 70ee4df5a2..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceHotArchivist.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive; - -import org.apache.commons.collections4.CollectionUtils; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -public class JobInstanceHotArchivist { - /** - * 对一个批次数据进行备份 - * - * @param start 数据起始记录ID - * @param stop 数据终止记录ID - * @return 备份结果 - */ - private AbstractJobInstanceArchiveTask.BackupResult backupRecords(List jobInstanceIds) throws IOException { - if (lastBackupId >= stop) { - // 说明数据已经备份过，跳过 - log.info("[{}] Record is already backup, skip. lastBackId: {}", tableName, lastBackupId); - return new AbstractJobInstanceArchiveTask.BackupResult(0L, 0L, 0L, 0L); - } - long startId = start; - if (lastBackupId > start) { - // 从上次备份结束的 ID 位置开始 - startId = lastBackupId; - } - - List recordList = new ArrayList<>(readRowLimit); - long backupRows = 0; - long readRows = 0; - long readCost = 0; - long writeCost = 0; - long offset = 0L; - List records; - do { - // 选取start= batchInsertRowSize) { - long writeStartTime = System.currentTimeMillis(); - int insertRows = insertAndReset(recordList); - writeCost += (System.currentTimeMillis() - writeStartTime); - backupRows += insertRows; - } - offset += readRowLimit; - - } while (records.size() == readRowLimit); - - if (CollectionUtils.isNotEmpty(recordList)) { - // 处理没有达到批量插入阈值的最后一个批次的数据 - int insertRows = insertAndReset(recordList); - backupRows += insertRows; - } - - updateArchiveProgress(stop); - - return new AbstractJobInstanceArchiveTask.BackupResult(readRows, backupRows, readCost, writeCost); - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java index b3dc084f8a..690de68f55 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java @@ -27,6 +27,7 @@ import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceRecordDAO; import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; +import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; import com.tencent.bk.job.backup.config.ArchiveProperties; import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; import com.tencent.bk.job.execute.model.tables.TaskInstance; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ArchiveTaskDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ArchiveTaskDAO.java index d0072417c2..3121162f1c 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ArchiveTaskDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ArchiveTaskDAO.java @@ -28,6 +28,7 @@ import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; import java.util.List; +import java.util.Map; /** * 归档任务 DAO @@ -45,7 +46,13 @@ public interface ArchiveTaskDAO { List listRunningTasks(ArchiveTaskTypeEnum taskType); - List listScheduleTasks(ArchiveTaskTypeEnum taskType, int limit); + /** + * 返回根据 db 分组的归档任务数量 + * + * @param taskType 归档任务类型 + * @return key: db 名称; value: 任务数量 + */ + Map countScheduleTasksGroupByDb(ArchiveTaskTypeEnum taskType); /** * 更新归档任务 @@ -53,4 +60,6 @@ public interface ArchiveTaskDAO { * @param archiveTask 归档任务 */ void updateTask(JobInstanceArchiveTaskInfo archiveTask); + + JobInstanceArchiveTaskInfo getFirstScheduleArchiveTaskByDb(ArchiveTaskTypeEnum taskType, String dbNodeId); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/JobInstanceHotRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/JobInstanceHotRecordDAO.java index 104bcfe11f..d67b7397c1 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/JobInstanceHotRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/JobInstanceHotRecordDAO.java @@ -24,7 +24,7 @@ package com.tencent.bk.job.backup.archive.dao; -import com.tencent.bk.job.backup.archive.dao.impl.RecordResultSet; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; import org.jooq.Record; import org.jooq.Table; import org.jooq.TableField; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobExecuteVerticalShardingDSLContextProvider.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ds/JobExecuteVerticalShardingDSLContextProvider.java similarity index 98% rename from src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobExecuteVerticalShardingDSLContextProvider.java rename to src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ds/JobExecuteVerticalShardingDSLContextProvider.java index 7e206ab247..e3fe8cc36d 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobExecuteVerticalShardingDSLContextProvider.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ds/JobExecuteVerticalShardingDSLContextProvider.java @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.backup.archive; +package com.tencent.bk.job.backup.archive.dao.ds; import com.tencent.bk.job.common.mysql.dynamic.ds.VerticalShardingDSLContextProvider; import lombok.extern.slf4j.Slf4j; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceHotRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceHotRecordDAO.java index 793e369c06..7c423045f6 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceHotRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceHotRecordDAO.java @@ -55,7 +55,7 @@ public List listRecords(Collection jobInstanceIds, Long readRowLimit) { return query(getTable(), buildBasicConditions(jobInstanceIds), readRowLimit); } - protected List listRecordsByConditions(List conditions, Long readRowLimit) { + public List listRecordsByConditions(List conditions, Long readRowLimit) { return query(getTable(), conditions, readRowLimit); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java index 1ee04d5ae1..d6db19f3d2 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java @@ -35,15 +35,19 @@ import com.tencent.bk.job.common.mysql.jooq.JooqDataTypeUtil; import org.jooq.DSLContext; import org.jooq.Record; +import org.jooq.Record2; import org.jooq.Result; import org.jooq.TableField; import org.jooq.UpdateSetMoreStep; +import org.jooq.impl.DSL; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; @Repository public class ArchiveTaskDAOImpl implements ArchiveTaskDAO { @@ -54,6 +58,7 @@ public class ArchiveTaskDAOImpl implements ArchiveTaskDAO { private static final TableField[] ALL_FIELDS = { T.TASK_TYPE, T.DATA_NODE, + T.DB_NODE, T.DAY, T.HOUR, T.FROM_TIMESTAMP, @@ -105,6 +110,7 @@ public void saveArchiveTask(JobInstanceArchiveTaskInfo jobInstanceArchiveTaskInf T, T.TASK_TYPE, T.DATA_NODE, + T.DB_NODE, T.DAY, T.HOUR, T.FROM_TIMESTAMP, @@ -116,6 +122,7 @@ public void saveArchiveTask(JobInstanceArchiveTaskInfo jobInstanceArchiveTaskInf .values( JooqDataTypeUtil.toByte(jobInstanceArchiveTaskInfo.getTaskType().getType()), jobInstanceArchiveTaskInfo.getDbDataNode().toDataNodeId(), + jobInstanceArchiveTaskInfo.getDbDataNode().toDbNodeId(), jobInstanceArchiveTaskInfo.getDay(), JooqDataTypeUtil.toByte(jobInstanceArchiveTaskInfo.getHour()), jobInstanceArchiveTaskInfo.getFromTimestamp(), @@ -143,21 +150,20 @@ public List listRunningTasks(ArchiveTaskTypeEnum tas } @Override - public List listScheduleTasks(ArchiveTaskTypeEnum taskType, int limit) { - Result result = ctx.select(ALL_FIELDS) + public Map countScheduleTasksGroupByDb(ArchiveTaskTypeEnum taskType) { + Result> result = ctx.select(T.DB_NODE, DSL.count().as("task_count")) .from(T) .where(T.STATUS.in( JooqDataTypeUtil.toByte(ArchiveTaskStatusEnum.PENDING.getStatus()), JooqDataTypeUtil.toByte(ArchiveTaskStatusEnum.SUSPENDED.getStatus()), JooqDataTypeUtil.toByte(ArchiveTaskStatusEnum.FAIL.getStatus()))) .and(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(taskType.getType()))) - .orderBy(T.DAY.desc(), T.HOUR.desc()) - .limit(limit) + .groupBy(T.DB_NODE) .fetch(); - List tasks = new ArrayList<>(result.size()); - result.forEach(record -> tasks.add(extract(record))); - return tasks; + Map dbAndTaskCount = new HashMap<>(); + result.forEach(record -> dbAndTaskCount.put(record.get(T.DB_NODE), (Integer) record.get("task_count"))); + return dbAndTaskCount; } @Override @@ -185,4 +191,20 @@ public void updateTask(JobInstanceArchiveTaskInfo archiveTask) { .and(T.HOUR.eq(archiveTask.getHour().byteValue())) .execute(); } + + @Override + public JobInstanceArchiveTaskInfo getFirstScheduleArchiveTaskByDb(ArchiveTaskTypeEnum taskType, String dbNodeId) { + Record record = ctx.select(ALL_FIELDS) + .from(T) + .where(T.STATUS.in( + JooqDataTypeUtil.toByte(ArchiveTaskStatusEnum.PENDING.getStatus()), + JooqDataTypeUtil.toByte(ArchiveTaskStatusEnum.SUSPENDED.getStatus()), + JooqDataTypeUtil.toByte(ArchiveTaskStatusEnum.FAIL.getStatus()))) + .and(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(taskType.getType()))) + .and(T.DB_NODE.eq(dbNodeId)) + .orderBy(T.DAY.desc(), T.HOUR.desc(), T.DATA_NODE) + .fetchOne(); + + return extract(record); + } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ExecuteArchiveDAOImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ExecuteArchiveDAOImpl.java deleted file mode 100644 index 0421d13285..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ExecuteArchiveDAOImpl.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.dao.impl; - -import com.tencent.bk.job.backup.archive.dao.ExecuteArchiveDAO; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.jooq.DSLContext; -import org.jooq.Loader; -import org.jooq.LoaderError; -import org.jooq.TableRecord; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.List; - - -@Slf4j -public class ExecuteArchiveDAOImpl implements ExecuteArchiveDAO { - - private static final Logger ARCHIVE_FAILED_LOGGER = LoggerFactory.getLogger("ArchiveFailedLogger"); - - private final DSLContext context; - - public ExecuteArchiveDAOImpl(DSLContext context) { - log.info("Init ExecuteArchiveDAO."); - this.context = context; - } - - @Override - public Integer batchInsert(List> recordList, int bulkSize) throws IOException { - long start = System.currentTimeMillis(); - int successInsertedRecords = 0; - String table = recordList.get(0).getTable().getName(); - boolean success = true; - try { - Loader loader = - context.loadInto(recordList.get(0).getTable()) - // 由于这里是批量写入，jooq 不允许使用 onDuplicateKeyIgnore/onDuplicateKeyUpdate. - // 否则会报错"Cannot apply bulk loading with onDuplicateKey flags" - // 所以这里暂时使用 onDuplicateKeyError 错误处理方式，等后续流程进一步判断是否是主键冲突错误 - // issue 参考：https://github.com/jOOQ/jOOQ/issues/12740 - .onDuplicateKeyError() - .bulkAfter(bulkSize) - .loadRecords(recordList) - .fieldsCorresponding() - .execute(); - successInsertedRecords = loader.stored(); - String bulkInsertResult = successInsertedRecords == recordList.size() ? "success" : "fail"; - log.info( - "InsertBulk: Load {} data|result|{}|executed|{}|processed|{}|stored|{}|ignored|{}|errors|{}", - table, - bulkInsertResult, - loader.executed(), - loader.processed(), - loader.stored(), - loader.ignored(), - loader.errors().size() - ); - if (CollectionUtils.isNotEmpty(loader.errors())) { - for (LoaderError error : loader.errors()) { - ARCHIVE_FAILED_LOGGER.error("Error while load {} data, exception: {}， error row: {}", table, - error.exception().getMessage(), error.row()); - } - if (hasDuplicateError(loader.errors())) { - // 如果存在主键冲突的数据，尝试每一条记录单独插入，就可以使用 onDuplicateKeyIgnore 错误处理方式 - successInsertedRecords = insertSingle(recordList); - } - } - } catch (IOException e) { - String errorMsg = String.format("Error while loading %s data!", table); - log.error(errorMsg, e); - success = false; - throw e; - } finally { - log.info("Load data to {} done! success: {}, total: {}, inserted: {}, cost: {}ms", table, success, - recordList.size(), successInsertedRecords, System.currentTimeMillis() - start); - } - - return successInsertedRecords; - } - - private boolean hasDuplicateError(List errors) { - // 通过 mysql 执行的错误消息判断是否是由于数据唯一性冲突引起的 - return errors.stream().anyMatch( - error -> error.exception().getMessage() != null - && error.exception().getMessage().contains("Duplicate entry")); - } - - private int insertSingle(List> recordList) throws IOException { - int successInsertedRecords; - String table = recordList.get(0).getTable().getName(); - try { - Loader loader = - context.loadInto(recordList.get(0).getTable()) - .onDuplicateKeyIgnore() - .loadRecords(recordList) - .fieldsCorresponding() - .execute(); - successInsertedRecords = loader.stored() + loader.ignored(); - String bulkInsertResult = successInsertedRecords == recordList.size() ? "success" : "fail"; - log.info( - "InsertSingle: Load {} data|result|{}|executed|{}|processed|{}|stored|{}|ignored|{}|errors|{}", - table, - bulkInsertResult, - loader.executed(), - loader.processed(), - loader.stored(), - loader.ignored(), - loader.errors().size() - ); - if (CollectionUtils.isNotEmpty(loader.errors())) { - for (LoaderError error : loader.errors()) { - ARCHIVE_FAILED_LOGGER.error("InsertSingle: Error while load {} data, exception: {}, error row: {}", - table, error.row(), error.exception().getMessage()); - } - } - } catch (IOException e) { - String errorMsg = String.format("Error while loading %s data!", table); - log.error(errorMsg, e); - throw e; - } - return successInsertedRecords; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/FileSourceTaskLogRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/FileSourceTaskLogRecordDAO.java index 993a698155..99292ee398 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/FileSourceTaskLogRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/FileSourceTaskLogRecordDAO.java @@ -25,6 +25,8 @@ package com.tencent.bk.job.backup.archive.dao.impl; +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; import com.tencent.bk.job.execute.model.tables.FileSourceTaskLog; import com.tencent.bk.job.execute.model.tables.records.FileSourceTaskLogRecord; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileAgentTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileAgentTaskRecordDAO.java index 91cc76c771..07b28b5b35 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileAgentTaskRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileAgentTaskRecordDAO.java @@ -24,6 +24,8 @@ package com.tencent.bk.job.backup.archive.dao.impl; +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; import com.tencent.bk.job.execute.model.tables.GseFileAgentTask; import com.tencent.bk.job.execute.model.tables.records.GseFileAgentTaskRecord; @@ -59,8 +61,8 @@ public Table getTable() { } @Override - public TableField getArchiveIdField() { - return TABLE.STEP_INSTANCE_ID; + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; } @Override diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileExecuteObjTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileExecuteObjTaskRecordDAO.java index ce860a61c0..7e2814a2e3 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileExecuteObjTaskRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileExecuteObjTaskRecordDAO.java @@ -24,6 +24,8 @@ package com.tencent.bk.job.backup.archive.dao.impl; +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; import com.tencent.bk.job.execute.model.tables.GseFileExecuteObjTask; import com.tencent.bk.job.execute.model.tables.records.GseFileExecuteObjTaskRecord; @@ -72,7 +74,7 @@ public TableField getJobInstanceIdField() { } public RecordResultSet executeQuery(Collection jobInstanceIds, - long readRowLimit) { + long readRowLimit) { return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( this, jobInstanceIds, diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptAgentTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptAgentTaskRecordDAO.java index 1f19b87a07..9342c48621 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptAgentTaskRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptAgentTaskRecordDAO.java @@ -24,6 +24,8 @@ package com.tencent.bk.job.backup.archive.dao.impl; +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; import com.tencent.bk.job.execute.model.tables.GseScriptAgentTask; import com.tencent.bk.job.execute.model.tables.records.GseScriptAgentTaskRecord; @@ -59,8 +61,8 @@ public Table getTable() { } @Override - public TableField getArchiveIdField() { - return TABLE.STEP_INSTANCE_ID; + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; } @Override @@ -68,14 +70,9 @@ protected Collection> getListRecordsOrderFields() { return ORDER_FIELDS; } - @Override - public TableField getJobInstanceIdField() { - return TABLE.TASK_INSTANCE_ID; - } - @Override public RecordResultSet executeQuery(Collection jobInstanceIds, - long readRowLimit) { + long readRowLimit) { return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( this, jobInstanceIds, diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptExecuteObjTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptExecuteObjTaskRecordDAO.java index 9d4edf0135..2c9367d4ee 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptExecuteObjTaskRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptExecuteObjTaskRecordDAO.java @@ -24,6 +24,8 @@ package com.tencent.bk.job.backup.archive.dao.impl; +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; import com.tencent.bk.job.execute.model.tables.GseScriptExecuteObjTask; import com.tencent.bk.job.execute.model.tables.records.GseScriptExecuteObjTaskRecord; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseTaskRecordDAO.java index 80f7240dcf..95872e8f81 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseTaskRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseTaskRecordDAO.java @@ -24,6 +24,8 @@ package com.tencent.bk.job.backup.archive.dao.impl; +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; import com.tencent.bk.job.execute.model.tables.GseTask; import com.tencent.bk.job.execute.model.tables.records.GseTaskRecord; @@ -67,7 +69,7 @@ public TableField getJobInstanceIdField() { @Override public RecordResultSet executeQuery(Collection jobInstanceIds, - long readRowLimit) { + long readRowLimit) { return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( this, jobInstanceIds, diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/OperationLogRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/OperationLogRecordDAO.java index 29b53805a1..b02d09e6e0 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/OperationLogRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/OperationLogRecordDAO.java @@ -24,6 +24,8 @@ package com.tencent.bk.job.backup.archive.dao.impl; +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; import com.tencent.bk.job.execute.model.tables.OperationLog; import com.tencent.bk.job.execute.model.tables.records.OperationLogRecord; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/RollingConfigRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/RollingConfigRecordDAO.java index 20ecb80ebc..cd6908dc12 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/RollingConfigRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/RollingConfigRecordDAO.java @@ -24,6 +24,8 @@ package com.tencent.bk.job.backup.archive.dao.impl; +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; import com.tencent.bk.job.execute.model.tables.RollingConfig; import com.tencent.bk.job.execute.model.tables.records.RollingConfigRecord; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceConfirmRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceConfirmRecordDAO.java index 07934ce9c9..5a7c758522 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceConfirmRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceConfirmRecordDAO.java @@ -24,6 +24,8 @@ package com.tencent.bk.job.backup.archive.dao.impl; +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; import com.tencent.bk.job.execute.model.tables.StepInstanceConfirm; import com.tencent.bk.job.execute.model.tables.records.StepInstanceConfirmRecord; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceFileRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceFileRecordDAO.java index e10b712068..f528a0784c 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceFileRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceFileRecordDAO.java @@ -24,6 +24,8 @@ package com.tencent.bk.job.backup.archive.dao.impl; +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; import com.tencent.bk.job.execute.model.tables.StepInstanceFile; import com.tencent.bk.job.execute.model.tables.records.StepInstanceFileRecord; @@ -71,7 +73,7 @@ protected Collection> getListRecordsOrderFields() { @Override public RecordResultSet executeQuery(Collection jobInstanceIds, - long readRowLimit) { + long readRowLimit) { return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( this, jobInstanceIds, diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRecordDAO.java index 14c995fdf6..0488a8e747 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRecordDAO.java @@ -24,6 +24,8 @@ package com.tencent.bk.job.backup.archive.dao.impl; +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; import com.tencent.bk.job.execute.model.tables.StepInstance; import com.tencent.bk.job.execute.model.tables.records.StepInstanceRecord; @@ -71,7 +73,7 @@ protected Collection> getListRecordsOrderFields() { @Override public RecordResultSet executeQuery(Collection jobInstanceIds, - long readRowLimit) { + long readRowLimit) { return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( this, jobInstanceIds, diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRollingTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRollingTaskRecordDAO.java index d80b2188a4..eb19f5661b 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRollingTaskRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRollingTaskRecordDAO.java @@ -24,6 +24,8 @@ package com.tencent.bk.job.backup.archive.dao.impl; +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; import com.tencent.bk.job.execute.model.tables.StepInstanceRollingTask; import com.tencent.bk.job.execute.model.tables.records.StepInstanceRollingTaskRecord; @@ -69,7 +71,7 @@ protected Collection> getListRecordsOrderFields() { @Override public RecordResultSet executeQuery(Collection jobInstanceIds, - long readRowLimit) { + long readRowLimit) { return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( this, jobInstanceIds, diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceScriptRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceScriptRecordDAO.java index 2b41c56695..5e6ea82fc7 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceScriptRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceScriptRecordDAO.java @@ -24,6 +24,8 @@ package com.tencent.bk.job.backup.archive.dao.impl; +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; import com.tencent.bk.job.execute.model.tables.StepInstanceScript; import com.tencent.bk.job.execute.model.tables.records.StepInstanceScriptRecord; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceVariableRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceVariableRecordDAO.java index 74f207cbf4..654f1104db 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceVariableRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceVariableRecordDAO.java @@ -24,6 +24,8 @@ package com.tencent.bk.job.backup.archive.dao.impl; +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; import com.tencent.bk.job.execute.model.tables.StepInstanceVariable; import com.tencent.bk.job.execute.model.tables.records.StepInstanceVariableRecord; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceHostRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceHostRecordDAO.java index 6b746187da..d1897e3279 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceHostRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceHostRecordDAO.java @@ -24,6 +24,8 @@ package com.tencent.bk.job.backup.archive.dao.impl; +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; import com.tencent.bk.job.execute.model.tables.TaskInstanceHost; import com.tencent.bk.job.execute.model.tables.records.TaskInstanceHostRecord; @@ -67,7 +69,7 @@ protected Collection> getListRecordsOrderFields() { @Override public RecordResultSet executeQuery(Collection jobInstanceIds, - long readRowLimit) { + long readRowLimit) { return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( this, jobInstanceIds, diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceRecordDAO.java index 388499f460..e74d07ac12 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceRecordDAO.java @@ -24,6 +24,8 @@ package com.tencent.bk.job.backup.archive.dao.impl; +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; import com.tencent.bk.job.execute.model.tables.TaskInstance; import com.tencent.bk.job.execute.model.tables.records.TaskInstanceRecord; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceVariableRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceVariableRecordDAO.java index 41553e3b5a..b48a28cdb6 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceVariableRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceVariableRecordDAO.java @@ -24,6 +24,8 @@ package com.tencent.bk.job.backup.archive.dao.impl; +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; import com.tencent.bk.job.execute.model.tables.TaskInstanceVariable; import com.tencent.bk.job.execute.model.tables.records.TaskInstanceVariableRecord; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceRecordResultSet.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/AbstractJobInstanceRecordResultSet.java similarity index 96% rename from src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceRecordResultSet.java rename to src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/AbstractJobInstanceRecordResultSet.java index f0b052ada4..f4e0563106 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceRecordResultSet.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/AbstractJobInstanceRecordResultSet.java @@ -22,8 +22,9 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.backup.archive.dao.impl; +package com.tencent.bk.job.backup.archive.dao.resultset; +import com.tencent.bk.job.backup.archive.dao.impl.AbstractJobInstanceHotRecordDAO; import org.apache.commons.collections4.CollectionUtils; import org.jooq.Condition; import org.jooq.Record; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/JobInstanceRecordResultSetFactory.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/JobInstanceRecordResultSetFactory.java similarity index 96% rename from src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/JobInstanceRecordResultSetFactory.java rename to src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/JobInstanceRecordResultSetFactory.java index a45160f50e..970196964c 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/JobInstanceRecordResultSetFactory.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/JobInstanceRecordResultSetFactory.java @@ -22,8 +22,9 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.backup.archive.dao.impl; +package com.tencent.bk.job.backup.archive.dao.resultset; +import com.tencent.bk.job.backup.archive.dao.impl.AbstractJobInstanceHotRecordDAO; import org.jooq.Condition; import org.jooq.Record; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/RecordResultSet.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/RecordResultSet.java similarity index 96% rename from src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/RecordResultSet.java rename to src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/RecordResultSet.java index c030b749a9..e5d7e72fc3 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/RecordResultSet.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/RecordResultSet.java @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.backup.archive.dao.impl; +package com.tencent.bk.job.backup.archive.dao.resultset; import org.jooq.Record; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/AbstractJobInstanceSubTableArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/AbstractJobInstanceSubTableArchiver.java index 8b014c75b7..86fa0f4a3b 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/AbstractJobInstanceSubTableArchiver.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/AbstractJobInstanceSubTableArchiver.java @@ -28,7 +28,7 @@ import com.tencent.bk.job.backup.archive.JobInstanceSubTableArchiver; import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; import com.tencent.bk.job.backup.archive.dao.impl.AbstractJobInstanceHotRecordDAO; -import com.tencent.bk.job.backup.archive.dao.impl.RecordResultSet; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; import org.apache.commons.collections4.CollectionUtils; import org.jooq.TableRecord; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DbDataNode.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DbDataNode.java index 665569695b..4f5b70ac0e 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DbDataNode.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DbDataNode.java @@ -52,6 +52,8 @@ public class DbDataNode { */ private Integer tableIndex; + public static final String STANDALONE_DS_NAME = "ds_standalone"; + public DbDataNode(DbDataNodeTypeEnum type, String dataSource, Integer dbIndex, Integer tableIndex) { this.type = type; this.dataSource = dataSource; @@ -62,7 +64,7 @@ public DbDataNode(DbDataNodeTypeEnum type, String dataSource, Integer dbIndex, I public String toDataNodeId() { switch (type) { case STANDALONE: - return type + ":" + "ds_standalone"; + return type + ":" + STANDALONE_DS_NAME; case SHARDING: return type + ":" + dataSource + ":" + dbIndex + ":" + tableIndex; default: @@ -70,12 +72,23 @@ public String toDataNodeId() { } } + public String toDbNodeId() { + switch (type) { + case STANDALONE: + return STANDALONE_DS_NAME; + case SHARDING: + return dataSource + ":" + dbIndex; + default: + throw new IllegalArgumentException("Invalid DbDataNodeTypeEnum"); + } + } + public static DbDataNode fromDataNodeId(String dataNodeId) { String[] dataNodeParts = dataNodeId.split(":"); DbDataNodeTypeEnum dbDataNodeType = DbDataNodeTypeEnum.valOf(Integer.parseInt(dataNodeParts[0])); switch (dbDataNodeType) { case STANDALONE: - return new DbDataNode(dbDataNodeType, null, null, null); + return new DbDataNode(dbDataNodeType, STANDALONE_DS_NAME, null, null); case SHARDING: return new DbDataNode( dbDataNodeType, @@ -88,6 +101,14 @@ public static DbDataNode fromDataNodeId(String dataNodeId) { } } + public static DbDataNode standaloneDbDatNode() { + return new DbDataNode(DbDataNodeTypeEnum.STANDALONE, STANDALONE_DS_NAME, null, null); + } + + public static DbDataNode shardingDbDatNode(String dataSource, Integer dbIndex, Integer tableIndex) { + return new DbDataNode(DbDataNodeTypeEnum.SHARDING, dataSource, dbIndex, tableIndex); + } + @Override public DbDataNode clone() { return new DbDataNode(type, dataSource, dbIndex, tableIndex); diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskService.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/service/ArchiveTaskService.java similarity index 81% rename from src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskService.java rename to src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/service/ArchiveTaskService.java index b541428b03..2f04573941 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskService.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/service/ArchiveTaskService.java @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.backup.archive; +package com.tencent.bk.job.backup.archive.service; import com.tencent.bk.job.backup.archive.dao.ArchiveTaskDAO; import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; @@ -30,6 +30,7 @@ import org.springframework.stereotype.Service; import java.util.List; +import java.util.Map; @Service public class ArchiveTaskService { @@ -57,11 +58,21 @@ public List listRunningTasks(ArchiveTaskTypeEnum tas return archiveTaskDAO.listRunningTasks(taskType); } - public List listScheduleTasks(ArchiveTaskTypeEnum taskType, int limit) { - return archiveTaskDAO.listScheduleTasks(taskType, limit); + /** + * 返回根据 db 分组的归档任务数量 + * + * @param taskType 归档任务类型 + * @return key: db 名称; value: 任务数量 + */ + public Map countScheduleTasksGroupByDb(ArchiveTaskTypeEnum taskType) { + return archiveTaskDAO.countScheduleTasksGroupByDb(taskType); } public void updateTask(JobInstanceArchiveTaskInfo archiveTask) { archiveTaskDAO.updateTask(archiveTask); } + + public JobInstanceArchiveTaskInfo getFirstScheduleArchiveTaskByDb(ArchiveTaskTypeEnum taskType, String dbNodeId) { + return archiveTaskDAO.getFirstScheduleArchiveTaskByDb(taskType, dbNodeId); + } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java index 992984d4fa..bf65edaa20 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java @@ -26,8 +26,8 @@ import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; import com.tencent.bk.job.backup.archive.ArchiveTaskLock; -import com.tencent.bk.job.backup.archive.ArchiveTaskService; import com.tencent.bk.job.backup.archive.JobInstanceArchiveTaskGenerator; +import com.tencent.bk.job.backup.archive.JobInstanceArchiveTaskScheduleLock; import com.tencent.bk.job.backup.archive.JobInstanceArchiveTaskScheduler; import com.tencent.bk.job.backup.archive.JobInstanceSubTableArchivers; import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; @@ -66,6 +66,8 @@ import com.tencent.bk.job.backup.archive.impl.TaskInstanceArchiver; import com.tencent.bk.job.backup.archive.impl.TaskInstanceHostArchiver; import com.tencent.bk.job.backup.archive.impl.TaskInstanceVariableArchiver; +import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; +import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; @@ -495,14 +497,27 @@ public JobInstanceArchiveTaskGenerator jobInstanceArchiveTaskGenerator(ArchiveTa @Bean @ConditionalOnExpression("${job.backup.archive.execute.enabled:false}") - public JobInstanceArchiveTaskScheduler jobInstanceArchiveTaskScheduler(ArchiveTaskService archiveTaskService, - TaskInstanceRecordDAO taskInstanceRecordDAO, - ArchiveProperties archiveProperties) { + public JobInstanceArchiveTaskScheduler jobInstanceArchiveTaskScheduler( + ArchiveTaskService archiveTaskService, + TaskInstanceRecordDAO taskInstanceRecordDAO, + ArchiveProperties archiveProperties, + JobInstanceArchiveTaskScheduleLock jobInstanceArchiveTaskScheduleLock, + JobInstanceSubTableArchivers jobInstanceSubTableArchivers, + JobInstanceColdDAO jobInstanceColdDAO, + ArchiveTaskLock archiveTaskLock, + ArchiveErrorTaskCounter archiveErrorTaskCounter, + ArchiveTablePropsStorage archiveTablePropsStorage) { + return new JobInstanceArchiveTaskScheduler( archiveTaskService, taskInstanceRecordDAO, archiveProperties, - - ); + jobInstanceArchiveTaskScheduleLock, + jobInstanceSubTableArchivers, + jobInstanceColdDAO, + archiveTaskLock, + archiveErrorTaskCounter, + archiveTablePropsStorage + ); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteHotDbConfiguration.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteHotDbConfiguration.java index c383d60980..d73ff4d915 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteHotDbConfiguration.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteHotDbConfiguration.java @@ -24,7 +24,7 @@ package com.tencent.bk.job.backup.config; -import com.tencent.bk.job.backup.archive.JobExecuteVerticalShardingDSLContextProvider; +import com.tencent.bk.job.backup.archive.dao.ds.JobExecuteVerticalShardingDSLContextProvider; import com.tencent.bk.job.common.mysql.dynamic.ds.DataSourceMode; import com.tencent.bk.job.common.mysql.dynamic.ds.StandaloneDSLContextProvider; import com.tencent.bk.job.common.mysql.dynamic.ds.VerticalShardingDSLContextProvider; diff --git a/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluatorTest.java b/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluatorTest.java new file mode 100644 index 0000000000..6a99ef91dc --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluatorTest.java @@ -0,0 +1,99 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.model.DbDataNode; +import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; +import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +class ArchiveDbNodePriorityEvaluatorTest { + + @Test + void testStandaloneDb() { + + List runningTasks = new ArrayList<>(); + runningTasks.add(genTask(DbDataNode.standaloneDbDatNode(), 20240808, 2)); + runningTasks.add(genTask(DbDataNode.standaloneDbDatNode(), 20240808, 2)); + runningTasks.add(genTask(DbDataNode.standaloneDbDatNode(), 20240808, 3)); + + Map scheduleTaskCountGroupByDb = new HashMap<>(); + scheduleTaskCountGroupByDb.put(DbDataNode.STANDALONE_DS_NAME, 2); + + ArchiveDbNodePriorityEvaluator.DbNodeTasksInfo dbNodeTasksInfo = + ArchiveDbNodePriorityEvaluator.evaluatePriorityDbNode(runningTasks, scheduleTaskCountGroupByDb); + assertThat(dbNodeTasksInfo).isNotNull(); + assertThat(dbNodeTasksInfo.getDbNodeId()).isEqualTo(DbDataNode.STANDALONE_DS_NAME); + assertThat(dbNodeTasksInfo.getRunningTaskCount()).isEqualTo(3); + assertThat(dbNodeTasksInfo.getScheduleTaskCount()).isEqualTo(2); + } + + @Test + void testShardingDb() { + + List runningTasks = new ArrayList<>(); + Map scheduleTaskCountGroupByDb = new HashMap<>(); + + runningTasks.add(genTask(DbDataNode.shardingDbDatNode("ds", 0, 1), 20240808, 1)); + runningTasks.add(genTask(DbDataNode.shardingDbDatNode("ds", 1, 0), 20240808, 1)); + runningTasks.add(genTask(DbDataNode.shardingDbDatNode("ds", 1, 1), 20240808, 1)); + scheduleTaskCountGroupByDb.put("ds:0", 2); + scheduleTaskCountGroupByDb.put("ds:1", 1); + + ArchiveDbNodePriorityEvaluator.DbNodeTasksInfo dbNodeTasksInfo = + ArchiveDbNodePriorityEvaluator.evaluatePriorityDbNode(runningTasks, scheduleTaskCountGroupByDb); + assertThat(dbNodeTasksInfo).isNotNull(); + assertThat(dbNodeTasksInfo.getDbNodeId()).isEqualTo("ds:0"); + + runningTasks.clear(); + scheduleTaskCountGroupByDb.clear(); + + runningTasks.add(genTask(DbDataNode.shardingDbDatNode("ds", 0, 1), 20240808, 1)); + runningTasks.add(genTask(DbDataNode.shardingDbDatNode("ds", 1, 1), 20240808, 1)); + scheduleTaskCountGroupByDb.put("ds:0", 2); + scheduleTaskCountGroupByDb.put("ds:1", 1); + + dbNodeTasksInfo = ArchiveDbNodePriorityEvaluator.evaluatePriorityDbNode(runningTasks, scheduleTaskCountGroupByDb); + assertThat(dbNodeTasksInfo).isNotNull(); + assertThat(dbNodeTasksInfo.getDbNodeId()).isEqualTo("ds:0"); + + } + + private JobInstanceArchiveTaskInfo genTask(DbDataNode dataNode, Integer day, Integer hour) { + JobInstanceArchiveTaskInfo task = new JobInstanceArchiveTaskInfo(); + task.setTaskType(ArchiveTaskTypeEnum.JOB_INSTANCE); + task.setDbDataNode(dataNode); + task.setDay(day); + task.setHour(hour); + return task; + } +} diff --git a/support-files/sql/job-backup/0005_job_backup_20241115-1000_V3.11.2_mysql.sql b/support-files/sql/job-backup/0005_job_backup_20241115-1000_V3.11.2_mysql.sql index 0130618ced..fd26490269 100644 --- a/support-files/sql/job-backup/0005_job_backup_20241115-1000_V3.11.2_mysql.sql +++ b/support-files/sql/job-backup/0005_job_backup_20241115-1000_V3.11.2_mysql.sql @@ -8,6 +8,7 @@ CREATE TABLE IF NOT EXISTS `archive_task` ( `id` BIGINT(20) NOT NULL AUTO_INCREMENT, `task_type` TINYINT(2), `data_node` VARCHAR(128) NOT NULL, + `db_node` VARCHAR(64) NOT NULL, `day` INT(8), `hour` TINYINT(2), `from_timestamp` BIGINT(20) NOT NULL, @@ -17,6 +18,7 @@ CREATE TABLE IF NOT EXISTS `archive_task` ( `create_time` BIGINT(20) NOT NULL, `last_update_time` BIGINT(20) NOT NULL, PRIMARY KEY (`id`) USING BTREE, - UNIQUE KEY `uk_type_node_day_hour` (`task_type`,`data_node`,`day`,`hour`) + UNIQUE KEY `uk_type_node_day_hour` (`task_type`,`data_node`,`day`,`hour`), + KEY `idx_type_status_db` (`task_type`,`status`,`db_node`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; From bb7ccb8be787003b9962aeb4eebc72e98c26fd8b Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Mon, 18 Nov 2024 21:02:46 +0800 Subject: [PATCH 27/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../AbstractJobInstanceArchiveTask.java | 16 +++++- .../ArchiveDbNodePriorityEvaluator.java | 4 +- .../archive/ArchiveTaskDoneCallback.java | 32 ++++++++++++ .../archive/ArchiveTaskStopCallback.java | 32 ++++++++++++ .../archive/JobInstanceArchiveCronJobs.java | 8 +-- .../archive/JobInstanceArchiveTask.java | 10 +++- .../JobInstanceArchiveTaskGenerator.java | 9 ++-- .../JobInstanceArchiveTaskScheduler.java | 52 +++++++++++-------- .../JobInstanceMainDataArchiveTask.java | 3 ++ ...skCounter.java => TaskCountDownLatch.java} | 22 ++------ .../AbstractJobInstanceRecordResultSet.java | 2 +- .../dao/resultset/RecordResultSet.java | 6 +-- .../AbstractJobInstanceSubTableArchiver.java | 11 +++- .../ArchiveDbNodePriorityEvaluatorTest.java | 6 +-- 14 files changed, 151 insertions(+), 62 deletions(-) create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskDoneCallback.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskStopCallback.java rename src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/{StopTaskCounter.java => TaskCountDownLatch.java} (90%) diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java index 1b3396fd57..8707227646 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java @@ -79,6 +79,8 @@ public abstract class AbstractJobInstanceArchiveTask> i * 同步锁 */ private final Object stopMonitor = new Object(); + private volatile ArchiveTaskStopCallback stopCallback = null; + private volatile ArchiveTaskDoneCallback archiveTaskDoneCallback; public AbstractJobInstanceArchiveTask(JobInstanceColdDAO jobInstanceColdDAO, @@ -106,9 +108,10 @@ public void execute() { } @Override - public void stop() { + public void stop(ArchiveTaskStopCallback stopCallback) { synchronized (stopMonitor) { this.stopFlag = true; + this.stopCallback = stopCallback; } } @@ -122,7 +125,9 @@ private void stopTask() { synchronized (stopMonitor) { if (!isStopped) { isStopped = true; - StopTaskCounter.getInstance().decrement(taskId); + if (stopCallback != null) { + stopCallback.callback(); + } log.info("Stop archive task successfully, taskId: {}", taskId); } else { log.info("Archive task is stopped, taskId: {}", taskId); @@ -130,6 +135,10 @@ private void stopTask() { } } + @Override + public void registerDoneCallback(ArchiveTaskDoneCallback archiveTaskDoneCallback) { + this.archiveTaskDoneCallback = archiveTaskDoneCallback; + } private void archive() { try { @@ -164,6 +173,9 @@ private void archive() { taskId, JsonUtils.toJson(archiveTaskSummary) ); + if (archiveTaskDoneCallback != null) { + archiveTaskDoneCallback.callback(); + } if (checkStopFlag()) { stopTask(); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluator.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluator.java index 4723f27555..c2fb3df5b2 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluator.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluator.java @@ -45,8 +45,8 @@ public class ArchiveDbNodePriorityEvaluator { * @param scheduleTaskCountGroupByDb 待调度归档任务计数（根据 db) * @return DbNodeTasksInfo */ - public static DbNodeTasksInfo evaluatePriorityDbNode(List runningTasks, - Map scheduleTaskCountGroupByDb) { + public static DbNodeTasksInfo evaluateHighestPriorityDbNode(List runningTasks, + Map scheduleTaskCountGroupByDb) { Map runningTaskCountGroupByDb = groupRunningTaskCountByDb(runningTasks); List dbNodeTasksInfos = new ArrayList<>(); diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskDoneCallback.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskDoneCallback.java new file mode 100644 index 0000000000..94185a1e22 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskDoneCallback.java @@ -0,0 +1,32 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +/** + * 归档任务执行完成回调 + */ +public interface ArchiveTaskDoneCallback { + void callback(); +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskStopCallback.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskStopCallback.java new file mode 100644 index 0000000000..364582b6aa --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskStopCallback.java @@ -0,0 +1,32 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +/** + * 归档任务被终止回调 + */ +public interface ArchiveTaskStopCallback { + void callback(); +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveCronJobs.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveCronJobs.java index bbcae195c4..9ec93f61cc 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveCronJobs.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveCronJobs.java @@ -48,9 +48,9 @@ public JobInstanceArchiveCronJobs(JobInstanceArchiveTaskGenerator jobInstanceArc } /** - * 定时创建归档任务 + * 定时创建归档任务,每天0 点触发一次 */ - @Scheduled(cron = "${job.backup.archive.execute.cron:0,6,12,18 0 0 * * *}") + @Scheduled(cron = "0 0 0 * * *") public void generateArchiveTask() { log.info("Generate archive task start..."); jobInstanceArchiveTaskGenerator.generate(); @@ -58,9 +58,9 @@ public void generateArchiveTask() { } /** - * 定时调度并执行归档任务 + * 定时调度并执行归档任务，默认每小时第 1 分钟触发一次 */ - @Scheduled(cron = "${job.backup.archive.execute.cron:1,7,13,19 0 0 * * *}") + @Scheduled(cron = "${job.backup.archive.execute.cron: 0 1 * * * *}") public void scheduleAndExecuteArchiveTask() { log.info("Schedule and execute archive task start..."); jobInstanceArchiveTaskScheduler.schedule(); diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTask.java index e99513471b..80f50c80a3 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTask.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTask.java @@ -34,12 +34,18 @@ public interface JobInstanceArchiveTask { */ void execute(); - /** * 归档任务被终止时候调用 + * + * @param stopCallback 回调 */ - void stop(); + void stop(ArchiveTaskStopCallback stopCallback); + + void registerDoneCallback(ArchiveTaskDoneCallback archiveTaskDoneCallback); + /** + * 获取任务 ID + */ String getTaskId(); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java index 33e0898d4f..8c89f9dd67 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java @@ -73,17 +73,14 @@ public void generate() { List archiveTaskList = new ArrayList<>(); LocalDateTime startDateTime = computeArchiveStartDateTime(); - LocalDateTime endDateTime = unixTimestampToUtcLocalDateTime( - getEndTime(archiveProperties.getKeepDays())); + LocalDateTime endDateTime = unixTimestampToUtcLocalDateTime(getEndTime(archiveProperties.getKeepDays())); + // 创建归档任务 while (startDateTime.isBefore(endDateTime)) { // 水平分库分表 if (isHorizontalShardingEnabled()) { - // 作业实例数据归档任务 + // 作业实例数据归档任务,现版本暂不支持 archiveTaskList.addAll(buildArchiveTasksForShardingDataNodes(ArchiveTaskTypeEnum.JOB_INSTANCE, startDateTime, archiveProperties.getTasks().getTaskInstance().getShardingDataNodes())); - // 作业实例按业务冗余数据归档 - archiveTaskList.addAll(buildArchiveTasksForShardingDataNodes(ArchiveTaskTypeEnum.JOB_INSTANCE_APP, - startDateTime, archiveProperties.getTasks().getTaskInstance().getShardingDataNodes())); } else { // 单db DbDataNode dbDataNode = new DbDataNode(DbDataNodeTypeEnum.STANDALONE, null, null, null); diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java index 1559f76552..69314b5cd4 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java @@ -42,7 +42,6 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; /** * 作业执行历史归档任务调度 @@ -65,8 +64,6 @@ public class JobInstanceArchiveTaskScheduler implements SmartLifecycle { private final ArchiveErrorTaskCounter archiveErrorTaskCounter; private final ArchiveTablePropsStorage archiveTablePropsStorage; - private final AtomicInteger runningTasksCount = new AtomicInteger(0); - private final Object lifecycleMonitor = new Object(); /** @@ -114,7 +111,7 @@ public JobInstanceArchiveTaskScheduler(ArchiveTaskService archiveTaskService, public void schedule() { try { if (isScheduling()) { - log.info("Scheduler is already working"); + log.info("Scheduler is working"); return; } if (!isActive()) { @@ -131,12 +128,12 @@ public void schedule() { continue; } - // 获取待调度的任务列表 + // 获取待调度的任务信息(按照 DB 节点计数) Map scheduleTasksGroupByDb = archiveTaskService.countScheduleTasksGroupByDb(ArchiveTaskTypeEnum.JOB_INSTANCE); if (scheduleTasksGroupByDb.isEmpty()) { // 所有任务都已经被调度完成，退出本次任务调度 - log.info("All archive task is done!"); + log.info("All archive task is done! Exit schedule"); return; } @@ -146,10 +143,10 @@ public void schedule() { // 对待调度的任务进行优先级排序，保证同一个 db 上的归档任务数量尽可能均衡，避免出现db 热点 ArchiveDbNodePriorityEvaluator.DbNodeTasksInfo highestPriorityDbNodeTasksInfo = - ArchiveDbNodePriorityEvaluator.evaluatePriorityDbNode(runningTasks, scheduleTasksGroupByDb); + ArchiveDbNodePriorityEvaluator.evaluateHighestPriorityDbNode(runningTasks, scheduleTasksGroupByDb); if (highestPriorityDbNodeTasksInfo.getRunningTaskCount() >= archiveProperties.getConcurrent()) { - // 休眠一分钟，等待并行任务减少 - log.info("Running archive task count exceed concurrent limit : {}, wait 60s", + // 休眠5分钟，等待并行任务减少 + log.info("Running archive task count exceed concurrent limit : {}, wait 300s", archiveProperties.getConcurrent()); ThreadUtils.sleep(1000 * 60L); continue; @@ -171,7 +168,7 @@ public void schedule() { } private void startArchiveTask(JobInstanceArchiveTaskInfo archiveTaskInfo) { - log.info("Start JobInstanceArchiveTask : {}", archiveTaskInfo); + log.info("Start JobInstanceArchiveTask, taskId: {}", archiveTaskInfo.buildTaskUniqueId()); JobInstanceMainDataArchiveTask archiveTask = new JobInstanceMainDataArchiveTask( taskInstanceRecordDAO, jobInstanceSubTableArchivers, @@ -183,6 +180,12 @@ private void startArchiveTask(JobInstanceArchiveTaskInfo archiveTaskInfo) { archiveTaskService, archiveTablePropsStorage ); + // 注册任务完成回调函数 + archiveTask.registerDoneCallback(() -> { + synchronized (lifecycleMonitor) { + scheduledTasks.remove(archiveTask.getTaskId()); + } + }); synchronized (lifecycleMonitor) { if (!isActive()) { log.info("JobInstanceArchiveTaskScheduler is not active, skip"); @@ -190,9 +193,9 @@ private void startArchiveTask(JobInstanceArchiveTaskInfo archiveTaskInfo) { } scheduledTasks.put(archiveTask.getTaskId(), archiveTask); } - runningTasksCount.incrementAndGet(); ArchiveTaskWorker worker = new ArchiveTaskWorker(archiveTask); worker.start(); + log.info("Start JobInstanceArchiveTask success, taskId: {}", archiveTaskInfo.buildTaskUniqueId()); } /** @@ -230,39 +233,40 @@ public void stop() { private void stopTasksGraceful() { log.info("Stop archive tasks graceful - start"); long start = System.currentTimeMillis(); - StopTaskCounter stopTaskCounter = null; + TaskCountDownLatch taskCountDownLatch = null; synchronized (lifecycleMonitor) { if (!this.scheduledTasks.isEmpty()) { log.info("Stop archive tasks, size: {}, tasks: {}", scheduledTasks.size(), scheduledTasks); - stopTaskCounter = StopTaskCounter.getInstance(); - stopTaskCounter.initCounter(scheduledTasks.keySet()); + taskCountDownLatch = new TaskCountDownLatch(scheduledTasks.keySet()); } for (JobInstanceArchiveTask task : scheduledTasks.values()) { - shutdownExecutor.execute(new StopTask(task)); + shutdownExecutor.execute(new StopTask(task, taskCountDownLatch)); } } try { - if (stopTaskCounter != null) { - stopTaskCounter.waitingForAllTasksDone(); + if (taskCountDownLatch != null) { + taskCountDownLatch.waitingForAllTasksDone(); } } catch (Throwable e) { - log.error("Stop tasks caught exception", e); + log.error("Stop archive tasks caught exception", e); } long end = System.currentTimeMillis(); - log.info("Stop tasks graceful - end, cost: {}", end - start); + log.info("Stop archive tasks graceful - end, cost: {}", end - start); } private static final class StopTask implements Runnable { private final JobInstanceArchiveTask task; + private final TaskCountDownLatch taskCountDownLatch; - StopTask(JobInstanceArchiveTask task) { + StopTask(JobInstanceArchiveTask task, TaskCountDownLatch taskCountDownLatch) { this.task = task; + this.taskCountDownLatch = taskCountDownLatch; } @Override public void run() { try { - task.stop(); + task.stop(() -> taskCountDownLatch.decrement(task.getTaskId())); } catch (Throwable e) { String errorMsg = "Stop archive task caught exception, task: " + task; log.warn(errorMsg, e); @@ -277,12 +281,18 @@ public boolean isRunning() { } } + /** + * 判断是否处于激活状态 + */ private boolean isActive() { synchronized (this.lifecycleMonitor) { return this.active; } } + /** + * 判断是否正在进行任务调度中 + */ private boolean isScheduling() { synchronized (this.lifecycleMonitor) { return this.scheduling; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java index 690de68f55..7c0ff43f14 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java @@ -89,8 +89,11 @@ protected void deleteJobInstanceHotData(List jobInstanceIds) { tableArchiver.deleteRecords(jobInstanceIds); }); // 删除主表数据 + long startTime = System.currentTimeMillis(); taskInstanceRecordDAO.deleteRecords(jobInstanceIds, archiveTablePropsStorage.getDeleteLimitRowCount(TaskInstance.TASK_INSTANCE.getName())); + log.info("Delete {}, taskInstanceIdSize: {}, cost: {}ms", "task_instance", + jobInstanceIds.size(), System.currentTimeMillis() - startTime); } @Override diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/StopTaskCounter.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/TaskCountDownLatch.java similarity index 90% rename from src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/StopTaskCounter.java rename to src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/TaskCountDownLatch.java index 200dff8ab9..cd17028ff2 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/StopTaskCounter.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/TaskCountDownLatch.java @@ -33,24 +33,17 @@ import java.util.concurrent.CountDownLatch; /** - * 停止任务计数 + * 任务 CountDownLatch 组件，用于等待所有任务完成后再退出 */ @Slf4j -public class StopTaskCounter { +public class TaskCountDownLatch { private final Object taskMonitor = new Object(); - private CountDownLatch latch; + private final CountDownLatch latch; private final Set taskIds = new HashSet<>(); private volatile boolean monitorInitial = false; private volatile boolean isAllTaskDone = false; - private StopTaskCounter() { - } - - public static StopTaskCounter getInstance() { - return Inner.instance; - } - - void initCounter(Set taskIds) { + public TaskCountDownLatch(Set taskIds) { this.taskIds.addAll(taskIds); this.latch = new CountDownLatch(taskIds.size()); startMonitor(); @@ -69,7 +62,7 @@ public void decrement(String taskId) { } } - void waitingForAllTasksDone() { + public void waitingForAllTasksDone() { try { log.info("Waiting for all tasks done! total: {}", latch.getCount()); this.latch.await(); @@ -106,9 +99,4 @@ private void destroy() { taskIds); } } - - private static class Inner { - private static final StopTaskCounter instance = new StopTaskCounter(); - } - } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/AbstractJobInstanceRecordResultSet.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/AbstractJobInstanceRecordResultSet.java index f4e0563106..c4a39ed2cb 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/AbstractJobInstanceRecordResultSet.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/AbstractJobInstanceRecordResultSet.java @@ -61,7 +61,7 @@ public AbstractJobInstanceRecordResultSet(AbstractJobInstanceHotRecordDAO job } @Override - public boolean hasNext() { + public boolean next() { if (!hasNext) { return false; } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/RecordResultSet.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/RecordResultSet.java index e5d7e72fc3..80aa53ae9c 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/RecordResultSet.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/RecordResultSet.java @@ -35,12 +35,12 @@ */ public interface RecordResultSet { /** - * 是否还需要继续查询 + * 将查询游标指向下一次查询 */ - boolean hasNext(); + boolean next(); /** - * 获取本次查询的表记录 + * 获取下一次查询的表记录 */ List getRecords(); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/AbstractJobInstanceSubTableArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/AbstractJobInstanceSubTableArchiver.java index 86fa0f4a3b..0287ade551 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/AbstractJobInstanceSubTableArchiver.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/AbstractJobInstanceSubTableArchiver.java @@ -29,11 +29,13 @@ import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; import com.tencent.bk.job.backup.archive.dao.impl.AbstractJobInstanceHotRecordDAO; import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.jooq.TableRecord; import java.util.List; +@Slf4j public class AbstractJobInstanceSubTableArchiver implements JobInstanceSubTableArchiver { private final JobInstanceColdDAO jobInstanceColdDAO; @@ -60,8 +62,12 @@ public void backupRecords(List jobInstanceIds) { RecordResultSet> recordResultSet = jobInstanceHotRecordDAO.executeQuery(jobInstanceIds, archiveTablePropsStorage.getReadRowLimit(tableName)); - while (recordResultSet.hasNext()) { + long startTime = System.currentTimeMillis(); + while (recordResultSet.next()) { List> records = recordResultSet.getRecords(); + long readEndTime = System.currentTimeMillis(); + log.info("Read {}, recordSize: {}, cost: {}ms", tableName, + CollectionUtils.isEmpty(records) ? 0 : records.size(), readEndTime - startTime); if (CollectionUtils.isNotEmpty(records)) { jobInstanceColdDAO.batchInsert(records, archiveTablePropsStorage.getBatchInsertRowSize(tableName)); @@ -71,7 +77,10 @@ public void backupRecords(List jobInstanceIds) { @Override public void deleteRecords(List jobInstanceIds) { + long startTime = System.currentTimeMillis(); jobInstanceHotRecordDAO.deleteRecords(jobInstanceIds, archiveTablePropsStorage.getDeleteLimitRowCount(tableName)); + log.info("Delete {}, taskInstanceIdSize: {}, cost: {}ms", tableName, + jobInstanceIds.size(), System.currentTimeMillis() - startTime); } } diff --git a/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluatorTest.java b/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluatorTest.java index 6a99ef91dc..3ab9c408ee 100644 --- a/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluatorTest.java +++ b/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluatorTest.java @@ -50,7 +50,7 @@ void testStandaloneDb() { scheduleTaskCountGroupByDb.put(DbDataNode.STANDALONE_DS_NAME, 2); ArchiveDbNodePriorityEvaluator.DbNodeTasksInfo dbNodeTasksInfo = - ArchiveDbNodePriorityEvaluator.evaluatePriorityDbNode(runningTasks, scheduleTaskCountGroupByDb); + ArchiveDbNodePriorityEvaluator.evaluateHighestPriorityDbNode(runningTasks, scheduleTaskCountGroupByDb); assertThat(dbNodeTasksInfo).isNotNull(); assertThat(dbNodeTasksInfo.getDbNodeId()).isEqualTo(DbDataNode.STANDALONE_DS_NAME); assertThat(dbNodeTasksInfo.getRunningTaskCount()).isEqualTo(3); @@ -70,7 +70,7 @@ void testShardingDb() { scheduleTaskCountGroupByDb.put("ds:1", 1); ArchiveDbNodePriorityEvaluator.DbNodeTasksInfo dbNodeTasksInfo = - ArchiveDbNodePriorityEvaluator.evaluatePriorityDbNode(runningTasks, scheduleTaskCountGroupByDb); + ArchiveDbNodePriorityEvaluator.evaluateHighestPriorityDbNode(runningTasks, scheduleTaskCountGroupByDb); assertThat(dbNodeTasksInfo).isNotNull(); assertThat(dbNodeTasksInfo.getDbNodeId()).isEqualTo("ds:0"); @@ -82,7 +82,7 @@ void testShardingDb() { scheduleTaskCountGroupByDb.put("ds:0", 2); scheduleTaskCountGroupByDb.put("ds:1", 1); - dbNodeTasksInfo = ArchiveDbNodePriorityEvaluator.evaluatePriorityDbNode(runningTasks, scheduleTaskCountGroupByDb); + dbNodeTasksInfo = ArchiveDbNodePriorityEvaluator.evaluateHighestPriorityDbNode(runningTasks, scheduleTaskCountGroupByDb); assertThat(dbNodeTasksInfo).isNotNull(); assertThat(dbNodeTasksInfo.getDbNodeId()).isEqualTo("ds:0"); From fe9f0a76d6cc52bde0cd3500dff401a56cc0f526 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Mon, 18 Nov 2024 21:43:14 +0800 Subject: [PATCH 28/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../archive/JobInstanceArchiveCronJobs.java | 17 +++++-- .../JobInstanceArchiveTaskScheduler.java | 6 +-- .../backup/config/ArchiveConfiguration.java | 44 ++++++++++--------- .../job/backup/config/ArchiveProperties.java | 11 ++--- .../config/ExecuteColdDbConfiguration.java | 16 +++++++ .../execute/JobExecuteBootApplication.java | 2 + .../kubernetes/charts/bk-job/values.yaml | 7 ++- 7 files changed, 67 insertions(+), 36 deletions(-) diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveCronJobs.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveCronJobs.java index 9ec93f61cc..386fd87e28 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveCronJobs.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveCronJobs.java @@ -24,15 +24,14 @@ package com.tencent.bk.job.backup.archive; +import com.tencent.bk.job.backup.config.ArchiveProperties; import lombok.extern.slf4j.Slf4j; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Component; /** * 归档定时任务 */ -@Component @EnableScheduling @Slf4j public class JobInstanceArchiveCronJobs { @@ -41,17 +40,24 @@ public class JobInstanceArchiveCronJobs { private final JobInstanceArchiveTaskScheduler jobInstanceArchiveTaskScheduler; + private final ArchiveProperties archiveProperties; + public JobInstanceArchiveCronJobs(JobInstanceArchiveTaskGenerator jobInstanceArchiveTaskGenerator, - JobInstanceArchiveTaskScheduler jobInstanceArchiveTaskScheduler) { + JobInstanceArchiveTaskScheduler jobInstanceArchiveTaskScheduler, + ArchiveProperties archiveProperties) { this.jobInstanceArchiveTaskGenerator = jobInstanceArchiveTaskGenerator; this.jobInstanceArchiveTaskScheduler = jobInstanceArchiveTaskScheduler; + this.archiveProperties = archiveProperties; } /** - * 定时创建归档任务,每天0 点触发一次 + * 定时创建归档任务,每天 0 点触发一次 */ @Scheduled(cron = "0 0 0 * * *") public void generateArchiveTask() { + if (!archiveProperties.isEnabled()) { + return; + } log.info("Generate archive task start..."); jobInstanceArchiveTaskGenerator.generate(); log.info("Generate archive task done"); @@ -62,6 +68,9 @@ public void generateArchiveTask() { */ @Scheduled(cron = "${job.backup.archive.execute.cron: 0 1 * * * *}") public void scheduleAndExecuteArchiveTask() { + if (!archiveProperties.isEnabled()) { + return; + } log.info("Schedule and execute archive task start..."); jobInstanceArchiveTaskScheduler.schedule(); log.info("Schedule and execute archive task done"); diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java index 69314b5cd4..9592253670 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java @@ -144,10 +144,10 @@ public void schedule() { // 对待调度的任务进行优先级排序，保证同一个 db 上的归档任务数量尽可能均衡，避免出现db 热点 ArchiveDbNodePriorityEvaluator.DbNodeTasksInfo highestPriorityDbNodeTasksInfo = ArchiveDbNodePriorityEvaluator.evaluateHighestPriorityDbNode(runningTasks, scheduleTasksGroupByDb); - if (highestPriorityDbNodeTasksInfo.getRunningTaskCount() >= archiveProperties.getConcurrent()) { + int taskConcurrent = archiveProperties.getTasks().getTaskInstance().getConcurrent(); + if (highestPriorityDbNodeTasksInfo.getRunningTaskCount() >= taskConcurrent) { // 休眠5分钟，等待并行任务减少 - log.info("Running archive task count exceed concurrent limit : {}, wait 300s", - archiveProperties.getConcurrent()); + log.info("Running archive task count exceed concurrent limit : {}, wait 300s", taskConcurrent); ThreadUtils.sleep(1000 * 60L); continue; } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java index bf65edaa20..96a78d77bb 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java @@ -26,6 +26,7 @@ import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; import com.tencent.bk.job.backup.archive.ArchiveTaskLock; +import com.tencent.bk.job.backup.archive.JobInstanceArchiveCronJobs; import com.tencent.bk.job.backup.archive.JobInstanceArchiveTaskGenerator; import com.tencent.bk.job.backup.archive.JobInstanceArchiveTaskScheduleLock; import com.tencent.bk.job.backup.archive.JobInstanceArchiveTaskScheduler; @@ -37,7 +38,6 @@ import com.tencent.bk.job.backup.archive.dao.impl.GseScriptAgentTaskRecordDAO; import com.tencent.bk.job.backup.archive.dao.impl.GseScriptExecuteObjTaskRecordDAO; import com.tencent.bk.job.backup.archive.dao.impl.GseTaskRecordDAO; -import com.tencent.bk.job.backup.archive.dao.impl.JobInstanceColdDAOImpl; import com.tencent.bk.job.backup.archive.dao.impl.OperationLogRecordDAO; import com.tencent.bk.job.backup.archive.dao.impl.RollingConfigRecordDAO; import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceConfirmRecordDAO; @@ -70,13 +70,11 @@ import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; import lombok.extern.slf4j.Slf4j; -import org.jooq.DSLContext; import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.data.redis.core.StringRedisTemplate; @@ -90,13 +88,13 @@ @Slf4j @EnableConfigurationProperties(ArchiveProperties.class) @Import({ExecuteHotDbConfiguration.class, ExecuteColdDbConfiguration.class}) +@ConditionalOnExpression("${job.backup.archive.execute.enabled:false}") public class ArchiveConfiguration { /** * job-execute DB 配置 */ @Configuration - @ConditionalOnExpression("${job.backup.archive.execute.enabled:false}") public static class ExecuteDaoAutoConfig { @Bean(name = "taskInstanceRecordDAO") @@ -461,33 +459,18 @@ public JobInstanceSubTableArchivers jobInstanceSubTableArchivers( } - /** - * job-execute 归档冷 DB 配置 - */ - @Configuration - @Conditional(ExecuteColdDbConfiguration.JobExecuteColdDbInitCondition.class) - public static class ExecuteBackupDAOConfig { - @Bean(name = "execute-archive-dao") - public JobInstanceColdDAOImpl jobInstanceColdDAO( - @Qualifier("job-execute-archive-dsl-context") DSLContext context) { - log.info("Init ExecuteArchiveDAO"); - return new JobInstanceColdDAOImpl(context); - } - } @Bean - @ConditionalOnExpression("${job.backup.archive.execute.enabled:false}") public ArchiveTaskLock archiveTaskLock(StringRedisTemplate redisTemplate) { log.info("Init ArchiveTaskLock"); return new ArchiveTaskLock(redisTemplate); } - @Bean - @ConditionalOnExpression("${job.backup.archive.execute.enabled:false}") public JobInstanceArchiveTaskGenerator jobInstanceArchiveTaskGenerator(ArchiveTaskService archiveTaskService, TaskInstanceRecordDAO taskInstanceRecordDAO, ArchiveProperties archiveProperties) { + log.info("Init JobInstanceArchiveTaskGenerator"); return new JobInstanceArchiveTaskGenerator( archiveTaskService, taskInstanceRecordDAO, @@ -496,7 +479,6 @@ public JobInstanceArchiveTaskGenerator jobInstanceArchiveTaskGenerator(ArchiveTa } @Bean - @ConditionalOnExpression("${job.backup.archive.execute.enabled:false}") public JobInstanceArchiveTaskScheduler jobInstanceArchiveTaskScheduler( ArchiveTaskService archiveTaskService, TaskInstanceRecordDAO taskInstanceRecordDAO, @@ -508,6 +490,7 @@ public JobInstanceArchiveTaskScheduler jobInstanceArchiveTaskScheduler( ArchiveErrorTaskCounter archiveErrorTaskCounter, ArchiveTablePropsStorage archiveTablePropsStorage) { + log.info("Init JobInstanceArchiveTaskScheduler"); return new JobInstanceArchiveTaskScheduler( archiveTaskService, taskInstanceRecordDAO, @@ -520,4 +503,23 @@ public JobInstanceArchiveTaskScheduler jobInstanceArchiveTaskScheduler( archiveTablePropsStorage ); } + + @Bean + public JobInstanceArchiveCronJobs jobInstanceArchiveCronJobs( + JobInstanceArchiveTaskGenerator jobInstanceArchiveTaskGenerator, + JobInstanceArchiveTaskScheduler jobInstanceArchiveTaskScheduler, + ArchiveProperties archiveProperties) { + log.info("Init JobInstanceArchiveCronJobs"); + return new JobInstanceArchiveCronJobs( + jobInstanceArchiveTaskGenerator, + jobInstanceArchiveTaskScheduler, + archiveProperties + ); + } + + @Bean + public JobInstanceArchiveTaskScheduleLock jobInstanceArchiveTaskScheduleLock() { + log.info("Init JobInstanceArchiveTaskScheduleLock"); + return new JobInstanceArchiveTaskScheduleLock(); + } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveProperties.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveProperties.java index e28322072a..d7efe10472 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveProperties.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveProperties.java @@ -55,12 +55,6 @@ public class ArchiveProperties { * 触发时间 CRON 表达式 */ private String cron; - - /** - * 归档任务并行执行数量 - */ - private Integer concurrent = 10; - /** * DB数据保留天数 */ @@ -137,7 +131,10 @@ public static class ArchiveTaskConfig { * 要归档的分库分表数据节点配置 */ private List shardingDataNodes; - + /** + * 归档任务并行执行数量 + */ + private Integer concurrent = 6; } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteColdDbConfiguration.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteColdDbConfiguration.java index ec06946b6c..b30b0dd979 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteColdDbConfiguration.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteColdDbConfiguration.java @@ -24,7 +24,9 @@ package com.tencent.bk.job.backup.config; +import com.tencent.bk.job.backup.archive.dao.impl.JobInstanceColdDAOImpl; import com.tencent.bk.job.backup.constant.ArchiveModeEnum; +import lombok.extern.slf4j.Slf4j; import org.jooq.ConnectionProvider; import org.jooq.DSLContext; import org.jooq.SQLDialect; @@ -48,6 +50,7 @@ */ @Configuration(value = "executeColdDbConfiguration") @Conditional(ExecuteColdDbConfiguration.JobExecuteColdDbInitCondition.class) +@Slf4j public class ExecuteColdDbConfiguration { @Qualifier("job-execute-archive-source") @@ -112,5 +115,18 @@ static class ArchiveModeBackupOnlyCondition { } } + /** + * job-execute 归档冷 DB 配置 + */ + @Configuration + public static class ExecuteBackupDAOConfig { + @Bean(name = "execute-archive-dao") + public JobInstanceColdDAOImpl jobInstanceColdDAO( + @Qualifier("job-execute-archive-dsl-context") DSLContext context) { + log.info("Init ExecuteArchiveDAO"); + return new JobInstanceColdDAOImpl(context); + } + } + } diff --git a/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/JobExecuteBootApplication.java b/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/JobExecuteBootApplication.java index 29faab633f..f5c0d81c4c 100644 --- a/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/JobExecuteBootApplication.java +++ b/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/JobExecuteBootApplication.java @@ -25,6 +25,7 @@ package com.tencent.bk.job.execute; import com.tencent.bk.job.common.service.boot.JobBootApplication; +import com.tencent.devops.leaf.plugin.annotation.EnableLeafServer; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration; import org.springframework.boot.autoconfigure.jooq.JooqAutoConfiguration; @@ -41,6 +42,7 @@ "com.tencent.bk.job.file_gateway.api" } ) +@EnableLeafServer public class JobExecuteBootApplication { public static void main(String[] args) { diff --git a/support-files/kubernetes/charts/bk-job/values.yaml b/support-files/kubernetes/charts/bk-job/values.yaml index 478086d881..466054611b 100644 --- a/support-files/kubernetes/charts/bk-job/values.yaml +++ b/support-files/kubernetes/charts/bk-job/values.yaml @@ -1237,7 +1237,12 @@ backupConfig: deleteRowLimit: 1000 tasks: job_instance: - + dataSourceMode: standalone + concurrent: 6 + # shardingDsDataNodes: + # - dataSourceName: ds + # dbCount: 2 + # tableCount: 31 image: registry: hub.bktencent.com repository: blueking/job-backup From bcf0f8649c8cb11ecb2fe7813831446039744fcd Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Mon, 18 Nov 2024 22:18:29 +0800 Subject: [PATCH 29/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../JobInstanceArchiveTaskGenerator.java | 4 ++-- .../JobInstanceArchiveTaskScheduler.java | 2 +- .../job/backup/config/ArchiveProperties.java | 2 +- .../config/ExecuteColdDbConfiguration.java | 22 +++++++------------ .../templates/job-execute/configmap.yaml | 1 + 5 files changed, 13 insertions(+), 18 deletions(-) diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java index 8c89f9dd67..c2b23cbd61 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java @@ -80,7 +80,7 @@ public void generate() { if (isHorizontalShardingEnabled()) { // 作业实例数据归档任务,现版本暂不支持 archiveTaskList.addAll(buildArchiveTasksForShardingDataNodes(ArchiveTaskTypeEnum.JOB_INSTANCE, - startDateTime, archiveProperties.getTasks().getTaskInstance().getShardingDataNodes())); + startDateTime, archiveProperties.getTasks().getJobInstance().getShardingDataNodes())); } else { // 单db DbDataNode dbDataNode = new DbDataNode(DbDataNodeTypeEnum.STANDALONE, null, null, null); @@ -153,7 +153,7 @@ private LocalDateTime computeArchiveStartDateTime() { } private boolean isHorizontalShardingEnabled() { - return archiveProperties.getTasks().getTaskInstance().getDataSourceMode() + return archiveProperties.getTasks().getJobInstance().getDataSourceMode() .equals(DataSourceMode.Constants.HORIZONTAL_SHARDING); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java index 9592253670..056046b4e9 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java @@ -144,7 +144,7 @@ public void schedule() { // 对待调度的任务进行优先级排序，保证同一个 db 上的归档任务数量尽可能均衡，避免出现db 热点 ArchiveDbNodePriorityEvaluator.DbNodeTasksInfo highestPriorityDbNodeTasksInfo = ArchiveDbNodePriorityEvaluator.evaluateHighestPriorityDbNode(runningTasks, scheduleTasksGroupByDb); - int taskConcurrent = archiveProperties.getTasks().getTaskInstance().getConcurrent(); + int taskConcurrent = archiveProperties.getTasks().getJobInstance().getConcurrent(); if (highestPriorityDbNodeTasksInfo.getRunningTaskCount() >= taskConcurrent) { // 休眠5分钟，等待并行任务减少 log.info("Running archive task count exceed concurrent limit : {}, wait 300s", taskConcurrent); diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveProperties.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveProperties.java index d7efe10472..973626becf 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveProperties.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveProperties.java @@ -115,7 +115,7 @@ public static class ArchiveTasksConfig { /** * 作业实例数据归档任务配置 */ - private ArchiveTaskConfig taskInstance; + private ArchiveTaskConfig jobInstance; } @Data diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteColdDbConfiguration.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteColdDbConfiguration.java index b30b0dd979..9c0f37a767 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteColdDbConfiguration.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteColdDbConfiguration.java @@ -57,6 +57,7 @@ public class ExecuteColdDbConfiguration { @Bean(name = "job-execute-archive-source") @ConfigurationProperties(prefix = "spring.datasource.job-execute-archive") public DataSource executeArchiveDataSource() { + log.info("Init job-execute-archive cold datasource"); return DataSourceBuilder.create().build(); } @@ -81,6 +82,13 @@ public ConnectionProvider executeArchiveConnectionProvider( return new DataSourceConnectionProvider(dataSource); } + @Bean(name = "execute-archive-dao") + public JobInstanceColdDAOImpl jobInstanceColdDAO( + @Qualifier("job-execute-archive-dsl-context") DSLContext context) { + log.info("Init JobInstanceColdDAO"); + return new JobInstanceColdDAOImpl(context); + } + static class JobExecuteColdDbInitCondition extends AllNestedConditions { public JobExecuteColdDbInitCondition() { super(ConfigurationPhase.PARSE_CONFIGURATION); @@ -115,18 +123,4 @@ static class ArchiveModeBackupOnlyCondition { } } - /** - * job-execute 归档冷 DB 配置 - */ - @Configuration - public static class ExecuteBackupDAOConfig { - @Bean(name = "execute-archive-dao") - public JobInstanceColdDAOImpl jobInstanceColdDAO( - @Qualifier("job-execute-archive-dsl-context") DSLContext context) { - log.info("Init ExecuteArchiveDAO"); - return new JobInstanceColdDAOImpl(context); - } - } - - } diff --git a/support-files/kubernetes/charts/bk-job/templates/job-execute/configmap.yaml b/support-files/kubernetes/charts/bk-job/templates/job-execute/configmap.yaml index e780988df4..f76474e32b 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-execute/configmap.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-execute/configmap.yaml @@ -253,6 +253,7 @@ data: {{ if .Values.executeConfig.leaf.enabled }} # 分布式 ID （leaf 组件配置) leaf: + enabled: true segment: allocStrategyDaoBeanName: jooqLeafIdAllocator enable: true From 85c49c9749a386eea23c25cb4c29b8eddefa40b8 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Mon, 18 Nov 2024 22:19:11 +0800 Subject: [PATCH 30/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- support-files/kubernetes/charts/bk-job/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/support-files/kubernetes/charts/bk-job/values.yaml b/support-files/kubernetes/charts/bk-job/values.yaml index 466054611b..6a77813b61 100644 --- a/support-files/kubernetes/charts/bk-job/values.yaml +++ b/support-files/kubernetes/charts/bk-job/values.yaml @@ -1236,7 +1236,7 @@ backupConfig: # 每次执行删除的最大行数。为了减少 MySQL 事务时长，DB 性能受限时可适当降低该值 deleteRowLimit: 1000 tasks: - job_instance: + jobInstance: dataSourceMode: standalone concurrent: 6 # shardingDsDataNodes: From 984e607488e5fcf046e79eb77f58a6171b04ffc7 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Mon, 18 Nov 2024 22:32:58 +0800 Subject: [PATCH 31/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tencent/bk/job/backup/config/ArchiveConfiguration.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java index 96a78d77bb..13aa96461b 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java @@ -238,7 +238,7 @@ public OperationLogRecordDAO operationLogRecordDAO( } @Bean - public OperationLogArchiver taskInstanceVariableArchiver( + public OperationLogArchiver operationLogArchiver( ObjectProvider jobInstanceColdDAOObjectProvider, OperationLogRecordDAO operationLogRecordDAO, ArchiveTablePropsStorage archiveTablePropsStorage @@ -257,7 +257,7 @@ public FileSourceTaskLogRecordDAO fileSourceTaskLogRecordDAO( } @Bean - public FileSourceTaskLogArchiver taskInstanceVariableArchiver( + public FileSourceTaskLogArchiver fileSourceTaskLogArchiver( ObjectProvider jobInstanceColdDAOObjectProvider, FileSourceTaskLogRecordDAO fileSourceTaskLogRecordDAO, ArchiveTablePropsStorage archiveTablePropsStorage @@ -352,7 +352,7 @@ public GseFileExecuteObjTaskRecordDAO gseFileExecuteObjTaskRecordDAO( } @Bean - public GseFileExecuteObjTaskArchiver gseScriptExecuteObjTaskArchiver( + public GseFileExecuteObjTaskArchiver gseFileExecuteObjTaskArchiver( ObjectProvider jobInstanceColdDAOObjectProvider, GseFileExecuteObjTaskRecordDAO gseFileExecuteObjTaskRecordDAO, ArchiveTablePropsStorage archiveTablePropsStorage From b981c5d053e8d1abe5db7a7d4ad57c5151135e20 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Tue, 19 Nov 2024 15:55:17 +0800 Subject: [PATCH 32/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/mysql/dynamic/id/IdGenType.java | 4 +- .../backup/config/ArchiveConfiguration.java | 4 +- .../execute/config/IdGenConfiguration.java | 2 +- .../dao/common/PropBasedDynamicIdGen.java | 2 +- .../kubernetes/charts/bk-job/values.yaml | 3 +- ...ob_execute_20241115-1000_V3.11.2_mysql.sql | 57 +++++++++++++++---- 6 files changed, 53 insertions(+), 19 deletions(-) diff --git a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/id/IdGenType.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/id/IdGenType.java index e36123c524..77f91d13ae 100644 --- a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/id/IdGenType.java +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/id/IdGenType.java @@ -29,11 +29,11 @@ @Getter public enum IdGenType { AUTO_INCREMENT(IdGenType.Constants.AUTO_INCREMENT), - SEGMENT(IdGenType.Constants.SEGMENT); + SEGMENT(IdGenType.Constants.LEAF_SEGMENT); public static class Constants { public final static String AUTO_INCREMENT = "auto_increment"; - public final static String SEGMENT = "segment"; + public final static String LEAF_SEGMENT = "leaf_segment"; } private final String type; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java index 13aa96461b..9c120737e7 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java @@ -485,7 +485,7 @@ public JobInstanceArchiveTaskScheduler jobInstanceArchiveTaskScheduler( ArchiveProperties archiveProperties, JobInstanceArchiveTaskScheduleLock jobInstanceArchiveTaskScheduleLock, JobInstanceSubTableArchivers jobInstanceSubTableArchivers, - JobInstanceColdDAO jobInstanceColdDAO, + ObjectProvider jobInstanceColdDAOObjectProvider, ArchiveTaskLock archiveTaskLock, ArchiveErrorTaskCounter archiveErrorTaskCounter, ArchiveTablePropsStorage archiveTablePropsStorage) { @@ -497,7 +497,7 @@ public JobInstanceArchiveTaskScheduler jobInstanceArchiveTaskScheduler( archiveProperties, jobInstanceArchiveTaskScheduleLock, jobInstanceSubTableArchivers, - jobInstanceColdDAO, + jobInstanceColdDAOObjectProvider.getIfAvailable(), archiveTaskLock, archiveErrorTaskCounter, archiveTablePropsStorage diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/IdGenConfiguration.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/IdGenConfiguration.java index 3e9dd4d7c9..e7e024913b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/IdGenConfiguration.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/IdGenConfiguration.java @@ -78,7 +78,7 @@ public SegmentIdGenCondition() { super(ConfigurationPhase.REGISTER_BEAN); } - @ConditionalOnProperty(value = "idGen.type", havingValue = IdGenType.Constants.SEGMENT) + @ConditionalOnProperty(value = "idGen.type", havingValue = IdGenType.Constants.LEAF_SEGMENT) static class IdGenTypeCondition { } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/PropBasedDynamicIdGen.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/PropBasedDynamicIdGen.java index 8ea22a6bc4..f5ff65220f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/PropBasedDynamicIdGen.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/PropBasedDynamicIdGen.java @@ -49,7 +49,7 @@ public PropBasedDynamicIdGen( candidateIdGens.put(IdGenType.Constants.AUTO_INCREMENT, autoIncrementIdGenerator); } if (segmentIdGenerator != null) { - candidateIdGens.put(IdGenType.Constants.SEGMENT, segmentIdGenerator); + candidateIdGens.put(IdGenType.Constants.LEAF_SEGMENT, segmentIdGenerator); } super.initCandidateComponents(candidateIdGens); } diff --git a/support-files/kubernetes/charts/bk-job/values.yaml b/support-files/kubernetes/charts/bk-job/values.yaml index 6a77813b61..1b9425562a 100644 --- a/support-files/kubernetes/charts/bk-job/values.yaml +++ b/support-files/kubernetes/charts/bk-job/values.yaml @@ -1026,10 +1026,11 @@ executeConfig: # 分布式ID组件 leaf 配置 leaf: enabled: false + # id生成器配置 idGen: migration: enabled: false - # id生成器配置。auto_increment: 自增长，segment: leaf segment id + # auto_increment: 自增长;leaf_segment: leaf segment id type: auto_increment ## job-crontab定时任务配置 diff --git a/support-files/sql/job-execute/0026_job_execute_20241115-1000_V3.11.2_mysql.sql b/support-files/sql/job-execute/0026_job_execute_20241115-1000_V3.11.2_mysql.sql index 6f62d29889..e7108faf3f 100644 --- a/support-files/sql/job-execute/0026_job_execute_20241115-1000_V3.11.2_mysql.sql +++ b/support-files/sql/job-execute/0026_job_execute_20241115-1000_V3.11.2_mysql.sql @@ -286,8 +286,8 @@ DROP PROCEDURE IF EXISTS job_add_task_instance_id; DELIMITER -CREATE PROCEDURE job_add_task_instance_id() -BEGIN +CREATE PROCEDURE job_add_task_instance_id(IN fromStepInstanceId BIGINT, IN endStepInstanceId BIGINT) +label:BEGIN DECLARE minId BIGINT; DECLARE maxId BIGINT; @@ -295,13 +295,31 @@ BEGIN DECLARE endId BIGINT; SET AUTOCOMMIT = 0; + -- 如果 step_instance 表为空，无需变更 + IF NOT EXISTS (SELECT 1 FROM step_instance LIMIT 1) THEN + LEAVE label; + END IF; + + -- 如果 gse_task 表中不存在task_instance_id = 0，说明已经执行过该变更 + IF NOT EXISTS (SELECT 1 FROM gse_task WHERE task_instance_id = 0 LIMIT 1) THEN + LEAVE label; + END IF; + SELECT MIN(id), MAX(id) INTO minId, maxId FROM step_instance; - SET fromId = minId; + IF fromStepInstanceId > 0 THEN + SET minId = fromStepInstanceId; + END IF; + + IF endStepInstanceId > 0 THEN + SET maxId = endStepInstanceId; + END IF; + + SET fromId = minId - 1; WHILE fromId <= maxId DO - SET endId = fromId + 999; + SELECT MIN(t.id),MAX(t.id) INTO fromId,endId FROM (SELECT id FROM step_instance WHERE id > fromId ORDER BY id asc LIMIT 1000) t; UPDATE file_source_task_log t1 INNER JOIN ( @@ -386,14 +404,14 @@ BEGIN COMMIT; - SET fromId = endId + 1; + SET fromId = endId; END WHILE; END DELIMITER ; COMMIT; -CALL job_add_task_instance_id(); +CALL job_add_task_instance_id(-1,-1); DROP PROCEDURE IF EXISTS job_add_task_instance_id; @@ -402,22 +420,37 @@ DROP PROCEDURE IF EXISTS job_update_task_instance_host_data; DELIMITER -CREATE PROCEDURE job_update_task_instance_host_data() -BEGIN +CREATE PROCEDURE job_update_task_instance_host_data(IN fromTaskInstanceId BIGINT, IN endTaskInstanceId BIGINT) +label:BEGIN DECLARE minId BIGINT; DECLARE maxId BIGINT; DECLARE fromId BIGINT; DECLARE endId BIGINT; + IF NOT EXISTS (SELECT 1 FROM task_instance LIMIT 1) THEN + LEAVE label; + END IF; + + IF NOT EXISTS (SELECT 1 FROM task_instance_host WHERE app_id = 0 LIMIT 1) THEN + LEAVE label; + END IF; + SET AUTOCOMMIT = 0; SELECT MIN(id), MAX(id) INTO minId, maxId FROM task_instance; + IF fromTaskInstanceId > 0 THEN + SET minId = fromTaskInstanceId; + END IF; + + IF endTaskInstanceId > 0 THEN + SET maxId = endTaskInstanceId; + END IF; - SET fromId = minId; + SET fromId = minId - 1; WHILE fromId <= maxId DO - SET endId = fromId + 999; + SELECT MIN(t.id),MAX(t.id) INTO fromId,endId FROM (SELECT id FROM task_instance where id > fromId ORDER BY id asc LIMIT 1000) t; UPDATE task_instance_host t1 INNER JOIN ( @@ -430,13 +463,13 @@ BEGIN COMMIT; - SET fromId = endId + 1; + SET fromId = endId; END WHILE; END DELIMITER ; COMMIT; -CALL job_update_task_instance_host_data(); +CALL job_update_task_instance_host_data(-1,-1); DROP PROCEDURE IF EXISTS job_update_task_instance_host_data; From faa571c660a5bd30124b0b209fe82000cd19c788 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Tue, 19 Nov 2024 16:07:07 +0800 Subject: [PATCH 33/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java index 025416c58e..387f055ab4 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java @@ -62,7 +62,7 @@ public static Condition build(Long taskInstanceId, if (resourceScope == null) { log.info("TaskInstanceIdDynamicCondition : Empty resource scope!"); // 无法根据业务决定是否使用 task_instance_id 作为查询条件。为了不影响请求正常处理,直接返回 TRUE Condition - // (不会影响 DAO 查询，task_instance_id 仅作为分片功能实用，实际业务数据关系并不强依赖 task_instance_id) + // (不会影响 DAO 查询，task_instance_id 仅作为分片功能，实际业务数据关系并不强依赖 task_instance_id) return DSL.trueCondition(); } if (FeatureToggle.checkFeature( From 17ee608771d4e53a3ca6e8eee4dae2f133a2f64e Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Tue, 19 Nov 2024 17:12:31 +0800 Subject: [PATCH 34/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dao/common/AbstractPropBasedDynamicComponent.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AbstractPropBasedDynamicComponent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AbstractPropBasedDynamicComponent.java index 9bf634ef7e..4a42d95fbb 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AbstractPropBasedDynamicComponent.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AbstractPropBasedDynamicComponent.java @@ -111,6 +111,7 @@ public AbstractPropBasedDynamicComponent( // 注册监听属性变化 this.propToggleStore.addPropChangeEventListener(migrateTargetPropName, this); + checkInit(); } protected void initCandidateComponents(Map candidateComponents) { @@ -122,7 +123,7 @@ protected void initCandidateComponents(Map candidateComponents) { * 如果组件处于迁移状态中，会阻塞当前线程直到切换完成 */ public C getCurrent(boolean blockWhenMigration) { - checkInit(); +// checkInit(); if (status == MigrationStatus.MIGRATING && blockWhenMigration) { // 如果组件正在迁移中，并且 blockWhenMigration = true, 需要等待迁移完成；当前线程阻塞 try { @@ -176,10 +177,10 @@ public void handlePropChangeEvent(String propName, PropToggle currentValue) { } private void migrateComponent(C targetComponent) { - long startTime = System.currentTimeMillis(); - log.info("Migrate component from {} to {} start...", current.getClass(), targetComponent.getClass()); boolean success = false; try { + long startTime = System.currentTimeMillis(); + log.info("Migrate component from {} to {} start...", current.getClass(), targetComponent.getClass()); PropToggle migrateServiceInstanceCountProp = propToggleStore.getPropToggle(PROP_NAME_SERVICE_INSTANCE_COUNT); if (migrateServiceInstanceCountProp == null) { From d0b786f1619269deb0bb0b71e73d074abf00879d Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Tue, 19 Nov 2024 17:33:42 +0800 Subject: [PATCH 35/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../toggle/prop/InMemoryPropToggleStore.java | 37 +++++++-------- .../prop/PropLoadApplicationRunner.java | 47 ------------------- .../prop/PropToggleAutoConfiguration.java | 9 +--- 3 files changed, 19 insertions(+), 74 deletions(-) delete mode 100644 src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/PropLoadApplicationRunner.java diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/InMemoryPropToggleStore.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/InMemoryPropToggleStore.java index cf18427d9e..390949b532 100644 --- a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/InMemoryPropToggleStore.java +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/InMemoryPropToggleStore.java @@ -53,35 +53,29 @@ public class InMemoryPropToggleStore implements PropToggleStore { * key: prop_name; value: PropToggle */ private volatile Map propToggles = new HashMap<>(); - /** - * 是否初始化 - */ - private volatile boolean isInitial = false; private final Map> propEventListeners = new HashMap<>(); - @Override + public InMemoryPropToggleStore(PropToggleProperties propToggleProperties) { + log.info("Init PropToggleProperties : {}", JsonUtils.toJson(propToggleProperties)); + loadAllPropToggles(propToggleProperties); + log.info("Init PropToggleProperties successfully"); + } + public PropToggle getPropToggle(String propName) { - if (!isInitial) { - synchronized (this) { - if (!isInitial) { - init(); - } - } - } return propToggles.get(propName); } @Override public void init() { - loadAllPropToggles(); + } @Override public boolean handleConfigChange(Set changedKeys, boolean ignoreException) { boolean loadResult = true; try { - loadAllPropToggles(); + loadAllPropToggles(getRealTimePropToggleProperties()); } catch (Throwable e) { log.warn("Load prop config error", e); loadResult = false; @@ -122,11 +116,16 @@ private Set getUniqueChangedPropNames(Set changedKeys) { } - private void loadAllPropToggles() { + /** + * 获取 PropToggleProperties 实时配置 + */ + private PropToggleProperties getRealTimePropToggleProperties() { + return ApplicationContextRegister.getBean(PropToggleProperties.class); + } + + private void loadAllPropToggles(PropToggleProperties propToggleProperties) { synchronized (this) { log.info("Load prop toggle start ..."); - PropToggleProperties propToggleProperties = - ApplicationContextRegister.getBean(PropToggleProperties.class); if (propToggleProperties.getProps() == null || propToggleProperties.getProps().isEmpty()) { log.info("Prop toggle config empty!"); @@ -144,8 +143,6 @@ private void loadAllPropToggles() { // 使用新的配置完全替换老的配置 propToggles = tmpPropToggles; log.info("Load prop toggle config done! props: {}", propToggles); - - isInitial = true; } } @@ -184,7 +181,7 @@ private PropToggle parsePropToggle(String propName, return propToggle; } - + @Override public void addPropChangeEventListener(String propName, PropChangeEventListener propChangeEventListener) { diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/PropLoadApplicationRunner.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/PropLoadApplicationRunner.java deleted file mode 100644 index 9dc5232870..0000000000 --- a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/PropLoadApplicationRunner.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.service.toggle.prop; - -import com.tencent.bk.job.common.util.toggle.prop.PropToggleStore; -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.ApplicationArguments; -import org.springframework.boot.ApplicationRunner; - -@Slf4j -public class PropLoadApplicationRunner implements ApplicationRunner { - private final PropToggleStore propToggleStore; - - public PropLoadApplicationRunner(PropToggleStore propToggleStore) { - this.propToggleStore = propToggleStore; - } - - @Override - public void run(ApplicationArguments args) throws Exception { - // 初始化配置；如果初始化错误，那么抛出异常终止程序启动 - log.info("PropLoadApplicationRunner start"); - propToggleStore.init(); - log.info("PropLoadApplicationRunner run success"); - } -} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/PropToggleAutoConfiguration.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/PropToggleAutoConfiguration.java index 80d0c7cb63..543831c274 100644 --- a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/PropToggleAutoConfiguration.java +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/PropToggleAutoConfiguration.java @@ -34,12 +34,7 @@ public class PropToggleAutoConfiguration { @Bean - public PropToggleStore propToggleStore() { - return new InMemoryPropToggleStore(); - } - - @Bean - public PropLoadApplicationRunner propLoadApplicationRunner(PropToggleStore propToggleStore) { - return new PropLoadApplicationRunner(propToggleStore); + public PropToggleStore propToggleStore(PropToggleProperties propToggleProperties) { + return new InMemoryPropToggleStore(propToggleProperties); } } From 4ec691399da978536073e4089c81bc6d2138b6a0 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Tue, 19 Nov 2024 17:59:04 +0800 Subject: [PATCH 36/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../FeatureToggleAutoConfiguration.java | 9 +--- .../toggle/feature/InMemoryFeatureStore.java | 43 +++++++------------ .../toggle/prop/InMemoryPropToggleStore.java | 9 +--- .../config/RefreshableConfigStore.java | 4 +- .../AbstractPropBasedDynamicComponent.java | 42 ++++++++---------- 5 files changed, 39 insertions(+), 68 deletions(-) diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/FeatureToggleAutoConfiguration.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/FeatureToggleAutoConfiguration.java index d74c657c59..1bc201cd7f 100644 --- a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/FeatureToggleAutoConfiguration.java +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/FeatureToggleAutoConfiguration.java @@ -37,17 +37,12 @@ public class FeatureToggleAutoConfiguration { @Bean - public FeatureStore featureStore() { - return new InMemoryFeatureStore(); + public FeatureStore featureStore(FeatureToggleProperties featureToggleProperties) { + return new InMemoryFeatureStore(featureToggleProperties); } @Bean public FeatureManager featureManager(FeatureStore featureStore, MeterRegistry meterRegistry) { return new DefaultFeatureManager(featureStore, meterRegistry); } - - @Bean - public FeatureLoadApplicationRunner featureLoadApplicationRunner(FeatureStore featureStore) { - return new FeatureLoadApplicationRunner(featureStore); - } } diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/InMemoryFeatureStore.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/InMemoryFeatureStore.java index 92c42a0cb7..5eef5287f4 100644 --- a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/InMemoryFeatureStore.java +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/InMemoryFeatureStore.java @@ -53,33 +53,23 @@ public class InMemoryFeatureStore implements FeatureStore { * key: featureId; value: Feature */ private volatile Map features = new HashMap<>(); - /** - * 是否初始化 - */ - private volatile boolean isInitial = false; - @Override - public Feature getFeature(String featureId) { - if (!isInitial) { - synchronized (this) { - if (!isInitial) { - init(); - } - } - } - return features.get(featureId); + public InMemoryFeatureStore(FeatureToggleProperties featureToggleProperties) { + log.info("Init InMemoryFeatureStore, properties : {}", JsonUtils.toJson(featureToggleProperties)); + loadAllFeatures(featureToggleProperties); + log.info("Init InMemoryFeatureStore successfully"); } @Override - public void init() { - loadAllFeatures(); + public Feature getFeature(String featureId) { + return features.get(featureId); } @Override public boolean handleConfigChange(Set changedKeys, boolean ignoreException) { boolean loadResult = true; try { - loadAllFeatures(); + loadAllFeatures(getRealTimeFeatureToggleProperties()); } catch (Throwable e) { log.warn("Load feature config error", e); loadResult = false; @@ -92,11 +82,16 @@ public boolean handleConfigChange(Set changedKeys, boolean ignoreExcepti return loadResult; } - private void loadAllFeatures() { + /** + * 获取 FeatureToggleProperties 实时配置 + */ + private FeatureToggleProperties getRealTimeFeatureToggleProperties() { + return ApplicationContextRegister.getBean(FeatureToggleProperties.class); + } + + private void loadAllFeatures(FeatureToggleProperties featureToggleProperties) { synchronized (this) { log.info("Load feature toggle start ..."); - FeatureToggleProperties featureToggleProperties = - ApplicationContextRegister.getBean(FeatureToggleProperties.class); if (featureToggleProperties.getFeatures() == null || featureToggleProperties.getFeatures().isEmpty()) { log.info("Feature toggle config empty!"); @@ -114,7 +109,6 @@ private void loadAllFeatures() { // 使用新的配置完全替换老的配置 features = tmpFeatures; log.info("Load feature toggle config done! features: {}", features); - isInitial = true; } } @@ -154,13 +148,6 @@ private Feature parseFeatureConfig( @Override public List listFeatures() { - if (!isInitial) { - synchronized (this) { - if (!isInitial) { - init(); - } - } - } return new ArrayList<>(features.values()); } diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/InMemoryPropToggleStore.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/InMemoryPropToggleStore.java index 390949b532..1278138faf 100644 --- a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/InMemoryPropToggleStore.java +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/InMemoryPropToggleStore.java @@ -57,20 +57,15 @@ public class InMemoryPropToggleStore implements PropToggleStore { private final Map> propEventListeners = new HashMap<>(); public InMemoryPropToggleStore(PropToggleProperties propToggleProperties) { - log.info("Init PropToggleProperties : {}", JsonUtils.toJson(propToggleProperties)); + log.info("Init InMemoryPropToggleStore, properties : {}", JsonUtils.toJson(propToggleProperties)); loadAllPropToggles(propToggleProperties); - log.info("Init PropToggleProperties successfully"); + log.info("Init InMemoryPropToggleStore successfully"); } public PropToggle getPropToggle(String propName) { return propToggles.get(propName); } - @Override - public void init() { - - } - @Override public boolean handleConfigChange(Set changedKeys, boolean ignoreException) { boolean loadResult = true; diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/refreshable/config/RefreshableConfigStore.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/refreshable/config/RefreshableConfigStore.java index 31fb667ec7..997945f88e 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/refreshable/config/RefreshableConfigStore.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/refreshable/config/RefreshableConfigStore.java @@ -34,7 +34,9 @@ public interface RefreshableConfigStore { /** * 初始化配置仓库，用于加载初始配置 */ - void init(); + default void init() { + + } /** * 重载变更的配置 diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AbstractPropBasedDynamicComponent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AbstractPropBasedDynamicComponent.java index 4a42d95fbb..3c2733bc38 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AbstractPropBasedDynamicComponent.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AbstractPropBasedDynamicComponent.java @@ -31,8 +31,9 @@ import com.tencent.bk.job.common.util.toggle.prop.PropChangeEventListener; import com.tencent.bk.job.common.util.toggle.prop.PropToggle; import com.tencent.bk.job.common.util.toggle.prop.PropToggleStore; -import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.data.redis.core.RedisTemplate; import java.util.ArrayList; @@ -51,8 +52,9 @@ /** * 基于属性动态控制的组件 */ -@Slf4j + public abstract class AbstractPropBasedDynamicComponent implements PropChangeEventListener { + private final Logger log = LoggerFactory.getLogger(this.getClass()); private Map candidateComponents; @@ -66,8 +68,6 @@ public abstract class AbstractPropBasedDynamicComponent implements PropChange private final PropToggleStore propToggleStore; - private volatile boolean isInitial = false; - /** * 动态属性名称 - 迁移目标 */ @@ -111,11 +111,23 @@ public AbstractPropBasedDynamicComponent( // 注册监听属性变化 this.propToggleStore.addPropChangeEventListener(migrateTargetPropName, this); - checkInit(); } protected void initCandidateComponents(Map candidateComponents) { this.candidateComponents = candidateComponents; + initCurrentComponent(); + } + + private void initCurrentComponent() { + // 从配置文件读取组件名称，并初始化 + String propValue = propToggleStore.getPropToggle(migrateTargetPropName).getDefaultValue(); + log.info("Init default component by prop: {}:{}", migrateTargetPropName, propValue); + this.current = getComponentByProp(this.candidateComponents, propValue); + if (this.current == null) { + log.error("No match component for {}", propValue); + throw new IllegalStateException("Unsupported component define by prop: " + propValue); + } + log.info("Use {} as default component", current.getClass()); } /** @@ -123,7 +135,6 @@ protected void initCandidateComponents(Map candidateComponents) { * 如果组件处于迁移状态中，会阻塞当前线程直到切换完成 */ public C getCurrent(boolean blockWhenMigration) { -// checkInit(); if (status == MigrationStatus.MIGRATING && blockWhenMigration) { // 如果组件正在迁移中，并且 blockWhenMigration = true, 需要等待迁移完成；当前线程阻塞 try { @@ -139,25 +150,6 @@ public C getCurrent(boolean blockWhenMigration) { return current; } - private void checkInit() { - if (!isInitial) { - synchronized (this) { - if (!isInitial) { - // 从配置文件读取组件名称，并初始化 - String propValue = propToggleStore.getPropToggle(migrateTargetPropName).getDefaultValue(); - log.info("Init default component by prop: {}:{}", migrateTargetPropName, propValue); - this.current = getComponentByProp(this.candidateComponents, propValue); - if (this.current == null) { - log.error("No match component for {}", propValue); - throw new IllegalArgumentException("Unsupported component define by prop: " + propValue); - } - log.info("Use {} as default component", current.getClass()); - this.isInitial = true; - } - } - } - } - @Override public void handlePropChangeEvent(String propName, PropToggle currentValue) { From 769cff0ab2d778a663b50df525949884f2b3c49b Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Tue, 19 Nov 2024 19:28:37 +0800 Subject: [PATCH 37/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../feature/FeatureLoadApplicationRunner.java | 50 ------------------- .../api/web/WebExecuteTaskResource.java | 2 +- .../web/WebTaskExecutionResultResource.java | 2 +- .../api/web/WebTaskInstanceResource.java | 2 +- .../web/impl/WebExecuteTaskResourceImpl.java | 15 +++++- .../WebTaskExecutionResultResourceImpl.java | 6 ++- .../web/impl/WebTaskInstanceResourceImpl.java | 10 +++- .../AbstractPropBasedDynamicComponent.java | 4 +- .../web/WebGlobalSettingsQueryResource.java | 2 +- .../api/web/WebGlobalSettingsResource.java | 2 +- 10 files changed, 33 insertions(+), 62 deletions(-) delete mode 100644 src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/FeatureLoadApplicationRunner.java diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/FeatureLoadApplicationRunner.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/FeatureLoadApplicationRunner.java deleted file mode 100644 index 828eab774c..0000000000 --- a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/FeatureLoadApplicationRunner.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.service.toggle.feature; - -import com.tencent.bk.job.common.util.toggle.feature.FeatureStore; -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.ApplicationArguments; -import org.springframework.boot.ApplicationRunner; - -/** - * 特性开关配置加载 ApplicationRunner - */ -@Slf4j -public class FeatureLoadApplicationRunner implements ApplicationRunner { - private final FeatureStore featureStore; - - public FeatureLoadApplicationRunner(FeatureStore featureStore) { - this.featureStore = featureStore; - } - - @Override - public void run(ApplicationArguments args) throws Exception { - // 初始化特性开关配置；如果初始化错误，那么抛出异常终止程序启动 - log.info("FeatureLoadApplicationRunner start"); - featureStore.init(); - log.info("FeatureLoadApplicationRunner run success"); - } -} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebExecuteTaskResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebExecuteTaskResource.java index c0a65cc4d2..71d1ef919e 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebExecuteTaskResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebExecuteTaskResource.java @@ -144,7 +144,7 @@ Response fastPushFile( @ApiOperation(value = "执行作业步骤操作", produces = "application/json") @PostMapping(value = {"/do-step-operation/stepInstanceId/{stepInstanceId}"}) @Deprecated - @CompatibleImplementation(name = "dao_add_task_instance_id", deprecatedVersion = "3.10.x", + @CompatibleImplementation(name = "dao_add_task_instance_id", deprecatedVersion = "3.11.x", type = CompatibleType.DEPLOY, explain = "发布完成后可以删除") Response doStepOperation( @ApiParam("用户名，网关自动传入") diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java index 6f5bb76489..ec5c769ac7 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java @@ -521,7 +521,7 @@ Response> getExecuteObjectsByResultType( @ApiOperation(value = "获取步骤执行历史", produces = "application/json") @GetMapping(value = {"/step-execution-history/{stepInstanceId}"}) @Deprecated - @CompatibleImplementation(name = "dao_add_task_instance_id", deprecatedVersion = "3.10.x", type = CompatibleType.DEPLOY, + @CompatibleImplementation(name = "dao_add_task_instance_id", deprecatedVersion = "3.11.x", type = CompatibleType.DEPLOY, explain = "发布完成后可以删除") Response> listStepExecutionHistory( @ApiParam("用户名，网关自动传入") diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskInstanceResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskInstanceResource.java index ed881adab7..1ef8006fef 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskInstanceResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskInstanceResource.java @@ -56,7 +56,7 @@ @WebAPI public interface WebTaskInstanceResource { @Deprecated - @CompatibleImplementation(name = "dao_add_task_instance_id", deprecatedVersion = "3.10.x", + @CompatibleImplementation(name = "dao_add_task_instance_id", deprecatedVersion = "3.11.x", type = CompatibleType.DEPLOY, explain = "发布完成后可以删除") @ApiOperation(value = "获取作业步骤实例详情", produces = "application/json") @GetMapping(value = {"/scope/{scopeType}/{scopeId}/task-instance/step_instance/{stepInstanceId}"}) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java index 359231c2a4..c0c763e8cb 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java @@ -26,6 +26,8 @@ import com.tencent.bk.audit.annotations.AuditEntry; import com.tencent.bk.audit.annotations.AuditRequestBody; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; import com.tencent.bk.job.common.exception.InvalidParamException; @@ -55,6 +57,7 @@ import com.tencent.bk.job.execute.model.FastTaskDTO; import com.tencent.bk.job.execute.model.FileDetailDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.StepOperationDTO; import com.tencent.bk.job.execute.model.StepRollingConfigDTO; @@ -71,6 +74,7 @@ import com.tencent.bk.job.execute.model.web.vo.StepExecuteVO; import com.tencent.bk.job.execute.model.web.vo.StepOperationVO; import com.tencent.bk.job.execute.model.web.vo.TaskExecuteVO; +import com.tencent.bk.job.execute.service.StepInstanceService; import com.tencent.bk.job.execute.service.TaskExecuteService; import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; @@ -96,10 +100,13 @@ @Slf4j public class WebExecuteTaskResourceImpl implements WebExecuteTaskResource { private final TaskExecuteService taskExecuteService; + private final StepInstanceService stepInstanceService; @Autowired - public WebExecuteTaskResourceImpl(TaskExecuteService taskExecuteService) { + public WebExecuteTaskResourceImpl(TaskExecuteService taskExecuteService, + StepInstanceService stepInstanceService) { this.taskExecuteService = taskExecuteService; + this.stepInstanceService = stepInstanceService; } @Override @@ -526,14 +533,18 @@ private List convertFileSource(List file } @Override + @CompatibleImplementation(name = "dao_add_task_instance_id", deprecatedVersion = "3.11.x", + type = CompatibleType.DEPLOY, explain = "发布完成后可以删除") public Response doStepOperation(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, Long stepInstanceId, WebStepOperation operation) { + // 兼容代码，部署完成后删除 + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstanceById(stepInstanceId); return doStepOperationV2(username, appResourceScope, scopeType, scopeId, - null, stepInstanceId, operation); + stepInstance.getTaskInstanceId(), stepInstanceId, operation); } @Override diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java index 89a22d87e7..f1ddd71a46 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java @@ -991,8 +991,10 @@ public Response> listStepExecutionHistory(String use String scopeId, Long stepInstanceId, Integer batch) { - - return listStepExecutionHistoryV2(username, appResourceScope, scopeType, scopeId, null, stepInstanceId, batch); + // 兼容代码，部署完成后删除 + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstanceById(stepInstanceId); + return listStepExecutionHistoryV2(username, appResourceScope, scopeType, scopeId, + stepInstance.getTaskInstanceId(), stepInstanceId, batch); } @Override diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java index c2aac679c7..b63a90eb4f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java @@ -25,6 +25,8 @@ package com.tencent.bk.job.execute.api.web.impl; import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; import com.tencent.bk.job.common.exception.InvalidParamException; @@ -115,13 +117,17 @@ public WebTaskInstanceResourceImpl(TaskInstanceService taskInstanceService, @Override @AuditEntry(actionId = ActionId.VIEW_HISTORY) + @CompatibleImplementation(name = "dao_add_task_instance_id", deprecatedVersion = "3.11.x", + type = CompatibleType.DEPLOY, explain = "发布完成后可以删除") public Response getStepInstanceDetail(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, Long stepInstanceId) { - - return getStepInstanceDetailV2(username, appResourceScope, scopeType, scopeId, null, stepInstanceId); + // 兼容代码，部署完成后删除 + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstanceById(stepInstanceId); + return getStepInstanceDetailV2(username, appResourceScope, scopeType, scopeId, + stepInstance.getTaskInstanceId(), stepInstanceId); } @Override diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AbstractPropBasedDynamicComponent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AbstractPropBasedDynamicComponent.java index 3c2733bc38..d96c15675d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AbstractPropBasedDynamicComponent.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AbstractPropBasedDynamicComponent.java @@ -55,7 +55,9 @@ public abstract class AbstractPropBasedDynamicComponent implements PropChangeEventListener { private final Logger log = LoggerFactory.getLogger(this.getClass()); - + /** + * 候选的组件。key: 属性名称；value: 组件实例 + */ private Map candidateComponents; private volatile MigrationStatus status = NOT_START; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebGlobalSettingsQueryResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebGlobalSettingsQueryResource.java index 6a4ce42869..268675c7fd 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebGlobalSettingsQueryResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebGlobalSettingsQueryResource.java @@ -115,7 +115,7 @@ Response getCMDBAppIndexUrl( ); - @CompatibleImplementation(name = "platform_info", deprecatedVersion = "3.10.x", type = CompatibleType.DEPLOY, + @CompatibleImplementation(name = "platform_info", deprecatedVersion = "3.11.x", type = CompatibleType.DEPLOY, explain = "发布完成后可以删除") @ApiOperation(value = "获取渲染后的平台设置", produces = "application/json") @GetMapping("/platformInfo") diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebGlobalSettingsResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebGlobalSettingsResource.java index 93fb1832fe..aeb4f81e83 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebGlobalSettingsResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebGlobalSettingsResource.java @@ -233,7 +233,7 @@ Response getFileUploadSettings( String username ); - @CompatibleImplementation(name = "platform_info", deprecatedVersion = "3.10.x", type = CompatibleType.DEPLOY, + @CompatibleImplementation(name = "platform_info", deprecatedVersion = "3.11.x", type = CompatibleType.DEPLOY, explain = "发布完成后可以删除") @ApiOperation(value = "获取平台信息-包含默认配置", produces = "application/json") @GetMapping("/platformInfoWithDefault") From 33f458fe4b75f62b3cf7439dc52bfba605653fd8 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Tue, 19 Nov 2024 19:35:20 +0800 Subject: [PATCH 38/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bk/job/backup/archive/JobInstanceArchiveCronJobs.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveCronJobs.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveCronJobs.java index 386fd87e28..ec962b564e 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveCronJobs.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveCronJobs.java @@ -53,7 +53,7 @@ public JobInstanceArchiveCronJobs(JobInstanceArchiveTaskGenerator jobInstanceArc /** * 定时创建归档任务,每天 0 点触发一次 */ - @Scheduled(cron = "0 0 0 * * *") + @Scheduled(cron = "0 * * * * *") public void generateArchiveTask() { if (!archiveProperties.isEnabled()) { return; From f406f1d778b511a32049a0ac8d83c00eedb958dc Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Tue, 19 Nov 2024 20:24:17 +0800 Subject: [PATCH 39/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mysql/dynamic/ds/DSLContextProvider.java | 3 +++ .../JobInstanceArchiveTaskGenerator.java | 25 +++++++++++++++---- .../impl/AbstractJobInstanceHotRecordDAO.java | 5 ++++ .../dao/impl/TaskInstanceRecordDAO.java | 19 ++++++++++++++ 4 files changed, 47 insertions(+), 5 deletions(-) diff --git a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/DSLContextProvider.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/DSLContextProvider.java index b8a1d17d7e..2a0160f74c 100644 --- a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/DSLContextProvider.java +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/DSLContextProvider.java @@ -26,6 +26,9 @@ import org.jooq.DSLContext; +/** + * Jooq DSLContext 提供者 + */ public interface DSLContextProvider { DSLContext get(String tableName); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java index c2b23cbd61..d0de19e8c6 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java @@ -70,11 +70,24 @@ public JobInstanceArchiveTaskGenerator(ArchiveTaskService archiveTaskService, @JobTransactional(transactionManager = "jobBackupTransactionManager") public void generate() { + if (taskInstanceRecordDAO.isTableEmpty()) { + log.info("Job instance table is empty and does not require processing"); + return; + } List archiveTaskList = new ArrayList<>(); + // 归档起始时间 LocalDateTime startDateTime = computeArchiveStartDateTime(); - LocalDateTime endDateTime = unixTimestampToUtcLocalDateTime(getEndTime(archiveProperties.getKeepDays())); - // 创建归档任务 + // 归档结束时间 + LocalDateTime endDateTime = unixTimestampToUtcLocalDateTime( + computeArchiveEndTime(archiveProperties.getKeepDays())); + if (endDateTime.isBefore(startDateTime) || endDateTime.equals(startDateTime)) { + log.info("Archive endTime is before startTime, does not require processing"); + return; + } + + log.info("Generate job instance archive task, startDateTime: {}, endDateTime: {}", startDateTime, endDateTime); + // 创建归档任务。每个基础归档任务定义为：一个数据节点（db+表）+ 日期 + 小时 while (startDateTime.isBefore(endDateTime)) { // 水平分库分表 if (isHorizontalShardingEnabled()) { @@ -83,7 +96,7 @@ public void generate() { startDateTime, archiveProperties.getTasks().getJobInstance().getShardingDataNodes())); } else { // 单db - DbDataNode dbDataNode = new DbDataNode(DbDataNodeTypeEnum.STANDALONE, null, null, null); + DbDataNode dbDataNode = DbDataNode.standaloneDbDatNode(); archiveTaskList.add(buildArchiveTask(ArchiveTaskTypeEnum.JOB_INSTANCE, startDateTime, dbDataNode)); } @@ -93,6 +106,8 @@ public void generate() { if (CollectionUtils.isNotEmpty(archiveTaskList)) { archiveTaskList.forEach(archiveTaskService::saveArchiveTask); log.info("Add archive tasks : {}", JsonUtils.toJson(archiveTaskList)); + } else { + log.info("No new archive tasks are generated"); } } @@ -142,7 +157,7 @@ private LocalDateTime computeArchiveStartDateTime() { JobInstanceArchiveTaskInfo latestArchiveTask = archiveTaskService.getLatestArchiveTask(ArchiveTaskTypeEnum.JOB_INSTANCE); if (latestArchiveTask == null) { - log.info("Latest archive task is empty, try compute from task_instance table record"); + log.info("Latest archive task is empty, try compute from table min job create time"); Long minJobCreateTime = taskInstanceRecordDAO.getMinJobCreateTime(); startDateTime = toHourlyRoundDown(unixTimestampToUtcLocalDateTime(minJobCreateTime)); } else { @@ -165,7 +180,7 @@ private int computeHour(LocalDateTime dateTime) { return dateTime.getHour(); } - private Long getEndTime(int archiveDays) { + private Long computeArchiveEndTime(int archiveDays) { DateTime now = DateTime.now(); // 置为前一天天 24:00:00 long todayMaxMills = now.minusMillis(now.getMillisOfDay()).getMillis(); diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceHotRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceHotRecordDAO.java index 7c423045f6..27db0ffc2d 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceHotRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceHotRecordDAO.java @@ -39,6 +39,11 @@ import java.util.Collection; import java.util.List; +/** + * 作业实例热数据查询 DAO 基础抽象实现 + * + * @param 表记录 + */ public abstract class AbstractJobInstanceHotRecordDAO implements JobInstanceHotRecordDAO { protected final DSLContextProvider dslContextProvider; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceRecordDAO.java index e74d07ac12..3e02ef3b90 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceRecordDAO.java @@ -66,6 +66,9 @@ public Table getTable() { return TABLE; } + /** + * 获取表中作业的最早创建时间 + */ public Long getMinJobCreateTime() { Record1 record = dsl().select(min(TABLE.CREATE_TIME)) @@ -80,6 +83,22 @@ public Long getMinJobCreateTime() { return Long.MAX_VALUE; } + /** + * 是否为空表 + */ + public boolean isTableEmpty() { + return !dsl().fetchExists(TABLE); + } + + /** + * 按时间范围+作业实例 ID，顺序读取表记录 + * + * @param fromTimestamp 开始时间(include) + * @param endTimestamp 开始时间(exclude) + * @param fromJobInstanceId 起始作业实例 ID + * @param limit 读取最大行数 + * @return 记录 + */ public List readSortedJobInstanceFromHotDB(Long fromTimestamp, Long endTimestamp, Long fromJobInstanceId, From 7c4940e1e6fd63cbcb90f1ea5e03f2cdbd458eb8 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Wed, 20 Nov 2024 12:34:55 +0800 Subject: [PATCH 40/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/test/java/FeatureToggleTest.java | 2 +- .../archive/JobInstanceArchiveTaskGenerator.java | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/backend/commons/common-service/src/test/java/FeatureToggleTest.java b/src/backend/commons/common-service/src/test/java/FeatureToggleTest.java index 094a19af9b..97029e6495 100644 --- a/src/backend/commons/common-service/src/test/java/FeatureToggleTest.java +++ b/src/backend/commons/common-service/src/test/java/FeatureToggleTest.java @@ -62,7 +62,7 @@ static void beforeAll() { MockedStatic mockedStatic = Mockito.mockStatic(ApplicationContextRegister.class); mockedStatic.when(() -> ApplicationContextRegister.getBean(FeatureToggleProperties.class)) .thenReturn(featureToggleProperties); - FeatureStore mockFeatureStore = new InMemoryFeatureStore(); + FeatureStore mockFeatureStore = new InMemoryFeatureStore(featureToggleProperties); mockFeatureStore.init(); mockedStatic.close(); diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java index d0de19e8c6..7d2805a0ff 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java @@ -97,7 +97,10 @@ public void generate() { } else { // 单db DbDataNode dbDataNode = DbDataNode.standaloneDbDatNode(); - archiveTaskList.add(buildArchiveTask(ArchiveTaskTypeEnum.JOB_INSTANCE, startDateTime, dbDataNode)); + JobInstanceArchiveTaskInfo archiveTaskInfo = + buildArchiveTask(ArchiveTaskTypeEnum.JOB_INSTANCE, startDateTime, dbDataNode); + archiveTaskList.add(archiveTaskInfo); + log.info("Add JobInstanceArchiveTaskInfo: {}", archiveTaskInfo.buildTaskUniqueId()); } startDateTime = startDateTime.plusHours(1L); @@ -127,7 +130,10 @@ private List buildArchiveTasksForShardingDataNodes( DbDataNode dbDataNode = new DbDataNode(DbDataNodeTypeEnum.SHARDING, dataSource, dbNodeIndex, tableNodeIndex); // 作业实例数据归档任务 - tasks.add(buildArchiveTask(archiveTaskType, startDateTime, dbDataNode)); + JobInstanceArchiveTaskInfo archiveTaskInfo = + buildArchiveTask(archiveTaskType, startDateTime, dbDataNode); + tasks.add(archiveTaskInfo); + log.info("Add JobInstanceArchiveTaskInfo: {}", archiveTaskInfo.buildTaskUniqueId()); } } }); From a662b19a6852e3798b3ea2177243d61f26442cf5 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Wed, 20 Nov 2024 12:48:48 +0800 Subject: [PATCH 41/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../JobInstanceArchiveTaskGenerator.java | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java index 7d2805a0ff..52f6c507d3 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java @@ -77,33 +77,34 @@ public void generate() { List archiveTaskList = new ArrayList<>(); // 归档起始时间 - LocalDateTime startDateTime = computeArchiveStartDateTime(); + LocalDateTime archiveStartDateTime = computeArchiveStartDateTime(); // 归档结束时间 - LocalDateTime endDateTime = unixTimestampToUtcLocalDateTime( - computeArchiveEndTime(archiveProperties.getKeepDays())); - if (endDateTime.isBefore(startDateTime) || endDateTime.equals(startDateTime)) { - log.info("Archive endTime is before startTime, does not require processing"); + LocalDateTime archiveEndDateTime = computeArchiveEndTime(archiveProperties.getKeepDays()); + if (archiveEndDateTime.isBefore(archiveStartDateTime) || archiveEndDateTime.equals(archiveStartDateTime)) { + log.info("Archive endTime is before startTime, does not require processing. startTime: {}, endTime: {}", + archiveStartDateTime, archiveEndDateTime); return; } - log.info("Generate job instance archive task, startDateTime: {}, endDateTime: {}", startDateTime, endDateTime); + log.info("Generate job instance archive task, archiveStartDateTime: {}, archiveEndDateTime: {}", + archiveStartDateTime, archiveEndDateTime); // 创建归档任务。每个基础归档任务定义为：一个数据节点（db+表）+ 日期 + 小时 - while (startDateTime.isBefore(endDateTime)) { + while (archiveStartDateTime.isBefore(archiveEndDateTime)) { // 水平分库分表 if (isHorizontalShardingEnabled()) { // 作业实例数据归档任务,现版本暂不支持 archiveTaskList.addAll(buildArchiveTasksForShardingDataNodes(ArchiveTaskTypeEnum.JOB_INSTANCE, - startDateTime, archiveProperties.getTasks().getJobInstance().getShardingDataNodes())); + archiveStartDateTime, archiveProperties.getTasks().getJobInstance().getShardingDataNodes())); } else { // 单db DbDataNode dbDataNode = DbDataNode.standaloneDbDatNode(); JobInstanceArchiveTaskInfo archiveTaskInfo = - buildArchiveTask(ArchiveTaskTypeEnum.JOB_INSTANCE, startDateTime, dbDataNode); + buildArchiveTask(ArchiveTaskTypeEnum.JOB_INSTANCE, archiveStartDateTime, dbDataNode); archiveTaskList.add(archiveTaskInfo); log.info("Add JobInstanceArchiveTaskInfo: {}", archiveTaskInfo.buildTaskUniqueId()); } - startDateTime = startDateTime.plusHours(1L); + archiveStartDateTime = archiveStartDateTime.plusHours(1L); } if (CollectionUtils.isNotEmpty(archiveTaskList)) { @@ -186,14 +187,14 @@ private int computeHour(LocalDateTime dateTime) { return dateTime.getHour(); } - private Long computeArchiveEndTime(int archiveDays) { + private LocalDateTime computeArchiveEndTime(int archiveDays) { DateTime now = DateTime.now(); // 置为前一天天 24:00:00 long todayMaxMills = now.minusMillis(now.getMillisOfDay()).getMillis(); //减掉当前xx天后 long archiveMills = archiveDays * 24 * 3600 * 1000L; - return todayMaxMills - archiveMills; + return unixTimestampToUtcLocalDateTime(todayMaxMills - archiveMills); } private LocalDateTime unixTimestampToUtcLocalDateTime(long unixTimestamp) { From 15382b9beb0c10b5564b31d213f696305f67080c Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Wed, 20 Nov 2024 15:20:59 +0800 Subject: [PATCH 42/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../JobInstanceArchiveTaskGenerator.java | 45 +++-------- .../archive/service/ArchiveTaskService.java | 2 +- .../archive/util/ArchiveDateTimeUtil.java | 58 ++++++++++++++ .../ArchiveDbNodePriorityEvaluatorTest.java | 3 +- .../archive/util/ArchiveDateTimeUtilTest.java | 78 +++++++++++++++++++ .../web/WebTaskExecutionResultResource.java | 4 +- 6 files changed, 152 insertions(+), 38 deletions(-) create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtil.java create mode 100644 src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtilTest.java diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java index 52f6c507d3..3f7d3869fe 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java @@ -28,19 +28,17 @@ import com.tencent.bk.job.backup.archive.model.DbDataNode; import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; +import com.tencent.bk.job.backup.archive.util.ArchiveDateTimeUtil; import com.tencent.bk.job.backup.config.ArchiveProperties; import com.tencent.bk.job.backup.constant.ArchiveTaskStatusEnum; import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; import com.tencent.bk.job.backup.constant.DbDataNodeTypeEnum; import com.tencent.bk.job.common.mysql.JobTransactional; import com.tencent.bk.job.common.mysql.dynamic.ds.DataSourceMode; -import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.common.util.json.JsonUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; -import org.joda.time.DateTime; -import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.util.ArrayList; @@ -146,8 +144,8 @@ private JobInstanceArchiveTaskInfo buildArchiveTask(ArchiveTaskTypeEnum archiveT LocalDateTime startDateTime, DbDataNode dbDataNode) { JobInstanceArchiveTaskInfo archiveTask = new JobInstanceArchiveTaskInfo(); - int day = computeDay(startDateTime); - int hour = computeHour(startDateTime); + int day = ArchiveDateTimeUtil.computeDay(startDateTime); + int hour = ArchiveDateTimeUtil.computeHour(startDateTime); archiveTask.setDay(day); archiveTask.setHour(hour); archiveTask.setFromTimestamp(1000 * startDateTime.toEpochSecond(ZoneOffset.UTC)); @@ -164,11 +162,14 @@ private LocalDateTime computeArchiveStartDateTime() { JobInstanceArchiveTaskInfo latestArchiveTask = archiveTaskService.getLatestArchiveTask(ArchiveTaskTypeEnum.JOB_INSTANCE); if (latestArchiveTask == null) { + // 从表数据中的 job_create_time 计算归档任务开始时间 log.info("Latest archive task is empty, try compute from table min job create time"); - Long minJobCreateTime = taskInstanceRecordDAO.getMinJobCreateTime(); - startDateTime = toHourlyRoundDown(unixTimestampToUtcLocalDateTime(minJobCreateTime)); + Long minJobCreateTimeMills = taskInstanceRecordDAO.getMinJobCreateTime(); + startDateTime = ArchiveDateTimeUtil.toHourlyRoundDown( + ArchiveDateTimeUtil.unixTimestampMillToLocalDateTime(minJobCreateTimeMills)); } else { - startDateTime = unixTimestampToUtcLocalDateTime(latestArchiveTask.getToTimestamp()); + // 根据最新的归档任务计算开始 + startDateTime = ArchiveDateTimeUtil.unixTimestampMillToLocalDateTime(latestArchiveTask.getToTimestamp()); } return startDateTime; @@ -179,32 +180,8 @@ private boolean isHorizontalShardingEnabled() { .equals(DataSourceMode.Constants.HORIZONTAL_SHARDING); } - private int computeDay(LocalDateTime dateTime) { - return Integer.parseInt(DateUtils.formatLocalDateTime(dateTime, "%Y%m%d")); - } - - private int computeHour(LocalDateTime dateTime) { - return dateTime.getHour(); - } - private LocalDateTime computeArchiveEndTime(int archiveDays) { - DateTime now = DateTime.now(); - // 置为前一天天 24:00:00 - long todayMaxMills = now.minusMillis(now.getMillisOfDay()).getMillis(); - - //减掉当前xx天后 - long archiveMills = archiveDays * 24 * 3600 * 1000L; - return unixTimestampToUtcLocalDateTime(todayMaxMills - archiveMills); - } - - private LocalDateTime unixTimestampToUtcLocalDateTime(long unixTimestamp) { - // 创建一个 Instant 对象，表示从 1970-01-01T00:00:00Z 开始的指定秒数 - Instant instant = Instant.ofEpochSecond(unixTimestamp); - // 将 Instant 对象转换为 UTC 时区的 LocalDateTime 对象 - return LocalDateTime.ofInstant(instant, ZoneOffset.UTC); - } - - private LocalDateTime toHourlyRoundDown(LocalDateTime localDateTime) { - return localDateTime.withMinute(0).withSecond(0).withNano(0); + LocalDateTime now = LocalDateTime.now(); + return ArchiveDateTimeUtil.computeStartOfDayBeforeDays(now, archiveDays); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/service/ArchiveTaskService.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/service/ArchiveTaskService.java index 2f04573941..b8de1de8eb 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/service/ArchiveTaskService.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/service/ArchiveTaskService.java @@ -42,7 +42,7 @@ public ArchiveTaskService(ArchiveTaskDAO archiveTaskDAO) { } /** - * 获取最新的归档任务 + * 获取最新创建的归档任务 * * @param taskType 归档任务类型 */ diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtil.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtil.java new file mode 100644 index 0000000000..cd7f97551c --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtil.java @@ -0,0 +1,58 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.util; + +import com.tencent.bk.job.common.util.date.DateUtils; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; + +public class ArchiveDateTimeUtil { + public static int computeDay(LocalDateTime dateTime) { + return Integer.parseInt(DateUtils.formatLocalDateTime(dateTime, "yyyyMMdd")); + } + + public static int computeHour(LocalDateTime dateTime) { + return dateTime.getHour(); + } + + public static LocalDateTime computeStartOfDayBeforeDays(LocalDateTime now, int beforeDays) { + // 当前日期的最早时间 + LocalDateTime startOfDay = now.toLocalDate().atStartOfDay(); + return startOfDay.minusDays(beforeDays); + } + + public static LocalDateTime unixTimestampMillToLocalDateTime(long unixTimestampMill) { + // 创建一个 Instant 对象，表示从 1970-01-01T00:00:00Z 开始的指定秒数 + Instant instant = Instant.ofEpochMilli(unixTimestampMill); + // 将 Instant 对象转换为 当前时区的 LocalDateTime 对象 + return LocalDateTime.ofInstant(instant, ZoneId.systemDefault()); + } + + public static LocalDateTime toHourlyRoundDown(LocalDateTime localDateTime) { + return localDateTime.withMinute(0).withSecond(0).withNano(0); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluatorTest.java b/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluatorTest.java index 3ab9c408ee..2ed1c53b4e 100644 --- a/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluatorTest.java +++ b/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluatorTest.java @@ -82,7 +82,8 @@ void testShardingDb() { scheduleTaskCountGroupByDb.put("ds:0", 2); scheduleTaskCountGroupByDb.put("ds:1", 1); - dbNodeTasksInfo = ArchiveDbNodePriorityEvaluator.evaluateHighestPriorityDbNode(runningTasks, scheduleTaskCountGroupByDb); + dbNodeTasksInfo = ArchiveDbNodePriorityEvaluator.evaluateHighestPriorityDbNode( + runningTasks, scheduleTaskCountGroupByDb); assertThat(dbNodeTasksInfo).isNotNull(); assertThat(dbNodeTasksInfo.getDbNodeId()).isEqualTo("ds:0"); diff --git a/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtilTest.java b/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtilTest.java new file mode 100644 index 0000000000..106fde2d00 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtilTest.java @@ -0,0 +1,78 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.util; + +import org.junit.jupiter.api.Test; + +import java.time.LocalDateTime; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +class ArchiveDateTimeUtilTest { + + @Test + void computeDay() { + LocalDateTime dateTime = LocalDateTime.of(2024, 11, 11, 1, 11, 11); + assertThat(ArchiveDateTimeUtil.computeDay(dateTime)).isEqualTo(20241111); + + dateTime = LocalDateTime.of(2024, 8, 12, 0, 0, 0); + assertThat(ArchiveDateTimeUtil.computeDay(dateTime)).isEqualTo(20240812); + } + + @Test + void computeHour() { + LocalDateTime dateTime = LocalDateTime.of(2024, 11, 11, 1, 11, 11); + assertThat(ArchiveDateTimeUtil.computeHour(dateTime)).isEqualTo(1); + + dateTime = LocalDateTime.of(2024, 8, 12, 0, 0, 0); + assertThat(ArchiveDateTimeUtil.computeHour(dateTime)).isEqualTo(0); + + dateTime = LocalDateTime.of(2024, 8, 12, 0, 0, 1); + assertThat(ArchiveDateTimeUtil.computeHour(dateTime)).isEqualTo(0); + } + + @Test + void computeMaxLocalDatTimeBeforeDays() { + LocalDateTime dateTime = LocalDateTime.of(2024, 11, 11, 1, 11, 11); + LocalDateTime result = ArchiveDateTimeUtil.computeStartOfDayBeforeDays(dateTime, 1); + assertThat(result).isEqualTo(LocalDateTime.of(2024, 11, 10, 0, 0, 0)); + + result = ArchiveDateTimeUtil.computeStartOfDayBeforeDays(dateTime, 30); + assertThat(result).isEqualTo(LocalDateTime.of(2024, 10, 12, 0, 0, 0)); + } + + @Test + void unixTimestampMillToLocalDateTime() { + LocalDateTime dateTime = ArchiveDateTimeUtil.unixTimestampMillToLocalDateTime(1732072271000L); + assertThat(dateTime).isEqualTo(LocalDateTime.of(2024, 11, 20, 11, 11, 11)); + } + + @Test + void toHourlyRoundDown() { + LocalDateTime dateTime = + ArchiveDateTimeUtil.toHourlyRoundDown(LocalDateTime.of(2024, 11, 20, 11, 1, 1)); + assertThat(dateTime).isEqualTo(LocalDateTime.of(2024, 11, 20, 11, 0, 0)); + } +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java index ec5c769ac7..5d665f075c 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java @@ -521,8 +521,8 @@ Response> getExecuteObjectsByResultType( @ApiOperation(value = "获取步骤执行历史", produces = "application/json") @GetMapping(value = {"/step-execution-history/{stepInstanceId}"}) @Deprecated - @CompatibleImplementation(name = "dao_add_task_instance_id", deprecatedVersion = "3.11.x", type = CompatibleType.DEPLOY, - explain = "发布完成后可以删除") + @CompatibleImplementation(name = "dao_add_task_instance_id", deprecatedVersion = "3.11.x", + type = CompatibleType.DEPLOY, explain = "发布完成后可以删除") Response> listStepExecutionHistory( @ApiParam("用户名，网关自动传入") @RequestHeader("username") From cfaf616562c75d83fed9e1490350076bced4e946 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Wed, 20 Nov 2024 15:26:54 +0800 Subject: [PATCH 43/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../job/backup/archive/JobInstanceArchiveTaskGenerator.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java index 3f7d3869fe..264fa5ad6e 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java @@ -148,8 +148,9 @@ private JobInstanceArchiveTaskInfo buildArchiveTask(ArchiveTaskTypeEnum archiveT int hour = ArchiveDateTimeUtil.computeHour(startDateTime); archiveTask.setDay(day); archiveTask.setHour(hour); - archiveTask.setFromTimestamp(1000 * startDateTime.toEpochSecond(ZoneOffset.UTC)); - archiveTask.setToTimestamp(1000L * startDateTime.plusHours(1L).toEpochSecond(ZoneOffset.UTC)); + long fromTimestamp = 1000 * startDateTime.toEpochSecond(ZoneOffset.UTC); + archiveTask.setFromTimestamp(fromTimestamp); + archiveTask.setToTimestamp(fromTimestamp + 1000 * 3600L); archiveTask.setTaskType(archiveTaskType); archiveTask.setDbDataNode(dbDataNode); archiveTask.setStatus(ArchiveTaskStatusEnum.PENDING); From 83aea6ae21f6fe2cc0952a65c098391ddda8a940 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Wed, 20 Nov 2024 15:56:18 +0800 Subject: [PATCH 44/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../archive/JobInstanceArchiveTaskGenerator.java | 1 + .../backup/archive/util/ArchiveDateTimeUtil.java | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java index 264fa5ad6e..e060b91dbf 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java @@ -166,6 +166,7 @@ private LocalDateTime computeArchiveStartDateTime() { // 从表数据中的 job_create_time 计算归档任务开始时间 log.info("Latest archive task is empty, try compute from table min job create time"); Long minJobCreateTimeMills = taskInstanceRecordDAO.getMinJobCreateTime(); + log.info("Min job create time in db is : {}", minJobCreateTimeMills); startDateTime = ArchiveDateTimeUtil.toHourlyRoundDown( ArchiveDateTimeUtil.unixTimestampMillToLocalDateTime(minJobCreateTimeMills)); } else { diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtil.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtil.java index cd7f97551c..b04c199725 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtil.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtil.java @@ -30,11 +30,26 @@ import java.time.LocalDateTime; import java.time.ZoneId; +/** + * 归档日期计算工具类 + */ public class ArchiveDateTimeUtil { + /** + * 计算天（日期），返回 yyyyMMdd 格式的数字 + * + * @param dateTime 日期 + * @return 天（日期） + */ public static int computeDay(LocalDateTime dateTime) { return Integer.parseInt(DateUtils.formatLocalDateTime(dateTime, "yyyyMMdd")); } + /** + * 计算小时 + * + * @param dateTime 日期 + * @return 小时 + */ public static int computeHour(LocalDateTime dateTime) { return dateTime.getHour(); } From 68c51b951bd9674ca1d86338b5c70de255675000 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Wed, 20 Nov 2024 16:51:53 +0800 Subject: [PATCH 45/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java index d6db19f3d2..2b71211b18 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java @@ -80,6 +80,7 @@ public JobInstanceArchiveTaskInfo getLatestArchiveTask(ArchiveTaskTypeEnum taskT .from(T) .where(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(taskType.getType()))) .orderBy(T.DAY.desc(), T.HOUR.desc()) + .limit(1) .fetchOne(); return extract(record); From 08ceb394e503858fe1ec0954fc4e20423c549392 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Wed, 20 Nov 2024 16:58:32 +0800 Subject: [PATCH 46/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java index 2b71211b18..407c763fd3 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java @@ -204,6 +204,7 @@ public JobInstanceArchiveTaskInfo getFirstScheduleArchiveTaskByDb(ArchiveTaskTyp .and(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(taskType.getType()))) .and(T.DB_NODE.eq(dbNodeId)) .orderBy(T.DAY.desc(), T.HOUR.desc(), T.DATA_NODE) + .limit(1) .fetchOne(); return extract(record); From 8c7522fccfaae3ccb1dd455f7bb9e4676d7e5f72 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Wed, 20 Nov 2024 17:04:27 +0800 Subject: [PATCH 47/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/tencent/bk/job/backup/archive/model/DbDataNode.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DbDataNode.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DbDataNode.java index 4f5b70ac0e..2968bd8240 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DbDataNode.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DbDataNode.java @@ -64,9 +64,9 @@ public DbDataNode(DbDataNodeTypeEnum type, String dataSource, Integer dbIndex, I public String toDataNodeId() { switch (type) { case STANDALONE: - return type + ":" + STANDALONE_DS_NAME; + return type.getValue() + ":" + STANDALONE_DS_NAME; case SHARDING: - return type + ":" + dataSource + ":" + dbIndex + ":" + tableIndex; + return type.getValue() + ":" + dataSource + ":" + dbIndex + ":" + tableIndex; default: throw new IllegalArgumentException("Invalid DbDataNodeTypeEnum"); } From 70488c8242cb560df1654f86245b5284c2b22d57 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Thu, 21 Nov 2024 08:55:07 +0800 Subject: [PATCH 48/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../JobInstanceArchiveTaskGenerator.java | 19 ++++++++++++------- .../archive/util/ArchiveDateTimeUtil.java | 7 +++++++ .../archive/util/ArchiveDateTimeUtilTest.java | 10 +++++++++- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java index e060b91dbf..47f50b9237 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java @@ -40,7 +40,7 @@ import org.apache.commons.collections4.CollectionUtils; import java.time.LocalDateTime; -import java.time.ZoneOffset; +import java.time.ZoneId; import java.util.ArrayList; import java.util.List; @@ -74,12 +74,14 @@ public void generate() { } List archiveTaskList = new ArrayList<>(); - // 归档起始时间 + log.info("Compute archive task generate startDateTime and endDateTime begin..."); + // 归档任务创建范围-起始时间 LocalDateTime archiveStartDateTime = computeArchiveStartDateTime(); - // 归档结束时间 + // 归档任务创建范围-结束时间 LocalDateTime archiveEndDateTime = computeArchiveEndTime(archiveProperties.getKeepDays()); if (archiveEndDateTime.isBefore(archiveStartDateTime) || archiveEndDateTime.equals(archiveStartDateTime)) { - log.info("Archive endTime is before startTime, does not require processing. startTime: {}, endTime: {}", + log.info("Archive endTime is before startTime, does not require generating archive task." + + " startTime: {}, endTime: {}", archiveStartDateTime, archiveEndDateTime); return; } @@ -88,6 +90,7 @@ public void generate() { archiveStartDateTime, archiveEndDateTime); // 创建归档任务。每个基础归档任务定义为：一个数据节点（db+表）+ 日期 + 小时 while (archiveStartDateTime.isBefore(archiveEndDateTime)) { + log.info("Generate archive task for datetime : {}", archiveStartDateTime); // 水平分库分表 if (isHorizontalShardingEnabled()) { // 作业实例数据归档任务,现版本暂不支持 @@ -99,7 +102,7 @@ public void generate() { JobInstanceArchiveTaskInfo archiveTaskInfo = buildArchiveTask(ArchiveTaskTypeEnum.JOB_INSTANCE, archiveStartDateTime, dbDataNode); archiveTaskList.add(archiveTaskInfo); - log.info("Add JobInstanceArchiveTaskInfo: {}", archiveTaskInfo.buildTaskUniqueId()); + log.info("Add JobInstanceArchiveTaskInfo: {}", JsonUtils.toJson(archiveTaskInfo)); } archiveStartDateTime = archiveStartDateTime.plusHours(1L); @@ -132,7 +135,7 @@ private List buildArchiveTasksForShardingDataNodes( JobInstanceArchiveTaskInfo archiveTaskInfo = buildArchiveTask(archiveTaskType, startDateTime, dbDataNode); tasks.add(archiveTaskInfo); - log.info("Add JobInstanceArchiveTaskInfo: {}", archiveTaskInfo.buildTaskUniqueId()); + log.info("Add JobInstanceArchiveTaskInfo: {}", JsonUtils.toJson(archiveTaskInfo)); } } }); @@ -148,7 +151,7 @@ private JobInstanceArchiveTaskInfo buildArchiveTask(ArchiveTaskTypeEnum archiveT int hour = ArchiveDateTimeUtil.computeHour(startDateTime); archiveTask.setDay(day); archiveTask.setHour(hour); - long fromTimestamp = 1000 * startDateTime.toEpochSecond(ZoneOffset.UTC); + long fromTimestamp = ArchiveDateTimeUtil.toTimestampMillsAtZone(startDateTime, ZoneId.systemDefault()); archiveTask.setFromTimestamp(fromTimestamp); archiveTask.setToTimestamp(fromTimestamp + 1000 * 3600L); archiveTask.setTaskType(archiveTaskType); @@ -171,6 +174,7 @@ private LocalDateTime computeArchiveStartDateTime() { ArchiveDateTimeUtil.unixTimestampMillToLocalDateTime(minJobCreateTimeMills)); } else { // 根据最新的归档任务计算开始 + log.info("Compute archive from latest generated archive task: {}", JsonUtils.toJson(latestArchiveTask)); startDateTime = ArchiveDateTimeUtil.unixTimestampMillToLocalDateTime(latestArchiveTask.getToTimestamp()); } @@ -183,6 +187,7 @@ private boolean isHorizontalShardingEnabled() { } private LocalDateTime computeArchiveEndTime(int archiveDays) { + log.info("Compute archive task generate end time before {} days", archiveDays); LocalDateTime now = LocalDateTime.now(); return ArchiveDateTimeUtil.computeStartOfDayBeforeDays(now, archiveDays); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtil.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtil.java index b04c199725..a0e196c96b 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtil.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtil.java @@ -28,6 +28,7 @@ import java.time.Instant; import java.time.LocalDateTime; +import java.time.OffsetDateTime; import java.time.ZoneId; /** @@ -70,4 +71,10 @@ public static LocalDateTime unixTimestampMillToLocalDateTime(long unixTimestampM public static LocalDateTime toHourlyRoundDown(LocalDateTime localDateTime) { return localDateTime.withMinute(0).withSecond(0).withNano(0); } + + public static long toTimestampMillsAtZone(LocalDateTime localDateTime, ZoneId zoneId) { + OffsetDateTime offsetDateTime = + localDateTime.atOffset(zoneId.getRules().getOffset(localDateTime)); + return 1000 * offsetDateTime.toEpochSecond(); + } } diff --git a/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtilTest.java b/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtilTest.java index 106fde2d00..9c6048542b 100644 --- a/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtilTest.java +++ b/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtilTest.java @@ -27,6 +27,7 @@ import org.junit.jupiter.api.Test; import java.time.LocalDateTime; +import java.time.ZoneId; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; @@ -54,7 +55,7 @@ void computeHour() { } @Test - void computeMaxLocalDatTimeBeforeDays() { + void computeStartOfDayBeforeDays() { LocalDateTime dateTime = LocalDateTime.of(2024, 11, 11, 1, 11, 11); LocalDateTime result = ArchiveDateTimeUtil.computeStartOfDayBeforeDays(dateTime, 1); assertThat(result).isEqualTo(LocalDateTime.of(2024, 11, 10, 0, 0, 0)); @@ -75,4 +76,11 @@ void toHourlyRoundDown() { ArchiveDateTimeUtil.toHourlyRoundDown(LocalDateTime.of(2024, 11, 20, 11, 1, 1)); assertThat(dateTime).isEqualTo(LocalDateTime.of(2024, 11, 20, 11, 0, 0)); } + + @Test + void toTimestampMillsAtZone() { + LocalDateTime dateTime = LocalDateTime.of(2024, 11, 11, 1, 11, 11); + assertThat(ArchiveDateTimeUtil.toTimestampMillsAtZone(dateTime, ZoneId.of("GMT+8"))) + .isEqualTo(1731258671000L); + } } From 8243d2152741643e00c9a224bbca3ead367674d7 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Thu, 21 Nov 2024 10:33:04 +0800 Subject: [PATCH 49/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../AbstractJobInstanceArchiveTask.java | 38 +++++++++++++------ .../JobInstanceArchiveTaskGenerator.java | 12 +++--- .../JobInstanceArchiveTaskScheduleLock.java | 9 ++++- .../JobInstanceArchiveTaskScheduler.java | 7 +++- .../archive/dao/impl/ArchiveTaskDAOImpl.java | 2 +- .../AbstractJobInstanceSubTableArchiver.java | 6 +-- .../archive/service/ArchiveTaskService.java | 10 +++++ 7 files changed, 57 insertions(+), 27 deletions(-) diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java index 8707227646..941a63dee5 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java @@ -128,9 +128,9 @@ private void stopTask() { if (stopCallback != null) { stopCallback.callback(); } - log.info("Stop archive task successfully, taskId: {}", taskId); + log.info("[{}] Stop archive task successfully", taskId); } else { - log.info("Archive task is stopped, taskId: {}", taskId); + log.info("[{}] Archive task is stopped", taskId); } } } @@ -154,7 +154,7 @@ private void archive() { backupAndDelete(); } catch (Throwable e) { String msg = MessageFormatter.format( - "Error while execute archive task : {}", + "[{}] Error while execute archive task", taskId ).getMessage(); log.error(msg, e); @@ -199,30 +199,30 @@ private void backupAndDelete() { if (checkStopFlag()) { stopTask(); } - if (progress != null) { - jobInstanceRecords = readSortedJobInstanceFromHotDB(progress.getTimestamp(), - archiveTaskInfo.getToTimestamp(), progress.getId(), readLimit); - } else { - jobInstanceRecords = readSortedJobInstanceFromHotDB(archiveTaskInfo.getFromTimestamp(), - archiveTaskInfo.getToTimestamp(), null, readLimit); - } + jobInstanceRecords = readJobInstanceRecords(readLimit); if (CollectionUtils.isEmpty(jobInstanceRecords)) { - updateArchiveProgress(ArchiveTaskStatusEnum.SUCCESS, null); + updateArchiveProgress(ArchiveTaskStatusEnum.SUCCESS, progress); return; } - archivedJobInstanceCount++; + archivedJobInstanceCount += jobInstanceRecords.size(); List jobInstanceIds = jobInstanceRecords.stream().map(this::extractJobInstanceId).collect(Collectors.toList()); // 写入数据到冷 db if (backupEnabled) { + long backupStartTime = System.currentTimeMillis(); backupJobInstanceToColdDb(jobInstanceRecords); + log.info("[{}] Backup to cold db, jobInstanceRecordSize: {}, cost: {}", + taskId, jobInstanceRecords.size(), System.currentTimeMillis() - backupStartTime); } // 从热 db 删除数据 if (deleteEnabled) { + long deleteStartTime = System.currentTimeMillis(); deleteJobInstanceHotData(jobInstanceIds); + log.info("[{}] Delete hot db, jobInstanceRecordSize: {}, cost: {}", + taskId, jobInstanceRecords.size(), System.currentTimeMillis() - deleteStartTime); } // 更新归档进度 @@ -241,6 +241,20 @@ private void backupAndDelete() { } } + private List readJobInstanceRecords(int readLimit) { + List jobInstanceRecords; + long readStartTime = System.currentTimeMillis(); + Long fromTime = progress == null ? archiveTaskInfo.getFromTimestamp() : progress.getTimestamp(); + Long fromTaskInstanceId = progress != null ? progress.getId() : null; + jobInstanceRecords = readSortedJobInstanceFromHotDB(fromTime, + archiveTaskInfo.getToTimestamp(), fromTaskInstanceId, readLimit); + log.info("[{}] Read sorted job instance from hot db, fromJobCreateTime: {}, toJobCreatTime: {}, " + + "fromJobInstanceId: {}, resultSize: {}, cost: {} ms", + taskId, progress.getTimestamp(), archiveTaskInfo.getToTimestamp(), progress.getId(), + jobInstanceRecords.size(), System.currentTimeMillis() - readStartTime); + return jobInstanceRecords; + } + /** * 备份作业实例数据到冷存储 * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java index 47f50b9237..2b30cbfb7c 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java @@ -33,7 +33,6 @@ import com.tencent.bk.job.backup.constant.ArchiveTaskStatusEnum; import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; import com.tencent.bk.job.backup.constant.DbDataNodeTypeEnum; -import com.tencent.bk.job.common.mysql.JobTransactional; import com.tencent.bk.job.common.mysql.dynamic.ds.DataSourceMode; import com.tencent.bk.job.common.util.json.JsonUtils; import lombok.extern.slf4j.Slf4j; @@ -66,7 +65,7 @@ public JobInstanceArchiveTaskGenerator(ArchiveTaskService archiveTaskService, this.archiveProperties = archiveProperties; } - @JobTransactional(transactionManager = "jobBackupTransactionManager") + public void generate() { if (taskInstanceRecordDAO.isTableEmpty()) { log.info("Job instance table is empty and does not require processing"); @@ -74,7 +73,7 @@ public void generate() { } List archiveTaskList = new ArrayList<>(); - log.info("Compute archive task generate startDateTime and endDateTime begin..."); + log.info("Compute archive task generate startDateTime and endDateTime"); // 归档任务创建范围-起始时间 LocalDateTime archiveStartDateTime = computeArchiveStartDateTime(); // 归档任务创建范围-结束时间 @@ -86,8 +85,7 @@ public void generate() { return; } - log.info("Generate job instance archive task, archiveStartDateTime: {}, archiveEndDateTime: {}", - archiveStartDateTime, archiveEndDateTime); + log.info("Generate job instance archive tasks between {} and {}", archiveStartDateTime, archiveEndDateTime); // 创建归档任务。每个基础归档任务定义为：一个数据节点（db+表）+ 日期 + 小时 while (archiveStartDateTime.isBefore(archiveEndDateTime)) { log.info("Generate archive task for datetime : {}", archiveStartDateTime); @@ -109,8 +107,8 @@ public void generate() { } if (CollectionUtils.isNotEmpty(archiveTaskList)) { - archiveTaskList.forEach(archiveTaskService::saveArchiveTask); - log.info("Add archive tasks : {}", JsonUtils.toJson(archiveTaskList)); + archiveTaskService.saveArchiveTasks(archiveTaskList); + log.info("Generate archive tasks : {}", JsonUtils.toJson(archiveTaskList)); } else { log.info("No new archive tasks are generated"); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduleLock.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduleLock.java index e1aa2d41e7..b31448f2dc 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduleLock.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduleLock.java @@ -71,8 +71,13 @@ public synchronized boolean lock() { public synchronized void unlock() { - LockUtils.releaseDistributedLock(LOCK_KEY, lockRequestId); - this.lockRequestId = null; + boolean success = LockUtils.releaseDistributedLock(LOCK_KEY, lockRequestId); + if (success) { + log.info("Release job instance archive schedule lock successfully"); + this.lockRequestId = null; + } else { + log.warn("Release job instance archive schedule lock fail"); + } } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java index 056046b4e9..25b68fc48c 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java @@ -32,6 +32,7 @@ import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; import com.tencent.bk.job.common.util.ThreadUtils; +import com.tencent.bk.job.common.util.json.JsonUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.context.SmartLifecycle; @@ -133,9 +134,10 @@ public void schedule() { archiveTaskService.countScheduleTasksGroupByDb(ArchiveTaskTypeEnum.JOB_INSTANCE); if (scheduleTasksGroupByDb.isEmpty()) { // 所有任务都已经被调度完成，退出本次任务调度 - log.info("All archive task is done! Exit schedule"); + log.info("No archive task need scheduling! Exit"); return; } + log.info("Count archive task group by db, result: {}", scheduleTasksGroupByDb); // 获取正在执行中的任务列表 List runningTasks = @@ -168,7 +170,8 @@ public void schedule() { } private void startArchiveTask(JobInstanceArchiveTaskInfo archiveTaskInfo) { - log.info("Start JobInstanceArchiveTask, taskId: {}", archiveTaskInfo.buildTaskUniqueId()); + log.info("Start JobInstanceArchiveTask, taskId: {}, taskInfo: {}", + archiveTaskInfo.buildTaskUniqueId(), JsonUtils.toJson(archiveTaskInfo)); JobInstanceMainDataArchiveTask archiveTask = new JobInstanceMainDataArchiveTask( taskInstanceRecordDAO, jobInstanceSubTableArchivers, diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java index 407c763fd3..8d5c88a79d 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java @@ -203,7 +203,7 @@ public JobInstanceArchiveTaskInfo getFirstScheduleArchiveTaskByDb(ArchiveTaskTyp JooqDataTypeUtil.toByte(ArchiveTaskStatusEnum.FAIL.getStatus()))) .and(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(taskType.getType()))) .and(T.DB_NODE.eq(dbNodeId)) - .orderBy(T.DAY.desc(), T.HOUR.desc(), T.DATA_NODE) + .orderBy(T.DAY.asc(), T.HOUR.asc(), T.DATA_NODE) .limit(1) .fetchOne(); diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/AbstractJobInstanceSubTableArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/AbstractJobInstanceSubTableArchiver.java index 0287ade551..bd56e64163 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/AbstractJobInstanceSubTableArchiver.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/AbstractJobInstanceSubTableArchiver.java @@ -78,9 +78,9 @@ public void backupRecords(List jobInstanceIds) { @Override public void deleteRecords(List jobInstanceIds) { long startTime = System.currentTimeMillis(); - jobInstanceHotRecordDAO.deleteRecords(jobInstanceIds, + int deleteRows = jobInstanceHotRecordDAO.deleteRecords(jobInstanceIds, archiveTablePropsStorage.getDeleteLimitRowCount(tableName)); - log.info("Delete {}, taskInstanceIdSize: {}, cost: {}ms", tableName, - jobInstanceIds.size(), System.currentTimeMillis() - startTime); + log.info("Delete {}, taskInstanceIdSize: {}, deletedRows: {}, cost: {}ms", tableName, + jobInstanceIds.size(), deleteRows, System.currentTimeMillis() - startTime); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/service/ArchiveTaskService.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/service/ArchiveTaskService.java index b8de1de8eb..3cf9fe1711 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/service/ArchiveTaskService.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/service/ArchiveTaskService.java @@ -27,8 +27,11 @@ import com.tencent.bk.job.backup.archive.dao.ArchiveTaskDAO; import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; +import com.tencent.bk.job.common.mysql.JobTransactional; +import org.apache.commons.collections4.CollectionUtils; import org.springframework.stereotype.Service; +import java.util.Collection; import java.util.List; import java.util.Map; @@ -54,6 +57,13 @@ public void saveArchiveTask(JobInstanceArchiveTaskInfo jobInstanceArchiveTaskInf archiveTaskDAO.saveArchiveTask(jobInstanceArchiveTaskInfo); } + @JobTransactional(transactionManager = "jobBackupTransactionManager") + public void saveArchiveTasks(Collection archiveTaskList) { + if (CollectionUtils.isNotEmpty(archiveTaskList)) { + archiveTaskList.forEach(archiveTaskDAO::saveArchiveTask); + } + } + public List listRunningTasks(ArchiveTaskTypeEnum taskType) { return archiveTaskDAO.listRunningTasks(taskType); } From 54fed2430192fc61dc0368b12aa8efc5cc2fff94 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Thu, 21 Nov 2024 11:30:06 +0800 Subject: [PATCH 50/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...ArchiveProgressDAOImplIntegrationTest.java | 51 ---------- .../AbstractJobInstanceArchiveTask.java | 14 ++- ...DAO.java => ArchiveTaskContextHolder.java} | 26 ++++-- .../job/backup/archive/ArchiveTaskWorker.java | 15 ++- .../JobInstanceArchiveTaskScheduler.java | 9 +- .../dao/impl/ArchiveProgressDAOImpl.java | 92 ------------------- .../AbstractJobInstanceSubTableArchiver.java | 8 +- ...ogressDTO.java => ArchiveTaskContext.java} | 21 ++--- .../backup/config/ArchiveConfiguration.java | 7 +- .../backup/model/dto/ArchiveProgressDTO.java | 45 --------- .../job/backup/model/dto/ArchiveSummary.java | 78 ---------------- .../service/ArchiveProgressService.java | 35 ------- .../impl/ArchiveProgressServiceImpl.java | 65 ------------- 13 files changed, 68 insertions(+), 398 deletions(-) delete mode 100644 src/backend/job-backup/boot-job-backup/src/test/java/com/tencent/bk/job/backup/dao/execute/ArchiveProgressDAOImplIntegrationTest.java rename src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/{dao/ArchiveProgressDAO.java => ArchiveTaskContextHolder.java} (67%) delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveProgressDAOImpl.java rename src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/{ArchiveProgressDTO.java => ArchiveTaskContext.java} (83%) delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ArchiveProgressDTO.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ArchiveSummary.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/ArchiveProgressService.java delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/ArchiveProgressServiceImpl.java diff --git a/src/backend/job-backup/boot-job-backup/src/test/java/com/tencent/bk/job/backup/dao/execute/ArchiveProgressDAOImplIntegrationTest.java b/src/backend/job-backup/boot-job-backup/src/test/java/com/tencent/bk/job/backup/dao/execute/ArchiveProgressDAOImplIntegrationTest.java deleted file mode 100644 index ad8e954731..0000000000 --- a/src/backend/job-backup/boot-job-backup/src/test/java/com/tencent/bk/job/backup/dao/execute/ArchiveProgressDAOImplIntegrationTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.dao.execute; - -import org.junit.jupiter.api.Test; - -//@ExtendWith(SpringExtension.class) -//@SpringBootTest -//@ActiveProfiles("test") -//@TestPropertySource(locations = "classpath:test.properties") -//@SqlConfig(encoding = "utf-8") -//@Sql({"/init_archive_progress_data.sql"}) -class ArchiveProgressDAOImplIntegrationTest { - -// @Autowired -// private ArchiveProgressDAO archiveProgressDAO; - - @Test - void queryArchiveProgress() { -// ArchiveProgressDTO archiveProgress = archiveProgressDAO.queryArchiveProgress("task_instance"); -// assertThat(archiveProgress.getTableName()).isEqualTo("task_instance"); -// assertThat(archiveProgress.getProgress().getLastArchivedId()).isEqualTo(1000L); -// assertThat(archiveProgress.getLastModifyTime()).isEqualTo(1621166442000L); - } - - @Test - void saveArchiveProgress() { - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java index 941a63dee5..04006999ff 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java @@ -25,6 +25,7 @@ package com.tencent.bk.job.backup.archive; import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.model.ArchiveTaskContext; import com.tencent.bk.job.backup.archive.model.ArchiveTaskSummary; import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; import com.tencent.bk.job.backup.archive.model.TimeAndIdBasedArchiveProcess; @@ -142,6 +143,10 @@ public void registerDoneCallback(ArchiveTaskDoneCallback archiveTaskDoneCallback private void archive() { try { + // 设置归档任务上下文 + ArchiveTaskContextHolder.set(new ArchiveTaskContext(archiveTaskInfo)); + + // 获取分布式锁 if (!acquireLock()) { archiveTaskSummary.setSkip(!isAcquireLock); return; @@ -179,6 +184,7 @@ private void archive() { if (checkStopFlag()) { stopTask(); } + ArchiveTaskContextHolder.unset(); } } @@ -250,7 +256,7 @@ private List readJobInstanceRecords(int readLimit) { archiveTaskInfo.getToTimestamp(), fromTaskInstanceId, readLimit); log.info("[{}] Read sorted job instance from hot db, fromJobCreateTime: {}, toJobCreatTime: {}, " + "fromJobInstanceId: {}, resultSize: {}, cost: {} ms", - taskId, progress.getTimestamp(), archiveTaskInfo.getToTimestamp(), progress.getId(), + taskId, fromTime, archiveTaskInfo.getToTimestamp(), fromTaskInstanceId, jobInstanceRecords.size(), System.currentTimeMillis() - readStartTime); return jobInstanceRecords; } @@ -284,7 +290,7 @@ protected boolean isDeleteEnable() { private boolean acquireLock() { this.isAcquireLock = archiveTaskLock.lock(taskId); if (!isAcquireLock) { - log.info("[{}] Acquire lock fail", taskId); + log.info("[{}] Acquire archive task lock fail", taskId); } return isAcquireLock; } @@ -292,6 +298,10 @@ private boolean acquireLock() { private void updateArchiveProgress(ArchiveTaskStatusEnum taskStatus, TimeAndIdBasedArchiveProcess progress) { archiveTaskInfo.setStatus(taskStatus); archiveTaskInfo.setProcess(progress); + if (taskStatus != ArchiveTaskStatusEnum.RUNNING) { + log.info("[{}] Update archive task process, taskStatus: {}, process: {}", + taskId, taskStatus, progress); + } archiveTaskService.updateTask(archiveTaskInfo); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ArchiveProgressDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskContextHolder.java similarity index 67% rename from src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ArchiveProgressDAO.java rename to src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskContextHolder.java index 995ce38dbe..7134d0c8e1 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ArchiveProgressDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskContextHolder.java @@ -22,14 +22,28 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.backup.archive.dao; +package com.tencent.bk.job.backup.archive; -import com.tencent.bk.job.backup.model.dto.ArchiveProgressDTO; +import com.tencent.bk.job.backup.archive.model.ArchiveTaskContext; -public interface ArchiveProgressDAO { - ArchiveProgressDTO queryArchiveProgress(String table); +public class ArchiveTaskContextHolder { - void saveArchiveProgress(ArchiveProgressDTO archiveProgress); + private static final ThreadLocal HOLDER = new ThreadLocal<>(); - void saveDeleteProgress(ArchiveProgressDTO archiveProgress); + public static void set(ArchiveTaskContext archiveTaskContext) { + HOLDER.set(archiveTaskContext); + } + + public static void unset() { + HOLDER.remove(); + } + + public static ArchiveTaskContext get() { + return HOLDER.get(); + } + + public static String getArchiveTaskId() { + ArchiveTaskContext context = HOLDER.get(); + return context != null ? context.getArchiveTaskInfo().buildTaskUniqueId() : null; + } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskWorker.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskWorker.java index 261ee743c9..309c2cb537 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskWorker.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskWorker.java @@ -25,6 +25,8 @@ package com.tencent.bk.job.backup.archive; import lombok.extern.slf4j.Slf4j; +import org.springframework.cloud.sleuth.Span; +import org.springframework.cloud.sleuth.Tracer; /** * 归档任务执行线程 @@ -33,18 +35,25 @@ public class ArchiveTaskWorker extends Thread { private final JobInstanceArchiveTask archiveTask; + private final Tracer tracer; - public ArchiveTaskWorker(JobInstanceArchiveTask archiveTask) { - this.setName("ArchiveWorker-" + archiveTask.getTaskId()); + public ArchiveTaskWorker(JobInstanceArchiveTask archiveTask, Tracer tracer) { + this.tracer = tracer; this.archiveTask = archiveTask; + this.setName("ArchiveWorker-" + archiveTask.getTaskId()); } @Override public void run() { - try { + Span span = tracer.nextSpan().name("archive-task"); + + try (Tracer.SpanInScope ignored = tracer.withSpan(span.start())) { archiveTask.execute(); } catch (Throwable e) { + span.error(e); log.warn("Thread interrupted!"); + } finally { + span.end(); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java index 25b68fc48c..09f93187ce 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java @@ -34,6 +34,7 @@ import com.tencent.bk.job.common.util.ThreadUtils; import com.tencent.bk.job.common.util.json.JsonUtils; import lombok.extern.slf4j.Slf4j; +import org.springframework.cloud.sleuth.Tracer; import org.springframework.context.SmartLifecycle; import java.util.List; @@ -65,6 +66,8 @@ public class JobInstanceArchiveTaskScheduler implements SmartLifecycle { private final ArchiveErrorTaskCounter archiveErrorTaskCounter; private final ArchiveTablePropsStorage archiveTablePropsStorage; + private final Tracer tracer; + private final Object lifecycleMonitor = new Object(); /** @@ -97,7 +100,8 @@ public JobInstanceArchiveTaskScheduler(ArchiveTaskService archiveTaskService, JobInstanceColdDAO jobInstanceColdDAO, ArchiveTaskLock archiveTaskLock, ArchiveErrorTaskCounter archiveErrorTaskCounter, - ArchiveTablePropsStorage archiveTablePropsStorage) { + ArchiveTablePropsStorage archiveTablePropsStorage, + Tracer tracer) { this.archiveTaskService = archiveTaskService; this.taskInstanceRecordDAO = taskInstanceRecordDAO; this.archiveProperties = archiveProperties; @@ -107,6 +111,7 @@ public JobInstanceArchiveTaskScheduler(ArchiveTaskService archiveTaskService, this.archiveTaskLock = archiveTaskLock; this.archiveErrorTaskCounter = archiveErrorTaskCounter; this.archiveTablePropsStorage = archiveTablePropsStorage; + this.tracer = tracer; } public void schedule() { @@ -196,7 +201,7 @@ private void startArchiveTask(JobInstanceArchiveTaskInfo archiveTaskInfo) { } scheduledTasks.put(archiveTask.getTaskId(), archiveTask); } - ArchiveTaskWorker worker = new ArchiveTaskWorker(archiveTask); + ArchiveTaskWorker worker = new ArchiveTaskWorker(archiveTask, tracer); worker.start(); log.info("Start JobInstanceArchiveTask success, taskId: {}", archiveTaskInfo.buildTaskUniqueId()); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveProgressDAOImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveProgressDAOImpl.java deleted file mode 100644 index ff8124fa8d..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveProgressDAOImpl.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.dao.impl; - -import com.tencent.bk.job.backup.archive.dao.ArchiveProgressDAO; -import com.tencent.bk.job.backup.model.dto.ArchiveProgressDTO; -import com.tencent.bk.job.backup.model.tables.ArchiveProgress; -import org.jooq.DSLContext; -import org.jooq.Record; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Repository; - -@Repository -public class ArchiveProgressDAOImpl implements ArchiveProgressDAO { - - private final DSLContext ctx; - private final ArchiveProgress T = ArchiveProgress.ARCHIVE_PROGRESS; - - @Autowired - public ArchiveProgressDAOImpl(@Qualifier("job-backup-dsl-context") DSLContext ctx) { - this.ctx = ctx; - } - - @Override - public ArchiveProgressDTO queryArchiveProgress(String table) { - Record record = ctx.select(T.TABLE_NAME, T.LAST_ARCHIVED_ID, T.LAST_ARCHIVE_TIME, - T.LAST_DELETED_ID, T.LAST_DELETE_TIME) - .from(T) - .where(T.TABLE_NAME.eq(table)) - .fetchOne(); - - return extract(record); - } - - private ArchiveProgressDTO extract(Record record) { - if (record == null) { - return null; - } - ArchiveProgressDTO archiveProgress = new ArchiveProgressDTO(); - archiveProgress.setTableName(record.get(T.TABLE_NAME)); - archiveProgress.setLastBackupId(record.get(T.LAST_ARCHIVED_ID)); - archiveProgress.setLastBackupTime(record.get(T.LAST_ARCHIVE_TIME)); - archiveProgress.setLastDeletedId(record.get(T.LAST_DELETED_ID)); - archiveProgress.setLastDeleteTime(record.get(T.LAST_DELETE_TIME)); - return archiveProgress; - } - - @Override - public void saveArchiveProgress(ArchiveProgressDTO archiveProgress) { - ctx.insertInto(T, T.TABLE_NAME, T.LAST_ARCHIVED_ID, T.LAST_ARCHIVE_TIME) - .values(archiveProgress.getTableName(), archiveProgress.getLastBackupId(), - archiveProgress.getLastBackupTime()) - .onDuplicateKeyUpdate() - .set(T.LAST_ARCHIVED_ID, archiveProgress.getLastBackupId()) - .set(T.LAST_ARCHIVE_TIME, archiveProgress.getLastBackupTime()) - .execute(); - } - - @Override - public void saveDeleteProgress(ArchiveProgressDTO archiveProgress) { - ctx.insertInto(T, T.TABLE_NAME, T.LAST_DELETED_ID, T.LAST_DELETE_TIME) - .values(archiveProgress.getTableName(), archiveProgress.getLastDeletedId(), - archiveProgress.getLastDeleteTime()) - .onDuplicateKeyUpdate() - .set(T.LAST_DELETED_ID, archiveProgress.getLastDeletedId()) - .set(T.LAST_DELETE_TIME, archiveProgress.getLastDeleteTime()) - .execute(); - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/AbstractJobInstanceSubTableArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/AbstractJobInstanceSubTableArchiver.java index bd56e64163..1a1863e4b9 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/AbstractJobInstanceSubTableArchiver.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/AbstractJobInstanceSubTableArchiver.java @@ -25,6 +25,7 @@ package com.tencent.bk.job.backup.archive.impl; import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.ArchiveTaskContextHolder; import com.tencent.bk.job.backup.archive.JobInstanceSubTableArchiver; import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; import com.tencent.bk.job.backup.archive.dao.impl.AbstractJobInstanceHotRecordDAO; @@ -66,8 +67,8 @@ public void backupRecords(List jobInstanceIds) { while (recordResultSet.next()) { List> records = recordResultSet.getRecords(); long readEndTime = System.currentTimeMillis(); - log.info("Read {}, recordSize: {}, cost: {}ms", tableName, - CollectionUtils.isEmpty(records) ? 0 : records.size(), readEndTime - startTime); + log.info("[{}] Read {}, recordSize: {}, cost: {}ms", ArchiveTaskContextHolder.getArchiveTaskId(), + tableName, CollectionUtils.isEmpty(records) ? 0 : records.size(), readEndTime - startTime); if (CollectionUtils.isNotEmpty(records)) { jobInstanceColdDAO.batchInsert(records, archiveTablePropsStorage.getBatchInsertRowSize(tableName)); @@ -80,7 +81,8 @@ public void deleteRecords(List jobInstanceIds) { long startTime = System.currentTimeMillis(); int deleteRows = jobInstanceHotRecordDAO.deleteRecords(jobInstanceIds, archiveTablePropsStorage.getDeleteLimitRowCount(tableName)); - log.info("Delete {}, taskInstanceIdSize: {}, deletedRows: {}, cost: {}ms", tableName, + log.info("[{}] Delete {}, taskInstanceIdSize: {}, deletedRows: {}, cost: {}ms", + ArchiveTaskContextHolder.getArchiveTaskId(), tableName, jobInstanceIds.size(), deleteRows, System.currentTimeMillis() - startTime); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveProgressDTO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTaskContext.java similarity index 83% rename from src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveProgressDTO.java rename to src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTaskContext.java index c05924bc36..b8cc245a52 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveProgressDTO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTaskContext.java @@ -25,21 +25,14 @@ package com.tencent.bk.job.backup.archive.model; import lombok.Data; -import lombok.ToString; @Data -@ToString -public class ArchiveProgressDTO { - private String tableName; - /** - * 最后备份ID - */ - private Long lastBackupId; +public class ArchiveTaskContext { - /** - * 最后删除ID - */ - private Long lastDeletedId; - private Long lastBackupTime; - private Long lastDeleteTime; + private JobInstanceArchiveTaskInfo archiveTaskInfo; + + + public ArchiveTaskContext(JobInstanceArchiveTaskInfo archiveTaskInfo) { + this.archiveTaskInfo = archiveTaskInfo; + } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java index 9c120737e7..3987430f13 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java @@ -74,6 +74,7 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.sleuth.Tracer; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; @@ -488,7 +489,8 @@ public JobInstanceArchiveTaskScheduler jobInstanceArchiveTaskScheduler( ObjectProvider jobInstanceColdDAOObjectProvider, ArchiveTaskLock archiveTaskLock, ArchiveErrorTaskCounter archiveErrorTaskCounter, - ArchiveTablePropsStorage archiveTablePropsStorage) { + ArchiveTablePropsStorage archiveTablePropsStorage, + Tracer tracer) { log.info("Init JobInstanceArchiveTaskScheduler"); return new JobInstanceArchiveTaskScheduler( @@ -500,7 +502,8 @@ public JobInstanceArchiveTaskScheduler jobInstanceArchiveTaskScheduler( jobInstanceColdDAOObjectProvider.getIfAvailable(), archiveTaskLock, archiveErrorTaskCounter, - archiveTablePropsStorage + archiveTablePropsStorage, + tracer ); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ArchiveProgressDTO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ArchiveProgressDTO.java deleted file mode 100644 index 11f0901081..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ArchiveProgressDTO.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.model.dto; - -import lombok.Data; -import lombok.ToString; - -@Data -@ToString -public class ArchiveProgressDTO { - private String tableName; - /** - * 最后备份ID - */ - private Long lastBackupId; - - /** - * 最后删除ID - */ - private Long lastDeletedId; - private Long lastBackupTime; - private Long lastDeleteTime; -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ArchiveSummary.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ArchiveSummary.java deleted file mode 100644 index 072b7fec94..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ArchiveSummary.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.model.dto; - -import com.fasterxml.jackson.annotation.JsonInclude; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@JsonInclude(JsonInclude.Include.NON_EMPTY) -@NoArgsConstructor -public class ArchiveSummary { - private boolean enabled; - private String tableName; - private String archiveEndDate; - private boolean skip; - private boolean success; - - private String archiveMode; - - /** - * 备份数据-查询原数据耗时（单位毫秒) - */ - private Long backupReadCost; - /** - * 备份数据-写入数据到归档 db 耗时（单位毫秒) - */ - private Long backupWriteCost; - /** - * 删除热数据耗时（单位毫秒) - */ - private Long deleteCost; - /** - * 归档总耗时（单位毫秒) - */ - private Long archiveCost; - - private Long archiveIdStart; - private Long archiveIdEnd; - private Long needArchiveRecordSize; - - private Long lastBackupId; - private Long backupRecordSize; - - private Long lastDeletedId; - private Long deleteRecordSize; - - /** - * 归档详细说明信息 - */ - private String message; - - public ArchiveSummary(String tableName) { - this.tableName = tableName; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/ArchiveProgressService.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/ArchiveProgressService.java deleted file mode 100644 index 522ad9c72c..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/ArchiveProgressService.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.service; - -import com.tencent.bk.job.backup.model.dto.ArchiveProgressDTO; - -public interface ArchiveProgressService { - ArchiveProgressDTO queryArchiveProgress(String table); - - void saveArchiveProgress(ArchiveProgressDTO archiveProgress); - - void saveDeleteProgress(ArchiveProgressDTO deleteProgress); -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/ArchiveProgressServiceImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/ArchiveProgressServiceImpl.java deleted file mode 100644 index e3f998beac..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/ArchiveProgressServiceImpl.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.service.impl; - -import com.tencent.bk.job.backup.archive.dao.ArchiveProgressDAO; -import com.tencent.bk.job.backup.model.dto.ArchiveProgressDTO; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -@Slf4j -@Service -public class ArchiveProgressServiceImpl implements ArchiveProgressService { - private final ArchiveProgressDAO archiveProgressDAO; - - @Autowired - public ArchiveProgressServiceImpl(ArchiveProgressDAO archiveProgressDAO) { - this.archiveProgressDAO = archiveProgressDAO; - } - - - @Override - public ArchiveProgressDTO queryArchiveProgress(String table) { - return archiveProgressDAO.queryArchiveProgress(table); - } - - @Override - public void saveArchiveProgress(ArchiveProgressDTO archiveProgress) { - if (archiveProgress.getLastBackupTime() == null) { - archiveProgress.setLastBackupTime(System.currentTimeMillis()); - } - archiveProgressDAO.saveArchiveProgress(archiveProgress); - } - - @Override - public void saveDeleteProgress(ArchiveProgressDTO deleteProgress) { - if (deleteProgress.getLastDeleteTime() == null) { - deleteProgress.setLastDeleteTime(System.currentTimeMillis()); - } - archiveProgressDAO.saveDeleteProgress(deleteProgress); - } -} From 8425a928a6cbd7787cab98009224d8df4c73fe5b Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Thu, 21 Nov 2024 12:23:51 +0800 Subject: [PATCH 51/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../AbstractJobInstanceArchiveTask.java | 13 +- ...kLock.java => ArchiveTaskExecuteLock.java} | 4 +- .../JobInstanceArchiveTaskScheduler.java | 112 +++++++++++------- .../JobInstanceMainDataArchiveTask.java | 4 +- .../backup/archive/dao/ArchiveTaskDAO.java | 7 ++ .../archive/dao/impl/ArchiveTaskDAOImpl.java | 12 ++ .../impl/TaskInstanceHostArchiver.java | 7 ++ .../archive/service/ArchiveTaskService.java | 9 ++ .../backup/config/ArchiveConfiguration.java | 10 +- 9 files changed, 120 insertions(+), 58 deletions(-) rename src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/{ArchiveTaskLock.java => ArchiveTaskExecuteLock.java} (98%) diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java index 04006999ff..7340ec931c 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java @@ -52,9 +52,10 @@ public abstract class AbstractJobInstanceArchiveTask> implements JobInstanceArchiveTask { protected JobInstanceColdDAO jobInstanceColdDAO; protected final ArchiveProperties archiveProperties; - private final ArchiveTaskLock archiveTaskLock; + private final ArchiveTaskExecuteLock archiveTaskExecuteLock; protected final ArchiveErrorTaskCounter archiveErrorTaskCounter; protected final ArchiveTaskService archiveTaskService; + protected final ArchiveTablePropsStorage archiveTablePropsStorage; protected String taskId; @@ -86,14 +87,14 @@ public abstract class AbstractJobInstanceArchiveTask> i public AbstractJobInstanceArchiveTask(JobInstanceColdDAO jobInstanceColdDAO, ArchiveProperties archiveProperties, - ArchiveTaskLock archiveTaskLock, + ArchiveTaskExecuteLock archiveTaskExecuteLock, ArchiveErrorTaskCounter archiveErrorTaskCounter, JobInstanceArchiveTaskInfo archiveTaskInfo, ArchiveTaskService archiveTaskService, ArchiveTablePropsStorage archiveTablePropsStorage) { this.jobInstanceColdDAO = jobInstanceColdDAO; this.archiveProperties = archiveProperties; - this.archiveTaskLock = archiveTaskLock; + this.archiveTaskExecuteLock = archiveTaskExecuteLock; this.archiveErrorTaskCounter = archiveErrorTaskCounter; this.archiveTaskInfo = archiveTaskInfo; this.archiveTaskService = archiveTaskService; @@ -126,6 +127,8 @@ private void stopTask() { synchronized (stopMonitor) { if (!isStopped) { isStopped = true; + // 更新归档任务状态为暂停，用于后续调度 + archiveTaskService.updateArchiveTaskSuspendedStatus(archiveTaskInfo); if (stopCallback != null) { stopCallback.callback(); } @@ -171,7 +174,7 @@ private void archive() { updateArchiveProgress(ArchiveTaskStatusEnum.FAIL, null); } finally { if (this.isAcquireLock) { - archiveTaskLock.unlock(taskId); + archiveTaskExecuteLock.unlock(taskId); } log.info( "[{}] Archive finished, result: {}", @@ -288,7 +291,7 @@ protected boolean isDeleteEnable() { } private boolean acquireLock() { - this.isAcquireLock = archiveTaskLock.lock(taskId); + this.isAcquireLock = archiveTaskExecuteLock.lock(taskId); if (!isAcquireLock) { log.info("[{}] Acquire archive task lock fail", taskId); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskLock.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskExecuteLock.java similarity index 98% rename from src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskLock.java rename to src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskExecuteLock.java index 9455c466a2..15134acb88 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskLock.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskExecuteLock.java @@ -39,7 +39,7 @@ * 归档任务执行分布式锁 */ @Slf4j -public class ArchiveTaskLock { +public class ArchiveTaskExecuteLock { private final String ARCHIVE_LOCK_KEY_PREFIX = "archive:task:execute"; /** * 归档任务锁时间 1h @@ -63,7 +63,7 @@ public class ArchiveTaskLock { private final StringRedisTemplate redisTemplate; - public ArchiveTaskLock(StringRedisTemplate redisTemplate) { + public ArchiveTaskExecuteLock(StringRedisTemplate redisTemplate) { this.redisTemplate = redisTemplate; } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java index 09f93187ce..43baf68119 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java @@ -36,6 +36,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.cloud.sleuth.Tracer; import org.springframework.context.SmartLifecycle; +import org.springframework.util.StopWatch; import java.util.List; import java.util.Map; @@ -62,7 +63,7 @@ public class JobInstanceArchiveTaskScheduler implements SmartLifecycle { private final JobInstanceSubTableArchivers jobInstanceSubTableArchivers; private final JobInstanceColdDAO jobInstanceColdDAO; - private final ArchiveTaskLock archiveTaskLock; + private final ArchiveTaskExecuteLock archiveTaskExecuteLock; private final ArchiveErrorTaskCounter archiveErrorTaskCounter; private final ArchiveTablePropsStorage archiveTablePropsStorage; @@ -98,7 +99,7 @@ public JobInstanceArchiveTaskScheduler(ArchiveTaskService archiveTaskService, JobInstanceArchiveTaskScheduleLock jobInstanceArchiveTaskScheduleLock, JobInstanceSubTableArchivers jobInstanceSubTableArchivers, JobInstanceColdDAO jobInstanceColdDAO, - ArchiveTaskLock archiveTaskLock, + ArchiveTaskExecuteLock archiveTaskExecuteLock, ArchiveErrorTaskCounter archiveErrorTaskCounter, ArchiveTablePropsStorage archiveTablePropsStorage, Tracer tracer) { @@ -108,7 +109,7 @@ public JobInstanceArchiveTaskScheduler(ArchiveTaskService archiveTaskService, this.jobInstanceArchiveTaskScheduleLock = jobInstanceArchiveTaskScheduleLock; this.jobInstanceSubTableArchivers = jobInstanceSubTableArchivers; this.jobInstanceColdDAO = jobInstanceColdDAO; - this.archiveTaskLock = archiveTaskLock; + this.archiveTaskExecuteLock = archiveTaskExecuteLock; this.archiveErrorTaskCounter = archiveErrorTaskCounter; this.archiveTablePropsStorage = archiveTablePropsStorage; this.tracer = tracer; @@ -120,56 +121,79 @@ public void schedule() { log.info("Scheduler is working"); return; } + this.scheduling = true; if (!isActive()) { log.info("JobInstanceArchiveTaskScheduler is not active, skip"); return; } + StopWatch watch = new StopWatch("archive-task-schedule"); while (true) { - // 获取归档任务调度锁 - boolean locked = jobInstanceArchiveTaskScheduleLock.lock(); - if (!locked) { - log.info("Get lock fail, wait 1s"); - ThreadUtils.sleep(1000L); - continue; + try { + // 获取归档任务调度锁 + boolean locked = jobInstanceArchiveTaskScheduleLock.lock(); + if (!locked) { + log.info("Get lock fail, wait 1s"); + ThreadUtils.sleep(1000L); + continue; + } + + + // 获取待调度的任务信息(按照 DB 节点计数) + watch.start("countScheduleTasks"); + Map scheduleTasksGroupByDb = + archiveTaskService.countScheduleTasksGroupByDb(ArchiveTaskTypeEnum.JOB_INSTANCE); + if (scheduleTasksGroupByDb.isEmpty()) { + // 所有任务都已经被调度完成，退出本次任务调度 + log.info("No archive task need scheduling! Exit"); + return; + } + watch.stop(); + log.info("Count archive task group by db, result: {}", scheduleTasksGroupByDb); + + // 获取正在执行中的任务列表 + watch.start("queryRunningTasks"); + List runningTasks = + archiveTaskService.listRunningTasks(ArchiveTaskTypeEnum.JOB_INSTANCE); + watch.stop(); + + // 对待调度的任务进行优先级排序，保证同一个 db 上的归档任务数量尽可能均衡，避免出现db 热点 + watch.start("evaluateTaskPriority"); + ArchiveDbNodePriorityEvaluator.DbNodeTasksInfo highestPriorityDbNodeTasksInfo = + ArchiveDbNodePriorityEvaluator.evaluateHighestPriorityDbNode(runningTasks, + scheduleTasksGroupByDb); + watch.stop(); + int taskConcurrent = archiveProperties.getTasks().getJobInstance().getConcurrent(); + if (highestPriorityDbNodeTasksInfo.getRunningTaskCount() >= taskConcurrent) { + // 休眠5分钟，等待并行任务减少 + log.info("Running archive task count exceed concurrent limit : {}, wait 300s", taskConcurrent); + ThreadUtils.sleep(1000 * 60L); + continue; + } + + // 获取优先级最高的归档任务 + watch.start("getHighestPriorityTaskFromDb"); + String dbNodeId = highestPriorityDbNodeTasksInfo.getDbNodeId(); + JobInstanceArchiveTaskInfo archiveTaskInfo = + archiveTaskService.getFirstScheduleArchiveTaskByDb(ArchiveTaskTypeEnum.JOB_INSTANCE, dbNodeId); + watch.stop(); + + // 启动任务 + watch.start("startArchiveTask"); + startArchiveTask(archiveTaskInfo); + watch.stop(); + } finally { + jobInstanceArchiveTaskScheduleLock.unlock(); + if (watch.isRunning()) { + watch.stop(); + } + if (watch.getTotalTimeMillis() > 1000) { + log.info("Schedule archive task slow, cost statistics: {}", watch.prettyPrint()); + } } - - // 获取待调度的任务信息(按照 DB 节点计数) - Map scheduleTasksGroupByDb = - archiveTaskService.countScheduleTasksGroupByDb(ArchiveTaskTypeEnum.JOB_INSTANCE); - if (scheduleTasksGroupByDb.isEmpty()) { - // 所有任务都已经被调度完成，退出本次任务调度 - log.info("No archive task need scheduling! Exit"); - return; - } - log.info("Count archive task group by db, result: {}", scheduleTasksGroupByDb); - - // 获取正在执行中的任务列表 - List runningTasks = - archiveTaskService.listRunningTasks(ArchiveTaskTypeEnum.JOB_INSTANCE); - - // 对待调度的任务进行优先级排序，保证同一个 db 上的归档任务数量尽可能均衡，避免出现db 热点 - ArchiveDbNodePriorityEvaluator.DbNodeTasksInfo highestPriorityDbNodeTasksInfo = - ArchiveDbNodePriorityEvaluator.evaluateHighestPriorityDbNode(runningTasks, scheduleTasksGroupByDb); - int taskConcurrent = archiveProperties.getTasks().getJobInstance().getConcurrent(); - if (highestPriorityDbNodeTasksInfo.getRunningTaskCount() >= taskConcurrent) { - // 休眠5分钟，等待并行任务减少 - log.info("Running archive task count exceed concurrent limit : {}, wait 300s", taskConcurrent); - ThreadUtils.sleep(1000 * 60L); - continue; - } - - // 获取优先级最高的归档任务 - String dbNodeId = highestPriorityDbNodeTasksInfo.getDbNodeId(); - JobInstanceArchiveTaskInfo archiveTaskInfo = - archiveTaskService.getFirstScheduleArchiveTaskByDb(ArchiveTaskTypeEnum.JOB_INSTANCE, dbNodeId); - - // 启动任务 - startArchiveTask(archiveTaskInfo); } } finally { this.scheduling = false; - jobInstanceArchiveTaskScheduleLock.unlock(); } } @@ -182,7 +206,7 @@ private void startArchiveTask(JobInstanceArchiveTaskInfo archiveTaskInfo) { jobInstanceSubTableArchivers, jobInstanceColdDAO, archiveProperties, - archiveTaskLock, + archiveTaskExecuteLock, archiveErrorTaskCounter, archiveTaskInfo, archiveTaskService, diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java index 7c0ff43f14..09aed3f954 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java @@ -51,7 +51,7 @@ public JobInstanceMainDataArchiveTask(TaskInstanceRecordDAO taskInstanceRecordDA JobInstanceSubTableArchivers jobInstanceSubTableArchivers, JobInstanceColdDAO jobInstanceColdDAO, ArchiveProperties archiveProperties, - ArchiveTaskLock archiveTaskLock, + ArchiveTaskExecuteLock archiveTaskExecuteLock, ArchiveErrorTaskCounter archiveErrorTaskCounter, JobInstanceArchiveTaskInfo archiveTask, ArchiveTaskService archiveTaskService, @@ -59,7 +59,7 @@ public JobInstanceMainDataArchiveTask(TaskInstanceRecordDAO taskInstanceRecordDA super( jobInstanceColdDAO, archiveProperties, - archiveTaskLock, + archiveTaskExecuteLock, archiveErrorTaskCounter, archiveTask, archiveTaskService, diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ArchiveTaskDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ArchiveTaskDAO.java index 3121162f1c..72bab538ce 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ArchiveTaskDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ArchiveTaskDAO.java @@ -62,4 +62,11 @@ public interface ArchiveTaskDAO { void updateTask(JobInstanceArchiveTaskInfo archiveTask); JobInstanceArchiveTaskInfo getFirstScheduleArchiveTaskByDb(ArchiveTaskTypeEnum taskType, String dbNodeId); + + /** + * 设置归档任务状态为暂停 + * + * @param archiveTask 归档任务 + */ + void updateArchiveTaskSuspendedStatus(JobInstanceArchiveTaskInfo archiveTask); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java index 8d5c88a79d..3b39bec8ae 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java @@ -209,4 +209,16 @@ public JobInstanceArchiveTaskInfo getFirstScheduleArchiveTaskByDb(ArchiveTaskTyp return extract(record); } + + @Override + public void updateArchiveTaskSuspendedStatus(JobInstanceArchiveTaskInfo archiveTask) { + ctx.update(T) + .set(T.STATUS, JooqDataTypeUtil.toByte(ArchiveTaskStatusEnum.SUSPENDED.getStatus())) + .where(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(archiveTask.getTaskType().getType()))) + .and(T.DATA_NODE.eq(archiveTask.getDbDataNode().toDataNodeId())) + .and(T.DAY.eq(archiveTask.getDay())) + .and(T.HOUR.eq(archiveTask.getHour().byteValue())) + .and(T.STATUS.eq(JooqDataTypeUtil.toByte(ArchiveTaskStatusEnum.RUNNING.getStatus()))) + .execute(); + } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceHostArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceHostArchiver.java index 15d02ac59b..79408214c0 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceHostArchiver.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceHostArchiver.java @@ -28,6 +28,8 @@ import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceHostRecordDAO; +import java.util.List; + public class TaskInstanceHostArchiver extends AbstractJobInstanceSubTableArchiver { @@ -41,4 +43,9 @@ public TaskInstanceHostArchiver( archiveTablePropsStorage ); } + + @Override + public void backupRecords(List jobInstanceIds) { + // task_instance_host 表的数据暂时无需备份到冷存储 + } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/service/ArchiveTaskService.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/service/ArchiveTaskService.java index 3cf9fe1711..2c20e7ddfa 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/service/ArchiveTaskService.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/service/ArchiveTaskService.java @@ -85,4 +85,13 @@ public void updateTask(JobInstanceArchiveTaskInfo archiveTask) { public JobInstanceArchiveTaskInfo getFirstScheduleArchiveTaskByDb(ArchiveTaskTypeEnum taskType, String dbNodeId) { return archiveTaskDAO.getFirstScheduleArchiveTaskByDb(taskType, dbNodeId); } + + /** + * 设置归档任务状态为暂停 + * + * @param archiveTask 归档任务 + */ + public void updateArchiveTaskSuspendedStatus(JobInstanceArchiveTaskInfo archiveTask) { + archiveTaskDAO.updateArchiveTaskSuspendedStatus(archiveTask); + } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java index 3987430f13..8949bf9b0e 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java @@ -25,7 +25,7 @@ package com.tencent.bk.job.backup.config; import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; -import com.tencent.bk.job.backup.archive.ArchiveTaskLock; +import com.tencent.bk.job.backup.archive.ArchiveTaskExecuteLock; import com.tencent.bk.job.backup.archive.JobInstanceArchiveCronJobs; import com.tencent.bk.job.backup.archive.JobInstanceArchiveTaskGenerator; import com.tencent.bk.job.backup.archive.JobInstanceArchiveTaskScheduleLock; @@ -462,9 +462,9 @@ public JobInstanceSubTableArchivers jobInstanceSubTableArchivers( @Bean - public ArchiveTaskLock archiveTaskLock(StringRedisTemplate redisTemplate) { + public ArchiveTaskExecuteLock archiveTaskLock(StringRedisTemplate redisTemplate) { log.info("Init ArchiveTaskLock"); - return new ArchiveTaskLock(redisTemplate); + return new ArchiveTaskExecuteLock(redisTemplate); } @Bean @@ -487,7 +487,7 @@ public JobInstanceArchiveTaskScheduler jobInstanceArchiveTaskScheduler( JobInstanceArchiveTaskScheduleLock jobInstanceArchiveTaskScheduleLock, JobInstanceSubTableArchivers jobInstanceSubTableArchivers, ObjectProvider jobInstanceColdDAOObjectProvider, - ArchiveTaskLock archiveTaskLock, + ArchiveTaskExecuteLock archiveTaskExecuteLock, ArchiveErrorTaskCounter archiveErrorTaskCounter, ArchiveTablePropsStorage archiveTablePropsStorage, Tracer tracer) { @@ -500,7 +500,7 @@ public JobInstanceArchiveTaskScheduler jobInstanceArchiveTaskScheduler( jobInstanceArchiveTaskScheduleLock, jobInstanceSubTableArchivers, jobInstanceColdDAOObjectProvider.getIfAvailable(), - archiveTaskLock, + archiveTaskExecuteLock, archiveErrorTaskCounter, archiveTablePropsStorage, tracer From 70a7d205c6a47f1f4b34562a14542977e4a3261d Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Thu, 21 Nov 2024 14:46:19 +0800 Subject: [PATCH 52/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/tencent/bk/job/backup/archive/TaskCountDownLatch.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/TaskCountDownLatch.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/TaskCountDownLatch.java index cd17028ff2..336ec65c81 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/TaskCountDownLatch.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/TaskCountDownLatch.java @@ -75,8 +75,8 @@ public void waitingForAllTasksDone() { private synchronized void startMonitor() { if (!monitorInitial) { - log.info("Start StopTaskCounter monitor ..."); - Thread monitorThread = new Thread("StopTaskCounterMonitor") { + log.info("Start TaskCountDownLatch monitor ..."); + Thread monitorThread = new Thread("TaskCountDownLatchMonitor") { @Override public void run() { while (!isAllTaskDone) { From c190aa49f4fb2cd01e93de8229cd223cdff012f4 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Thu, 21 Nov 2024 18:15:44 +0800 Subject: [PATCH 53/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../redis/util/HeartBeatRedisLockConfig.java | 7 + .../AbstractJobInstanceArchiveTask.java | 1 + .../archive/ArchiveTaskExecuteLock.java | 139 ------------------ .../JobInstanceArchiveTaskGenerator.java | 105 +++++++------ .../JobInstanceArchiveTaskScheduler.java | 2 + .../JobInstanceMainDataArchiveTask.java | 1 + .../util/lock/ArchiveTaskExecuteLock.java | 58 ++++++++ .../lock/FairDistributeLock.java} | 38 +++-- .../util/lock/HeartBeatRedisLocks.java | 101 +++++++++++++ .../JobInstanceArchiveTaskGenerateLock.java | 69 +++++++++ .../JobInstanceArchiveTaskScheduleLock.java | 42 ++++++ .../backup/config/ArchiveConfiguration.java | 24 ++- 12 files changed, 385 insertions(+), 202 deletions(-) delete mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskExecuteLock.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/ArchiveTaskExecuteLock.java rename src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/{JobInstanceArchiveTaskScheduleLock.java => util/lock/FairDistributeLock.java} (72%) create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/HeartBeatRedisLocks.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/JobInstanceArchiveTaskGenerateLock.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/JobInstanceArchiveTaskScheduleLock.java diff --git a/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/HeartBeatRedisLockConfig.java b/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/HeartBeatRedisLockConfig.java index 2a07b1b209..d6648b4480 100644 --- a/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/HeartBeatRedisLockConfig.java +++ b/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/HeartBeatRedisLockConfig.java @@ -26,6 +26,7 @@ import com.tencent.bk.job.common.util.JobUUID; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.Setter; /** @@ -33,6 +34,7 @@ */ @Setter @Getter +@NoArgsConstructor public class HeartBeatRedisLockConfig { /** @@ -54,4 +56,9 @@ public static HeartBeatRedisLockConfig getDefault() { return INSTANCE; } + public HeartBeatRedisLockConfig(String heartBeatThreadName, long expireTimeMillis, long periodMillis) { + this.heartBeatThreadName = heartBeatThreadName; + this.expireTimeMillis = expireTimeMillis; + this.periodMillis = periodMillis; + } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java index 7340ec931c..112111a321 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java @@ -30,6 +30,7 @@ import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; import com.tencent.bk.job.backup.archive.model.TimeAndIdBasedArchiveProcess; import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; +import com.tencent.bk.job.backup.archive.util.lock.ArchiveTaskExecuteLock; import com.tencent.bk.job.backup.config.ArchiveProperties; import com.tencent.bk.job.backup.constant.ArchiveModeEnum; import com.tencent.bk.job.backup.constant.ArchiveTaskStatusEnum; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskExecuteLock.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskExecuteLock.java deleted file mode 100644 index 15134acb88..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskExecuteLock.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive; - -import com.tencent.bk.job.common.redis.util.LockUtils; -import com.tencent.bk.job.common.redis.util.RedisKeyHeartBeatThread; -import com.tencent.bk.job.common.util.ThreadUtils; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.data.redis.core.StringRedisTemplate; - -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; - -/** - * 归档任务执行分布式锁 - */ -@Slf4j -public class ArchiveTaskExecuteLock { - private final String ARCHIVE_LOCK_KEY_PREFIX = "archive:task:execute"; - /** - * 归档任务锁时间 1h - */ - private final Long LOCK_TIME = 3600 * 1000L; - /** - * 最小获取锁间隔时间；为了保证在分布式系统中多个节点都能均匀获取到任务，会优先让空闲的节点获取到任务 - */ - private final long MIN_ACQUIRE_LOCK_INTERVAL_MS = 1000L; - - private volatile long lastAcquireLockTimeMS = 0L; - /** - * Key: taskId ; Value: 分布式锁请求 requestId - */ - private final Map locks = new ConcurrentHashMap<>(); - /** - * 分布式锁保持，避免超时失效的心跳线程 - * Key: taskId ; Value: 心跳线程 - */ - private final Map lockKeepThreads = new ConcurrentHashMap<>(); - - private final StringRedisTemplate redisTemplate; - - public ArchiveTaskExecuteLock(StringRedisTemplate redisTemplate) { - this.redisTemplate = redisTemplate; - } - - public synchronized boolean lock(String taskId) { - while (System.currentTimeMillis() - lastAcquireLockTimeMS < MIN_ACQUIRE_LOCK_INTERVAL_MS) { - ThreadUtils.sleep(10L); - } - String lockRequestId = UUID.randomUUID().toString(); - String archiveLockKey = ARCHIVE_LOCK_KEY_PREFIX + "_" + taskId; - if (!LockUtils.tryGetDistributedLock(archiveLockKey, lockRequestId, LOCK_TIME)) { - log.info("Acquire archive task lock failed! taskId: {}", taskId); - return false; - } else { - log.info("Acquire archive task lock successfully! taskId: {}", taskId); - this.lastAcquireLockTimeMS = System.currentTimeMillis(); - this.locks.put(taskId, lockRequestId); - startRedisKeyHeartBeatThread(taskId, archiveLockKey, lockRequestId); - return true; - } - } - - private void startRedisKeyHeartBeatThread(String taskId, - String archiveLockKey, - String lockRequestId) { - // 开一个心跳子线程，维持锁状态不会因为超时失效 - String realArchiveLockKey = LockUtils.LOCK_KEY_PREFIX + archiveLockKey; - RedisKeyHeartBeatThread redisKeyHeartBeatThread = new RedisKeyHeartBeatThread( - redisTemplate, - realArchiveLockKey, - lockRequestId, - LOCK_TIME, - 30 * 60 * 1000L - ); - redisKeyHeartBeatThread.setName("[ArchiveTask-" + taskId + "]-redisKeyHeartBeatThread"); - lockKeepThreads.put(taskId, redisKeyHeartBeatThread); - - log.info("Start redis key heart beat thread for ArchiveTask:{}", taskId); - redisKeyHeartBeatThread.start(); - } - - private void stopRedisKeyHeartBeatThread(String taskId) { - RedisKeyHeartBeatThread heartBeatThread = lockKeepThreads.get(taskId); - if (heartBeatThread == null) { - log.error("RedisKeyHeartBeatThread for table {} not exist", taskId); - return; - } - log.info("Stop redis key heart beat thread for ArchiveTask:{}", taskId); - heartBeatThread.stopAtOnce(); - lockKeepThreads.remove(taskId); - } - - public synchronized void unlock(String taskId, String lockRequestId) { - // 先停止分布式锁维持线程 - stopRedisKeyHeartBeatThread(taskId); - - String archiveLockKey = ARCHIVE_LOCK_KEY_PREFIX + "_" + taskId; - LockUtils.releaseDistributedLock(archiveLockKey, lockRequestId); - this.locks.remove(taskId); - } - - public synchronized void unlock(String taskId) { - String lockRequestId = this.locks.get(taskId); - if (StringUtils.isNotEmpty(lockRequestId)) { - unlock(taskId, lockRequestId); - } - } - - public void unlockAll() { - this.locks.forEach(this::unlock); - this.locks.clear(); - } - -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java index 2b30cbfb7c..ec1ce3133d 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java @@ -29,11 +29,13 @@ import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; import com.tencent.bk.job.backup.archive.util.ArchiveDateTimeUtil; +import com.tencent.bk.job.backup.archive.util.lock.JobInstanceArchiveTaskGenerateLock; import com.tencent.bk.job.backup.config.ArchiveProperties; import com.tencent.bk.job.backup.constant.ArchiveTaskStatusEnum; import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; import com.tencent.bk.job.backup.constant.DbDataNodeTypeEnum; import com.tencent.bk.job.common.mysql.dynamic.ds.DataSourceMode; +import com.tencent.bk.job.common.redis.util.LockResult; import com.tencent.bk.job.common.util.json.JsonUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -55,63 +57,84 @@ public class JobInstanceArchiveTaskGenerator { private final ArchiveProperties archiveProperties; + private final JobInstanceArchiveTaskGenerateLock archiveTaskGenerateLock; + public JobInstanceArchiveTaskGenerator(ArchiveTaskService archiveTaskService, TaskInstanceRecordDAO taskInstanceRecordDAO, - ArchiveProperties archiveProperties) { + ArchiveProperties archiveProperties, + JobInstanceArchiveTaskGenerateLock archiveTaskGenerateLock) { this.archiveTaskService = archiveTaskService; this.taskInstanceRecordDAO = taskInstanceRecordDAO; this.archiveProperties = archiveProperties; + this.archiveTaskGenerateLock = archiveTaskGenerateLock; } public void generate() { - if (taskInstanceRecordDAO.isTableEmpty()) { - log.info("Job instance table is empty and does not require processing"); - return; - } - List archiveTaskList = new ArrayList<>(); - - log.info("Compute archive task generate startDateTime and endDateTime"); - // 归档任务创建范围-起始时间 - LocalDateTime archiveStartDateTime = computeArchiveStartDateTime(); - // 归档任务创建范围-结束时间 - LocalDateTime archiveEndDateTime = computeArchiveEndTime(archiveProperties.getKeepDays()); - if (archiveEndDateTime.isBefore(archiveStartDateTime) || archiveEndDateTime.equals(archiveStartDateTime)) { - log.info("Archive endTime is before startTime, does not require generating archive task." + - " startTime: {}, endTime: {}", - archiveStartDateTime, archiveEndDateTime); - return; - } + LockResult lockResult = null; + try { + lockResult = archiveTaskGenerateLock.lock(); + if (!lockResult.isLockGotten()) { + log.info( + "Archive task generate lock gotten by another process: {}, return", + lockResult.getLockValue() + ); + return; + } - log.info("Generate job instance archive tasks between {} and {}", archiveStartDateTime, archiveEndDateTime); - // 创建归档任务。每个基础归档任务定义为：一个数据节点（db+表）+ 日期 + 小时 - while (archiveStartDateTime.isBefore(archiveEndDateTime)) { - log.info("Generate archive task for datetime : {}", archiveStartDateTime); - // 水平分库分表 - if (isHorizontalShardingEnabled()) { - // 作业实例数据归档任务,现版本暂不支持 - archiveTaskList.addAll(buildArchiveTasksForShardingDataNodes(ArchiveTaskTypeEnum.JOB_INSTANCE, - archiveStartDateTime, archiveProperties.getTasks().getJobInstance().getShardingDataNodes())); - } else { - // 单db - DbDataNode dbDataNode = DbDataNode.standaloneDbDatNode(); - JobInstanceArchiveTaskInfo archiveTaskInfo = - buildArchiveTask(ArchiveTaskTypeEnum.JOB_INSTANCE, archiveStartDateTime, dbDataNode); - archiveTaskList.add(archiveTaskInfo); - log.info("Add JobInstanceArchiveTaskInfo: {}", JsonUtils.toJson(archiveTaskInfo)); + if (taskInstanceRecordDAO.isTableEmpty()) { + log.info("Job instance table is empty and does not require processing"); + return; + } + List archiveTaskList = new ArrayList<>(); + + log.info("Compute archive task generate startDateTime and endDateTime"); + // 归档任务创建范围-起始时间 + LocalDateTime archiveStartDateTime = computeArchiveStartDateTime(); + // 归档任务创建范围-结束时间 + LocalDateTime archiveEndDateTime = computeArchiveEndTime(archiveProperties.getKeepDays()); + if (archiveEndDateTime.isBefore(archiveStartDateTime) || archiveEndDateTime.equals(archiveStartDateTime)) { + log.info("Archive endTime is before startTime, does not require generating archive task." + + " startTime: {}, endTime: {}", + archiveStartDateTime, archiveEndDateTime); + return; } - archiveStartDateTime = archiveStartDateTime.plusHours(1L); - } + log.info("Generate job instance archive tasks between {} and {}", archiveStartDateTime, archiveEndDateTime); + // 创建归档任务。每个基础归档任务定义为：一个数据节点（db+表）+ 日期 + 小时 + while (archiveStartDateTime.isBefore(archiveEndDateTime)) { + log.info("Generate archive task for datetime : {}", archiveStartDateTime); + // 水平分库分表 + if (isHorizontalShardingEnabled()) { + // 作业实例数据归档任务,现版本暂不支持 + archiveTaskList.addAll(buildArchiveTasksForShardingDataNodes(ArchiveTaskTypeEnum.JOB_INSTANCE, + archiveStartDateTime, archiveProperties.getTasks().getJobInstance().getShardingDataNodes())); + } else { + // 单db + DbDataNode dbDataNode = DbDataNode.standaloneDbDatNode(); + JobInstanceArchiveTaskInfo archiveTaskInfo = + buildArchiveTask(ArchiveTaskTypeEnum.JOB_INSTANCE, archiveStartDateTime, dbDataNode); + archiveTaskList.add(archiveTaskInfo); + log.info("Add JobInstanceArchiveTaskInfo: {}", JsonUtils.toJson(archiveTaskInfo)); + } - if (CollectionUtils.isNotEmpty(archiveTaskList)) { - archiveTaskService.saveArchiveTasks(archiveTaskList); - log.info("Generate archive tasks : {}", JsonUtils.toJson(archiveTaskList)); - } else { - log.info("No new archive tasks are generated"); + archiveStartDateTime = archiveStartDateTime.plusHours(1L); + } + + if (CollectionUtils.isNotEmpty(archiveTaskList)) { + archiveTaskService.saveArchiveTasks(archiveTaskList); + log.info("Generate archive tasks : {}", JsonUtils.toJson(archiveTaskList)); + } else { + log.info("No new archive tasks are generated"); + } + } finally { + if (lockResult != null) { + lockResult.tryToRelease(); + } } + } private List buildArchiveTasksForShardingDataNodes( diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java index 43baf68119..3cbca8eb79 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java @@ -28,6 +28,8 @@ import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceRecordDAO; import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; +import com.tencent.bk.job.backup.archive.util.lock.ArchiveTaskExecuteLock; +import com.tencent.bk.job.backup.archive.util.lock.JobInstanceArchiveTaskScheduleLock; import com.tencent.bk.job.backup.config.ArchiveProperties; import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java index 09aed3f954..abd1968196 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java @@ -28,6 +28,7 @@ import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceRecordDAO; import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; +import com.tencent.bk.job.backup.archive.util.lock.ArchiveTaskExecuteLock; import com.tencent.bk.job.backup.config.ArchiveProperties; import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; import com.tencent.bk.job.execute.model.tables.TaskInstance; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/ArchiveTaskExecuteLock.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/ArchiveTaskExecuteLock.java new file mode 100644 index 0000000000..9ac272d783 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/ArchiveTaskExecuteLock.java @@ -0,0 +1,58 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.util.lock; + +import com.tencent.bk.job.common.redis.util.HeartBeatRedisLockConfig; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.StringRedisTemplate; + +/** + * 归档任务执行分布式锁 + */ +@Slf4j +public class ArchiveTaskExecuteLock { + + private final HeartBeatRedisLocks locks; + + public ArchiveTaskExecuteLock(StringRedisTemplate redisTemplate) { + locks = new HeartBeatRedisLocks( + "archive:task:execute", + redisTemplate, + new HeartBeatRedisLockConfig( + "RedisKeyHeartBeatThread-archive:task:execute", + 3600 * 1000L, // 1h 超时时间 + 600 * 1000L // 10min 续期一次 + ) + ); + } + + public boolean lock(String taskId) { + return locks.lock(taskId); + } + + public void unlock(String taskId) { + locks.unlock(taskId); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduleLock.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/FairDistributeLock.java similarity index 72% rename from src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduleLock.java rename to src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/FairDistributeLock.java index b31448f2dc..26cc202259 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduleLock.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/FairDistributeLock.java @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.backup.archive; +package com.tencent.bk.job.backup.archive.util.lock; import com.tencent.bk.job.common.redis.util.LockUtils; import com.tencent.bk.job.common.util.ThreadUtils; @@ -31,15 +31,20 @@ import java.util.UUID; /** - * 归档任务调度分布式锁 + * 分布式锁（公平锁，非饥饿抢占方式) */ @Slf4j -public class JobInstanceArchiveTaskScheduleLock { - private final String LOCK_KEY = "job:instance:archive:task:schedule"; +public class FairDistributeLock { /** - * 锁时间（60s) + * 锁名称 */ - private final Long LOCK_TIME = 60 * 1000L; + private final String lockName; + + private final String lockKey; + /** + * 锁时间（毫秒) + */ + private final Long lockMills; /** * 最小获取锁间隔时间 */ @@ -49,7 +54,10 @@ public class JobInstanceArchiveTaskScheduleLock { private volatile String lockRequestId = null; - public JobInstanceArchiveTaskScheduleLock() { + public FairDistributeLock(String lockName, String lockKey, long lockMills) { + this.lockName = lockName; + this.lockKey = lockKey; + this.lockMills = lockMills; } public synchronized boolean lock() { @@ -58,11 +66,11 @@ public synchronized boolean lock() { ThreadUtils.sleep(100L); } String lockRequestId = UUID.randomUUID().toString(); - if (!LockUtils.tryGetDistributedLock(LOCK_KEY, lockRequestId, LOCK_TIME)) { - log.info("Acquire job instance archive task schedule lock failed!"); + if (!LockUtils.tryGetDistributedLock(lockKey, lockRequestId, lockMills)) { + log.info("[{}] Acquire lock failed!", lockName); return false; } else { - log.info("Acquire job instance archive task schedule lock successfully!"); + log.info("[{}] Acquire lock successfully!", lockName); this.lastAcquireLockTimeMS = System.currentTimeMillis(); this.lockRequestId = lockRequestId; return true; @@ -70,14 +78,14 @@ public synchronized boolean lock() { } - public synchronized void unlock() { - boolean success = LockUtils.releaseDistributedLock(LOCK_KEY, lockRequestId); + public synchronized boolean unlock() { + boolean success = LockUtils.releaseDistributedLock(lockKey, lockRequestId); if (success) { - log.info("Release job instance archive schedule lock successfully"); + log.info("[{}] Release lock successfully", lockName); this.lockRequestId = null; } else { - log.warn("Release job instance archive schedule lock fail"); + log.warn("[{}] Release job instance archive schedule lock fail", lockName); } + return success; } - } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/HeartBeatRedisLocks.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/HeartBeatRedisLocks.java new file mode 100644 index 0000000000..13f1d43493 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/HeartBeatRedisLocks.java @@ -0,0 +1,101 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.util.lock; + +import com.tencent.bk.job.common.redis.util.HeartBeatRedisLock; +import com.tencent.bk.job.common.redis.util.HeartBeatRedisLockConfig; +import com.tencent.bk.job.common.redis.util.LockResult; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.StringRedisTemplate; + +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 分布式锁(自动续期) + */ +@Slf4j +public class HeartBeatRedisLocks { + /** + * 锁 key 前缀 + */ + private final String lockKeyPrefix; + /** + * Key: lockKey ; Value: 分布式锁 + */ + private final Map locks = new ConcurrentHashMap<>(); + + + private final StringRedisTemplate redisTemplate; + + private final HeartBeatRedisLockConfig heartBeatRedisLockConfig; + + public HeartBeatRedisLocks(String lockKeyPrefix, + StringRedisTemplate redisTemplate, + HeartBeatRedisLockConfig heartBeatRedisLockConfig) { + this.lockKeyPrefix = lockKeyPrefix; + this.redisTemplate = redisTemplate; + if (heartBeatRedisLockConfig == null) { + this.heartBeatRedisLockConfig = HeartBeatRedisLockConfig.getDefault(); + } else { + this.heartBeatRedisLockConfig = heartBeatRedisLockConfig; + } + } + + public synchronized boolean lock(String lockKey) { + String lockRequestId = UUID.randomUUID().toString(); + String actualLockKey = buildActualLockKey(lockKey); + HeartBeatRedisLock redisLock = + new HeartBeatRedisLock(redisTemplate, actualLockKey, lockRequestId, heartBeatRedisLockConfig); + + LockResult lockResult = redisLock.lock(); + if (!lockResult.isLockGotten()) { + return false; + } + + this.locks.put(lockKey, lockResult); + return true; + } + + private String buildActualLockKey(String lockKey) { + return lockKeyPrefix + ":" + lockKey; + } + + + public synchronized void unlock(String lockKey) { + LockResult lockResult = this.locks.get(lockKey); + if (lockResult == null) { + log.warn("RedisLock is not found, lockKey: {}", lockKey); + return; + } + try { + lockResult.tryToRelease(); + } finally { + locks.remove(lockKey); + } + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/JobInstanceArchiveTaskGenerateLock.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/JobInstanceArchiveTaskGenerateLock.java new file mode 100644 index 0000000000..8b396eb41f --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/JobInstanceArchiveTaskGenerateLock.java @@ -0,0 +1,69 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.util.lock; + +import com.tencent.bk.job.common.redis.util.HeartBeatRedisLock; +import com.tencent.bk.job.common.redis.util.HeartBeatRedisLockConfig; +import com.tencent.bk.job.common.redis.util.LockResult; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.StringRedisTemplate; + +import java.util.UUID; + +/** + * 归档任务创建分布式锁 + */ +@Slf4j +public class JobInstanceArchiveTaskGenerateLock { + + private final StringRedisTemplate redisTemplate; + + private static final String REDIS_LOCK_KEY = "job:instance:archive:task:generate"; + + private final HeartBeatRedisLockConfig heartBeatRedisLockConfig; + + private volatile HeartBeatRedisLock redisLock = null; + + private volatile String lockRequestId = null; + + public JobInstanceArchiveTaskGenerateLock(StringRedisTemplate redisTemplate) { + this.redisTemplate = redisTemplate; + this.heartBeatRedisLockConfig = new HeartBeatRedisLockConfig( + "RedisKeyHeartBeatThread-job:instance:archive:task:generate", + 60 * 1000L, // 60s 超时时间 + 10 * 1000L // 10s 续期一次 + ); + } + + public LockResult lock() { + if (lockRequestId != null) { + log.warn("Lock is held by another process"); + return LockResult.fail(lockRequestId); + } + this.lockRequestId = UUID.randomUUID().toString(); + redisLock = new HeartBeatRedisLock(redisTemplate, REDIS_LOCK_KEY, lockRequestId, heartBeatRedisLockConfig); + return redisLock.lock(); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/JobInstanceArchiveTaskScheduleLock.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/JobInstanceArchiveTaskScheduleLock.java new file mode 100644 index 0000000000..cb9d2a04b0 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/JobInstanceArchiveTaskScheduleLock.java @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.util.lock; + +import lombok.extern.slf4j.Slf4j; + +/** + * 归档任务调度分布式锁 + */ +@Slf4j +public class JobInstanceArchiveTaskScheduleLock extends FairDistributeLock { + + public JobInstanceArchiveTaskScheduleLock() { + super( + "job:instance:archive:task:schedule:lock", + "job:instance:archive:task:schedule", + 60 * 1000L + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java index 8949bf9b0e..958d66a484 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java @@ -25,10 +25,8 @@ package com.tencent.bk.job.backup.config; import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; -import com.tencent.bk.job.backup.archive.ArchiveTaskExecuteLock; import com.tencent.bk.job.backup.archive.JobInstanceArchiveCronJobs; import com.tencent.bk.job.backup.archive.JobInstanceArchiveTaskGenerator; -import com.tencent.bk.job.backup.archive.JobInstanceArchiveTaskScheduleLock; import com.tencent.bk.job.backup.archive.JobInstanceArchiveTaskScheduler; import com.tencent.bk.job.backup.archive.JobInstanceSubTableArchivers; import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; @@ -67,6 +65,9 @@ import com.tencent.bk.job.backup.archive.impl.TaskInstanceHostArchiver; import com.tencent.bk.job.backup.archive.impl.TaskInstanceVariableArchiver; import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; +import com.tencent.bk.job.backup.archive.util.lock.ArchiveTaskExecuteLock; +import com.tencent.bk.job.backup.archive.util.lock.JobInstanceArchiveTaskGenerateLock; +import com.tencent.bk.job.backup.archive.util.lock.JobInstanceArchiveTaskScheduleLock; import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; import lombok.extern.slf4j.Slf4j; @@ -463,19 +464,28 @@ public JobInstanceSubTableArchivers jobInstanceSubTableArchivers( @Bean public ArchiveTaskExecuteLock archiveTaskLock(StringRedisTemplate redisTemplate) { - log.info("Init ArchiveTaskLock"); + log.info("Init ArchiveTaskExecuteLock"); return new ArchiveTaskExecuteLock(redisTemplate); } @Bean - public JobInstanceArchiveTaskGenerator jobInstanceArchiveTaskGenerator(ArchiveTaskService archiveTaskService, - TaskInstanceRecordDAO taskInstanceRecordDAO, - ArchiveProperties archiveProperties) { + public JobInstanceArchiveTaskGenerateLock jobInstanceArchiveTaskGenerateLock(StringRedisTemplate redisTemplate) { + return new JobInstanceArchiveTaskGenerateLock(redisTemplate); + } + + @Bean + public JobInstanceArchiveTaskGenerator jobInstanceArchiveTaskGenerator( + ArchiveTaskService archiveTaskService, + TaskInstanceRecordDAO taskInstanceRecordDAO, + ArchiveProperties archiveProperties, + JobInstanceArchiveTaskGenerateLock jobInstanceArchiveTaskGenerateLock) { + log.info("Init JobInstanceArchiveTaskGenerator"); return new JobInstanceArchiveTaskGenerator( archiveTaskService, taskInstanceRecordDAO, - archiveProperties + archiveProperties, + jobInstanceArchiveTaskGenerateLock ); } From a34e41fa7c4a779c1f4456818f92bcd4d8506cb1 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Fri, 22 Nov 2024 09:10:43 +0800 Subject: [PATCH 54/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../JobInstanceArchiveTaskGenerator.java | 15 +++---- .../JobInstanceArchiveTaskScheduler.java | 7 ++- .../backup/archive/util/ServiceNodeInfo.java | 31 +++++++++++++ .../archive/util/lock/FairDistributeLock.java | 4 +- .../util/lock/HeartBeatRedisLocks.java | 4 +- .../JobInstanceArchiveTaskGenerateLock.java | 41 ++++++++++++------ .../backup/archive/util/lock/LockUtil.java | 43 +++++++++++++++++++ 7 files changed, 116 insertions(+), 29 deletions(-) create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ServiceNodeInfo.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/LockUtil.java diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java index ec1ce3133d..7efbeae1b4 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java @@ -35,7 +35,6 @@ import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; import com.tencent.bk.job.backup.constant.DbDataNodeTypeEnum; import com.tencent.bk.job.common.mysql.dynamic.ds.DataSourceMode; -import com.tencent.bk.job.common.redis.util.LockResult; import com.tencent.bk.job.common.util.json.JsonUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -73,14 +72,10 @@ public JobInstanceArchiveTaskGenerator(ArchiveTaskService archiveTaskService, public void generate() { - LockResult lockResult = null; + boolean locked = false; try { - lockResult = archiveTaskGenerateLock.lock(); - if (!lockResult.isLockGotten()) { - log.info( - "Archive task generate lock gotten by another process: {}, return", - lockResult.getLockValue() - ); + locked = archiveTaskGenerateLock.lock(); + if (!locked) { return; } @@ -130,8 +125,8 @@ public void generate() { log.info("No new archive tasks are generated"); } } finally { - if (lockResult != null) { - lockResult.tryToRelease(); + if (locked) { + archiveTaskGenerateLock.unlock(); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java index 3cbca8eb79..01b02b5f2b 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java @@ -131,9 +131,10 @@ public void schedule() { StopWatch watch = new StopWatch("archive-task-schedule"); while (true) { + boolean locked = false; try { // 获取归档任务调度锁 - boolean locked = jobInstanceArchiveTaskScheduleLock.lock(); + locked = jobInstanceArchiveTaskScheduleLock.lock(); if (!locked) { log.info("Get lock fail, wait 1s"); ThreadUtils.sleep(1000L); @@ -185,7 +186,9 @@ public void schedule() { startArchiveTask(archiveTaskInfo); watch.stop(); } finally { - jobInstanceArchiveTaskScheduleLock.unlock(); + if (locked) { + jobInstanceArchiveTaskScheduleLock.unlock(); + } if (watch.isRunning()) { watch.stop(); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ServiceNodeInfo.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ServiceNodeInfo.java new file mode 100644 index 0000000000..e3e81cd550 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ServiceNodeInfo.java @@ -0,0 +1,31 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.util; + +import com.tencent.bk.job.common.util.ip.IpUtils; + +public class ServiceNodeInfo { + public static final String NODE_IP = IpUtils.getFirstMachineIP(); +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/FairDistributeLock.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/FairDistributeLock.java index 26cc202259..7538d52f8f 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/FairDistributeLock.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/FairDistributeLock.java @@ -28,8 +28,6 @@ import com.tencent.bk.job.common.util.ThreadUtils; import lombok.extern.slf4j.Slf4j; -import java.util.UUID; - /** * 分布式锁（公平锁，非饥饿抢占方式) */ @@ -65,7 +63,7 @@ public synchronized boolean lock() { // 为了保证在分布式系统中多个节点都能均匀获取到任务，需要最小获取锁间隔时间，让其他服务节点优先获取任务锁 ThreadUtils.sleep(100L); } - String lockRequestId = UUID.randomUUID().toString(); + String lockRequestId = LockUtil.generateLockRequestId(); if (!LockUtils.tryGetDistributedLock(lockKey, lockRequestId, lockMills)) { log.info("[{}] Acquire lock failed!", lockName); return false; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/HeartBeatRedisLocks.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/HeartBeatRedisLocks.java index 13f1d43493..9256a7f29b 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/HeartBeatRedisLocks.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/HeartBeatRedisLocks.java @@ -31,7 +31,6 @@ import org.springframework.data.redis.core.StringRedisTemplate; import java.util.Map; -import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; /** @@ -66,13 +65,14 @@ public HeartBeatRedisLocks(String lockKeyPrefix, } public synchronized boolean lock(String lockKey) { - String lockRequestId = UUID.randomUUID().toString(); + String lockRequestId = LockUtil.generateLockRequestId(); String actualLockKey = buildActualLockKey(lockKey); HeartBeatRedisLock redisLock = new HeartBeatRedisLock(redisTemplate, actualLockKey, lockRequestId, heartBeatRedisLockConfig); LockResult lockResult = redisLock.lock(); if (!lockResult.isLockGotten()) { + log.warn("Lock is held by another process: {}", lockResult.getLockValue()); return false; } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/JobInstanceArchiveTaskGenerateLock.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/JobInstanceArchiveTaskGenerateLock.java index 8b396eb41f..f4f2aab34f 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/JobInstanceArchiveTaskGenerateLock.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/JobInstanceArchiveTaskGenerateLock.java @@ -30,8 +30,6 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.core.StringRedisTemplate; -import java.util.UUID; - /** * 归档任务创建分布式锁 */ @@ -44,9 +42,7 @@ public class JobInstanceArchiveTaskGenerateLock { private final HeartBeatRedisLockConfig heartBeatRedisLockConfig; - private volatile HeartBeatRedisLock redisLock = null; - - private volatile String lockRequestId = null; + private volatile LockResult lockResult = null; public JobInstanceArchiveTaskGenerateLock(StringRedisTemplate redisTemplate) { this.redisTemplate = redisTemplate; @@ -57,13 +53,34 @@ public JobInstanceArchiveTaskGenerateLock(StringRedisTemplate redisTemplate) { ); } - public LockResult lock() { - if (lockRequestId != null) { - log.warn("Lock is held by another process"); - return LockResult.fail(lockRequestId); + public synchronized boolean lock() { + if (this.lockResult != null) { + log.warn("JobInstanceArchiveTaskGenerateLock is held by another process: {}", + lockResult.getLockValue()); + return false; + } + + String lockRequestId = LockUtil.generateLockRequestId(); + HeartBeatRedisLock redisLock = new HeartBeatRedisLock( + redisTemplate, REDIS_LOCK_KEY, lockRequestId, heartBeatRedisLockConfig); + LockResult lockResult = redisLock.lock(); + if (!lockResult.isLockGotten()) { + return false; + } + + this.lockResult = lockResult; + return true; + } + + public synchronized void unlock() { + if (this.lockResult == null) { + log.warn("JobInstanceArchiveTaskGenerateLock is not found"); + return; + } + try { + lockResult.tryToRelease(); + } finally { + this.lockResult = null; } - this.lockRequestId = UUID.randomUUID().toString(); - redisLock = new HeartBeatRedisLock(redisTemplate, REDIS_LOCK_KEY, lockRequestId, heartBeatRedisLockConfig); - return redisLock.lock(); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/LockUtil.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/LockUtil.java new file mode 100644 index 0000000000..81e3ef0ff6 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/LockUtil.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.util.lock; + +import com.tencent.bk.job.backup.archive.util.ServiceNodeInfo; + +import java.util.UUID; + +/** + * 分布式锁工具类 + */ +public class LockUtil { + /** + * 生成随机的分布式锁 requestId + * + */ + public static String generateLockRequestId() { + return ServiceNodeInfo.NODE_IP + "_" + UUID.randomUUID(); + } + +} From d4ef3aa19d7d1b6b0a817c356c25ce32bcf561a6 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Fri, 22 Nov 2024 16:37:45 +0800 Subject: [PATCH 55/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- support-files/kubernetes/charts/bk-job/values.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/support-files/kubernetes/charts/bk-job/values.yaml b/support-files/kubernetes/charts/bk-job/values.yaml index 1b9425562a..21a19891a8 100644 --- a/support-files/kubernetes/charts/bk-job/values.yaml +++ b/support-files/kubernetes/charts/bk-job/values.yaml @@ -635,6 +635,9 @@ job: # 是否支持容器执行 containerExecution: enabled: false + # 分库分表迁移使用，控制是否在mysql 查询条件中加入 task_instance_id 字段作为查询条件 + daoAddTaskInstanceId: + enabled: true trace: report: # 是否上报Trace数据至监控平台APM应用，默认不上报 From 049a505414dbc5608b7292cd5284ace423083fe3 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Sat, 23 Nov 2024 11:54:35 +0800 Subject: [PATCH 56/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit code review 处理 --- .../AbstractJobInstanceArchiveTask.java | 84 +++++---- .../archive/JobInstanceArchiveTask.java | 7 + .../JobInstanceArchiveTaskGenerator.java | 12 +- .../JobInstanceArchiveTaskScheduler.java | 30 ++-- .../JobInstanceMainDataArchiveTask.java | 36 +--- .../backup/archive/TaskCountDownLatch.java | 17 +- .../archive/dao/JobInstanceHotRecordDAO.java | 9 + .../impl/AbstractJobInstanceHotRecordDAO.java | 18 +- ... AbstractJobInstanceMainHotRecordDAO.java} | 76 +++------ .../archive/dao/impl/ArchiveTaskDAOImpl.java | 21 +-- .../dao/impl/FileSourceTaskLogRecordDAO.java | 8 +- .../dao/impl/GseFileAgentTaskRecordDAO.java | 8 +- .../impl/GseFileExecuteObjTaskRecordDAO.java | 8 +- .../dao/impl/GseScriptAgentTaskRecordDAO.java | 8 +- .../GseScriptExecuteObjTaskRecordDAO.java | 8 +- .../archive/dao/impl/GseTaskRecordDAO.java | 8 +- .../dao/impl/JobInstanceHotRecordDAO.java | 68 ++++++++ .../dao/impl/OperationLogRecordDAO.java | 8 +- .../dao/impl/RollingConfigRecordDAO.java | 8 +- .../impl/StepInstanceConfirmRecordDAO.java | 8 +- .../dao/impl/StepInstanceFileRecordDAO.java | 7 +- .../dao/impl/StepInstanceRecordDAO.java | 8 +- .../StepInstanceRollingTaskRecordDAO.java | 7 +- .../dao/impl/StepInstanceScriptRecordDAO.java | 7 +- .../impl/StepInstanceVariableRecordDAO.java | 7 +- .../dao/impl/TaskInstanceHostRecordDAO.java | 7 +- .../impl/TaskInstanceVariableRecordDAO.java | 8 +- .../archive/impl/TaskInstanceArchiver.java | 4 +- .../job/backup/archive/model/DbDataNode.java | 4 +- .../model/JobInstanceArchiveTaskInfo.java | 8 +- .../backup/config/ArchiveConfiguration.java | 12 +- .../job/backup/config/ArchiveProperties.java | 12 +- .../ArchiveDbNodePriorityEvaluatorTest.java | 16 +- .../kubernetes/charts/bk-job/values.yaml | 4 +- ...ob_execute_20241115-1000_V3.11.2_mysql.sql | 160 +++--------------- 35 files changed, 285 insertions(+), 436 deletions(-) rename src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/{TaskInstanceRecordDAO.java => AbstractJobInstanceMainHotRecordDAO.java} (58%) create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/JobInstanceHotRecordDAO.java diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java index 112111a321..7f176192b5 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java @@ -25,6 +25,7 @@ package com.tencent.bk.job.backup.archive; import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.AbstractJobInstanceMainHotRecordDAO; import com.tencent.bk.job.backup.archive.model.ArchiveTaskContext; import com.tencent.bk.job.backup.archive.model.ArchiveTaskSummary; import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; @@ -42,6 +43,7 @@ import org.slf4j.helpers.MessageFormatter; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; /** @@ -51,6 +53,14 @@ */ @Slf4j public abstract class AbstractJobInstanceArchiveTask> implements JobInstanceArchiveTask { + + /** + * 作业实例主表 DAO + */ + protected AbstractJobInstanceMainHotRecordDAO jobInstanceMainRecordDAO; + /** + * 冷 DB DAO + */ protected JobInstanceColdDAO jobInstanceColdDAO; protected final ArchiveProperties archiveProperties; private final ArchiveTaskExecuteLock archiveTaskExecuteLock; @@ -84,15 +94,21 @@ public abstract class AbstractJobInstanceArchiveTask> i private final Object stopMonitor = new Object(); private volatile ArchiveTaskStopCallback stopCallback = null; private volatile ArchiveTaskDoneCallback archiveTaskDoneCallback; + /** + * 任务标识-是否已被任务调度强制终止 + */ + private AtomicBoolean forceStoppedByScheduler = new AtomicBoolean(false); - public AbstractJobInstanceArchiveTask(JobInstanceColdDAO jobInstanceColdDAO, + public AbstractJobInstanceArchiveTask(AbstractJobInstanceMainHotRecordDAO jobInstanceMainRecordDAO, + JobInstanceColdDAO jobInstanceColdDAO, ArchiveProperties archiveProperties, ArchiveTaskExecuteLock archiveTaskExecuteLock, ArchiveErrorTaskCounter archiveErrorTaskCounter, JobInstanceArchiveTaskInfo archiveTaskInfo, ArchiveTaskService archiveTaskService, ArchiveTablePropsStorage archiveTablePropsStorage) { + this.jobInstanceMainRecordDAO = jobInstanceMainRecordDAO; this.jobInstanceColdDAO = jobInstanceColdDAO; this.archiveProperties = archiveProperties; this.archiveTaskExecuteLock = archiveTaskExecuteLock; @@ -157,8 +173,7 @@ private void archive() { } log.info("[{}] Start archive task", taskId); - archiveTaskInfo.setStatus(ArchiveTaskStatusEnum.RUNNING); - archiveTaskService.updateTask(archiveTaskInfo); + updateArchiveTask(ArchiveTaskStatusEnum.RUNNING, null); // 归档 backupAndDelete(); } catch (Throwable e) { @@ -172,7 +187,7 @@ private void archive() { archiveErrorTaskCounter.increment(); // 更新归档任务状态 - updateArchiveProgress(ArchiveTaskStatusEnum.FAIL, null); + updateArchiveTask(ArchiveTaskStatusEnum.FAIL, null); } finally { if (this.isAcquireLock) { archiveTaskExecuteLock.unlock(taskId); @@ -195,8 +210,8 @@ private void archive() { private void backupAndDelete() { boolean backupEnabled = isBackupEnable(); boolean deleteEnabled = isDeleteEnable(); - - int readLimit = 1000; + // 获取主表对应的 readRowLimit + int readLimit = archiveTablePropsStorage.getReadRowLimit(jobInstanceMainRecordDAO.getTable().getName()); long archivedJobInstanceCount = 0; long startTime = System.currentTimeMillis(); @@ -212,7 +227,7 @@ private void backupAndDelete() { jobInstanceRecords = readJobInstanceRecords(readLimit); if (CollectionUtils.isEmpty(jobInstanceRecords)) { - updateArchiveProgress(ArchiveTaskStatusEnum.SUCCESS, progress); + updateArchiveTask(ArchiveTaskStatusEnum.SUCCESS, null); return; } archivedJobInstanceCount += jobInstanceRecords.size(); @@ -241,8 +256,7 @@ private void backupAndDelete() { Long lastJobInstanceId = extractJobInstanceId(lastRecord); progress = new TimeAndIdBasedArchiveProcess(lastTimestamp, lastJobInstanceId); boolean isFinished = jobInstanceRecords.size() < readLimit; - updateArchiveProgress(isFinished ? ArchiveTaskStatusEnum.SUCCESS : ArchiveTaskStatusEnum.RUNNING, - progress); + updateArchiveTask(isFinished ? ArchiveTaskStatusEnum.SUCCESS : ArchiveTaskStatusEnum.RUNNING, progress); } while (jobInstanceRecords.size() == readLimit); } finally { long archiveCost = System.currentTimeMillis() - startTime; @@ -256,7 +270,7 @@ private List readJobInstanceRecords(int readLimit) { long readStartTime = System.currentTimeMillis(); Long fromTime = progress == null ? archiveTaskInfo.getFromTimestamp() : progress.getTimestamp(); Long fromTaskInstanceId = progress != null ? progress.getId() : null; - jobInstanceRecords = readSortedJobInstanceFromHotDB(fromTime, + jobInstanceRecords = jobInstanceMainRecordDAO.readSortedJobInstanceFromHotDB(fromTime, archiveTaskInfo.getToTimestamp(), fromTaskInstanceId, readLimit); log.info("[{}] Read sorted job instance from hot db, fromJobCreateTime: {}, toJobCreatTime: {}, " + "fromJobInstanceId: {}, resultSize: {}, cost: {} ms", @@ -299,12 +313,23 @@ private boolean acquireLock() { return isAcquireLock; } - private void updateArchiveProgress(ArchiveTaskStatusEnum taskStatus, TimeAndIdBasedArchiveProcess progress) { - archiveTaskInfo.setStatus(taskStatus); - archiveTaskInfo.setProcess(progress); - if (taskStatus != ArchiveTaskStatusEnum.RUNNING) { - log.info("[{}] Update archive task process, taskStatus: {}, process: {}", - taskId, taskStatus, progress); + private void updateArchiveTask(ArchiveTaskStatusEnum status, + TimeAndIdBasedArchiveProcess progress) { + if (status != null) { + archiveTaskInfo.setStatus(status); + } + if (progress != null) { + archiveTaskInfo.setProcess(progress); + } + if (forceStoppedByScheduler.get()) { + // 如果已经被归档任务调度强制终止了，就不能再去更新 db，引起数据不一致 + log.info("[{}] Archive task is force stopped by scheduler, do not update archive task again", taskId); + return; + } + + if (archiveTaskInfo.getStatus() != ArchiveTaskStatusEnum.RUNNING) { + log.info("[{}] Update archive task, taskStatus: {}, process: {}", + taskId, archiveTaskInfo.getStatus(), progress); } archiveTaskService.updateTask(archiveTaskInfo); } @@ -314,31 +339,26 @@ public String getTaskId() { return this.taskId; } - /** - * 从热 db 读取作业实例熟悉，按照时间+ID 的顺序排序 - * - * @param fromTimestamp 时间范围-起始-作业实例创建时间（include) - * @param endTimestamp 时间范围-起始-作业实例创建时间（exclude) - * @param fromJobInstanceId 作业实例 ID-起始 (exclude) - * @param limit 读取记录最大数量 - * @return 作业实例记录 - */ - protected abstract List readSortedJobInstanceFromHotDB(Long fromTimestamp, - Long endTimestamp, - Long fromJobInstanceId, - int limit); - /** * 从作业实例记录中提取作业实例 ID * * @param record 作业实例记录 */ - protected abstract Long extractJobInstanceId(T record); + protected Long extractJobInstanceId(T record) { + return record.get(jobInstanceMainRecordDAO.getJobInstanceIdField()); + } /** * 从作业实例记录中提取作业实例创建时间 * * @param record 作业实例记录 */ - protected abstract Long extractJobInstanceCreateTime(T record); + protected Long extractJobInstanceCreateTime(T record) { + return record.get(jobInstanceMainRecordDAO.getJobInstanceCreateTimeField()); + } + + @Override + public JobInstanceArchiveTaskInfo getJobInstanceArchiveTaskInfo() { + return archiveTaskInfo; + } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTask.java index 80f50c80a3..fbb049435b 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTask.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTask.java @@ -24,6 +24,8 @@ package com.tencent.bk.job.backup.archive; +import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; + /** * 作业执行实例归档任务 */ @@ -48,4 +50,9 @@ public interface JobInstanceArchiveTask { */ String getTaskId(); + /** + * 获取归档任务信息 + */ + JobInstanceArchiveTaskInfo getJobInstanceArchiveTaskInfo(); + } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java index 7efbeae1b4..a13f170a4a 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java @@ -24,7 +24,7 @@ package com.tencent.bk.job.backup.archive; -import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.JobInstanceHotRecordDAO; import com.tencent.bk.job.backup.archive.model.DbDataNode; import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; @@ -52,7 +52,7 @@ public class JobInstanceArchiveTaskGenerator { private final ArchiveTaskService archiveTaskService; - private final TaskInstanceRecordDAO taskInstanceRecordDAO; + private final JobInstanceHotRecordDAO taskInstanceRecordDAO; private final ArchiveProperties archiveProperties; @@ -60,7 +60,7 @@ public class JobInstanceArchiveTaskGenerator { public JobInstanceArchiveTaskGenerator(ArchiveTaskService archiveTaskService, - TaskInstanceRecordDAO taskInstanceRecordDAO, + JobInstanceHotRecordDAO taskInstanceRecordDAO, ArchiveProperties archiveProperties, JobInstanceArchiveTaskGenerateLock archiveTaskGenerateLock) { @@ -108,7 +108,7 @@ public void generate() { archiveStartDateTime, archiveProperties.getTasks().getJobInstance().getShardingDataNodes())); } else { // 单db - DbDataNode dbDataNode = DbDataNode.standaloneDbDatNode(); + DbDataNode dbDataNode = DbDataNode.standaloneDbDataNode(); JobInstanceArchiveTaskInfo archiveTaskInfo = buildArchiveTask(ArchiveTaskTypeEnum.JOB_INSTANCE, archiveStartDateTime, dbDataNode); archiveTaskList.add(archiveTaskInfo); @@ -124,6 +124,8 @@ public void generate() { } else { log.info("No new archive tasks are generated"); } + } catch (Throwable e) { + log.error("Generate archive task caught exception", e); } finally { if (locked) { archiveTaskGenerateLock.unlock(); @@ -184,7 +186,7 @@ private LocalDateTime computeArchiveStartDateTime() { if (latestArchiveTask == null) { // 从表数据中的 job_create_time 计算归档任务开始时间 log.info("Latest archive task is empty, try compute from table min job create time"); - Long minJobCreateTimeMills = taskInstanceRecordDAO.getMinJobCreateTime(); + Long minJobCreateTimeMills = taskInstanceRecordDAO.getMinJobInstanceCreateTime(); log.info("Min job create time in db is : {}", minJobCreateTimeMills); startDateTime = ArchiveDateTimeUtil.toHourlyRoundDown( ArchiveDateTimeUtil.unixTimestampMillToLocalDateTime(minJobCreateTimeMills)); diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java index 01b02b5f2b..2a553a87bf 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java @@ -25,7 +25,7 @@ package com.tencent.bk.job.backup.archive; import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; -import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.JobInstanceHotRecordDAO; import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; import com.tencent.bk.job.backup.archive.util.lock.ArchiveTaskExecuteLock; @@ -56,7 +56,7 @@ public class JobInstanceArchiveTaskScheduler implements SmartLifecycle { private final ArchiveTaskService archiveTaskService; - private final TaskInstanceRecordDAO taskInstanceRecordDAO; + private final JobInstanceHotRecordDAO taskInstanceRecordDAO; private final ArchiveProperties archiveProperties; @@ -77,10 +77,6 @@ public class JobInstanceArchiveTaskScheduler implements SmartLifecycle { * 作业执行历史归档任务调度组件是否处于活动状态 */ private volatile boolean active; - /** - * 组件是否正在运行(用于 Spring Lifecycle isRunning 判断) - */ - private volatile boolean running = false; /** * 是否正在进行任务调度中 */ @@ -96,7 +92,7 @@ public class JobInstanceArchiveTaskScheduler implements SmartLifecycle { public JobInstanceArchiveTaskScheduler(ArchiveTaskService archiveTaskService, - TaskInstanceRecordDAO taskInstanceRecordDAO, + JobInstanceHotRecordDAO taskInstanceRecordDAO, ArchiveProperties archiveProperties, JobInstanceArchiveTaskScheduleLock jobInstanceArchiveTaskScheduleLock, JobInstanceSubTableArchivers jobInstanceSubTableArchivers, @@ -164,13 +160,13 @@ public void schedule() { watch.start("evaluateTaskPriority"); ArchiveDbNodePriorityEvaluator.DbNodeTasksInfo highestPriorityDbNodeTasksInfo = ArchiveDbNodePriorityEvaluator.evaluateHighestPriorityDbNode(runningTasks, - scheduleTasksGroupByDb); + scheduleTasksGroupByDb); watch.stop(); int taskConcurrent = archiveProperties.getTasks().getJobInstance().getConcurrent(); if (highestPriorityDbNodeTasksInfo.getRunningTaskCount() >= taskConcurrent) { // 休眠5分钟，等待并行任务减少 log.info("Running archive task count exceed concurrent limit : {}, wait 300s", taskConcurrent); - ThreadUtils.sleep(1000 * 60L); + ThreadUtils.sleep(1000 * 300L); continue; } @@ -197,6 +193,8 @@ public void schedule() { } } } + } catch (Throwable e) { + log.error("Schedule archive task caught exception", e); } finally { this.scheduling = false; } @@ -245,7 +243,6 @@ public void start() { } synchronized (lifecycleMonitor) { this.active = true; - this.running = true; } } @@ -261,7 +258,6 @@ public void stop() { return; } this.active = false; - this.running = false; } stopTasksGraceful(); log.info("JobInstanceArchiveTaskScheduler stop successfully!"); @@ -282,7 +278,15 @@ private void stopTasksGraceful() { } try { if (taskCountDownLatch != null) { - taskCountDownLatch.waitingForAllTasksDone(); + // 等待任务结束，最多等待 2min + boolean isAllTaskStopped = taskCountDownLatch.waitingForAllTasksDone(120); + if (!isAllTaskStopped) { + log.info("Force update archive task status to suspended"); + for (JobInstanceArchiveTask task : scheduledTasks.values()) { + archiveTaskService.updateArchiveTaskSuspendedStatus(task.getJobInstanceArchiveTaskInfo()); + log.info("Force update archive task status to suspended, taskId: {}", task.getTaskId()); + } + } } } catch (Throwable e) { log.error("Stop archive tasks caught exception", e); @@ -314,7 +318,7 @@ public void run() { @Override public boolean isRunning() { synchronized (this.lifecycleMonitor) { - return (this.running); + return (this.active); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java index abd1968196..4aa9d5f36a 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java @@ -25,7 +25,7 @@ package com.tencent.bk.job.backup.archive; import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; -import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.JobInstanceHotRecordDAO; import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; import com.tencent.bk.job.backup.archive.util.lock.ArchiveTaskExecuteLock; @@ -44,11 +44,9 @@ @Slf4j public class JobInstanceMainDataArchiveTask extends AbstractJobInstanceArchiveTask { - private final TaskInstanceRecordDAO taskInstanceRecordDAO; - private final JobInstanceSubTableArchivers jobInstanceSubTableArchivers; - public JobInstanceMainDataArchiveTask(TaskInstanceRecordDAO taskInstanceRecordDAO, + public JobInstanceMainDataArchiveTask(JobInstanceHotRecordDAO jobInstanceHotRecordDAO, JobInstanceSubTableArchivers jobInstanceSubTableArchivers, JobInstanceColdDAO jobInstanceColdDAO, ArchiveProperties archiveProperties, @@ -58,6 +56,7 @@ public JobInstanceMainDataArchiveTask(TaskInstanceRecordDAO taskInstanceRecordDA ArchiveTaskService archiveTaskService, ArchiveTablePropsStorage archiveTablePropsStorage) { super( + jobInstanceHotRecordDAO, jobInstanceColdDAO, archiveProperties, archiveTaskExecuteLock, @@ -66,7 +65,6 @@ public JobInstanceMainDataArchiveTask(TaskInstanceRecordDAO taskInstanceRecordDA archiveTaskService, archiveTablePropsStorage ); - this.taskInstanceRecordDAO = taskInstanceRecordDAO; this.jobInstanceSubTableArchivers = jobInstanceSubTableArchivers; } @@ -76,7 +74,8 @@ protected void backupJobInstanceToColdDb(List jobInstanceRec List jobInstanceIds = jobInstanceRecords.stream().map(this::extractJobInstanceId).collect(Collectors.toList()); // 备份主表数据 - jobInstanceColdDAO.batchInsert(jobInstanceRecords, 1000); + jobInstanceColdDAO.batchInsert(jobInstanceRecords, + archiveTablePropsStorage.getBatchInsertRowSize(jobInstanceMainRecordDAO.getTable().getName())); // 备份子表数据 jobInstanceSubTableArchivers.getAll().forEach(tableArchiver -> { tableArchiver.backupRecords(jobInstanceIds); @@ -91,32 +90,9 @@ protected void deleteJobInstanceHotData(List jobInstanceIds) { }); // 删除主表数据 long startTime = System.currentTimeMillis(); - taskInstanceRecordDAO.deleteRecords(jobInstanceIds, + jobInstanceMainRecordDAO.deleteRecords(jobInstanceIds, archiveTablePropsStorage.getDeleteLimitRowCount(TaskInstance.TASK_INSTANCE.getName())); log.info("Delete {}, taskInstanceIdSize: {}, cost: {}ms", "task_instance", jobInstanceIds.size(), System.currentTimeMillis() - startTime); } - - @Override - protected List readSortedJobInstanceFromHotDB(Long fromTimestamp, - Long endTimestamp, - Long fromJobInstanceId, - int limit) { - return taskInstanceRecordDAO.readSortedJobInstanceFromHotDB( - fromTimestamp, - endTimestamp, - fromJobInstanceId, - limit - ); - } - - @Override - protected Long extractJobInstanceId(TaskInstanceRecord record) { - return record.get(TaskInstance.TASK_INSTANCE.ID); - } - - @Override - protected Long extractJobInstanceCreateTime(TaskInstanceRecord record) { - return record.get(TaskInstance.TASK_INSTANCE.CREATE_TIME); - } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/TaskCountDownLatch.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/TaskCountDownLatch.java index 336ec65c81..4f470ae77b 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/TaskCountDownLatch.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/TaskCountDownLatch.java @@ -31,6 +31,7 @@ import java.util.HashSet; import java.util.Set; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; /** * 任务 CountDownLatch 组件，用于等待所有任务完成后再退出 @@ -62,14 +63,20 @@ public void decrement(String taskId) { } } - public void waitingForAllTasksDone() { + public boolean waitingForAllTasksDone(long timeoutSeconds) { try { log.info("Waiting for all tasks done! total: {}", latch.getCount()); - this.latch.await(); - this.isAllTaskDone = true; - log.info("All tasks are stopped!"); + isAllTaskDone = latch.await(timeoutSeconds, TimeUnit.SECONDS); + if (isAllTaskDone) { + log.info("All tasks have been completed"); + } else { + log.info("Some tasks did not end within the timeout period, timeout: {}, notCompletedTaskIds: {}", + timeoutSeconds, taskIds); + } + return isAllTaskDone; } catch (InterruptedException e) { - log.warn("Stop task counter wait interrupted", e); + log.warn("Task count down latch wait interrupted", e); + return false; } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/JobInstanceHotRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/JobInstanceHotRecordDAO.java index d67b7397c1..c3571e4049 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/JobInstanceHotRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/JobInstanceHotRecordDAO.java @@ -53,6 +53,15 @@ public interface JobInstanceHotRecordDAO { */ TableField getJobInstanceIdField(); + /** + * 获取表中作业实例创建时间对应的字段（仅主表存在） + * + * @return ID字段 + */ + default TableField getJobInstanceCreateTimeField() { + return null; + } + /** * 根据作业实例 ID 列表获取表记录 * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceHotRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceHotRecordDAO.java index 27db0ffc2d..aef42ffcdb 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceHotRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceHotRecordDAO.java @@ -48,25 +48,25 @@ public abstract class AbstractJobInstanceHotRecordDAO implemen protected final DSLContextProvider dslContextProvider; - protected final String tableName; + protected final Table table; - public AbstractJobInstanceHotRecordDAO(DSLContextProvider dslContextProvider, String tableName) { + public AbstractJobInstanceHotRecordDAO(DSLContextProvider dslContextProvider, Table table) { this.dslContextProvider = dslContextProvider; - this.tableName = tableName; + this.table = table; } @Override public List listRecords(Collection jobInstanceIds, Long readRowLimit) { - return query(getTable(), buildBasicConditions(jobInstanceIds), readRowLimit); + return query(table, buildBasicConditions(jobInstanceIds), readRowLimit); } public List listRecordsByConditions(List conditions, Long readRowLimit) { - return query(getTable(), conditions, readRowLimit); + return query(table, conditions, readRowLimit); } @Override public int deleteRecords(Collection jobInstanceIds, long maxLimitedDeleteRows) { - return deleteWithLimit(getTable(), buildBasicConditions(jobInstanceIds), maxLimitedDeleteRows); + return deleteWithLimit(table, buildBasicConditions(jobInstanceIds), maxLimitedDeleteRows); } public List buildBasicConditions(Collection jobInstanceIds) { @@ -109,11 +109,13 @@ protected List query(Table table, } - public abstract Table getTable(); + public Table getTable() { + return this.table; + } protected abstract Collection> getListRecordsOrderFields(); protected DSLContext dsl() { - return this.dslContextProvider.get(tableName); + return this.dslContextProvider.get(table.getName()); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceMainHotRecordDAO.java similarity index 58% rename from src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceRecordDAO.java rename to src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceMainHotRecordDAO.java index 3e02ef3b90..2e89e73dbd 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceMainHotRecordDAO.java @@ -27,52 +27,35 @@ import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; -import com.tencent.bk.job.execute.model.tables.TaskInstance; -import com.tencent.bk.job.execute.model.tables.records.TaskInstanceRecord; -import org.jooq.OrderField; import org.jooq.Record; import org.jooq.Record1; import org.jooq.Result; import org.jooq.SelectConditionStep; import org.jooq.Table; -import org.jooq.TableField; -import java.util.ArrayList; import java.util.Collection; import java.util.List; import static org.jooq.impl.DSL.min; /** - * task_instance DAO + * 作业实例 - 主表 - 热数据查询 DAO 基础抽象实现 + * + * @param */ -public class TaskInstanceRecordDAO extends AbstractJobInstanceHotRecordDAO { - - private static final TaskInstance TABLE = TaskInstance.TASK_INSTANCE; - - private static final List> ORDER_FIELDS = new ArrayList<>(); - - static { - ORDER_FIELDS.add(TaskInstance.TASK_INSTANCE.ID.asc()); - } - +public abstract class AbstractJobInstanceMainHotRecordDAO extends AbstractJobInstanceHotRecordDAO { - public TaskInstanceRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); - } - - @Override - public Table getTable() { - return TABLE; + public AbstractJobInstanceMainHotRecordDAO(DSLContextProvider dslContextProvider, Table table) { + super(dslContextProvider, table); } /** - * 获取表中作业的最早创建时间 + * 获取表中作业实例的最早创建时间 */ - public Long getMinJobCreateTime() { + public Long getMinJobInstanceCreateTime() { Record1 record = - dsl().select(min(TABLE.CREATE_TIME)) - .from(TABLE) + dsl().select(min(getJobInstanceCreateTimeField())) + .from(getTable()) .fetchOne(); if (record != null) { Long minJobCreateTime = (Long) record.get(0); @@ -87,7 +70,7 @@ public Long getMinJobCreateTime() { * 是否为空表 */ public boolean isTableEmpty() { - return !dsl().fetchExists(TABLE); + return !dsl().fetchExists(getTable()); } /** @@ -99,42 +82,35 @@ public boolean isTableEmpty() { * @param limit 读取最大行数 * @return 记录 */ - public List readSortedJobInstanceFromHotDB(Long fromTimestamp, - Long endTimestamp, - Long fromJobInstanceId, - int limit) { + public List readSortedJobInstanceFromHotDB(Long fromTimestamp, + Long endTimestamp, + Long fromJobInstanceId, + int limit) { SelectConditionStep selectConditionStep = dsl().select() - .from(TABLE) - .where(TABLE.CREATE_TIME.greaterOrEqual(fromTimestamp)) - .and(TABLE.CREATE_TIME.lessThan(endTimestamp)); + .from(getTable()) + .where(getJobInstanceCreateTimeField().greaterOrEqual(fromTimestamp)) + .and(getJobInstanceCreateTimeField().lessThan(endTimestamp)); if (fromJobInstanceId != null) { - selectConditionStep.and(TABLE.ID.greaterThan(fromJobInstanceId)); + selectConditionStep.and(getJobInstanceIdField().greaterThan(fromJobInstanceId)); } - Result result = selectConditionStep.orderBy(TABLE.CREATE_TIME.asc(), TABLE.ID.asc()) + Result result = selectConditionStep.orderBy( + getJobInstanceCreateTimeField().asc(), + getJobInstanceIdField().asc() + ) .limit(limit) .fetch(); - return result.into(TABLE); + return result.into(getTable()); } - @Override - public TableField getJobInstanceIdField() { - return TABLE.ID; - } - - protected Collection> getListRecordsOrderFields() { - return ORDER_FIELDS; - } @Override - public RecordResultSet executeQuery(Collection jobInstanceIds, - long readRowLimit) { + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { return JobInstanceRecordResultSetFactory.createOneQueryResultSet( this, jobInstanceIds, readRowLimit ); } - - } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java index 3b39bec8ae..ef51c7ceac 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java @@ -169,10 +169,7 @@ public Map countScheduleTasksGroupByDb(ArchiveTaskTypeEnum task @Override public void updateTask(JobInstanceArchiveTaskInfo archiveTask) { - if (archiveTask.getStatus() == null && archiveTask.getProcess() == null) { - // 无需更新 - return; - } + boolean update = false; // 按需更新 UpdateSetMoreStep updateSetMoreStep; updateSetMoreStep = ctx .update(T) @@ -180,17 +177,21 @@ public void updateTask(JobInstanceArchiveTaskInfo archiveTask) { if (archiveTask.getStatus() != null) { updateSetMoreStep .set(T.STATUS, JooqDataTypeUtil.toByte(archiveTask.getStatus().getStatus())); + update = true; } if (archiveTask.getProcess() != null) { updateSetMoreStep .set(T.PROCESS, archiveTask.getProcess().toPersistentProcess()); + update = true; + } + if (update) { + updateSetMoreStep + .where(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(archiveTask.getTaskType().getType()))) + .and(T.DATA_NODE.eq(archiveTask.getDbDataNode().toDataNodeId())) + .and(T.DAY.eq(archiveTask.getDay())) + .and(T.HOUR.eq(archiveTask.getHour().byteValue())) + .execute(); } - updateSetMoreStep - .where(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(archiveTask.getTaskType().getType()))) - .and(T.DATA_NODE.eq(archiveTask.getDbDataNode().toDataNodeId())) - .and(T.DAY.eq(archiveTask.getDay())) - .and(T.HOUR.eq(archiveTask.getHour().byteValue())) - .execute(); } @Override diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/FileSourceTaskLogRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/FileSourceTaskLogRecordDAO.java index 99292ee398..6b9f93b183 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/FileSourceTaskLogRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/FileSourceTaskLogRecordDAO.java @@ -32,7 +32,6 @@ import com.tencent.bk.job.execute.model.tables.records.FileSourceTaskLogRecord; import org.jooq.Condition; import org.jooq.OrderField; -import org.jooq.Table; import org.jooq.TableField; import java.util.ArrayList; @@ -53,12 +52,7 @@ public class FileSourceTaskLogRecordDAO extends AbstractJobInstanceHotRecordDAO< } public FileSourceTaskLogRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); - } - - @Override - public Table getTable() { - return TABLE; + super(dslContextProvider, TABLE); } @Override diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileAgentTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileAgentTaskRecordDAO.java index 07b28b5b35..24949b4302 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileAgentTaskRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileAgentTaskRecordDAO.java @@ -31,7 +31,6 @@ import com.tencent.bk.job.execute.model.tables.records.GseFileAgentTaskRecord; import org.jooq.Condition; import org.jooq.OrderField; -import org.jooq.Table; import org.jooq.TableField; import java.util.ArrayList; @@ -52,12 +51,7 @@ public class GseFileAgentTaskRecordDAO extends AbstractJobInstanceHotRecordDAO getTable() { - return TABLE; + super(dslContextProvider, TABLE); } @Override diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileExecuteObjTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileExecuteObjTaskRecordDAO.java index 7e2814a2e3..875e2e938c 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileExecuteObjTaskRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileExecuteObjTaskRecordDAO.java @@ -31,7 +31,6 @@ import com.tencent.bk.job.execute.model.tables.records.GseFileExecuteObjTaskRecord; import org.jooq.Condition; import org.jooq.OrderField; -import org.jooq.Table; import org.jooq.TableField; import java.util.ArrayList; @@ -55,12 +54,7 @@ public class GseFileExecuteObjTaskRecordDAO extends AbstractJobInstanceHotRecord } public GseFileExecuteObjTaskRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); - } - - @Override - public Table getTable() { - return TABLE; + super(dslContextProvider, TABLE); } @Override diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptAgentTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptAgentTaskRecordDAO.java index 9342c48621..acdde19c44 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptAgentTaskRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptAgentTaskRecordDAO.java @@ -31,7 +31,6 @@ import com.tencent.bk.job.execute.model.tables.records.GseScriptAgentTaskRecord; import org.jooq.Condition; import org.jooq.OrderField; -import org.jooq.Table; import org.jooq.TableField; import java.util.ArrayList; @@ -52,12 +51,7 @@ public class GseScriptAgentTaskRecordDAO extends AbstractJobInstanceHotRecordDAO } public GseScriptAgentTaskRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); - } - - @Override - public Table getTable() { - return TABLE; + super(dslContextProvider, TABLE); } @Override diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptExecuteObjTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptExecuteObjTaskRecordDAO.java index 2c9367d4ee..e1e4a28c74 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptExecuteObjTaskRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptExecuteObjTaskRecordDAO.java @@ -31,7 +31,6 @@ import com.tencent.bk.job.execute.model.tables.records.GseScriptExecuteObjTaskRecord; import org.jooq.Condition; import org.jooq.OrderField; -import org.jooq.Table; import org.jooq.TableField; import java.util.ArrayList; @@ -52,12 +51,7 @@ public class GseScriptExecuteObjTaskRecordDAO extends AbstractJobInstanceHotReco } public GseScriptExecuteObjTaskRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); - } - - @Override - public Table getTable() { - return TABLE; + super(dslContextProvider, TABLE); } @Override diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseTaskRecordDAO.java index 95872e8f81..8eac2d8dc3 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseTaskRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseTaskRecordDAO.java @@ -31,7 +31,6 @@ import com.tencent.bk.job.execute.model.tables.records.GseTaskRecord; import org.jooq.Condition; import org.jooq.OrderField; -import org.jooq.Table; import org.jooq.TableField; import java.util.ArrayList; @@ -49,12 +48,7 @@ public class GseTaskRecordDAO extends AbstractJobInstanceHotRecordDAO getTable() { - return TABLE; + super(dslContextProvider, TABLE); } @Override diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/JobInstanceHotRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/JobInstanceHotRecordDAO.java new file mode 100644 index 0000000000..1edf16f4f5 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/JobInstanceHotRecordDAO.java @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.TaskInstance; +import com.tencent.bk.job.execute.model.tables.records.TaskInstanceRecord; +import org.jooq.OrderField; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * task_instance DAO + */ +public class JobInstanceHotRecordDAO extends AbstractJobInstanceMainHotRecordDAO { + + private static final TaskInstance TABLE = TaskInstance.TASK_INSTANCE; + + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(TaskInstance.TASK_INSTANCE.ID.asc()); + } + + public JobInstanceHotRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE); + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.ID; + } + + @Override + public TableField getJobInstanceCreateTimeField() { + return TABLE.CREATE_TIME; + } + + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/OperationLogRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/OperationLogRecordDAO.java index b02d09e6e0..a1cf111c9c 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/OperationLogRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/OperationLogRecordDAO.java @@ -31,7 +31,6 @@ import com.tencent.bk.job.execute.model.tables.records.OperationLogRecord; import org.jooq.Condition; import org.jooq.OrderField; -import org.jooq.Table; import org.jooq.TableField; import java.util.ArrayList; @@ -53,12 +52,7 @@ public class OperationLogRecordDAO extends AbstractJobInstanceHotRecordDAO getTable() { - return TABLE; + super(dslContextProvider, TABLE); } @Override diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/RollingConfigRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/RollingConfigRecordDAO.java index cd6908dc12..e376e842a5 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/RollingConfigRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/RollingConfigRecordDAO.java @@ -30,7 +30,6 @@ import com.tencent.bk.job.execute.model.tables.RollingConfig; import com.tencent.bk.job.execute.model.tables.records.RollingConfigRecord; import org.jooq.OrderField; -import org.jooq.Table; import org.jooq.TableField; import java.util.ArrayList; @@ -51,12 +50,7 @@ public class RollingConfigRecordDAO extends AbstractJobInstanceHotRecordDAO getTable() { - return TABLE; + super(dslContextProvider, TABLE); } @Override diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceConfirmRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceConfirmRecordDAO.java index 5a7c758522..22669fca42 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceConfirmRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceConfirmRecordDAO.java @@ -31,7 +31,6 @@ import com.tencent.bk.job.execute.model.tables.records.StepInstanceConfirmRecord; import org.jooq.Condition; import org.jooq.OrderField; -import org.jooq.Table; import org.jooq.TableField; import java.util.ArrayList; @@ -55,12 +54,7 @@ public class StepInstanceConfirmRecordDAO extends AbstractJobInstanceHotRecordDA public StepInstanceConfirmRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); - } - - @Override - public Table getTable() { - return TABLE; + super(dslContextProvider, TABLE); } @Override diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceFileRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceFileRecordDAO.java index f528a0784c..7806bdcbef 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceFileRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceFileRecordDAO.java @@ -31,7 +31,6 @@ import com.tencent.bk.job.execute.model.tables.records.StepInstanceFileRecord; import org.jooq.Condition; import org.jooq.OrderField; -import org.jooq.Table; import org.jooq.TableField; import java.util.ArrayList; @@ -54,13 +53,9 @@ public class StepInstanceFileRecordDAO extends AbstractJobInstanceHotRecordDAO getTable() { - return TABLE; - } @Override public TableField getJobInstanceIdField() { diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRecordDAO.java index 0488a8e747..31fe2845f0 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRecordDAO.java @@ -31,7 +31,6 @@ import com.tencent.bk.job.execute.model.tables.records.StepInstanceRecord; import org.jooq.Condition; import org.jooq.OrderField; -import org.jooq.Table; import org.jooq.TableField; import java.util.ArrayList; @@ -53,12 +52,7 @@ public class StepInstanceRecordDAO extends AbstractJobInstanceHotRecordDAO getTable() { - return TABLE; + super(dslContextProvider, TABLE); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRollingTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRollingTaskRecordDAO.java index eb19f5661b..fc986c5aee 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRollingTaskRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRollingTaskRecordDAO.java @@ -31,7 +31,6 @@ import com.tencent.bk.job.execute.model.tables.records.StepInstanceRollingTaskRecord; import org.jooq.Condition; import org.jooq.OrderField; -import org.jooq.Table; import org.jooq.TableField; import java.util.ArrayList; @@ -52,13 +51,9 @@ public class StepInstanceRollingTaskRecordDAO extends AbstractJobInstanceHotReco } public StepInstanceRollingTaskRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); + super(dslContextProvider, TABLE); } - @Override - public Table getTable() { - return TABLE; - } @Override public TableField getJobInstanceIdField() { diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceScriptRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceScriptRecordDAO.java index 5e6ea82fc7..7990a5a8c2 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceScriptRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceScriptRecordDAO.java @@ -31,7 +31,6 @@ import com.tencent.bk.job.execute.model.tables.records.StepInstanceScriptRecord; import org.jooq.Condition; import org.jooq.OrderField; -import org.jooq.Table; import org.jooq.TableField; import java.util.ArrayList; @@ -54,13 +53,9 @@ public class StepInstanceScriptRecordDAO extends AbstractJobInstanceHotRecordDAO public StepInstanceScriptRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); + super(dslContextProvider, TABLE); } - @Override - public Table getTable() { - return TABLE; - } @Override public TableField getJobInstanceIdField() { diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceVariableRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceVariableRecordDAO.java index 654f1104db..e130df1444 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceVariableRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceVariableRecordDAO.java @@ -31,7 +31,6 @@ import com.tencent.bk.job.execute.model.tables.records.StepInstanceVariableRecord; import org.jooq.Condition; import org.jooq.OrderField; -import org.jooq.Table; import org.jooq.TableField; import java.util.ArrayList; @@ -52,13 +51,9 @@ public class StepInstanceVariableRecordDAO extends AbstractJobInstanceHotRecordD } public StepInstanceVariableRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); + super(dslContextProvider, TABLE); } - @Override - public Table getTable() { - return TABLE; - } @Override public TableField getJobInstanceIdField() { diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceHostRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceHostRecordDAO.java index d1897e3279..1624e7045d 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceHostRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceHostRecordDAO.java @@ -31,7 +31,6 @@ import com.tencent.bk.job.execute.model.tables.records.TaskInstanceHostRecord; import org.jooq.Condition; import org.jooq.OrderField; -import org.jooq.Table; import org.jooq.TableField; import java.util.ArrayList; @@ -50,13 +49,9 @@ public class TaskInstanceHostRecordDAO extends AbstractJobInstanceHotRecordDAO getTable() { - return TABLE; - } @Override public TableField getJobInstanceIdField() { diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceVariableRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceVariableRecordDAO.java index b48a28cdb6..146bc1858a 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceVariableRecordDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceVariableRecordDAO.java @@ -31,7 +31,6 @@ import com.tencent.bk.job.execute.model.tables.records.TaskInstanceVariableRecord; import org.jooq.Condition; import org.jooq.OrderField; -import org.jooq.Table; import org.jooq.TableField; import java.util.ArrayList; @@ -53,12 +52,7 @@ public class TaskInstanceVariableRecordDAO extends AbstractJobInstanceHotRecordD } public TaskInstanceVariableRecordDAO(DSLContextProvider dslContextProvider) { - super(dslContextProvider, TABLE.getName()); - } - - @Override - public Table getTable() { - return TABLE; + super(dslContextProvider, TABLE); } @Override diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceArchiver.java index 6ffff81e9a..2227c2d7f4 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceArchiver.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceArchiver.java @@ -26,14 +26,14 @@ import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; -import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.JobInstanceHotRecordDAO; public class TaskInstanceArchiver extends AbstractJobInstanceSubTableArchiver { public TaskInstanceArchiver( JobInstanceColdDAO jobInstanceColdDAO, - TaskInstanceRecordDAO jobInstanceHotRecordDAO, + JobInstanceHotRecordDAO jobInstanceHotRecordDAO, ArchiveTablePropsStorage archiveTablePropsStorage) { super( jobInstanceColdDAO, diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DbDataNode.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DbDataNode.java index 2968bd8240..d2a7116a42 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DbDataNode.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DbDataNode.java @@ -101,11 +101,11 @@ public static DbDataNode fromDataNodeId(String dataNodeId) { } } - public static DbDataNode standaloneDbDatNode() { + public static DbDataNode standaloneDbDataNode() { return new DbDataNode(DbDataNodeTypeEnum.STANDALONE, STANDALONE_DS_NAME, null, null); } - public static DbDataNode shardingDbDatNode(String dataSource, Integer dbIndex, Integer tableIndex) { + public static DbDataNode shardingDbDataNode(String dataSource, Integer dbIndex, Integer tableIndex) { return new DbDataNode(DbDataNodeTypeEnum.SHARDING, dataSource, dbIndex, tableIndex); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/JobInstanceArchiveTaskInfo.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/JobInstanceArchiveTaskInfo.java index 0b3d694717..a19d00c223 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/JobInstanceArchiveTaskInfo.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/JobInstanceArchiveTaskInfo.java @@ -45,19 +45,19 @@ public class JobInstanceArchiveTaskInfo { private DbDataNode dbDataNode; /** - * 归档任务所在天.比如 20240806 + * 归档数据所在天.比如 20240806 */ private Integer day; /** - * 归档任务所在小时。 1-24 + * 归档数据所在小时。 0-23 */ private Integer hour; /** - * 归档任务时间范围-from timestamp + * 归档数据时间范围-from timestamp */ private Long fromTimestamp; /** - * 归档任务时间范围-to timestamp + * 归档数据时间范围-to timestamp */ private Long toTimestamp; /** diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java index 958d66a484..5e7b0f5ceb 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java @@ -36,6 +36,7 @@ import com.tencent.bk.job.backup.archive.dao.impl.GseScriptAgentTaskRecordDAO; import com.tencent.bk.job.backup.archive.dao.impl.GseScriptExecuteObjTaskRecordDAO; import com.tencent.bk.job.backup.archive.dao.impl.GseTaskRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.JobInstanceHotRecordDAO; import com.tencent.bk.job.backup.archive.dao.impl.OperationLogRecordDAO; import com.tencent.bk.job.backup.archive.dao.impl.RollingConfigRecordDAO; import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceConfirmRecordDAO; @@ -45,7 +46,6 @@ import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceScriptRecordDAO; import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceVariableRecordDAO; import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceHostRecordDAO; -import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceRecordDAO; import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceVariableRecordDAO; import com.tencent.bk.job.backup.archive.impl.FileSourceTaskLogArchiver; import com.tencent.bk.job.backup.archive.impl.GseFileAgentTaskArchiver; @@ -100,16 +100,16 @@ public class ArchiveConfiguration { public static class ExecuteDaoAutoConfig { @Bean(name = "taskInstanceRecordDAO") - public TaskInstanceRecordDAO taskInstanceRecordDAO( + public JobInstanceHotRecordDAO taskInstanceRecordDAO( @Qualifier("job-execute-dsl-context-provider") DSLContextProvider dslContextProvider) { log.info("Init TaskInstanceRecordDAO"); - return new TaskInstanceRecordDAO(dslContextProvider); + return new JobInstanceHotRecordDAO(dslContextProvider); } @Bean public TaskInstanceArchiver taskInstanceArchiver( ObjectProvider jobInstanceColdDAOObjectProvider, - TaskInstanceRecordDAO taskInstanceRecordDAO, + JobInstanceHotRecordDAO taskInstanceRecordDAO, ArchiveTablePropsStorage archiveTablePropsStorage ) { return new TaskInstanceArchiver( @@ -476,7 +476,7 @@ public JobInstanceArchiveTaskGenerateLock jobInstanceArchiveTaskGenerateLock(Str @Bean public JobInstanceArchiveTaskGenerator jobInstanceArchiveTaskGenerator( ArchiveTaskService archiveTaskService, - TaskInstanceRecordDAO taskInstanceRecordDAO, + JobInstanceHotRecordDAO taskInstanceRecordDAO, ArchiveProperties archiveProperties, JobInstanceArchiveTaskGenerateLock jobInstanceArchiveTaskGenerateLock) { @@ -492,7 +492,7 @@ public JobInstanceArchiveTaskGenerator jobInstanceArchiveTaskGenerator( @Bean public JobInstanceArchiveTaskScheduler jobInstanceArchiveTaskScheduler( ArchiveTaskService archiveTaskService, - TaskInstanceRecordDAO taskInstanceRecordDAO, + JobInstanceHotRecordDAO taskInstanceRecordDAO, ArchiveProperties archiveProperties, JobInstanceArchiveTaskScheduleLock jobInstanceArchiveTaskScheduleLock, JobInstanceSubTableArchivers jobInstanceSubTableArchivers, diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveProperties.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveProperties.java index 973626becf..f32f5a50f9 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveProperties.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveProperties.java @@ -60,11 +60,6 @@ public class ArchiveProperties { */ private int keepDays = 30; - /** - * 归档数据读取时每次读取的数据量（单个表），服务内存受限时可适当降低该值 - */ - private int readIdStepSize = 1000; - /** * 归档数据写入归档库时每次写入的数据量（单个表），服务内存受限时可适当降低该值 */ @@ -78,7 +73,7 @@ public class ArchiveProperties { /** * 每批次从 db 表中读取的记录数量 */ - private int readRowLimit = 10000; + private int readRowLimit = 1000; private Map tableConfigs; @@ -89,11 +84,6 @@ public class ArchiveProperties { @Data public static class TableConfig { - /** - * 归档数据读取时每次读取的数据量（单个表），服务内存受限时可适当降低该值 - */ - private Integer readIdStepSize; - /** * 归档数据写入归档库时每次写入的数据量（单个表），服务内存受限时可适当降低该值 */ diff --git a/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluatorTest.java b/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluatorTest.java index 2ed1c53b4e..ad47d69c7a 100644 --- a/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluatorTest.java +++ b/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluatorTest.java @@ -42,9 +42,9 @@ class ArchiveDbNodePriorityEvaluatorTest { void testStandaloneDb() { List runningTasks = new ArrayList<>(); - runningTasks.add(genTask(DbDataNode.standaloneDbDatNode(), 20240808, 2)); - runningTasks.add(genTask(DbDataNode.standaloneDbDatNode(), 20240808, 2)); - runningTasks.add(genTask(DbDataNode.standaloneDbDatNode(), 20240808, 3)); + runningTasks.add(genTask(DbDataNode.standaloneDbDataNode(), 20240808, 2)); + runningTasks.add(genTask(DbDataNode.standaloneDbDataNode(), 20240808, 2)); + runningTasks.add(genTask(DbDataNode.standaloneDbDataNode(), 20240808, 3)); Map scheduleTaskCountGroupByDb = new HashMap<>(); scheduleTaskCountGroupByDb.put(DbDataNode.STANDALONE_DS_NAME, 2); @@ -63,9 +63,9 @@ void testShardingDb() { List runningTasks = new ArrayList<>(); Map scheduleTaskCountGroupByDb = new HashMap<>(); - runningTasks.add(genTask(DbDataNode.shardingDbDatNode("ds", 0, 1), 20240808, 1)); - runningTasks.add(genTask(DbDataNode.shardingDbDatNode("ds", 1, 0), 20240808, 1)); - runningTasks.add(genTask(DbDataNode.shardingDbDatNode("ds", 1, 1), 20240808, 1)); + runningTasks.add(genTask(DbDataNode.shardingDbDataNode("ds", 0, 1), 20240808, 1)); + runningTasks.add(genTask(DbDataNode.shardingDbDataNode("ds", 1, 0), 20240808, 1)); + runningTasks.add(genTask(DbDataNode.shardingDbDataNode("ds", 1, 1), 20240808, 1)); scheduleTaskCountGroupByDb.put("ds:0", 2); scheduleTaskCountGroupByDb.put("ds:1", 1); @@ -77,8 +77,8 @@ void testShardingDb() { runningTasks.clear(); scheduleTaskCountGroupByDb.clear(); - runningTasks.add(genTask(DbDataNode.shardingDbDatNode("ds", 0, 1), 20240808, 1)); - runningTasks.add(genTask(DbDataNode.shardingDbDatNode("ds", 1, 1), 20240808, 1)); + runningTasks.add(genTask(DbDataNode.shardingDbDataNode("ds", 0, 1), 20240808, 1)); + runningTasks.add(genTask(DbDataNode.shardingDbDataNode("ds", 1, 1), 20240808, 1)); scheduleTaskCountGroupByDb.put("ds:0", 2); scheduleTaskCountGroupByDb.put("ds:1", 1); diff --git a/support-files/kubernetes/charts/bk-job/values.yaml b/support-files/kubernetes/charts/bk-job/values.yaml index 21a19891a8..32b0ec9b00 100644 --- a/support-files/kubernetes/charts/bk-job/values.yaml +++ b/support-files/kubernetes/charts/bk-job/values.yaml @@ -1233,8 +1233,8 @@ backupConfig: cron: 0 0 4 * * * # 热库中的数据保留时间（天） keeyDays: 30 - # 归档数据读取时每次读取的数据量（单个表），服务内存受限时可适当降低该值 - readIdStepSize: 1000 + # 每批次从 db 表中读取的记录数量 + readRowLimit: 1000 # 归档数据写入归档库时每次写入的数据量（单个表），服务内存受限时可适当降低该值 batchInsertRowSize: 1000 # 每次执行删除的最大行数。为了减少 MySQL 事务时长，DB 性能受限时可适当降低该值 diff --git a/support-files/sql/job-execute/0026_job_execute_20241115-1000_V3.11.2_mysql.sql b/support-files/sql/job-execute/0026_job_execute_20241115-1000_V3.11.2_mysql.sql index e7108faf3f..4cc0edfa3b 100644 --- a/support-files/sql/job-execute/0026_job_execute_20241115-1000_V3.11.2_mysql.sql +++ b/support-files/sql/job-execute/0026_job_execute_20241115-1000_V3.11.2_mysql.sql @@ -14,22 +14,6 @@ BEGIN SET AUTOCOMMIT = 0; SELECT DATABASE() INTO db; - -- task_instance - IF NOT EXISTS(SELECT 1 - FROM information_schema.statistics - WHERE TABLE_SCHEMA = db - AND TABLE_NAME = 'task_instance' - AND INDEX_NAME = 'idx_create_time_id') THEN - ALTER TABLE task_instance ADD INDEX `idx_create_time_id` (`create_time`,`id`); - END IF; - IF EXISTS(SELECT 1 - FROM information_schema.statistics - WHERE TABLE_SCHEMA = db - AND TABLE_NAME = 'task_instance' - AND INDEX_NAME = 'idx_create_time') THEN - ALTER TABLE task_instance DROP INDEX `idx_create_time`; - END IF; - -- task_instance_host IF NOT EXISTS(SELECT 1 FROM information_schema.COLUMNS @@ -38,21 +22,12 @@ BEGIN AND COLUMN_NAME = 'app_id') THEN ALTER TABLE task_instance_host ADD COLUMN `app_id` bigint(20) NOT NULL DEFAULT '0'; END IF; - - -- step_instance IF NOT EXISTS(SELECT 1 FROM information_schema.statistics WHERE TABLE_SCHEMA = db - AND TABLE_NAME = 'step_instance' - AND INDEX_NAME = 'idx_task_instance_id_id') THEN - ALTER TABLE step_instance ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); - END IF; - IF EXISTS(SELECT 1 - FROM information_schema.statistics - WHERE TABLE_SCHEMA = db - AND TABLE_NAME = 'step_instance' - AND INDEX_NAME = 'idx_task_instance_id') THEN - ALTER TABLE step_instance DROP INDEX `idx_task_instance_id`; + AND TABLE_NAME = 'task_instance_host' + AND INDEX_NAME = 'idx_app_id_task_instance_id') THEN + ALTER TABLE task_instance_host ADD INDEX `idx_app_id_task_instance_id` (`app_id`,`task_instance_id`); END IF; -- step_instance_script @@ -67,8 +42,8 @@ BEGIN FROM information_schema.statistics WHERE TABLE_SCHEMA = db AND TABLE_NAME = 'step_instance_script' - AND INDEX_NAME = 'idx_task_instance_id_step_instance_id') THEN - ALTER TABLE step_instance_script ADD INDEX `idx_task_instance_id_step_instance_id` (`task_instance_id`,`step_instance_id`); + AND INDEX_NAME = 'idx_task_instance_id') THEN + ALTER TABLE step_instance_script ADD INDEX `idx_task_instance_id` (`task_instance_id`); END IF; -- step_instance_file @@ -83,8 +58,8 @@ BEGIN FROM information_schema.statistics WHERE TABLE_SCHEMA = db AND TABLE_NAME = 'step_instance_file' - AND INDEX_NAME = 'idx_task_instance_id_step_instance_id') THEN - ALTER TABLE step_instance_file ADD INDEX `idx_task_instance_id_step_instance_id` (`task_instance_id`,`step_instance_id`); + AND INDEX_NAME = 'idx_task_instance_id') THEN + ALTER TABLE step_instance_file ADD INDEX `idx_task_instance_id` (`task_instance_id`); END IF; -- step_instance_confirm @@ -99,8 +74,8 @@ BEGIN FROM information_schema.statistics WHERE TABLE_SCHEMA = db AND TABLE_NAME = 'step_instance_confirm' - AND INDEX_NAME = 'idx_task_instance_id_step_instance_id') THEN - ALTER TABLE step_instance_confirm ADD INDEX `idx_task_instance_id_step_instance_id` (`task_instance_id`,`step_instance_id`); + AND INDEX_NAME = 'idx_task_instance_id') THEN + ALTER TABLE step_instance_confirm ADD INDEX `idx_task_instance_id` (`task_instance_id`); END IF; -- gse_task @@ -115,8 +90,8 @@ BEGIN FROM information_schema.statistics WHERE TABLE_SCHEMA = db AND TABLE_NAME = 'gse_task' - AND INDEX_NAME = 'idx_task_instance_id_id') THEN - ALTER TABLE gse_task ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); + AND INDEX_NAME = 'idx_task_instance_id') THEN + ALTER TABLE gse_task ADD INDEX `idx_task_instance_id` (`task_instance_id`); END IF; -- gse_script_agent_task @@ -131,8 +106,8 @@ BEGIN FROM information_schema.statistics WHERE TABLE_SCHEMA = db AND TABLE_NAME = 'gse_script_agent_task' - AND INDEX_NAME = 'idx_task_instance_id_id') THEN - ALTER TABLE gse_script_agent_task ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); + AND INDEX_NAME = 'idx_task_instance_id') THEN + ALTER TABLE gse_script_agent_task ADD INDEX `idx_task_instance_id` (`task_instance_id`); END IF; -- gse_file_agent_task @@ -147,39 +122,8 @@ BEGIN FROM information_schema.statistics WHERE TABLE_SCHEMA = db AND TABLE_NAME = 'gse_file_agent_task' - AND INDEX_NAME = 'idx_task_instance_id_id') THEN - ALTER TABLE gse_file_agent_task ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); - END IF; - -- gse_script_execute_obj_task - IF NOT EXISTS(SELECT 1 - FROM information_schema.statistics - WHERE TABLE_SCHEMA = db - AND TABLE_NAME = 'gse_script_execute_obj_task' - AND INDEX_NAME = 'idx_task_instance_id_id') THEN - ALTER TABLE gse_script_execute_obj_task ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); - END IF; - IF EXISTS(SELECT 1 - FROM information_schema.statistics - WHERE TABLE_SCHEMA = db - AND TABLE_NAME = 'gse_script_execute_obj_task' - AND INDEX_NAME = 'idx_task_instance_id') THEN - ALTER TABLE gse_script_execute_obj_task DROP INDEX `idx_task_instance_id`; - END IF; - - -- gse_file_execute_obj_task - IF NOT EXISTS(SELECT 1 - FROM information_schema.statistics - WHERE TABLE_SCHEMA = db - AND TABLE_NAME = 'gse_file_execute_obj_task' - AND INDEX_NAME = 'idx_task_instance_id_id') THEN - ALTER TABLE gse_file_execute_obj_task ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); - END IF; - IF EXISTS(SELECT 1 - FROM information_schema.statistics - WHERE TABLE_SCHEMA = db - AND TABLE_NAME = 'gse_file_execute_obj_task' AND INDEX_NAME = 'idx_task_instance_id') THEN - ALTER TABLE gse_file_execute_obj_task DROP INDEX `idx_task_instance_id`; + ALTER TABLE gse_file_agent_task ADD INDEX `idx_task_instance_id` (`task_instance_id`); END IF; -- file_source_task_log @@ -194,17 +138,8 @@ BEGIN FROM information_schema.statistics WHERE TABLE_SCHEMA = db AND TABLE_NAME = 'file_source_task_log' - AND INDEX_NAME = 'idx_task_instance_id_id') THEN - ALTER TABLE file_source_task_log ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); - END IF; - - -- rolling_config - IF NOT EXISTS(SELECT 1 - FROM information_schema.statistics - WHERE TABLE_SCHEMA = db - AND TABLE_NAME = 'operation_log' - AND INDEX_NAME = 'idx_task_instance_id_id') THEN - ALTER TABLE rolling_config ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); + AND INDEX_NAME = 'idx_task_instance_id') THEN + ALTER TABLE file_source_task_log ADD INDEX `idx_task_instance_id` (`task_instance_id`); END IF; -- step_instance_rolling_task @@ -219,56 +154,8 @@ BEGIN FROM information_schema.statistics WHERE TABLE_SCHEMA = db AND TABLE_NAME = 'step_instance_rolling_task' - AND INDEX_NAME = 'idx_task_instance_id_id') THEN - ALTER TABLE step_instance_rolling_task ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); - END IF; - - -- operation_log - IF NOT EXISTS(SELECT 1 - FROM information_schema.statistics - WHERE TABLE_SCHEMA = db - AND TABLE_NAME = 'operation_log' - AND INDEX_NAME = 'idx_task_instance_id_id') THEN - ALTER TABLE operation_log ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); - END IF; - IF EXISTS(SELECT 1 - FROM information_schema.statistics - WHERE TABLE_SCHEMA = db - AND TABLE_NAME = 'operation_log' AND INDEX_NAME = 'idx_task_instance_id') THEN - ALTER TABLE operation_log DROP INDEX `idx_task_instance_id`; - END IF; - - -- task_instance_variable - IF NOT EXISTS(SELECT 1 - FROM information_schema.statistics - WHERE TABLE_SCHEMA = db - AND TABLE_NAME = 'task_instance_variable' - AND INDEX_NAME = 'idx_task_instance_id_id') THEN - ALTER TABLE task_instance_variable ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); - END IF; - IF EXISTS(SELECT 1 - FROM information_schema.statistics - WHERE TABLE_SCHEMA = db - AND TABLE_NAME = 'task_instance_variable' - AND INDEX_NAME = 'idx_task_instance_id') THEN - ALTER TABLE task_instance_variable DROP INDEX `idx_task_instance_id`; - END IF; - - -- step_instance_variable - IF NOT EXISTS(SELECT 1 - FROM information_schema.statistics - WHERE TABLE_SCHEMA = db - AND TABLE_NAME = 'step_instance_variable' - AND INDEX_NAME = 'idx_task_instance_id_id') THEN - ALTER TABLE step_instance_variable ADD INDEX `idx_task_instance_id_id` (`task_instance_id`,`id`); - END IF; - IF EXISTS(SELECT 1 - FROM information_schema.statistics - WHERE TABLE_SCHEMA = db - AND TABLE_NAME = 'step_instance_variable' - AND INDEX_NAME = 'idx_task_instance_id') THEN - ALTER TABLE step_instance_variable DROP INDEX `idx_task_instance_id`; + ALTER TABLE step_instance_rolling_task ADD INDEX `idx_task_instance_id` (`task_instance_id`); END IF; COMMIT; @@ -319,7 +206,7 @@ label:BEGIN SET fromId = minId - 1; WHILE fromId <= maxId DO - SELECT MIN(t.id),MAX(t.id) INTO fromId,endId FROM (SELECT id FROM step_instance WHERE id > fromId ORDER BY id asc LIMIT 1000) t; + SELECT MIN(t.id),MAX(t.id) INTO fromId,endId FROM (SELECT id FROM step_instance WHERE id > fromId AND id <= maxId ORDER BY id asc LIMIT 1000) t; UPDATE file_source_task_log t1 INNER JOIN ( @@ -393,15 +280,6 @@ label:BEGIN SET t1.task_instance_id = tmp.task_instance_id WHERE t1.task_instance_id = 0; - UPDATE step_instance_variable t1 - INNER JOIN ( - SELECT id, task_instance_id - FROM step_instance - WHERE id BETWEEN fromId AND endId) - AS tmp ON t1.step_instance_id = tmp.id - SET t1.task_instance_id = tmp.task_instance_id - WHERE t1.task_instance_id = 0; - COMMIT; SET fromId = endId; @@ -450,7 +328,7 @@ label:BEGIN SET fromId = minId - 1; WHILE fromId <= maxId DO - SELECT MIN(t.id),MAX(t.id) INTO fromId,endId FROM (SELECT id FROM task_instance where id > fromId ORDER BY id asc LIMIT 1000) t; + SELECT MIN(t.id),MAX(t.id) INTO fromId,endId FROM (SELECT id FROM task_instance WHERE id > fromId AND id <= maxId ORDER BY id asc LIMIT 1000) t; UPDATE task_instance_host t1 INNER JOIN ( From 2df90114c9b96b2e4562cd314d01521843bf68e2 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Sat, 23 Nov 2024 13:03:22 +0800 Subject: [PATCH 57/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 归档时区 --- .../JobInstanceArchiveTaskGenerator.java | 36 ++++++++++++++++--- .../archive/util/ArchiveDateTimeUtil.java | 8 ++--- .../job/backup/config/ArchiveProperties.java | 6 ++++ .../archive/util/ArchiveDateTimeUtilTest.java | 3 +- 4 files changed, 43 insertions(+), 10 deletions(-) diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java index a13f170a4a..abe9fc4875 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java @@ -38,7 +38,9 @@ import com.tencent.bk.job.common.util.json.JsonUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import java.time.DateTimeException; import java.time.LocalDateTime; import java.time.ZoneId; import java.util.ArrayList; @@ -58,16 +60,21 @@ public class JobInstanceArchiveTaskGenerator { private final JobInstanceArchiveTaskGenerateLock archiveTaskGenerateLock; + /** + * 归档数据时间范围计算所依据的时区 + */ + private final ZoneId archiveZoneId; + public JobInstanceArchiveTaskGenerator(ArchiveTaskService archiveTaskService, JobInstanceHotRecordDAO taskInstanceRecordDAO, ArchiveProperties archiveProperties, JobInstanceArchiveTaskGenerateLock archiveTaskGenerateLock) { - this.archiveTaskService = archiveTaskService; this.taskInstanceRecordDAO = taskInstanceRecordDAO; this.archiveProperties = archiveProperties; this.archiveTaskGenerateLock = archiveTaskGenerateLock; + archiveZoneId = getArchiveBasedTimeZone(archiveProperties); } @@ -169,7 +176,7 @@ private JobInstanceArchiveTaskInfo buildArchiveTask(ArchiveTaskTypeEnum archiveT int hour = ArchiveDateTimeUtil.computeHour(startDateTime); archiveTask.setDay(day); archiveTask.setHour(hour); - long fromTimestamp = ArchiveDateTimeUtil.toTimestampMillsAtZone(startDateTime, ZoneId.systemDefault()); + long fromTimestamp = ArchiveDateTimeUtil.toTimestampMillsAtZone(startDateTime, archiveZoneId); archiveTask.setFromTimestamp(fromTimestamp); archiveTask.setToTimestamp(fromTimestamp + 1000 * 3600L); archiveTask.setTaskType(archiveTaskType); @@ -189,16 +196,35 @@ private LocalDateTime computeArchiveStartDateTime() { Long minJobCreateTimeMills = taskInstanceRecordDAO.getMinJobInstanceCreateTime(); log.info("Min job create time in db is : {}", minJobCreateTimeMills); startDateTime = ArchiveDateTimeUtil.toHourlyRoundDown( - ArchiveDateTimeUtil.unixTimestampMillToLocalDateTime(minJobCreateTimeMills)); + ArchiveDateTimeUtil.unixTimestampMillToZoneDateTime(minJobCreateTimeMills, archiveZoneId)); } else { // 根据最新的归档任务计算开始 log.info("Compute archive from latest generated archive task: {}", JsonUtils.toJson(latestArchiveTask)); - startDateTime = ArchiveDateTimeUtil.unixTimestampMillToLocalDateTime(latestArchiveTask.getToTimestamp()); + startDateTime = ArchiveDateTimeUtil.unixTimestampMillToZoneDateTime( + latestArchiveTask.getToTimestamp(), archiveZoneId); } return startDateTime; } + /** + * 获取归档数据时间范围计算所依据的时区 + * + * @param archiveProperties 归档配置 + * @return 时区 + */ + private ZoneId getArchiveBasedTimeZone(ArchiveProperties archiveProperties) throws DateTimeException { + ZoneId zoneId; + if (StringUtils.isBlank(archiveProperties.getTimeZone())) { + zoneId = ZoneId.systemDefault(); + log.info("Use system zone as archive base time zone, zoneId: {}", zoneId); + return zoneId; + } + zoneId = ZoneId.of(archiveProperties.getTimeZone()); + log.info("Use configured zone as archive base time zone, zoneId: {}", zoneId); + return zoneId; + } + private boolean isHorizontalShardingEnabled() { return archiveProperties.getTasks().getJobInstance().getDataSourceMode() .equals(DataSourceMode.Constants.HORIZONTAL_SHARDING); @@ -206,7 +232,7 @@ private boolean isHorizontalShardingEnabled() { private LocalDateTime computeArchiveEndTime(int archiveDays) { log.info("Compute archive task generate end time before {} days", archiveDays); - LocalDateTime now = LocalDateTime.now(); + LocalDateTime now = LocalDateTime.now(archiveZoneId); return ArchiveDateTimeUtil.computeStartOfDayBeforeDays(now, archiveDays); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtil.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtil.java index a0e196c96b..66cb706eba 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtil.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtil.java @@ -61,11 +61,11 @@ public static LocalDateTime computeStartOfDayBeforeDays(LocalDateTime now, int b return startOfDay.minusDays(beforeDays); } - public static LocalDateTime unixTimestampMillToLocalDateTime(long unixTimestampMill) { - // 创建一个 Instant 对象，表示从 1970-01-01T00:00:00Z 开始的指定秒数 + public static LocalDateTime unixTimestampMillToZoneDateTime(long unixTimestampMill, ZoneId zoneId) { + // 创建一个 Instant 对象，表示从 1970-01-01T00:00:00Z 开始的指定毫秒数 Instant instant = Instant.ofEpochMilli(unixTimestampMill); - // 将 Instant 对象转换为 当前时区的 LocalDateTime 对象 - return LocalDateTime.ofInstant(instant, ZoneId.systemDefault()); + // 将 Instant 对象转换为 对应时区的 LocalDateTime 对象 + return LocalDateTime.ofInstant(instant, zoneId); } public static LocalDateTime toHourlyRoundDown(LocalDateTime localDateTime) { diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveProperties.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveProperties.java index f32f5a50f9..6737fe2a5d 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveProperties.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveProperties.java @@ -55,11 +55,17 @@ public class ArchiveProperties { * 触发时间 CRON 表达式 */ private String cron; + /** * DB数据保留天数 */ private int keepDays = 30; + /** + * 归档数据时间范围计算所依据的时区，如果不指定默认为系统时区 + */ + private String timeZone; + /** * 归档数据写入归档库时每次写入的数据量（单个表），服务内存受限时可适当降低该值 */ diff --git a/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtilTest.java b/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtilTest.java index 9c6048542b..fc106ac271 100644 --- a/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtilTest.java +++ b/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtilTest.java @@ -66,7 +66,8 @@ void computeStartOfDayBeforeDays() { @Test void unixTimestampMillToLocalDateTime() { - LocalDateTime dateTime = ArchiveDateTimeUtil.unixTimestampMillToLocalDateTime(1732072271000L); + LocalDateTime dateTime = ArchiveDateTimeUtil.unixTimestampMillToZoneDateTime( + 1732072271000L, ZoneId.of("GMT+8")); assertThat(dateTime).isEqualTo(LocalDateTime.of(2024, 11, 20, 11, 11, 11)); } From 659ce74f4d4d1ee964d2b7d5a01019dde1845da8 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Sat, 23 Nov 2024 13:34:24 +0800 Subject: [PATCH 58/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 强制终止归档任务 --- .../AbstractJobInstanceArchiveTask.java | 20 ++++++++++++++++--- .../job/backup/archive/ArchiveTaskWorker.java | 1 + .../archive/JobInstanceArchiveTask.java | 20 ++++++++++++++----- .../JobInstanceArchiveTaskScheduler.java | 4 +--- 4 files changed, 34 insertions(+), 11 deletions(-) diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java index 7f176192b5..07e1eaaf90 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java @@ -97,7 +97,11 @@ public abstract class AbstractJobInstanceArchiveTask> i /** * 任务标识-是否已被任务调度强制终止 */ - private AtomicBoolean forceStoppedByScheduler = new AtomicBoolean(false); + private final AtomicBoolean forceStoppedByScheduler = new AtomicBoolean(false); + /** + * 当前归档线程 + */ + private ArchiveTaskWorker archiveTaskWorker; public AbstractJobInstanceArchiveTask(AbstractJobInstanceMainHotRecordDAO jobInstanceMainRecordDAO, @@ -156,6 +160,16 @@ private void stopTask() { } } + @Override + public void forceStopAtOnce() { + log.info("Force stop archive task at once. taskId: {}", taskId); + forceStoppedByScheduler.set(true); + // 更新归档任务状态为“暂停” + archiveTaskService.updateArchiveTaskSuspendedStatus(archiveTaskInfo); + // 打断当前线程，退出执行 + archiveTaskWorker.interrupt(); + } + @Override public void registerDoneCallback(ArchiveTaskDoneCallback archiveTaskDoneCallback) { this.archiveTaskDoneCallback = archiveTaskDoneCallback; @@ -358,7 +372,7 @@ protected Long extractJobInstanceCreateTime(T record) { } @Override - public JobInstanceArchiveTaskInfo getJobInstanceArchiveTaskInfo() { - return archiveTaskInfo; + public void initArchiveTaskWorker(ArchiveTaskWorker archiveTaskWorker) { + this.archiveTaskWorker = archiveTaskWorker; } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskWorker.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskWorker.java index 309c2cb537..0af099ab8a 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskWorker.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskWorker.java @@ -40,6 +40,7 @@ public class ArchiveTaskWorker extends Thread { public ArchiveTaskWorker(JobInstanceArchiveTask archiveTask, Tracer tracer) { this.tracer = tracer; this.archiveTask = archiveTask; + this.archiveTask.initArchiveTaskWorker(this); this.setName("ArchiveWorker-" + archiveTask.getTaskId()); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTask.java index fbb049435b..c1c7c85e9b 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTask.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTask.java @@ -24,8 +24,6 @@ package com.tencent.bk.job.backup.archive; -import com.tencent.bk.job.backup.archive.model.JobInstanceArchiveTaskInfo; - /** * 作业执行实例归档任务 */ @@ -37,12 +35,22 @@ public interface JobInstanceArchiveTask { void execute(); /** - * 归档任务被终止时候调用 + * 优雅终止任务 * * @param stopCallback 回调 */ void stop(ArchiveTaskStopCallback stopCallback); + /** + * 强制终止 + */ + void forceStopAtOnce(); + + /** + * 注册任务完成回调函数 + * + * @param archiveTaskDoneCallback 回调函数 + */ void registerDoneCallback(ArchiveTaskDoneCallback archiveTaskDoneCallback); /** @@ -51,8 +59,10 @@ public interface JobInstanceArchiveTask { String getTaskId(); /** - * 获取归档任务信息 + * 设置归档任务的 worker 信息 + * + * @param archiveTaskWorker 归档任务执行线程 */ - JobInstanceArchiveTaskInfo getJobInstanceArchiveTaskInfo(); + void initArchiveTaskWorker(ArchiveTaskWorker archiveTaskWorker); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java index 2a553a87bf..df64df00d6 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java @@ -281,10 +281,8 @@ private void stopTasksGraceful() { // 等待任务结束，最多等待 2min boolean isAllTaskStopped = taskCountDownLatch.waitingForAllTasksDone(120); if (!isAllTaskStopped) { - log.info("Force update archive task status to suspended"); for (JobInstanceArchiveTask task : scheduledTasks.values()) { - archiveTaskService.updateArchiveTaskSuspendedStatus(task.getJobInstanceArchiveTaskInfo()); - log.info("Force update archive task status to suspended, taskId: {}", task.getTaskId()); + task.forceStopAtOnce(); } } } From 9cd40b6c7e06f8d4aea4ebfa9acc30a786bd4b43 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Mon, 25 Nov 2024 11:25:00 +0800 Subject: [PATCH 59/59] =?UTF-8?q?perf:=20=E5=88=86=E5=BA=93=E5=88=86?= =?UTF-8?q?=E8=A1=A8=E6=94=B9=E9=80=A0-=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E5=AD=90=E8=A1=A8=E5=8A=A0=E5=85=A5=20task?= =?UTF-8?q?=5Finstance=5Fid=20=E5=AD=97=E6=AE=B5=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 优化 JobInstanceArchiveTaskScheduler StopWatch --- .../bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java index df64df00d6..83338af36c 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java @@ -125,8 +125,8 @@ public void schedule() { return; } - StopWatch watch = new StopWatch("archive-task-schedule"); while (true) { + StopWatch watch = new StopWatch("archive-task-schedule"); boolean locked = false; try { // 获取归档任务调度锁