perf: 分库分表改造前置准备 #2991 #2994
perf: 分库分表改造前置准备 #2991 #2994
Conversation
| */ | ||
| SHARDING(1); | ||
|
|
||
| private final int value; |
There was a problem hiding this comment.
这个枚举类的value并不会作为单独的字段写入DB，没有太大的实际意义，建议去掉value直接使用name()，拼接到DataNodeId中语义更清晰，避免排查问题时做一次value->含义的思维转换。
There was a problem hiding this comment.
会存储到 archive_task.data_node 字段，该字段上面会创建索引。基于性能考虑，用这种简短的表示比较好一些。
| */ | ||
| private Integer day; | ||
| /** | ||
| * 归档任务所在小时。 1-24 |
There was a problem hiding this comment.
此处描述与代码实现不一致，建议修改为：
归档数据所在小时，0-23
| private DbDataNode dbDataNode; | ||
|
|
||
| /** | ||
| * 归档任务所在天.比如 20240806 |
There was a problem hiding this comment.
建议修改为：
归档数据所在天，比如 20240806
| */ | ||
| private Integer hour; | ||
| /** | ||
| * 归档任务时间范围-from timestamp |
There was a problem hiding this comment.
描述的对象是待归档数据而非归档任务本身，建议将“任务”修改为“数据”
| */ | ||
| private Long fromTimestamp; | ||
| /** | ||
| * 归档任务时间范围-to timestamp |
There was a problem hiding this comment.
描述的对象是待归档数据而非归档任务本身，建议将“任务”修改为“数据”
| } | ||
|
|
||
| /** | ||
| * 从热 db 读取作业实例熟悉，按照时间+ID 的顺序排序 |
| boolean backupEnabled = isBackupEnable(); | ||
| boolean deleteEnabled = isDeleteEnable(); | ||
|
|
||
| int readLimit = 1000; |
There was a problem hiding this comment.
读取批量大小在这里写死的，是否应该使用ArchiveProperties.readRowLimit及相关子表配置项？
| List<Long> jobInstanceIds = | ||
| jobInstanceRecords.stream().map(this::extractJobInstanceId).collect(Collectors.toList()); | ||
| // 备份主表数据 | ||
| jobInstanceColdDAO.batchInsert(jobInstanceRecords, 1000); |
There was a problem hiding this comment.
读取批量大小在这里写死的，是否应该使用ArchiveProperties.readRowLimit及相关子表配置项？
| import java.util.Map; | ||
|
|
||
| @Getter | ||
| @Setter | ||
| @ToString | ||
| @ConfigurationProperties(prefix = "job.backup.archive.execute") | ||
| public class ArchiveDBProperties { | ||
| public class ArchiveProperties { |
There was a problem hiding this comment.
该类中的readIdStepSize配置项已不再被使用，是否可以删除？
| /** | ||
| * 组件是否正在运行(用于 Spring Lifecycle isRunning 判断) | ||
| */ | ||
| private volatile boolean running = false; |
There was a problem hiding this comment.
active与running变量的初始值、赋值位置完全一致，建议合并为同一个。
| public void waitingForAllTasksDone() { | ||
| try { | ||
| log.info("Waiting for all tasks done! total: {}", latch.getCount()); | ||
| this.latch.await(); |
| if (highestPriorityDbNodeTasksInfo.getRunningTaskCount() >= taskConcurrent) { | ||
| // 休眠5分钟，等待并行任务减少 | ||
| log.info("Running archive task count exceed concurrent limit : {}, wait 300s", taskConcurrent); | ||
| ThreadUtils.sleep(1000 * 60L); |
| } | ||
| } | ||
| } | ||
| } finally { |
| } else { | ||
| log.info("No new archive tasks are generated"); | ||
| } | ||
| } finally { |
wangyu096
force-pushed
the
issue_2991
branch
from
a0ef8a8 to
d4ef3aa
Compare
| } | ||
| } | ||
|
|
||
| public static DbDataNode standaloneDbDatNode() { |
There was a problem hiding this comment.
是不是少了个a，DatNode->DataNode
| return new DbDataNode(DbDataNodeTypeEnum.STANDALONE, STANDALONE_DS_NAME, null, null); | ||
| } | ||
|
|
||
| public static DbDataNode shardingDbDatNode(String dataSource, Integer dbIndex, Integer tableIndex) { |
There was a problem hiding this comment.
是不是少了个a，DatNode->DataNode
优化 JobInstanceArchiveTaskScheduler StopWatch
背景
在 Job 支持分库分表之前，需要做一些准备工作，比如对所有表添加分片键并更新数据等。
分库分表整体设计
详细设计说明
作业相关的表加入 task_instance_id 作为分片键
常规的 DAO 、Service 层方法添加 task_instance_id 参数，可不用细看。重点需要 Review的代码如下：
使用分布式主键框架美团 leaf 替换 MySQL 自增长主键
重点 review 代码：
执行历史归档改造