ak
/
xx-job
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

调度中心移除SQL中的 "now()" 函数；集群部署时不再依赖DB时钟，仅需要保证调度中心应用节点时钟一致即可；

This commit is contained in:
xuxueli 2019-11-19 18:22:00 +08:00
parent a375ebd92e
commit 71cf3f611f
15 changed files with 67 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
doc/XXL-JOB官方文档.md
View File

@ -475,7 +475,6 @@ XXL-JOB是一个轻量级分布式任务调度平台，其核心设计目标是
调度中心集群部署时，几点要求和建议：
- DB配置保持一致；
- 登陆账号配置保持一致；
- 集群机器时钟保持一致（单机集群忽视）；
- 建议：推荐通过nginx为调度中心集群做负载均衡，分配域名。调度中心访问、执行器回调配置、调用API服务等操作均通过该域名进行。
@ -1588,10 +1587,11 @@ Tips: 历史版本(V1.3.x)目前已经Release至稳定版本, 进入维护阶段
- 16、任务告警组件分页参数无效问题修复；
- 17、DB脚本默认编码改为utf8mb4，修复字符乱码问题(建议Mysql版本5.7+)；
- 18、调度中心任务平均分配，触发组件每次获取与线程池数量相关数量的任务，避免大量任务集中在单个调度中心集群节点；
- 19、[ING]xxl-rpc服务端线程优化，降低线程内存开销；
- 20、[ING]调度日志优化：支持设置日志保留天数，过期日志天维度记录报表，并清理；调度报表汇总实时数据和报表；
- 21、[ING]调度中心日志删除，改为分页获取ID，根据ID删除的方式；
- 22、[ING]任务回调改为restful方式；
- 19、调度中心移除SQL中的 "now()" 函数；集群部署时不再依赖DB时钟，仅需要保证调度中心应用节点时钟一致即可；
- 20、[ING]xxl-rpc服务端线程优化，降低线程内存开销；
- 21、[ING]调度日志优化：支持设置日志保留天数，过期日志天维度记录报表，并清理；调度报表汇总实时数据和报表；
- 22、[ING]调度中心日志删除，改为分页获取ID，根据ID删除的方式；
- 23、[ING]任务回调改为restful方式；
### TODO LIST

6
doc/db/tables_xxl_job.sql
View File

@ -59,8 +59,8 @@ CREATE TABLE `xxl_job_logglue` (
`glue_type` varchar(50) DEFAULT NULL COMMENT 'GLUE类型',
`glue_source` mediumtext COMMENT 'GLUE源代码',
`glue_remark` varchar(128) NOT NULL COMMENT 'GLUE备注',
`add_time` timestamp NULL DEFAULT NULL,
`update_time` timestamp NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP,
`add_time` datetime DEFAULT NULL,
`update_time` datetime DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
@ -69,7 +69,7 @@ CREATE TABLE `xxl_job_registry` (
`registry_group` varchar(50) NOT NULL,
`registry_key` varchar(255) NOT NULL,
`registry_value` varchar(255) NOT NULL,
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
`update_time` datetime DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `i_g_k_v` (`registry_group`,`registry_key`,`registry_value`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

5
xxl-job-admin/src/main/java/com/xxl/job/admin/controller/JobCodeController.java
View File

@ -72,6 +72,8 @@ public class JobCodeController {
exists_jobInfo.setGlueSource(glueSource);
exists_jobInfo.setGlueRemark(glueRemark);
exists_jobInfo.setGlueUpdatetime(new Date());
exists_jobInfo.setUpdateTime(new Date());
xxlJobInfoDao.update(exists_jobInfo);
// log old code
@ -80,6 +82,9 @@ public class JobCodeController {
xxlJobLogGlue.setGlueType(exists_jobInfo.getGlueType());
xxlJobLogGlue.setGlueSource(glueSource);
xxlJobLogGlue.setGlueRemark(glueRemark);
xxlJobLogGlue.setAddTime(new Date());
xxlJobLogGlue.setUpdateTime(new Date());
xxlJobLogGlueDao.save(xxlJobLogGlue);
// remove code backup more than 30

7
xxl-job-admin/src/main/java/com/xxl/job/admin/controller/JobGroupController.java
View File

@ -14,10 +14,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.*;
/**
* job group controller
@ -119,7 +116,7 @@ public class JobGroupController {
private List<String> findRegistryByAppName(String appNameParam){
HashMap<String, List<String>> appAddressMap = new HashMap<String, List<String>>();
List<XxlJobRegistry> list = xxlJobRegistryDao.findAll(RegistryConfig.DEAD_TIMEOUT);
List<XxlJobRegistry> list = xxlJobRegistryDao.findAll(RegistryConfig.DEAD_TIMEOUT, new Date());
if (list != null) {
for (XxlJobRegistry item: list) {
if (RegistryConfig.RegistType.EXECUTOR.name().equals(item.getRegistryGroup())) {

14
xxl-job-admin/src/main/java/com/xxl/job/admin/core/model/XxlJobLogGlue.java
View File

@ -1,5 +1,7 @@
package com.xxl.job.admin.core.model;
import java.util.Date;
/**
* xxl-job log for glue, used to track job code process
* @author xuxueli 2016-5-19 17:57:46
@ -11,8 +13,8 @@ public class XxlJobLogGlue {
private String glueType; // GLUE类型 #com.xxl.job.core.glue.GlueTypeEnum
private String glueSource;
private String glueRemark;
private String addTime;
private String updateTime;
private Date addTime;
private Date updateTime;
public int getId() {
return id;
@ -54,19 +56,19 @@ public class XxlJobLogGlue {
this.glueRemark = glueRemark;
}
public String getAddTime() {
public Date getAddTime() {
return addTime;
}
public void setAddTime(String addTime) {
public void setAddTime(Date addTime) {
this.addTime = addTime;
}
public String getUpdateTime() {
public Date getUpdateTime() {
return updateTime;
}
public void setUpdateTime(String updateTime) {
public void setUpdateTime(Date updateTime) {
this.updateTime = updateTime;
}

9
xxl-job-admin/src/main/java/com/xxl/job/admin/core/thread/JobRegistryMonitorHelper.java
View File

@ -7,10 +7,7 @@ import com.xxl.job.core.enums.RegistryConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.*;
import java.util.concurrent.TimeUnit;
/**
@ -38,14 +35,14 @@ public class JobRegistryMonitorHelper {
if (groupList!=null && !groupList.isEmpty()) {
// remove dead address (admin/executor)
List<Integer> ids = XxlJobAdminConfig.getAdminConfig().getXxlJobRegistryDao().findDead(RegistryConfig.DEAD_TIMEOUT);
List<Integer> ids = XxlJobAdminConfig.getAdminConfig().getXxlJobRegistryDao().findDead(RegistryConfig.DEAD_TIMEOUT, new Date());
if (ids!=null && ids.size()>0) {
XxlJobAdminConfig.getAdminConfig().getXxlJobRegistryDao().removeDead(ids);
}
// fresh online address (admin/executor)
HashMap<String, List<String>> appAddressMap = new HashMap<String, List<String>>();
List<XxlJobRegistry> list = XxlJobAdminConfig.getAdminConfig().getXxlJobRegistryDao().findAll(RegistryConfig.DEAD_TIMEOUT);
List<XxlJobRegistry> list = XxlJobAdminConfig.getAdminConfig().getXxlJobRegistryDao().findAll(RegistryConfig.DEAD_TIMEOUT, new Date());
if (list != null) {
for (XxlJobRegistry item: list) {
if (RegistryConfig.RegistType.EXECUTOR.name().equals(item.getRegistryGroup())) {

13
xxl-job-admin/src/main/java/com/xxl/job/admin/dao/XxlJobRegistryDao.java
View File

@ -4,6 +4,7 @@ import com.xxl.job.admin.core.model.XxlJobRegistry;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import java.util.Date;
import java.util.List;
/**
@ -12,19 +13,23 @@ import java.util.List;
@Mapper
public interface XxlJobRegistryDao {
public List<Integer> findDead(@Param("timeout") int timeout);
public List<Integer> findDead(@Param("timeout") int timeout,
@Param("nowTime") Date nowTime);
public int removeDead(@Param("ids") List<Integer> ids);
public List<XxlJobRegistry> findAll(@Param("timeout") int timeout);
public List<XxlJobRegistry> findAll(@Param("timeout") int timeout,
@Param("nowTime") Date nowTime);
public int registryUpdate(@Param("registryGroup") String registryGroup,
@Param("registryKey") String registryKey,
@Param("registryValue") String registryValue);
@Param("registryValue") String registryValue,
@Param("updateTime") Date updateTime);
public int registrySave(@Param("registryGroup") String registryGroup,
@Param("registryKey") String registryKey,
@Param("registryValue") String registryValue);
@Param("registryValue") String registryValue,
@Param("updateTime") Date updateTime);
public int registryDelete(@Param("registryGroup") String registGroup,
@Param("registryKey") String registryKey,

4
xxl-job-admin/src/main/java/com/xxl/job/admin/service/impl/AdminBizImpl.java
View File

@ -126,9 +126,9 @@ public class AdminBizImpl implements AdminBiz {
@Override
public ReturnT<String> registry(RegistryParam registryParam) {
int ret = xxlJobRegistryDao.registryUpdate(registryParam.getRegistGroup(), registryParam.getRegistryKey(), registryParam.getRegistryValue());
int ret = xxlJobRegistryDao.registryUpdate(registryParam.getRegistGroup(), registryParam.getRegistryKey(), registryParam.getRegistryValue(), new Date());
if (ret < 1) {
xxlJobRegistryDao.registrySave(registryParam.getRegistGroup(), registryParam.getRegistryKey(), registryParam.getRegistryValue());
xxlJobRegistryDao.registrySave(registryParam.getRegistGroup(), registryParam.getRegistryKey(), registryParam.getRegistryValue(), new Date());
// fresh
freshGroupRegistryInfo(registryParam);

7
xxl-job-admin/src/main/java/com/xxl/job/admin/service/impl/XxlJobServiceImpl.java
View File

@ -117,6 +117,9 @@ public class XxlJobServiceImpl implements XxlJobService {
}
// add in db
jobInfo.setAddTime(new Date());
jobInfo.setUpdateTime(new Date());
jobInfo.setGlueUpdatetime(new Date());
xxlJobInfoDao.save(jobInfo);
if (jobInfo.getId() < 1) {
return new ReturnT<String>(ReturnT.FAIL_CODE, (I18nUtil.getString("jobinfo_field_add")+I18nUtil.getString("system_fail")) );
@ -220,6 +223,8 @@ public class XxlJobServiceImpl implements XxlJobService {
exists_jobInfo.setExecutorFailRetryCount(jobInfo.getExecutorFailRetryCount());
exists_jobInfo.setChildJobId(jobInfo.getChildJobId());
exists_jobInfo.setTriggerNextTime(nextTriggerTime);
exists_jobInfo.setUpdateTime(new Date());
xxlJobInfoDao.update(exists_jobInfo);
@ -260,6 +265,7 @@ public class XxlJobServiceImpl implements XxlJobService {
xxlJobInfo.setTriggerLastTime(0);
xxlJobInfo.setTriggerNextTime(nextTriggerTime);
xxlJobInfo.setUpdateTime(new Date());
xxlJobInfoDao.update(xxlJobInfo);
return ReturnT.SUCCESS;
}
@ -272,6 +278,7 @@ public class XxlJobServiceImpl implements XxlJobService {
xxlJobInfo.setTriggerLastTime(0);
xxlJobInfo.setTriggerNextTime(0);
xxlJobInfo.setUpdateTime(new Date());
xxlJobInfoDao.update(xxlJobInfo);
return ReturnT.SUCCESS;
}

8
xxl-job-admin/src/main/resources/mybatis-mapper/XxlJobInfoMapper.xml
View File

@ -133,8 +133,8 @@
#{jobGroup},
#{jobCron},
#{jobDesc},
NOW(),
NOW(),
#{addTime},
#{updateTime},
#{author},
#{alarmEmail},
#{executorRouteStrategy},
@ -146,7 +146,7 @@
#{glueType},
#{glueSource},
#{glueRemark},
NOW(),
#{glueUpdatetime},
#{childJobId},
#{triggerStatus},
#{triggerLastTime},
@ -170,7 +170,7 @@
job_group = #{jobGroup},
job_cron = #{jobCron},
job_desc = #{jobDesc},
update_time = NOW(),
update_time = #{updateTime},
author = #{author},
alarm_email = #{alarmEmail},
executor_route_strategy = #{executorRouteStrategy},

4
xxl-job-admin/src/main/resources/mybatis-mapper/XxlJobLogGlueMapper.xml
View File

@ -36,8 +36,8 @@
#{glueType},
#{glueSource},
#{glueRemark},
now(),
now()
#{addTime},
#{updateTime}
);
<!--<selectKey resultType="java.lang.Integer" order="AFTER" keyProperty="id">
SELECT LAST_INSERT_ID()

8
xxl-job-admin/src/main/resources/mybatis-mapper/XxlJobRegistryMapper.xml
View File

@ -22,7 +22,7 @@
<select id="findDead" parameterType="java.lang.Integer" resultType="java.lang.Integer" >
SELECT t.id
FROM xxl_job_registry AS t
WHERE t.update_time <![CDATA[ < ]]> DATE_ADD(NOW(),INTERVAL -#{timeout} SECOND)
WHERE t.update_time <![CDATA[ < ]]> DATE_ADD(#{nowTime},INTERVAL -#{timeout} SECOND)
</select>
<delete id="removeDead" parameterType="java.lang.Integer" >
@ -36,12 +36,12 @@
<select id="findAll" parameterType="java.lang.Integer" resultMap="XxlJobRegistry">
SELECT <include refid="Base_Column_List" />
FROM xxl_job_registry AS t
WHERE t.update_time <![CDATA[ > ]]> DATE_ADD(NOW(),INTERVAL -#{timeout} SECOND)
WHERE t.update_time <![CDATA[ > ]]> DATE_ADD(#{nowTime},INTERVAL -#{timeout} SECOND)
</select>
<update id="registryUpdate" >
UPDATE xxl_job_registry
SET `update_time` = NOW()
SET `update_time` = #{updateTime}
WHERE `registry_group` = #{registryGroup}
AND `registry_key` = #{registryKey}
AND `registry_value` = #{registryValue}
@ -49,7 +49,7 @@
<insert id="registrySave" >
INSERT INTO xxl_job_registry( `registry_group` , `registry_key` , `registry_value`, `update_time`)
VALUES( #{registryGroup} , #{registryKey} , #{registryValue}, NOW())
VALUES( #{registryGroup} , #{registryKey} , #{registryValue}, #{updateTime})
</insert>
<delete id="registryDelete" >

7
xxl-job-admin/src/test/java/com/xxl/job/admin/dao/XxlJobInfoDaoTest.java
View File

@ -4,8 +4,6 @@ import com.xxl.job.admin.core.model.XxlJobInfo;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.context.junit4.SpringRunner;
import javax.annotation.Resource;
@ -47,6 +45,10 @@ public class XxlJobInfoDaoTest {
info.setGlueRemark("setGlueRemark");
info.setChildJobId("1");
info.setAddTime(new Date());
info.setUpdateTime(new Date());
info.setGlueUpdatetime(new Date());
int count = xxlJobInfoDao.save(info);
XxlJobInfo info2 = xxlJobInfoDao.loadById(info.getId());
@ -64,6 +66,7 @@ public class XxlJobInfoDaoTest {
info2.setGlueUpdatetime(new Date());
info2.setChildJobId("1");
info2.setUpdateTime(new Date());
int item2 = xxlJobInfoDao.update(info2);
xxlJobInfoDao.delete(info2.getId());

7
xxl-job-admin/src/test/java/com/xxl/job/admin/dao/XxlJobLogGlueDaoTest.java
View File

@ -1,15 +1,13 @@
package com.xxl.job.admin.dao;
import com.xxl.job.admin.core.model.XxlJobLogGlue;
import com.xxl.job.admin.dao.XxlJobLogGlueDao;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.context.junit4.SpringRunner;
import javax.annotation.Resource;
import java.util.Date;
import java.util.List;
@RunWith(SpringRunner.class)
@ -26,6 +24,9 @@ public class XxlJobLogGlueDaoTest {
logGlue.setGlueType("1");
logGlue.setGlueSource("1");
logGlue.setGlueRemark("1");
logGlue.setAddTime(new Date());
logGlue.setUpdateTime(new Date());
int ret = xxlJobLogGlueDao.save(logGlue);
List<XxlJobLogGlue> list = xxlJobLogGlueDao.findByJobId(1);

7
xxl-job-admin/src/test/java/com/xxl/job/admin/dao/XxlJobRegistryDaoTest.java
View File

@ -8,6 +8,7 @@ import org.springframework.test.context.junit4.SpringRunner;
import javax.annotation.Resource;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
@RunWith(SpringRunner.class)
@ -19,12 +20,12 @@ public class XxlJobRegistryDaoTest {
@Test
public void test(){
int ret = xxlJobRegistryDao.registryUpdate("g1", "k1", "v1");
int ret = xxlJobRegistryDao.registryUpdate("g1", "k1", "v1", new Date());
if (ret < 1) {
ret = xxlJobRegistryDao.registrySave("g1", "k1", "v1");
ret = xxlJobRegistryDao.registrySave("g1", "k1", "v1", new Date());
}
List<XxlJobRegistry> list = xxlJobRegistryDao.findAll(1);
List<XxlJobRegistry> list = xxlJobRegistryDao.findAll(1, new Date());
int ret2 = xxlJobRegistryDao.removeDead(Arrays.asList(1));
}