调度中心任务监控线程销毁时,批量对失败任务告警,防止告警信息丢失;
This commit is contained in:
parent
667d486899
commit
edcea479bc
|
@ -987,9 +987,11 @@ Tips: 历史版本(V1.3.x)目前已经Release至稳定版本, 进入维护阶段
|
||||||
- 2、规范项目目录,方便扩展多执行器;
|
- 2、规范项目目录,方便扩展多执行器;
|
||||||
- 3、新增JFinal类型执行器sample示例项目;
|
- 3、新增JFinal类型执行器sample示例项目;
|
||||||
- 4、执行器手动设置IP时将会绑定Host;
|
- 4、执行器手动设置IP时将会绑定Host;
|
||||||
- 5、项目主页搭建,提供中英文文档;
|
- 5、项目主页搭建,提供中英文文档(http://www.xuxueli.com/xxl-job);
|
||||||
- 6、执行器回调线程优化,线程销毁前批量回调队列中数据,防止任务结果丢失;
|
- 6、执行器回调线程销毁前, 批量回调队列中数据,防止任务结果丢失;
|
||||||
- 7、执行器注册线程优化,线程销毁时主动摘除注册机器信息,提高执行器注册的实时性;
|
- 7、执行器注册线程销毁时, 主动摘除注册机器信息,提高执行器注册的实时性;
|
||||||
|
- 8、调度中心任务监控线程销毁时,批量对失败任务告警,防止告警信息丢失;
|
||||||
|
- 9、调度中心API服务:支持API方式触发任务执行;
|
||||||
|
|
||||||
### TODO LIST
|
### TODO LIST
|
||||||
- 1、任务权限管理:执行器为粒度分配权限,核心操作校验权限;
|
- 1、任务权限管理:执行器为粒度分配权限,核心操作校验权限;
|
||||||
|
@ -1002,8 +1004,7 @@ Tips: 历史版本(V1.3.x)目前已经Release至稳定版本, 进入维护阶段
|
||||||
- 8、springboot 和 docker镜像,并且推送docker镜像到中央仓库,更进一步实现产品开箱即用;
|
- 8、springboot 和 docker镜像,并且推送docker镜像到中央仓库,更进一步实现产品开箱即用;
|
||||||
- 9、国际化:调度中心界面。
|
- 9、国际化:调度中心界面。
|
||||||
- 10、执行器摘除:执行器销毁时,主动通知调度中心并摘除对应执行器节点,提高执行器状态感知的时效性。
|
- 10、执行器摘除:执行器销毁时,主动通知调度中心并摘除对应执行器节点,提高执行器状态感知的时效性。
|
||||||
- 11、调度中心API服务:支持API方式触发任务执行;
|
- 11、任务类方法"IJobHandler.execute"的参数类型改为"string",进一步方便参数传递;任务注解和任务类统一并改为"JobHandler"";
|
||||||
- 12、任务参数类型改为string,进一步方便参数传递;
|
|
||||||
|
|
||||||
## 七、其他
|
## 七、其他
|
||||||
|
|
||||||
|
|
|
@ -10,10 +10,9 @@ import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import java.text.MessageFormat;
|
import java.text.MessageFormat;
|
||||||
import java.util.Arrays;
|
import java.util.*;
|
||||||
import java.util.HashSet;
|
import java.util.concurrent.LinkedBlockingQueue;
|
||||||
import java.util.Set;
|
import java.util.concurrent.TimeUnit;
|
||||||
import java.util.concurrent.*;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* job monitor instance
|
* job monitor instance
|
||||||
|
@ -36,28 +35,64 @@ public class JobFailMonitorHelper {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
|
|
||||||
|
// monitor
|
||||||
while (!toStop) {
|
while (!toStop) {
|
||||||
try {
|
try {
|
||||||
logger.debug(">>>>>>>>>>> job monitor beat ... ");
|
|
||||||
Integer jobLogId = JobFailMonitorHelper.instance.queue.take();
|
Integer jobLogId = JobFailMonitorHelper.instance.queue.take();
|
||||||
if (jobLogId != null && jobLogId > 0) {
|
if (jobLogId != null && jobLogId > 0) {
|
||||||
logger.debug(">>>>>>>>>>> job monitor heat success, JobLogId:{}", jobLogId);
|
|
||||||
XxlJobLog log = XxlJobDynamicScheduler.xxlJobLogDao.load(jobLogId);
|
XxlJobLog log = XxlJobDynamicScheduler.xxlJobLogDao.load(jobLogId);
|
||||||
if (log!=null) {
|
if (log!=null) {
|
||||||
if (ReturnT.SUCCESS_CODE==log.getTriggerCode() && log.getHandleCode()==0) {
|
if (ReturnT.SUCCESS_CODE==log.getTriggerCode() && log.getHandleCode()==0) {
|
||||||
// running
|
// job running, wait + again monitor
|
||||||
try {
|
|
||||||
TimeUnit.SECONDS.sleep(10);
|
TimeUnit.SECONDS.sleep(10);
|
||||||
} catch (InterruptedException e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
JobFailMonitorHelper.monitor(jobLogId);
|
JobFailMonitorHelper.monitor(jobLogId);
|
||||||
|
logger.info(">>>>>>>>>>> job monitor, job running, JobLogId:{}", jobLogId);
|
||||||
}
|
}
|
||||||
if (ReturnT.SUCCESS_CODE==log.getTriggerCode() && ReturnT.SUCCESS_CODE==log.getHandleCode()) {
|
if (ReturnT.SUCCESS_CODE==log.getTriggerCode() && ReturnT.SUCCESS_CODE==log.getHandleCode()) {
|
||||||
// pass
|
// job success, pass
|
||||||
|
logger.info(">>>>>>>>>>> job monitor, job success, JobLogId:{}", jobLogId);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (ReturnT.FAIL_CODE == log.getTriggerCode()|| ReturnT.FAIL_CODE==log.getHandleCode()) {
|
if (ReturnT.FAIL_CODE == log.getTriggerCode()|| ReturnT.FAIL_CODE==log.getHandleCode()) {
|
||||||
XxlJobInfo info = XxlJobDynamicScheduler.xxlJobInfoDao.loadById(log.getJobId());
|
// job fail,
|
||||||
|
sendMonitorEmail(log);
|
||||||
|
logger.info(">>>>>>>>>>> job monitor, job fail, JobLogId:{}", jobLogId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error("job monitor error:{}", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// monitor all clear
|
||||||
|
List<Integer> jobLogIdList = new ArrayList<Integer>();
|
||||||
|
int drainToNum = getInstance().queue.drainTo(jobLogIdList);
|
||||||
|
if (jobLogIdList!=null && jobLogIdList.size()>0) {
|
||||||
|
for (Integer jobLogId: jobLogIdList) {
|
||||||
|
XxlJobLog log = XxlJobDynamicScheduler.xxlJobLogDao.load(jobLogId);
|
||||||
|
if (ReturnT.FAIL_CODE == log.getTriggerCode()|| ReturnT.FAIL_CODE==log.getHandleCode()) {
|
||||||
|
// job fail,
|
||||||
|
sendMonitorEmail(log);
|
||||||
|
logger.info(">>>>>>>>>>> job monitor last, job fail, JobLogId:{}", jobLogId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
});
|
||||||
|
monitorThread.setDaemon(true);
|
||||||
|
monitorThread.start();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* send monitor email
|
||||||
|
* @param jobLog
|
||||||
|
*/
|
||||||
|
private void sendMonitorEmail(XxlJobLog jobLog){
|
||||||
|
XxlJobInfo info = XxlJobDynamicScheduler.xxlJobInfoDao.loadById(jobLog.getJobId());
|
||||||
if (info!=null && info.getAlarmEmail()!=null && info.getAlarmEmail().trim().length()>0) {
|
if (info!=null && info.getAlarmEmail()!=null && info.getAlarmEmail().trim().length()>0) {
|
||||||
|
|
||||||
Set<String> emailSet = new HashSet<String>(Arrays.asList(info.getAlarmEmail().split(",")));
|
Set<String> emailSet = new HashSet<String>(Arrays.asList(info.getAlarmEmail().split(",")));
|
||||||
|
@ -69,21 +104,16 @@ public class JobFailMonitorHelper {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.error("job monitor error:{}", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
monitorThread.setDaemon(true);
|
|
||||||
monitorThread.start();
|
|
||||||
}
|
|
||||||
|
|
||||||
public void toStop(){
|
public void toStop(){
|
||||||
toStop = true;
|
toStop = true;
|
||||||
//monitorThread.interrupt();
|
// interrupt and wait
|
||||||
|
monitorThread.interrupt();
|
||||||
|
try {
|
||||||
|
monitorThread.join();
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
logger.error(e.getMessage(), e);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// producer
|
// producer
|
||||||
|
|
|
@ -91,7 +91,13 @@ public class JobRegistryMonitorHelper {
|
||||||
|
|
||||||
public void toStop(){
|
public void toStop(){
|
||||||
toStop = true;
|
toStop = true;
|
||||||
//registryThread.interrupt();
|
// interrupt and wait
|
||||||
|
registryThread.interrupt();
|
||||||
|
try {
|
||||||
|
registryThread.join();
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
logger.error(e.getMessage(), e);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,6 +4,7 @@ import com.xxl.job.admin.controller.JobApiController;
|
||||||
import com.xxl.job.admin.core.model.XxlJobInfo;
|
import com.xxl.job.admin.core.model.XxlJobInfo;
|
||||||
import com.xxl.job.admin.core.model.XxlJobLog;
|
import com.xxl.job.admin.core.model.XxlJobLog;
|
||||||
import com.xxl.job.admin.core.schedule.XxlJobDynamicScheduler;
|
import com.xxl.job.admin.core.schedule.XxlJobDynamicScheduler;
|
||||||
|
import com.xxl.job.admin.core.trigger.XxlJobTrigger;
|
||||||
import com.xxl.job.admin.dao.XxlJobInfoDao;
|
import com.xxl.job.admin.dao.XxlJobInfoDao;
|
||||||
import com.xxl.job.admin.dao.XxlJobLogDao;
|
import com.xxl.job.admin.dao.XxlJobLogDao;
|
||||||
import com.xxl.job.admin.dao.XxlJobRegistryDao;
|
import com.xxl.job.admin.dao.XxlJobRegistryDao;
|
||||||
|
@ -124,4 +125,11 @@ public class AdminBizImpl implements AdminBiz {
|
||||||
return ReturnT.SUCCESS;
|
return ReturnT.SUCCESS;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ReturnT<String> triggerJob(int jobId) {
|
||||||
|
// TODO (thread queue trigger)
|
||||||
|
|
||||||
|
return ReturnT.SUCCESS;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -14,26 +14,37 @@ import org.junit.Test;
|
||||||
*/
|
*/
|
||||||
public class AdminBizTest {
|
public class AdminBizTest {
|
||||||
|
|
||||||
|
// admin-client
|
||||||
|
private static String addressUrl = "http://127.0.0.1:8080/xxl-job-admin".concat(AdminBiz.MAPPING);
|
||||||
|
private static String accessToken = null;
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void registryTest() throws Exception {
|
public void registryTest() throws Exception {
|
||||||
|
|
||||||
// admin-client
|
|
||||||
String addressUrl = "http://127.0.0.1:8080/xxl-job-admin".concat(AdminBiz.MAPPING);
|
|
||||||
String accessToken = null;
|
|
||||||
AdminBiz adminBiz = (AdminBiz) new NetComClientProxy(AdminBiz.class, addressUrl, accessToken).getObject();
|
AdminBiz adminBiz = (AdminBiz) new NetComClientProxy(AdminBiz.class, addressUrl, accessToken).getObject();
|
||||||
|
|
||||||
// test executor registry
|
// test executor registry
|
||||||
RegistryParam registryParam = new RegistryParam(RegistryConfig.RegistType.EXECUTOR.name(), "xxl-job-executor-example", "127.0.0.1:9999");
|
RegistryParam registryParam = new RegistryParam(RegistryConfig.RegistType.EXECUTOR.name(), "xxl-job-executor-example", "127.0.0.1:9999");
|
||||||
ReturnT<String> returnT = adminBiz.registry(registryParam);
|
ReturnT<String> returnT = adminBiz.registry(registryParam);
|
||||||
Assert.assertTrue(returnT.getCode() == ReturnT.SUCCESS_CODE);
|
Assert.assertTrue(returnT.getCode() == ReturnT.SUCCESS_CODE);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void registryRemove() throws Exception {
|
||||||
|
AdminBiz adminBiz = (AdminBiz) new NetComClientProxy(AdminBiz.class, addressUrl, accessToken).getObject();
|
||||||
|
|
||||||
// test executor registry remove
|
// test executor registry remove
|
||||||
registryParam = new RegistryParam(RegistryConfig.RegistType.EXECUTOR.name(), "xxl-job-executor-example", "127.0.0.1:9999");
|
RegistryParam registryParam = new RegistryParam(RegistryConfig.RegistType.EXECUTOR.name(), "xxl-job-executor-example", "127.0.0.1:9999");
|
||||||
returnT = adminBiz.registryRemove(registryParam);
|
ReturnT<String> returnT = adminBiz.registryRemove(registryParam);
|
||||||
Assert.assertTrue(returnT.getCode() == ReturnT.SUCCESS_CODE);
|
Assert.assertTrue(returnT.getCode() == ReturnT.SUCCESS_CODE);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void triggerJob() throws Exception {
|
||||||
|
AdminBiz adminBiz = (AdminBiz) new NetComClientProxy(AdminBiz.class, addressUrl, accessToken).getObject();
|
||||||
|
|
||||||
|
int jobId = 1;
|
||||||
|
ReturnT<String> returnT = adminBiz.triggerJob(1);
|
||||||
|
Assert.assertTrue(returnT.getCode() == ReturnT.SUCCESS_CODE);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -37,4 +37,13 @@ public interface AdminBiz {
|
||||||
*/
|
*/
|
||||||
public ReturnT<String> registryRemove(RegistryParam registryParam);
|
public ReturnT<String> registryRemove(RegistryParam registryParam);
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* trigger job for once
|
||||||
|
*
|
||||||
|
* @param jobId
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public ReturnT<String> triggerJob(int jobId);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue