From 935cc1276faf89e62fdab456a7896130472ac902 Mon Sep 17 00:00:00 2001 From: "xueli.xue" Date: Thu, 27 Apr 2017 16:49:43 +0800 Subject: [PATCH] =?UTF-8?q?=E8=84=9A=E6=9C=AC=E4=BB=BB=E5=8A=A1=E5=AE=9E?= =?UTF-8?q?=E7=8E=B0=EF=BC=9AShell=E5=92=8CPython?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../template/jobinfo/jobinfo.index.ftl | 16 ++-- .../job/core/biz/impl/ExecutorBizImpl.java | 77 ++++----------- .../core/handler/impl/ScriptJobHandler.java | 55 +++++++++++ .../com/xxl/job/core/util/ScriptUtil.java | 93 ++++++++++--------- 4 files changed, 128 insertions(+), 113 deletions(-) create mode 100644 xxl-job-core/src/main/java/com/xxl/job/core/handler/impl/ScriptJobHandler.java diff --git a/xxl-job-admin/src/main/webapp/WEB-INF/template/jobinfo/jobinfo.index.ftl b/xxl-job-admin/src/main/webapp/WEB-INF/template/jobinfo/jobinfo.index.ftl index 2f2a898c..4e152b81 100644 --- a/xxl-job-admin/src/main/webapp/WEB-INF/template/jobinfo/jobinfo.index.ftl +++ b/xxl-job-admin/src/main/webapp/WEB-INF/template/jobinfo/jobinfo.index.ftl @@ -193,17 +193,15 @@ public class DemoGlueJobHandler extends IJobHandler { diff --git a/xxl-job-core/src/main/java/com/xxl/job/core/biz/impl/ExecutorBizImpl.java b/xxl-job-core/src/main/java/com/xxl/job/core/biz/impl/ExecutorBizImpl.java index d4b4c752..2fb40dc9 100644 --- a/xxl-job-core/src/main/java/com/xxl/job/core/biz/impl/ExecutorBizImpl.java +++ b/xxl-job-core/src/main/java/com/xxl/job/core/biz/impl/ExecutorBizImpl.java @@ -9,15 +9,12 @@ import com.xxl.job.core.glue.GlueFactory; import com.xxl.job.core.glue.GlueTypeEnum; import com.xxl.job.core.handler.IJobHandler; import com.xxl.job.core.handler.impl.GlueJobHandler; +import com.xxl.job.core.handler.impl.ScriptJobHandler; import com.xxl.job.core.log.XxlJobFileAppender; import com.xxl.job.core.thread.JobThread; -import com.xxl.job.core.util.ScriptUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; import java.util.Date; /** @@ -85,11 +82,11 @@ public class ExecutorBizImpl implements ExecutorBiz { } else if (GlueTypeEnum.GLUE_GROOVY==GlueTypeEnum.match(triggerParam.getGlueType())) { - // valid exists job thread：change handler or glue timeout, need kill old thread + // valid exists job thread：change handler or gluesource updated, need kill old thread if (jobThread != null && !(jobThread.getHandler() instanceof GlueJobHandler && ((GlueJobHandler) jobThread.getHandler()).getGlueUpdatetime()==triggerParam.getGlueUpdatetime() )) { - // change glue model or glue timeout, kill old job thread + // change glue model or gluesource updated, kill old job thread jobThread.toStop("更换任务模式或JobHandler,终止旧任务线程"); jobThread.interrupt(); XxlJobExecutor.removeJobThread(triggerParam.getJobId()); @@ -107,61 +104,25 @@ public class ExecutorBizImpl implements ExecutorBiz { } jobThread = XxlJobExecutor.registJobThread(triggerParam.getJobId(), new GlueJobHandler(jobHandler, triggerParam.getGlueUpdatetime())); } - } else if (GlueTypeEnum.GLUE_SHELL==GlueTypeEnum.match(triggerParam.getGlueType())) { - - // make path - String scriptPath = XxlJobFileAppender.filePath + "gluesource/"; - String scriptFileName = triggerParam.getJobId() + "_" + triggerParam.getGlueUpdatetime() + ".sh"; - - // valid file - File scriptFile = new File(scriptPath, scriptFileName); - if (!scriptFile.exists()) { - // valid glue source - if (triggerParam.getGlueSource()==null) { - return new ReturnT(ReturnT.FAIL_CODE, "glueSource is null."); - } - - // .../gluesource/ - File scriptPathDir = new File(scriptPath); - if (!scriptPathDir.exists()) { - scriptPathDir.mkdirs(); - } - - // .../gluesource/666-156465656.sh - scriptFile = new File(scriptPath, scriptFileName); - FileOutputStream fos = null; - try { - scriptFile.createNewFile(); - - fos = new FileOutputStream(scriptFile, true); - fos.write(triggerParam.getGlueSource().getBytes("utf-8")); - fos.flush(); - } catch (IOException e) { - logger.error(e.getMessage(), e); - return new ReturnT(ReturnT.FAIL_CODE, e.getMessage()); - } finally { - if (fos != null) { - try { - fos.close(); - } catch (IOException e) { - logger.error(e.getMessage(), e); - } - } - } + } else if (GlueTypeEnum.GLUE_SHELL==GlueTypeEnum.match(triggerParam.getGlueType()) + || GlueTypeEnum.GLUE_PYTHON==GlueTypeEnum.match(triggerParam.getGlueType()) ) { + // valid exists job thread：change script or gluesource updated, need kill old thread + if (jobThread != null && + !(jobThread.getHandler() instanceof ScriptJobHandler + && ((ScriptJobHandler) jobThread.getHandler()).getGlueUpdatetime()==triggerParam.getGlueUpdatetime() )) { + // change glue model or gluesource updated, kill old job thread + jobThread.toStop("更换任务模式或JobHandler,终止旧任务线程"); + jobThread.interrupt(); + XxlJobExecutor.removeJobThread(triggerParam.getJobId()); + jobThread = null; } - // log File - String logFileName = XxlJobFileAppender.makeLogFileName(new Date(triggerParam.getLogDateTim()), triggerParam.getLogId()); - - // run script - ScriptUtil.execToFile("python", scriptFile.getName(), (XxlJobFileAppender.filePath + logFileName) ); - - return ReturnT.FAIL; - } else if (GlueTypeEnum.GLUE_PYTHON==GlueTypeEnum.match(triggerParam.getGlueType())) { - String scriptFilePath = XxlJobFileAppender.filePath + "gluesource/" + triggerParam.getJobId() + "_" + triggerParam.getGlueUpdatetime() + ".py"; - - + // make thread: new or exists invalid + if (jobThread == null) { + ScriptJobHandler scriptJobHandler = new ScriptJobHandler(triggerParam.getJobId(), triggerParam.getGlueUpdatetime(), triggerParam.getGlueSource(), GlueTypeEnum.match(triggerParam.getGlueType())); + jobThread = XxlJobExecutor.registJobThread(triggerParam.getJobId(), scriptJobHandler); + } } else { return new ReturnT(ReturnT.FAIL_CODE, "glueType[" + triggerParam.getGlueType() + "] is not valid."); } diff --git a/xxl-job-core/src/main/java/com/xxl/job/core/handler/impl/ScriptJobHandler.java b/xxl-job-core/src/main/java/com/xxl/job/core/handler/impl/ScriptJobHandler.java new file mode 100644 index 00000000..5c7f38ea --- /dev/null +++ b/xxl-job-core/src/main/java/com/xxl/job/core/handler/impl/ScriptJobHandler.java @@ -0,0 +1,55 @@ +package com.xxl.job.core.handler.impl; + +import com.xxl.job.core.biz.model.ReturnT; +import com.xxl.job.core.glue.GlueTypeEnum; +import com.xxl.job.core.handler.IJobHandler; +import com.xxl.job.core.log.XxlJobFileAppender; +import com.xxl.job.core.util.ScriptUtil; + +/** + * Created by xuxueli on 17/4/27. + */ +public class ScriptJobHandler extends IJobHandler { + + private int jobId; + private long glueUpdatetime; + private String gluesource; + private GlueTypeEnum glueType; + + public ScriptJobHandler(int jobId, long glueUpdatetime, String gluesource, GlueTypeEnum glueType){ + this.jobId = jobId; + this.glueUpdatetime = glueUpdatetime; + this.gluesource = gluesource; + this.glueType = glueType; + } + + public long getGlueUpdatetime() { + return glueUpdatetime; + } + + @Override + public ReturnT execute(String... params) throws Exception { + + // cmd + script-file-name + String cmd = "bash"; + String scriptFileName = null; + if (GlueTypeEnum.GLUE_SHELL == glueType) { + cmd = "bash"; + scriptFileName = XxlJobFileAppender.filePath.concat("gluesource/").concat(String.valueOf(jobId)).concat("_").concat(String.valueOf(glueUpdatetime)).concat(".sh"); + } else if (GlueTypeEnum.GLUE_PYTHON == glueType) { + cmd = "python"; + scriptFileName = XxlJobFileAppender.filePath.concat("gluesource/").concat(String.valueOf(jobId)).concat("_").concat(String.valueOf(glueUpdatetime)).concat(".py"); + } + + // make script file + ScriptUtil.markScriptFile(scriptFileName, gluesource); + + // log file + String logFileName = XxlJobFileAppender.filePath.concat(XxlJobFileAppender.contextHolder.get()); + + // invoke + ScriptUtil.execToFile(cmd, scriptFileName, logFileName); + return ReturnT.SUCCESS; + } + +} diff --git a/xxl-job-core/src/main/java/com/xxl/job/core/util/ScriptUtil.java b/xxl-job-core/src/main/java/com/xxl/job/core/util/ScriptUtil.java index a5518646..580cdb4b 100644 --- a/xxl-job-core/src/main/java/com/xxl/job/core/util/ScriptUtil.java +++ b/xxl-job-core/src/main/java/com/xxl/job/core/util/ScriptUtil.java @@ -1,49 +1,57 @@ package com.xxl.job.core.util; +import com.xxl.job.core.log.XxlJobFileAppender; import org.apache.commons.exec.CommandLine; import org.apache.commons.exec.DefaultExecutor; import org.apache.commons.exec.PumpStreamHandler; import java.io.File; import java.io.FileOutputStream; +import java.io.IOException; /** * 1、内嵌编译器如"PythonInterpreter"无法引用扩展包，因此推荐使用java调用控制台进程方式"Runtime.getRuntime().exec()"来运行脚本(shell或python)； * 2、因为通过java调用控制台进程方式实现，需要保证目标机器PATH路径正确配置对应编译器； * 3、暂时脚本执行日志只能在脚本执行结束后一次性获取，无法保证实时性；因此为确保日志实时性，可改为将脚本打印的日志存储在指定的日志文件上； - * - * 知识点： - * 1、日志输出到日志文件：[>>logfile 2>&1]：将错误输出2以及标准输出1都一起以附加写方式导入logfile文件 - * 2、python 异常输出优先级高于标准输出，体现在Log文件中，因此推荐通过logging方式打日志保持和异常信息一致；否则用prinf日志顺序会错乱 + * 4、python 异常输出优先级高于标准输出，体现在Log文件中，因此推荐通过logging方式打日志保持和异常信息一致；否则用prinf日志顺序会错乱 * * Created by xuxueli on 17/2/25. */ public class ScriptUtil { - private static String pyCmd = "python"; - private static String shllCmd = "bash"; - private static String pyFile = "/Users/xuxueli/workspaces/idea-git-workspace/github/xxl-incubator/xxl-util/src/main/resources/script/pytest.py"; - private static String shellFile = "/Users/xuxueli/workspaces/idea-git-workspace/github/xxl-incubator/xxl-util/src/main/resources/script/shelltest.sh"; - private static String pyLogFile = "/Users/xuxueli/Downloads/tmp/pylog.log"; - private static String shLogFile = "/Users/xuxueli/Downloads/tmp/shlog.log"; - - public static void main(String[] args) { - - String command = pyCmd; - String filename = pyFile; - String logFile = pyLogFile; - if (false) { - command = shllCmd; - filename = shellFile; - logFile = shLogFile; + /** + * make script file + * + * @param scriptFileName + * @param content + * @throws IOException + */ + public static void markScriptFile(String scriptFileName, String content) throws IOException { + // filePath/ + File filePathDir = new File(XxlJobFileAppender.filePath); + if (!filePathDir.exists()) { + filePathDir.mkdirs(); } - execToFile(command, filename, logFile); + // filePath/gluesource/ + File filePathSourceDir = new File(filePathDir, "gluesource"); + if (!filePathSourceDir.exists()) { + filePathSourceDir.mkdirs(); + } - } - - public static File markScriptFile(){ - return null; + // make file, filePath/gluesource/666-123456789.py + FileOutputStream fileOutputStream = null; + try { + fileOutputStream = new FileOutputStream(scriptFileName); + fileOutputStream.write(content.getBytes("UTF-8")); + fileOutputStream.close(); + } catch (Exception e) { + throw e; + }finally{ + if(fileOutputStream != null){ + fileOutputStream.close(); + } + } } /** @@ -58,29 +66,22 @@ public class ScriptUtil { * @param scriptFile * @param logFile */ - public static void execToFile(String command, String scriptFile, String logFile){ - try { - // 标准输出：print （null if watchdog timeout） - // 错误输出：logging + 异常 （still exists if watchdog timeout） - // 标准输出 - FileOutputStream fileOutputStream = new FileOutputStream(logFile); - PumpStreamHandler streamHandler = new PumpStreamHandler(fileOutputStream, fileOutputStream, null); + public static void execToFile(String command, String scriptFile, String logFile) throws IOException { + // 标准输出：print （null if watchdog timeout） + // 错误输出：logging + 异常 （still exists if watchdog timeout） + // 标准输入 + FileOutputStream fileOutputStream = new FileOutputStream(logFile, true); + PumpStreamHandler streamHandler = new PumpStreamHandler(fileOutputStream, fileOutputStream, null); - // command - CommandLine commandline = new CommandLine(command); - commandline.addArgument(scriptFile); + // command + CommandLine commandline = new CommandLine(command); + commandline.addArgument(scriptFile); - // exec - DefaultExecutor exec = new DefaultExecutor(); - exec.setExitValues(null); - exec.setStreamHandler(streamHandler); - int exitValue = exec.execute(commandline); - } catch (Exception e) { - e.printStackTrace(); - } - /*Process process = Runtime.getRuntime().exec(cmdarray); - IOUtils.copy(process.getInputStream(), out); - IOUtils.copy(process.getErrorStream(), out);*/ + // exec + DefaultExecutor exec = new DefaultExecutor(); + exec.setExitValues(null); + exec.setStreamHandler(streamHandler); + int exitValue = exec.execute(commandline); } }