master
xueli.xue 8 years ago
parent 1b3ad6fbbe
commit d2eafe20cd
  1. 7
      xxl-job-core/pom.xml
  2. 60
      xxl-job-core/src/main/java/com/xxl/job/core/biz/impl/ExecutorBizImpl.java
  3. 2
      xxl-job-core/src/main/java/com/xxl/job/core/executor/XxlJobExecutor.java
  4. 27
      xxl-job-core/src/main/java/com/xxl/job/core/glue/GlueFactory.java
  5. 32
      xxl-job-core/src/main/java/com/xxl/job/core/glue/loader/GlueLoader.java
  6. 60
      xxl-job-core/src/main/java/com/xxl/job/core/glue/loader/impl/DbGlueLoader.java
  7. 86
      xxl-job-core/src/main/java/com/xxl/job/core/util/ScriptUtil.java
  8. 7
      xxl-job-executor-example/src/main/resources/applicationcontext-xxl-job.xml

@ -79,6 +79,13 @@
<version>2.4.5</version>
</dependency>
<!-- commons-exec -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-exec</artifactId>
<version>1.3</version>
</dependency>
</dependencies>
</project>

@ -11,9 +11,13 @@ import com.xxl.job.core.handler.IJobHandler;
import com.xxl.job.core.handler.impl.GlueJobHandler;
import com.xxl.job.core.log.XxlJobFileAppender;
import com.xxl.job.core.thread.JobThread;
import com.xxl.job.core.util.ScriptUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Date;
/**
@ -103,7 +107,63 @@ public class ExecutorBizImpl implements ExecutorBiz {
}
jobThread = XxlJobExecutor.registJobThread(triggerParam.getJobId(), new GlueJobHandler(jobHandler, triggerParam.getGlueUpdatetime()));
}
} else if (GlueTypeEnum.GLUE_SHELL==GlueTypeEnum.match(triggerParam.getGlueType())) {
// make path
String scriptPath = XxlJobFileAppender.filePath + "gluesource/";
String scriptFileName = triggerParam.getJobId() + "_" + triggerParam.getGlueUpdatetime() + ".sh";
// valid file
File scriptFile = new File(scriptPath, scriptFileName);
if (!scriptFile.exists()) {
// valid glue source
if (triggerParam.getGlueSource()==null) {
return new ReturnT<String>(ReturnT.FAIL_CODE, "glueSource is null.");
}
// .../gluesource/
File scriptPathDir = new File(scriptPath);
if (!scriptPathDir.exists()) {
scriptPathDir.mkdirs();
}
// .../gluesource/666-156465656.sh
scriptFile = new File(scriptPath, scriptFileName);
FileOutputStream fos = null;
try {
scriptFile.createNewFile();
fos = new FileOutputStream(scriptFile, true);
fos.write(triggerParam.getGlueSource().getBytes("utf-8"));
fos.flush();
} catch (IOException e) {
logger.error(e.getMessage(), e);
return new ReturnT<String>(ReturnT.FAIL_CODE, e.getMessage());
} finally {
if (fos != null) {
try {
fos.close();
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
}
}
}
// log File
String logFileName = XxlJobFileAppender.makeLogFileName(new Date(triggerParam.getLogDateTim()), triggerParam.getLogId());
// run script
ScriptUtil.execToFile("python", scriptFile.getName(), (XxlJobFileAppender.filePath + logFileName) );
return ReturnT.FAIL;
} else if (GlueTypeEnum.GLUE_PYTHON==GlueTypeEnum.match(triggerParam.getGlueType())) {
String scriptFilePath = XxlJobFileAppender.filePath + "gluesource/" + triggerParam.getJobId() + "_" + triggerParam.getGlueUpdatetime() + ".py";
} else {
return new ReturnT<String>(ReturnT.FAIL_CODE, "glueType[" + triggerParam.getGlueType() + "] is not valid.");
}
// push data to queue

@ -78,8 +78,10 @@ public class XxlJobExecutor implements ApplicationContextAware, ApplicationListe
}
// ---------------------------------- init job handler ------------------------------------
public static ApplicationContext applicationContext;
@Override
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
XxlJobExecutor.applicationContext = applicationContext;
// init job handler action
Map<String, Object> serviceBeanMap = applicationContext.getBeansWithAnnotation(JobHander.class);

@ -1,14 +1,12 @@
package com.xxl.job.core.glue;
import com.xxl.job.core.executor.XxlJobExecutor;
import com.xxl.job.core.handler.IJobHandler;
import groovy.lang.GroovyClassLoader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.core.annotation.AnnotationUtils;
import javax.annotation.Resource;
@ -19,7 +17,7 @@ import java.lang.reflect.Modifier;
* glue factory, product class/object by name
* @author xuxueli 2016-1-2 20:02:27
*/
public class GlueFactory implements ApplicationContextAware {
public class GlueFactory {
private static Logger logger = LoggerFactory.getLogger(GlueFactory.class);
/**
@ -28,18 +26,11 @@ public class GlueFactory implements ApplicationContextAware {
private GroovyClassLoader groovyClassLoader = new GroovyClassLoader();
// ----------------------------- spring support -----------------------------
private static ApplicationContext applicationContext;
private static GlueFactory glueFactory;
private static GlueFactory glueFactory = new GlueFactory();
public static GlueFactory getInstance(){
return glueFactory;
}
@Override
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
GlueFactory.applicationContext = applicationContext;
GlueFactory.glueFactory = (GlueFactory) applicationContext.getBean("glueFactory");
}
/**
* inject action of spring
* @param instance
@ -61,21 +52,21 @@ public class GlueFactory implements ApplicationContextAware {
try {
Resource resource = AnnotationUtils.getAnnotation(field, Resource.class);
if (resource.name()!=null && resource.name().length()>0){
fieldBean = applicationContext.getBean(resource.name());
fieldBean = XxlJobExecutor.applicationContext.getBean(resource.name());
} else {
fieldBean = applicationContext.getBean(field.getName());
fieldBean = XxlJobExecutor.applicationContext.getBean(field.getName());
}
} catch (Exception e) {
}
if (fieldBean==null ) {
fieldBean = applicationContext.getBean(field.getType());
fieldBean = XxlJobExecutor.applicationContext.getBean(field.getType());
}
} else if (AnnotationUtils.getAnnotation(field, Autowired.class) != null) {
Qualifier qualifier = AnnotationUtils.getAnnotation(field, Qualifier.class);
if (qualifier!=null && qualifier.value()!=null && qualifier.value().length()>0) {
fieldBean = applicationContext.getBean(qualifier.value());
fieldBean = XxlJobExecutor.applicationContext.getBean(qualifier.value());
} else {
fieldBean = applicationContext.getBean(field.getType());
fieldBean = XxlJobExecutor.applicationContext.getBean(field.getType());
}
}

@ -1,16 +1,16 @@
package com.xxl.job.core.glue.loader;
/**
* code source loader
* @author xuxueli 2016-1-2 20:01:39
*/
public interface GlueLoader {
/**
* load code source by name, ensure every load is the latest.
* @param jobId
* @return code source
*/
public String load(int jobId);
}
//package com.xxl.job.core.glue.loader;
//
///**
// * code source loader
// * @author xuxueli 2016-1-2 20:01:39
// */
//public interface GlueLoader {
//
// /**
// * load code source by name, ensure every load is the latest.
// * @param jobId
// * @return code source
// */
// public String load(int jobId);
//
//}

@ -1,30 +1,30 @@
package com.xxl.job.core.glue.loader.impl;
import com.xxl.job.core.glue.loader.GlueLoader;
import com.xxl.job.core.util.DBUtil;
import javax.sql.DataSource;
import java.util.List;
import java.util.Map;
/**
* Created by xuxueli on 16/9/30.
*/
public class DbGlueLoader implements GlueLoader {
private DataSource dataSource;
public void setDataSource(DataSource dataSource) {
this.dataSource = dataSource;
}
@Override
public String load(int jobId) {
String sql = "SELECT glue_source FROM XXL_JOB_QRTZ_TRIGGER_INFO WHERE id = ?";
List<Map<String, Object>> result = DBUtil.query(dataSource, sql, new Object[]{jobId});
if (result!=null && result.size()==1 && result.get(0)!=null && result.get(0).get("glue_source")!=null ) {
return (String) result.get(0).get("glue_source");
}
return null;
}
}
//package com.xxl.job.core.glue.loader.impl;
//
//import com.xxl.job.core.glue.loader.GlueLoader;
//import com.xxl.job.core.util.DBUtil;
//
//import javax.sql.DataSource;
//import java.util.List;
//import java.util.Map;
//
///**
// * Created by xuxueli on 16/9/30.
// */
//public class DbGlueLoader implements GlueLoader {
//
// private DataSource dataSource;
// public void setDataSource(DataSource dataSource) {
// this.dataSource = dataSource;
// }
//
// @Override
// public String load(int jobId) {
// String sql = "SELECT glue_source FROM XXL_JOB_QRTZ_TRIGGER_INFO WHERE id = ?";
// List<Map<String, Object>> result = DBUtil.query(dataSource, sql, new Object[]{jobId});
// if (result!=null && result.size()==1 && result.get(0)!=null && result.get(0).get("glue_source")!=null ) {
// return (String) result.get(0).get("glue_source");
// }
// return null;
// }
//
//}

@ -0,0 +1,86 @@
package com.xxl.job.core.util;
import org.apache.commons.exec.CommandLine;
import org.apache.commons.exec.DefaultExecutor;
import org.apache.commons.exec.PumpStreamHandler;
import java.io.File;
import java.io.FileOutputStream;
/**
* 1内嵌编译器如"PythonInterpreter"无法引用扩展包因此推荐使用java调用控制台进程方式"Runtime.getRuntime().exec()"来运行脚本(shell或python)
* 2因为通过java调用控制台进程方式实现需要保证目标机器PATH路径正确配置对应编译器
* 3暂时脚本执行日志只能在脚本执行结束后一次性获取无法保证实时性因此为确保日志实时性可改为将脚本打印的日志存储在指定的日志文件上
*
* 知识点
* 1日志输出到日志文件[>>logfile 2>&1]将错误输出2以及标准输出1都一起以附加写方式导入logfile文件
* 2python 异常输出优先级高于标准输出体现在Log文件中因此推荐通过logging方式打日志保持和异常信息一致否则用prinf日志顺序会错乱
*
* Created by xuxueli on 17/2/25.
*/
public class ScriptUtil {
private static String pyCmd = "python";
private static String shllCmd = "bash";
private static String pyFile = "/Users/xuxueli/workspaces/idea-git-workspace/github/xxl-incubator/xxl-util/src/main/resources/script/pytest.py";
private static String shellFile = "/Users/xuxueli/workspaces/idea-git-workspace/github/xxl-incubator/xxl-util/src/main/resources/script/shelltest.sh";
private static String pyLogFile = "/Users/xuxueli/Downloads/tmp/pylog.log";
private static String shLogFile = "/Users/xuxueli/Downloads/tmp/shlog.log";
public static void main(String[] args) {
String command = pyCmd;
String filename = pyFile;
String logFile = pyLogFile;
if (false) {
command = shllCmd;
filename = shellFile;
logFile = shLogFile;
}
execToFile(command, filename, logFile);
}
public static File markScriptFile(){
return null;
}
/**
* 日志文件输出方式
*
* 优点支持将目标数据实时输出到指定日志文件中去
* 缺点
* 标准输出和错误输出优先级固定可能和脚本中顺序不一致
* Java无法实时获取
*
* @param command
* @param scriptFile
* @param logFile
*/
public static void execToFile(String command, String scriptFile, String logFile){
try {
// 标准输出:print (null if watchdog timeout)
// 错误输出:logging + 异常 (still exists if watchdog timeout)
// 标准输出
FileOutputStream fileOutputStream = new FileOutputStream(logFile);
PumpStreamHandler streamHandler = new PumpStreamHandler(fileOutputStream, fileOutputStream, null);
// command
CommandLine commandline = new CommandLine(command);
commandline.addArgument(scriptFile);
// exec
DefaultExecutor exec = new DefaultExecutor();
exec.setExitValues(null);
exec.setStreamHandler(streamHandler);
int exitValue = exec.execute(commandline);
} catch (Exception e) {
e.printStackTrace();
}
/*Process process = Runtime.getRuntime().exec(cmdarray);
IOUtils.copy(process.getInputStream(), out);
IOUtils.copy(process.getErrorStream(), out);*/
}
}

@ -38,14 +38,9 @@
</property>
</bean>
<!-- ********************************* "GlueFactory" 配置, 仅在启动 "GLUE模式任务" 时才需要, 否则可删除 ********************************* -->
<!-- 配置03、GlueFactory -->
<bean id="glueFactory" class="com.xxl.job.core.glue.GlueFactory" />
<!-- ********************************* "XXL-JOB公共数据源" 配置, 仅在启动 "DbRegistHelper" 时才需要, 否则可删除 ********************************* -->
<!-- 配置04、XXL-JOB公共数据源 -->
<!-- 配置03、XXL-JOB公共数据源 -->
<bean id="xxlJobDataSource" class="com.mchange.v2.c3p0.ComboPooledDataSource" destroy-method="close">
<property name="driverClass" value="${xxl.job.db.driverClass}" />
<property name="jdbcUrl" value="${xxl.job.db.url}" />

Loading…
Cancel
Save