调度报表优化:新增日志报表的存储表,三天内的任务日志会以每分钟一次的频率异步同步至报表中;任务报表仅读取报表数据,极大提升加载速度;

master
xuxueli 5 years ago
parent 24cc161b96
commit 7ab75aa817
  1. 3
      doc/XXL-JOB官方文档.md
  2. 10
      doc/db/tables_xxl_job.sql
  3. 11
      xxl-job-admin/src/main/java/com/xxl/job/admin/core/conf/XxlJobAdminConfig.java
  4. 54
      xxl-job-admin/src/main/java/com/xxl/job/admin/core/model/XxlJobLogReport.java
  5. 11
      xxl-job-admin/src/main/java/com/xxl/job/admin/core/scheduler/XxlJobScheduler.java
  6. 12
      xxl-job-admin/src/main/java/com/xxl/job/admin/core/thread/JobFailMonitorHelper.java
  7. 121
      xxl-job-admin/src/main/java/com/xxl/job/admin/core/thread/JobLogReportHelper.java
  8. 1
      xxl-job-admin/src/main/java/com/xxl/job/admin/core/thread/JobScheduleHelper.java
  9. 6
      xxl-job-admin/src/main/java/com/xxl/job/admin/dao/XxlJobLogDao.java
  10. 26
      xxl-job-admin/src/main/java/com/xxl/job/admin/dao/XxlJobLogReportDao.java
  11. 53
      xxl-job-admin/src/main/java/com/xxl/job/admin/service/impl/XxlJobServiceImpl.java
  12. 21
      xxl-job-admin/src/main/resources/mybatis-mapper/XxlJobLogMapper.xml
  13. 62
      xxl-job-admin/src/main/resources/mybatis-mapper/XxlJobLogReportMapper.xml
  14. 4
      xxl-job-admin/src/test/java/com/xxl/job/admin/dao/XxlJobLogDaoTest.java
  15. 8
      xxl-job-core/src/main/java/com/xxl/job/core/util/DateUtil.java

@ -1598,7 +1598,8 @@ Tips: 历史版本(V1.3.x)目前已经Release至稳定版本, 进入维护阶段
- 20、xxl-rpc服务端线程优化,降低线程内存开销;
- 21、调度中心回调API服务改为restful方式;
- 22、调度中心日志删除优化,改为分页获取ID并根据ID删除的方式,避免批量删除海量日志导致死锁问题;
- 23、[ING]调度日志优化:支持设置日志保留天数,过期日志天维度记录报表,并清理;调度报表汇总实时数据和报表;
- 23、调度报表优化:新增日志报表的存储表,三天内的任务日志会以每分钟一次的频率异步同步至报表中;任务报表仅读取报表数据,极大提升加载速度;
- 24、[ing]调度日志优化:支持设置日志保留天数,过期日志天维度记录报表,并清理;调度报表汇总实时数据和报表;

@ -53,6 +53,16 @@ CREATE TABLE `xxl_job_log` (
KEY `I_handle_code` (`handle_code`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
CREATE TABLE `xxl_job_log_report` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`trigger_day` datetime DEFAULT NULL COMMENT '调度-时间',
`running_count` int(11) NOT NULL DEFAULT '0' COMMENT '运行中-日志数量',
`suc_count` int(11) NOT NULL DEFAULT '0' COMMENT '执行成功-日志数量',
`fail_count` int(11) NOT NULL DEFAULT '0' COMMENT '执行失败-日志数量',
PRIMARY KEY (`id`),
UNIQUE KEY `i_trigger_day` (`trigger_day`) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
CREATE TABLE `xxl_job_logglue` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`job_id` int(11) NOT NULL COMMENT '任务,主键ID',

@ -1,10 +1,7 @@
package com.xxl.job.admin.core.conf;
import com.xxl.job.admin.core.scheduler.XxlJobScheduler;
import com.xxl.job.admin.dao.XxlJobGroupDao;
import com.xxl.job.admin.dao.XxlJobInfoDao;
import com.xxl.job.admin.dao.XxlJobLogDao;
import com.xxl.job.admin.dao.XxlJobRegistryDao;
import com.xxl.job.admin.dao.*;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Value;
@ -76,6 +73,8 @@ public class XxlJobAdminConfig implements InitializingBean, DisposableBean {
@Resource
private XxlJobGroupDao xxlJobGroupDao;
@Resource
private XxlJobLogReportDao xxlJobLogReportDao;
@Resource
private JavaMailSender mailSender;
@Resource
private DataSource dataSource;
@ -123,6 +122,10 @@ public class XxlJobAdminConfig implements InitializingBean, DisposableBean {
return xxlJobGroupDao;
}
public XxlJobLogReportDao getXxlJobLogReportDao() {
return xxlJobLogReportDao;
}
public JavaMailSender getMailSender() {
return mailSender;
}

@ -0,0 +1,54 @@
package com.xxl.job.admin.core.model;
import java.util.Date;
public class XxlJobLogReport {
private int id;
private Date triggerDay;
private int runningCount;
private int sucCount;
private int failCount;
public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
public Date getTriggerDay() {
return triggerDay;
}
public void setTriggerDay(Date triggerDay) {
this.triggerDay = triggerDay;
}
public int getRunningCount() {
return runningCount;
}
public void setRunningCount(int runningCount) {
this.runningCount = runningCount;
}
public int getSucCount() {
return sucCount;
}
public void setSucCount(int sucCount) {
this.sucCount = sucCount;
}
public int getFailCount() {
return failCount;
}
public void setFailCount(int failCount) {
this.failCount = failCount;
}
}

@ -1,10 +1,7 @@
package com.xxl.job.admin.core.scheduler;
import com.xxl.job.admin.core.conf.XxlJobAdminConfig;
import com.xxl.job.admin.core.thread.JobFailMonitorHelper;
import com.xxl.job.admin.core.thread.JobRegistryMonitorHelper;
import com.xxl.job.admin.core.thread.JobScheduleHelper;
import com.xxl.job.admin.core.thread.JobTriggerPoolHelper;
import com.xxl.job.admin.core.thread.*;
import com.xxl.job.admin.core.util.I18nUtil;
import com.xxl.job.core.biz.ExecutorBiz;
import com.xxl.job.core.enums.ExecutorBlockStrategyEnum;
@ -40,6 +37,9 @@ public class XxlJobScheduler {
// admin trigger pool start
JobTriggerPoolHelper.toStart();
// admin log report start
JobLogReportHelper.getInstance().start();
// start-schedule
JobScheduleHelper.getInstance().start();
@ -52,6 +52,9 @@ public class XxlJobScheduler {
// stop-schedule
JobScheduleHelper.getInstance().toStop();
// admin log report stop
JobLogReportHelper.getInstance().toStop();
// admin trigger pool stop
JobTriggerPoolHelper.toStop();

@ -85,13 +85,21 @@ public class JobFailMonitorHelper {
}
}
TimeUnit.SECONDS.sleep(10);
} catch (Exception e) {
if (!toStop) {
logger.error(">>>>>>>>>>> xxl-job, job fail monitor thread error:{}", e);
}
}
}
try {
TimeUnit.SECONDS.sleep(10);
} catch (Exception e) {
if (!toStop) {
logger.error(e.getMessage(), e);
}
}
}
logger.info(">>>>>>>>>>> xxl-job, job fail monitor thread stop");

@ -0,0 +1,121 @@
package com.xxl.job.admin.core.thread;
import com.xxl.job.admin.core.conf.XxlJobAdminConfig;
import com.xxl.job.admin.core.model.XxlJobLogReport;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Calendar;
import java.util.Date;
import java.util.Map;
import java.util.concurrent.TimeUnit;
/**
* job log report helper
*
* @author xuxueli 2019-11-22
*/
public class JobLogReportHelper {
private static Logger logger = LoggerFactory.getLogger(JobLogReportHelper.class);
private static JobLogReportHelper instance = new JobLogReportHelper();
public static JobLogReportHelper getInstance(){
return instance;
}
private Thread logrThread;
private volatile boolean toStop = false;
public void start(){
logrThread = new Thread(new Runnable() {
@Override
public void run() {
// monitor
while (!toStop) {
try {
// refresh log report in 3 days
for (int i = 0; i < 3; i++) {
// today
Calendar itemDay = Calendar.getInstance();
itemDay.add(Calendar.DAY_OF_MONTH, -i);
itemDay.set(Calendar.HOUR_OF_DAY, 0);
itemDay.set(Calendar.MINUTE, 0);
itemDay.set(Calendar.SECOND, 0);
itemDay.set(Calendar.MILLISECOND, 0);
Date todayFrom = itemDay.getTime();
itemDay.set(Calendar.HOUR_OF_DAY, 23);
itemDay.set(Calendar.MINUTE, 59);
itemDay.set(Calendar.SECOND, 59);
itemDay.set(Calendar.MILLISECOND, 999);
Date todayTo = itemDay.getTime();
// refresh log-report every minute
XxlJobLogReport xxlJobLogReport = new XxlJobLogReport();
xxlJobLogReport.setTriggerDay(todayFrom);
xxlJobLogReport.setRunningCount(0);
xxlJobLogReport.setSucCount(0);
xxlJobLogReport.setFailCount(0);
Map<String, Object> triggerCountMap = XxlJobAdminConfig.getAdminConfig().getXxlJobLogDao().findLogReport(todayFrom, todayTo);
if (triggerCountMap!=null && triggerCountMap.size()>0) {
int triggerDayCount = triggerCountMap.containsKey("triggerDayCount")?Integer.valueOf(String.valueOf(triggerCountMap.get("triggerDayCount"))):0;
int triggerDayCountRunning = triggerCountMap.containsKey("triggerDayCountRunning")?Integer.valueOf(String.valueOf(triggerCountMap.get("triggerDayCountRunning"))):0;
int triggerDayCountSuc = triggerCountMap.containsKey("triggerDayCountSuc")?Integer.valueOf(String.valueOf(triggerCountMap.get("triggerDayCountSuc"))):0;
int triggerDayCountFail = triggerDayCount - triggerDayCountRunning - triggerDayCountSuc;
xxlJobLogReport.setRunningCount(triggerDayCountRunning);
xxlJobLogReport.setSucCount(triggerDayCountSuc);
xxlJobLogReport.setFailCount(triggerDayCountFail);
}
// do refresh
int ret = XxlJobAdminConfig.getAdminConfig().getXxlJobLogReportDao().update(xxlJobLogReport);
if (ret < 1) {
XxlJobAdminConfig.getAdminConfig().getXxlJobLogReportDao().save(xxlJobLogReport);
}
}
} catch (Exception e) {
if (!toStop) {
logger.error(">>>>>>>>>>> xxl-job, job log report thread error:{}", e);
}
}
try {
TimeUnit.MINUTES.sleep(1);
} catch (Exception e) {
if (!toStop) {
logger.error(e.getMessage(), e);
}
}
}
logger.info(">>>>>>>>>>> xxl-job, job log report thread stop");
}
});
logrThread.setDaemon(true);
logrThread.setName("xxl-job, admin JobLogReportHelper");
logrThread.start();
}
public void toStop(){
toStop = true;
// interrupt and wait
logrThread.interrupt();
try {
logrThread.join();
} catch (InterruptedException e) {
logger.error(e.getMessage(), e);
}
}
}

@ -84,6 +84,7 @@ public class JobScheduleHelper {
// time-ring jump
if (nowTime > jobInfo.getTriggerNextTime() + PRE_READ_MS) {
// 2.1、trigger-expire > 5s:pass && make next-trigger-time
logger.warn(">>>>>>>>>>> xxl-job, schedule misfire, jobId = " + jobInfo.getId());
// fresh next
refreshNextValidTime(jobInfo, new Date());

@ -41,10 +41,8 @@ public interface XxlJobLogDao {
public int delete(@Param("jobId") int jobId);
public int triggerCountByHandleCode(@Param("handleCode") int handleCode);
public List<Map<String, Object>> triggerCountByDay(@Param("from") Date from,
@Param("to") Date to);
public Map<String, Object> findLogReport(@Param("from") Date from,
@Param("to") Date to);
public List<Long> findClearLogIds(@Param("jobGroup") int jobGroup,
@Param("jobId") int jobId,

@ -0,0 +1,26 @@
package com.xxl.job.admin.dao;
import com.xxl.job.admin.core.model.XxlJobLogReport;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import java.util.Date;
import java.util.List;
/**
* job log
* @author xuxueli 2019-11-22
*/
@Mapper
public interface XxlJobLogReportDao {
public int save(XxlJobLogReport xxlJobLogReport);
public int update(XxlJobLogReport xxlJobLogReport);
public List<XxlJobLogReport> queryLogReport(@Param("triggerDayFrom") Date triggerDayFrom,
@Param("triggerDayTo") Date triggerDayTo);
public XxlJobLogReport queryLogReportTotal();
}

@ -3,13 +3,11 @@ package com.xxl.job.admin.service.impl;
import com.xxl.job.admin.core.model.XxlJobGroup;
import com.xxl.job.admin.core.model.XxlJobInfo;
import com.xxl.job.admin.core.cron.CronExpression;
import com.xxl.job.admin.core.model.XxlJobLogReport;
import com.xxl.job.admin.core.route.ExecutorRouteStrategyEnum;
import com.xxl.job.admin.core.thread.JobScheduleHelper;
import com.xxl.job.admin.core.util.I18nUtil;
import com.xxl.job.admin.dao.XxlJobGroupDao;
import com.xxl.job.admin.dao.XxlJobInfoDao;
import com.xxl.job.admin.dao.XxlJobLogDao;
import com.xxl.job.admin.dao.XxlJobLogGlueDao;
import com.xxl.job.admin.dao.*;
import com.xxl.job.admin.service.XxlJobService;
import com.xxl.job.core.biz.model.ReturnT;
import com.xxl.job.core.enums.ExecutorBlockStrategyEnum;
@ -40,6 +38,8 @@ public class XxlJobServiceImpl implements XxlJobService {
public XxlJobLogDao xxlJobLogDao;
@Resource
private XxlJobLogGlueDao xxlJobLogGlueDao;
@Resource
private XxlJobLogReportDao xxlJobLogReportDao;
@Override
public Map<String, Object> pageList(int start, int length, int jobGroup, int triggerStatus, String jobDesc, String executorHandler, String author) {
@ -287,8 +287,13 @@ public class XxlJobServiceImpl implements XxlJobService {
public Map<String, Object> dashboardInfo() {
int jobInfoCount = xxlJobInfoDao.findAllCount();
int jobLogCount = xxlJobLogDao.triggerCountByHandleCode(-1);
int jobLogSuccessCount = xxlJobLogDao.triggerCountByHandleCode(ReturnT.SUCCESS_CODE);
int jobLogCount = 0;
int jobLogSuccessCount = 0;
XxlJobLogReport xxlJobLogReport = xxlJobLogReportDao.queryLogReportTotal();
if (xxlJobLogReport != null) {
jobLogCount = xxlJobLogReport.getRunningCount() + xxlJobLogReport.getSucCount() + xxlJobLogReport.getFailCount();
jobLogSuccessCount = xxlJobLogReport.getSucCount();
}
// executor count
Set<String> executorAddressSet = new HashSet<String>();
@ -312,15 +317,8 @@ public class XxlJobServiceImpl implements XxlJobService {
return dashboardMap;
}
private static final String TRIGGER_CHART_DATA_CACHE = "trigger_chart_data_cache";
@Override
public ReturnT<Map<String, Object>> chartInfo(Date startDate, Date endDate) {
/*// get cache
String cacheKey = TRIGGER_CHART_DATA_CACHE + "_" + startDate.getTime() + "_" + endDate.getTime();
Map<String, Object> chartInfo = (Map<String, Object>) LocalCacheUtil.get(cacheKey);
if (chartInfo != null) {
return new ReturnT<Map<String, Object>>(chartInfo);
}*/
// process
List<String> triggerDayList = new ArrayList<String>();
@ -331,14 +329,14 @@ public class XxlJobServiceImpl implements XxlJobService {
int triggerCountSucTotal = 0;
int triggerCountFailTotal = 0;
List<Map<String, Object>> triggerCountMapAll = xxlJobLogDao.triggerCountByDay(startDate, endDate);
if (triggerCountMapAll!=null && triggerCountMapAll.size()>0) {
for (Map<String, Object> item: triggerCountMapAll) {
String day = String.valueOf(item.get("triggerDay"));
int triggerDayCount = Integer.valueOf(String.valueOf(item.get("triggerDayCount")));
int triggerDayCountRunning = Integer.valueOf(String.valueOf(item.get("triggerDayCountRunning")));
int triggerDayCountSuc = Integer.valueOf(String.valueOf(item.get("triggerDayCountSuc")));
int triggerDayCountFail = triggerDayCount - triggerDayCountRunning - triggerDayCountSuc;
List<XxlJobLogReport> logReportList = xxlJobLogReportDao.queryLogReport(startDate, endDate);
if (logReportList!=null && logReportList.size()>0) {
for (XxlJobLogReport item: logReportList) {
String day = DateUtil.formatDate(item.getTriggerDay());
int triggerDayCountRunning = item.getRunningCount();
int triggerDayCountSuc = item.getSucCount();
int triggerDayCountFail = item.getFailCount();
triggerDayList.add(day);
triggerDayCountRunningList.add(triggerDayCountRunning);
@ -350,12 +348,12 @@ public class XxlJobServiceImpl implements XxlJobService {
triggerCountFailTotal += triggerDayCountFail;
}
} else {
for (int i = 4; i > -1; i--) {
triggerDayList.add(DateUtil.formatDate(DateUtil.addDays(new Date(), -i)));
for (int i = -6; i <= 0; i++) {
triggerDayList.add(DateUtil.formatDate(DateUtil.addDays(new Date(), i)));
triggerDayCountRunningList.add(0);
triggerDayCountSucList.add(0);
triggerDayCountFailList.add(0);
}
triggerDayCountSucList.add(0);
triggerDayCountFailList.add(0);
}
}
Map<String, Object> result = new HashMap<String, Object>();
@ -368,9 +366,6 @@ public class XxlJobServiceImpl implements XxlJobService {
result.put("triggerCountSucTotal", triggerCountSucTotal);
result.put("triggerCountFailTotal", triggerCountFailTotal);
/*// set cache
LocalCacheUtil.set(cacheKey, result, 60*1000); // cache 60s*/
return new ReturnT<Map<String, Object>>(result);
}

@ -164,17 +164,7 @@
WHERE job_id = #{jobId}
</delete>
<select id="triggerCountByHandleCode" resultType="int" >
SELECT count(1)
FROM xxl_job_log AS t
<trim prefix="WHERE" prefixOverrides="AND | OR" >
<if test="handleCode gt 0">
AND t.handle_code = #{handleCode}
</if>
</trim>
</select>
<select id="triggerCountByDay" resultType="java.util.Map" >
<!--<select id="triggerCountByDay" resultType="java.util.Map" >
SELECT
DATE_FORMAT(trigger_time,'%Y-%m-%d') triggerDay,
COUNT(handle_code) triggerDayCount,
@ -184,6 +174,15 @@
WHERE trigger_time BETWEEN #{from} and #{to}
GROUP BY triggerDay
ORDER BY triggerDay
</select>-->
<select id="findLogReport" resultType="java.util.Map" >
SELECT
COUNT(handle_code) triggerDayCount,
SUM(CASE WHEN (trigger_code in (0, 200) and handle_code = 0) then 1 else 0 end) as triggerDayCountRunning,
SUM(CASE WHEN handle_code = 200 then 1 else 0 end) as triggerDayCountSuc
FROM xxl_job_log
WHERE trigger_time BETWEEN #{from} and #{to}
</select>
<select id="findClearLogIds" resultType="long" >

@ -0,0 +1,62 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.xxl.job.admin.dao.XxlJobLogReportDao">
<resultMap id="XxlJobLogReport" type="com.xxl.job.admin.core.model.XxlJobLogReport" >
<result column="id" property="id" />
<result column="trigger_day" property="triggerDay" />
<result column="running_count" property="runningCount" />
<result column="suc_count" property="sucCount" />
<result column="fail_count" property="failCount" />
</resultMap>
<sql id="Base_Column_List">
t.id,
t.trigger_day,
t.running_count,
t.suc_count,
t.fail_count
</sql>
<insert id="save" parameterType="com.xxl.job.admin.core.model.XxlJobLogReport" useGeneratedKeys="true" keyProperty="id" >
INSERT INTO xxl_job_log_report (
`trigger_day`,
`running_count`,
`suc_count`,
`fail_count`
) VALUES (
#{triggerDay},
#{runningCount},
#{sucCount},
#{failCount}
);
<!--<selectKey resultType="java.lang.Integer" order="AFTER" keyProperty="id">
SELECT LAST_INSERT_ID()
</selectKey>-->
</insert>
<update id="update" >
UPDATE xxl_job_log_report
SET `running_count` = #{runningCount},
`suc_count` = #{sucCount},
`fail_count` = #{failCount}
WHERE `trigger_day` = #{triggerDay}
</update>
<select id="queryLogReport" resultMap="XxlJobLogReport">
SELECT <include refid="Base_Column_List" />
FROM xxl_job_log_report AS t
WHERE t.trigger_day between #{triggerDayFrom} and #{triggerDayTo}
ORDER BY t.trigger_day ASC
</select>
<select id="queryLogReportTotal" resultMap="XxlJobLogReport">
SELECT
SUM(running_count) running_count,
SUM(suc_count) suc_count,
SUM(fail_count) fail_count
FROM xxl_job_log_report AS t
</select>
</mapper>

@ -9,7 +9,6 @@ import org.springframework.test.context.junit4.SpringRunner;
import javax.annotation.Resource;
import java.util.Date;
import java.util.List;
import java.util.Map;
@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@ -47,13 +46,10 @@ public class XxlJobLogDaoTest {
dto = xxlJobLogDao.load(log.getId());
List<Map<String, Object>> list2 = xxlJobLogDao.triggerCountByDay(new Date(new Date().getTime() + 30*24*60*60*1000), new Date());
List<Long> ret4 = xxlJobLogDao.findClearLogIds(1, 1, new Date(), 100, 100);
int ret2 = xxlJobLogDao.delete(log.getJobId());
int ret3 = xxlJobLogDao.triggerCountByHandleCode(-1);
}
}

@ -123,10 +123,6 @@ public class DateUtil {
// ---------------------- add date ----------------------
public static Date addDays(final Date date, final int amount) {
return add(date, Calendar.DAY_OF_MONTH, amount);
}
public static Date addYears(final Date date, final int amount) {
return add(date, Calendar.YEAR, amount);
}
@ -135,6 +131,10 @@ public class DateUtil {
return add(date, Calendar.MONTH, amount);
}
public static Date addDays(final Date date, final int amount) {
return add(date, Calendar.DAY_OF_MONTH, amount);
}
private static Date add(final Date date, final int calendarField, final int amount) {
if (date == null) {
return null;

Loading…
Cancel
Save