|
|
|
|
@ -10,10 +10,9 @@ import org.slf4j.Logger; |
|
|
|
|
import org.slf4j.LoggerFactory; |
|
|
|
|
|
|
|
|
|
import java.text.MessageFormat; |
|
|
|
|
import java.util.Arrays; |
|
|
|
|
import java.util.HashSet; |
|
|
|
|
import java.util.Set; |
|
|
|
|
import java.util.concurrent.*; |
|
|
|
|
import java.util.*; |
|
|
|
|
import java.util.concurrent.LinkedBlockingQueue; |
|
|
|
|
import java.util.concurrent.TimeUnit; |
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
* job monitor instance |
|
|
|
|
@ -36,38 +35,30 @@ public class JobFailMonitorHelper { |
|
|
|
|
|
|
|
|
|
@Override |
|
|
|
|
public void run() { |
|
|
|
|
|
|
|
|
|
// monitor
|
|
|
|
|
while (!toStop) { |
|
|
|
|
try { |
|
|
|
|
logger.debug(">>>>>>>>>>> job monitor beat ... "); |
|
|
|
|
Integer jobLogId = JobFailMonitorHelper.instance.queue.take(); |
|
|
|
|
if (jobLogId != null && jobLogId > 0) { |
|
|
|
|
logger.debug(">>>>>>>>>>> job monitor heat success, JobLogId:{}", jobLogId); |
|
|
|
|
XxlJobLog log = XxlJobDynamicScheduler.xxlJobLogDao.load(jobLogId); |
|
|
|
|
if (log!=null) { |
|
|
|
|
if (ReturnT.SUCCESS_CODE==log.getTriggerCode() && log.getHandleCode()==0) { |
|
|
|
|
// running
|
|
|
|
|
try { |
|
|
|
|
TimeUnit.SECONDS.sleep(10); |
|
|
|
|
} catch (InterruptedException e) { |
|
|
|
|
e.printStackTrace(); |
|
|
|
|
} |
|
|
|
|
// job running, wait + again monitor
|
|
|
|
|
TimeUnit.SECONDS.sleep(10); |
|
|
|
|
|
|
|
|
|
JobFailMonitorHelper.monitor(jobLogId); |
|
|
|
|
logger.info(">>>>>>>>>>> job monitor, job running, JobLogId:{}", jobLogId); |
|
|
|
|
} |
|
|
|
|
if (ReturnT.SUCCESS_CODE==log.getTriggerCode() && ReturnT.SUCCESS_CODE==log.getHandleCode()) { |
|
|
|
|
// pass
|
|
|
|
|
// job success, pass
|
|
|
|
|
logger.info(">>>>>>>>>>> job monitor, job success, JobLogId:{}", jobLogId); |
|
|
|
|
} |
|
|
|
|
if (ReturnT.FAIL_CODE == log.getTriggerCode()|| ReturnT.FAIL_CODE==log.getHandleCode()) { |
|
|
|
|
XxlJobInfo info = XxlJobDynamicScheduler.xxlJobInfoDao.loadById(log.getJobId()); |
|
|
|
|
if (info!=null && info.getAlarmEmail()!=null && info.getAlarmEmail().trim().length()>0) { |
|
|
|
|
|
|
|
|
|
Set<String> emailSet = new HashSet<String>(Arrays.asList(info.getAlarmEmail().split(","))); |
|
|
|
|
for (String email: emailSet) { |
|
|
|
|
String title = "《调度监控报警》(任务调度中心XXL-JOB)"; |
|
|
|
|
XxlJobGroup group = XxlJobDynamicScheduler.xxlJobGroupDao.load(Integer.valueOf(info.getJobGroup())); |
|
|
|
|
String content = MessageFormat.format("任务调度失败, 执行器名称:{0}, 任务描述:{1}.", group!=null?group.getTitle():"null", info.getJobDesc()); |
|
|
|
|
MailUtil.sendMail(email, title, content, false, null); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
if (ReturnT.FAIL_CODE == log.getTriggerCode()|| ReturnT.FAIL_CODE==log.getHandleCode()) { |
|
|
|
|
// job fail,
|
|
|
|
|
sendMonitorEmail(log); |
|
|
|
|
logger.info(">>>>>>>>>>> job monitor, job fail, JobLogId:{}", jobLogId); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
@ -75,15 +66,54 @@ public class JobFailMonitorHelper { |
|
|
|
|
logger.error("job monitor error:{}", e); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// monitor all clear
|
|
|
|
|
List<Integer> jobLogIdList = new ArrayList<Integer>(); |
|
|
|
|
int drainToNum = getInstance().queue.drainTo(jobLogIdList); |
|
|
|
|
if (jobLogIdList!=null && jobLogIdList.size()>0) { |
|
|
|
|
for (Integer jobLogId: jobLogIdList) { |
|
|
|
|
XxlJobLog log = XxlJobDynamicScheduler.xxlJobLogDao.load(jobLogId); |
|
|
|
|
if (ReturnT.FAIL_CODE == log.getTriggerCode()|| ReturnT.FAIL_CODE==log.getHandleCode()) { |
|
|
|
|
// job fail,
|
|
|
|
|
sendMonitorEmail(log); |
|
|
|
|
logger.info(">>>>>>>>>>> job monitor last, job fail, JobLogId:{}", jobLogId); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
}); |
|
|
|
|
monitorThread.setDaemon(true); |
|
|
|
|
monitorThread.start(); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
* send monitor email |
|
|
|
|
* @param jobLog |
|
|
|
|
*/ |
|
|
|
|
private void sendMonitorEmail(XxlJobLog jobLog){ |
|
|
|
|
XxlJobInfo info = XxlJobDynamicScheduler.xxlJobInfoDao.loadById(jobLog.getJobId()); |
|
|
|
|
if (info!=null && info.getAlarmEmail()!=null && info.getAlarmEmail().trim().length()>0) { |
|
|
|
|
|
|
|
|
|
Set<String> emailSet = new HashSet<String>(Arrays.asList(info.getAlarmEmail().split(","))); |
|
|
|
|
for (String email: emailSet) { |
|
|
|
|
String title = "《调度监控报警》(任务调度中心XXL-JOB)"; |
|
|
|
|
XxlJobGroup group = XxlJobDynamicScheduler.xxlJobGroupDao.load(Integer.valueOf(info.getJobGroup())); |
|
|
|
|
String content = MessageFormat.format("任务调度失败, 执行器名称:{0}, 任务描述:{1}.", group!=null?group.getTitle():"null", info.getJobDesc()); |
|
|
|
|
MailUtil.sendMail(email, title, content, false, null); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public void toStop(){ |
|
|
|
|
toStop = true; |
|
|
|
|
//monitorThread.interrupt();
|
|
|
|
|
// interrupt and wait
|
|
|
|
|
monitorThread.interrupt(); |
|
|
|
|
try { |
|
|
|
|
monitorThread.join(); |
|
|
|
|
} catch (InterruptedException e) { |
|
|
|
|
logger.error(e.getMessage(), e); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// producer
|
|
|
|
|
|
xuxueli
7 years ago