|
|
@@ -10,10 +10,9 @@ import org.slf4j.Logger;
|
|
10
|
10
|
import org.slf4j.LoggerFactory;
|
|
11
|
11
|
|
|
12
|
12
|
import java.text.MessageFormat;
|
|
13
|
|
-import java.util.Arrays;
|
|
14
|
|
-import java.util.HashSet;
|
|
15
|
|
-import java.util.Set;
|
|
16
|
|
-import java.util.concurrent.*;
|
|
|
13
|
+import java.util.*;
|
|
|
14
|
+import java.util.concurrent.LinkedBlockingQueue;
|
|
|
15
|
+import java.util.concurrent.TimeUnit;
|
|
17
|
16
|
|
|
18
|
17
|
/**
|
|
19
|
18
|
* job monitor instance
|
|
|
@@ -36,38 +35,30 @@ public class JobFailMonitorHelper {
|
|
36
|
35
|
|
|
37
|
36
|
@Override
|
|
38
|
37
|
public void run() {
|
|
|
38
|
+
|
|
|
39
|
+ // monitor
|
|
39
|
40
|
while (!toStop) {
|
|
40
|
41
|
try {
|
|
41
|
|
- logger.debug(">>>>>>>>>>> job monitor beat ... ");
|
|
42
|
42
|
Integer jobLogId = JobFailMonitorHelper.instance.queue.take();
|
|
43
|
43
|
if (jobLogId != null && jobLogId > 0) {
|
|
44
|
|
- logger.debug(">>>>>>>>>>> job monitor heat success, JobLogId:{}", jobLogId);
|
|
45
|
44
|
XxlJobLog log = XxlJobDynamicScheduler.xxlJobLogDao.load(jobLogId);
|
|
46
|
45
|
if (log!=null) {
|
|
47
|
46
|
if (ReturnT.SUCCESS_CODE==log.getTriggerCode() && log.getHandleCode()==0) {
|
|
48
|
|
- // running
|
|
49
|
|
- try {
|
|
50
|
|
- TimeUnit.SECONDS.sleep(10);
|
|
51
|
|
- } catch (InterruptedException e) {
|
|
52
|
|
- e.printStackTrace();
|
|
53
|
|
- }
|
|
|
47
|
+ // job running, wait + again monitor
|
|
|
48
|
+ TimeUnit.SECONDS.sleep(10);
|
|
|
49
|
+
|
|
54
|
50
|
JobFailMonitorHelper.monitor(jobLogId);
|
|
|
51
|
+ logger.info(">>>>>>>>>>> job monitor, job running, JobLogId:{}", jobLogId);
|
|
55
|
52
|
}
|
|
56
|
53
|
if (ReturnT.SUCCESS_CODE==log.getTriggerCode() && ReturnT.SUCCESS_CODE==log.getHandleCode()) {
|
|
57
|
|
- // pass
|
|
|
54
|
+ // job success, pass
|
|
|
55
|
+ logger.info(">>>>>>>>>>> job monitor, job success, JobLogId:{}", jobLogId);
|
|
58
|
56
|
}
|
|
59
|
|
- if (ReturnT.FAIL_CODE == log.getTriggerCode()|| ReturnT.FAIL_CODE==log.getHandleCode()) {
|
|
60
|
|
- XxlJobInfo info = XxlJobDynamicScheduler.xxlJobInfoDao.loadById(log.getJobId());
|
|
61
|
|
- if (info!=null && info.getAlarmEmail()!=null && info.getAlarmEmail().trim().length()>0) {
|
|
62
|
57
|
|
|
63
|
|
- Set<String> emailSet = new HashSet<String>(Arrays.asList(info.getAlarmEmail().split(",")));
|
|
64
|
|
- for (String email: emailSet) {
|
|
65
|
|
- String title = "《调度监控报警》(任务调度中心XXL-JOB)";
|
|
66
|
|
- XxlJobGroup group = XxlJobDynamicScheduler.xxlJobGroupDao.load(Integer.valueOf(info.getJobGroup()));
|
|
67
|
|
- String content = MessageFormat.format("任务调度失败, 执行器名称:{0}, 任务描述:{1}.", group!=null?group.getTitle():"null", info.getJobDesc());
|
|
68
|
|
- MailUtil.sendMail(email, title, content, false, null);
|
|
69
|
|
- }
|
|
70
|
|
- }
|
|
|
58
|
+ if (ReturnT.FAIL_CODE == log.getTriggerCode()|| ReturnT.FAIL_CODE==log.getHandleCode()) {
|
|
|
59
|
+ // job fail,
|
|
|
60
|
+ sendMonitorEmail(log);
|
|
|
61
|
+ logger.info(">>>>>>>>>>> job monitor, job fail, JobLogId:{}", jobLogId);
|
|
71
|
62
|
}
|
|
72
|
63
|
}
|
|
73
|
64
|
}
|
|
|
@@ -75,15 +66,54 @@ public class JobFailMonitorHelper {
|
|
75
|
66
|
logger.error("job monitor error:{}", e);
|
|
76
|
67
|
}
|
|
77
|
68
|
}
|
|
|
69
|
+
|
|
|
70
|
+ // monitor all clear
|
|
|
71
|
+ List<Integer> jobLogIdList = new ArrayList<Integer>();
|
|
|
72
|
+ int drainToNum = getInstance().queue.drainTo(jobLogIdList);
|
|
|
73
|
+ if (jobLogIdList!=null && jobLogIdList.size()>0) {
|
|
|
74
|
+ for (Integer jobLogId: jobLogIdList) {
|
|
|
75
|
+ XxlJobLog log = XxlJobDynamicScheduler.xxlJobLogDao.load(jobLogId);
|
|
|
76
|
+ if (ReturnT.FAIL_CODE == log.getTriggerCode()|| ReturnT.FAIL_CODE==log.getHandleCode()) {
|
|
|
77
|
+ // job fail,
|
|
|
78
|
+ sendMonitorEmail(log);
|
|
|
79
|
+ logger.info(">>>>>>>>>>> job monitor last, job fail, JobLogId:{}", jobLogId);
|
|
|
80
|
+ }
|
|
|
81
|
+ }
|
|
|
82
|
+ }
|
|
|
83
|
+
|
|
78
|
84
|
}
|
|
79
|
85
|
});
|
|
80
|
86
|
monitorThread.setDaemon(true);
|
|
81
|
87
|
monitorThread.start();
|
|
82
|
88
|
}
|
|
83
|
89
|
|
|
|
90
|
+ /**
|
|
|
91
|
+ * send monitor email
|
|
|
92
|
+ * @param jobLog
|
|
|
93
|
+ */
|
|
|
94
|
+ private void sendMonitorEmail(XxlJobLog jobLog){
|
|
|
95
|
+ XxlJobInfo info = XxlJobDynamicScheduler.xxlJobInfoDao.loadById(jobLog.getJobId());
|
|
|
96
|
+ if (info!=null && info.getAlarmEmail()!=null && info.getAlarmEmail().trim().length()>0) {
|
|
|
97
|
+
|
|
|
98
|
+ Set<String> emailSet = new HashSet<String>(Arrays.asList(info.getAlarmEmail().split(",")));
|
|
|
99
|
+ for (String email: emailSet) {
|
|
|
100
|
+ String title = "《调度监控报警》(任务调度中心XXL-JOB)";
|
|
|
101
|
+ XxlJobGroup group = XxlJobDynamicScheduler.xxlJobGroupDao.load(Integer.valueOf(info.getJobGroup()));
|
|
|
102
|
+ String content = MessageFormat.format("任务调度失败, 执行器名称:{0}, 任务描述:{1}.", group!=null?group.getTitle():"null", info.getJobDesc());
|
|
|
103
|
+ MailUtil.sendMail(email, title, content, false, null);
|
|
|
104
|
+ }
|
|
|
105
|
+ }
|
|
|
106
|
+ }
|
|
|
107
|
+
|
|
84
|
108
|
public void toStop(){
|
|
85
|
109
|
toStop = true;
|
|
86
|
|
- //monitorThread.interrupt();
|
|
|
110
|
+ // interrupt and wait
|
|
|
111
|
+ monitorThread.interrupt();
|
|
|
112
|
+ try {
|
|
|
113
|
+ monitorThread.join();
|
|
|
114
|
+ } catch (InterruptedException e) {
|
|
|
115
|
+ logger.error(e.getMessage(), e);
|
|
|
116
|
+ }
|
|
87
|
117
|
}
|
|
88
|
118
|
|
|
89
|
119
|
// producer
|
xuxueli