调度报表优化：新增日志报表的存储表，三天内的任务日志会以每分钟一次的频率异步同步至报表中；任务报表仅读取报表数据，极大提升加载速度；

doc/XXL-JOB官方文档.md

 - 20、xxl-rpc服务端线程优化，降低线程内存开销；
 - 21、调度中心回调API服务改为restful方式；
 - 22、调度中心日志删除优化，改为分页获取ID并根据ID删除的方式，避免批量删除海量日志导致死锁问题；
-- 23、[ING]调度日志优化：支持设置日志保留天数，过期日志天维度记录报表，并清理；调度报表汇总实时数据和报表；
+- 23、调度报表优化：新增日志报表的存储表，三天内的任务日志会以每分钟一次的频率异步同步至报表中；任务报表仅读取报表数据，极大提升加载速度；
+- 24、[ing]调度日志优化：支持设置日志保留天数，过期日志天维度记录报表，并清理；调度报表汇总实时数据和报表；
 
 
 

doc/db/tables_xxl_job.sql

   KEY `I_handle_code` (`handle_code`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
+CREATE TABLE `xxl_job_log_report` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `trigger_day` datetime DEFAULT NULL COMMENT '调度-时间',
+  `running_count` int(11) NOT NULL DEFAULT '0' COMMENT '运行中-日志数量',
+  `suc_count` int(11) NOT NULL DEFAULT '0' COMMENT '执行成功-日志数量',
+  `fail_count` int(11) NOT NULL DEFAULT '0' COMMENT '执行失败-日志数量',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `i_trigger_day` (`trigger_day`) USING BTREE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
 CREATE TABLE `xxl_job_logglue` (
   `id` int(11) NOT NULL AUTO_INCREMENT,
   `job_id` int(11) NOT NULL COMMENT '任务，主键ID',

xxl-job-admin/src/main/java/com/xxl/job/admin/core/conf/XxlJobAdminConfig.java

 package com.xxl.job.admin.core.conf;
 
 import com.xxl.job.admin.core.scheduler.XxlJobScheduler;
-import com.xxl.job.admin.dao.XxlJobGroupDao;
-import com.xxl.job.admin.dao.XxlJobInfoDao;
-import com.xxl.job.admin.dao.XxlJobLogDao;
-import com.xxl.job.admin.dao.XxlJobRegistryDao;
+import com.xxl.job.admin.dao.*;
 import org.springframework.beans.factory.DisposableBean;
 import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Value;
     @Resource
     private XxlJobGroupDao xxlJobGroupDao;
     @Resource
+    private XxlJobLogReportDao xxlJobLogReportDao;
+    @Resource
     private JavaMailSender mailSender;
     @Resource
     private DataSource dataSource;
         return xxlJobGroupDao;
     }
 
+    public XxlJobLogReportDao getXxlJobLogReportDao() {
+        return xxlJobLogReportDao;
+    }
+
     public JavaMailSender getMailSender() {
         return mailSender;
     }

xxl-job-admin/src/main/java/com/xxl/job/admin/core/model/XxlJobLogReport.java

+package com.xxl.job.admin.core.model;
+
+import java.util.Date;
+
+public class XxlJobLogReport {
+
+    private int id;
+
+    private Date triggerDay;
+
+    private int runningCount;
+    private int sucCount;
+    private int failCount;
+
+    public int getId() {
+        return id;
+    }
+
+    public void setId(int id) {
+        this.id = id;
+    }
+
+    public Date getTriggerDay() {
+        return triggerDay;
+    }
+
+    public void setTriggerDay(Date triggerDay) {
+        this.triggerDay = triggerDay;
+    }
+
+    public int getRunningCount() {
+        return runningCount;
+    }
+
+    public void setRunningCount(int runningCount) {
+        this.runningCount = runningCount;
+    }
+
+    public int getSucCount() {
+        return sucCount;
+    }
+
+    public void setSucCount(int sucCount) {
+        this.sucCount = sucCount;
+    }
+
+    public int getFailCount() {
+        return failCount;
+    }
+
+    public void setFailCount(int failCount) {
+        this.failCount = failCount;
+    }
+}

xxl-job-admin/src/main/java/com/xxl/job/admin/core/scheduler/XxlJobScheduler.java

 package com.xxl.job.admin.core.scheduler;
 
 import com.xxl.job.admin.core.conf.XxlJobAdminConfig;
-import com.xxl.job.admin.core.thread.JobFailMonitorHelper;
-import com.xxl.job.admin.core.thread.JobRegistryMonitorHelper;
-import com.xxl.job.admin.core.thread.JobScheduleHelper;
-import com.xxl.job.admin.core.thread.JobTriggerPoolHelper;
+import com.xxl.job.admin.core.thread.*;
 import com.xxl.job.admin.core.util.I18nUtil;
 import com.xxl.job.core.biz.ExecutorBiz;
 import com.xxl.job.core.enums.ExecutorBlockStrategyEnum;
         // admin trigger pool start
         JobTriggerPoolHelper.toStart();
 
+        // admin log report start
+        JobLogReportHelper.getInstance().start();
+
         // start-schedule
         JobScheduleHelper.getInstance().start();
 
         // stop-schedule
         JobScheduleHelper.getInstance().toStop();
 
+        // admin log report stop
+        JobLogReportHelper.getInstance().toStop();
+
         // admin trigger pool stop
         JobTriggerPoolHelper.toStop();
 

xxl-job-admin/src/main/java/com/xxl/job/admin/core/thread/JobFailMonitorHelper.java

 							}
 						}
 
-						TimeUnit.SECONDS.sleep(10);
 					} catch (Exception e) {
 						if (!toStop) {
 							logger.error(">>>>>>>>>>> xxl-job, job fail monitor thread error:{}", e);
 						}
 					}
-				}
+
+                    try {
+                        TimeUnit.SECONDS.sleep(10);
+                    } catch (Exception e) {
+                        if (!toStop) {
+                            logger.error(e.getMessage(), e);
+                        }
+                    }
+
+                }
 
 				logger.info(">>>>>>>>>>> xxl-job, job fail monitor thread stop");
 

xxl-job-admin/src/main/java/com/xxl/job/admin/core/thread/JobLogReportHelper.java

+package com.xxl.job.admin.core.thread;
+
+import com.xxl.job.admin.core.conf.XxlJobAdminConfig;
+import com.xxl.job.admin.core.model.XxlJobLogReport;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Calendar;
+import java.util.Date;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * job log report helper
+ *
+ * @author xuxueli 2019-11-22
+ */
+public class JobLogReportHelper {
+    private static Logger logger = LoggerFactory.getLogger(JobLogReportHelper.class);
+
+    private static JobLogReportHelper instance = new JobLogReportHelper();
+    public static JobLogReportHelper getInstance(){
+        return instance;
+    }
+
+
+    private Thread logrThread;
+    private volatile boolean toStop = false;
+    public void start(){
+        logrThread = new Thread(new Runnable() {
+
+            @Override
+            public void run() {
+
+                // monitor
+                while (!toStop) {
+                    try {
+
+                        // refresh log report in 3 days
+                        for (int i = 0; i < 3; i++) {
+
+                            // today
+                            Calendar itemDay = Calendar.getInstance();
+                            itemDay.add(Calendar.DAY_OF_MONTH, -i);
+                            itemDay.set(Calendar.HOUR_OF_DAY, 0);
+                            itemDay.set(Calendar.MINUTE, 0);
+                            itemDay.set(Calendar.SECOND, 0);
+                            itemDay.set(Calendar.MILLISECOND, 0);
+
+                            Date todayFrom = itemDay.getTime();
+
+                            itemDay.set(Calendar.HOUR_OF_DAY, 23);
+                            itemDay.set(Calendar.MINUTE, 59);
+                            itemDay.set(Calendar.SECOND, 59);
+                            itemDay.set(Calendar.MILLISECOND, 999);
+
+                            Date todayTo = itemDay.getTime();
+
+                            // refresh log-report every minute
+                            XxlJobLogReport xxlJobLogReport = new XxlJobLogReport();
+                            xxlJobLogReport.setTriggerDay(todayFrom);
+                            xxlJobLogReport.setRunningCount(0);
+                            xxlJobLogReport.setSucCount(0);
+                            xxlJobLogReport.setFailCount(0);
+
+                            Map<String, Object> triggerCountMap = XxlJobAdminConfig.getAdminConfig().getXxlJobLogDao().findLogReport(todayFrom, todayTo);
+                            if (triggerCountMap!=null && triggerCountMap.size()>0) {
+                                int triggerDayCount = triggerCountMap.containsKey("triggerDayCount")?Integer.valueOf(String.valueOf(triggerCountMap.get("triggerDayCount"))):0;
+                                int triggerDayCountRunning = triggerCountMap.containsKey("triggerDayCountRunning")?Integer.valueOf(String.valueOf(triggerCountMap.get("triggerDayCountRunning"))):0;
+                                int triggerDayCountSuc = triggerCountMap.containsKey("triggerDayCountSuc")?Integer.valueOf(String.valueOf(triggerCountMap.get("triggerDayCountSuc"))):0;
+                                int triggerDayCountFail = triggerDayCount - triggerDayCountRunning - triggerDayCountSuc;
+
+                                xxlJobLogReport.setRunningCount(triggerDayCountRunning);
+                                xxlJobLogReport.setSucCount(triggerDayCountSuc);
+                                xxlJobLogReport.setFailCount(triggerDayCountFail);
+                            }
+
+                            // do refresh
+                            int ret = XxlJobAdminConfig.getAdminConfig().getXxlJobLogReportDao().update(xxlJobLogReport);
+                            if (ret < 1) {
+                                XxlJobAdminConfig.getAdminConfig().getXxlJobLogReportDao().save(xxlJobLogReport);
+                            }
+                        }
+
+                    } catch (Exception e) {
+                        if (!toStop) {
+                            logger.error(">>>>>>>>>>> xxl-job, job log report thread error:{}", e);
+                        }
+                    }
+
+                    try {
+                        TimeUnit.MINUTES.sleep(1);
+                    } catch (Exception e) {
+                        if (!toStop) {
+                            logger.error(e.getMessage(), e);
+                        }
+                    }
+
+                }
+
+                logger.info(">>>>>>>>>>> xxl-job, job log report thread stop");
+
+            }
+        });
+        logrThread.setDaemon(true);
+        logrThread.setName("xxl-job, admin JobLogReportHelper");
+        logrThread.start();
+    }
+
+    public void toStop(){
+        toStop = true;
+        // interrupt and wait
+        logrThread.interrupt();
+        try {
+            logrThread.join();
+        } catch (InterruptedException e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+}

xxl-job-admin/src/main/java/com/xxl/job/admin/core/thread/JobScheduleHelper.java

                                 // time-ring jump
                                 if (nowTime > jobInfo.getTriggerNextTime() + PRE_READ_MS) {
                                     // 2.1、trigger-expire > 5s：pass && make next-trigger-time
+                                    logger.warn(">>>>>>>>>>> xxl-job, schedule misfire, jobId = " + jobInfo.getId());
 
                                     // fresh next
                                     refreshNextValidTime(jobInfo, new Date());

xxl-job-admin/src/main/java/com/xxl/job/admin/dao/XxlJobLogDao.java

 	
 	public int delete(@Param("jobId") int jobId);
 
-	public int triggerCountByHandleCode(@Param("handleCode") int handleCode);
-
-	public List<Map<String, Object>> triggerCountByDay(@Param("from") Date from,
-													   @Param("to") Date to);
+	public Map<String, Object> findLogReport(@Param("from") Date from,
+											 @Param("to") Date to);
 
 	public List<Long> findClearLogIds(@Param("jobGroup") int jobGroup,
 									  @Param("jobId") int jobId,

xxl-job-admin/src/main/java/com/xxl/job/admin/dao/XxlJobLogReportDao.java

+package com.xxl.job.admin.dao;
+
+import com.xxl.job.admin.core.model.XxlJobLogReport;
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.Date;
+import java.util.List;
+
+/**
+ * job log
+ * @author xuxueli 2019-11-22
+ */
+@Mapper
+public interface XxlJobLogReportDao {
+
+	public int save(XxlJobLogReport xxlJobLogReport);
+
+	public int update(XxlJobLogReport xxlJobLogReport);
+
+	public List<XxlJobLogReport> queryLogReport(@Param("triggerDayFrom") Date triggerDayFrom,
+												@Param("triggerDayTo") Date triggerDayTo);
+
+	public XxlJobLogReport queryLogReportTotal();
+
+}

xxl-job-admin/src/main/java/com/xxl/job/admin/service/impl/XxlJobServiceImpl.java

 import com.xxl.job.admin.core.model.XxlJobGroup;
 import com.xxl.job.admin.core.model.XxlJobInfo;
 import com.xxl.job.admin.core.cron.CronExpression;
+import com.xxl.job.admin.core.model.XxlJobLogReport;
 import com.xxl.job.admin.core.route.ExecutorRouteStrategyEnum;
 import com.xxl.job.admin.core.thread.JobScheduleHelper;
 import com.xxl.job.admin.core.util.I18nUtil;
-import com.xxl.job.admin.dao.XxlJobGroupDao;
-import com.xxl.job.admin.dao.XxlJobInfoDao;
-import com.xxl.job.admin.dao.XxlJobLogDao;
-import com.xxl.job.admin.dao.XxlJobLogGlueDao;
+import com.xxl.job.admin.dao.*;
 import com.xxl.job.admin.service.XxlJobService;
 import com.xxl.job.core.biz.model.ReturnT;
 import com.xxl.job.core.enums.ExecutorBlockStrategyEnum;
 	public XxlJobLogDao xxlJobLogDao;
 	@Resource
 	private XxlJobLogGlueDao xxlJobLogGlueDao;
+	@Resource
+	private XxlJobLogReportDao xxlJobLogReportDao;
 	
 	@Override
 	public Map<String, Object> pageList(int start, int length, int jobGroup, int triggerStatus, String jobDesc, String executorHandler, String author) {
 	public Map<String, Object> dashboardInfo() {
 
 		int jobInfoCount = xxlJobInfoDao.findAllCount();
-		int jobLogCount = xxlJobLogDao.triggerCountByHandleCode(-1);
-		int jobLogSuccessCount = xxlJobLogDao.triggerCountByHandleCode(ReturnT.SUCCESS_CODE);
+		int jobLogCount = 0;
+		int jobLogSuccessCount = 0;
+		XxlJobLogReport xxlJobLogReport = xxlJobLogReportDao.queryLogReportTotal();
+		if (xxlJobLogReport != null) {
+			jobLogCount = xxlJobLogReport.getRunningCount() + xxlJobLogReport.getSucCount() + xxlJobLogReport.getFailCount();
+			jobLogSuccessCount = xxlJobLogReport.getSucCount();
+		}
 
 		// executor count
 		Set<String> executorAddressSet = new HashSet<String>();
 		return dashboardMap;
 	}
 
-	private static final String TRIGGER_CHART_DATA_CACHE = "trigger_chart_data_cache";
 	@Override
 	public ReturnT<Map<String, Object>> chartInfo(Date startDate, Date endDate) {
-		/*// get cache
-		String cacheKey = TRIGGER_CHART_DATA_CACHE + "_" + startDate.getTime() + "_" + endDate.getTime();
-		Map<String, Object> chartInfo = (Map<String, Object>) LocalCacheUtil.get(cacheKey);
-		if (chartInfo != null) {
-			return new ReturnT<Map<String, Object>>(chartInfo);
-		}*/
 
 		// process
 		List<String> triggerDayList = new ArrayList<String>();
 		int triggerCountSucTotal = 0;
 		int triggerCountFailTotal = 0;
 
-		List<Map<String, Object>> triggerCountMapAll = xxlJobLogDao.triggerCountByDay(startDate, endDate);
-		if (triggerCountMapAll!=null && triggerCountMapAll.size()>0) {
-			for (Map<String, Object> item: triggerCountMapAll) {
-				String day = String.valueOf(item.get("triggerDay"));
-				int triggerDayCount = Integer.valueOf(String.valueOf(item.get("triggerDayCount")));
-				int triggerDayCountRunning = Integer.valueOf(String.valueOf(item.get("triggerDayCountRunning")));
-				int triggerDayCountSuc = Integer.valueOf(String.valueOf(item.get("triggerDayCountSuc")));
-				int triggerDayCountFail = triggerDayCount - triggerDayCountRunning - triggerDayCountSuc;
+		List<XxlJobLogReport> logReportList = xxlJobLogReportDao.queryLogReport(startDate, endDate);
+
+		if (logReportList!=null && logReportList.size()>0) {
+			for (XxlJobLogReport item: logReportList) {
+				String day = DateUtil.formatDate(item.getTriggerDay());
+				int triggerDayCountRunning = item.getRunningCount();
+				int triggerDayCountSuc = item.getSucCount();
+				int triggerDayCountFail = item.getFailCount();
 
 				triggerDayList.add(day);
 				triggerDayCountRunningList.add(triggerDayCountRunning);
 				triggerCountFailTotal += triggerDayCountFail;
 			}
 		} else {
-            for (int i = 4; i > -1; i--) {
-                triggerDayList.add(DateUtil.formatDate(DateUtil.addDays(new Date(), -i)));
+			for (int i = -6; i <= 0; i++) {
+				triggerDayList.add(DateUtil.formatDate(DateUtil.addDays(new Date(), i)));
 				triggerDayCountRunningList.add(0);
-                triggerDayCountSucList.add(0);
-                triggerDayCountFailList.add(0);
-            }
+				triggerDayCountSucList.add(0);
+				triggerDayCountFailList.add(0);
+			}
 		}
 
 		Map<String, Object> result = new HashMap<String, Object>();
 		result.put("triggerCountSucTotal", triggerCountSucTotal);
 		result.put("triggerCountFailTotal", triggerCountFailTotal);
 
-		/*// set cache
-		LocalCacheUtil.set(cacheKey, result, 60*1000);     // cache 60s*/
-
 		return new ReturnT<Map<String, Object>>(result);
 	}
 

xxl-job-admin/src/main/resources/mybatis-mapper/XxlJobLogMapper.xml

 		WHERE job_id = #{jobId}
 	</delete>
 
-	<select id="triggerCountByHandleCode" resultType="int" >
-		SELECT count(1)
-		FROM xxl_job_log AS t
-		<trim prefix="WHERE" prefixOverrides="AND | OR" >
-			<if test="handleCode gt 0">
-				AND t.handle_code = #{handleCode}
-			</if>
-		</trim>
-	</select>
-
-    <select id="triggerCountByDay" resultType="java.util.Map" >
+    <!--<select id="triggerCountByDay" resultType="java.util.Map" >
 		SELECT
 			DATE_FORMAT(trigger_time,'%Y-%m-%d') triggerDay,
 			COUNT(handle_code) triggerDayCount,
 		WHERE trigger_time BETWEEN #{from} and #{to}
 		GROUP BY triggerDay
 		ORDER BY triggerDay
+    </select>-->
+
+    <select id="findLogReport" resultType="java.util.Map" >
+		SELECT
+			COUNT(handle_code) triggerDayCount,
+			SUM(CASE WHEN (trigger_code in (0, 200) and handle_code = 0) then 1 else 0 end) as triggerDayCountRunning,
+			SUM(CASE WHEN handle_code = 200 then 1 else 0 end) as triggerDayCountSuc
+		FROM xxl_job_log
+		WHERE trigger_time BETWEEN #{from} and #{to}
     </select>
 
 	<select id="findClearLogIds" resultType="long" >

xxl-job-admin/src/main/resources/mybatis-mapper/XxlJobLogReportMapper.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" 
+	"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.xxl.job.admin.dao.XxlJobLogReportDao">
+	
+	<resultMap id="XxlJobLogReport" type="com.xxl.job.admin.core.model.XxlJobLogReport" >
+		<result column="id" property="id" />
+	    <result column="trigger_day" property="triggerDay" />
+		<result column="running_count" property="runningCount" />
+	    <result column="suc_count" property="sucCount" />
+	    <result column="fail_count" property="failCount" />
+	</resultMap>
+
+	<sql id="Base_Column_List">
+		t.id,
+		t.trigger_day,
+		t.running_count,
+		t.suc_count,
+		t.fail_count
+	</sql>
+	
+	<insert id="save" parameterType="com.xxl.job.admin.core.model.XxlJobLogReport" useGeneratedKeys="true" keyProperty="id" >
+		INSERT INTO xxl_job_log_report (
+			`trigger_day`,
+			`running_count`,
+			`suc_count`,
+			`fail_count`
+		) VALUES (
+			#{triggerDay},
+			#{runningCount},
+			#{sucCount},
+			#{failCount}
+		);
+		<!--<selectKey resultType="java.lang.Integer" order="AFTER" keyProperty="id">
+			SELECT LAST_INSERT_ID() 
+		</selectKey>-->
+	</insert>
+
+	<update id="update" >
+        UPDATE xxl_job_log_report
+        SET `running_count` = #{runningCount},
+        	`suc_count` = #{sucCount},
+        	`fail_count` = #{failCount}
+        WHERE `trigger_day` = #{triggerDay}
+    </update>
+
+	<select id="queryLogReport" resultMap="XxlJobLogReport">
+		SELECT <include refid="Base_Column_List" />
+		FROM xxl_job_log_report AS t
+		WHERE t.trigger_day between #{triggerDayFrom} and #{triggerDayTo}
+		ORDER BY t.trigger_day ASC
+	</select>
+
+	<select id="queryLogReportTotal" resultMap="XxlJobLogReport">
+		SELECT
+			SUM(running_count) running_count,
+			SUM(suc_count) suc_count,
+			SUM(fail_count) fail_count
+		FROM xxl_job_log_report AS t
+	</select>
+
+</mapper>

xxl-job-admin/src/test/java/com/xxl/job/admin/dao/XxlJobLogDaoTest.java

 import javax.annotation.Resource;
 import java.util.Date;
 import java.util.List;
-import java.util.Map;
 
 @RunWith(SpringRunner.class)
 @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
         dto = xxlJobLogDao.load(log.getId());
 
 
-        List<Map<String, Object>> list2 = xxlJobLogDao.triggerCountByDay(new Date(new Date().getTime() + 30*24*60*60*1000), new Date());
-
         List<Long> ret4 = xxlJobLogDao.findClearLogIds(1, 1, new Date(), 100, 100);
 
         int ret2 = xxlJobLogDao.delete(log.getJobId());
 
-        int ret3 = xxlJobLogDao.triggerCountByHandleCode(-1);
     }
 
 }

xxl-job-core/src/main/java/com/xxl/job/core/util/DateUtil.java

 
     // ---------------------- add date ----------------------
 
-    public static Date addDays(final Date date, final int amount) {
-        return add(date, Calendar.DAY_OF_MONTH, amount);
-    }
-
     public static Date addYears(final Date date, final int amount) {
         return add(date, Calendar.YEAR, amount);
     }
         return add(date, Calendar.MONTH, amount);
     }
 
+    public static Date addDays(final Date date, final int amount) {
+        return add(date, Calendar.DAY_OF_MONTH, amount);
+    }
+
     private static Date add(final Date date, final int calendarField, final int amount) {
         if (date == null) {
             return null;