浏览代码

调度中心日志自动清理功能:调度中心新增配置项("xxl.job.logretentiondays")日志保存天数,过期日志自动清理;解决海量日志情况下日志表慢SQL问题;限制大于等于7时生效,否则关闭清理功能,默认为30;

xuxueli 5 年前
父节点
当前提交
1aad0a2676

+ 9 - 1
doc/XXL-JOB官方文档.md 查看文件

464
     ### xxl-job, i18n (default empty as chinese, "en" as english)
464
     ### xxl-job, i18n (default empty as chinese, "en" as english)
465
     xxl.job.i18n=
465
     xxl.job.i18n=
466
     
466
     
467
+    ## xxl-job, 调度线程池最大线程配置
468
+    xxl.job.triggerpool.fast.max=200
469
+    xxl.job.triggerpool.slow.max=100
470
+    
471
+    ### xxl-job, 日志保存天数,过期日志自动清理;限制大于等于7时生效,否则,如-1,关闭自动清理功能;
472
+    xxl.job.logretentiondays=30
473
+    
467
     
474
     
468
 
475
 
469
 #### 步骤二:部署项目:
476
 #### 步骤二:部署项目:
1599
 - 21、调度中心回调API服务改为restful方式;
1606
 - 21、调度中心回调API服务改为restful方式;
1600
 - 22、调度中心日志删除优化,改为分页获取ID并根据ID删除的方式,避免批量删除海量日志导致死锁问题;
1607
 - 22、调度中心日志删除优化,改为分页获取ID并根据ID删除的方式,避免批量删除海量日志导致死锁问题;
1601
 - 23、调度报表优化:新增日志报表的存储表,三天内的任务日志会以每分钟一次的频率异步同步至报表中;任务报表仅读取报表数据,极大提升加载速度;
1608
 - 23、调度报表优化:新增日志报表的存储表,三天内的任务日志会以每分钟一次的频率异步同步至报表中;任务报表仅读取报表数据,极大提升加载速度;
1602
-- 24、[ing]调度日志优化:支持设置日志保留天数,过期日志天维度记录报表,并清理;调度报表汇总实时数据和报表;
1609
+- 24、调度中心日志自动清理功能:调度中心新增配置项("xxl.job.logretentiondays")日志保存天数,过期日志自动清理;解决海量日志情况下日志表慢SQL问题;限制大于等于7时生效,否则关闭清理功能,默认为30;
1610
+
1603
 
1611
 
1604
 
1612
 
1605
 
1613
 

+ 10 - 0
xxl-job-admin/src/main/java/com/xxl/job/admin/core/conf/XxlJobAdminConfig.java 查看文件

62
     @Value("${xxl.job.triggerpool.slow.max}")
62
     @Value("${xxl.job.triggerpool.slow.max}")
63
     private int triggerPoolSlowMax;
63
     private int triggerPoolSlowMax;
64
 
64
 
65
+    @Value("${xxl.job.logretentiondays}")
66
+    private int logretentiondays;
67
+
65
     // dao, service
68
     // dao, service
66
 
69
 
67
     @Resource
70
     @Resource
106
         return triggerPoolSlowMax;
109
         return triggerPoolSlowMax;
107
     }
110
     }
108
 
111
 
112
+    public int getLogretentiondays() {
113
+        if (logretentiondays < 7) {
114
+            return -1;  // Limit greater than or equal to 7, otherwise close
115
+        }
116
+        return logretentiondays;
117
+    }
118
+
109
     public XxlJobLogDao getXxlJobLogDao() {
119
     public XxlJobLogDao getXxlJobLogDao() {
110
         return xxlJobLogDao;
120
         return xxlJobLogDao;
111
     }
121
     }

+ 33 - 2
xxl-job-admin/src/main/java/com/xxl/job/admin/core/thread/JobLogReportHelper.java 查看文件

7
 
7
 
8
 import java.util.Calendar;
8
 import java.util.Calendar;
9
 import java.util.Date;
9
 import java.util.Date;
10
+import java.util.List;
10
 import java.util.Map;
11
 import java.util.Map;
11
 import java.util.concurrent.TimeUnit;
12
 import java.util.concurrent.TimeUnit;
12
 
13
 
32
             @Override
33
             @Override
33
             public void run() {
34
             public void run() {
34
 
35
 
35
-                // monitor
36
+                // last clean log time
37
+                long lastCleanLogTime = 0;
38
+
39
+
36
                 while (!toStop) {
40
                 while (!toStop) {
41
+
42
+                    // 1、log-report refresh: refresh log report in 3 days
37
                     try {
43
                     try {
38
 
44
 
39
-                        // refresh log report in 3 days
40
                         for (int i = 0; i < 3; i++) {
45
                         for (int i = 0; i < 3; i++) {
41
 
46
 
42
                             // today
47
                             // today
88
                         }
93
                         }
89
                     }
94
                     }
90
 
95
 
96
+                    // 2、log-clean: switch open & once each day
97
+                    if (XxlJobAdminConfig.getAdminConfig().getLogretentiondays()>0
98
+                            && System.currentTimeMillis() - lastCleanLogTime > 24*60*60*1000) {
99
+
100
+                        // expire-time
101
+                        Calendar expiredDay = Calendar.getInstance();
102
+                        expiredDay.add(Calendar.DAY_OF_MONTH, -1 * XxlJobAdminConfig.getAdminConfig().getLogretentiondays());
103
+                        expiredDay.set(Calendar.HOUR_OF_DAY, 0);
104
+                        expiredDay.set(Calendar.MINUTE, 0);
105
+                        expiredDay.set(Calendar.SECOND, 0);
106
+                        expiredDay.set(Calendar.MILLISECOND, 0);
107
+                        Date clearBeforeTime = expiredDay.getTime();
108
+
109
+                        // clean expired log
110
+                        List<Long> logIds = null;
111
+                        do {
112
+                            logIds = XxlJobAdminConfig.getAdminConfig().getXxlJobLogDao().findClearLogIds(0, 0, clearBeforeTime, 0, 1000);
113
+                            if (logIds!=null && logIds.size()>0) {
114
+                                XxlJobAdminConfig.getAdminConfig().getXxlJobLogDao().clearLog(logIds);
115
+                            }
116
+                        } while (logIds!=null && logIds.size()>0);
117
+
118
+                        // update clean time
119
+                        lastCleanLogTime = System.currentTimeMillis();
120
+                    }
121
+
91
                     try {
122
                     try {
92
                         TimeUnit.MINUTES.sleep(1);
123
                         TimeUnit.MINUTES.sleep(1);
93
                     } catch (Exception e) {
124
                     } catch (Exception e) {