Sfoglia il codice sorgente

调度中心日志自动清理功能:调度中心新增配置项("xxl.job.logretentiondays")日志保存天数,过期日志自动清理;解决海量日志情况下日志表慢SQL问题;限制大于等于7时生效,否则关闭清理功能,默认为30;

xuxueli 5 anni fa
parent
commit
1aad0a2676

+ 9 - 1
doc/XXL-JOB官方文档.md Vedi File

@@ -464,6 +464,13 @@ XXL-JOB是一个轻量级分布式任务调度平台,其核心设计目标是
464 464
     ### xxl-job, i18n (default empty as chinese, "en" as english)
465 465
     xxl.job.i18n=
466 466
     
467
+    ## xxl-job, 调度线程池最大线程配置
468
+    xxl.job.triggerpool.fast.max=200
469
+    xxl.job.triggerpool.slow.max=100
470
+    
471
+    ### xxl-job, 日志保存天数,过期日志自动清理;限制大于等于7时生效,否则,如-1,关闭自动清理功能;
472
+    xxl.job.logretentiondays=30
473
+    
467 474
     
468 475
 
469 476
 #### 步骤二:部署项目:
@@ -1599,7 +1606,8 @@ Tips: 历史版本(V1.3.x)目前已经Release至稳定版本, 进入维护阶段
1599 1606
 - 21、调度中心回调API服务改为restful方式;
1600 1607
 - 22、调度中心日志删除优化,改为分页获取ID并根据ID删除的方式,避免批量删除海量日志导致死锁问题;
1601 1608
 - 23、调度报表优化:新增日志报表的存储表,三天内的任务日志会以每分钟一次的频率异步同步至报表中;任务报表仅读取报表数据,极大提升加载速度;
1602
-- 24、[ing]调度日志优化:支持设置日志保留天数,过期日志天维度记录报表,并清理;调度报表汇总实时数据和报表;
1609
+- 24、调度中心日志自动清理功能:调度中心新增配置项("xxl.job.logretentiondays")日志保存天数,过期日志自动清理;解决海量日志情况下日志表慢SQL问题;限制大于等于7时生效,否则关闭清理功能,默认为30;
1610
+
1603 1611
 
1604 1612
 
1605 1613
 

+ 10 - 0
xxl-job-admin/src/main/java/com/xxl/job/admin/core/conf/XxlJobAdminConfig.java Vedi File

@@ -62,6 +62,9 @@ public class XxlJobAdminConfig implements InitializingBean, DisposableBean {
62 62
     @Value("${xxl.job.triggerpool.slow.max}")
63 63
     private int triggerPoolSlowMax;
64 64
 
65
+    @Value("${xxl.job.logretentiondays}")
66
+    private int logretentiondays;
67
+
65 68
     // dao, service
66 69
 
67 70
     @Resource
@@ -106,6 +109,13 @@ public class XxlJobAdminConfig implements InitializingBean, DisposableBean {
106 109
         return triggerPoolSlowMax;
107 110
     }
108 111
 
112
+    public int getLogretentiondays() {
113
+        if (logretentiondays < 7) {
114
+            return -1;  // Limit greater than or equal to 7, otherwise close
115
+        }
116
+        return logretentiondays;
117
+    }
118
+
109 119
     public XxlJobLogDao getXxlJobLogDao() {
110 120
         return xxlJobLogDao;
111 121
     }

+ 33 - 2
xxl-job-admin/src/main/java/com/xxl/job/admin/core/thread/JobLogReportHelper.java Vedi File

@@ -7,6 +7,7 @@ import org.slf4j.LoggerFactory;
7 7
 
8 8
 import java.util.Calendar;
9 9
 import java.util.Date;
10
+import java.util.List;
10 11
 import java.util.Map;
11 12
 import java.util.concurrent.TimeUnit;
12 13
 
@@ -32,11 +33,15 @@ public class JobLogReportHelper {
32 33
             @Override
33 34
             public void run() {
34 35
 
35
-                // monitor
36
+                // last clean log time
37
+                long lastCleanLogTime = 0;
38
+
39
+
36 40
                 while (!toStop) {
41
+
42
+                    // 1、log-report refresh: refresh log report in 3 days
37 43
                     try {
38 44
 
39
-                        // refresh log report in 3 days
40 45
                         for (int i = 0; i < 3; i++) {
41 46
 
42 47
                             // today
@@ -88,6 +93,32 @@ public class JobLogReportHelper {
88 93
                         }
89 94
                     }
90 95
 
96
+                    // 2、log-clean: switch open & once each day
97
+                    if (XxlJobAdminConfig.getAdminConfig().getLogretentiondays()>0
98
+                            && System.currentTimeMillis() - lastCleanLogTime > 24*60*60*1000) {
99
+
100
+                        // expire-time
101
+                        Calendar expiredDay = Calendar.getInstance();
102
+                        expiredDay.add(Calendar.DAY_OF_MONTH, -1 * XxlJobAdminConfig.getAdminConfig().getLogretentiondays());
103
+                        expiredDay.set(Calendar.HOUR_OF_DAY, 0);
104
+                        expiredDay.set(Calendar.MINUTE, 0);
105
+                        expiredDay.set(Calendar.SECOND, 0);
106
+                        expiredDay.set(Calendar.MILLISECOND, 0);
107
+                        Date clearBeforeTime = expiredDay.getTime();
108
+
109
+                        // clean expired log
110
+                        List<Long> logIds = null;
111
+                        do {
112
+                            logIds = XxlJobAdminConfig.getAdminConfig().getXxlJobLogDao().findClearLogIds(0, 0, clearBeforeTime, 0, 1000);
113
+                            if (logIds!=null && logIds.size()>0) {
114
+                                XxlJobAdminConfig.getAdminConfig().getXxlJobLogDao().clearLog(logIds);
115
+                            }
116
+                        } while (logIds!=null && logIds.size()>0);
117
+
118
+                        // update clean time
119
+                        lastCleanLogTime = System.currentTimeMillis();
120
+                    }
121
+
91 122
                     try {
92 123
                         TimeUnit.MINUTES.sleep(1);
93 124
                     } catch (Exception e) {