Sfoglia il codice sorgente

调度中心日志删除优化,改为分页获取ID并根据ID删除的方式,避免批量删除海量日志导致死锁问题

xuxueli 5 anni fa
parent
commit
e73be8175c

+ 3 - 2
doc/XXL-JOB官方文档.md Vedi File

1590
 - 19、调度中心移除SQL中的 "now()" 函数;集群部署时不再依赖DB时钟,仅需要保证调度中心应用节点时钟一致即可;
1590
 - 19、调度中心移除SQL中的 "now()" 函数;集群部署时不再依赖DB时钟,仅需要保证调度中心应用节点时钟一致即可;
1591
 - 20、xxl-rpc服务端线程优化,降低线程内存开销;
1591
 - 20、xxl-rpc服务端线程优化,降低线程内存开销;
1592
 - 21、调度中心回调API服务改为restful方式;
1592
 - 21、调度中心回调API服务改为restful方式;
1593
-- 22、[ING]调度日志优化:支持设置日志保留天数,过期日志天维度记录报表,并清理;调度报表汇总实时数据和报表
1594
-- 23、[ING]调度中心日志删除,改为分页获取ID,根据ID删除的方式
1593
+- 22、调度中心日志删除优化,改为分页获取ID并根据ID删除的方式,避免批量删除海量日志导致死锁问题
1594
+- 23、[ING]调度日志优化:支持设置日志保留天数,过期日志天维度记录报表,并清理;调度报表汇总实时数据和报表
1595
 
1595
 
1596
 
1596
 
1597
 
1597
 
1632
 - 34、通讯调整;双向HTTP,回调和其他API自定义AccessToken,Restful,执行器复用容器端口;
1632
 - 34、通讯调整;双向HTTP,回调和其他API自定义AccessToken,Restful,执行器复用容器端口;
1633
 - 35、父子任务参数传递;流程任务等,透传动态参数;
1633
 - 35、父子任务参数传递;流程任务等,透传动态参数;
1634
 - 36、任务操作API服务调整为和回调服务一致,降低接入成本;
1634
 - 36、任务操作API服务调整为和回调服务一致,降低接入成本;
1635
+- 37、新增执行器描述、任务描述属性;
1635
 
1636
 
1636
 
1637
 
1637
 ## 七、其他
1638
 ## 七、其他

+ 8 - 1
xxl-job-admin/src/main/java/com/xxl/job/admin/controller/JobLogController.java Vedi File

216
 			return new ReturnT<String>(ReturnT.FAIL_CODE, I18nUtil.getString("joblog_clean_type_unvalid"));
216
 			return new ReturnT<String>(ReturnT.FAIL_CODE, I18nUtil.getString("joblog_clean_type_unvalid"));
217
 		}
217
 		}
218
 
218
 
219
-		xxlJobLogDao.clearLog(jobGroup, jobId, clearBeforeTime, clearBeforeNum);
219
+		List<Long> logIds = null;
220
+		do {
221
+			logIds = xxlJobLogDao.findClearLogIds(jobGroup, jobId, clearBeforeTime, clearBeforeNum, 1000);
222
+			if (logIds!=null && logIds.size()>0) {
223
+				xxlJobLogDao.clearLog(logIds);
224
+			}
225
+		} while (logIds!=null && logIds.size()>0);
226
+
220
 		return ReturnT.SUCCESS;
227
 		return ReturnT.SUCCESS;
221
 	}
228
 	}
222
 
229
 

+ 6 - 4
xxl-job-admin/src/main/java/com/xxl/job/admin/dao/XxlJobLogDao.java Vedi File

46
 	public List<Map<String, Object>> triggerCountByDay(@Param("from") Date from,
46
 	public List<Map<String, Object>> triggerCountByDay(@Param("from") Date from,
47
 													   @Param("to") Date to);
47
 													   @Param("to") Date to);
48
 
48
 
49
-	public int clearLog(@Param("jobGroup") int jobGroup,
50
-						@Param("jobId") int jobId,
51
-						@Param("clearBeforeTime") Date clearBeforeTime,
52
-						@Param("clearBeforeNum") int clearBeforeNum);
49
+	public List<Long> findClearLogIds(@Param("jobGroup") int jobGroup,
50
+									  @Param("jobId") int jobId,
51
+									  @Param("clearBeforeTime") Date clearBeforeTime,
52
+									  @Param("clearBeforeNum") int clearBeforeNum,
53
+									  @Param("pagesize") int pagesize);
54
+	public int clearLog(@Param("logIds") List<Long> logIds);
53
 
55
 
54
 	public List<Long> findFailJobLogIds(@Param("pagesize") int pagesize);
56
 	public List<Long> findFailJobLogIds(@Param("pagesize") int pagesize);
55
 
57
 

+ 25 - 15
xxl-job-admin/src/main/resources/mybatis-mapper/XxlJobLogMapper.xml Vedi File

186
 		ORDER BY triggerDay
186
 		ORDER BY triggerDay
187
     </select>
187
     </select>
188
 
188
 
189
-	<delete id="clearLog" >
190
-		delete from xxl_job_log
189
+	<select id="findClearLogIds" resultType="long" >
190
+		SELECT id FROM xxl_job_log
191
 		<trim prefix="WHERE" prefixOverrides="AND | OR" >
191
 		<trim prefix="WHERE" prefixOverrides="AND | OR" >
192
 			<if test="jobGroup gt 0">
192
 			<if test="jobGroup gt 0">
193
 				AND job_group = #{jobGroup}
193
 				AND job_group = #{jobGroup}
200
 			</if>
200
 			</if>
201
 			<if test="clearBeforeNum gt 0">
201
 			<if test="clearBeforeNum gt 0">
202
 				AND id NOT in(
202
 				AND id NOT in(
203
-					SELECT id FROM(
204
-						SELECT id FROM xxl_job_log AS t
205
-						<trim prefix="WHERE" prefixOverrides="AND | OR" >
206
-							<if test="jobGroup gt 0">
207
-								AND t.job_group = #{jobGroup}
208
-							</if>
209
-							<if test="jobId gt 0">
210
-								AND t.job_id = #{jobId}
211
-							</if>
212
-						</trim>
213
-						ORDER BY t.trigger_time desc
214
-						LIMIT 0, #{clearBeforeNum}
215
-					) t1
203
+				SELECT id FROM(
204
+				SELECT id FROM xxl_job_log AS t
205
+				<trim prefix="WHERE" prefixOverrides="AND | OR" >
206
+					<if test="jobGroup gt 0">
207
+						AND t.job_group = #{jobGroup}
208
+					</if>
209
+					<if test="jobId gt 0">
210
+						AND t.job_id = #{jobId}
211
+					</if>
212
+				</trim>
213
+				ORDER BY t.trigger_time desc
214
+				LIMIT 0, #{clearBeforeNum}
215
+				) t1
216
 				)
216
 				)
217
 			</if>
217
 			</if>
218
 		</trim>
218
 		</trim>
219
+		order by id asc
220
+		LIMIT #{pagesize}
221
+	</select>
222
+
223
+	<delete id="clearLog" >
224
+		delete from xxl_job_log
225
+		WHERE id in
226
+		<foreach collection="logIds" item="item" open="(" close=")" separator="," >
227
+			#{item}
228
+		</foreach>
219
 	</delete>
229
 	</delete>
220
 
230
 
221
 	<select id="findFailJobLogIds" resultType="long" >
231
 	<select id="findFailJobLogIds" resultType="long" >