| 
															
																@@ -10,10 +10,9 @@ import org.slf4j.Logger; 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 import org.slf4j.LoggerFactory; 
															 | 
															
															 | 
															
																 import org.slf4j.LoggerFactory; 
															 | 
														
													
												
													
														
															| 
															 | 
															
																  
															 | 
															
															 | 
															
																  
															 | 
														
													
												
													
														
															| 
															 | 
															
																 import java.text.MessageFormat; 
															 | 
															
															 | 
															
																 import java.text.MessageFormat; 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-import java.util.Arrays; 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-import java.util.HashSet; 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-import java.util.Set; 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-import java.util.concurrent.*; 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+import java.util.*; 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+import java.util.concurrent.LinkedBlockingQueue; 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+import java.util.concurrent.TimeUnit; 
															 | 
														
													
												
													
														
															| 
															 | 
															
																  
															 | 
															
															 | 
															
																  
															 | 
														
													
												
													
														
															| 
															 | 
															
																 /** 
															 | 
															
															 | 
															
																 /** 
															 | 
														
													
												
													
														
															| 
															 | 
															
																  * job monitor instance 
															 | 
															
															 | 
															
																  * job monitor instance 
															 | 
														
													
												
											
												
													
														
															 | 
															
																@@ -36,38 +35,30 @@ public class JobFailMonitorHelper { 
															 | 
														
													
												
													
														
															| 
															 | 
															
																  
															 | 
															
															 | 
															
																  
															 | 
														
													
												
													
														
															| 
															 | 
															
																 			@Override 
															 | 
															
															 | 
															
																 			@Override 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 			public void run() { 
															 | 
															
															 | 
															
																 			public void run() { 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+				// monitor 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 				while (!toStop) { 
															 | 
															
															 | 
															
																 				while (!toStop) { 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 					try { 
															 | 
															
															 | 
															
																 					try { 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-						logger.debug(">>>>>>>>>>> job monitor beat ... "); 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 						Integer jobLogId = JobFailMonitorHelper.instance.queue.take(); 
															 | 
															
															 | 
															
																 						Integer jobLogId = JobFailMonitorHelper.instance.queue.take(); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 						if (jobLogId != null && jobLogId > 0) { 
															 | 
															
															 | 
															
																 						if (jobLogId != null && jobLogId > 0) { 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-							logger.debug(">>>>>>>>>>> job monitor heat success, JobLogId:{}", jobLogId); 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 							XxlJobLog log = XxlJobDynamicScheduler.xxlJobLogDao.load(jobLogId); 
															 | 
															
															 | 
															
																 							XxlJobLog log = XxlJobDynamicScheduler.xxlJobLogDao.load(jobLogId); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 							if (log!=null) { 
															 | 
															
															 | 
															
																 							if (log!=null) { 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 								if (ReturnT.SUCCESS_CODE==log.getTriggerCode() && log.getHandleCode()==0) { 
															 | 
															
															 | 
															
																 								if (ReturnT.SUCCESS_CODE==log.getTriggerCode() && log.getHandleCode()==0) { 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-									// running 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-									try { 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-										TimeUnit.SECONDS.sleep(10); 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-									} catch (InterruptedException e) { 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-										e.printStackTrace(); 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-									} 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+									// job running, wait + again monitor 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+									TimeUnit.SECONDS.sleep(10); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 									JobFailMonitorHelper.monitor(jobLogId); 
															 | 
															
															 | 
															
																 									JobFailMonitorHelper.monitor(jobLogId); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+									logger.info(">>>>>>>>>>> job monitor, job running, JobLogId:{}", jobLogId); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 								} 
															 | 
															
															 | 
															
																 								} 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 								if (ReturnT.SUCCESS_CODE==log.getTriggerCode() && ReturnT.SUCCESS_CODE==log.getHandleCode()) { 
															 | 
															
															 | 
															
																 								if (ReturnT.SUCCESS_CODE==log.getTriggerCode() && ReturnT.SUCCESS_CODE==log.getHandleCode()) { 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-									// pass 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+									// job success, pass 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+									logger.info(">>>>>>>>>>> job monitor, job success, JobLogId:{}", jobLogId); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 								} 
															 | 
															
															 | 
															
																 								} 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-								if (ReturnT.FAIL_CODE == log.getTriggerCode()|| ReturnT.FAIL_CODE==log.getHandleCode()) { 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-									XxlJobInfo info = XxlJobDynamicScheduler.xxlJobInfoDao.loadById(log.getJobId()); 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-									if (info!=null && info.getAlarmEmail()!=null && info.getAlarmEmail().trim().length()>0) { 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																  
															 | 
															
															 | 
															
																  
															 | 
														
													
												
													
														
															| 
															 | 
															
																-										Set<String> emailSet = new HashSet<String>(Arrays.asList(info.getAlarmEmail().split(","))); 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-										for (String email: emailSet) { 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-											String title = "《调度监控报警》(任务调度中心XXL-JOB)"; 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-											XxlJobGroup group = XxlJobDynamicScheduler.xxlJobGroupDao.load(Integer.valueOf(info.getJobGroup())); 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-											String content = MessageFormat.format("任务调度失败, 执行器名称:{0}, 任务描述:{1}.", group!=null?group.getTitle():"null", info.getJobDesc()); 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-											MailUtil.sendMail(email, title, content, false, null); 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-										} 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-									} 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+								if (ReturnT.FAIL_CODE == log.getTriggerCode()|| ReturnT.FAIL_CODE==log.getHandleCode()) { 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+									// job fail, 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+									sendMonitorEmail(log); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+									logger.info(">>>>>>>>>>> job monitor, job fail, JobLogId:{}", jobLogId); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 								} 
															 | 
															
															 | 
															
																 								} 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 							} 
															 | 
															
															 | 
															
																 							} 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 						} 
															 | 
															
															 | 
															
																 						} 
															 | 
														
													
												
											
												
													
														
															 | 
															
																@@ -75,15 +66,54 @@ public class JobFailMonitorHelper { 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 						logger.error("job monitor error:{}", e); 
															 | 
															
															 | 
															
																 						logger.error("job monitor error:{}", e); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 					} 
															 | 
															
															 | 
															
																 					} 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 				} 
															 | 
															
															 | 
															
																 				} 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+				// monitor all clear 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+				List<Integer> jobLogIdList = new ArrayList<Integer>(); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+				int drainToNum = getInstance().queue.drainTo(jobLogIdList); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+				if (jobLogIdList!=null && jobLogIdList.size()>0) { 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+					for (Integer jobLogId: jobLogIdList) { 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+						XxlJobLog log = XxlJobDynamicScheduler.xxlJobLogDao.load(jobLogId); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+						if (ReturnT.FAIL_CODE == log.getTriggerCode()|| ReturnT.FAIL_CODE==log.getHandleCode()) { 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+							// job fail, 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+							sendMonitorEmail(log); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+							logger.info(">>>>>>>>>>> job monitor last, job fail, JobLogId:{}", jobLogId); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+						} 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+					} 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+				} 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 			} 
															 | 
															
															 | 
															
																 			} 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 		}); 
															 | 
															
															 | 
															
																 		}); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 		monitorThread.setDaemon(true); 
															 | 
															
															 | 
															
																 		monitorThread.setDaemon(true); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 		monitorThread.start(); 
															 | 
															
															 | 
															
																 		monitorThread.start(); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 	} 
															 | 
															
															 | 
															
																 	} 
															 | 
														
													
												
													
														
															| 
															 | 
															
																  
															 | 
															
															 | 
															
																  
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+	/** 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+	 * send monitor email 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+	 * @param jobLog 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+	 */ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+	private void sendMonitorEmail(XxlJobLog jobLog){ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+		XxlJobInfo info = XxlJobDynamicScheduler.xxlJobInfoDao.loadById(jobLog.getJobId()); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+		if (info!=null && info.getAlarmEmail()!=null && info.getAlarmEmail().trim().length()>0) { 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+			Set<String> emailSet = new HashSet<String>(Arrays.asList(info.getAlarmEmail().split(","))); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+			for (String email: emailSet) { 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+				String title = "《调度监控报警》(任务调度中心XXL-JOB)"; 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+				XxlJobGroup group = XxlJobDynamicScheduler.xxlJobGroupDao.load(Integer.valueOf(info.getJobGroup())); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+				String content = MessageFormat.format("任务调度失败, 执行器名称:{0}, 任务描述:{1}.", group!=null?group.getTitle():"null", info.getJobDesc()); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+				MailUtil.sendMail(email, title, content, false, null); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+			} 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+		} 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+	} 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 	public void toStop(){ 
															 | 
															
															 | 
															
																 	public void toStop(){ 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 		toStop = true; 
															 | 
															
															 | 
															
																 		toStop = true; 
															 | 
														
													
												
													
														
															| 
															 | 
															
																-		//monitorThread.interrupt(); 
															 | 
															
															 | 
															
																 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+		// interrupt and wait 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+		monitorThread.interrupt(); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+		try { 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+			monitorThread.join(); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+		} catch (InterruptedException e) { 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+			logger.error(e.getMessage(), e); 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 
															 | 
															
															 | 
															
																+		} 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 	} 
															 | 
															
															 | 
															
																 	} 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 	 
															 | 
															
															 | 
															
																 	 
															 | 
														
													
												
													
														
															| 
															 | 
															
																 	// producer 
															 | 
															
															 | 
															
																 	// producer 
															 |