Server log analysis tool (identify, extract, merge, and count exception information)

demand ： I need to check yesterday's log today , And pay attention to the exceptions , In order to deal with some of these problems .
Manual reading takes a long time , Generally, there are more than ten log files a day , It's impossible to see one by one , And a lot of log information is info Information , No need to pay attention . So a gadget is implemented to automatically read all log files , Extract the exception information , For business exceptions, the same are merged together , In this way, you only need to view the generated exception information file , It saves a lot of time . Here is the code implementation ：

import com.google.common.collect.Maps;
import org.springframework.util.CollectionUtils;

import java.io.*;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.zip.GZIPInputStream;

/** * @author visy.wang * @desc  Log reading  * @date 2020/11/25 10:39 */
public class LogReader {
    

    // Keywords for extracting exception information 
	private static final String[] KEY_WORDS = {
    "xception", "at "};
	// Save statistics of business exceptions 
	private static Map<String, Integer> errMap = Maps.newHashMap();

	/** *  Read exception information  * @param folder  Folder path  * @param ignores  Ignored keywords （ If there is a keyword in it, skip ） */
	private static void readException(String folder, String... ignores) throws Exception{
    

		File dir = new File(folder);
		File[] fileArr = dir.listFiles();
		if(fileArr==null || fileArr.length==0){
    
			System.out.println(" The folder is empty ");
			return;
		}
		
		// decompression 
		for(File in: fileArr){
    
			if(!in.isDirectory() && in.getName().endsWith(".gz")){
    
				unGzip(in);
			}
		}

		// Filter & Sort 
		List<File> files = Arrays.stream(fileArr).filter(f -> {
    
			String name = f.getName();
			// Only deal with debug.*.log file , Modifiable 
			return !f.isDirectory() && name.startsWith("debug") && name.endsWith(".log");
		}).sorted(Comparator.comparing(f->{
    
		    // File sorting ,  You can delete this part of the code without sorting , Or modify other sorting rules 
		    // Name example ：debug.2020-11-24.1.log, Extract numbers 1 Used to sort 
			String name = f.getName();
			int secondIndex = name.indexOf(".",name.indexOf(".")+1);
			String index = name.substring(secondIndex+1, name.lastIndexOf("."));
			return Integer.parseInt(index);
		})).collect(Collectors.toList());

		if(CollectionUtils.isEmpty(files)){
    
			System.out.println(" Log file not found ");
			return;
		}

		// Create output file 
		String yesterday = LocalDate.now().minusDays(1).format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
		String outputFileName = "exceptions."+ yesterday +".log";
		File outputFile = new File(folder+"\\"+outputFileName);
		if(!outputFile.exists()){
    
			if(!outputFile.createNewFile()){
    
				System.out.println(outputFileName+" Create failure ！！！");
			}
		}
		FileWriter writer = new FileWriter(outputFile);

		for(File file: files){
    
			String fileName = file.getName();
			System.out.println(" Reading ："+fileName+" ...");

			writer.write("LOG_FILE_NAME-------------------->"+ fileName+"\n");

			String tempStr;
			BufferedReader reader = new BufferedReader(new FileReader(file));
			boolean flag = false;
			while ((tempStr = reader.readLine()) != null) {
    
				if(isIncludes(tempStr, KEY_WORDS) && !isIncludes(tempStr, ignores)){
    
					// Control business exceptions to print only once （ Merge ）
					//ServiceException Is the custom business exception name of my project , It can be modified according to its own name 
					if(tempStr.contains("ServiceException:")){
    
						String errMsg = tempStr.substring(tempStr.indexOf(":")+1).trim();
						Integer count;
						if((count=errMap.get(errMsg))==null){
    
							errMap.put(errMsg, 1);
							flag = false;
						}else{
    
							errMap.put(errMsg, count+1);
							flag = true;
							continue;
						}
					}else{
    
						if(tempStr.contains("at ")){
    
							if(flag){
    
								continue;
							}
						}else{
    
							flag = false;
						}
					}
					writer.write(tempStr+"\n");
				}
			}
			reader.close();
		}

		writer.write("\n\n-------------- Business exception statistics -------------------\n");
		for(String k : errMap.keySet()){
    
			writer.write(k+": "+ errMap.get(k)+" Time \n");
		}

		writer.close();

		System.out.println(" Exception file generation completed !!!");

	}

	// Whether to find the keyword 
	private static boolean isIncludes(String tempStr, String[] keyWords){
    
		for(String keyWord: keyWords){
    
			if(tempStr.contains(keyWord)){
    
				return true;
			}
		}
		return false;
	}
	
	// decompression 
	private static void unGzip(File inFile) throws Exception{
    
		System.out.println(" Decompressing ："+ inFile.getName() + "...");
		// establish gzip Compressed file input stream 
		FileInputStream fin = new FileInputStream(inFile);
		// establish gzip Decompression workflow 
		GZIPInputStream gzin = new GZIPInputStream(fin);
		// Establish the output stream of decompressed files 
		String path = inFile.getAbsolutePath();
		String outPath = path.substring(0, path.lastIndexOf('.'));
		FileOutputStream fout = new FileOutputStream(outPath);

		int num;
		byte[] buf=new byte[1024];

		while ((num = gzin.read(buf,0,buf.length)) != -1) {
    
			fout.write(buf,0,num);
		}

		gzin.close();
		fout.close();
		fin.close();

		// Delete source file 
		inFile.delete();
	}

	public static void main(String[] args) throws Exception{
    
		// Put all the log files to be analyzed on the desktop logs Under the folder , The file must be decompressed .log file 
		// After the analysis, it will be in logs Generate under folder exceptions.yyyy-MM-dd.log file , You can see it. 
		// All business exceptions will only print the exception information once , At the end of the file, there are the number of exceptions 
		readException("C:\\Users\\admin\\Desktop\\logs", "aaa", "bbb");
	}
}