网站的构思,新媒体宣传推广方案,杭州外贸网站制作,深圳公共资源交易平台日志分析
场景
运维嫌弃生产环境打印日志过多#xff0c;而且日志存储需要费用#xff0c;让我们减少打印日志大小#xff0c;所以需要分析日志在哪里打印的过多
解决方案
读取生产日志文件#xff0c;统计分析打印日志的地方#xff0c;最后删除代码中打印日志的地方…日志分析
场景
运维嫌弃生产环境打印日志过多而且日志存储需要费用让我们减少打印日志大小所以需要分析日志在哪里打印的过多
解决方案
读取生产日志文件统计分析打印日志的地方最后删除代码中打印日志的地方
实现-LogParAnalyzer
public class LogParAnalyzer {//日志原始文件private File log;private ListPattern list;private ExecutorService executorService;//生成的分割文件private String subPath D:\\split\\;private ListFile files;public LogParAnalyzer(File log, ListString patterns) {this.log log;executorService Executors.newFixedThreadPool(30);list new ArrayList();try {for (String pattern : patterns) {Pattern p Pattern.compile(pattern);list.add(p);}} catch (Exception e) {throw new RuntimeException(e);}}public void analyze() throws Exception {// 使用 try-with-resources 自动关闭 BufferedReaderint chunkSize 100000;try (BufferedReader reader new BufferedReader(new FileReader(log))) {File file new File(subPath);if (!file.exists()) {file.mkdirs();}String line;ListCompletableFuture? task new ArrayList();int cur 0;ListString list new ArrayList();AtomicInteger batch new AtomicInteger(0);while ((line reader.readLine()) ! null) {//sb 会通过Arrays.copy复制字节数组内存频繁复制list.add(line);cur;if ((cur % chunkSize) 0) {//深拷贝ListString tt list.stream().map(String::new).collect(Collectors.toList());list.clear();CompletableFuture f CompletableFuture.runAsync(() - processChunk(tt, batch.get()), executorService);task.add(f);batch.incrementAndGet();}}if (list.size()0) {CompletableFuture f CompletableFuture.runAsync(() - processChunk(list, batch.get()), executorService);task.add(f);}//等待所有任务结束CompletableFuture.allOf(task.toArray(new CompletableFuture[0])).get();System.out.println(task execute finished);}}private void processChunk(ListString lines, int batch) {try {System.out.println(Thread.currentThread().getName() execute batch.txt start);MapString, AtomicInteger map new HashMap();try (BufferedWriter writer Files.newBufferedWriter(Paths.get(subPath batch .txt))) {lines.forEach(line - {for (Pattern pattern : list) {Matcher matcher pattern.matcher(line);if (matcher.find()) {String group matcher.group(0);map.computeIfAbsent(group, k - new AtomicInteger(1)).incrementAndGet();}}if (map.size() 0) {//每个文件只保存100前100条writeBatchToFile(writer, map);}});}System.out.println(Thread.currentThread().getName() execute batch.txt end);} catch (Exception e) {e.printStackTrace();}}private void writeBatchToFile(BufferedWriter writer, MapString, AtomicInteger map) {MapString, AtomicInteger limit limit(map, 100);try {for (Map.EntryString, AtomicInteger entry : limit.entrySet()) {writer.write(entry.getKey() entry.getValue().get());writer.newLine();}} catch (IOException e) {e.printStackTrace();}//清除缓存map.clear();limit.clear();}public void mergeAndSort() throws Exception {files Files.list(Paths.get(subPath)).map(Path::toFile).filter(f - f.length() 0).collect(Collectors.toList());// 创建 ForkJoinPoolForkJoinPool forkJoinPool new ForkJoinPool();MergeFileTask mergeFileTask new MergeFileTask(files.toArray(new File[0]), forkJoinPool);Path finalPath mergeFileTask.invoke();System.out.println(final path: finalPath.toAbsolutePath());try (BufferedReader reader Files.newBufferedReader(finalPath)) {String line;while ((line reader.readLine()) ! null) {System.out.println(line);}}mergeFileTask.finished();}public void finished() throws IOException {if (!CollectionUtils.isEmpty(files)){files.stream().parallel().forEach(File::delete);}Files.deleteIfExists(Paths.get(subPath));}public MapString, AtomicInteger limit(MapString, AtomicInteger map, int limit) {// 排序并过滤结果return map.entrySet().stream().sorted(Map.Entry.comparingByValue(Comparator.comparing(AtomicInteger::get).reversed())).limit(limit).collect(Collectors.toMap(Map.Entry::getKey,Map.Entry::getValue,(oldValue, newValue) - oldValue, // 解决键冲突LinkedHashMap::new));}}测试
Test
public void bb() throws Exception {StopWatch stopWatch new StopWatch();stopWatch.start();File log new File(E:\\log_total.log\\log_total.log);//2023-09-26 11:10:00.123 INFO - none --- [main] com.example.service.UserService.create:42 - User service started successfully.//匹配出 com.example.service.UserService.create:42//c.y.c.w..*指文件名太长缩写ListString list Arrays.asList((com\\.xx\\.[\\w\\.\\*]*:\\d*), (c\\.y\\.c\\.[\\w\\.\\*]*:\\d*));LogParAnalyzer logAnalyzer new LogParAnalyzer(log, list);logAnalyzer.analyze();logAnalyzer.mergeAndSort();logAnalyzer.finished();stopWatch.stop();System.out.println(stopWatch.prettyPrint());//最后统计格式//c.y.c.s.service.impl.Service.complete:98 count: 6
}good luck!
