|
@@ -0,0 +1,170 @@
|
|
|
+package com.xzl.web.utils;
|
|
|
+
|
|
|
+import com.alibaba.fastjson.JSON;
|
|
|
+import com.xzl.common.utils.DateUtils;
|
|
|
+import com.xzl.common.utils.file.FileUtils;
|
|
|
+import com.xzl.web.model.dataGovernance.entity.SysDataLog;
|
|
|
+import org.apache.commons.collections.MapUtils;
|
|
|
+
|
|
|
+import java.io.BufferedReader;
|
|
|
+import java.io.File;
|
|
|
+import java.io.FileFilter;
|
|
|
+import java.io.FileInputStream;
|
|
|
+import java.io.FileNotFoundException;
|
|
|
+import java.io.FileOutputStream;
|
|
|
+import java.io.FileReader;
|
|
|
+import java.io.IOException;
|
|
|
+import java.sql.Connection;
|
|
|
+import java.sql.Date;
|
|
|
+import java.sql.DriverManager;
|
|
|
+import java.sql.PreparedStatement;
|
|
|
+import java.sql.SQLException;
|
|
|
+import java.sql.Statement;
|
|
|
+import java.util.ArrayList;
|
|
|
+import java.util.List;
|
|
|
+import java.util.Map;
|
|
|
+import java.util.zip.GZIPInputStream;
|
|
|
+
|
|
|
+public class AuditLogUtils {
|
|
|
+
|
|
|
+
|
|
|
+ public static String getFileName(String f) {
|
|
|
+ String fname = "";
|
|
|
+ int i = f.lastIndexOf('.');
|
|
|
+
|
|
|
+ if (i > 0 && i < f.length() - 1) {
|
|
|
+ fname = f.substring(0, i);
|
|
|
+ }
|
|
|
+ return fname;
|
|
|
+ }
|
|
|
+
|
|
|
+ public static List<String> read(File logfile) {
|
|
|
+ List<String> data = new ArrayList<>();
|
|
|
+ // 按行读日志
|
|
|
+ try {
|
|
|
+ // 创建FileReader和BufferedReader对象来读取日志文件
|
|
|
+ FileReader fileReader = new FileReader(logfile);
|
|
|
+ BufferedReader bufferedReader = new BufferedReader(fileReader);
|
|
|
+ // 逐行读取日志文件内容并分析处理
|
|
|
+ String line;
|
|
|
+ while ((line = bufferedReader.readLine()) != null) {
|
|
|
+ data.add(line);
|
|
|
+ }
|
|
|
+ // 关闭文件流
|
|
|
+ bufferedReader.close();
|
|
|
+ fileReader.close();
|
|
|
+ } catch (IOException e) {
|
|
|
+ e.printStackTrace();
|
|
|
+ }
|
|
|
+ return data;
|
|
|
+ }
|
|
|
+
|
|
|
+ public static List<String> uncompress(File gzfile) {
|
|
|
+ List<String> data = new ArrayList<>();
|
|
|
+ // 解压文件
|
|
|
+ GZIPInputStream in = null;
|
|
|
+ try {
|
|
|
+ in = new GZIPInputStream(new FileInputStream(gzfile));
|
|
|
+ System.out.println("Open the output file.");
|
|
|
+ String outFileName = getFileName(gzfile.getPath());
|
|
|
+ FileOutputStream out = null;
|
|
|
+ try {
|
|
|
+ out = new FileOutputStream(outFileName);
|
|
|
+ } catch (FileNotFoundException e) {
|
|
|
+ System.err.println("Could not write to file. " + outFileName);
|
|
|
+ }
|
|
|
+
|
|
|
+ System.out.println("Transfering bytes from compressed file to the output file.");
|
|
|
+ byte[] buf = new byte[1024];
|
|
|
+ int len;
|
|
|
+ while ((len = in.read(buf)) > 0) {
|
|
|
+ out.write(buf, 0, len);
|
|
|
+ }
|
|
|
+
|
|
|
+ System.out.println("Closing the file and stream");
|
|
|
+ in.close();
|
|
|
+ out.close();
|
|
|
+
|
|
|
+ // 删除 gz文件
|
|
|
+ FileUtils.deleteFile(gzfile.getPath());
|
|
|
+ return read(new File(outFileName));
|
|
|
+ } catch (IOException e) {
|
|
|
+ e.printStackTrace();
|
|
|
+ System.exit(1);
|
|
|
+ }
|
|
|
+ return data;
|
|
|
+ }
|
|
|
+
|
|
|
+ public static void main(String[] args) {
|
|
|
+ String logPath = "/usr/local/file";
|
|
|
+ /*
|
|
|
+ File logDir = new File(logPath);
|
|
|
+ if (logDir.exists()) {
|
|
|
+ File[] files = logDir.listFiles((o) -> o.getName().toLowerCase().endsWith(".gz"));
|
|
|
+ for (File gzfile : files) {
|
|
|
+ List<Map> data = uncompress(gzfile);
|
|
|
+ System.out.println(data.size());
|
|
|
+ }
|
|
|
+ }
|
|
|
+ */
|
|
|
+// List<String> data = read(new File(logPath, "mysql-audit.json-10.70.192.124-20240117000101"));
|
|
|
+ List<String> data = read(new File(logPath, "mysql-audit.json-10.70.192.124-20240118000101"));
|
|
|
+// System.out.println(data.size());
|
|
|
+ long beginTime = System.currentTimeMillis();
|
|
|
+ saveLog(data);
|
|
|
+ System.out.println("耗时 : " + (System.currentTimeMillis() - beginTime) / 1000 + " 秒");
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ private static void saveLog(List<String> data) {
|
|
|
+ // 连接数据库
|
|
|
+
|
|
|
+ int count = 1000;
|
|
|
+ Connection conn = null;
|
|
|
+ try {
|
|
|
+ Class.forName("com.mysql.cj.jdbc.Driver");
|
|
|
+ conn = DriverManager.getConnection("jdbc:mysql://10.70.192.124:3306/xtdb?allowMultiQueries=true&useUnicode=true&characterEncoding=UTF-8&useSSL=false", "audit", "xtyc@2023@xzl");
|
|
|
+ // 开始事务
|
|
|
+ conn.setAutoCommit(false);
|
|
|
+ String sql = "INSERT INTO audit_log (log_content, user, ip, method, create_time) VALUE (?,?,?,?,?)";
|
|
|
+ PreparedStatement ps = conn.prepareStatement(sql);
|
|
|
+ int size = data.size();
|
|
|
+ for (int i = 0; i < size; i++) {
|
|
|
+ // 添加到批处理队列
|
|
|
+ String line = data.get(i);
|
|
|
+ Map row = JSON.parseObject(line);
|
|
|
+ ps.setString(1, line);
|
|
|
+ ps.setString(2, MapUtils.getString(row, "user"));
|
|
|
+ ps.setString(3, MapUtils.getString(row, "ip"));
|
|
|
+ ps.setString(4, MapUtils.getString(row, "cmd"));
|
|
|
+ ps.setString(5, DateUtils.parseDateToStr(DateUtils.YYYY_MM_DD_HH_MM_SS, new Date(Long.valueOf(MapUtils.getString(row, "date")))));
|
|
|
+ ps.addBatch();
|
|
|
+ if (i % count == 0) {
|
|
|
+ int[] result = ps.executeBatch();
|
|
|
+ conn.commit();
|
|
|
+ ps.clearBatch();
|
|
|
+ System.out.println("已提交 " + result.length + " 条记录");
|
|
|
+ }
|
|
|
+ }
|
|
|
+ int[] result = ps.executeBatch();
|
|
|
+ System.out.println("已提交 " + result.length + " 条记录");
|
|
|
+ // 结束事务并自动提交更改
|
|
|
+ conn.commit();
|
|
|
+ } catch (SQLException | ClassNotFoundException e) {
|
|
|
+ e.printStackTrace();
|
|
|
+ // 发生错误时回滚事务
|
|
|
+ try {
|
|
|
+ conn.rollback();
|
|
|
+ } catch (SQLException ex) {
|
|
|
+ ex.printStackTrace();
|
|
|
+ }
|
|
|
+ } finally {
|
|
|
+ // 关闭连接
|
|
|
+ try {
|
|
|
+ conn.close();
|
|
|
+ } catch (SQLException e) {
|
|
|
+ e.printStackTrace();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|