xzl-admin/pom.xml

@@ -135,6 +135,19 @@
             <artifactId>pinyin4j</artifactId>
             <version>2.5.1</version>
         </dependency>
+
+<!--        网页处理-->
+        <dependency>
+            <groupId>org.jsoup</groupId>
+            <artifactId>jsoup</artifactId>
+            <version>1.17.2</version> <!-- 推荐使用最新版本 -->
+        </dependency>
+
+        <dependency>
+            <groupId>org.seleniumhq.selenium</groupId>
+            <artifactId>selenium-java</artifactId>
+            <version>3.141.59</version>
+        </dependency>
     </dependencies>
 
     <build>

xzl-admin/src/main/java/com/xzl/HtmlTextExtractor.java

@@ -0,0 +1,79 @@
+package com.xzl;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class HtmlTextExtractor {
+    // 1. 定义HTML处理相关正则
+    // 匹配所有HTML标签（包括自闭合标签，如<br/>）
+    private static final Pattern HTML_TAG_PATTERN = Pattern.compile("<[^>]+>");
+    // 匹配HTML实体（如&nbsp;、&lt;）
+    private static final Pattern HTML_ENTITY_PATTERN = Pattern.compile("&[#a-zA-Z0-9]+;");
+
+    public static void main(String[] args) {
+        // 测试用HTML内容（含标签、实体、多余空白）
+        String htmlContent = "  <div class=\"article\">\n" +
+                "    <h1>Java提取HTML文本</h1>\n" +
+                "    <p>这是一段带<span style=\"color:red\">标签</span>的文本，&nbsp;包含&nbsp;空格实体。</p>\n" +
+                "    <p>  多余空格、\t制表符、\n换行符都需要优化  </p>\n" +
+                "  </div>";
+
+        // 2. 提取并优化文本
+        String pureText = extractAndOptimizeText(htmlContent);
+
+        // 3. 输出结果
+        System.out.println("优化后的纯文本：");
+        System.out.println(pureText);
+    }
+
+    /**
+     * 提取HTML中的纯文本并优化
+     * @param html 原始HTML字符串
+     * @return 优化后的纯文本
+     */
+    public static String extractAndOptimizeText(String html) {
+        if (html == null || html.trim().isEmpty()) {
+            return "";
+        }
+
+        String text = html;
+
+        // 步骤1：移除所有HTML标签
+        Matcher tagMatcher = HTML_TAG_PATTERN.matcher(text);
+        text = tagMatcher.replaceAll("");
+
+        // 步骤2：替换HTML实体为普通字符（如&nbsp;→空格，&lt;→<）
+        text = replaceHtmlEntities(text);
+
+        // 步骤3：优化文本（去多余空白、首尾空格）
+        text = optimizeText(text);
+
+        return text;
+    }
+
+    /**
+     * 替换常见HTML实体为普通字符
+     */
+    private static String replaceHtmlEntities(String text) {
+        // 常见实体映射（可根据需求扩展，如&copy;→©）
+        return text.replace("&nbsp;", " ")
+                   .replace("&lt;", "<")
+                   .replace("&gt;", ">")
+                   .replace("&amp;", "&")
+                   .replace("&quot;", "\"")
+                   .replace("&apos;", "'");
+    }
+
+    /**
+     * 优化文本：移除多余空白、过滤无意义字符
+     */
+    private static String optimizeText(String text) {
+        // 1. 替换连续空白（空格、换行、制表符）为单个空格
+        text = text.replaceAll("\\s+", " ");
+        // 2. （可选）过滤特殊符号（如★、▌，保留字母/数字/中文）
+        // text = text.replaceAll("[^a-zA-Z0-9\\u4e00-\\u9fa5\\s,.!?；。！？]", "");
+        // 3. 去除首尾空白
+        text = text.trim();
+        return text;
+    }
+}

xzl-admin/src/main/java/com/xzl/JsoupHtmlExtractor.java

@@ -0,0 +1,83 @@
+package com.xzl;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.safety.Safelist;
+
+import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.stream.Stream;
+
+public class JsoupHtmlExtractor {
+    private static String filePath = "C:\\Users\\GZ\\Desktop\\test.txt";
+
+    public static void main(String[] args) {
+
+
+
+//        String filePath = "example.txt";
+        // 使用 try-with-resources 确保 BufferedReader 被自动关闭
+        try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) {
+            StringWriter writer = new StringWriter();
+            String line;
+
+            // 按行读取并写入 StringWriter
+            while ((line = reader.readLine()) != null) {
+                writer.write(line);
+                writer.write(System.lineSeparator()); // 写入换行符
+            }
+
+            // 从 StringWriter 中获取完整的字符串
+            String content = writer.toString();
+
+            System.out.println("文件内容:");
+            System.out.println(content);
+
+            // 提取并优化文本
+            String pureText = extractAndOptimizeWithJsoup(content);
+
+            System.out.println("Jsoup优化后的纯文本：");
+            System.out.println(pureText);
+
+        } catch (IOException e) {
+            System.err.println("读取文件时发生错误: " + e.getMessage());
+            e.printStackTrace();
+        }
+    }
+
+
+
+
+    /**
+     * 用Jsoup提取HTML纯文本并优化
+     */
+    public static String extractAndOptimizeWithJsoup(String html) {
+        if (html == null || html.trim().isEmpty()) {
+            return "";
+        }
+
+        // 步骤1：解析HTML（忽略文档类型、编码，自动处理嵌套）
+        Document doc = Jsoup.parse(html);
+
+        // 步骤2：提取纯文本（自动移除标签、注释、脚本）
+        // 方式1：直接获取所有文本（保留段落逻辑，用换行分隔）
+        // String text = doc.text();
+        // 方式2：更灵活的过滤（用Safelist.none()表示不保留任何标签，仅文本）
+        String text = Jsoup.clean(html, Safelist.none());
+
+        // 步骤3：优化文本（同原生方案，去多余空白）
+        text = optimizeText(text);
+
+        return text;
+    }
+
+    /**
+     * 文本优化（复用方案一的逻辑）
+     */
+    private static String optimizeText(String text) {
+        text = text.replaceAll("\\s+", " "); // 连续空白→单个空格
+        text = text.trim(); // 首尾去空
+        return text;
+    }
+}

xzl-admin/src/main/java/com/xzl/XzlApplication.java

@@ -1,6 +1,5 @@
 package com.xzl;
 
-import com.alibaba.fastjson.JSONObject;
 import com.xzl.common.core.domain.entity.MindMapNode;
 import com.xzl.common.utils.SecurityUtils;
 import org.apache.commons.lang3.ObjectUtils;
@@ -25,95 +24,6 @@ public class XzlApplication {
     SpringApplication.run(XzlApplication.class, args);
     System.out.println(SecurityUtils.encryptPassword("123456"));
     System.out.println("系统启动成功 ...");
-    String data ="{  \n" +
-            "  \"\": \"层级1节点\",  \n" +
-            "  \"\": \"层级1备注\",  \n" +
-            "  \"sonNode\": [  \n" +
-            "    {  \n" +
-            "      \"contentTxt\": \"层级2节点\",  \n" +
-            "      \"remark\": \"层级2备注\",  \n" +
-            "      \"sonNode\": [  \n" +
-            "        {  \n" +
-            "          \"contentTxt\": \"层级3节点\",  \n" +
-            "          \"remark\": \"层级3备注\",  \n" +
-            "          \"sonNode\": []  \n" +
-            "        },  \n" +
-            "        {  \n" +
-            "          \"contentTxt\": \"层级3节点\",  \n" +
-            "          \"remark\": \"层级3备注\",  \n" +
-            "          \"sonNode\": []  \n" +
-            "        }  \n" +
-            "      ]  \n" +
-            "    },  \n" +
-            "    {  \n" +
-            "      \"contentTxt\": \"层级2节点\",  \n" +
-            "      \"remark\": \"层级2备注\",  \n" +
-            "      \"sonNode\": []  \n" +
-            "    }  \n" +
-            "  ],\n" +
-            "  \"contentTxt\": \"层级1节点2\",  \n" +
-            "  \"remark\": \"层级1备注2\",\n" +
-            "  \"sonNode\": [\n" +
-            "        {\n" +
-            "            \"contentTxt\": \"层级2节点2\",\n" +
-            "            \"remark\": \"层级2备注2\",\n" +
-            "            \"sonNode\": [\n" +
-            "                {\n" +
-            "                    \"contentTxt\": \"层级3节点2\",\n" +
-            "                    \"remark\": \"层级3备注2\",\n" +
-            "                    \"sonNode\": []\n" +
-            "                },\n" +
-            "                {\n" +
-            "                    \"contentTxt\": \"层级3节点2\",\n" +
-            "                    \"remark\": \"层级3备注2\",\n" +
-            "                    \"sonNode\": []\n" +
-            "                }\n" +
-            "            ]\n" +
-            "        },\n" +
-            "        {\n" +
-            "            \"contentTxt\": \"层级2节点2\",\n" +
-            "            \"remark\": \"层级2备注2\",\n" +
-            "            \"sonNode\": []\n" +
-            "        }\n" +
-            "    ]\n" +
-            "}\n";
-    List<MindMapNode> mindMapNodes = JSONObject.parseArray(data, MindMapNode.class);
-      try {
-          createMindMap(mindMapNodes,"测试！", new OutputStream() {
-            @Override
-            public void write(int b) throws IOException {
-
-            }
-          });
-      } catch (IOException e) {
-          throw new RuntimeException(e);
-      } catch (CoreException e) {
-          throw new RuntimeException(e);
-      }
-//创建思维导图的工作空间
-    IWorkbookBuilder workbookBuilder = Core.getWorkbookBuilder();
-    IWorkbook workbook = workbookBuilder.createWorkbook();
-//获得默认sheet
-    ISheet primarySheet = workbook.getPrimarySheet();
-
-// 创建根节点
-    ITopic rootTopic = primarySheet.getRootTopic();
-    ITopic topicSon = workbook.createTopic();
-
-    //设置根主题的标题
-    rootTopic.setTitleText("测试！");
-// 根主题对象
-    rootTopic.setStructureClass("org.xmind.ui.logic.left");
-    topicSon.setTitleText(mindMapNodes.get(1).getContentTxt());
-    // 备注
-    IPlainNotesContent plainContent = (IPlainNotesContent) workbook.createNotesContent(INotes.PLAIN);
-    plainContent.setTextContent(mindMapNodes.get(1).getRemark());
-    INotes notes = topicSon.getNotes();
-    notes.setContent(INotes.PLAIN, plainContent);
-    buildNode(mindMapNodes.get(1).getSonNode(),topicSon,workbook);
-    topicSon.add(topicSon);
-//组装节点到根节点
-    rootTopic.add(topicSon);
 
   }
 
@@ -125,19 +35,19 @@ public class XzlApplication {
    * @throws IOException
    * @throws CoreException
    */
-  public static void createMindMap(List<MindMapNode> nodes, String root,OutputStream os) throws IOException, CoreException {
-    //创建脑图工作空间
-    IWorkbookBuilder workbookBuilder = Core.getWorkbookBuilder();
-    IWorkbook workbook = workbookBuilder.createWorkbook();
-    //获得默认sheet
-    ISheet primarySheet = workbook.getPrimarySheet();
-    ITopic rootTopic = primarySheet.getRootTopic();
-    //根节点的标题
-    rootTopic.setTitleText(root);
-    rootTopic.setStructureClass("org.xmind.ui.logic.right");
-    buildNode(nodes,rootTopic,workbook);
-    workbook.save(os);
-  }
+//  public static void createMindMap(List<MindMapNode> nodes, String root,OutputStream os) throws IOException, CoreException {
+//    //创建脑图工作空间
+//    IWorkbookBuilder workbookBuilder = Core.getWorkbookBuilder();
+//    IWorkbook workbook = workbookBuilder.createWorkbook();
+//    //获得默认sheet
+//    ISheet primarySheet = workbook.getPrimarySheet();
+//    ITopic rootTopic = primarySheet.getRootTopic();
+//    //根节点的标题
+//    rootTopic.setTitleText(root);
+//    rootTopic.setStructureClass("org.xmind.ui.logic.right");
+//    buildNode(nodes,rootTopic,workbook);
+//    workbook.save(os);
+//  }
 
   /**
    * 创建脑图节点
@@ -145,22 +55,22 @@ public class XzlApplication {
    * @param topic 根节点
    * @param workbook 工作空间
    */
-  private static void buildNode(List<MindMapNode> mindMapNodes, ITopic topic, IWorkbook workbook) {
-    if (ObjectUtils.isEmpty(mindMapNodes)) {
-      return;
-    } else {
-      for (MindMapNode mindMapNode : mindMapNodes) {
-        ITopic topicSon = workbook.createTopic();
-        topicSon.setTitleText(mindMapNode.getContentTxt());
-        // 备注
-        IPlainNotesContent plainContent = (IPlainNotesContent) workbook.createNotesContent(INotes.PLAIN);
-        plainContent.setTextContent(mindMapNode.getRemark());
-        INotes notes = topicSon.getNotes();
-        notes.setContent(INotes.PLAIN, plainContent);
-        buildNode(mindMapNode.getSonNode(),topicSon,workbook);
-        topic.add(topicSon);
-      }
-    }
-  }
+//  private static void buildNode(List<MindMapNode> mindMapNodes, ITopic topic, IWorkbook workbook) {
+//    if (ObjectUtils.isEmpty(mindMapNodes)) {
+//      return;
+//    } else {
+//      for (MindMapNode mindMapNode : mindMapNodes) {
+//        ITopic topicSon = workbook.createTopic();
+//        topicSon.setTitleText(mindMapNode.getContentTxt());
+//        // 备注
+//        IPlainNotesContent plainContent = (IPlainNotesContent) workbook.createNotesContent(INotes.PLAIN);
+//        plainContent.setTextContent(mindMapNode.getRemark());
+//        INotes notes = topicSon.getNotes();
+//        notes.setContent(INotes.PLAIN, plainContent);
+//        buildNode(mindMapNode.getSonNode(),topicSon,workbook);
+//        topic.add(topicSon);
+//      }
+//    }
+//  }
 
 }

xzl-admin/src/main/java/com/xzl/web/controller/KnowledgeFileController.java

@@ -30,6 +30,7 @@ import org.springframework.web.multipart.MultipartFile;
 @RestController
 @RequestMapping("/system/file")
 public class KnowledgeFileController extends BaseController {
+
     @Autowired
     private KnowledgeFileService knowledgeFileService;
 
@@ -41,7 +42,7 @@ public class KnowledgeFileController extends BaseController {
     /**
      * 查询文件列表
      */
-    @PreAuthorize("@ss.hasPermi('system:user:list')")
+    @PreAuthorize("@ss.hasPermi('system:file:list')")
     @GetMapping("/list")
     public TableDataInfo list(
             KnowledgeFile knowledge,

xzl-admin/src/main/java/com/xzl/web/controller/MonitorAreaController.java

@@ -20,20 +20,17 @@ public class MonitorAreaController {
     @Autowired
     private MonitorAreaService monitorAreaService;
 
-
     @RequestMapping(value = "/infoList",method = RequestMethod.GET)
     public InfoListVO infoList(@RequestParam(value = "id",required = false)String id,
                                @RequestParam(value = "areaType",required = false)String areaType){
         return monitorAreaService.infoList(id,areaType);
     }
 
-
     @RequestMapping(value = "/saveArea",method = RequestMethod.POST)
     public Map<String,String> saveArea(@RequestBody MonitorArea monitorArea){
         return monitorAreaService.saveArea(monitorArea);
     }
 
-
     @RequestMapping(value = "/deleteArea",method = RequestMethod.DELETE)
     public Map<String,String> deleteArea(@RequestBody List<String> ids){
         return monitorAreaService.deleteArea(ids);

xzl-admin/src/main/java/com/xzl/web/controller/SysFileFolderController.java

@@ -1,9 +1,5 @@
 package com.xzl.web.controller;
 
-
-
-
-
 import java.util.List;
 import javax.servlet.http.HttpServletResponse;
 import com.xzl.common.core.domain.entity.SysFileFolder;
@@ -31,11 +27,10 @@ import com.xzl.common.core.page.TableDataInfo;
 @RestController
 @RequestMapping("/system/folder")
 public class SysFileFolderController extends BaseController {
+
     @Autowired
     private ISysFileFolderService sysFileFolderService;
 
-
-
     /**
      * 获取知识库目录结构（Markdown格式）
      */
@@ -66,7 +61,8 @@ public class SysFileFolderController extends BaseController {
     /**
      * 查询文件文件夹（麻城知识库四级结构）列表
      */
-    @PreAuthorize("@ss.hasPermi('system:user:list')")
+//    @PreAuthorize("@ss.hasPermi('folder')")
+    @PreAuthorize("@ss.hasPermi('system:folder:list')")
     @GetMapping("/list")
     public TableDataInfo list(SysFileFolder sysFileFolder) {
         startPage();
@@ -131,7 +127,6 @@ public class SysFileFolderController extends BaseController {
         return toAjax(sysFileFolderService.deleteSysFileFolderByFolderIds(folderIds));
     }
 
-
     /**
      * 获取部门树列表
      */

xzl-admin/src/main/java/com/xzl/web/controller/SysSpiderSourceDataController.java

@@ -0,0 +1,143 @@
+package com.xzl.web.controller;
+
+import java.util.List;
+import javax.servlet.http.HttpServletResponse;
+
+
+import com.xzl.common.exception.ServiceException;
+import com.xzl.common.utils.StringUtils;
+import com.xzl.common.utils.poi.ExcelUtil;
+import com.xzl.system.domain.SysSpiderSourceData;
+import com.xzl.web.service.ISysSpiderSourceDataService;
+import io.swagger.annotations.ApiOperation;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.DeleteMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+import com.xzl.common.annotation.Log;
+import com.xzl.common.core.controller.BaseController;
+import com.xzl.common.core.domain.AjaxResult;
+import com.xzl.common.enums.BusinessType;
+
+
+import com.xzl.common.core.page.TableDataInfo;
+
+/**
+ * 原始采集数据Controller
+ *
+ * @author xzl
+ * @date 2025-11-13
+ */
+@RestController
+@RequestMapping("/spiderData/sourceData")
+public class SysSpiderSourceDataController extends BaseController
+{
+    @Autowired
+    private ISysSpiderSourceDataService sysSpiderSourceDataService;
+
+    /**
+     * 触发页面采集
+     * @param  前端传入的采集参数（包含 pageUrl）
+     * @return 采集结果（成功/失败信息）
+     */
+    @PostMapping("/collect") // 接口路径与前端 spiderData.js 中配置的一致
+    @ApiOperation("触发页面采集") // Swagger 接口说明（可选）
+    public AjaxResult collectPage(@RequestBody SysSpiderSourceData  sysSpiderSourceData) {
+        // 调用业务层执行采集逻辑（核心）
+        String pageUrl = sysSpiderSourceData.getPageUrl();
+        boolean empty = StringUtils.isEmpty(pageUrl);
+        if (empty){
+            return AjaxResult.error("url不能为空");
+        }
+        // 正则规则：支持 http/https，域名含字母/数字/短横线，允许端口和路径
+//        String urlRegex = "^(https?:\\/\\/)?([\\da-z.-]+)\\.([a-z.]{2,6})(:[0-9]{1,5})?(\\/[\\w.-]*)*$";
+        String urlRegex = "^(https?:\\/\\/)?([\\da-z.-]+)\\.([a-z.]{2,6})(:[0-9]{1,5})?(/[\\w\\-./?%&=]*)?$";
+
+        if (!pageUrl.matches(urlRegex)) {
+            throw new ServiceException("URL格式非法，请输入类似 https://www.xxx.com 的地址");
+        }
+        try {
+            sysSpiderSourceDataService.collectPage(sysSpiderSourceData.getPageUrl());
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+//        spiderCollectService.collectPage(collectDTO.getPageUrl());
+        // 返回成功结果（Result 是全局统一响应类）
+        return AjaxResult.success("采集任务已启动，正在处理");
+    }
+
+    /**
+     * 查询原始采集数据列表
+     */
+    @PreAuthorize("@ss.hasPermi('spiderData:sourceData:list')")
+    @GetMapping("/list")
+    public TableDataInfo list(SysSpiderSourceData sysSpiderSourceData)
+    {
+        startPage();
+        List<SysSpiderSourceData> list = sysSpiderSourceDataService.selectSysSpiderSourceDataList(sysSpiderSourceData);
+        return getDataTable(list);
+    }
+
+    /**
+     * 导出原始采集数据列表
+     */
+    @PreAuthorize("@ss.hasPermi('spiderData:sourceData:export')")
+    @Log(title = "原始采集数据", businessType = BusinessType.EXPORT)
+    @PostMapping("/export")
+    public void export(HttpServletResponse response, SysSpiderSourceData sysSpiderSourceData)
+    {
+        List<SysSpiderSourceData> list = sysSpiderSourceDataService.selectSysSpiderSourceDataList(sysSpiderSourceData);
+        ExcelUtil<SysSpiderSourceData> util = new ExcelUtil<SysSpiderSourceData>(SysSpiderSourceData.class);
+        util.exportExcel(response, list, "原始采集数据数据");
+    }
+
+    /**
+     * 获取原始采集数据详细信息
+     */
+    @PreAuthorize("@ss.hasPermi('spiderData:sourceData:query')")
+    @GetMapping(value = "/{id}")
+    public AjaxResult getInfo(@PathVariable("id") Long id)
+    {
+        return success(sysSpiderSourceDataService.selectSysSpiderSourceDataById(id));
+    }
+
+    /**
+     * 新增原始采集数据
+     */
+    @PreAuthorize("@ss.hasPermi('spiderData:sourceData:add')")
+    @Log(title = "原始采集数据", businessType = BusinessType.INSERT)
+    @PostMapping
+    public AjaxResult add(@RequestBody SysSpiderSourceData sysSpiderSourceData)
+    {
+        return toAjax(sysSpiderSourceDataService.insertSysSpiderSourceData(sysSpiderSourceData));
+    }
+
+    /**
+     * 修改原始采集数据
+     */
+    @PreAuthorize("@ss.hasPermi('spiderData:sourceData:edit')")
+    @Log(title = "原始采集数据", businessType = BusinessType.UPDATE)
+    @PutMapping
+    public AjaxResult edit(@RequestBody SysSpiderSourceData sysSpiderSourceData)
+    {
+        return toAjax(sysSpiderSourceDataService.updateSysSpiderSourceData(sysSpiderSourceData));
+    }
+
+    /**
+     * 删除原始采集数据
+     */
+    @PreAuthorize("@ss.hasPermi('spiderData:sourceData:remove')")
+    @Log(title = "原始采集数据", businessType = BusinessType.DELETE)
+	@DeleteMapping("/{ids}")
+    public AjaxResult remove(@PathVariable Long[] ids)
+    {
+        return toAjax(sysSpiderSourceDataService.deleteSysSpiderSourceDataByIds(ids));
+    }
+
+}

xzl-admin/src/main/java/com/xzl/web/mapper/SysSpiderSourceDataMapper.java

@@ -0,0 +1,61 @@
+package com.xzl.web.mapper;
+
+import java.util.List;
+import com.xzl.system.domain.SysSpiderSourceData;
+
+/**
+ * 原始采集数据Mapper接口
+ *
+ * @author xzl
+ * @date 2025-11-13
+ */
+public interface SysSpiderSourceDataMapper
+{
+    /**
+     * 查询原始采集数据
+     *
+     * @param id 原始采集数据主键
+     * @return 原始采集数据
+     */
+    public SysSpiderSourceData selectSysSpiderSourceDataById(Long id);
+
+    /**
+     * 查询原始采集数据列表
+     *
+     * @param sysSpiderSourceData 原始采集数据
+     * @return 原始采集数据集合
+     */
+    public List<SysSpiderSourceData> selectSysSpiderSourceDataList(SysSpiderSourceData sysSpiderSourceData);
+
+    /**
+     * 新增原始采集数据
+     *
+     * @param sysSpiderSourceData 原始采集数据
+     * @return 结果
+     */
+    public int insertSysSpiderSourceData(SysSpiderSourceData sysSpiderSourceData);
+
+    /**
+     * 修改原始采集数据
+     *
+     * @param sysSpiderSourceData 原始采集数据
+     * @return 结果
+     */
+    public int updateSysSpiderSourceData(SysSpiderSourceData sysSpiderSourceData);
+
+    /**
+     * 删除原始采集数据
+     *
+     * @param id 原始采集数据主键
+     * @return 结果
+     */
+    public int deleteSysSpiderSourceDataById(Long id);
+
+    /**
+     * 批量删除原始采集数据
+     *
+     * @param ids 需要删除的数据主键集合
+     * @return 结果
+     */
+    public int deleteSysSpiderSourceDataByIds(Long[] ids);
+}

xzl-admin/src/main/java/com/xzl/web/service/ISysSpiderSourceDataService.java

@@ -0,0 +1,63 @@
+package com.xzl.web.service;
+
+import java.util.List;
+import com.xzl.system.domain.SysSpiderSourceData;
+
+/**
+ * 原始采集数据Service接口
+ *
+ * @author xzl
+ * @date 2025-11-13
+ */
+public interface ISysSpiderSourceDataService
+{
+    /**
+     * 查询原始采集数据
+     *
+     * @param id 原始采集数据主键
+     * @return 原始采集数据
+     */
+    public SysSpiderSourceData selectSysSpiderSourceDataById(Long id);
+
+    /**
+     * 查询原始采集数据列表
+     *
+     * @param sysSpiderSourceData 原始采集数据
+     * @return 原始采集数据集合
+     */
+    public List<SysSpiderSourceData> selectSysSpiderSourceDataList(SysSpiderSourceData sysSpiderSourceData);
+
+    /**
+     * 新增原始采集数据
+     *
+     * @param sysSpiderSourceData 原始采集数据
+     * @return 结果
+     */
+    public int insertSysSpiderSourceData(SysSpiderSourceData sysSpiderSourceData);
+
+    /**
+     * 修改原始采集数据
+     *
+     * @param sysSpiderSourceData 原始采集数据
+     * @return 结果
+     */
+    public int updateSysSpiderSourceData(SysSpiderSourceData sysSpiderSourceData);
+
+    /**
+     * 批量删除原始采集数据
+     *
+     * @param ids 需要删除的原始采集数据主键集合
+     * @return 结果
+     */
+    public int deleteSysSpiderSourceDataByIds(Long[] ids);
+
+    /**
+     * 删除原始采集数据信息
+     *
+     * @param id 原始采集数据主键
+     * @return 结果
+     */
+    public int deleteSysSpiderSourceDataById(Long id);
+
+    public void collectPage(String pageUrl);
+}

xzl-admin/src/main/java/com/xzl/web/service/impl/SysFileFolderServiceImpl.java

@@ -20,7 +20,6 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import com.xzl.web.mapper.SysFileFolderMapper;
 
-import javax.xml.ws.Response;
 
 
 /**

xzl-admin/src/main/java/com/xzl/web/service/impl/SysSpiderSourceDataServiceImpl.java

@@ -0,0 +1,184 @@
+package com.xzl.web.service.impl;
+
+import java.util.*;
+
+
+import com.xzl.web.mapper.SysSpiderSourceDataMapper;
+import com.xzl.web.service.ISysSpiderSourceDataService;
+import com.xzl.web.utils.SeleniumUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import org.openqa.selenium.chrome.ChromeOptions;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import com.xzl.system.domain.SysSpiderSourceData;
+
+
+/**
+ * 原始采集数据Service业务层处理
+ *
+ * @author xzl
+ * @date 2025-11-13
+ */
+@Slf4j
+@Service
+public class SysSpiderSourceDataServiceImpl implements ISysSpiderSourceDataService {
+    @Autowired
+    private SysSpiderSourceDataMapper sysSpiderSourceDataMapper;
+
+    /**
+     * 查询原始采集数据
+     *
+     * @param id 原始采集数据主键
+     * @return 原始采集数据
+     */
+    @Override
+    public SysSpiderSourceData selectSysSpiderSourceDataById(Long id) {
+        return sysSpiderSourceDataMapper.selectSysSpiderSourceDataById(id);
+    }
+
+    /**
+     * 查询原始采集数据列表
+     *
+     * @param sysSpiderSourceData 原始采集数据
+     * @return 原始采集数据
+     */
+    @Override
+    public List<SysSpiderSourceData> selectSysSpiderSourceDataList(SysSpiderSourceData sysSpiderSourceData) {
+        return sysSpiderSourceDataMapper.selectSysSpiderSourceDataList(sysSpiderSourceData);
+    }
+
+    /**
+     * 新增原始采集数据
+     *
+     * @param sysSpiderSourceData 原始采集数据
+     * @return 结果
+     */
+    @Override
+    public int insertSysSpiderSourceData(SysSpiderSourceData sysSpiderSourceData) {
+        return sysSpiderSourceDataMapper.insertSysSpiderSourceData(sysSpiderSourceData);
+    }
+
+    /**
+     * 修改原始采集数据
+     *
+     * @param sysSpiderSourceData 原始采集数据
+     * @return 结果
+     */
+    @Override
+    public int updateSysSpiderSourceData(SysSpiderSourceData sysSpiderSourceData) {
+        return sysSpiderSourceDataMapper.updateSysSpiderSourceData(sysSpiderSourceData);
+    }
+
+    /**
+     * 批量删除原始采集数据
+     *
+     * @param ids 需要删除的原始采集数据主键
+     * @return 结果
+     */
+    @Override
+    public int deleteSysSpiderSourceDataByIds(Long[] ids) {
+        return sysSpiderSourceDataMapper.deleteSysSpiderSourceDataByIds(ids);
+    }
+
+    /**
+     * 删除原始采集数据信息
+     *
+     * @param id 原始采集数据主键
+     * @return 结果
+     */
+    @Override
+    public int deleteSysSpiderSourceDataById(Long id) {
+        return sysSpiderSourceDataMapper.deleteSysSpiderSourceDataById(id);
+    }
+
+    @Override
+    public void collectPage(String pageUrl) {
+        try {
+            // 1. 打印采集日志（便于调试）
+            log.info("开始采集页面：{}", pageUrl);
+
+            System.setProperty("webdriver.chrome.driver", "D:\\chromedriver.exe");
+
+            // 浏览器配置（无头模式、禁用图片、设置超时）
+            ChromeOptions options = new ChromeOptions();
+            options.addArguments("--headless=new");
+            options.addArguments("--disable-gpu");
+            options.addArguments("--window-size=1920,1080");
+//            options.addArguments("--disable-images");
+//            options.addArguments("--disable-javascript"); // 尝试禁用JS
+            options.addArguments("--no-sandbox");
+            options.addArguments("--disable-dev-shm-usage");
+            String htmlContent = SeleniumUtils.getPageSource(pageUrl);
+            if (htmlContent == null || htmlContent.isEmpty()) {
+                throw new IllegalArgumentException("网页内容为空");
+            }
+            SeleniumUtils.closeDriver();
+            SysSpiderSourceData sysSpiderSourceData = new SysSpiderSourceData();
+            sysSpiderSourceData.setPageUrl(pageUrl);
+            sysSpiderSourceData.setRawContent(htmlContent);
+            sysSpiderSourceData.setRawAttachments(getHtmlFileLinkAndName(htmlContent));
+            sysSpiderSourceData.setTaskId(UUID.randomUUID().toString());
+            sysSpiderSourceData.setRawContentFilter(SeleniumUtils.extractAndOptimizeWithJsoup(htmlContent));
+            sysSpiderSourceDataMapper.insertSysSpiderSourceData(sysSpiderSourceData);
+            // 5. 采集完成日志
+            log.info("页面采集完成：{}", pageUrl);
+
+        } catch (Exception e) {
+            // 捕获采集异常，打印日志（避免程序崩溃）
+            log.error("页面采集失败：{}，错误信息：{}", pageUrl, e.getMessage(), e);
+            // 抛出业务异常（由全局异常处理器捕获并返回给前端）
+            throw new RuntimeException("采集失败：" + e.getMessage());
+        }
+    }
+
+
+
+    public String getHtmlFileLinkAndName(String htmlContent) {
+        ArrayList<String> attrCollections = new ArrayList<>();
+        attrCollections.add(".pdf");
+        attrCollections.add(".xlsx");
+        attrCollections.add(".xls");
+        attrCollections.add(".doc");
+        attrCollections.add(".docx");
+
+        Map<String, String> pdfMap = new HashMap<>();
+
+        Document parse = Jsoup.parse(htmlContent);
+        Elements aTags = parse.select("a");
+        for (Element aTag : aTags) {
+            String href = aTag.attr("href");
+            String text = aTag.text().trim();
+
+            // 检查 href 和文本是否有效
+            if (href != null && !href.isEmpty() && !text.isEmpty()) {
+                // 将 href 转换为小写，以便进行不区分大小写的后缀匹配
+                String lowerHref = href.toLowerCase();
+
+                // --- 核心修改点 ---
+                // 检查 href 是否以 attrCollections 中的任何一个后缀结尾
+                boolean isTargetFile = false;
+                for (String suffix : attrCollections) {
+                    if (lowerHref.endsWith(suffix)) {
+                        isTargetFile = true;
+                        break; // 找到一个匹配项即可，无需继续循环
+                    }
+                }
+
+                // 如果是目标文件类型，则存入 Map
+                if (isTargetFile) {
+                    // 处理相对路径，转换为绝对路径
+                    String absoluteUrl = aTag.absUrl("href");
+                    // 存入 Map。如果文件名（text）重复，后面的会覆盖前面的
+                    pdfMap.put(text, absoluteUrl);
+                }
+            }
+        }
+        return pdfMap.toString();
+    }
+
+}

xzl-admin/src/main/java/com/xzl/web/service/impl/UserPortraitServiceImpl.java

@@ -63,12 +63,12 @@ public class UserPortraitServiceImpl implements UserPortraitService {
       return null;
     }
     SysUser sysUser = userService.selectUserByUserName(username);
-    String nlpName = sysUser.getNlpName();
-    String nlpPwd = sysUser.getNlpPwd();
+//    String nlpName = sysUser.getNlpName();
+//    String nlpPwd = sysUser.getNlpPwd();
     // 调用登录
     Map loginForm = new HashMap();
-    loginForm.put("username", nlpName);
-    loginForm.put("password", nlpPwd);
+//    loginForm.put("username", nlpName);
+//    loginForm.put("password", nlpPwd);
     loginForm.put("userAgent", 1);
     loginForm.put("client", "zdwxxtyc");
     loginForm.put("saveUsername", true);
@@ -359,8 +359,8 @@ public class UserPortraitServiceImpl implements UserPortraitService {
     }
     SysUser user = new SysUser();
     user.setUserId(sysUser.getUserId());
-    user.setNlpName(nlpUserName);
-    user.setNlpPwd(nlpPassword);
+//    user.setNlpName(nlpUserName);
+//    user.setNlpPwd(nlpPassword);
     userService.updateUser(user);
     rs.put("success", 1);
     rs.put("msg", "密码修改成功");

xzl-admin/src/main/java/com/xzl/web/utils/SeleniumUtils.java

@@ -0,0 +1,120 @@
+package com.xzl.web.utils;
+
+
+import lombok.extern.slf4j.Slf4j;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.safety.Safelist;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.chrome.ChromeDriver;
+import org.openqa.selenium.chrome.ChromeOptions;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Selenium 浏览器工具类（Chrome）
+ */
+@Slf4j
+public class SeleniumUtils {
+
+//    private static final Log log = LogFactory.get(SeleniumUtils.class);
+    private static WebDriver driver;
+
+    /**
+     * 初始化 Chrome 浏览器（无头模式）
+     */
+    public static void initDriver() {
+        try {
+            // 1. 手动指定 ChromeDriver 路径（Linux 部署路径）
+            System.setProperty("webdriver.chrome.driver", "D:\\chromedriver.exe");
+
+            // 浏览器配置（无头模式、禁用图片、设置超时）
+            ChromeOptions options = new ChromeOptions();
+            options.addArguments("--headless=new");
+            options.addArguments("--disable-gpu");
+            options.addArguments("--window-size=1920,1080");
+//            options.addArguments("--disable-images");
+            options.addArguments("--disable-javascript"); // 尝试禁用JS
+            options.addArguments("--no-sandbox");
+            options.addArguments("--disable-dev-shm-usage");
+
+            // 初始化驱动并设置超时
+            driver = new ChromeDriver(options);
+            driver.manage().timeouts().pageLoadTimeout(60, TimeUnit.SECONDS); // 页面加载超时30秒
+            driver.manage().timeouts().implicitlyWait(10, TimeUnit.SECONDS); // 元素查找超时10秒
+            log.info("Chrome 浏览器初始化成功");
+        } catch (Exception e) {
+            log.error("Chrome 浏览器初始化失败", e);
+            throw new RuntimeException("浏览器启动失败：" + e.getMessage());
+        }
+    }
+
+    /**
+     * 访问网页并获取 HTML 源码
+     * @param url 目标网页URL
+     * @return HTML 源码字符串
+     */
+    public static String getPageSource(String url) {
+        if (driver == null) {
+            initDriver();
+        }
+        try {
+            log.info("Selenium 访问网页：{}", url);
+            driver.get(url); // 模拟浏览器打开网页（支持动态渲染）
+            Thread.sleep(2000); // 等待2秒，确保动态内容加载完成（可按需调整）
+            return driver.getPageSource(); // 获取渲染后的完整HTML
+        } catch (Exception e) {
+            log.error("访问网页失败：{}", url, e);
+            throw new RuntimeException("网页访问失败：" + e.getMessage());
+        }
+    }
+
+    /**
+     * 关闭浏览器并释放资源
+     */
+    public static void closeDriver() {
+        if (driver != null) {
+            try {
+                driver.quit();
+                log.info("Chrome 浏览器已关闭");
+            } catch (Exception e) {
+                log.error("浏览器关闭失败", e);
+            } finally {
+                driver = null;
+            }
+        }
+    }
+
+
+    /**
+     * 用Jsoup提取HTML纯文本并优化
+     */
+    public static String extractAndOptimizeWithJsoup(String html) {
+        if (html == null || html.trim().isEmpty()) {
+            return "";
+        }
+
+        // 步骤1：解析HTML（忽略文档类型、编码，自动处理嵌套）
+        Document doc = Jsoup.parse(html);
+
+        // 步骤2：提取纯文本（自动移除标签、注释、脚本）
+        // 方式1：直接获取所有文本（保留段落逻辑，用换行分隔）
+        // String text = doc.text();
+        // 方式2：更灵活的过滤（用Safelist.none()表示不保留任何标签，仅文本）
+        String text = Jsoup.clean(html, Safelist.none());
+
+        // 步骤3：优化文本（同原生方案，去多余空白）
+        text = optimizeText(text);
+
+        return text;
+    }
+
+    /**
+     * 文本优化（复用方案一的逻辑）
+     */
+    private static String optimizeText(String text) {
+        text = text.replaceAll("\\s+", " "); // 连续空白→单个空格
+        text = text.trim(); // 首尾去空
+        return text;
+    }
+}

xzl-admin/src/main/resources/application-dev.yml

@@ -11,7 +11,7 @@ spring:
 #                username: jerry
  #               password: zjr38zjR@
                 username: root
-                password: 123456
+                password: root
 #                url: jdbc:mysql://localhost:3306/xtdb?allowMultiQueries=true&useUnicode=true&characterEncoding=UTF-8&useSSL=false
 #                username: root
 #                password: 1234
@@ -75,3 +75,7 @@ minio:
     access-key: minioadmin
     secret-key: minioadmin
     bucket-name: mc-kb
+
+chromeDriver:
+    driverPath: D:\chromedriver.exe
+

xzl-admin/src/main/resources/mapper/SysSpiderSourceDataMapper.xml

@@ -0,0 +1,81 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE mapper
+PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
+"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.xzl.web.mapper.SysSpiderSourceDataMapper">
+
+    <resultMap type="SysSpiderSourceData" id="SysSpiderSourceDataResult">
+        <result property="id"    column="id"    />
+        <result property="pageUrl"    column="page_url"    />
+        <result property="rawContent"    column="raw_content"    />
+        <result property="rawContentFilter"    column="raw_content_filter"    />
+        <result property="rawAttachments"    column="raw_attachments"    />
+        <result property="collectionTime"    column="collection_time"    />
+        <result property="taskId"    column="task_id"    />
+    </resultMap>
+
+    <sql id="selectSysSpiderSourceDataVo">
+        select id, page_url, raw_content, raw_content_filter,raw_attachments, collection_time, task_id from sys_spider_source_data
+    </sql>
+
+    <select id="selectSysSpiderSourceDataList" parameterType="SysSpiderSourceData" resultMap="SysSpiderSourceDataResult">
+        <include refid="selectSysSpiderSourceDataVo"/>
+        <where>
+            <if test="pageUrl != null  and pageUrl != ''"> and page_url like concat('%', #{pageUrl}, '%')</if>
+            <if test="rawContent != null  and rawContent != ''"> and raw_content like concat('%', #{rawContent}, '%')</if>
+            <if test="rawContentFilter != null  and rawContentFilter != ''"> and raw_content_filter like concat('%', #{rawContentFilter}, '%')</if>
+            <if test="rawAttachments != null  and rawAttachments != ''"> and raw_attachments like concat('%', #{rawAttachments}, '%')</if>
+            <if test="params.beginCollectionTime != null and params.beginCollectionTime != '' and params.endCollectionTime != null and params.endCollectionTime != ''"> and collection_time between #{params.beginCollectionTime} and #{params.endCollectionTime}</if>
+            <if test="taskId != null  and taskId != ''"> and task_id = #{taskId}</if>
+        </where>
+    </select>
+
+    <select id="selectSysSpiderSourceDataById" parameterType="Long" resultMap="SysSpiderSourceDataResult">
+        <include refid="selectSysSpiderSourceDataVo"/>
+        where id = #{id}
+    </select>
+
+    <insert id="insertSysSpiderSourceData" parameterType="SysSpiderSourceData" useGeneratedKeys="true" keyProperty="id">
+        insert into sys_spider_source_data
+        <trim prefix="(" suffix=")" suffixOverrides=",">
+            <if test="pageUrl != null">page_url,</if>
+            <if test="rawContent != null">raw_content,</if>
+            <if test="rawContentFilter != null">raw_content_filter,</if>
+            <if test="rawAttachments != null">raw_attachments,</if>
+            <if test="collectionTime != null">collection_time,</if>
+            <if test="taskId != null">task_id,</if>
+         </trim>
+        <trim prefix="values (" suffix=")" suffixOverrides=",">
+            <if test="pageUrl != null">#{pageUrl},</if>
+            <if test="rawContent != null">#{rawContent},</if>
+            <if test="rawContentFilter != null">#{rawContentFilter},</if>
+            <if test="rawAttachments != null">#{rawAttachments},</if>
+            <if test="collectionTime != null">#{collectionTime},</if>
+            <if test="taskId != null">#{taskId},</if>
+         </trim>
+    </insert>
+
+    <update id="updateSysSpiderSourceData" parameterType="SysSpiderSourceData">
+        update sys_spider_source_data
+        <trim prefix="SET" suffixOverrides=",">
+            <if test="pageUrl != null">page_url = #{pageUrl},</if>
+            <if test="rawContent != null">raw_content = #{rawContent},</if>
+            <if test="rawContentFilter != null">raw_content_filter = #{rawContentFilter},</if>
+            <if test="rawAttachments != null">raw_attachments = #{rawAttachments},</if>
+            <if test="collectionTime != null">collection_time = #{collectionTime},</if>
+            <if test="taskId != null">task_id = #{taskId},</if>
+        </trim>
+        where id = #{id}
+    </update>
+
+    <delete id="deleteSysSpiderSourceDataById" parameterType="Long">
+        delete from sys_spider_source_data where id = #{id}
+    </delete>
+
+    <delete id="deleteSysSpiderSourceDataByIds" parameterType="String">
+        delete from sys_spider_source_data where id in
+        <foreach item="id" collection="array" open="(" separator="," close=")">
+            #{id}
+        </foreach>
+    </delete>
+</mapper>

xzl-admin/src/main/resources/mapper/UserPortraitMapper.xml

@@ -5,7 +5,6 @@
         select distinct goods_name from d_order
     </select>
 
-
     <select id="getUserByDingUnionId" parameterType="string" resultType="com.xzl.common.core.domain.entity.SysUser">
         select user_name as "userName",password from sys_user where ding_union_id=#{unionId}
     </select>
@@ -30,9 +29,9 @@
     where d1.dept_id =  d2.dept_id
   </update>
 
-    <update id="syncDeptForDingtalk">
-    update sys_user e, sys_dept d
-    set e.dept_id = d.dept_id
-    where e.ding_dept_id = d.ding_id
-  </update>
-</mapper>
+<!--    <update id="syncDeptForDingtalk">-->
+<!--    update sys_user e, sys_dept d-->
+<!--    set e.dept_id = d.dept_id-->
+<!--    where e.ding_dept_id = d.ding_id-->
+<!--  </update>-->
+</mapper>

xzl-common/src/main/java/com/xzl/common/core/domain/entity/SysUser.java

@@ -128,8 +128,8 @@ public class SysUser extends BaseEntity {
   private Long dingDeptId;
   private String dingUnionId;
 
-  private String nlpName;
-  private String nlpPwd;
+//  private String nlpName;
+//  private String nlpPwd;
 
   public SysUser() {
 
@@ -315,21 +315,21 @@ public class SysUser extends BaseEntity {
     this.dingUnionId = dingUnionId;
   }
 
-  public String getNlpName() {
-    return nlpName;
-  }
-
-  public void setNlpName(String nlpName) {
-    this.nlpName = nlpName;
-  }
-
-  public String getNlpPwd() {
-    return nlpPwd;
-  }
-
-  public void setNlpPwd(String nlpPwd) {
-    this.nlpPwd = nlpPwd;
-  }
+//  public String getNlpName() {
+//    return nlpName;
+//  }
+//
+//  public void setNlpName(String nlpName) {
+//    this.nlpName = nlpName;
+//  }
+
+//  public String getNlpPwd() {
+//    return nlpPwd;
+//  }
+//
+//  public void setNlpPwd(String nlpPwd) {
+//    this.nlpPwd = nlpPwd;
+//  }
 
   @Override
   public String toString() {

xzl-framework/src/main/java/com/xzl/framework/web/service/SysLoginService.java

@@ -33,7 +33,7 @@ import com.xzl.system.service.ISysUserService;
 
 /**
  * 登录校验方法
- * 
+ *
  * @author xzl
  */
 @Component
@@ -47,7 +47,7 @@ public class SysLoginService
 
     @Autowired
     private RedisCache redisCache;
-    
+
     @Autowired
     private ISysUserService userService;
 
@@ -59,7 +59,7 @@ public class SysLoginService
 
     /**
      * 登录验证
-     * 
+     *
      * @param username 用户名
      * @param password 密码
      * @param code 验证码
@@ -107,7 +107,7 @@ public class SysLoginService
 
     /**
      * 校验验证码
-     * 
+     *
      * @param username 用户名
      * @param code 验证码
      * @param uuid 唯一标识
@@ -154,6 +154,7 @@ public class SysLoginService
             AsyncManager.me().execute(AsyncFactory.recordLogininfor(username, Constants.LOGIN_FAIL, MessageUtils.message("user.password.not.match")));
             throw new UserPasswordNotMatchException();
         }
+
         // 用户名不在指定范围内 错误
         if (username.length() < UserConstants.USERNAME_MIN_LENGTH
                 || username.length() > UserConstants.USERNAME_MAX_LENGTH)

xzl-system/src/main/java/com/xzl/system/domain/SysSpiderSourceData.java

@@ -0,0 +1,131 @@
+package com.xzl.system.domain;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.xzl.common.annotation.Excel;
+import com.xzl.common.core.domain.BaseEntity;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
+
+import java.util.Date;
+
+/**
+ * 原始采集数据对象 sys_spider_source_data
+ *
+ * @author xzl
+ * @date 2025-11-13
+ */
+public class SysSpiderSourceData extends BaseEntity {
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * 原始数据ID
+     */
+    private Long id;
+
+    /**
+     * 采集页面URL
+     */
+    @Excel(name = "采集页面URL")
+    private String pageUrl;
+
+    /**
+     * 原始文本内容（HTML/JSON等）
+     */
+    @Excel(name = "原始文本内容", readConverterExp = "H=TML/JSON等")
+    private String rawContent;
+
+    /**
+     * 原始文本内容（HTML/JSON等）
+     */
+    @Excel(name = "原始文本内容", readConverterExp = "H=TML/JSON等")
+    private String rawContentFilter;
+
+    /**
+     * 原始附件信息（JSON格式：文件名、URL等）
+     */
+    @Excel(name = "原始附件信息", readConverterExp = "J=SON格式：文件名、URL等")
+    private String rawAttachments;
+
+    /**
+     * 采集时间
+     */
+    @JsonFormat(pattern = "yyyy-MM-dd")
+    @Excel(name = "采集时间", width = 30, dateFormat = "yyyy-MM-dd")
+    private Date collectionTime;
+
+    /**
+     * 关联采集任务ID（便于追溯）
+     */
+    @Excel(name = "关联采集任务ID", readConverterExp = "便=于追溯")
+    private String taskId;
+
+    public void setId(Long id) {
+        this.id = id;
+    }
+
+    public Long getId() {
+        return id;
+    }
+
+    public void setPageUrl(String pageUrl) {
+        this.pageUrl = pageUrl;
+    }
+
+    public String getPageUrl() {
+        return pageUrl;
+    }
+
+    public void setRawContent(String rawContent) {
+        this.rawContent = rawContent;
+    }
+
+    public void setRawContentFilter(String rawContentFilter) {
+        this.rawContentFilter = rawContentFilter;
+    }
+
+    public String getRawContent() {
+        return rawContent;
+    }
+
+    public String getRawContentFilter() {
+        return rawContentFilter;
+    }
+
+    public void setRawAttachments(String rawAttachments) {
+        this.rawAttachments = rawAttachments;
+    }
+
+    public String getRawAttachments() {
+        return rawAttachments;
+    }
+
+    public void setCollectionTime(Date collectionTime) {
+        this.collectionTime = collectionTime;
+    }
+
+    public Date getCollectionTime() {
+        return collectionTime;
+    }
+
+    public void setTaskId(String taskId) {
+        this.taskId = taskId;
+    }
+
+    public String getTaskId() {
+        return taskId;
+    }
+
+    @Override
+    public String toString() {
+        return new ToStringBuilder(this, ToStringStyle.MULTI_LINE_STYLE)
+                .append("id", getId())
+                .append("pageUrl", getPageUrl())
+                .append("rawContent", getRawContent())
+                .append("rawContentFilter", getRawContentFilter())
+                .append("rawAttachments", getRawAttachments())
+                .append("collectionTime", getCollectionTime())
+                .append("taskId", getTaskId())
+                .toString();
+    }
+
+}

xzl-system/src/main/resources/mapper/system/SysDeptMapper.xml

@@ -23,13 +23,12 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
 		<result property="dingId" column="ding_id" />
 		<result property="dingParentId" column="ding_parent_id" />
 	</resultMap>
-	
+
 	<sql id="selectDeptVo">
-        select d.dept_id, d.parent_id, d.ancestors, d.dept_name, d.order_num, d.leader, d.phone, d.email, d.status, d.del_flag, d.create_by, d.create_time ,
-        d.ding_id, d.ding_parent_id
+        select d.dept_id, d.parent_id, d.ancestors, d.dept_name, d.order_num, d.leader, d.phone, d.email, d.status, d.del_flag, d.create_by, d.create_time
         from sys_dept d
     </sql>
-    
+
 	<select id="selectDeptList" parameterType="SysDept" resultMap="SysDeptResult">
         <include refid="selectDeptVo"/>
         where d.del_flag = '0'
@@ -49,7 +48,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
 		${params.dataScope}
 		order by d.parent_id, d.order_num
     </select>
-    
+
     <select id="selectDeptListByRoleId" resultType="Long">
 		select d.dept_id
 		from sys_dept d
@@ -60,36 +59,36 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
             </if>
 		order by d.parent_id, d.order_num
 	</select>
-    
+
     <select id="selectDeptById" parameterType="Long" resultMap="SysDeptResult">
 		select d.dept_id, d.parent_id, d.ancestors, d.dept_name, d.order_num, d.leader, d.phone, d.email, d.status,
 			(select dept_name from sys_dept where dept_id = d.parent_id) parent_name
 		from sys_dept d
 		where d.dept_id = #{deptId}
 	</select>
-    
+
     <select id="checkDeptExistUser" parameterType="Long" resultType="int">
 		select count(1) from sys_user where dept_id = #{deptId} and del_flag = '0'
 	</select>
-	
+
 	<select id="hasChildByDeptId" parameterType="Long" resultType="int">
 		select count(1) from sys_dept
 		where del_flag = '0' and parent_id = #{deptId} limit 1
 	</select>
-	
+
 	<select id="selectChildrenDeptById" parameterType="Long" resultMap="SysDeptResult">
 		select * from sys_dept where find_in_set(#{deptId}, ancestors)
 	</select>
-	
+
 	<select id="selectNormalChildrenDeptById" parameterType="Long" resultType="int">
 		select count(*) from sys_dept where status = 0 and del_flag = '0' and find_in_set(#{deptId}, ancestors)
 	</select>
-	
+
 	<select id="checkDeptNameUnique" resultMap="SysDeptResult">
 	    <include refid="selectDeptVo"/>
 		where dept_name=#{deptName} and parent_id = #{parentId} and del_flag = '0' limit 1
 	</select>
-    
+
     <insert id="insertDept" parameterType="SysDept">
  		insert into sys_dept(
  			<if test="deptId != null and deptId != 0">dept_id,</if>
@@ -121,7 +120,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
  			sysdate()
  		)
 	</insert>
-	
+
 	<update id="updateDept" parameterType="SysDept">
  		update sys_dept
  		<set>
@@ -140,7 +139,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
  		</set>
  		where dept_id = #{deptId}
 	</update>
-	
+
 	<update id="updateDeptChildren" parameterType="java.util.List">
 	    update sys_dept set ancestors =
 	    <foreach collection="depts" item="item" index="index"
@@ -153,16 +152,16 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
 	        #{item.deptId}
 	    </foreach>
 	</update>
-	 
+
 	<update id="updateDeptStatusNormal" parameterType="Long">
- 	    update sys_dept set status = '0' where dept_id in 
+ 	    update sys_dept set status = '0' where dept_id in
  	    <foreach collection="array" item="deptId" open="(" separator="," close=")">
         	#{deptId}
         </foreach>
 	</update>
-	
+
 	<delete id="deleteDeptById" parameterType="Long">
 		update sys_dept set del_flag = '2' where dept_id = #{deptId}
 	</delete>
 
-</mapper> 
+</mapper>

xzl-system/src/main/resources/mapper/system/SysUserMapper.xml

@@ -53,7 +53,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
 	<sql id="selectUserVo">
         select u.user_id, u.dept_id, u.user_name, u.nick_name, u.email, u.avatar, u.phonenumber, u.password, u.sex, u.status, u.del_flag, u.login_ip, u.login_date, u.create_by, u.create_time, u.remark,
         d.dept_id, d.parent_id, d.ancestors, d.dept_name, d.order_num, d.leader, d.status as dept_status,
-        r.role_id, r.role_name, r.role_key, r.role_sort, r.data_scope, r.status as role_status, u.nlp_name, u.nlp_pwd
+        r.role_id, r.role_name, r.role_key, r.role_sort, r.data_scope, r.status as role_status
         from sys_user u
 		    left join sys_dept d on u.dept_id = d.dept_id
 		    left join sys_user_role ur on u.user_id = ur.user_id
@@ -61,8 +61,8 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
     </sql>
 
     <select id="selectUserList" parameterType="SysUser" resultMap="SysUserResult">
-		select u.user_id, u.dept_id, u.nick_name, u.user_name, u.email, u.avatar, u.phonenumber, u.sex, u.status, u.del_flag, u.login_ip, u.login_date, u.create_by, u.create_time, u.remark, d.dept_name, d.leader ,
-		u.nlp_name, u.nlp_pwd
+		select u.user_id, u.dept_id, u.nick_name, u.user_name, u.email, u.avatar, u.phonenumber, u.sex, u.status, u.del_flag, u.login_ip, u.login_date, u.create_by, u.create_time, u.remark, d.dept_name, d.leader
+
 		from sys_user u
 		left join sys_dept d on u.dept_id = d.dept_id
 		where u.del_flag = '0'
@@ -164,8 +164,8 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
  			<if test="remark != null and remark != ''">remark,</if>
  			<if test="dingDeptId != null and dingDeptId != ''">ding_dept_id,</if>
  			<if test="dingUnionId != null and dingUnionId != ''">ding_union_id,</if>
- 			<if test="nlpName != null and nlpName != ''">nlp_name,</if>
- 			<if test="nlpPwd != null and nlpPwd != ''">nlp_pwd,</if>
+<!-- 			<if test="nlpName != null and nlpName != ''">nlp_name,</if>-->
+<!-- 			<if test="nlpPwd != null and nlpPwd != ''">nlp_pwd,</if>-->
  			create_time
  		)values(
  			<if test="userId != null and userId != ''">#{userId},</if>
@@ -182,8 +182,8 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
  			<if test="remark != null and remark != ''">#{remark},</if>
 			<if test="dingDeptId != null and dingDeptId != ''">#{dingDeptId},</if>
 			<if test="dingUnionId != null and dingUnionId != ''">#{dingUnionId},</if>
-		    <if test="nlpName != null and nlpName != ''">#{nlpName},</if>
-		    <if test="nlpPwd != null and nlpPwd != ''">#{nlpPwd},</if>
+<!--		    <if test="nlpName != null and nlpName != ''">#{nlpName},</if>-->
+<!--		    <if test="nlpPwd != null and nlpPwd != ''">#{nlpPwd},</if>-->
 		    sysdate()
  		)
 	</insert>
@@ -206,8 +206,8 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
  			<if test="remark != null">remark = #{remark},</if>
 			<if test="dingDeptId != null and dingDeptId != ''">ding_dept_id = #{dingDeptId},</if>
 			<if test="dingUnionId != null and dingUnionId != ''">ding_union_id = #{dingUnionId},</if>
-			<if test="nlpName != null and nlpName != ''">nlp_name = #{nlpName},</if>
-			<if test="nlpPwd != null and nlpPwd != ''">nlp_pwd = #{nlpPwd},</if>
+<!--			<if test="nlpName != null and nlpName != ''">nlp_name = #{nlpName},</if>-->
+<!--			<if test="nlpPwd != null and nlpPwd != ''">nlp_pwd = #{nlpPwd},</if>-->
 			update_time = sysdate()
  		</set>
  		where user_id = #{userId}

xzl-ui/src/api/spiderData/spiderData.js

@@ -0,0 +1,55 @@
+import request from '@/utils/request'
+
+// 查询原始采集数据列表
+export function listSpiderData(query) {
+  return request({
+    url: '/spiderData/sourceData/list',
+    method: 'get',
+    params: query
+  })
+}
+
+// 查询原始采集数据详细
+export function getSpiderData(id) {
+  return request({
+    url: '/spiderData/sourceData/' + id,
+    method: 'get'
+  })
+}
+
+// 新增原始采集数据
+export function addSpiderData(data) {
+  return request({
+    url: '/spiderData/sourceData',
+    method: 'post',
+    data: data
+  })
+}
+
+// 修改原始采集数据
+export function updateSpiderData(data) {
+  return request({
+    url: '/spiderData/sourceData',
+    method: 'put',
+    data: data
+  })
+}
+
+// 删除原始采集数据
+export function delSpiderData(id) {
+  return request({
+    url: '/spiderData/sourceData/' + id,
+    method: 'delete'
+  })
+}
+
+// 新增：采集接口函数
+export function spiderCollect(data) {
+  return request({
+    url: '/spiderData/sourceData/collect', // 后端采集接口地址，需与后端协商确认
+    method: 'post',
+    data: data // 传递pageUrl等参数
+  })
+}
+
+

xzl-ui/src/router/index.js

@@ -216,7 +216,6 @@ export const dynamicRoutes = [
     ]
   },
 
-
 ]
 
 // 防止连续点击多次路由报错
@@ -236,3 +235,4 @@ export default new Router({
   scrollBehavior: () => ({ y: 0 }),
   routes: constantRoutes
 })
+

xzl-ui/src/views/logistics4.vue

@@ -1,4 +1,4 @@
-<template>
+g<template>
   <div id="screen" >
     <iframe frameborder="no"  src="http://10.70.192.123/webroot/decision/view/form?viewlet=大屏新版20240226/营销业务看板.frm" width="100%" height="100%"></iframe>
 

xzl-ui/src/views/spiderData/sourceData/index.vue

@@ -0,0 +1,534 @@
+<template>
+  <div class="app-container">
+    <!-- 搜索表单 -->
+    <el-form
+      :model="queryParams"
+      ref="queryForm"
+      size="small"
+      :inline="true"
+      v-show="showSearch"
+      label-width="68px"
+    >
+      <el-form-item label="采集页面URL" prop="pageUrl">
+        <el-input
+          v-model="queryParams.pageUrl"
+          placeholder="请输入采集页面URL"
+          clearable
+          @keyup.enter.native="handleQuery"
+        />
+      </el-form-item>
+      <el-form-item label="采集时间">
+        <el-date-picker
+          v-model="daterangeCollectionTime"
+          style="width: 240px"
+          value-format="yyyy-MM-dd"
+          type="daterange"
+          range-separator="-"
+          start-placeholder="开始日期"
+          end-placeholder="结束日期"
+        ></el-date-picker>
+      </el-form-item>
+      <el-form-item label="关联采集任务ID" prop="taskId">
+        <el-input
+          v-model="queryParams.taskId"
+          placeholder="请输入关联采集任务ID"
+          clearable
+          @keyup.enter.native="handleQuery"
+        />
+      </el-form-item>
+      <el-form-item>
+        <el-button type="primary" icon="el-icon-search" size="mini" @click="handleQuery">搜索</el-button>
+        <el-button icon="el-icon-refresh" size="mini" @click="resetQuery">重置</el-button>
+        <el-button icon="el-icon-document" size="mini" @click="spiderQuery">采集</el-button>
+      </el-form-item>
+    </el-form>
+
+    <!-- 操作按钮区 -->
+    <el-row :gutter="10" class="mb8">
+      <el-col :span="1.5">
+        <el-button
+          type="primary"
+          plain
+          icon="el-icon-plus"
+          size="mini"
+          @click="handleAdd"
+          v-hasPermi="['spiderData:sourceData:add']"
+        >新增
+        </el-button>
+      </el-col>
+      <el-col :span="1.5">
+        <el-button
+          type="success"
+          plain
+          icon="el-icon-edit"
+          size="mini"
+          :disabled="!hasSingleSelection"
+          @click="handleUpdate"
+          v-hasPermi="['spiderData:sourceData:edit']"
+        >修改
+        </el-button>
+      </el-col>
+      <el-col :span="1.5">
+        <el-button
+          type="danger"
+          plain
+          icon="el-icon-delete"
+          size="mini"
+          :disabled="!hasMultipleSelection"
+          @click="handleDelete"
+          v-hasPermi="['spiderData:sourceData:remove']"
+        >删除
+        </el-button>
+      </el-col>
+      <el-col :span="1.5">
+        <el-button
+          type="warning"
+          plain
+          icon="el-icon-download"
+          size="mini"
+          @click="handleExport"
+          v-hasPermi="['spiderData:sourceData:export']"
+        >导出
+        </el-button>
+      </el-col>
+      <right-toolbar :showSearch.sync="showSearch" @queryTable="getList"></right-toolbar>
+    </el-row>
+
+    <!-- 数据表格 -->
+    <el-table
+      v-loading="loading"
+      :data="spiderDataList"
+      @selection-change="handleSelectionChange"
+      row-key="id"
+    >
+      <el-table-column type="selection" width="55" align="center"/>
+
+      <el-table-column label="采集页面URL" align="center" prop="pageUrl" width="300">
+        <template #default="scope">
+          <el-link
+            type="primary"
+            :href="scope.row.pageUrl"
+            target="_blank"
+            :underline="true"
+            class="text-ellipsis"
+          >
+            {{ formatLongText(scope.row.pageUrl, 30) }}
+          </el-link>
+        </template>
+      </el-table-column>
+
+      <el-table-column label="原始文本内容" align="center" prop="rawContent" width="200">
+        <template #default="scope">
+          <el-link
+            type="primary"
+            @click="openRichTextNewPage(scope.row.rawContent, '原始文本内容')"
+            :underline="false"
+            class="text-ellipsis"
+          >
+            {{ formatLongText(scope.row.rawContent, 50) }}
+          </el-link>
+        </template>
+      </el-table-column>
+
+      <el-table-column label="优化文本内容" align="center" prop="rawContentFilter" width="200">
+        <template #default="scope">
+          <el-link
+            type="primary"
+            @click="openRichTextNewPage(scope.row.rawContentFilter, '优化文本内容')"
+            :underline="false"
+            class="text-ellipsis"
+          >
+            {{ formatLongText(scope.row.rawContentFilter, 50) }}
+          </el-link>
+        </template>
+      </el-table-column>
+
+      <el-table-column label="原始附件信息" align="center" prop="rawAttachments" width="200">
+        <template #default="scope">
+          <el-link
+            type="primary"
+            @click="openRichTextNewPage(scope.row.rawAttachments, '原始附件信息')"
+            :underline="false"
+            class="text-ellipsis"
+          >
+            {{ formatLongText(scope.row.rawAttachments, 50) }}
+          </el-link>
+        </template>
+      </el-table-column>
+
+      <el-table-column label="采集时间" align="center" prop="collectionTime" width="180">
+        <template #default="scope">
+          <span>{{ parseTime(scope.row.collectionTime, '{y}-{m}-{d}') }}</span>
+        </template>
+      </el-table-column>
+
+      <el-table-column label="关联采集任务ID" align="center" prop="taskId"/>
+
+      <el-table-column label="操作" align="center" class-name="small-padding fixed-width">
+        <template #default="scope">
+          <el-button
+            size="mini"
+            type="text"
+            icon="el-icon-edit"
+            @click="handleUpdate(scope.row)"
+            v-hasPermi="['spiderData:sourceData:edit']"
+          >修改
+          </el-button>
+          <el-button
+            size="mini"
+            type="text"
+            icon="el-icon-delete"
+            @click="handleDelete(scope.row)"
+            v-hasPermi="['spiderData:sourceData:remove']"
+          >删除
+          </el-button>
+        </template>
+      </el-table-column>
+    </el-table>
+
+    <!-- 分页组件 -->
+    <pagination
+      v-show="total > 0"
+      :total="total"
+      :page.sync="queryParams.pageNum"
+      :limit.sync="queryParams.pageSize"
+      @pagination="getList"
+    />
+
+    <!-- 添加或修改原始采集数据对话框 -->
+    <el-dialog :title="title" :visible.sync="open" width="1000px" append-to-body>
+      <el-form ref="form" :model="form" :rules="rules" label-width="80px">
+        <el-form-item label="采集页面URL" prop="pageUrl">
+          <el-input v-model="form.pageUrl" placeholder="请输入采集页面URL"/>
+        </el-form-item>
+        <el-form-item label="原始文本内容">
+          <editor v-model="form.rawContent" :min-height="192"/>
+        </el-form-item>
+        <el-form-item label="优化文本内容">
+          <editor v-model="form.rawContentFilter" :min-height="192"/>
+        </el-form-item>
+        <el-form-item label="原始附件信息" prop="rawAttachments">
+          <el-input v-model="form.rawAttachments" type="textarea" placeholder="请输入内容"/>
+        </el-form-item>
+      </el-form>
+      <div slot="footer" class="dialog-footer">
+        <el-button type="primary" @click="submitForm">确 定</el-button>
+        <el-button @click="cancel">取 消</el-button>
+      </div>
+    </el-dialog>
+  </div>
+</template>
+
+
+<script>
+import {
+  listSpiderData,
+  getSpiderData,
+  delSpiderData,
+  addSpiderData,
+  updateSpiderData,
+  spiderCollect
+} from "@/api/spiderData/spiderData";
+
+// 常量定义
+const DEFAULT_PAGE_SIZE = 10;
+const DEFAULT_PAGE_NUM = 1;
+
+export default {
+  name: "SpiderData",
+  data() {
+    return {
+      // 遮罩层
+      loading: true,
+      // 选中数组
+      ids: [],
+      // 显示搜索条件
+      showSearch: true,
+      // 总条数
+      total: 0,
+      // 原始采集数据表格数据
+      spiderDataList: [],
+      // 弹出层标题
+      title: "",
+      // 是否显示弹出层
+      open: false,
+      // 采集时间范围
+      daterangeCollectionTime: [],
+      // 查询参数
+      queryParams: {
+        pageNum: DEFAULT_PAGE_NUM,
+        pageSize: DEFAULT_PAGE_SIZE,
+        pageUrl: null,
+        rawContent: null,
+        rawContentFilter: null,
+        rawAttachments: null,
+        collectionTime: null,
+        taskId: null
+      },
+      // 表单参数
+      form: {},
+      // 表单校验
+      rules: {},
+      // 新增附件弹窗相关数据
+      attachmentsDialogVisible: false,
+      attachments: []
+    };
+  },
+  computed: {
+    // 计算属性：是否有单个选中项
+    hasSingleSelection() {
+      return this.ids.length === 1;
+    },
+    // 计算属性：是否有选中项（用于删除）
+    hasMultipleSelection() {
+      return this.ids.length > 0;
+    }
+  },
+  created() {
+    this.getList();
+  },
+  methods: {
+    /** 格式化长文本，超过指定长度显示省略号 */
+    formatLongText(text = '', maxLength) {
+      if (!text) return '无';
+      return text.length > maxLength
+        ? `${text.substring(0, maxLength)}...`
+        : text;
+    },
+
+    /** 打开富文本新页面（新窗口） */
+    openRichTextNewPage(content, title = '文本内容') {
+      // 创建一个临时HTML内容
+      const htmlContent = `
+        <!DOCTYPE html>
+        <html lang="zh-CN">
+        <head>
+          <meta charset="UTF-8">
+          <meta name="viewport" content="width=device-width, initial-scale=1.0">
+          <title>${title}</title>
+          <style>
+            body {
+              font-family: Arial, sans-serif;
+              padding: 20px;
+              line-height: 1.6;
+            }
+            .container {
+              max-width: 1200px;
+              margin: 0 auto;
+            }
+            .title {
+              color: #333;
+              border-bottom: 1px solid #eee;
+              padding-bottom: 10px;
+              margin-bottom: 20px;
+            }
+          </style>
+        </head>
+        <body>
+          <div class="container">
+            <h2 class="title">${title}</h2>
+            <div class="content">${content || '无内容'}</div>
+          </div>
+        </body>
+        </html>
+      `;
+
+      // 创建Blob对象
+      const blob = new Blob([htmlContent], { type: 'text/html' });
+      // 创建临时URL
+      const url = URL.createObjectURL(blob);
+      // 打开新窗口
+      window.open(url, '_blank');
+
+      // 释放URL对象
+      setTimeout(() => {
+        URL.revokeObjectURL(url);
+      }, 1000);
+    },
+
+    /** 查询原始采集数据列表 */
+    getList() {
+      this.loading = true;
+      // 处理查询参数
+      const params = { ...this.queryParams };
+
+      // 添加时间范围查询参数
+      if (this.daterangeCollectionTime && this.daterangeCollectionTime.length === 2) {
+        params.params = {
+          beginCollectionTime: this.daterangeCollectionTime[0],
+          endCollectionTime: this.daterangeCollectionTime[1]
+        };
+      }
+
+      listSpiderData(params)
+        .then(response => {
+          this.spiderDataList = response.rows || [];
+          this.total = response.total || 0;
+        })
+        .catch(error => {
+          console.error('获取数据列表失败:', error);
+          this.$modal.msgError('获取数据失败，请重试');
+        })
+        .finally(() => {
+          this.loading = false;
+        });
+    },
+
+    /** 采集按钮操作 */
+    spiderQuery() {
+      // 校验：若pageUrl为空，给出提示
+      if (!this.queryParams.pageUrl) {
+        this.$modal.msgWarning("请输入采集页面URL");
+        return;
+      }
+
+      // 显示加载状态
+      this.loading = true;
+
+      // 调用采集接口
+      spiderCollect({ pageUrl: this.queryParams.pageUrl })
+        .then(() => {
+          this.$modal.msgSuccess("采集请求已发送，处理中...");
+          this.getList(); // 刷新列表
+        })
+        .catch(error => {
+          this.$modal.msgError(`采集失败：${error.response?.data?.msg || "服务器异常"}`);
+        })
+        .finally(() => {
+          this.loading = false;
+        });
+    },
+
+    // 取消按钮
+    cancel() {
+      this.open = false;
+      this.reset();
+    },
+
+    // 表单重置
+    reset() {
+      this.form = {
+        id: null,
+        pageUrl: null,
+        rawContent: null,
+        rawContentFilter: null,
+        rawAttachments: null,
+        collectionTime: null,
+        taskId: null
+      };
+      this.resetForm("form");
+    },
+
+    /** 搜索按钮操作 */
+    handleQuery() {
+      this.queryParams.pageNum = DEFAULT_PAGE_NUM;
+      this.getList();
+    },
+
+    /** 重置按钮操作 */
+    resetQuery() {
+      this.daterangeCollectionTime = [];
+      this.resetForm("queryForm");
+      this.handleQuery();
+    },
+
+    // 多选框选中数据
+    handleSelectionChange(selection) {
+      this.ids = selection.map(item => item.id);
+    },
+
+    /** 新增按钮操作 */
+    handleAdd() {
+      this.reset();
+      this.open = true;
+      this.title = "添加原始采集数据";
+    },
+
+    /** 修改按钮操作 */
+    handleUpdate(row) {
+      this.reset();
+      const id = row?.id || this.ids[0];
+      if (!id) return;
+
+      getSpiderData(id)
+        .then(response => {
+          this.form = response.data;
+          this.open = true;
+          this.title = "修改原始采集数据";
+        })
+        .catch(error => {
+          console.error('获取数据详情失败:', error);
+          this.$modal.msgError('获取数据详情失败，请重试');
+        });
+    },
+
+    /** 提交按钮 */
+    submitForm() {
+      this.$refs["form"].validate(valid => {
+        if (valid) {
+          const isEdit = this.form.id != null;
+          const apiMethod = isEdit ? updateSpiderData : addSpiderData;
+
+          apiMethod(this.form)
+            .then(() => {
+              this.$modal.msgSuccess(isEdit ? "修改成功" : "新增成功");
+              this.open = false;
+              this.getList();
+            })
+            .catch(error => {
+              console.error(`${isEdit ? '修改' : '新增'}失败:`, error);
+              this.$modal.msgError(`${isEdit ? '修改' : '新增'}失败，请重试`);
+            });
+        }
+      });
+    },
+
+    /** 删除按钮操作 */
+    handleDelete(row) {
+      const ids = row?.id || this.ids;
+      if (!ids || (Array.isArray(ids) && ids.length === 0)) return;
+
+      this.$modal.confirm(`是否确认删除原始采集数据编号为"${ids}"的数据项？`)
+        .then(() => delSpiderData(ids))
+        .then(() => {
+          this.getList();
+          this.$modal.msgSuccess("删除成功");
+        })
+        .catch(() => {
+          // 取消删除时的处理
+        });
+    },
+
+    /** 导出按钮操作 */
+    handleExport() {
+      this.download(
+        'spiderData/sourceData/export',
+        { ...this.queryParams },
+        `spiderData_${new Date().getTime()}.xlsx`
+      );
+    }
+  }
+};
+</script>
+
+<style scoped>
+/* 附件列表样式，避免布局错乱 */
+.attachment-item {
+  padding: 12px 0;
+  border-bottom: 1px solid #f5f5f5;
+}
+.attachment-item:last-child {
+  border-bottom: none;
+}
+.text-ellipsis {
+  display: block;
+  white-space: nowrap;
+  overflow: hidden;
+  text-overflow: ellipsis;
+}
+/* 容器样式 */
+.app-container {
+  padding: 16px;
+  box-sizing: border-box;
+}
+</style>

xzl-ui/src/views/spiderData/sourceData/index.vuebak

@@ -0,0 +1,475 @@
+<template>
+  <div class="app-container">
+    <el-form :model="queryParams" ref="queryForm" size="small" :inline="true" v-show="showSearch" label-width="68px">
+      <el-form-item label="采集页面URL" prop="pageUrl">
+        <el-input
+          v-model="queryParams.pageUrl"
+          placeholder="请输入采集页面URL"
+          clearable
+          @keyup.enter.native="handleQuery"
+        />
+      </el-form-item>
+      <el-form-item label="采集时间">
+        <el-date-picker
+          v-model="daterangeCollectionTime"
+          style="width: 240px"
+          value-format="yyyy-MM-dd"
+          type="daterange"
+          range-separator="-"
+          start-placeholder="开始日期"
+          end-placeholder="结束日期"
+        ></el-date-picker>
+      </el-form-item>
+      <el-form-item label="关联采集任务ID" prop="taskId">
+        <el-input
+          v-model="queryParams.taskId"
+          placeholder="请输入关联采集任务ID"
+          clearable
+          @keyup.enter.native="handleQuery"
+        />
+      </el-form-item>
+      <el-form-item>
+        <el-button type="primary" icon="el-icon-search" size="mini" @click="handleQuery">搜索</el-button>
+        <el-button icon="el-icon-refresh" size="mini" @click="resetQuery">重置</el-button>
+        <el-button icon="el-icon-document" size="mini" @click="spiderQuery">采集</el-button>
+      </el-form-item>
+    </el-form>
+
+    <el-row :gutter="10" class="mb8">
+      <el-col :span="1.5">
+        <el-button
+          type="primary"
+          plain
+          icon="el-icon-plus"
+          size="mini"
+          @click="handleAdd"
+          v-hasPermi="['spiderData:sourceData:add']"
+        >新增
+        </el-button>
+      </el-col>
+      <el-col :span="1.5">
+        <el-button
+          type="success"
+          plain
+          icon="el-icon-edit"
+          size="mini"
+          :disabled="single"
+          @click="handleUpdate"
+          v-hasPermi="['spiderData:sourceData:edit']"
+        >修改
+        </el-button>
+      </el-col>
+      <el-col :span="1.5">
+        <el-button
+          type="danger"
+          plain
+          icon="el-icon-delete"
+          size="mini"
+          :disabled="multiple"
+          @click="handleDelete"
+          v-hasPermi="['spiderData:sourceData:remove']"
+        >删除
+        </el-button>
+      </el-col>
+      <el-col :span="1.5">
+        <el-button
+          type="warning"
+          plain
+          icon="el-icon-download"
+          size="mini"
+          @click="handleExport"
+          v-hasPermi="['spiderData:sourceData:export']"
+        >导出
+        </el-button>
+      </el-col>
+      <right-toolbar :showSearch.sync="showSearch" @queryTable="getList"></right-toolbar>
+    </el-row>
+
+    <el-table v-loading="loading" :data="spiderDataList" @selection-change="handleSelectionChange">
+      <el-table-column type="selection" width="55" align="center"/>
+      <!-- <el-table-column label="原始数据ID" align="center" prop="id" /> -->
+      <!-- <el-table-column label="采集页面URL" align="center" prop="pageUrl" /> -->
+      <!-- 修改为以下代码 -->
+      <el-table-column label="采集页面URL" align="center" prop="pageUrl" width="300">
+        <template slot-scope="scope">
+          <el-link
+            type="primary"
+            :href="scope.row.pageUrl"
+            target="_blank"
+            :underline="true"
+            class="text-ellipsis"
+          >
+            {{
+              scope.row.pageUrl.length > 30 ? scope.row.pageUrl.substring(0, 30) + '...' : (scope.row.pageUrl || '无')
+            }}
+          </el-link>
+        </template>
+      </el-table-column>
+      <!--      <el-table-column label="原始文本内容" align="center" prop="rawContent" width="500" show-overflow-tooltip/>-->
+      <!-- 修改原始文本内容列的el-link部分 -->
+      <el-table-column label="原始文本内容" align="center" prop="rawContent" width="200">
+        <template slot-scope="scope">
+          <el-link
+            type="primary"
+            @click="openRichTextNewPage(scope.row.rawContent)"
+            :underline="false"
+            class="text-ellipsis"
+          >
+            {{
+              scope.row.rawContent.length > 50 ? scope.row.rawContent.substring(0, 50) + '...' : (scope.row.rawContent || '无')
+            }}
+          </el-link>
+        </template>
+      </el-table-column>
+      <!-- <el-table-column label="优化文本内容" align="center" prop="rawContentFilter" width="500" show-overflow-tooltip/> -->
+      <el-table-column label="优化文本内容" align="center" prop="rawContentFilter" width="200">
+        <template slot-scope="scope">
+          <el-link
+            type="primary"
+            @click="openRichTextNewPage(scope.row.rawContentFilter)"
+            :underline="false"
+            class="text-ellipsis"
+          >
+            {{
+              scope.row.rawContentFilter.length > 50 ? scope.row.rawContentFilter.substring(0, 50) + '...' : (scope.row.rawContentFilter || '无')
+            }}
+          </el-link>
+        </template>
+      </el-table-column>
+
+
+      <!-- <el-table-column label="原始附件信息" align="center" prop="rawAttachments" width="300" show-overflow-tooltip/> -->
+      <el-table-column label="原始附件信息" align="center" prop="rawAttachments" width="200">
+        <template slot-scope="scope">
+          <el-link
+            type="primary"
+            @click="openRichTextNewPage(scope.row.rawAttachments)"
+            :underline="false"
+            class="text-ellipsis"
+          >
+            {{
+              scope.row.rawAttachments.length > 50 ? scope.row.rawAttachments.substring(0, 50) + '...' : (scope.row.rawAttachments || '无')
+            }}
+          </el-link>
+        </template>
+      </el-table-column>
+
+
+      <el-table-column label="采集时间" align="center" prop="collectionTime" width="180">
+        <template slot-scope="scope">
+          <span>{{ parseTime(scope.row.collectionTime, '{y}-{m}-{d}') }}</span>
+        </template>
+      </el-table-column>
+      <el-table-column label="关联采集任务ID" align="center" prop="taskId"/>
+      <el-table-column label="操作" align="center" class-name="small-padding fixed-width">
+        <template slot-scope="scope">
+          <el-button
+            size="mini"
+            type="text"
+            icon="el-icon-edit"
+            @click="handleUpdate(scope.row)"
+            v-hasPermi="['spiderData:sourceData:edit']"
+          >修改
+          </el-button>
+          <el-button
+            size="mini"
+            type="text"
+            icon="el-icon-delete"
+            @click="handleDelete(scope.row)"
+            v-hasPermi="['spiderData:sourceData:remove']"
+          >删除
+          </el-button>
+        </template>
+      </el-table-column>
+    </el-table>
+
+    <pagination
+      v-show="total>0"
+      :total="total"
+      :page.sync="queryParams.pageNum"
+      :limit.sync="queryParams.pageSize"
+      @pagination="getList"
+    />
+
+    <!-- 添加或修改原始采集数据对话框 -->
+    <el-dialog :title="title" :visible.sync="open" width="1000px" append-to-body>
+      <el-form ref="form" :model="form" :rules="rules" label-width="80px">
+        <el-form-item label="采集页面URL" prop="pageUrl">
+          <el-input v-model="form.pageUrl" placeholder="请输入采集页面URL"/>
+        </el-form-item>
+        <el-form-item label="原始文本内容">
+          <editor v-model="form.rawContent" :min-height="192"/>
+        </el-form-item>
+        <el-form-item label="优化文本内容">
+          <editor v-model="form.rawContentFilter" :min-height="192"/>
+        </el-form-item>
+        <el-form-item label="原始附件信息" prop="rawAttachments">
+          <el-input v-model="form.rawAttachments" type="textarea" placeholder="请输入内容"/>
+        </el-form-item>
+      </el-form>
+      <div slot="footer" class="dialog-footer">
+        <el-button type="primary" @click="submitForm">确 定</el-button>
+        <el-button @click="cancel">取 消</el-button>
+      </div>
+    </el-dialog>
+  </div>
+</template>
+
+
+<script>
+import {
+  listSpiderData,
+  getSpiderData,
+  delSpiderData,
+  addSpiderData,
+  updateSpiderData,
+  spiderCollect
+} from "@/api/spiderData/spiderData";
+
+export default {
+  name: "SpiderData",
+  data() {
+    return {
+      // 遮罩层
+      loading: true,
+      // 选中数组
+      ids: [],
+      // 非单个禁用
+      single: true,
+      // 非多个禁用
+      multiple: true,
+      // 显示搜索条件
+      showSearch: true,
+      // 总条数
+      total: 0,
+      // 原始采集数据表格数据
+      spiderDataList: [],
+      // 弹出层标题
+      title: "",
+      // 是否显示弹出层
+      open: false,
+      // 原始附件信息时间范围
+      daterangeCollectionTime: [],
+      // 查询参数
+      queryParams: {
+        pageNum: 1,
+        pageSize: 10,
+        pageUrl: null,
+        rawContent: null,
+        rawContentFilter: null,
+        rawAttachments: null,
+        collectionTime: null,
+        taskId: null
+      },
+      // 表单参数
+      form: {},
+      // 表单校验
+      rules: {},
+      // 新增附件弹窗相关数据
+      attachmentsDialogVisible: false,
+      attachments: []
+    };
+  },
+  created() {
+    this.getList();
+  },
+  methods: {
+
+    /** 打开富文本新页面（新窗口） */
+    openRichTextNewPage(rawContent) {
+      // 创建一个临时HTML内容
+      const htmlContent = `
+      <!DOCTYPE html>
+      <html lang="zh-CN">
+      <head>
+        <meta charset="UTF-8">
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        <title>原始文本内容</title>
+        <style>
+          body {
+            font-family: Arial, sans-serif;
+            padding: 20px;
+            line-height: 1.6;
+          }
+          .container {
+            max-width: 1200px;
+            margin: 0 auto;
+          }
+          .title {
+            color: #333;
+            border-bottom: 1px solid #eee;
+            padding-bottom: 10px;
+            margin-bottom: 20px;
+          }
+        </style>
+      </head>
+      <body>
+        <div class="container">
+          <h2 class="title">原始文本内容</h2>
+          <div class="content">${rawContent || '无内容'}</div>
+        </div>
+      </body>
+      </html>
+    `;
+
+      // 创建Blob对象
+      const blob = new Blob([htmlContent], {type: 'text/html'});
+      // 创建临时URL
+      const url = URL.createObjectURL(blob);
+      // 打开新窗口
+      window.open(url, '_blank');
+
+      // 释放URL对象（可选，浏览器会自动回收，但手动释放更规范）
+      setTimeout(() => {
+        URL.revokeObjectURL(url);
+      }, 1000);
+    },
+    /** 查询原始采集数据列表 */
+    getList() {
+      this.loading = true;
+      this.queryParams.params = {};
+      if (null != this.daterangeCollectionTime && '' != this.daterangeCollectionTime) {
+        this.queryParams.params["beginCollectionTime"] = this.daterangeCollectionTime[0];
+        this.queryParams.params["endCollectionTime"] = this.daterangeCollectionTime[1];
+      }
+      listSpiderData(this.queryParams).then(response => {
+        this.spiderDataList = response.rows;
+        this.total = response.total;
+        this.loading = false;
+      });
+    },
+    /** 采集按钮操作 */
+    spiderQuery() {
+      // 校验：若pageUrl为空，给出提示
+      if (!this.queryParams.pageUrl) {
+        this.$modal.msgWarning("请输入采集页面URL");
+        return;
+      }
+
+      // 调用封装好的采集接口
+      spiderCollect({
+        pageUrl: this.queryParams.pageUrl // 传递采集页面URL
+      }).then(response => {
+        this.$modal.msgSuccess("采集请求已发送，处理中...");
+        // 若需要刷新列表，可调用getList()
+        this.getList();
+      }).catch(error => {
+        this.$modal.msgError("采集失败：" + (error.response?.data?.msg || "服务器异常"));
+      });
+    },
+    // 取消按钮
+    cancel() {
+      this.open = false;
+      this.reset();
+    },
+    // 表单重置
+    reset() {
+      this.form = {
+        id: null,
+        pageUrl: null,
+        rawContent: null,
+        rawContentFilter: null,
+        rawAttachments: null,
+        collectionTime: null,
+        taskId: null
+      };
+      this.resetForm("form");
+    },
+    /** 搜索按钮操作 */
+    handleQuery() {
+      this.queryParams.pageNum = 1;
+      this.getList();
+    },
+    /** 重置按钮操作 */
+    resetQuery() {
+      this.daterangeCollectionTime = [];
+      this.resetForm("queryForm");
+      this.handleQuery();
+    },
+    // 多选框选中数据
+    handleSelectionChange(selection) {
+      this.ids = selection.map(item => item.id)
+      this.single = selection.length !== 1
+      this.multiple = !selection.length
+    },
+    /** 新增按钮操作 */
+    handleAdd() {
+      this.reset();
+      this.open = true;
+      this.title = "添加原始采集数据";
+    },
+    /** 修改按钮操作 */
+    handleUpdate(row) {
+      this.reset();
+      const id = row.id || this.ids
+      getSpiderData(id).then(response => {
+        this.form = response.data;
+        this.open = true;
+        this.title = "修改原始采集数据";
+      });
+    },
+    /** 提交按钮 */
+    submitForm() {
+      this.$refs["form"].validate(valid => {
+        if (valid) {
+          if (this.form.id != null) {
+            updateSpiderData(this.form).then(response => {
+              this.$modal.msgSuccess("修改成功");
+              this.open = false;
+              this.getList();
+            });
+          } else {
+            addSpiderData(this.form).then(response => {
+              this.$modal.msgSuccess("新增成功");
+              this.open = false;
+              this.getList();
+            });
+          }
+        }
+      });
+    },
+    /** 删除按钮操作 */
+    handleDelete(row) {
+      const ids = row.id || this.ids;
+      this.$modal.confirm('是否确认删除原始采集数据编号为"' + ids + '"的数据项？').then(function () {
+        return delSpiderData(ids);
+      }).then(() => {
+        this.getList();
+        this.$modal.msgSuccess("删除成功");
+      }).catch(() => {
+      });
+    },
+    /** 导出按钮操作 */
+    handleExport() {
+      this.download('spiderData/sourceData/export', {
+        ...this.queryParams
+      }, `spiderData_${new Date().getTime()}.xlsx`)
+    }
+  }
+};
+
+</script>
+
+<!--
+<style scoped>
+/* 新增：附件列表样式，避免布局错乱 */
+.attachment-item {
+  padding: 12px 0;
+  border-bottom: 1px solid #f5f5f5;
+}
+.attachment-item:last-child {
+  border-bottom: none;
+}
+.text-ellipsis {
+  display: block;
+  white-space: nowrap;
+  overflow: hidden;
+  text-overflow: ellipsis;
+}
+/* 修复：表格滚动容器样式 */
+.app-container {
+  padding: 16px;
+  box-sizing: border-box;
+}
+</style>
+ -->