feat:邮件解析-支持解析邮件正文

service-base/src/main/java/com/simuwang/base/common/util/EmailUtil.java

@@ -158,9 +158,9 @@ public class EmailUtil {
                             savefile.getParentFile().setExecutable(true);
                         }
                         FileUtil.writeFile(savePath, partContent.toString());
-                        emailContentInfoDTO.setFileName(fileName);
-                        emailContentInfoDTO.setFilePath(savePath);
                     }
+                    emailContentInfoDTO.setFileName(fileName);
+                    emailContentInfoDTO.setFilePath(savePath);
                 }
             }
         } catch (MessagingException e) {

service-daq/src/main/java/com/simuwang/daq/service/AbstractEmailParser.java

@@ -36,7 +36,7 @@ public abstract class AbstractEmailParser {
             log.warn("单位净值或累计净值或资产净值或资产份额格式不正确 -> 解析到的数据:{}", fundNavDTO);
             return false;
         }
-        log.info("数据格式验证通过...");
+        log.info("数据格式验证通过 -> {}", fundNavDTO);
         return true;
     }
 }

service-daq/src/main/java/com/simuwang/daq/service/EmailParseService.java

@@ -96,6 +96,7 @@ public class EmailParseService {
                 log.warn("未采集到正文或附件");
                 continue;
             }
+            log.info("开始解析邮件数据 -> 邮件主题:{},邮件日期:{}", emailContentInfoDTOList.get(0).getEmailTitle(), emailContentInfoDTOList.get(0).getEmailDate());
             List<EmailFundNavDTO> emailFundNavDTOList = CollUtil.newArrayList();
             Map<EmailContentInfoDTO, List<EmailFundNavDTO>> fileNameNavMap = MapUtil.newHashMap();
             for (EmailContentInfoDTO emailContentInfoDTO : emailContentInfoDTOList) {
@@ -277,9 +278,10 @@ public class EmailParseService {
     }
 
     private void setNavParseStatus(EmailFundNavDTO fundNavDTO, String emailTitle, String emailDate) {
-        Integer navParseStatus = getDataParseStatus(fundNavDTO, emailTitle, emailDate);
+        Integer navParseStatus = getDataParseStatus(fundNavDTO);
         fundNavDTO.setParseStatus(navParseStatus);
         if (navParseStatus.equals(NavParseStatusConst.NAV_DEFICIENCY)) {
+            log.info("判断数据的解析状态 -> 邮件主题:{},邮件日期:{},数据解析状态:{}", emailTitle, emailDate, fundNavDTO.getParseStatus());
             return;
         }
         // 2.匹配基金
@@ -287,6 +289,7 @@ public class EmailParseService {
         if (CollUtil.isEmpty(fundIdList)) {
             fundNavDTO.setParseStatus(NavParseStatusConst.NOT_MATCH);
         }
+        log.info("判断数据的解析状态 -> 邮件主题:{},邮件日期:{},数据解析状态:{}", emailTitle, emailDate, fundNavDTO.getParseStatus());
         fundNavDTO.setFundIdList(fundIdList);
     }
 
@@ -301,7 +304,7 @@ public class EmailParseService {
             return fundIdList;
         }
         // 2.基金名称匹配
-        if(StrUtil.isNotBlank(fundName)){
+        if (StrUtil.isNotBlank(fundName)) {
             List<String> fundIds = fundInfomationMapper.queryFundByName(fundName);
             if (CollUtil.isNotEmpty(fundIds)) {
                 return fundIdList;
@@ -312,7 +315,7 @@ public class EmailParseService {
             }
         }
         // 3.备案编码匹配
-        if(StrUtil.isNotBlank(fundName)){
+        if (StrUtil.isNotBlank(fundName)) {
             List<String> fundIds = fundInfomationMapper.queryFundByRegisterNumber(registerNumber);
             if (CollUtil.isNotEmpty(fundIds)) {
                 return fundIds;
@@ -348,8 +351,7 @@ public class EmailParseService {
         return emailParseInfoDO;
     }
 
-    private Integer getDataParseStatus(EmailFundNavDTO fundNavDTO, String emailTitle, String emailDate) {
-        log.info("判断数据的解析状态 -> 邮件主题:{},邮件日期:{},净值数据:{}", emailTitle, emailDate, fundNavDTO);
+    private Integer getDataParseStatus(EmailFundNavDTO fundNavDTO) {
         // 1.单位净值或累计净值缺失
         if (StrUtil.isBlank(fundNavDTO.getNav()) || StrUtil.isBlank(fundNavDTO.getCumulativeNavWithdrawal())) {
             return NavParseStatusConst.NAV_DEFICIENCY;
@@ -443,9 +445,9 @@ public class EmailParseService {
                             savefile.getParentFile().mkdirs();
                             savefile.getParentFile().setExecutable(true);
                         }
-                        FileUtil.writeFile(filePath, content.toString());
-                        emailContentInfoDTO.setFilePath(filePath);
                     }
+                    FileUtil.writeFile(filePath, content.toString());
+                    emailContentInfoDTO.setFilePath(filePath);
                     emailContentInfoDTOList.add(emailContentInfoDTO);
                 }
                 if (CollUtil.isNotEmpty(emailContentInfoDTOList)) {

service-daq/src/main/java/com/simuwang/daq/service/NavEmailParser.java

@@ -2,6 +2,7 @@ package com.simuwang.daq.service;
 
 import cn.hutool.core.collection.CollUtil;
 import cn.hutool.core.date.DateUtil;
+import cn.hutool.core.exceptions.ExceptionUtil;
 import cn.hutool.core.lang.Pair;
 import cn.hutool.core.map.MapUtil;
 import cn.hutool.core.util.StrUtil;
@@ -15,6 +16,8 @@ import com.simuwang.base.pojo.dto.EmailFundNavDTO;
 import org.apache.poi.ss.usermodel.Cell;
 import org.apache.poi.ss.usermodel.Row;
 import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.ss.usermodel.Workbook;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
@@ -24,6 +27,8 @@ import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
 
 import java.io.File;
+import java.io.FileOutputStream;
+import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -56,7 +61,7 @@ public class NavEmailParser extends AbstractEmailParser {
         String emailContent = emailContentInfoDTO.getEmailContent();
         // 1.解析邮件正文
         if (StrUtil.isNotBlank(emailContent)) {
-            emailFundNavDTOList = parseEmailContent(emailContent, emailFieldMap);
+            emailFundNavDTOList = parseEmailContent(emailContentInfoDTO, emailContent, emailFieldMap);
         }
         // 2.解析邮件excel附件
         if (StrUtil.isNotBlank(emailContentInfoDTO.getFilePath()) && ExcelUtil.isExcel(emailContentInfoDTO.getFileName())) {
@@ -186,12 +191,12 @@ public class NavEmailParser extends AbstractEmailParser {
         fundNavDTO.setNav(fieldValueMap.get(EmailFieldConst.NAV));
         fundNavDTO.setCumulativeNavWithdrawal(fieldValueMap.get(EmailFieldConst.CUMULATIVE_NAV_WITHDRAWAL));
         String assetNet = fieldValueMap.get(EmailFieldConst.ASSET_NET);
-        if (StrUtil.isBlank(assetNet) || !StrUtil.isNumeric(assetNet)) {
+        if (StrUtil.isBlank(assetNet)) {
             assetNet = null;
         }
         fundNavDTO.setAssetNet(assetNet);
         String assetShares = fieldValueMap.get(EmailFieldConst.ASSET_NET);
-        if (StrUtil.isBlank(assetShares) || !StrUtil.isNumeric(assetShares)) {
+        if (StrUtil.isBlank(assetShares)) {
             assetShares = null;
         }
         fundNavDTO.setAssetShare(assetShares);
@@ -238,13 +243,13 @@ public class NavEmailParser extends AbstractEmailParser {
         emailFundNavDTO.setVirtualNav(virtualNav);
         String assetNet = columnFieldMap.get(EmailFieldConst.ASSET_NET) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.ASSET_NET)) != null ?
                 ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.ASSET_NET))) : null;
-        if (StrUtil.isBlank(assetNet) || !StrUtil.isNumeric(assetNet)) {
+        if (StrUtil.isBlank(assetNet)) {
             assetNet = null;
         }
         emailFundNavDTO.setAssetNet(assetNet);
         String assetShares = columnFieldMap.get(EmailFieldConst.ASSET_SHARE) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.ASSET_SHARE)) != null ?
                 ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.ASSET_SHARE))) : null;
-        if (StrUtil.isBlank(assetShares) || !StrUtil.isNumeric(assetShares)) {
+        if (StrUtil.isBlank(assetShares)) {
             assetShares = null;
         }
         emailFundNavDTO.setAssetShare(assetShares);
@@ -406,24 +411,44 @@ public class NavEmailParser extends AbstractEmailParser {
         return null;
     }
 
-    /**
-     * 解析邮件正文
-     *
-     * @param emailContent  邮件excel附件地址
-     * @param emailFieldMap 邮件字段识别规则映射表
-     * @return 解析到的净值数据
-     */
-    private List<EmailFundNavDTO> parseEmailContent(String emailContent, Map<String, List<String>> emailFieldMap) {
-        List<EmailFundNavDTO> fundNavDTOList = CollUtil.newArrayList();
+    private List<EmailFundNavDTO> parseEmailContent(EmailContentInfoDTO emailContentInfoDTO, String emailContent, Map<String, List<String>> emailFieldMap) {
         Document doc = Jsoup.parse(emailContent);
         Element table = doc.select("table").first();
         Elements rows = table.select("tr");
-        for (Element row : rows) {
-            Elements cells = row.select("td");
-            int cellSize = cells.size();
-            // todo 后续补充
+        String excelFilePath = "/data/file/content/" + emailContentInfoDTO.getEmailDate().substring(0, 10).replaceAll("-","")
+                + "/" + emailContentInfoDTO.getFileName().replace(".html", ".xlsx");
+        File savefile = new File(excelFilePath);
+        if (!savefile.exists()) {
+            if (!savefile.getParentFile().exists()) {
+                savefile.getParentFile().mkdirs();
+                savefile.getParentFile().setExecutable(true);
+            }
         }
-        return fundNavDTOList;
+        try (OutputStream outputStream = new FileOutputStream(savefile)) {
+            // 创建一个新的Excel工作簿
+            Workbook workbook = new XSSFWorkbook();
+            Sheet sheet = workbook.createSheet("Sheet1");
+            writeDataToSheet(sheet, rows);
+            // 将Excel工作簿写入输出流
+            workbook.write(outputStream);
+        } catch (Exception e) {
+            log.error("解析正文报错 -> 堆栈信息:{}", ExceptionUtil.stacktraceToString(e));
+        }
+        return parseExcelFile(excelFilePath, emailFieldMap);
     }
 
+    private void writeDataToSheet(Sheet sheet, Elements rows) {
+        int rowSize = rows.size();
+        for (int rowNum = 0; rowNum < rowSize; rowNum++) {
+            Row sheetRow = sheet.createRow(rowNum);
+
+            Element elementRow = rows.get(rowNum);
+            Elements cells = elementRow.select("td");
+            int cellSize = cells.size();
+            for (int cellNum = 0; cellNum< cellSize; cellNum++) {
+                Cell sheetRowCell = sheetRow.createCell(cellNum);
+                sheetRowCell.setCellValue(cells.get(cellNum).text());
+            }
+        }
+    }
 }

service-deploy/src/main/test/java/com/simuwang/datadaq/DataTrusteeApplicationTests.java

@@ -1,4 +1,4 @@
-package com.simuwang.datadaq;
+package java.com.simuwang.datadaq;
 
 import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.map.MapUtil;
@@ -17,7 +17,7 @@ import java.util.Map;
 class DataTrusteeApplicationTests {
 
     @Autowired
-    EmailParseService emailParseService;
+    private EmailParseService emailParseService;
 
     @Test
     public void test() {