feat:邮件解析-支持分级分类产品净值文件

service-base/src/main/java/com/simuwang/base/common/conts/EmailFieldConst.java

@@ -19,4 +19,7 @@ public class EmailFieldConst {
     public final static String PARENT_ASSET_SHARE = "parentAssetShare";
     public final static String PARENT_ASSET_NET = "parentAssetNet";
 
+    public final static String LEVEL_FUND_NAME = "levelFundName";
+    public final static String LEVEL_REGISTER_NUMBER = "levelRegisterNumber";
+
 }

service-base/src/main/java/com/simuwang/base/common/util/ExcelUtil.java

@@ -1,18 +1,25 @@
 package com.simuwang.base.common.util;
 
 import cn.hutool.core.collection.CollUtil;
+import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.util.StrUtil;
+import com.simuwang.base.common.conts.DateConst;
 import org.apache.poi.hssf.usermodel.HSSFWorkbook;
 import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.Row;
 import org.apache.poi.ss.usermodel.Sheet;
 import org.apache.poi.ss.usermodel.Workbook;
 import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.*;
 import java.text.NumberFormat;
+import java.util.Date;
 import java.util.List;
+
 import org.apache.commons.compress.archivers.ArchiveEntry;
 import org.apache.commons.compress.archivers.ArchiveInputStream;
 import org.apache.commons.compress.archivers.ArchiveStreamFactory;
@@ -155,7 +162,8 @@ public class ExcelUtil {
             case NUMERIC:
                 if (org.apache.poi.ss.usermodel.DateUtil.isCellDateFormatted(cell)) {
                     // 如果是日期格式的数字
-                    cellValue = cell.getDateCellValue().toString();
+                    Date dateCellValue = cell.getDateCellValue();
+                    cellValue = DateUtil.format(dateCellValue, DateConst.YYYY_MM_DD);
                 } else {
                     // 否则是纯数字
                     NumberFormat numberFormat = NumberFormat.getNumberInstance();
@@ -218,4 +226,54 @@ public class ExcelUtil {
         return filePathList;
     }
 
+    public static void writeDataToSheet(Sheet sheet, Elements rows) {
+        int rowSize = rows.size();
+        for (int rowNum = 0; rowNum < rowSize; rowNum++) {
+            Row sheetRow = sheet.createRow(rowNum);
+
+            Element elementRow = rows.get(rowNum);
+            Elements cells = elementRow.select("td");
+            int cellSize = cells.size();
+            for (int cellNum = 0; cellNum < cellSize; cellNum++) {
+                Cell sheetRowCell = sheetRow.createCell(cellNum);
+                sheetRowCell.setCellValue(cells.get(cellNum).text());
+            }
+        }
+    }
+
+
+    /**
+     * 获取优先级高的字段值
+     *
+     * @param sheetRow         行
+     * @param priorityPosition 字段位置
+     * @param basePosition     优先级搞的字段位置
+     * @return 优先级高的字段值
+     */
+    public static String getPriorityFieldValue(Row sheetRow, Integer priorityPosition, Integer basePosition) {
+        boolean hasPriorityValue = priorityPosition != null && sheetRow.getCell(priorityPosition) != null && StrUtil.isNotBlank(ExcelUtil.getCellValue(sheetRow.getCell(priorityPosition)));
+        if (hasPriorityValue) {
+            return ExcelUtil.getCellValue(sheetRow.getCell(priorityPosition));
+        }
+        return basePosition != null && sheetRow.getCell(basePosition) != null ? ExcelUtil.getCellValue(sheetRow.getCell(basePosition)) : null;
+    }
+
+    /**
+     * 去掉逗号
+     *
+     * @param numberData 数字型字符串
+     * @return 无逗号的数字型字符串
+     */
+    public static String numberDataStripCommas(String numberData) {
+        if (StrUtil.isBlank(numberData)) {
+            return null;
+        }
+        // pdf解析到的值带有",",比如:"10,656,097.37"
+        String data = numberData.replaceAll(",", "");
+        if (!StringUtil.isNumeric(data)) {
+            return null;
+        }
+        return data;
+    }
+
 }

service-daq/src/main/java/com/simuwang/daq/service/NavEmailParser.java

@@ -96,7 +96,7 @@ public class NavEmailParser extends AbstractEmailParser {
     }
 
     private List<EmailFundNavDTO> parsePackageFile(EmailContentInfoDTO emailContentInfoDTO, String fileName, String filePath, Map<String, List<String>> emailFieldMap) {
-        String destPath = filePath.substring(0, filePath.indexOf(fileName)) + fileName.replaceAll(".zip","").replaceAll(".ZIP","") ;
+        String destPath = filePath.substring(0, filePath.indexOf(fileName)) + fileName.replaceAll(".zip", "").replaceAll(".ZIP", "");
         log.info("压缩包地址:{},解压后文件地址:{}", filePath, destPath);
         List<String> dir = ExcelUtil.extractCompressedFiles(filePath, destPath);
         List<EmailFundNavDTO> emailFundNavDTOList = CollUtil.newArrayList();
@@ -220,7 +220,7 @@ public class NavEmailParser extends AbstractEmailParser {
             // 创建一个新的Excel工作簿
             Workbook workbook = new XSSFWorkbook();
             Sheet sheet = workbook.createSheet("Sheet1");
-            writeDataToSheet(sheet, rows);
+            ExcelUtil.writeDataToSheet(sheet, rows);
             // 将Excel工作簿写入输出流
             workbook.write(outputStream);
         } catch (Exception e) {
@@ -327,10 +327,10 @@ public class NavEmailParser extends AbstractEmailParser {
 
         // pdf解析到的值带有",",比如:"10,656,097.37"
         String assetNet = fieldValueMap.get(EmailFieldConst.ASSET_NET);
-        fundNavDTO.setAssetNet(numberDataHandler(assetNet));
+        fundNavDTO.setAssetNet(ExcelUtil.numberDataStripCommas(assetNet));
 
         String assetShares = fieldValueMap.get(EmailFieldConst.ASSET_NET);
-        fundNavDTO.setAssetNet(numberDataHandler(assetShares));
+        fundNavDTO.setAssetNet(ExcelUtil.numberDataStripCommas(assetShares));
 
         return fundNavDTO;
     }
@@ -359,17 +359,16 @@ public class NavEmailParser extends AbstractEmailParser {
         // 正常净值文件格式
         if (StrUtil.isNotBlank(priceDate) && !priceDate.contains("-")) {
             // 处理日期yyyyMMdd格式 -> 转成yyyy-MM-dd
-            priceDate = priceDate.replace("年","").replace("月","").replace("日","");
+            priceDate = priceDate.replace("年", "").replace("月", "").replace("日", "");
             priceDate = DateUtil.format(DateUtil.parse(priceDate, DateConst.YYYYMMDD), DateConst.YYYY_MM_DD);
         }
         emailFundNavDTO.setPriceDate(priceDate);
-        String fundName = columnFieldMap.get(EmailFieldConst.FUND_NAME) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.FUND_NAME)).getStringCellValue() != null ?
-                ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.FUND_NAME))) : null;
+        String fundName = ExcelUtil.getPriorityFieldValue(sheetRow, columnFieldMap.get(EmailFieldConst.LEVEL_FUND_NAME), columnFieldMap.get(EmailFieldConst.FUND_NAME));
         emailFundNavDTO.setFundName(fundName);
 
-        String registerNumber = columnFieldMap.get(EmailFieldConst.REGISTER_NUMBER) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.REGISTER_NUMBER)) != null ?
-                ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.REGISTER_NUMBER))) : null;
+        String registerNumber = ExcelUtil.getPriorityFieldValue(sheetRow, columnFieldMap.get(EmailFieldConst.LEVEL_REGISTER_NUMBER), columnFieldMap.get(EmailFieldConst.REGISTER_NUMBER));
         emailFundNavDTO.setRegisterNumber(registerNumber);
+
         emailFundNavDTO.setNav(nav);
         emailFundNavDTO.setCumulativeNavWithdrawal(cumulativeNavWithdrawal);
         String virtualNav = columnFieldMap.get(EmailFieldConst.VIRTUAL_NAV) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.VIRTUAL_NAV)) != null ?
@@ -378,28 +377,16 @@ public class NavEmailParser extends AbstractEmailParser {
 
         String assetNet = columnFieldMap.get(EmailFieldConst.ASSET_NET) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.ASSET_NET)) != null ?
                 ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.ASSET_NET))) : null;
-        emailFundNavDTO.setAssetNet(numberDataHandler(assetNet));
+        emailFundNavDTO.setAssetNet(ExcelUtil.numberDataStripCommas(assetNet));
 
         String assetShares = columnFieldMap.get(EmailFieldConst.ASSET_SHARE) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.ASSET_SHARE)) != null ?
                 ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.ASSET_SHARE))) : null;
-        emailFundNavDTO.setAssetShare(numberDataHandler(assetShares));
+        emailFundNavDTO.setAssetShare(ExcelUtil.numberDataStripCommas(assetShares));
 
         fundNavDTOList.add(emailFundNavDTO);
         return fundNavDTOList;
     }
 
-    private String numberDataHandler(String numberData) {
-        if (StrUtil.isBlank(numberData)) {
-            return null;
-        }
-        // pdf解析到的值带有",",比如:"10,656,097.37"
-        String data = numberData.replaceAll(",", "");
-        if (!StringUtil.isNumeric(data)) {
-            return null;
-        }
-        return data;
-    }
-
     private EmailFundNavDTO buildParentNav(Row sheetRow, Map<String, Integer> columnFieldMap, String priceDate) {
         EmailFundNavDTO emailFundNavDTO = new EmailFundNavDTO();
         String nav = columnFieldMap.get(EmailFieldConst.PARENT_NAV) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.PARENT_NAV)) != null ?
@@ -411,7 +398,7 @@ public class NavEmailParser extends AbstractEmailParser {
         }
         if (StrUtil.isNotBlank(priceDate) && !priceDate.contains("-")) {
             // 处理日期yyyyMMdd格式 -> 转成yyyy-MM-dd
-            priceDate = priceDate.replace("年","").replace("月","").replace("日","");
+            priceDate = priceDate.replace("年", "").replace("月", "").replace("日", "");
             priceDate = DateUtil.format(DateUtil.parse(priceDate, DateConst.YYYYMMDD), DateConst.YYYY_MM_DD);
         }
         emailFundNavDTO.setPriceDate(priceDate);
@@ -429,11 +416,11 @@ public class NavEmailParser extends AbstractEmailParser {
 
         String assetNet = columnFieldMap.get(EmailFieldConst.PARENT_ASSET_NET) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.PARENT_ASSET_NET)) != null ?
                 ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.PARENT_ASSET_NET))) : null;
-        emailFundNavDTO.setAssetNet(numberDataHandler(assetNet));
+        emailFundNavDTO.setAssetNet(ExcelUtil.numberDataStripCommas(assetNet));
 
         String assetShares = columnFieldMap.get(EmailFieldConst.PARENT_ASSET_SHARE) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.PARENT_ASSET_SHARE)) != null ?
                 ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.PARENT_ASSET_SHARE))) : null;
-        emailFundNavDTO.setAssetNet(numberDataHandler(assetShares));
+        emailFundNavDTO.setAssetNet(ExcelUtil.numberDataStripCommas(assetShares));
 
         return emailFundNavDTO;
     }
@@ -551,19 +538,4 @@ public class NavEmailParser extends AbstractEmailParser {
         }
         return null;
     }
-
-    private void writeDataToSheet(Sheet sheet, Elements rows) {
-        int rowSize = rows.size();
-        for (int rowNum = 0; rowNum < rowSize; rowNum++) {
-            Row sheetRow = sheet.createRow(rowNum);
-
-            Element elementRow = rows.get(rowNum);
-            Elements cells = elementRow.select("td");
-            int cellSize = cells.size();
-            for (int cellNum = 0; cellNum < cellSize; cellNum++) {
-                Cell sheetRowCell = sheetRow.createCell(cellNum);
-                sheetRowCell.setCellValue(cells.get(cellNum).text());
-            }
-        }
-    }
 }

service-deploy/src/main/test/java/com/simuwang/datadaq/DataTrusteeApplicationTests.java

@@ -32,8 +32,8 @@ class DataTrusteeApplicationTests {
         emailInfoDTO.setProtocol("imap");
         Map<Integer, List<String>> emailTypeMap = MapUtil.newHashMap();
         emailTypeMap.put(1, List.of("净值"));
-        Date startDate = DateUtil.parse("2024-09-11 11:10:00", DateConst.YYYY_MM_DD_HH_MM_SS);
-        Date endDate = DateUtil.parse("2024-09-11 12:00:00", DateConst.YYYY_MM_DD_HH_MM_SS);
+        Date startDate = DateUtil.parse("2024-09-12 11:10:00", DateConst.YYYY_MM_DD_HH_MM_SS);
+        Date endDate = DateUtil.parse("2024-09-12 12:00:00", DateConst.YYYY_MM_DD_HH_MM_SS);
         try {
             emailParseService.parseEmail(emailInfoDTO, startDate, endDate);
         } catch (Exception e) {