feat:邮件解析-支持份额基金净值文件格式

service-base/src/main/java/com/simuwang/base/common/conts/EmailFieldConst.java

@@ -11,4 +11,12 @@ public class EmailFieldConst {
     public final static String ASSET_SHARE = "assetShare";
     public final static String ASSET_NET = "assetNet";
 
+    public final static String PARENT_FUND_NAME = "parentFundName";
+    public final static String PARENT_REGISTER_NUMBER = "parentRegisterNumber";
+    public final static String PARENT_NAV = "parentNav";
+    public final static String PARENT_CUMULATIVE_NAV_WITHDRAWAL = "parentCumulativeNavWithdrawal";
+    public final static String PARENT_VIRTUAL_NAV = "parentVirtualNav";
+    public final static String PARENT_ASSET_SHARE = "parentAssetShare";
+    public final static String PARENT_ASSET_NET = "parentAssetNet";
+
 }

service-base/src/main/java/com/simuwang/base/common/util/ExcelUtil.java

@@ -2,6 +2,7 @@ package com.simuwang.base.common.util;
 
 import cn.hutool.core.util.StrUtil;
 import org.apache.poi.hssf.usermodel.HSSFWorkbook;
+import org.apache.poi.ss.usermodel.Cell;
 import org.apache.poi.ss.usermodel.Sheet;
 import org.apache.poi.ss.usermodel.Workbook;
 import org.apache.poi.xssf.usermodel.XSSFWorkbook;
@@ -9,6 +10,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.*;
+import java.text.NumberFormat;
 
 public class ExcelUtil {
 
@@ -133,4 +135,49 @@ public class ExcelUtil {
             return null;
         }
     }
+
+    public static String getCellValue(Cell cell) {
+        if (cell == null) {
+            return null;
+        }
+        String cellValue = "";
+
+        switch (cell.getCellTypeEnum()) {
+            case STRING:
+                cellValue = cell.getStringCellValue();
+                break;
+            case NUMERIC:
+                if (org.apache.poi.ss.usermodel.DateUtil.isCellDateFormatted(cell)) {
+                    // 如果是日期格式的数字
+                    cellValue = cell.getDateCellValue().toString();
+                } else {
+                    // 否则是纯数字
+                    NumberFormat numberFormat = NumberFormat.getNumberInstance();
+                    numberFormat.setMaximumFractionDigits(15);
+                    double formulaResult = cell.getNumericCellValue();
+                    cellValue = numberFormat.format(formulaResult).replaceAll(",","");
+                }
+                break;
+            case BOOLEAN:
+                cellValue = String.valueOf(cell.getBooleanCellValue());
+                break;
+            case FORMULA:
+                // 处理公式结果
+                try {
+                    cellValue = String.valueOf(cell.getNumericCellValue());
+                } catch (IllegalStateException e) {
+                    cellValue = cell.getStringCellValue();
+                }
+                break;
+            case BLANK:
+                break;
+            case ERROR:
+                cellValue = "ERROR: " + cell.getErrorCellValue();
+                break;
+            default:
+                cellValue = "";
+                break;
+        }
+        return cellValue;
+    }
 }

service-daq/src/main/java/com/simuwang/daq/service/EmailParseService.java

@@ -112,6 +112,12 @@ public class EmailParseService {
         }
     }
 
+    private List<EmailFundNavDTO> parseEmail(EmailContentInfoDTO emailContentInfoDTO, Map<String, List<String>> emailFieldMap) {
+        Integer emailType = emailContentInfoDTO.getEmailType();
+        AbstractEmailParser emailParser = emailParserFactory.getInstance(emailType);
+        return emailParser.parse(emailContentInfoDTO, emailFieldMap);
+    }
+
     private void saveRelatedTable(String emailAddress, List<EmailContentInfoDTO> emailContentInfoDTOList, Map<EmailContentInfoDTO, List<EmailFundNavDTO>> fileNameNavMap) {
         String emailTitle = CollUtil.isNotEmpty(emailContentInfoDTOList) ? emailContentInfoDTOList.get(0).getEmailTitle() : null;
         String emailDate = CollUtil.isNotEmpty(emailContentInfoDTOList) ? emailContentInfoDTOList.get(0).getEmailDate() : null;
@@ -241,6 +247,7 @@ public class EmailParseService {
             emailFundNavDO.setFundName(fundNavDTO.getFundName());
             emailFundNavDO.setRegisterNumber(fundNavDTO.getRegisterNumber());
             emailFundNavDO.setExceptionStatus(fundNavDTO.getParseStatus());
+            emailFundNavDO.setIsStored(isStored);
             emailFundNavDO.setIsvalid(1);
             emailFundNavDO.setCreatorId(0);
             emailFundNavDO.setCreateTime(parseDate);
@@ -341,12 +348,6 @@ public class EmailParseService {
         return emailParseInfoDO;
     }
 
-    private List<EmailFundNavDTO> parseEmail(EmailContentInfoDTO emailContentInfoDTO, Map<String, List<String>> emailFieldMap) {
-        Integer emailType = emailContentInfoDTO.getEmailType();
-        AbstractEmailParser emailParser = emailParserFactory.getInstance(emailType);
-        return emailParser.parse(emailContentInfoDTO, emailFieldMap);
-    }
-
     private Integer getDataParseStatus(EmailFundNavDTO fundNavDTO, String emailTitle, String emailDate) {
         log.info("判断数据的解析状态 -> 邮件主题:{},邮件日期:{},净值数据:{}", emailTitle, emailDate, fundNavDTO);
         // 1.单位净值或累计净值缺失

service-daq/src/main/java/com/simuwang/daq/service/NavEmailParser.java

@@ -24,6 +24,7 @@ import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
 
 import java.io.File;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -104,7 +105,7 @@ public class NavEmailParser extends AbstractEmailParser {
             // 遍历可能的数据行
             for (int rowNum = initRow + 1; rowNum <= lastRowNum; rowNum++) {
                 Row sheetRow = sheet.getRow(rowNum);
-                Optional.ofNullable(readSheetRowData(sheetRow, fieldColumnMap)).ifPresent(fundNavDTOList::add);
+                Optional.ofNullable(readSheetRowData(sheetRow, fieldColumnMap)).ifPresent(fundNavDTOList::addAll);
             }
         }
         if (dataDirectionType.equals(COLUMN_DIRECTION_TYPE)) {
@@ -197,45 +198,93 @@ public class NavEmailParser extends AbstractEmailParser {
         return fundNavDTO;
     }
 
-
-    private EmailFundNavDTO readSheetRowData(Row sheetRow, Map<String, Integer> columnFieldMap) {
+    private List<EmailFundNavDTO> readSheetRowData(Row sheetRow, Map<String, Integer> columnFieldMap) {
         if (sheetRow == null) {
             return null;
         }
         String nav = columnFieldMap.get(EmailFieldConst.NAV) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.NAV)) != null
-                ? sheetRow.getCell(columnFieldMap.get(EmailFieldConst.NAV)).getStringCellValue() : null;
+                ? ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.NAV))) : null;
         String cumulativeNavWithdrawal = columnFieldMap.get(EmailFieldConst.CUMULATIVE_NAV_WITHDRAWAL) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.CUMULATIVE_NAV_WITHDRAWAL)) != null ?
-                sheetRow.getCell(columnFieldMap.get(EmailFieldConst.CUMULATIVE_NAV_WITHDRAWAL)).getStringCellValue() : null;
-        if (StrUtil.isBlank(nav) || StrUtil.isBlank(cumulativeNavWithdrawal)) {
+                ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.CUMULATIVE_NAV_WITHDRAWAL))) : null;
+        if (StrUtil.isBlank(nav) && StrUtil.isBlank(cumulativeNavWithdrawal)) {
             return null;
         }
+        List<EmailFundNavDTO> fundNavDTOList = CollUtil.newArrayList();
         EmailFundNavDTO emailFundNavDTO = new EmailFundNavDTO();
         String priceDate = columnFieldMap.get(EmailFieldConst.PRICE_DATE) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.PRICE_DATE)) != null ?
-                sheetRow.getCell(columnFieldMap.get(EmailFieldConst.PRICE_DATE)).getStringCellValue() : null;
+                ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.PRICE_DATE))) : null;
+
+        // 份额基金净值文件格式
+        long parentFiledCount = columnFieldMap.keySet().stream().filter(e -> e.contains("parent")).count();
+        if (parentFiledCount >= 1) {
+            Optional.ofNullable(buildParentNav(sheetRow, columnFieldMap, priceDate)).ifPresent(fundNavDTOList::add);
+        }
+        // 正常净值文件格式
         if (StrUtil.isNotBlank(priceDate) && !priceDate.contains("-")) {
             // 处理日期yyyyMMdd格式 -> 转成yyyy-MM-dd
             priceDate = DateUtil.format(DateUtil.parse(priceDate, DateConst.YYYYMMDD), DateConst.YYYY_MM_DD);
         }
         emailFundNavDTO.setPriceDate(priceDate);
         String fundName = columnFieldMap.get(EmailFieldConst.FUND_NAME) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.FUND_NAME)).getStringCellValue() != null ?
-                sheetRow.getCell(columnFieldMap.get(EmailFieldConst.FUND_NAME)).getStringCellValue() : null;
+                ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.FUND_NAME))) : null;
         emailFundNavDTO.setFundName(fundName);
         String registerNumber = columnFieldMap.get(EmailFieldConst.REGISTER_NUMBER) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.REGISTER_NUMBER)) != null ?
-                sheetRow.getCell(columnFieldMap.get(EmailFieldConst.REGISTER_NUMBER)).getStringCellValue() : null;
+                ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.REGISTER_NUMBER))) : null;
         emailFundNavDTO.setRegisterNumber(registerNumber);
         emailFundNavDTO.setNav(nav);
         emailFundNavDTO.setCumulativeNavWithdrawal(cumulativeNavWithdrawal);
         String virtualNav = columnFieldMap.get(EmailFieldConst.VIRTUAL_NAV) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.VIRTUAL_NAV)) != null ?
-                sheetRow.getCell(columnFieldMap.get(EmailFieldConst.VIRTUAL_NAV)).getStringCellValue() : null;
+                ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.VIRTUAL_NAV))) : null;
         emailFundNavDTO.setVirtualNav(virtualNav);
         String assetNet = columnFieldMap.get(EmailFieldConst.ASSET_NET) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.ASSET_NET)) != null ?
-                sheetRow.getCell(columnFieldMap.get(EmailFieldConst.ASSET_NET)).getStringCellValue() : null;
+                ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.ASSET_NET))) : null;
         if (StrUtil.isBlank(assetNet) || !StrUtil.isNumeric(assetNet)) {
             assetNet = null;
         }
         emailFundNavDTO.setAssetNet(assetNet);
         String assetShares = columnFieldMap.get(EmailFieldConst.ASSET_SHARE) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.ASSET_SHARE)) != null ?
-                sheetRow.getCell(columnFieldMap.get(EmailFieldConst.ASSET_SHARE)).getStringCellValue() : null;
+                ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.ASSET_SHARE))) : null;
+        if (StrUtil.isBlank(assetShares) || !StrUtil.isNumeric(assetShares)) {
+            assetShares = null;
+        }
+        emailFundNavDTO.setAssetShare(assetShares);
+        fundNavDTOList.add(emailFundNavDTO);
+        return fundNavDTOList;
+    }
+
+    private EmailFundNavDTO buildParentNav(Row sheetRow, Map<String, Integer> columnFieldMap, String priceDate) {
+        EmailFundNavDTO emailFundNavDTO = new EmailFundNavDTO();
+        String nav = columnFieldMap.get(EmailFieldConst.PARENT_NAV) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.PARENT_NAV)) != null ?
+                ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.PARENT_NAV))) : null;
+        String cumulativeNavWithdrawal = columnFieldMap.get(EmailFieldConst.PARENT_CUMULATIVE_NAV_WITHDRAWAL) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.PARENT_CUMULATIVE_NAV_WITHDRAWAL)) != null ?
+                ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.PARENT_CUMULATIVE_NAV_WITHDRAWAL))) : null;
+        if (StrUtil.isBlank(nav) && StrUtil.isBlank(cumulativeNavWithdrawal)) {
+            return null;
+        }
+        if (StrUtil.isNotBlank(priceDate) && !priceDate.contains("-")) {
+            // 处理日期yyyyMMdd格式 -> 转成yyyy-MM-dd
+            priceDate = DateUtil.format(DateUtil.parse(priceDate, DateConst.YYYYMMDD), DateConst.YYYY_MM_DD);
+        }
+        emailFundNavDTO.setPriceDate(priceDate);
+        String fundName = columnFieldMap.get(EmailFieldConst.PARENT_FUND_NAME) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.PARENT_FUND_NAME)).getStringCellValue() != null ?
+                ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.PARENT_FUND_NAME))) : null;
+        emailFundNavDTO.setFundName(fundName);
+        String registerNumber = columnFieldMap.get(EmailFieldConst.PARENT_REGISTER_NUMBER) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.PARENT_REGISTER_NUMBER)) != null ?
+                ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.PARENT_REGISTER_NUMBER))) : null;
+        emailFundNavDTO.setRegisterNumber(registerNumber);
+        emailFundNavDTO.setNav(nav);
+        emailFundNavDTO.setCumulativeNavWithdrawal(cumulativeNavWithdrawal);
+        String virtualNav = columnFieldMap.get(EmailFieldConst.PARENT_VIRTUAL_NAV) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.PARENT_VIRTUAL_NAV)) != null ?
+                ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.PARENT_VIRTUAL_NAV))) : null;
+        emailFundNavDTO.setVirtualNav(virtualNav);
+        String assetNet = columnFieldMap.get(EmailFieldConst.PARENT_ASSET_NET) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.PARENT_ASSET_NET)) != null ?
+                ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.PARENT_ASSET_NET))) : null;
+        if (StrUtil.isBlank(assetNet) || !StrUtil.isNumeric(assetNet)) {
+            assetNet = null;
+        }
+        emailFundNavDTO.setAssetNet(assetNet);
+        String assetShares = columnFieldMap.get(EmailFieldConst.PARENT_ASSET_SHARE) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.PARENT_ASSET_SHARE)) != null ?
+                ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.PARENT_ASSET_SHARE))) : null;
         if (StrUtil.isBlank(assetShares) || !StrUtil.isNumeric(assetShares)) {
             assetShares = null;
         }
@@ -288,7 +337,7 @@ public class NavEmailParser extends AbstractEmailParser {
      * @return excel中表头所在的位置(行, 列)
      */
     private Map<String, Pair<Integer, Integer>> getFieldPosition(Sheet sheet, Map<String, List<String>> emailFieldMap) {
-        Map<String, Pair<Integer, Integer>> fieldPositionMap = MapUtil.newHashMap(8);
+        Map<String, List<Pair<Integer, Integer>>> tempFieldPositionMap = MapUtil.newHashMap();
         int lastRowNum = sheet.getLastRowNum();
         for (int rowNum = 0; rowNum < lastRowNum; rowNum++) {
             Row sheetRow = sheet.getRow(rowNum);
@@ -301,14 +350,36 @@ public class NavEmailParser extends AbstractEmailParser {
                 if (cell == null) {
                     continue;
                 }
-                String cellValue = cell.getStringCellValue();
+                String cellValue = ExcelUtil.getCellValue(cell);
                 String field = fieldMatch(cellValue, emailFieldMap);
-                // todo 考虑份额基金净值文件格式
                 if (StrUtil.isNotBlank(field)) {
-                    fieldPositionMap.put(field, new Pair<>(rowNum, cellNum));
+                    List<Pair<Integer, Integer>> pairList = tempFieldPositionMap.getOrDefault(field, new ArrayList<>());
+                    pairList.add(Pair.of(rowNum, cellNum));
+                    tempFieldPositionMap.put(field, pairList);
                 }
             }
         }
+        // 判断是不是份额基金净值文件格式(同时存在两个备案编码字段)
+        return handlerFieldPosition(tempFieldPositionMap);
+    }
+
+    private Map<String, Pair<Integer, Integer>> handlerFieldPosition(Map<String, List<Pair<Integer, Integer>>> tempFieldPositionMap) {
+        Map<String, Pair<Integer, Integer>> fieldPositionMap = MapUtil.newHashMap();
+        boolean hasParentField = tempFieldPositionMap.keySet().stream().anyMatch(e -> e.contains("parent"));
+        for (Map.Entry<String, List<Pair<Integer, Integer>>> entry : tempFieldPositionMap.entrySet()) {
+            List<Pair<Integer, Integer>> pairList = entry.getValue();
+            if (pairList.size() == 1) {
+                fieldPositionMap.put(entry.getKey(), pairList.get(0));
+                continue;
+            }
+            if ((!hasParentField && pairList.size() > 1)) {
+                fieldPositionMap.put(entry.getKey(), pairList.get(pairList.size() - 1));
+                continue;
+            }
+            if ((hasParentField && pairList.size() > 1)) {
+                fieldPositionMap.put(entry.getKey(), pairList.get(0));
+            }
+        }
         return fieldPositionMap;
     }
 

service-deploy/src/main/test/java/com/simuwang/datadaq/DataTrusteeApplicationTests.java

@@ -32,8 +32,8 @@ class DataTrusteeApplicationTests {
         emailInfoDTO.setProtocol("imap");
         Map<Integer, List<String>> emailTypeMap = MapUtil.newHashMap();
         emailTypeMap.put(1, List.of("净值"));
-        Date startDate = DateUtil.parse("2024-09-06 18:40:00", DateConst.YYYY_MM_DD_HH_MM_SS);
-        Date endDate = DateUtil.parse("2024-09-06 19:00:00", DateConst.YYYY_MM_DD_HH_MM_SS);
+        Date startDate = DateUtil.parse("2024-09-10 09:40:00", DateConst.YYYY_MM_DD_HH_MM_SS);
+        Date endDate = DateUtil.parse("2024-09-10 10:00:00", DateConst.YYYY_MM_DD_HH_MM_SS);
         try {
             emailParseService.parseEmail(emailInfoDTO, startDate, endDate);
         } catch (Exception e) {