feat:邮件解析-兼容母基金缺少代码和子基金净值同时存在“资产代码”和“协会备案编码”的情况

service-base/src/main/java/com/simuwang/base/common/util/StringUtil.java

@@ -8,7 +8,6 @@ import java.util.List;
 
 import cn.hutool.core.text.StrFormatter;
 import com.simuwang.base.common.conts.Constants;
-import org.apache.commons.lang3.CharSequenceUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.util.AntPathMatcher;
 
@@ -29,6 +28,9 @@ public class StringUtil {
     private static final char SEPARATOR = '_';
 
     public static String retainChineseCharacters(String value) {
+        if (StrUtil.isBlank(value)) {
+            return null;
+        }
         // 正则表达式匹配中文字符
         String regex = "[^\\u4e00-\\u9fa5]";
         // 使用空字符串替换所有非中文字符

service-base/src/main/java/com/simuwang/base/pojo/dto/FieldPositionDTO.java

@@ -0,0 +1,22 @@
+package com.simuwang.base.pojo.dto;
+
+import cn.hutool.core.lang.Pair;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class FieldPositionDTO {
+
+    /**
+     * 字段值
+     */
+    private String fieldValue;
+
+    /**
+     * 字段对应的行号，列号
+     */
+    Pair<Integer, Integer> pair;
+}

service-daq/src/main/java/com/simuwang/daq/service/NavEmailParser.java

@@ -13,6 +13,7 @@ import com.simuwang.base.common.util.ExcelUtil;
 import com.simuwang.base.common.util.StringUtil;
 import com.simuwang.base.pojo.dto.EmailContentInfoDTO;
 import com.simuwang.base.pojo.dto.EmailFundNavDTO;
+import com.simuwang.base.pojo.dto.FieldPositionDTO;
 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.poi.ss.usermodel.Cell;
@@ -63,7 +64,8 @@ public class NavEmailParser extends AbstractEmailParser {
     }
 
     @Override
-    public List<EmailFundNavDTO> parse(EmailContentInfoDTO emailContentInfoDTO, Map<String, List<String>> emailFieldMap) {
+    public List<EmailFundNavDTO>
+    parse(EmailContentInfoDTO emailContentInfoDTO, Map<String, List<String>> emailFieldMap) {
         List<EmailFundNavDTO> emailFundNavDTOList = CollUtil.newArrayList();
         String emailContent = emailContentInfoDTO.getEmailContent();
         // 1.解析邮件正文
@@ -471,7 +473,7 @@ public class NavEmailParser extends AbstractEmailParser {
      * @return excel中表头所在的位置(行, 列)
      */
     private Map<String, Pair<Integer, Integer>> getFieldPosition(Sheet sheet, Map<String, List<String>> emailFieldMap) {
-        Map<String, List<Pair<Integer, Integer>>> tempFieldPositionMap = MapUtil.newHashMap();
+        Map<String, List<FieldPositionDTO>> tempFieldPositionMap = MapUtil.newHashMap();
         int lastRowNum = sheet.getLastRowNum();
         for (int rowNum = 0; rowNum <= lastRowNum; rowNum++) {
             Row sheetRow = sheet.getRow(rowNum);
@@ -485,11 +487,13 @@ public class NavEmailParser extends AbstractEmailParser {
                     continue;
                 }
                 String cellValue = ExcelUtil.getCellValue(cell);
-                String field = fieldMatch(cellValue, emailFieldMap);
+                // 移除掉非中文字符
+                String newCellValue = StringUtil.retainChineseCharacters(cellValue);
+                String field = fieldMatch(newCellValue, emailFieldMap);
                 if (StrUtil.isNotBlank(field)) {
-                    List<Pair<Integer, Integer>> pairList = tempFieldPositionMap.getOrDefault(field, new ArrayList<>());
-                    pairList.add(Pair.of(rowNum, cellNum));
-                    tempFieldPositionMap.put(field, pairList);
+                    List<FieldPositionDTO> fieldPositionDTOList = tempFieldPositionMap.getOrDefault(field, new ArrayList<>());
+                    fieldPositionDTOList.add(new FieldPositionDTO(newCellValue, Pair.of(rowNum, cellNum)));
+                    tempFieldPositionMap.put(field, fieldPositionDTOList);
                 }
             }
         }
@@ -497,23 +501,39 @@ public class NavEmailParser extends AbstractEmailParser {
         return handlerFieldPosition(tempFieldPositionMap);
     }
 
-    private Map<String, Pair<Integer, Integer>> handlerFieldPosition(Map<String, List<Pair<Integer, Integer>>> tempFieldPositionMap) {
+    private Map<String, Pair<Integer, Integer>> handlerFieldPosition(Map<String, List<FieldPositionDTO>> tempFieldPositionMap) {
         Map<String, Pair<Integer, Integer>> fieldPositionMap = MapUtil.newHashMap();
         boolean hasParentField = tempFieldPositionMap.keySet().stream().anyMatch(e -> e.contains("parent"));
-        for (Map.Entry<String, List<Pair<Integer, Integer>>> entry : tempFieldPositionMap.entrySet()) {
-            List<Pair<Integer, Integer>> pairList = entry.getValue();
-            if (pairList.size() == 1) {
-                fieldPositionMap.put(entry.getKey(), pairList.get(0));
+        for (Map.Entry<String, List<FieldPositionDTO>> entry : tempFieldPositionMap.entrySet()) {
+            String field = entry.getKey();
+            List<FieldPositionDTO> fieldPositionDTOList = entry.getValue();
+            int size = fieldPositionDTOList.size();
+            if (size == 1) {
+                fieldPositionMap.put(field, fieldPositionDTOList.get(0).getPair());
                 continue;
             }
-            if ((!hasParentField && pairList.size() > 1)) {
-                fieldPositionMap.put(entry.getKey(), pairList.get(pairList.size() - 1));
+            if ((!hasParentField && size > 1)) {
+                if (EmailFieldConst.REGISTER_NUMBER.equals(field)) {
+                    Pair<Integer, Integer> pair = fieldPositionDTOList.stream()
+                            .filter(e -> !e.getFieldValue().contains("协会") && !e.getFieldValue().contains("备案")).map(FieldPositionDTO::getPair).findFirst().orElse(null);
+                    fieldPositionMap.put(field, pair);
+                } else {
+                    fieldPositionMap.put(field, fieldPositionDTOList.get(size - 1).getPair());
+                }
                 continue;
             }
-            if ((hasParentField && pairList.size() > 1)) {
-                fieldPositionMap.put(entry.getKey(), pairList.get(0));
+            if ((hasParentField && size > 1)) {
+                fieldPositionMap.put(field, fieldPositionDTOList.get(0).getPair());
             }
         }
+
+        // 母基金缺少代码的情况
+        if (hasParentField && fieldPositionMap.get(EmailFieldConst.PARENT_REGISTER_NUMBER) == null) {
+            List<FieldPositionDTO> fieldPositionDTOS = tempFieldPositionMap.get(EmailFieldConst.REGISTER_NUMBER);
+            Pair<Integer, Integer> parentRegisterNumberPair = fieldPositionDTOS.stream()
+                    .filter(e -> e.getFieldValue().contains("协会") || e.getFieldValue().contains("备案")).map(FieldPositionDTO::getPair).findFirst().orElse(null);
+            fieldPositionMap.put(EmailFieldConst.PARENT_REGISTER_NUMBER, parentRegisterNumberPair);
+        }
         return fieldPositionMap;
     }
 
@@ -528,11 +548,10 @@ public class NavEmailParser extends AbstractEmailParser {
         if (StrUtil.isBlank(cellValue)) {
             return null;
         }
-        String newCellValue = StringUtil.retainChineseCharacters(cellValue);
         for (Map.Entry<String, List<String>> fieldEntry : emailFieldMap.entrySet()) {
             List<String> fieldList = fieldEntry.getValue();
             for (String field : fieldList) {
-                if (newCellValue.equals(field)) {
+                if (cellValue.equals(field)) {
                     return fieldEntry.getKey();
                 }
             }

service-deploy/src/main/test/java/com/simuwang/datadaq/DataTrusteeApplicationTests.java

@@ -30,23 +30,19 @@ class DataTrusteeApplicationTests {
         emailInfoDTO.setUserId(2395446);
         emailInfoDTO.setAccount("mozuwen@simuwang.com");
         emailInfoDTO.setPassword("Mzw@0306");
-//        emailInfoDTO.setAccount("fofservice@simuwang.com");
-//        emailInfoDTO.setPassword("cJH@2021");
         emailInfoDTO.setHost("imap.exmail.qq.com");
         emailInfoDTO.setPort("993");
         emailInfoDTO.setProtocol("imap");
-
-        emailInfoDTO.setAccount("jjpj_test");
-        emailInfoDTO.setPassword("shzq#919");
-//        emailInfoDTO.setAccount("fofservice@simuwang.com");
-//        emailInfoDTO.setPassword("cJH@2021");
-        emailInfoDTO.setHost("mail.shzq.com");
-        emailInfoDTO.setPort("993");
-        emailInfoDTO.setProtocol("imap");
+//
+//        emailInfoDTO.setAccount("jjpj_test");
+//        emailInfoDTO.setPassword("shzq#919");
+//        emailInfoDTO.setHost("mail.shzq.com");
+//        emailInfoDTO.setPort("993");
+//        emailInfoDTO.setProtocol("imap");
         Map<Integer, List<String>> emailTypeMap = MapUtil.newHashMap();
         emailTypeMap.put(1, List.of("净值"));
-        Date startDate = DateUtil.parse("2024-09-19 11:20:00", DateConst.YYYY_MM_DD_HH_MM_SS);
-        Date endDate = DateUtil.parse("2024-09-19 16:00:00", DateConst.YYYY_MM_DD_HH_MM_SS);
+        Date startDate = DateUtil.parse("2024-09-19 20:56:00", DateConst.YYYY_MM_DD_HH_MM_SS);
+        Date endDate = DateUtil.parse("2024-09-19 21:00:00", DateConst.YYYY_MM_DD_HH_MM_SS);
         try {
             emailParseService.parseEmail(emailInfoDTO, startDate, endDate);
         } catch (Exception e) {