feat:邮件解析-支持zip估值表解析

service-daq/src/main/java/com/simuwang/daq/service/ValuationEmailParser.java

@@ -15,10 +15,7 @@ import com.simuwang.base.pojo.valuation.ValuationNeedParseParam;
 import org.springframework.stereotype.Component;
 
 import java.io.File;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
+import java.util.*;
 import java.util.stream.Collectors;
 
 /**
@@ -45,33 +42,76 @@ public class ValuationEmailParser extends AbstractEmailParser {
     @Override
     public List<EmailFundNavDTO> parse(EmailContentInfoDTO emailContentInfoDTO, Map<String, List<String>> emailFieldMap) {
         List<EmailFundNavDTO> emailFundNavDTOList = CollUtil.newArrayList();
-        boolean isSatisfiedParse  = emailContentInfoDTO != null && StrUtil.isNotBlank(emailContentInfoDTO.getFilePath())
-                && (ExcelUtil.isExcel(emailContentInfoDTO.getFileName()) || ExcelUtil.isPdf(emailContentInfoDTO.getFileName()));
+        boolean isSatisfiedParse = emailContentInfoDTO != null && StrUtil.isNotBlank(emailContentInfoDTO.getFileName())
+                && (ExcelUtil.isExcel(emailContentInfoDTO.getFileName()) || ExcelUtil.isPdf(emailContentInfoDTO.getFileName()) || ExcelUtil.isZip(emailContentInfoDTO.getFileName()));
         if (!isSatisfiedParse) {
             return emailFundNavDTOList;
         }
         List<ValuationNeedParseParam> valuationNeedParseParams = buildValuationNeedParseParam(emailContentInfoDTO);
         List<AssetsValuationResult.Record> recordList = valuationParseService.parseValuationExcel(valuationNeedParseParams);
         if (CollUtil.isNotEmpty(recordList)) {
-            List<AssetsValuationResult.Record> parseSucessList = recordList.stream()
+            List<AssetsValuationResult.Record> parseSuccessList = recordList.stream()
                     .filter(e -> e.getSuccess() == 1 || (StrUtil.isNotBlank(e.getMsg()) && "未匹配基金".equals(e.getMsg()))).collect(Collectors.toList());
-            EmailFundNavDTO fundNavDTO = convertToFundNavDTO(parseSucessList);
-            Optional.ofNullable(fundNavDTO).ifPresent(emailFundNavDTOList::add);
+            List<EmailFundNavDTO> fundNavDTOList = convertToFundNavDTO(parseSuccessList);
+            Optional.ofNullable(fundNavDTOList).ifPresent(emailFundNavDTOList::addAll);
         }
         return emailFundNavDTOList;
     }
 
     private List<ValuationNeedParseParam> buildValuationNeedParseParam(EmailContentInfoDTO emailContentInfoDTO) {
-        // pdf格式文件转成excel
-        File file = new File(emailContentInfoDTO.getFilePath());
-        ValuationNeedParseParam parseParam = new ValuationNeedParseParam();
-        parseParam.setFile(file);
-        parseParam.setFileUrl(emailContentInfoDTO.getFilePath());
-        parseParam.setOriginFileName(emailContentInfoDTO.getFileName());
-        parseParam.setFundId(null);
-        parseParam.setFromEmail(1);
-        transformPdfToExcel(parseParam);
-        return ListUtil.toList(parseParam);
+        String fileName = emailContentInfoDTO.getFileName();
+        String filePath = emailContentInfoDTO.getFilePath();
+        if (StrUtil.isNotBlank(fileName) && ExcelUtil.isZip(fileName)) {
+            List<String> filePathList = compressedFile(filePath, fileName);
+            if (CollUtil.isEmpty(filePathList)) {
+                return CollUtil.newArrayList();
+            }
+            List<ValuationNeedParseParam> parseParamList = CollUtil.newArrayList();
+            for (String path : filePathList) {
+                ValuationNeedParseParam parseParam = new ValuationNeedParseParam();
+                parseParam.setFile(new File(path));
+                parseParam.setFileUrl(path);
+                parseParam.setOriginFileName(fileName);
+                parseParam.setFundId(null);
+                parseParam.setFromEmail(1);
+                // pdf格式文件转成excel
+                transformPdfToExcel(parseParam);
+                parseParamList.add(parseParam);
+            }
+            return parseParamList;
+        } else {
+            File file = new File(filePath);
+            ValuationNeedParseParam parseParam = new ValuationNeedParseParam();
+            parseParam.setFile(file);
+            parseParam.setFileUrl(filePath);
+            parseParam.setOriginFileName(fileName);
+            parseParam.setFundId(null);
+            parseParam.setFromEmail(1);
+            // pdf格式文件转成excel
+            transformPdfToExcel(parseParam);
+            return ListUtil.toList(parseParam);
+        }
+    }
+
+    private List<String> compressedFile(String filePath, String fileName) {
+        List<String> filePathList = CollUtil.newArrayList();
+        String destPath = filePath.substring(0, filePath.indexOf(fileName)) + fileName.replaceAll(".zip", "").replaceAll(".ZIP", "");
+        List<String> compressedFiles = ExcelUtil.extractCompressedFiles(filePath, destPath);
+        for (String path : compressedFiles) {
+            boolean isSatisfiedParse = ExcelUtil.isExcel(path) || ExcelUtil.isPdf(path) || ExcelUtil.isZip(path);
+            if (!isSatisfiedParse) {
+                continue;
+            }
+            if (ExcelUtil.isZip(path)) {
+                filePathList.addAll(compressedFile(path, new File(path).getName()));
+            }
+            File file = new File(path);
+            if (file.isDirectory()) {
+                filePathList.addAll(Arrays.stream(Objects.requireNonNull(file.list())).toList());
+            }
+            filePathList.add(path);
+        }
+        return filePathList;
     }
 
     private void transformPdfToExcel(ValuationNeedParseParam valuationNeedParseParam) {
@@ -91,31 +131,36 @@ public class ValuationEmailParser extends AbstractEmailParser {
         valuationNeedParseParam.setFile(toExcelDTO != null ? toExcelDTO.getExcelFile() : null);
     }
 
-    private EmailFundNavDTO convertToFundNavDTO(List<AssetsValuationResult.Record> parseSucessList) {
-        if (CollUtil.isEmpty(parseSucessList)) {
+    private List<EmailFundNavDTO> convertToFundNavDTO(List<AssetsValuationResult.Record> parseSuccessList) {
+        if (CollUtil.isEmpty(parseSuccessList)) {
             return null;
         }
-        String fundName = parseSucessList.stream().map(AssetsValuationResult.Record::getParseValuationInfo)
-                .filter(Objects::nonNull).map(ParseValuationInfo::getFundName).distinct().findFirst().orElse(null);
-        String registerNumber = parseSucessList.stream().map(AssetsValuationResult.Record::getParseValuationInfo)
-                .filter(Objects::nonNull).map(ParseValuationInfo::getRegisterNumber).distinct().findFirst().orElse(null);
-        String valuationDate = parseSucessList.stream().map(AssetsValuationResult.Record::getDate).filter(Objects::nonNull).distinct().findFirst().orElse(null);
-        String nav = parseSucessList.stream().map(AssetsValuationResult.Record::getNav).filter(Objects::nonNull).distinct().findFirst().orElse(null);
-        String cumulativeNavWithdrawal = parseSucessList.stream().map(AssetsValuationResult.Record::getCumulativeNavWithdrawal).filter(Objects::nonNull).distinct().findFirst().orElse(null);
-        String assetNet = parseSucessList.stream().map(AssetsValuationResult.Record::getAssetNet).filter(Objects::nonNull).distinct().findFirst().orElse(null);
-        String assetShare = parseSucessList.stream().map(AssetsValuationResult.Record::getAssetShare).filter(Objects::nonNull).distinct().findFirst().orElse(null);
-        List<String> fundIdList = parseSucessList.stream().map(AssetsValuationResult.Record::getFundId).filter(Objects::nonNull).distinct().collect(Collectors.toList());
+        List<EmailFundNavDTO> emailFundNavDTOList = CollUtil.newArrayList();
+        Map<String, List<AssetsValuationResult.Record>> dateSuccessRecordMap = parseSuccessList.stream().collect(Collectors.groupingBy(AssetsValuationResult.Record::getDate));
+        for (Map.Entry<String, List<AssetsValuationResult.Record>> successRecordEntry : dateSuccessRecordMap.entrySet()) {
+            String date = successRecordEntry.getKey();
+            List<AssetsValuationResult.Record> recordList = successRecordEntry.getValue();
+            String fundName = recordList.stream().map(AssetsValuationResult.Record::getParseValuationInfo)
+                    .filter(Objects::nonNull).map(ParseValuationInfo::getFundName).distinct().findFirst().orElse(null);
+            String registerNumber = recordList.stream().map(AssetsValuationResult.Record::getParseValuationInfo)
+                    .filter(Objects::nonNull).map(ParseValuationInfo::getRegisterNumber).distinct().findFirst().orElse(null);
+            String nav = recordList.stream().map(AssetsValuationResult.Record::getNav).filter(Objects::nonNull).distinct().findFirst().orElse(null);
+            String cumulativeNavWithdrawal = recordList.stream().map(AssetsValuationResult.Record::getCumulativeNavWithdrawal).filter(Objects::nonNull).distinct().findFirst().orElse(null);
+            String assetNet = recordList.stream().map(AssetsValuationResult.Record::getAssetNet).filter(Objects::nonNull).distinct().findFirst().orElse(null);
+            String assetShare = recordList.stream().map(AssetsValuationResult.Record::getAssetShare).filter(Objects::nonNull).distinct().findFirst().orElse(null);
+            List<String> fundIdList = recordList.stream().map(AssetsValuationResult.Record::getFundId).filter(Objects::nonNull).distinct().collect(Collectors.toList());
 
-        EmailFundNavDTO fundNavDTO = new EmailFundNavDTO();
-        fundNavDTO.setFundName(fundName);
-        fundNavDTO.setRegisterNumber(registerNumber);
-        fundNavDTO.setPriceDate(valuationDate);
-        fundNavDTO.setNav(nav);
-        fundNavDTO.setCumulativeNavWithdrawal(cumulativeNavWithdrawal);
-        fundNavDTO.setAssetNet(assetNet);
-        fundNavDTO.setAssetShare(assetShare);
-        fundNavDTO.setFundIdList(fundIdList);
-        return fundNavDTO;
+            EmailFundNavDTO fundNavDTO = new EmailFundNavDTO();
+            fundNavDTO.setFundName(fundName);
+            fundNavDTO.setRegisterNumber(registerNumber);
+            fundNavDTO.setPriceDate(date);
+            fundNavDTO.setNav(nav);
+            fundNavDTO.setCumulativeNavWithdrawal(cumulativeNavWithdrawal);
+            fundNavDTO.setAssetNet(assetNet);
+            fundNavDTO.setAssetShare(assetShare);
+            fundNavDTO.setFundIdList(fundIdList);
+            emailFundNavDTOList.add(fundNavDTO);
+        }
+        return emailFundNavDTOList;
     }
-
 }

service-deploy/src/main/test/java/com/simuwang/datadaq/DataTrusteeApplicationTests.java

@@ -37,8 +37,8 @@ class DataTrusteeApplicationTests {
         emailInfoDTO.setProtocol("imap");
         Map<Integer, List<String>> emailTypeMap = MapUtil.newHashMap();
         emailTypeMap.put(1, List.of("净值"));
-        Date startDate = DateUtil.parse("2024-09-14 10:10:00", DateConst.YYYY_MM_DD_HH_MM_SS);
-        Date endDate = DateUtil.parse("2024-09-14 12:00:00", DateConst.YYYY_MM_DD_HH_MM_SS);
+        Date startDate = DateUtil.parse("2024-09-19 09:20:00", DateConst.YYYY_MM_DD_HH_MM_SS);
+        Date endDate = DateUtil.parse("2024-09-19 16:00:00", DateConst.YYYY_MM_DD_HH_MM_SS);
         try {
             emailParseService.parseEmail(emailInfoDTO, startDate, endDate);
         } catch (Exception e) {