fix:文件重复解析的判断逻辑优化（一个邮件的所有附件大小汇总）

mo-daq/src/main/java/com/smppw/modaq/domain/dto/EmailZipFileDTO.java

@@ -4,34 +4,37 @@ import cn.hutool.core.io.FileUtil;
 import lombok.Getter;
 import lombok.Setter;
 
+import java.io.File;
+
 @Getter
 public class EmailZipFileDTO {
     private final String emailTitle;
     private final String filename;
     private final String filepath;
-    private final int fileSize;
+    private final long fileSize;
     @Setter
     private Integer emailType;
 
     private final String extName;
 
-    public EmailZipFileDTO(String emailTitle, String filepath, int fileSize, Integer emailType) {
+    public EmailZipFileDTO(String emailTitle, String filepath, Integer emailType) {
+        File file = FileUtil.file(filepath);
         this.emailTitle = emailTitle;
         this.filepath = filepath;
         this.emailType = emailType;
-        this.fileSize = fileSize;
-        this.filename = FileUtil.getName(filepath);
-        this.extName = FileUtil.extName(filepath);
+        this.fileSize = FileUtil.size(file);
+        this.filename = FileUtil.getName(file);
+        this.extName = FileUtil.extName(file);
     }
 
-    public EmailZipFileDTO(String emailTitle, String filepath, String filename, int fileSize, Integer emailType) {
-        this.emailTitle = emailTitle;
-        this.filepath = filepath;
-        this.emailType = emailType;
-        this.filename = filename;
-        this.fileSize = fileSize;
-        this.extName = FileUtil.extName(filepath);
-    }
+//    public EmailZipFileDTO(String emailTitle, String filepath, String filename, int fileSize, Integer emailType) {
+//        this.emailTitle = emailTitle;
+//        this.filepath = filepath;
+//        this.emailType = emailType;
+//        this.filename = filename;
+//        this.fileSize = fileSize;
+//        this.extName = FileUtil.extName(filepath);
+//    }
 
     public EmailZipFileDTO(String emailTitle, EmailContentInfoDTO emailDto) {
         this.emailTitle = emailTitle;

mo-daq/src/main/java/com/smppw/modaq/domain/entity/EmailParseInfoDO.java

@@ -60,7 +60,7 @@ public class EmailParseInfoDO {
     /**
      * 附件文件大小,byte
      */
-    private Integer attrSize;
+    private Long attrSize;
     /**
      * 记录的有效性；1-有效；0-无效；
      */

mo-daq/src/main/java/com/smppw/modaq/domain/mapper/EmailFileInfoMapper.java

@@ -41,13 +41,13 @@ public interface EmailFileInfoMapper {
 
     int getAmacFilenameSuccessCount(@Param("emailTitle") String emailTitle,
                                     @Param("filename") String filename,
-                                    @Param("fileSize") int fileSize);
+                                    @Param("fileSize") long fileSize);
 
     int getWeeklyFilenameSuccessCount(@Param("emailTitle") String emailTitle,
                                       @Param("filename") String filename,
-                                      @Param("fileSize") int fileSize);
+                                      @Param("fileSize") long fileSize);
 
     int getOtherFilenameSuccessCount(@Param("emailTitle") String emailTitle,
                                      @Param("filename") String filename,
-                                     @Param("fileSize") int fileSize);
+                                     @Param("fileSize") long fileSize);
 }

mo-daq/src/main/java/com/smppw/modaq/domain/service/EmailParseService.java

@@ -143,19 +143,19 @@ public class EmailParseService {
             }
             log.info("开始解析邮件数据 -> 邮件主题:{},邮件日期:{}", emailContentInfoDTOList.get(0).getEmailTitle(), emailContentInfoDTOList.get(0).getEmailDate());
             Map<EmailContentInfoDTO, List<EmailZipFileDTO>> emailZipFileMap = MapUtil.newHashMap();
-            for (EmailContentInfoDTO emailContentInfoDTO : emailContentInfoDTOList) {
+            for (EmailContentInfoDTO emailDto : emailContentInfoDTOList) {
                 // 正文不用解压附件
-                if (emailContentInfoDTO.getFileName() != null && emailContentInfoDTO.getFileName().endsWith(".html")) {
-                    emailZipFileMap.put(emailContentInfoDTO, ListUtil.list(false));
+                if (emailDto.getFileName() != null && emailDto.getFileName().endsWith(".html")) {
+                    emailZipFileMap.put(emailDto, ListUtil.list(false));
                     continue;
                 }
                 try {
-                    List<EmailZipFileDTO> tempList = emailZipFileMap.getOrDefault(emailContentInfoDTO, ListUtil.list(false));
-                    tempList.addAll(this.parseZipEmail(emailContentInfoDTO));
-                    emailZipFileMap.put(emailContentInfoDTO, tempList);
+                    List<EmailZipFileDTO> tempList = emailZipFileMap.getOrDefault(emailDto, ListUtil.list(false));
+                    tempList.addAll(this.parseZipEmail(emailDto));
+                    emailZipFileMap.put(emailDto, tempList);
                 } catch (IOException e) {
                     log.error("压缩包解压失败:{}", ExceptionUtil.stacktraceToString(e));
-                    EmailParseInfoDO fail = buildEmailParseInfo(null, mailboxInfoDTO.getAccount(), emailContentInfoDTO);
+                    EmailParseInfoDO fail = buildEmailParseInfo(null, mailboxInfoDTO.getAccount(), emailDto, emailDto.getFileSize());
                     fail.setFailReason("压缩包解压失败");
                     fail.setParseStatus(EmailParseStatusConst.FAIL);
                     fail.setEmailKey(emailEntry.getKey());
@@ -203,12 +203,11 @@ public class EmailParseService {
         Integer emailType = emailContentInfoDTO.getEmailType();
         String filepath = emailContentInfoDTO.getFilePath();
         String emailTitle = emailContentInfoDTO.getEmailTitle();
-        int fileSize = emailContentInfoDTO.getFileSize();
 
         if (ArchiveUtil.isZip(filepath)) {
-            handleCompressedFiles(emailTitle, filepath, ".zip", emailType, fileSize, resultList);
+            handleCompressedFiles(emailTitle, filepath, ".zip", emailType, resultList);
         } else if (ArchiveUtil.isRAR(filepath)) {
-            handleCompressedFiles(emailTitle, filepath, ".rar", emailType, fileSize, resultList);
+            handleCompressedFiles(emailTitle, filepath, ".rar", emailType, resultList);
         } else {
             // 不是压缩包时
             EmailZipFileDTO dto = new EmailZipFileDTO(emailTitle, emailContentInfoDTO);
@@ -234,7 +233,7 @@ public class EmailParseService {
     }
 
     private void handleCompressedFiles(String emailTitle, String filepath, String extension,
-                                       Integer emailType, int fileSize, List<EmailZipFileDTO> resultList) throws IOException {
+                                       Integer emailType, List<EmailZipFileDTO> resultList) throws IOException {
         String destPath = getDestinationPath(filepath, extension);
 
         File destFile = new File(destPath);
@@ -262,13 +261,13 @@ public class EmailParseService {
                 String[] subDirs = file.list();
                 if (subDirs != null) {
                     for (String subDir : subDirs) {
-                        resultList.add(new EmailZipFileDTO(emailTitle, subDir, fileSize, emailType));
+                        resultList.add(new EmailZipFileDTO(emailTitle, subDir, emailType));
                     }
                 } else {
                     log.warn("目录 {} 下无文件", dir);
                 }
             } else {
-                resultList.add(new EmailZipFileDTO(emailTitle, dir, fileSize, emailType));
+                resultList.add(new EmailZipFileDTO(emailTitle, dir, emailType));
             }
         }
     }
@@ -312,35 +311,44 @@ public class EmailParseService {
             if (exts.contains("pdf") && exts.size() > 1) {
                 dtos.removeIf(e -> !Objects.equals("pdf", e.getExtName()));
             }
-
-            // 数据库已存在的数据过滤（邮件主题+报告名称+附件大小）
-            Iterator<EmailZipFileDTO> iterator = dtos.iterator();
-            while (iterator.hasNext()) {
-                EmailZipFileDTO dto = iterator.next();
+            // 移除逻辑
+            Iterator<EmailZipFileDTO> removeIterator = dtos.iterator();
+            while (removeIterator.hasNext()) {
+                EmailZipFileDTO dto = removeIterator.next();
                 String filename = dto.getFilename();
                 // 删除复核函或基金合同
                 if (filename.contains("复核函") || (filename.contains("基金合同") && !filename.contains("合同变更"))) {
                     log.warn("邮件{} 中的报告{} 是复核函或基金合同，不用解析上传。", emailTitle, filename);
-                    iterator.remove();
+                    removeIterator.remove();
                 }
+                // 不支持的类型
+                Integer emailType = dto.getEmailType();
+                if (!EmailTypeConst.SUPPORT_EMAIL_TYPES.contains(emailType)) {
+                    log.info("邮件{} 类型{} 不支持解析。", emailTitle, emailType);
+                    removeIterator.remove();
+                }
+            }
+
+            // 数据库已存在的数据过滤（邮件主题+报告名称+附件大小,压缩包文件大小汇总）
+            long totalSize = dtos.stream().map(EmailZipFileDTO::getFileSize).reduce(0L, Long::sum);
+            Iterator<EmailZipFileDTO> iterator = dtos.iterator();
+            while (iterator.hasNext()) {
+                EmailZipFileDTO dto = iterator.next();
+                String filename = dto.getFilename();
                 Integer emailType = dto.getEmailType();
-                int fileSize = dto.getFileSize();
                 int count = 0;
                 if (Objects.equals(emailType, EmailTypeConst.REPORT_LETTER_EMAIL_TYPE)) {
                     // 确认单
                     count = this.emailFileInfoMapper.getLetterFilenameSuccessCount(emailTitle, filename);
                 } else if (Objects.equals(emailType, EmailTypeConst.REPORT_EMAIL_TYPE)) {
                     // 定期报告
-                    count = this.emailFileInfoMapper.getAmacFilenameSuccessCount(emailTitle, filename, fileSize);
+                    count = this.emailFileInfoMapper.getAmacFilenameSuccessCount(emailTitle, filename, totalSize);
                 } else if (Objects.equals(emailType, EmailTypeConst.REPORT_WEEKLY_TYPE)) {
                     // 管理人周报
-                    count = this.emailFileInfoMapper.getWeeklyFilenameSuccessCount(emailTitle, filename, fileSize);
+                    count = this.emailFileInfoMapper.getWeeklyFilenameSuccessCount(emailTitle, filename, totalSize);
                 } else if (Objects.equals(emailType, EmailTypeConst.REPORT_OTHER_TYPE)) {
                     // 其他报告
-                    count = this.emailFileInfoMapper.getOtherFilenameSuccessCount(emailTitle, filename, fileSize);
-                } else {
-                    log.info("邮件{} 类型{} 不支持解析。", emailTitle, emailType);
-                    iterator.remove();
+                    count = this.emailFileInfoMapper.getOtherFilenameSuccessCount(emailTitle, filename, totalSize);
                 }
                 if (count > 0) {
                     iterator.remove();
@@ -356,7 +364,7 @@ public class EmailParseService {
             }
 
             Integer emailId = emailDto.getEmailId();
-            EmailParseInfoDO emailParseInfoDO = this.buildEmailParseInfo(emailId, emailAddress, emailDto);
+            EmailParseInfoDO emailParseInfoDO = this.buildEmailParseInfo(emailId, emailAddress, emailDto, totalSize);
             emailParseInfoDO.setEmailKey(emailKey);
             emailId = this.saveEmailParseInfo(emailParseInfoDO);
             if (emailId == null) {
@@ -646,7 +654,8 @@ public class EmailParseService {
         return emailParseInfoDO.getId();
     }
 
-    private EmailParseInfoDO buildEmailParseInfo(Integer emailId, String emailAddress, EmailContentInfoDTO emailContentInfoDTO) {
+    private EmailParseInfoDO buildEmailParseInfo(Integer emailId, String emailAddress,
+                                                 EmailContentInfoDTO emailContentInfoDTO, long totalSize) {
         EmailParseInfoDO emailParseInfoDO = new EmailParseInfoDO();
         emailParseInfoDO.setId(emailId);
         emailParseInfoDO.setSenderEmail(emailContentInfoDTO.getSenderEmail());
@@ -656,7 +665,7 @@ public class EmailParseService {
         emailParseInfoDO.setEmailTitle(emailContentInfoDTO.getEmailTitle());
         emailParseInfoDO.setEmailType(emailContentInfoDTO.getEmailType());
         emailParseInfoDO.setParseStatus(EmailParseStatusConst.SUCCESS);
-        emailParseInfoDO.setAttrSize(emailContentInfoDTO.getFileSize());
+        emailParseInfoDO.setAttrSize(totalSize);
         emailParseInfoDO.setIsvalid(1);
         emailParseInfoDO.setCreatorId(0);
         emailParseInfoDO.setCreateTime(new Date());

mo-daq/src/main/resources/application.yml

@@ -17,7 +17,7 @@ server:
 # 日志配置
 logging:
   level:
-    com.simuwang.base.mapper: debug
+    com.smppw.modaq.domain.mapper: debug
   config: classpath:logback.xml
 
 spring: