pref:邮件解析-完善优化重新解析功能

service-base/src/main/java/com/simuwang/base/mapper/EmailFileInfoMapper.java

@@ -1,7 +1,6 @@
 package com.simuwang.base.mapper;
 
 import com.simuwang.base.pojo.dos.EmailFileInfoDO;
-import com.simuwang.base.pojo.dos.EmailParseInfoDO;
 import com.simuwang.base.pojo.dto.query.EmailFileQuery;
 import com.simuwang.base.pojo.dto.query.FundFilePageQuery;
 import com.simuwang.base.pojo.vo.EmailParseDetailVO;
@@ -9,6 +8,7 @@ import com.simuwang.base.pojo.vo.FundFileInfoVO;
 import org.apache.ibatis.annotations.Mapper;
 import org.apache.ibatis.annotations.Param;
 
+import java.util.Date;
 import java.util.List;
 
 @Mapper
@@ -27,4 +27,6 @@ public interface EmailFileInfoMapper {
     long countFundFileInfo(FundFilePageQuery fundFilePageQuery);
 
     long countEmailDetailById(EmailFileQuery emailFileQuery);
+
+    void updateTimeById(@Param("id") Integer fileId, @Param("parseDate") Date parseDate);
 }

service-base/src/main/java/com/simuwang/base/mapper/EmailFundAssetMapper.java

@@ -15,4 +15,6 @@ public interface EmailFundAssetMapper {
     List<EmailFundAssetDO> searchAssetDetail(ParseDetailPageQuery parseDetailPageQuery);
 
     long countAssetDetail(ParseDetailPageQuery parseDetailPageQuery);
+
+    void deleteByFileId(@Param("fileId") Integer fileId);
 }

service-base/src/main/java/com/simuwang/base/mapper/EmailFundNavMapper.java

@@ -2,7 +2,6 @@ package com.simuwang.base.mapper;
 
 import com.simuwang.base.pojo.dos.EmailFundNavDO;
 import com.simuwang.base.pojo.dto.query.ParseDetailPageQuery;
-import com.simuwang.base.pojo.vo.EmailFundNavVO;
 import org.apache.ibatis.annotations.Mapper;
 import org.apache.ibatis.annotations.Param;
 
@@ -16,4 +15,6 @@ public interface EmailFundNavMapper {
     List<EmailFundNavDO> searchNavDetail(ParseDetailPageQuery parseDetailPageQuery);
 
     long countNavDetail(ParseDetailPageQuery parseDetailPageQuery);
+
+    void deleteByFileId(@Param("fileId") Integer fileId);
 }

service-base/src/main/java/com/simuwang/base/mapper/EmailParseInfoMapper.java

@@ -6,6 +6,7 @@ import com.simuwang.base.pojo.vo.EmailParseInfoVO;
 import org.apache.ibatis.annotations.Mapper;
 import org.apache.ibatis.annotations.Param;
 
+import java.util.Date;
 import java.util.List;
 
 @Mapper
@@ -24,4 +25,6 @@ public interface EmailParseInfoMapper {
     EmailParseInfoDO queryById(@Param("id") Integer id);
 
     long countEmailList(EmailParseQuery emailParseQuery);
+
+    void updateParseTime(@Param("id") Integer id, @Param("parseDate") Date parseDate);
 }

service-base/src/main/java/com/simuwang/base/pojo/dto/EmailContentInfoDTO.java

@@ -11,6 +11,10 @@ public class EmailContentInfoDTO implements Serializable, Cloneable {
     private static final long serialVersionUID = 202104140906313753L;
 
     /**
+     * 邮件id(重新解析邮件功能)
+     */
+    private Integer emailId;
+    /**
      * 邮箱地址
      */
     private String emailAddress;
@@ -54,6 +58,9 @@ public class EmailContentInfoDTO implements Serializable, Cloneable {
      * 收件人
      */
     private String toEmail;
-
+    /**
+     * 文件id(重新解析邮件功能)
+     */
+    private Integer fielId;
 }
 

service-base/src/main/resources/mapper/EmailFileInfoMapper.xml

@@ -29,9 +29,9 @@
     </select>
 
     <select id="queryByEmailId" resultMap="BaseResultMap">
-        select id, file_name, file_path,
+        select id, file_name, file_path
         from PPW_EMAIL.email_file_info
-        where email - id = #{emailId}
+        where email_id = #{emailId}
           and isvalid = 1
     </select>
     <select id="searchFundFileInfo" resultType="com.simuwang.base.pojo.vo.FundFileInfoVO">
@@ -124,4 +124,11 @@
           AND efi.id =#{fileId}) a
     </select>
 
+    <update id="updateTimeById">
+        update PPW_EMAIL.EMAIL_FILE_INFO
+        set updatetime = #{parseDate}
+        where isvalid = 1
+          and id = #{id}
+    </update>
+
 </mapper>

service-base/src/main/resources/mapper/EmailFundAssetMapper.xml

@@ -87,5 +87,12 @@
         )a
     </select>
 
+    <delete id="deleteByFileId">
+        update PPW_EMAIL.email_fund_asset
+        set isvalid   = 0,
+            updatetime= now()
+        where isvalid = 1
+          and file_id = #{fileId}
+    </delete>
 
 </mapper>

service-base/src/main/resources/mapper/EmailFundNavMapper.xml

@@ -108,5 +108,12 @@
         ) a
     </select>
 
+    <delete id="deleteByFileId">
+        update PPW_EMAIL.email_fund_nav
+        set isvalid   = 0,
+            updatetime= now()
+        where isvalid = 1
+          and file_id = #{fileId}
+    </delete>
 
 </mapper>

service-base/src/main/resources/mapper/EmailParseInfoMapper.xml

@@ -116,4 +116,13 @@
         <include refid="sqlwhereSearch"/>
     </select>
 
+    <update id="updateParseTime">
+        update PPW_EMAIL.email_parse_info
+        set updatetime = #{parseDate},
+            parse_date = #{parseDate}
+        where isvalid = 1
+          and id = #{id}
+    </update>
+
+
 </mapper>

service-daq/src/main/java/com/simuwang/daq/service/EmailParseApiServiceImpl.java

@@ -4,8 +4,10 @@ import cn.hutool.core.collection.CollUtil;
 import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.exceptions.ExceptionUtil;
 import cn.hutool.core.map.MapUtil;
+import cn.hutool.core.util.StrUtil;
 import com.simuwang.base.common.conts.DateConst;
 import com.simuwang.base.common.util.EmailUtil;
+import com.simuwang.base.common.util.ExcelUtil;
 import com.simuwang.base.mapper.EmailFileInfoMapper;
 import com.simuwang.base.mapper.EmailParseInfoMapper;
 import com.simuwang.base.pojo.dos.EmailFileInfoDO;
@@ -59,7 +61,7 @@ public class EmailParseApiServiceImpl implements EmailParseApiService {
             return;
         }
         List<EmailFileInfoDO> emailFileInfoDOList = emailFileInfoMapper.queryByEmailId(emailId);
-        if (CollUtil.isNotEmpty(emailFileInfoDOList)) {
+        if (CollUtil.isEmpty(emailFileInfoDOList)) {
             log.info("该邮件不存在附件 -> 邮件id:{}", emailId);
             return;
         }
@@ -69,7 +71,7 @@ public class EmailParseApiServiceImpl implements EmailParseApiService {
         Map<Integer, List<String>> emailTypeMap = emailParseService.getEmailType();
 
         // 解析流程
-        List<EmailContentInfoDTO> emailContentInfoDTOList = buildEmailContentInfoDTO(emailParseInfoDO, emailFileInfoDOList, emailTypeMap);
+        List<EmailContentInfoDTO> emailContentInfoDTOList = buildEmailContentInfoDTO(emailId, emailParseInfoDO, emailFileInfoDOList, emailTypeMap);
 
         List<EmailFundNavDTO> emailFundNavDTOList = CollUtil.newArrayList();
         Map<EmailContentInfoDTO, List<EmailFundNavDTO>> fileNameNavMap = MapUtil.newHashMap();
@@ -86,12 +88,14 @@ public class EmailParseApiServiceImpl implements EmailParseApiService {
         emailParseService.saveRelatedTable(emailParseInfoDO.getEmail(), emailContentInfoDTOList, fileNameNavMap);
     }
 
-    private List<EmailContentInfoDTO> buildEmailContentInfoDTO(EmailParseInfoDO emailParseInfoDO, List<EmailFileInfoDO> emailParseInfoDOList, Map<Integer, List<String>> emailTypeMap) {
+    private List<EmailContentInfoDTO> buildEmailContentInfoDTO(Integer emailId, EmailParseInfoDO emailParseInfoDO, List<EmailFileInfoDO> emailParseInfoDOList, Map<Integer, List<String>> emailTypeMap) {
         List<EmailContentInfoDTO> emailContentInfoDTOList = CollUtil.newArrayList();
         String emailDate = DateUtil.format(emailParseInfoDO.getEmailDate(), DateConst.YYYY_MM_DD_HH_MM_SS);
         String parseDate = DateUtil.format(new Date(), DateConst.YYYY_MM_DD_HH_MM_SS);
         for (EmailFileInfoDO fileInfoDO : emailParseInfoDOList) {
             EmailContentInfoDTO contentInfoDTO = new EmailContentInfoDTO();
+            contentInfoDTO.setEmailId(emailId);
+            contentInfoDTO.setFielId(fileInfoDO.getId());
             contentInfoDTO.setEmailAddress(emailParseInfoDO.getEmail());
             contentInfoDTO.setEmailDate(emailDate);
             contentInfoDTO.setEmailTitle(emailParseInfoDO.getEmailTitle());
@@ -108,18 +112,21 @@ public class EmailParseApiServiceImpl implements EmailParseApiService {
     }
 
     public static String getEmailContent(String filePath) {
-        StringBuilder content = new StringBuilder();
-        try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) {
-            String line;
-            while ((line = reader.readLine()) != null) {
-                // 追加每一行到StringBuilder中
-                content.append(line).append("\n");
+        if (StrUtil.isNotBlank(filePath) && ExcelUtil.isPdf(filePath)) {
+            StringBuilder content = new StringBuilder();
+            try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) {
+                String line;
+                while ((line = reader.readLine()) != null) {
+                    // 追加每一行到StringBuilder中
+                    content.append(line).append("\n");
+                }
+            } catch (IOException e) {
+                System.err.println("Error reading the file: " + e.getMessage());
+                return null;
             }
-        } catch (IOException e) {
-            System.err.println("Error reading the file: " + e.getMessage());
-            return null;
+            return content.toString();
         }
-        return content.toString();
+        return null;
     }
 
 }

service-daq/src/main/java/com/simuwang/daq/service/EmailParseService.java

@@ -130,16 +130,16 @@ public class EmailParseService {
         String emailTitle = CollUtil.isNotEmpty(emailContentInfoDTOList) ? emailContentInfoDTOList.get(0).getEmailTitle() : null;
         String emailDate = CollUtil.isNotEmpty(emailContentInfoDTOList) ? emailContentInfoDTOList.get(0).getEmailDate() : null;
         Integer emailType = CollUtil.isNotEmpty(emailContentInfoDTOList) ? emailContentInfoDTOList.get(0).getEmailType() : null;
-
+        Integer emailId = CollUtil.isNotEmpty(emailContentInfoDTOList) ? emailContentInfoDTOList.get(0).getEmailId() : null;
         Date parseDate = new Date();
         int emailParseStatus = EmailParseStatusConst.SUCCESS;
-        EmailParseInfoDO emailParseInfoDO = buildEmailParseInfo(emailAddress, emailDate, emailTitle, emailType, emailParseStatus, parseDate);
-        Integer emailId = saveEmailParseInfo(emailParseInfoDO);
+        EmailParseInfoDO emailParseInfoDO = buildEmailParseInfo(emailId, emailAddress, emailDate, emailTitle, emailType, emailParseStatus, parseDate);
+        emailId = saveEmailParseInfo(emailParseInfoDO);
 
         for (Map.Entry<EmailContentInfoDTO, List<EmailFundNavDTO>> fileNameNavEntry : fileNameNavMap.entrySet()) {
             // 保存邮件文件表
             EmailContentInfoDTO emailContentInfoDTO = fileNameNavEntry.getKey();
-            Integer fileId = saveEmailFileInfo(emailId, emailContentInfoDTO.getFileName(), emailContentInfoDTO.getFilePath(), parseDate);
+            Integer fileId = saveEmailFileInfo(emailId, emailContentInfoDTO.getFielId(), emailContentInfoDTO.getFileName(), emailContentInfoDTO.getFilePath(), parseDate);
 
             List<EmailFundNavDTO> fundNavDTOList = fileNameNavEntry.getValue();
             if (CollUtil.isEmpty(fundNavDTOList)) {
@@ -167,6 +167,8 @@ public class EmailParseService {
         List<EmailFundNavDO> emailFundNavDOList = fundNavDTOList.stream()
                 .map(e -> buildEmailFundNavDo(fileId, e, parseDate)).filter(CollUtil::isNotEmpty).flatMap(List::stream).collect(Collectors.toList());
         if (CollUtil.isNotEmpty(emailFundNavDOList)) {
+            // 先删除文件id下的净值数据(考虑到重新解析的需求,如果是首次解析,那么fiel_id下不存在净值数据)
+            emailFundNavMapper.deleteByFileId(fileId);
             emailFundNavMapper.batchInsert(emailFundNavDOList);
             List<NavDO> navDOList = emailFundNavDOList.stream().filter(e -> StrUtil.isNotBlank(e.getFundId()))
                     .map(e -> BeanUtil.copyProperties(e, NavDO.class)).collect(Collectors.toList());
@@ -176,6 +178,8 @@ public class EmailParseService {
         List<EmailFundAssetDO> emailFundAssetDOList = fundNavDTOList.stream()
                 .map(e -> buildEmailFundAssetDo(fileId, e, parseDate)).filter(CollUtil::isNotEmpty).flatMap(List::stream).collect(Collectors.toList());
         if (CollUtil.isNotEmpty(emailFundAssetDOList)) {
+            // 先删除file_id下的规模数据(考虑到重新解析的需求,如果是首次解析,那么fiel_id下不存在规模数据)
+            emailFundAssetMapper.deleteByFileId(fileId);
             emailFundAssetMapper.batchInsert(emailFundAssetDOList);
             List<AssetDO> assetDOList = emailFundAssetDOList.stream().filter(e -> StrUtil.isNotBlank(e.getFundId()))
                     .map(e -> BeanUtil.copyProperties(e, AssetDO.class)).collect(Collectors.toList());
@@ -321,14 +325,19 @@ public class EmailParseService {
         return fundNavDOList;
     }
 
-    private Integer saveEmailFileInfo(Integer emailId, String fileName, String filePath, Date parseDate) {
-        EmailFileInfoDO emailFileInfoDO = buildEmailFileInfoDO(emailId, fileName, filePath, parseDate);
+    private Integer saveEmailFileInfo(Integer emailId, Integer fileId, String fileName, String filePath, Date parseDate) {
+        EmailFileInfoDO emailFileInfoDO = buildEmailFileInfoDO(emailId, fileId, fileName, filePath, parseDate);
+        if (emailFileInfoDO.getId() != null) {
+            emailFileInfoMapper.updateTimeById(fileId, parseDate);
+            return emailFileInfoDO.getId();
+        }
         emailFileInfoMapper.insert(emailFileInfoDO);
         return emailFileInfoDO.getId();
     }
 
-    private EmailFileInfoDO buildEmailFileInfoDO(Integer emailId, String fileName, String filePath, Date parseDate) {
+    private EmailFileInfoDO buildEmailFileInfoDO(Integer emailId, Integer fileId, String fileName, String filePath, Date parseDate) {
         EmailFileInfoDO emailFileInfoDO = new EmailFileInfoDO();
+        emailFileInfoDO.setId(fileId);
         emailFileInfoDO.setEmailId(emailId);
         emailFileInfoDO.setFileName(fileName);
         emailFileInfoDO.setFilePath(filePath);
@@ -376,12 +385,19 @@ public class EmailParseService {
         if (emailParseInfoDO == null) {
             return null;
         }
+        // 重新邮件功能 -> 修改解析时间和更新时间
+        if (emailParseInfoDO.getId() != null) {
+            emailParseInfoMapper.updateParseTime(emailParseInfoDO.getId(), emailParseInfoDO.getParseDate());
+            return emailParseInfoDO.getId();
+        }
         emailParseInfoMapper.insert(emailParseInfoDO);
         return emailParseInfoDO.getId();
     }
 
-    private EmailParseInfoDO buildEmailParseInfo(String emailAddress, String emailDate, String emailTitle, Integer emailType, Integer parseStatus, Date parseDate) {
+    private EmailParseInfoDO buildEmailParseInfo(Integer emailId, String emailAddress, String emailDate,
+                                                 String emailTitle, Integer emailType, Integer parseStatus, Date parseDate) {
         EmailParseInfoDO emailParseInfoDO = new EmailParseInfoDO();
+        emailParseInfoDO.setId(emailId);
         emailParseInfoDO.setEmail(emailAddress);
         emailParseInfoDO.setEmailDate(DateUtil.parse(emailDate, DateConst.YYYY_MM_DD_HH_MM_SS));
         emailParseInfoDO.setParseDate(parseDate);

service-deploy/src/main/test/java/com/simuwang/datadaq/DataTrusteeApplicationTests.java

@@ -4,6 +4,7 @@ import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.map.MapUtil;
 import com.simuwang.base.common.conts.DateConst;
 import com.simuwang.base.pojo.dto.MailboxInfoDTO;
+import com.simuwang.daq.service.EmailParseApiService;
 import com.simuwang.daq.service.EmailParseService;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -19,6 +20,9 @@ class DataTrusteeApplicationTests {
     @Autowired
     private EmailParseService emailParseService;
 
+    @Autowired
+    private EmailParseApiService emailParseApiService;
+
     @Test
     public void test() {
         MailboxInfoDTO emailInfoDTO = new MailboxInfoDTO();
@@ -32,12 +36,17 @@ class DataTrusteeApplicationTests {
         emailInfoDTO.setProtocol("imap");
         Map<Integer, List<String>> emailTypeMap = MapUtil.newHashMap();
         emailTypeMap.put(1, List.of("净值"));
-        Date startDate = DateUtil.parse("2024-09-12 11:10:00", DateConst.YYYY_MM_DD_HH_MM_SS);
-        Date endDate = DateUtil.parse("2024-09-12 12:00:00", DateConst.YYYY_MM_DD_HH_MM_SS);
+        Date startDate = DateUtil.parse("2024-09-14 09:10:00", DateConst.YYYY_MM_DD_HH_MM_SS);
+        Date endDate = DateUtil.parse("2024-09-14 10:00:00", DateConst.YYYY_MM_DD_HH_MM_SS);
         try {
             emailParseService.parseEmail(emailInfoDTO, startDate, endDate);
         } catch (Exception e) {
             throw new RuntimeException(e);
         }
     }
+
+    @Test
+    public void testReparseEmail() {
+        emailParseApiService.reparseEmail(6);
+    }
 }