feat:邮件解析-保存净值数据和规模数据

service-base/src/main/java/com/simuwang/base/mapper/AssetMapper.java

@@ -0,0 +1,18 @@
+package com.simuwang.base.mapper;
+
+import com.simuwang.base.pojo.dos.AssetDO;
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
+
+@Mapper
+public interface AssetMapper {
+
+    void batchInsert(@Param("itemDoList") List<AssetDO> assetDOList);
+
+    void batchUpdate(@Param("itemDoList") List<AssetDO> assetDOList);
+
+    List<String> queryFundNavByDate(@Param("fundId") String fundId, @Param("priceDateList") List<String> priceDateList);
+
+}

service-base/src/main/java/com/simuwang/base/mapper/NavMapper.java

@@ -0,0 +1,18 @@
+package com.simuwang.base.mapper;
+
+import com.simuwang.base.pojo.dos.NavDO;
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
+
+@Mapper
+public interface NavMapper {
+
+    void batchInsert(@Param("itemDoList") List<NavDO> navDOList);
+
+    void batchUpdate(@Param("itemDoList") List<NavDO> navDOList);
+
+    List<String> queryFundNavByDate(@Param("fundId") String fundId, @Param("priceDateList") List<String> priceDateList);
+
+}

service-base/src/main/java/com/simuwang/base/pojo/dos/AssetDO.java

@@ -0,0 +1,64 @@
+package com.simuwang.base.pojo.dos;
+
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+import java.math.BigDecimal;
+import java.util.Date;
+
+@Data
+@TableName("asset")
+public class AssetDO {
+    /**
+     * 主键Id
+     */
+    @TableId(value = "id")
+    private Integer id;
+    /**
+     * 基金id
+     */
+    @TableField(value = "fund_id")
+    private String fundId;
+    /**
+     * 规模日期
+     */
+    @TableField(value = "price_date")
+    private Date priceDate;
+    /**
+     * 资产份额
+     */
+    @TableField(value = "asset_share")
+    private BigDecimal assetShare;
+    /**
+     * 资产净值(基金规模)
+     */
+    @TableField(value = "asset_net")
+    private BigDecimal assetNet;
+    /**
+     * 是否有效：0-无效，1-有效
+     */
+    @TableField(value = "isvalid")
+    private Integer isvalid;
+    /**
+     * 创建者Id
+     */
+    @TableField(value = "creatorid")
+    private Integer creatorId;
+    /**
+     * 创建时间
+     */
+    @TableField(value = "createtime")
+    private Date createTime;
+    /**
+     * 修改者Id
+     */
+    @TableField(value = "updaterid")
+    private Integer updaterId;
+    /**
+     * 更新时间
+     */
+    @TableField(value = "updatetime")
+    private Date updateTime;
+}

service-base/src/main/java/com/simuwang/base/pojo/dos/NavDO.java

@@ -0,0 +1,69 @@
+package com.simuwang.base.pojo.dos;
+
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+import java.math.BigDecimal;
+import java.util.Date;
+
+@Data
+@TableName("nav")
+public class NavDO {
+    /**
+     * 主键Id
+     */
+    @TableId(value = "id")
+    private Integer id;
+    /**
+     * 基金id
+     */
+    @TableField(value = "fund_id")
+    private String fundId;
+    /**
+     * 规模日期
+     */
+    @TableField(value = "price_date")
+    private Date priceDate;
+    /**
+     * 单位净值
+     */
+    @TableField(value = "nav")
+    private BigDecimal nav;
+    /**
+     * 复权净值
+     */
+    @TableField(value = "cumulative_nav")
+    private BigDecimal cumulativeNav;
+    /**
+     * 累计净值
+     */
+    @TableField(value = "cumulative_nav_withdrawal")
+    private BigDecimal cumulativeNavWithdrawal;
+    /**
+     * 是否有效：0-无效，1-有效
+     */
+    @TableField(value = "isvalid")
+    private Integer isvalid;
+    /**
+     * 创建者Id
+     */
+    @TableField(value = "creatorid")
+    private Integer creatorId;
+    /**
+     * 创建时间
+     */
+    @TableField(value = "createtime")
+    private Date createTime;
+    /**
+     * 修改者Id
+     */
+    @TableField(value = "updaterid")
+    private Integer updaterId;
+    /**
+     * 更新时间
+     */
+    @TableField(value = "updatetime")
+    private Date updateTime;
+}

service-base/src/main/resources/mapper/AssetMapper.xml

@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.simuwang.base.mapper.AssetMapper">
+    <resultMap id="BaseResultMap" type="com.simuwang.base.pojo.dos.AssetDO">
+        <id column="id" property="id"/>
+        <result column="fund_id" property="fundId"/>
+        <result column="price_date" property="priceDate"/>
+        <result column="asset_net" property="assetNet"/>
+        <result column="asset_share" property="assetShare"/>
+        <result column="isvalid" property="isvalid"/>
+        <result column="creatorid" property="creatorId"/>
+        <result column="createtime" property="createTime"/>
+        <result column="updaterid" property="updaterId"/>
+        <result column="updatetime" property="updateTime"/>
+    </resultMap>
+
+    <insert id="batchInsert" parameterType="com.simuwang.base.pojo.dos.AssetDO">
+        insert into PPW_EMAIL.asset(fund_id,price_date,asset_net,asset_share,
+                                              isvalid, creatorid, createtime, updaterid, updatetime)
+        values
+        <foreach collection="itemDoList" item="itemDo" index="index" separator=",">
+            (#{itemDo.fundId},#{itemDo.priceDate},#{itemDo.assetNet},#{itemDo.assetShare},
+            #{itemDo.isvalid}, #{itemDo.creatorId}, #{itemDo.createTime}, #{itemDo.updaterId}, #{itemDo.updateTime})
+        </foreach>
+    </insert>
+
+    <insert id="batchUpdate">
+        <foreach collection="itemDoList" item="itemDo" index="index" open="" close="" separator=";">
+            update PPW_EMAIL.asset
+            <set>
+                asset_net = #{itemDo.assetNet},
+                asset_share = #{itemDo.assetShare},
+                updatetime=#{itemDo.updateTime}
+            </set>
+            where isvalid = 1
+            and fund_id = #{itemDo.fundId}
+            and price_date = #{itemDo.priceDate}
+        </foreach>
+    </insert>
+
+    <select id="queryFundNavByDate" resultType="java.lang.String">
+        select price_date
+        from PPW_EMAIL.asset
+        where isvalid = 1
+        and fund_id = #{fundId}
+        and price_date in
+        <foreach collection="priceDateList" index="index" item="priceDate" separator="," open="(" close=")">
+            #{priceDate}
+        </foreach>
+    </select>
+
+</mapper>

service-base/src/main/resources/mapper/NavMapper.xml

@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.simuwang.base.mapper.NavMapper">
+    <resultMap id="BaseResultMap" type="com.simuwang.base.pojo.dos.NavDO">
+        <id column="id" property="id"/>
+        <result column="fund_id" property="fundId"/>
+        <result column="price_date" property="priceDate"/>
+        <result column="nav" property="nav"/>
+        <result column="cumulative_nav" property="cumulativeNav"/>
+        <result column="cumulative_nav_withdrawal" property="cumulativeNavWithdrawal"/>
+        <result column="isvalid" property="isvalid"/>
+        <result column="creatorid" property="creatorId"/>
+        <result column="createtime" property="createTime"/>
+        <result column="updaterid" property="updaterId"/>
+        <result column="updatetime" property="updateTime"/>
+    </resultMap>
+
+    <insert id="batchInsert" parameterType="com.simuwang.base.pojo.dos.NavDO">
+        insert into PPW_EMAIL.nav(fund_id,price_date,nav,cumulative_nav,cumulative_nav_withdrawal,
+                                  isvalid, creatorid, createtime, updaterid, updatetime)
+        values
+        <foreach collection="itemDoList" item="itemDo" index="index" separator=",">
+            (#{itemDo.fundId},#{itemDo.priceDate},#{itemDo.nav},#{itemDo.cumulativeNav},#{itemDo.cumulativeNavWithdrawal},
+            #{itemDo.isvalid}, #{itemDo.creatorId}, #{itemDo.createTime}, #{itemDo.updaterId}, #{itemDo.updateTime})
+        </foreach>
+
+    </insert>
+
+    <insert id="batchUpdate">
+        <foreach collection="itemDoList" item="itemDo" index="index" open="" close="" separator=";">
+            update PPW_EMAIL.nav
+            <set>
+                nav = #{itemDo.nav},
+                cumulative_nav_withdrawal = #{itemDo.cumulativeNavWithdrawal},
+                updatetime=#{itemDo.updateTime}
+            </set>
+            where isvalid = 1
+            and fund_id = #{itemDo.fundId}
+            and price_date = #{itemDo.priceDate}
+        </foreach>
+    </insert>
+
+    <select id="queryFundNavByDate" resultType="java.lang.String">
+        select price_date
+        from PPW_EMAIL.nav
+        where isvalid = 1
+          and fund_id = #{fundId}
+        and price_date in
+        <foreach collection="priceDateList" index="index" item="priceDate" separator="," open="(" close=")">
+            #{priceDate}
+        </foreach>
+    </select>
+
+
+</mapper>

service-daq/src/main/java/com/simuwang/daq/service/EmailParseService.java

@@ -1,5 +1,6 @@
 package com.simuwang.daq.service;
 
+import cn.hutool.core.bean.BeanUtil;
 import cn.hutool.core.collection.CollUtil;
 import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.exceptions.ExceptionUtil;
@@ -57,12 +58,15 @@ public class EmailParseService {
     private final EmailFundNavMapper emailFundNavMapper;
     private final EmailFundAssetMapper emailFundAssetMapper;
     private final FundAliasMapper fundAliasMapper;
+    private final AssetMapper assetMapper;
+    private final NavMapper navMapper;
 
     public EmailParseService(EmailTypeRuleMapper emailTypeRuleMapper, EmailRuleConfig emailRuleConfig,
                              EmailFieldMappingMapper emailFieldMapper, EmailParserFactory emailParserFactory,
                              EmailParseInfoMapper emailParseInfoMapper, FundInfoMapper fundInfoMapper,
                              EmailFileInfoMapper emailFileInfoMapper, EmailFundNavMapper emailFundNavMapper,
-                             EmailFundAssetMapper emailFundAssetMapper, FundAliasMapper fundAliasMapper) {
+                             EmailFundAssetMapper emailFundAssetMapper, FundAliasMapper fundAliasMapper,
+                             AssetMapper assetMapper, NavMapper navMapper) {
         this.emailTypeRuleMapper = emailTypeRuleMapper;
         this.emailRuleConfig = emailRuleConfig;
         this.emailFieldMapper = emailFieldMapper;
@@ -73,6 +77,8 @@ public class EmailParseService {
         this.emailFundNavMapper = emailFundNavMapper;
         this.emailFundAssetMapper = emailFundAssetMapper;
         this.fundAliasMapper = fundAliasMapper;
+        this.assetMapper = assetMapper;
+        this.navMapper = navMapper;
     }
 
     /**
@@ -144,7 +150,7 @@ public class EmailParseService {
             }
             for (EmailFundNavDTO fundNavDTO : fundNavDTOList) {
                 // 设置净值数据的解析状态
-                setNavParseStatus(fundNavDTO, emailTitle, emailDate);
+                setNavParseStatus(fundNavDTO, emailTitle);
             }
             // 保存净值表和规模表
             saveNavAndAssetNet(fileId, fundNavDTOList, parseDate);
@@ -162,15 +168,61 @@ public class EmailParseService {
         }
         // 净值数据
         List<EmailFundNavDO> emailFundNavDOList = fundNavDTOList.stream()
-                .map(e -> buildEmailFundNavDo(fileId, e, parseDate)).flatMap(List::stream).collect(Collectors.toList());
+                .map(e -> buildEmailFundNavDo(fileId, e, parseDate)).filter(CollUtil::isNotEmpty).flatMap(List::stream).collect(Collectors.toList());
         if (CollUtil.isNotEmpty(emailFundNavDOList)) {
             emailFundNavMapper.batchInsert(emailFundNavDOList);
+            List<NavDO> navDOList = emailFundNavDOList.stream().filter(e -> StrUtil.isNotBlank(e.getFundId()))
+                    .map(e -> BeanUtil.copyProperties(e, NavDO.class)).collect(Collectors.toList());
+            saveNavDo(navDOList);
         }
         // 保存规模数据
         List<EmailFundAssetDO> emailFundAssetDOList = fundNavDTOList.stream()
-                .map(e -> buildEmailFundAssetDo(fileId, e, parseDate)).flatMap(List::stream).collect(Collectors.toList());
+                .map(e -> buildEmailFundAssetDo(fileId, e, parseDate)).filter(CollUtil::isNotEmpty).flatMap(List::stream).collect(Collectors.toList());
         if (CollUtil.isNotEmpty(emailFundAssetDOList)) {
             emailFundAssetMapper.batchInsert(emailFundAssetDOList);
+            List<AssetDO> assetDOList = emailFundAssetDOList.stream().filter(e -> StrUtil.isNotBlank(e.getFundId()))
+                    .map(e -> BeanUtil.copyProperties(e, AssetDO.class)).collect(Collectors.toList());
+            saveAssetDo(assetDOList);
+        }
+    }
+
+    private void saveNavDo(List<NavDO> navDOList) {
+        if (CollUtil.isEmpty(navDOList)) {
+            return;
+        }
+        Map<String, List<NavDO>> fundIdNavMap = navDOList.stream().collect(Collectors.groupingBy(NavDO::getFundId));
+        for (Map.Entry<String, List<NavDO>> entry : fundIdNavMap.entrySet()) {
+            List<NavDO> navDOS = entry.getValue();
+            List<String> priceDateList = navDOS.stream().map(NavDO::getPriceDate).map(e -> DateUtil.format(e, DateConst.YYYY_MM_DD)).collect(Collectors.toList());
+            List<String> dateList = navMapper.queryFundNavByDate(entry.getKey(), priceDateList);
+            List<NavDO> updateNavDoList = navDOS.stream().filter(e -> dateList.contains(DateUtil.format(e.getPriceDate(), DateConst.YYYY_MM_DD))).collect(Collectors.toList());
+            List<NavDO> insertNavDoList = navDOS.stream().filter(e -> !dateList.contains(DateUtil.format(e.getPriceDate(), DateConst.YYYY_MM_DD))).collect(Collectors.toList());
+            if(CollUtil.isNotEmpty(insertNavDoList)){
+                navMapper.batchInsert(insertNavDoList);
+            }
+            if(CollUtil.isNotEmpty(updateNavDoList)){
+                navMapper.batchUpdate(updateNavDoList);
+            }
+        }
+    }
+
+    private void saveAssetDo(List<AssetDO> assetDOList) {
+        if (CollUtil.isEmpty(assetDOList)) {
+            return;
+        }
+        Map<String, List<AssetDO>> fundIdNavMap = assetDOList.stream().collect(Collectors.groupingBy(AssetDO::getFundId));
+        for (Map.Entry<String, List<AssetDO>> entry : fundIdNavMap.entrySet()) {
+            List<AssetDO> assetDOS = entry.getValue();
+            List<String> priceDateList = assetDOS.stream().map(AssetDO::getPriceDate).map(e -> DateUtil.format(e, DateConst.YYYY_MM_DD)).collect(Collectors.toList());
+            List<String> dateList = assetMapper.queryFundNavByDate(entry.getKey(), priceDateList);
+            List<AssetDO> updateAssetDoList = assetDOS.stream().filter(e -> dateList.contains(DateUtil.format(e.getPriceDate(), DateConst.YYYY_MM_DD))).collect(Collectors.toList());
+            List<AssetDO> insertAssetDoList = assetDOS.stream().filter(e -> !dateList.contains(DateUtil.format(e.getPriceDate(), DateConst.YYYY_MM_DD))).collect(Collectors.toList());
+            if(CollUtil.isNotEmpty(insertAssetDoList)){
+                assetMapper.batchInsert(insertAssetDoList);
+            }
+            if(CollUtil.isNotEmpty(updateAssetDoList)){
+                assetMapper.batchUpdate(updateAssetDoList);
+            }
         }
     }
 
@@ -222,6 +274,9 @@ public class EmailParseService {
         Date priceDate = DateUtil.parse(fundNavDTO.getPriceDate(), DateConst.YYYY_MM_DD);
         BigDecimal nav = StrUtil.isNotBlank(fundNavDTO.getNav()) ? new BigDecimal(fundNavDTO.getNav()) : null;
         BigDecimal cumulativeNavWithdrawal = StrUtil.isNotBlank(fundNavDTO.getCumulativeNavWithdrawal()) ? new BigDecimal(fundNavDTO.getCumulativeNavWithdrawal()) : null;
+        if (nav == null && cumulativeNavWithdrawal == null) {
+            return CollUtil.newArrayList();
+        }
         Integer isStored = fundNavDTO.getParseStatus() != null && !fundNavDTO.getParseStatus().equals(NavParseStatusConst.NAV_DEFICIENCY)
                 && !fundNavDTO.getParseStatus().equals(NavParseStatusConst.NOT_MATCH) ? 1 : 0;
         if (CollUtil.isNotEmpty(fundNavDTO.getFundIdList())) {
@@ -282,11 +337,10 @@ public class EmailParseService {
         return emailFileInfoDO;
     }
 
-    private void setNavParseStatus(EmailFundNavDTO fundNavDTO, String emailTitle, String emailDate) {
-        Integer navParseStatus = getDataParseStatus(fundNavDTO);
-        fundNavDTO.setParseStatus(navParseStatus);
-        if (navParseStatus.equals(NavParseStatusConst.NAV_DEFICIENCY)) {
-            log.info("判断数据的解析状态 -> 邮件主题:{},邮件日期:{},数据解析状态:{}", emailTitle, emailDate, fundNavDTO.getParseStatus());
+    private void setNavParseStatus(EmailFundNavDTO fundNavDTO, String emailTitle) {
+        // 1.单位净值或累计净值缺失
+        if (StrUtil.isBlank(fundNavDTO.getNav()) || StrUtil.isBlank(fundNavDTO.getCumulativeNavWithdrawal())) {
+            fundNavDTO.setParseStatus(NavParseStatusConst.NAV_DEFICIENCY);
             return;
         }
         // 2.匹配基金
@@ -295,6 +349,21 @@ public class EmailParseService {
             fundNavDTO.setParseStatus(NavParseStatusConst.NOT_MATCH);
         }
         fundNavDTO.setFundIdList(fundIdList);
+        // 考虑单独规模文件时 -> 无单位净值和累计净值
+        // 3.单位净值或累计净值不大于0
+        if (!emailTitle.contains("规模")) {
+            if (StrUtil.isBlank(fundNavDTO.getNav()) || StrUtil.isBlank(fundNavDTO.getCumulativeNavWithdrawal())
+                    || (fundNavDTO.getNav().compareTo("0") <= 0 || fundNavDTO.getCumulativeNavWithdrawal().compareTo("0") <= 0)) {
+                fundNavDTO.setParseStatus(NavParseStatusConst.NAV_NEGATIVE);
+                return;
+            }
+        }
+        // 4.资产净值不大于0
+        if (StrUtil.isNotBlank(fundNavDTO.getAssetNet()) && fundNavDTO.getAssetNet().compareTo("0") <= 0) {
+            fundNavDTO.setParseStatus(NavParseStatusConst.ASSET_NET_NEGATIVE);
+            return;
+        }
+        fundNavDTO.setParseStatus(NavParseStatusConst.SUCCESS);
     }
 
     private List<String> matchFund(String fundName, String registerNumber) {
@@ -356,22 +425,6 @@ public class EmailParseService {
         return emailParseInfoDO;
     }
 
-    private Integer getDataParseStatus(EmailFundNavDTO fundNavDTO) {
-        // 1.单位净值或累计净值缺失
-        if (StrUtil.isBlank(fundNavDTO.getNav()) || StrUtil.isBlank(fundNavDTO.getCumulativeNavWithdrawal())) {
-            return NavParseStatusConst.NAV_DEFICIENCY;
-        }
-        // 3.单位净值或累计净值不大于0
-        if (fundNavDTO.getNav().compareTo("0") <= 0 || fundNavDTO.getCumulativeNavWithdrawal().compareTo("0") <= 0) {
-            return NavParseStatusConst.NAV_NEGATIVE;
-        }
-        // 4.资产净值不大于0
-        if (StrUtil.isNotBlank(fundNavDTO.getAssetNet()) && fundNavDTO.getAssetNet().compareTo("0") <= 0) {
-            return NavParseStatusConst.ASSET_NET_NEGATIVE;
-        }
-        return NavParseStatusConst.SUCCESS;
-    }
-
     public Map<String, List<String>> getEmailFieldMapping() {
         List<EmailFieldMappingDO> emailFieldMappingDOList = emailFieldMapper.getEmailFieldMapping();
         return emailFieldMappingDOList.stream()

service-daq/src/main/java/com/simuwang/daq/service/NavEmailParser.java

@@ -343,7 +343,9 @@ public class NavEmailParser extends AbstractEmailParser {
                 ? ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.NAV))) : null;
         String cumulativeNavWithdrawal = columnFieldMap.get(EmailFieldConst.CUMULATIVE_NAV_WITHDRAWAL) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.CUMULATIVE_NAV_WITHDRAWAL)) != null ?
                 ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.CUMULATIVE_NAV_WITHDRAWAL))) : null;
-        if (StrUtil.isBlank(nav) && StrUtil.isBlank(cumulativeNavWithdrawal)) {
+        String assetNet = columnFieldMap.get(EmailFieldConst.ASSET_NET) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.ASSET_NET)) != null ?
+                ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.ASSET_NET))) : null;
+        if (StrUtil.isBlank(nav) && StrUtil.isBlank(cumulativeNavWithdrawal) && StrUtil.isBlank(assetNet)) {
             return null;
         }
         List<EmailFundNavDTO> fundNavDTOList = CollUtil.newArrayList();
@@ -375,8 +377,7 @@ public class NavEmailParser extends AbstractEmailParser {
                 ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.VIRTUAL_NAV))) : null;
         emailFundNavDTO.setVirtualNav(virtualNav);
 
-        String assetNet = columnFieldMap.get(EmailFieldConst.ASSET_NET) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.ASSET_NET)) != null ?
-                ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.ASSET_NET))) : null;
+
         emailFundNavDTO.setAssetNet(ExcelUtil.numberDataStripCommas(assetNet));
 
         String assetShares = columnFieldMap.get(EmailFieldConst.ASSET_SHARE) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.ASSET_SHARE)) != null ?