feat:优化净值缺失检测任务性能

service-base/src/main/java/com/simuwang/base/mapper/DeletionInfoMapper.java

@@ -39,6 +39,8 @@ public interface DeletionInfoMapper {
 
     DeletionInfoDO getDeletionInfoDO(DeletionInfoDO deletionInfoDO);
 
+    List<DeletionInfoDO> getDeletionInfo(@Param("fundId") String funId,@Param("deletionType")Integer code,@Param("list")List<String> tradeDateList);
+
     void saveDeletionInfoDO(DeletionInfoDO deletionInfoDO);
 
     void removeDistributeDeletion(@Param("fundId")String fundId);
@@ -55,4 +57,9 @@ public interface DeletionInfoMapper {
 
     List<FundDeletionInfoDO> getFundNavDeletion(@Param("deletionType") Integer deletionType, @Param("competitionId") Integer competitionId,
                                                 @Param("startDate") String startDate, @Param("endDate") String endDate);
+    void batchUpdateDeletionInfoDO(@Param("itemDoList") List<DeletionInfoDO> oldDeletionDOList);
+
+    void batchSaveDeletionInfoDO(@Param("itemDoList")List<DeletionInfoDO> insertDeletionInfoDO);
+
+    void deleteByDate(@Param("fundId")String fundId, @Param("deletionType") Integer deletionType, @Param("tradeDateList") List<String> tradeDateList);
 }

service-base/src/main/java/com/simuwang/base/mapper/TradeDateMapper.java

@@ -22,4 +22,6 @@ public interface TradeDateMapper {
     List<TradeDateDO> listNatureDays();
 
     List<IndexesTradeDate> listAllNatureDays();
+
+    List<TradeDateDO> listAllTradeDate(@Param("date")  String date);
 }

service-base/src/main/resources/mapper/DeletionInfoMapper.xml

@@ -298,4 +298,49 @@
           and t2.competition_id = #{competitionId}
           and t1.deletion_date <![CDATA[ <= ]]> #{endDate}
     </select>
+
+    <select id="getDeletionInfo" resultType="com.simuwang.base.pojo.dos.DeletionInfoDO"
+            parameterType="com.simuwang.base.pojo.dos.DeletionInfoDO">
+        select id,fund_id,deletion_type,deletion_date,remark,isvalid,creatorid,updaterid,createtime,updatetime
+        from deletion_info where fund_id=#{fundId} and deletion_type=#{deletionType} and deletion_date in
+        <foreach item="tradeDate" collection="list" open="(" separator="," close=")">
+            #{tradeDate}
+        </foreach>
+    </select>
+
+    <update id="batchUpdateDeletionInfoDO">
+        <foreach collection="itemDoList" item="itemDo" index="index" open="" close="" separator=";">
+            update deletion_info
+            <set>
+                fund_id=#{itemDo.fundId},
+                deletion_type = #{itemDo.deletionType},
+                deletion_date=#{itemDo.deletionDate},
+                remark=#{itemDo.remark},
+                updatetime=sysdate(),
+                isvalid=#{itemDo.isvalid},
+                updaterid=#{itemDo.updaterId}
+            </set>
+            where id=#{itemDo.id}
+        </foreach>
+    </update>
+
+    <insert id="batchSaveDeletionInfoDO">
+        insert into deletion_info(fund_id,deletion_type,deletion_date,isvalid,createtime,updatetime,is_send)
+        values
+        <foreach collection="itemDoList" item="itemDo" index="index" separator=",">
+            (#{itemDo.fundId},#{itemDo.deletionType},#{itemDo.deletionDate},#{itemDo.isvalid},#{itemDo.createTime},#{itemDo.updateTime},#{itemDo.isSend})
+        </foreach>
+    </insert>
+
+    <update id="deleteByDate">
+        update deletion_info set isvalid=0,
+        updatetime=now()
+        where fund_id=#{fundId}
+        and deletion_type = #{deletionType}
+        and deletion_date in
+        <foreach item="tradeDate" collection="tradeDateList" open="(" separator="," close=")">
+            #{tradeDate}
+        </foreach>
+    </update>
+
 </mapper>

service-base/src/main/resources/mapper/TradeDateMapper.xml

@@ -48,4 +48,21 @@
         order by trade_date asc
     </select>
 
+    <select id="listAllTradeDate" resultMap="BaseResultMap">
+        select id,
+               trade_date,
+               end_year,
+               yearmonth,
+               week_of_year,
+               year_week,
+               day_of_week,
+               isholiday,
+               isvalid,
+               createtime,
+               updatetime
+        from pvn_trade_date
+        where isvalid = 1
+          and isholiday = 0
+        and trade_date <![CDATA[ <= ]]> #{date}
+    </select>
 </mapper>

service-manage/src/main/java/com/simuwang/manage/service/DeletionService.java

@@ -1,6 +1,7 @@
 package com.simuwang.manage.service;
 
 import com.simuwang.base.common.support.MybatisPage;
+import com.simuwang.base.pojo.dos.TradeDateDO;
 import com.simuwang.base.pojo.dto.DeletionDownParam;
 import com.simuwang.base.pojo.dto.ExcelDeletionInfoDTO;
 import com.simuwang.base.pojo.dto.query.DeletionPageQuery;
@@ -30,4 +31,6 @@ public interface DeletionService {
     List<ExcelDeletionInfoDTO> selectFundDeletionInfoVOList(FundDeletionListVO fundDeletionListVO);
 
     void computeDeletion(DeletionDownParam deletionDownParam);
+
+    public void computeDeletionNew(String fundId, List<TradeDateDO> tradeDateDOList);
 }

service-manage/src/main/java/com/simuwang/manage/service/impl/DeletionServiceImpl.java

@@ -1,6 +1,8 @@
 package com.simuwang.manage.service.impl;
 
+import cn.hutool.core.collection.CollUtil;
 import cn.hutool.core.date.DateUtil;
+import com.simuwang.base.common.conts.DateConst;
 import com.simuwang.base.common.enums.DeletionType;
 import com.simuwang.base.common.enums.DistributeType;
 import com.simuwang.base.common.enums.Frequency;
@@ -141,8 +143,10 @@ public class DeletionServiceImpl implements DeletionService {
             return;
         }
         List<NavDO> navDOList = navMapper.selectNavByFundId(fundId);
-        List<AssetDO> assetDOList = assetMapper.selectAssetByFundId(fundId);
-        List<FundPositionDetailDO> fundPositionDetailDOList = fundPositionDetailMapper.fundPositionDetailByFundId(fundId);
+//        List<AssetDO> assetDOList = assetMapper.selectAssetByFundId(fundId);
+        List<AssetDO> assetDOList = CollUtil.newArrayList();
+        //List<FundPositionDetailDO> fundPositionDetailDOList = fundPositionDetailMapper.fundPositionDetailByFundId(fundId);
+        List<FundPositionDetailDO> fundPositionDetailDOList = CollUtil.newArrayList();
         //查询成立日到今天为止的交易日集合
         List<TradeDateDO> tradeDateDOList = tradeDateMapper.selectTradeDate(inceptionDate,today);
         if(deletionDownParam.getDeletionType() != null && deletionDownParam.getDeletionType().equals(DeletionType.NAV_DELETION.getCode())){
@@ -162,6 +166,15 @@ public class DeletionServiceImpl implements DeletionService {
 
     }
 
+    @Override
+    public void computeDeletionNew(String fundId,List<TradeDateDO> tradeDateDOList) {
+        // 净值报送频率为：周频
+        FundReportFrequencyDO fundReportFrequencyDO = new FundReportFrequencyDO();
+        fundReportFrequencyDO.setNavFrequency(2);
+        List<NavDO> navDOList = navMapper.selectNavByFundId(fundId);
+        navDeletion(fundId, navDOList, tradeDateDOList, fundReportFrequencyDO);
+    }
+
     private void distributionDeletion(String fundId, List<NavDO> navDOList) {
         if(navDOList.size() < 1){
             return;
@@ -364,57 +377,63 @@ public class DeletionServiceImpl implements DeletionService {
                 saveDeletionInfoDO(fundId,tradeDate,DeletionType.NAV_DELETION.getCode());
             }
         }
-        if(Frequency.WEEK == Frequency.getFrequencyByCode(fundReportFrequencyDO.getNavFrequency())){
-            Map<String,List<NavDO>> navListMap = navDOList.stream().collect(Collectors.groupingBy(e -> DateUtils.format(e.getPriceDate(),DateUtils.YYYY_MM_DD)));
-            TreeMap<Integer,List<NavDO>> weekNavListMap = new TreeMap<>();
+        List<String> deletionTradeDateList = CollUtil.newArrayList();
+        List<String> tradeDateList = CollUtil.newArrayList();
+        if (Frequency.WEEK == Frequency.getFrequencyByCode(fundReportFrequencyDO.getNavFrequency())) {
+            Map<String, List<NavDO>> navListMap = navDOList.stream().collect(Collectors.groupingBy(e -> DateUtils.format(e.getPriceDate(), DateUtils.YYYY_MM_DD)));
+            TreeMap<Integer, List<NavDO>> weekNavListMap = new TreeMap<>();
             //按周数整合
-            for(String priceDate : navListMap.keySet()){
-                Integer weekOfYear = Integer.parseInt(priceDate.substring(0,4)+DateUtil.weekOfYear(DateUtils.parse(priceDate,DateUtils.YYYY_MM_DD)));;
-                if(weekNavListMap.containsKey(weekOfYear)){
+            for (String priceDate : navListMap.keySet()) {
+                Integer weekOfYear = Integer.parseInt(priceDate.substring(0, 4) + DateUtil.weekOfYear(DateUtils.parse(priceDate, DateUtils.YYYY_MM_DD)));
+                if (weekNavListMap.containsKey(weekOfYear)) {
                     List<NavDO> navDOS = weekNavListMap.get(weekOfYear);
                     navDOS.addAll(navListMap.get(priceDate));
-                    weekNavListMap.put(weekOfYear,navDOS);
-                }else{
+                    weekNavListMap.put(weekOfYear, navDOS);
+                } else {
                     List<NavDO> navDOS = new ArrayList<>();
                     navDOS.addAll(navListMap.get(priceDate));
-                    weekNavListMap.put(weekOfYear,navDOS);
+                    weekNavListMap.put(weekOfYear, navDOS);
                 }
             }
-            Map<Integer,List<TradeDateDO>> tradeListMap = tradeDateDOList.stream().collect(Collectors.groupingBy(e -> e.getYearWeek()));
-            for(Integer weekOfYear : tradeListMap.keySet()){
+            Map<Integer, List<TradeDateDO>> tradeListMap = tradeDateDOList.stream().collect(Collectors.groupingBy(e -> e.getYearWeek()));
+            for (Integer weekOfYear : tradeListMap.keySet()) {
                 List<NavDO> navDOS = weekNavListMap.get(weekOfYear);
-                if(weekNavListMap.containsKey(weekOfYear)){
-                    for(TradeDateDO tradeDateDO : tradeListMap.get(weekOfYear)){
-                        String tradeDate = DateUtils.format(tradeDateDO.getTradeDate(),DateUtils.YYYY_MM_DD);
-                        deletionInfoMapper.updateRemark(fundId,DeletionType.NAV_DELETION.getCode(),tradeDate,DeletionType.NO_DELETION.getInfo(),null);
+                if (weekNavListMap.containsKey(weekOfYear)) {
+                    List<TradeDateDO> dateDOList = tradeListMap.get(weekOfYear);
+                    if(CollUtil.isNotEmpty(dateDOList)){
+                        tradeDateList.addAll(dateDOList.stream().map(e->DateUtil.format(e.getTradeDate(), DateConst.YYYY_MM_DD)).toList());
                     }
                     continue;
-                }else{
+                } else {
                     //防止频率变更导致数据异常，需要把当前日期下所在的频率全部无效掉
-                    if(StringUtil.isNotEmpty(navDOS)){
-                        for(int idx=0;idx < navDOS.size()-1 ; idx++){
+                    if (StringUtil.isNotEmpty(navDOS)) {
+                        for (int idx = 0; idx < navDOS.size() - 1; idx++) {
                             NavDO navDO = navDOS.get(idx);
-                            String tradeDate = DateUtils.format(navDO.getPriceDate(),DateUtils.YYYY_MM_DD);
-                            deletionInfoMapper.deleteDeletionRemark(fundId,DeletionType.NAV_DELETION.getCode(),tradeDate);
+                            String tradeDate = DateUtils.format(navDO.getPriceDate(), DateUtils.YYYY_MM_DD);
+                            deletionInfoMapper.deleteDeletionRemark(fundId, DeletionType.NAV_DELETION.getCode(), tradeDate);
                         }
                     }
                 }
                 //不包含的话，默认取每周的最后一个交易日作为周净值日期
                 List<TradeDateDO> tradeDateDOS = tradeListMap.get(weekOfYear);
                 String tradeDate = null;
-                if(tradeDateDOS.size() > 0){
-                    tradeDate = DateUtils.format(tradeDateDOS.get(tradeDateDOS.size()-1).getTradeDate(),DateUtils.YYYY_MM_DD);
+                if (tradeDateDOS.size() > 0) {
+                    tradeDate = DateUtils.format(tradeDateDOS.get(tradeDateDOS.size() - 1).getTradeDate(), DateUtils.YYYY_MM_DD);
                 }
-                if(tradeDateDOList.size() <= 3){
+                if (tradeDateDOList.size() <= 3) {
                     continue;
                 }
-                TradeDateDO tradeDateDO = tradeDateDOList.get(tradeDateDOList.size()-3);
-                if(tradeDateDO.getTradeDate().compareTo(DateUtils.parse(tradeDate,DateUtils.YYYY_MM_DD)) < 0){
+                TradeDateDO tradeDateDO = tradeDateDOList.get(tradeDateDOList.size() - 3);
+                if (tradeDateDO.getTradeDate().compareTo(DateUtils.parse(tradeDate, DateUtils.YYYY_MM_DD)) < 0) {
                     continue;
                 }
                 //写入缺失信息表
-                saveDeletionInfoDO(fundId,tradeDate,DeletionType.NAV_DELETION.getCode());
+                deletionTradeDateList.add(tradeDate);
+            }
+            if (CollUtil.isNotEmpty(tradeDateList)) {
+                deletionInfoMapper.deleteByDate(fundId, DeletionType.NAV_DELETION.getCode(), tradeDateList);
             }
+            batchSaveDeletionInfoDO(fundId, deletionTradeDateList, DeletionType.NAV_DELETION.getCode());
         }
         if(Frequency.MONTH == Frequency.getFrequencyByCode(fundReportFrequencyDO.getNavFrequency())){
             Map<String,List<NavDO>> navListMap = navDOList.stream().collect(Collectors.groupingBy(e -> DateUtils.format(e.getPriceDate(),DateUtils.YYYY_MM_DD)));
@@ -475,6 +494,44 @@ public class DeletionServiceImpl implements DeletionService {
         }
     }
 
+    private void batchSaveDeletionInfoDO(String fundId, List<String> tradeDateList, Integer code) {
+        if (tradeDateList.size() < 1) {
+            return;
+        }
+        List<DeletionInfoDO> oldDeletionDOList = deletionInfoMapper.getDeletionInfo(fundId, code, tradeDateList);
+        if (oldDeletionDOList != null && oldDeletionDOList.size() > 0) {
+            for (DeletionInfoDO oldDeletionDO : oldDeletionDOList) {
+                oldDeletionDO.setIsvalid(1);
+                oldDeletionDO.setUpdateTime(DateUtils.getNowDate());
+            }
+            deletionInfoMapper.batchUpdateDeletionInfoDO(oldDeletionDOList);
+        }
+
+        List<String> oldTradeDateList = oldDeletionDOList.stream().map(e -> e.getDeletionDate()).collect(Collectors.toList());
+        List<String> insertTradeDateList = new ArrayList<>();
+        for (String tradeDate : tradeDateList) {
+            if (oldTradeDateList.contains(tradeDate)) {
+                continue;
+            }
+            insertTradeDateList.add(tradeDate);
+        }
+        if (insertTradeDateList.size() > 0) {
+            List<DeletionInfoDO> insertDeletionInfoDO = new ArrayList<>();
+            for (String tradeDate : insertTradeDateList) {
+                DeletionInfoDO deletionInfoDO = new DeletionInfoDO();
+                deletionInfoDO.setFundId(fundId);
+                deletionInfoDO.setDeletionType(code);
+                deletionInfoDO.setDeletionDate(tradeDate);
+                deletionInfoDO.setIsvalid(1);
+                deletionInfoDO.setIsSend(0);
+                deletionInfoDO.setUpdateTime(DateUtils.getNowDate());
+                deletionInfoDO.setCreateTime(DateUtils.getNowDate());
+                insertDeletionInfoDO.add(deletionInfoDO);
+            }
+            deletionInfoMapper.batchSaveDeletionInfoDO(insertDeletionInfoDO);
+        }
+    }
+
     private void valuationDeletion(String fundId,List<FundPositionDetailDO> fundPositionDetailDOList, List<TradeDateDO> tradeDateDOList,FundReportFrequencyDO fundReportFrequencyDO) {
         //将现有数据无效掉
         deletionInfoMapper.deleteDeletion(fundId,DeletionType.VALUATION_DELETION.getCode());

service-manage/src/main/java/com/simuwang/manage/task/CompetitionTask.java

@@ -80,7 +80,7 @@ public class CompetitionTask {
             log.info("基金净值采集结果明细表格已输出 -> 文件:{}", filePath);
 
             // 通过邮件发送"基金净值采集结果明细表格"
-            if (competitionConfig.getMethod() == 2) {
+            if (competitionConfig.getMethod() == 1) {
                 MailboxInfoDTO mailboxInfoDTO = emailSystemConfigService.getFromEmailInfo();
                 String receivingMailbox = emailSystemConfigService.getRecipientEmail();
                 if (mailboxInfoDTO == null || StrUtil.isBlank(receivingMailbox)) {
@@ -166,7 +166,7 @@ public class CompetitionTask {
 
             Integer method = competitionConfig.getMethod();
             // 通过邮件发送"基金净值采集结果明细表格"
-            if (method == 2) {
+            if (method == 1) {
                 MailboxInfoDTO mailboxInfoDTO = emailSystemConfigService.getFromEmailInfo();
                 String receivingMailbox = emailSystemConfigService.getRecipientEmail();
                 if (mailboxInfoDTO == null || StrUtil.isBlank(receivingMailbox)) {

service-manage/src/main/java/com/simuwang/manage/task/FundDeletionTask.java

@@ -1,5 +1,7 @@
 package com.simuwang.manage.task;
 
+import cn.hutool.core.date.DateUtil;
+import com.simuwang.base.common.conts.DateConst;
 import com.simuwang.base.common.enums.TaskType;
 import com.simuwang.base.common.util.DateUtils;
 import com.simuwang.base.mapper.*;
@@ -30,22 +32,25 @@ public class FundDeletionTask {
     private DeletionService deletionService;
     @Autowired
     private EmailTaskInfoMapper emailTaskInfoMapper;
+    @Autowired
+    private TradeDateMapper tradeDateMapper;
 //    @Scheduled(cron = "0 0 5,12,19 * * ?")
     public void computeDeletion(){
         EmailTaskInfoDO emailTaskInfoDO = startEmailTask(null, 1);
         List<String> fundIdList = navMapper.getAllFundId();
-        try{
-            for(String fundId : fundIdList){
-                DeletionDownParam deletionDownParam = new DeletionDownParam();
-                deletionDownParam.setFundId(fundId);
-                deletionService.computeDeletion(deletionDownParam);
+        // 查出所有交易日序列
+        String toDay = DateUtil.format(new Date(), DateConst.YYYY_MM_DD);
+        List<TradeDateDO> tradeDateDOList = tradeDateMapper.listAllTradeDate(toDay);
+        try {
+            for (String fundId : fundIdList) {
+                deletionService.computeDeletionNew(fundId, tradeDateDOList);
             }
-        }catch (Exception e){
-            log.error(e.getMessage(),e);
-            endEmailTask(emailTaskInfoDO.getId(),-1);
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+            endEmailTask(emailTaskInfoDO.getId(), -1);
             return;
         }
-        endEmailTask(emailTaskInfoDO.getId(),2);
+        endEmailTask(emailTaskInfoDO.getId(), 2);
     }
     private void endEmailTask(Integer id, Integer taskStatus) {
         try{