feat: 振幅检测需求开发

service-base/src/main/java/com/simuwang/base/mapper/EmailFundNavMapper.java

@@ -35,4 +35,6 @@ public interface EmailFundNavMapper {
     List<EmailFundNavDO> selectNavByFundId(@Param("fundId") String sourceFundId);
 
     Long countEmailNavTotal();
+
+    void update(EmailFundNavDO fundNavDO);
 }

service-base/src/main/resources/mapper/EmailFundNavMapper.xml

@@ -47,6 +47,18 @@
             and id = #{itemDo.id}
         </foreach>
     </update>
+    <update id="update" parameterType="com.simuwang.base.pojo.dos.EmailFundNavDO">
+        update email_fund_nav
+        set
+            exception_status = #{exceptionStatus},
+            updatetime=#{updateTime},
+            is_stored=#{isStored}
+        where isvalid = 1 and price_date = #{priceDate} and fund_id=#{fundId}
+          and nav=#{nav} and cumulative_nav_withdrawal=#{cumulativeNavWithdrawal}
+        <if test="fileId != null and fileId !=''">
+            and file_id = #{fileId}
+        </if>
+    </update>
     <delete id="deleteByFileId" parameterType="java.lang.Integer">
         update email_fund_nav nav set isvalid = 0,updatetime=now() where file_id=#{fileId}
     </delete>

service-base/src/main/resources/mapper/NavMapper.xml

@@ -107,7 +107,7 @@
                isvalid, creatorid, createtime, updaterid, updatetime
         from nav
         where isvalid = 1
-          and fund_id = #{fundId} and price_date <![CDATA[ <= ]]> #{priceDate}
+          and fund_id = #{fundId} and price_date <![CDATA[ < ]]> #{priceDate}
         order by price_date desc
         limit 1
     </select>

service-daq/src/main/java/com/simuwang/daq/service/EmailParseService.java

@@ -4,6 +4,7 @@ import cn.hutool.core.bean.BeanUtil;
 import cn.hutool.core.collection.CollUtil;
 import cn.hutool.core.collection.CollectionUtil;
 import cn.hutool.core.collection.ListUtil;
+import cn.hutool.core.date.DatePattern;
 import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.exceptions.ExceptionUtil;
 import cn.hutool.core.map.MapUtil;
@@ -482,7 +483,7 @@ public class EmailParseService {
             emailFundNavMapper.batchInsert(emailFundNavDOList);
             List<NavDO> navDOList = emailFundNavDOList.stream().filter(e -> StrUtil.isNotBlank(e.getFundId()))
                     .map(e -> BeanUtil.copyProperties(e, NavDO.class)).collect(Collectors.toList());
-            saveNavDo(navDOList);
+            saveNavDo(navDOList,fileId);
         }
         // 保存规模数据
         List<EmailFundAssetDO> emailFundAssetDOList = fundNavDTOList.stream()
@@ -497,7 +498,7 @@ public class EmailParseService {
         }
     }
 
-    public void saveNavDo(List<NavDO> navDOList) {
+    public void saveNavDo(List<NavDO> navDOList,Integer fileId) {
         if (CollUtil.isEmpty(navDOList)) {
             return;
         }
@@ -508,21 +509,60 @@ public class EmailParseService {
             List<String> dateList = navMapper.queryFundNavByDate(entry.getKey(), priceDateList);
             List<NavDO> updateNavDoList = navDOS.stream().filter(e -> dateList.contains(DateUtil.format(e.getPriceDate(), DateConst.YYYY_MM_DD))).collect(Collectors.toList());
             List<NavDO> insertNavDoList = navDOS.stream().filter(e -> !dateList.contains(DateUtil.format(e.getPriceDate(), DateConst.YYYY_MM_DD))).collect(Collectors.toList());
-            if (CollUtil.isNotEmpty(insertNavDoList)) {
-                Map<Date, List<NavDO>> priceDateNavDoListMap = insertNavDoList.stream().collect(Collectors.groupingBy(NavDO::getPriceDate));
-                boolean hasDuplicationDateData = priceDateNavDoListMap.values().stream().map(List::size).anyMatch(e -> e > 1);
-                if (!hasDuplicationDateData) {
-                    navMapper.batchInsert(insertNavDoList);
-                }
-                // 要插入的数据中存在相同日期的数据 -> 只能一条一条的插入数据了
-                insertNavDoList.forEach(e -> saveNavDo(ListUtil.toList(e)));
+            //按照净值日期排序，方便后面的振幅检测
+            if (CollUtil.isNotEmpty(insertNavDoList) && insertNavDoList.size() > 1) {
+                insertNavDoList = insertNavDoList.stream().sorted(Comparator.comparing(NavDO::getPriceDate)).collect(Collectors.toList());
+            }
+            if (CollUtil.isNotEmpty(updateNavDoList) && updateNavDoList.size() > 1) {
+                updateNavDoList = updateNavDoList.stream().sorted(Comparator.comparing(NavDO::getPriceDate)).collect(Collectors.toList());
             }
-            if (CollUtil.isNotEmpty(updateNavDoList)) {
-                navMapper.batchUpdate(updateNavDoList);
+            //因为要做振幅检测，所以不能使用批量插入的方式
+            for(NavDO insertDO : insertNavDoList){
+               boolean  amplitudeException = amplitudeNav(insertDO);
+               if(amplitudeException){
+                   //超过净值振幅不能入库，同时需要修改采集库的入库状态和异常信息
+                   saveAmplitudeFundNav(insertDO,fileId);
+               }else{
+                   navMapper.batchInsert(ListUtil.toList(insertDO));
+               }
             }
+            for(NavDO updateDO : updateNavDoList){
+                boolean  amplitudeException = amplitudeNav(updateDO);
+                if(amplitudeException){
+                    //超过净值振幅不能入库，同时需要修改采集库的入库状态和异常信息
+                    saveAmplitudeFundNav(updateDO,fileId);
+                }else{
+                    navMapper.batchUpdate(ListUtil.toList(updateDO));
+                }
+            }
+//            if (CollUtil.isNotEmpty(insertNavDoList)) {
+//                Map<Date, List<NavDO>> priceDateNavDoListMap = insertNavDoList.stream().collect(Collectors.groupingBy(NavDO::getPriceDate));
+//                boolean hasDuplicationDateData = priceDateNavDoListMap.values().stream().map(List::size).anyMatch(e -> e > 1);
+//                if (!hasDuplicationDateData) {
+//                    navMapper.batchInsert(insertNavDoList);
+//                }
+//                // 要插入的数据中存在相同日期的数据 -> 只能一条一条的插入数据了
+//                insertNavDoList.forEach(e -> saveNavDo(ListUtil.toList(e)));
+//            }
+//            if (CollUtil.isNotEmpty(updateNavDoList)) {
+//                navMapper.batchUpdate(updateNavDoList);
+//            }
         }
     }
 
+    private void saveAmplitudeFundNav(NavDO insertDO,Integer fileId) {
+        EmailFundNavDO fundNavDO = new EmailFundNavDO();
+        fundNavDO.setFundId(insertDO.getFundId());
+        fundNavDO.setNav(insertDO.getNav());
+        fundNavDO.setPriceDate(insertDO.getPriceDate());
+        fundNavDO.setCumulativeNavWithdrawal(insertDO.getCumulativeNavWithdrawal());
+        fundNavDO.setFileId(fileId);
+        fundNavDO.setExceptionStatus(NavParseStatusConst.AMPLITUDE_EXCEPTION);
+        fundNavDO.setIsStored(0);
+        fundNavDO.setUpdateTime(new Date());
+        emailFundNavMapper.update(fundNavDO);
+    }
+
     public void saveAssetDo(List<AssetDO> assetDOList) {
         if (CollUtil.isEmpty(assetDOList)) {
             return;
@@ -706,25 +746,19 @@ public class EmailParseService {
         }
         fundNavDTO.setParseStatus(NavParseStatusConst.SUCCESS);
     }
-
-    private boolean amplitudeNav(EmailFundNavDTO fundNavDTO) {
-        if(CollUtil.isEmpty(fundNavDTO.getFundIdList())){
+    //振幅检测
+    private boolean amplitudeNav(NavDO navDO) {
+        String fundId = navDO.getFundId();
+        String priceDate = DateUtils.format(navDO.getPriceDate(), DateUtils.YYYY_MM_DD);
+        //上一期单位净值
+        NavDO preNavDO = navMapper.queryNavByFundIdDate(fundId,priceDate);
+        if(preNavDO == null){
             return false;
         }
-        List<String> fundIdList = fundNavDTO.getFundIdList();
-        String priceDate = fundNavDTO.getPriceDate();
-        for(String fundId : fundIdList){
-            NavDO navDO = navMapper.queryNavByFundIdDate(fundId,priceDate);
-            if(navDO == null){
-                continue;
-            }
-            //|（当期-上期）/ 上期 | >= 0.2
-            BigDecimal amplitudeRate = (new BigDecimal(fundNavDTO.getNav()).subtract(navDO.getNav())).divide(navDO.getNav());
-            Double absAmplitudeRate = Math.abs(amplitudeRate.doubleValue());
-            if(BigDecimal.valueOf(absAmplitudeRate).compareTo(new BigDecimal(0.2)) > 0 ){
-                fundNavDTO.setParseStatus(NavParseStatusConst.AMPLITUDE_EXCEPTION);
-                return true;
-            }
+        //|（当期-上期）/ 上期 | >= 0.2
+        BigDecimal amplitudeRate = navDO.getNav().subtract(preNavDO.getNav()).divide(preNavDO.getNav(),4, BigDecimal.ROUND_HALF_UP);
+        if(amplitudeRate.abs().compareTo(new BigDecimal(0.2)) > 0 ){
+            return true;
         }
         return false;
     }

service-deploy/src/main/resources/application.yml

@@ -26,11 +26,16 @@ spring:
   datasource:
     data-daq:
       # 指定为HikariDataSource
+#      type: com.zaxxer.hikari.HikariDataSource
+#      driver-class-name: com.alipay.oceanbase.obproxy.mysql.jdbc.Driver
+#      url:  jdbc:oceanbase://192.168.1.28:2881/ppw_email?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8&allowPublicKeyRetrieval=true
+#      username: root
+#      password: yIJ+L+WxdcugkhQEl+oM1t63Rpt+RnQ5
       type: com.zaxxer.hikari.HikariDataSource
-      driver-class-name: com.alipay.oceanbase.obproxy.mysql.jdbc.Driver
-      url:  jdbc:oceanbase://192.168.1.28:2881/ppw_email?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8&allowPublicKeyRetrieval=true
-      username: root
-      password: yIJ+L+WxdcugkhQEl+oM1t63Rpt+RnQ5
+      driver-class-name: dm.jdbc.driver.DmDriver
+      url: jdbc:dm://192.168.1.39:5236/PPW_EMAIL
+      username: PPW_EMAIL
+      password: nRkHI1oCRiAOckoaB/VryURnEo98Iy7k
       # hikari连接池配置 对应 HikariConfig 配置属性类
       hikari:
         pool-name: HikariCP-daq

service-deploy/src/test/java/com/simuwang/ApplicationTest.java

@@ -1,89 +1,89 @@
-package com.simuwang;
-
-import cn.hutool.core.collection.ListUtil;
-import cn.hutool.core.date.DateUtil;
-import com.simuwang.base.common.conts.DateConst;
-import com.simuwang.base.common.util.DateUtils;
-import com.simuwang.base.pojo.dto.MailboxInfoDTO;
-import com.simuwang.daq.service.EmailParseApiService;
-import com.simuwang.daq.service.EmailParseService;
-import org.jasypt.util.text.BasicTextEncryptor;
-import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.context.SpringBootTest;
-
-import java.util.ArrayList;
-import java.util.Date;
-
-@SpringBootTest(classes = Application.class)
-public class ApplicationTest {
-
-    @Autowired
-    private EmailParseService emailParseService;
-
-    @Autowired
-    private EmailParseApiService emailParseApiService;
-
-    @Test
-    public void test() {
-        MailboxInfoDTO emailInfoDTO = this.buildMailbox("", "");
+//package com.simuwang;
 //
-//        emailInfoDTO.setAccount("jjpj_test");
-//        emailInfoDTO.setPassword("shzq#919");
-//        emailInfoDTO.setHost("mail.shzq.com");
+//import cn.hutool.core.collection.ListUtil;
+//import cn.hutool.core.date.DateUtil;
+//import com.simuwang.base.common.conts.DateConst;
+//import com.simuwang.base.common.util.DateUtils;
+//import com.simuwang.base.pojo.dto.MailboxInfoDTO;
+//import com.simuwang.daq.service.EmailParseApiService;
+//import com.simuwang.daq.service.EmailParseService;
+//import org.jasypt.util.text.BasicTextEncryptor;
+//import org.junit.jupiter.api.Test;
+//import org.springframework.beans.factory.annotation.Autowired;
+//import org.springframework.boot.test.context.SpringBootTest;
+//
+//import java.util.ArrayList;
+//import java.util.Date;
+//
+//@SpringBootTest(classes = Application.class)
+//public class ApplicationTest {
+//
+//    @Autowired
+//    private EmailParseService emailParseService;
+//
+//    @Autowired
+//    private EmailParseApiService emailParseApiService;
+//
+//    @Test
+//    public void test() {
+//        MailboxInfoDTO emailInfoDTO = this.buildMailbox("", "");
+////
+////        emailInfoDTO.setAccount("jjpj_test");
+////        emailInfoDTO.setPassword("shzq#919");
+////        emailInfoDTO.setHost("mail.shzq.com");
+////        emailInfoDTO.setPort("993");
+////        emailInfoDTO.setProtocol("imap");
+//
+//        Date startDate = DateUtil.parse("2024-10-10 15:00:00", DateConst.YYYY_MM_DD_HH_MM_SS);
+//        Date endDate = DateUtil.parse("2024-10-10 16:40:00", DateConst.YYYY_MM_DD_HH_MM_SS);
+//        try {
+//            emailParseService.parseEmail(emailInfoDTO, startDate, endDate);
+//        } catch (Exception e) {
+//            throw new RuntimeException(e);
+//        }
+//    }
+//
+//    @Test
+//    public void reportTest() {
+//        MailboxInfoDTO emailInfoDTO = this.buildMailbox("x", "x");
+//        Date startDate = DateUtil.parse("2024-10-15 15:10:30", DateConst.YYYY_MM_DD_HH_MM_SS);
+//        Date endDate = DateUtil.parse("2024-10-15 17:50:30", DateConst.YYYY_MM_DD_HH_MM_SS);
+//        try {
+//            emailParseService.parseEmail(emailInfoDTO, startDate, endDate);
+//        } catch (Exception e) {
+//            throw new RuntimeException(e);
+//        }
+//    }
+//
+//    @Test
+//    public void testReparseEmail() {
+//        emailParseApiService.reparseEmail(593);
+//    }
+//
+//    @Test
+//    public void testReparseFile() {
+//        emailParseApiService.reparseFile(ListUtil.toList(40, 43));
+//    }
+//
+//    @Test
+//    public void testDateFormat() {
+//        String input = "Smppw@2024";
+//        BasicTextEncryptor textEncryptor = new BasicTextEncryptor();
+//        textEncryptor.setPassword("qwertyuiopasdfghjklzxcvbnm1234567890qwertyuiopasdfghjklzxcvbnm12"); // 用您自己的密钥替换"yourSecretKey"
+//        String encryptedString = textEncryptor.encrypt(input);
+//        String decrypt = textEncryptor.decrypt(encryptedString);
+//        System.out.println("加密后的字符串为: " + encryptedString);
+//        System.out.println("解密后的字符串为: " + decrypt);
+//    }
+//
+//    private MailboxInfoDTO buildMailbox(String account, String pwd) {
+//        MailboxInfoDTO emailInfoDTO = new MailboxInfoDTO();
+//        emailInfoDTO.setUserId(1);
+//        emailInfoDTO.setAccount(account);
+//        emailInfoDTO.setPassword(pwd);
+//        emailInfoDTO.setHost("imap.exmail.qq.com");
 //        emailInfoDTO.setPort("993");
 //        emailInfoDTO.setProtocol("imap");
-
-        Date startDate = DateUtil.parse("2024-10-10 15:00:00", DateConst.YYYY_MM_DD_HH_MM_SS);
-        Date endDate = DateUtil.parse("2024-10-10 16:40:00", DateConst.YYYY_MM_DD_HH_MM_SS);
-        try {
-            emailParseService.parseEmail(emailInfoDTO, startDate, endDate);
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    @Test
-    public void reportTest() {
-        MailboxInfoDTO emailInfoDTO = this.buildMailbox("x", "x");
-        Date startDate = DateUtil.parse("2024-10-15 15:10:30", DateConst.YYYY_MM_DD_HH_MM_SS);
-        Date endDate = DateUtil.parse("2024-10-15 17:50:30", DateConst.YYYY_MM_DD_HH_MM_SS);
-        try {
-            emailParseService.parseEmail(emailInfoDTO, startDate, endDate);
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    @Test
-    public void testReparseEmail() {
-        emailParseApiService.reparseEmail(593);
-    }
-
-    @Test
-    public void testReparseFile() {
-        emailParseApiService.reparseFile(ListUtil.toList(40, 43));
-    }
-
-    @Test
-    public void testDateFormat() {
-        String input = "IXO6gLlm4QgiuvHZ";
-        BasicTextEncryptor textEncryptor = new BasicTextEncryptor();
-        textEncryptor.setPassword("qwertyuiopasdfghjklzxcvbnm1234567890qwertyuiopasdfghjklzxcvbnm12"); // 用您自己的密钥替换"yourSecretKey"
-        String encryptedString = textEncryptor.encrypt(input);
-        String decrypt = textEncryptor.decrypt(encryptedString);
-        System.out.println("加密后的字符串为: " + encryptedString);
-        System.out.println("解密后的字符串为: " + decrypt);
-    }
-
-    private MailboxInfoDTO buildMailbox(String account, String pwd) {
-        MailboxInfoDTO emailInfoDTO = new MailboxInfoDTO();
-        emailInfoDTO.setUserId(1);
-        emailInfoDTO.setAccount(account);
-        emailInfoDTO.setPassword(pwd);
-        emailInfoDTO.setHost("imap.exmail.qq.com");
-        emailInfoDTO.setPort("993");
-        emailInfoDTO.setProtocol("imap");
-        return emailInfoDTO;
-    }
-}
+//        return emailInfoDTO;
+//    }
+//}

service-manage/src/main/java/com/simuwang/manage/service/impl/EmailFundNavServiceImpl.java

@@ -61,7 +61,7 @@ public class EmailFundNavServiceImpl implements EmailFundNavService {
             List<NavDO> navDOList = dataList.stream().filter(e -> StrUtil.isNotBlank(e.getFundId()))
                     .map(e -> BeanUtil.copyProperties(e, NavDO.class)).collect(Collectors.toList());
             navDOList.forEach(e -> e.setUpdateTime(DateUtils.getNowDate()));
-            emailParseService.saveNavDo(navDOList);
+            emailParseService.saveNavDo(navDOList,null);
             //还原历史基金的净值数据
             if(StringUtil.isNotEmpty(sourceFundId)){
                 List<String> priceDateList = new ArrayList<>();
@@ -75,7 +75,7 @@ public class EmailFundNavServiceImpl implements EmailFundNavService {
                 List<NavDO> sourcenavDOList = sourceNavList.stream().filter(e -> StrUtil.isNotBlank(e.getFundId()))
                         .map(e -> BeanUtil.copyProperties(e, NavDO.class)).collect(Collectors.toList());
                 sourcenavDOList.forEach(e -> e.setUpdateTime(DateUtils.getNowDate()));
-                emailParseService.saveNavDo(sourcenavDOList);
+                emailParseService.saveNavDo(sourcenavDOList,null);
             }
         }
     }