Merge branch 'test' of http://112.74.196.215:3000/Tech2/data-daq into test

service-base/src/main/java/com/simuwang/base/common/enums/EmailCron.java

@@ -9,7 +9,7 @@ import java.util.stream.Stream;
  * Description: ${DESCRIPTION}
  */
 public enum EmailCron {
-    EVERY_HOUR(2, "0 0 0/1 * * ?"), ONE_AM(1, "0 0 1 * * ?");
+    EVERY_HOUR(2, "0 0 0/1 * * ?"), ONE_AM(1, "0 0 1 * * ?"), ONE_MINUTE(3, "0 0/1 * * * ?");
 
     private final Integer text;
     private final String cron;

service-base/src/main/resources/mapper/DeletionInfoMapper.xml

@@ -53,7 +53,7 @@
     </update>
     <select id="searchDeletionList" resultMap="BaseResultMap"
             parameterType="com.simuwang.base.pojo.dto.query.DeletionPageQuery">
-        SELECT
+        SELECT distinct
         d.fund_id,
         info.fund_name,
         c.company_name,
@@ -62,10 +62,10 @@
         ddp.processed_num
         FROM
         PPW_EMAIL.deletion_info d
-        JOIN PPW_EMAIL.pvn_fund_info info
-        ON d.fund_id = info.fund_id
-        JOIN PPW_EMAIL.pvn_company_info c
-        ON info.trust_id = c.company_id
+        LEFT JOIN PPW_EMAIL.pvn_fund_info info
+        ON d.fund_id = info.fund_id and info.isvalid=1
+        LEFT JOIN PPW_EMAIL.pvn_company_info c
+        ON info.trust_id = c.company_id and c.isvalid=1
         LEFT JOIN
         (SELECT
         di.fund_id,
@@ -93,8 +93,6 @@
         ON ddp.fund_id = d.fund_id
         AND d.deletion_type = ddp.deletion_type
         WHERE d.isvalid = 1
-        AND info.isvalid = 1
-        AND c.isvalid = 1
         <if test="companyName != null and companyName !=''">
             and (c.company_name like concat('%',#{companyName},'%') or c.company_short_name like concat('%',#{companyName},'%'))
         </if>
@@ -115,13 +113,19 @@
     </select>
     <select id="countDeletion" resultType="java.lang.Long"
             parameterType="com.simuwang.base.pojo.dto.query.DeletionPageQuery">
-        select count(1) from (select distinct d.fund_id,d.deletion_type
+        select count(1) from (select distinct
+        d.fund_id,
+        info.fund_name,
+        c.company_name,
+        d.deletion_type,
+        ddn.deletion_num,
+        ddp.processed_num
         FROM
         PPW_EMAIL.deletion_info d
-        JOIN PPW_EMAIL.pvn_fund_info info
-        ON d.fund_id = info.fund_id
-        JOIN PPW_EMAIL.pvn_company_info c
-        ON info.trust_id = c.company_id
+        LEFT JOIN PPW_EMAIL.pvn_fund_info info
+        ON d.fund_id = info.fund_id and info.isvalid=1
+        LEFT JOIN PPW_EMAIL.pvn_company_info c
+        ON info.trust_id = c.company_id and c.isvalid=1
         LEFT JOIN
         (SELECT
         di.fund_id,
@@ -149,8 +153,6 @@
         ON ddp.fund_id = d.fund_id
         AND d.deletion_type = ddp.deletion_type
         WHERE d.isvalid = 1
-        AND info.isvalid = 1
-        AND c.isvalid = 1
         <if test="companyName != null and companyName !=''">
             and (c.company_name like concat('%',#{companyName},'%') or c.company_short_name like concat('%',#{companyName},'%'))
         </if>
@@ -188,6 +190,7 @@
             parameterType="com.simuwang.base.pojo.dto.query.FundDeletionPageQuery">
         select id,fund_id,deletion_type,deletion_date,remark,isvalid,creatorid,updaterid,createtime,updatetime
         from PPW_EMAIL.deletion_info where isvalid=1 and fund_id=#{fundId} and deletion_type=#{deletionType}
+        order by deletion_date desc
         limit #{offset},#{pageSize}
     </select>
     <select id="countFundDeletionList" resultType="java.lang.Long"
@@ -205,6 +208,7 @@
         <if test="deletionType != null">
             and deletion_type=#{deletionType}
         </if>
+        order by fund_id desc,deletion_type desc ,deletion_date desc
     </select>
     <select id="getDeletionInfoDO" resultType="com.simuwang.base.pojo.dos.DeletionInfoDO"
             parameterType="com.simuwang.base.pojo.dos.DeletionInfoDO">

service-base/src/main/resources/mapper/EmailFundNavMapper.xml

@@ -147,7 +147,7 @@
         <if test="emailTitle != null and emailTitle != ''">
             and parse.email_title like concat('%',#{emailTitle},'%')
         </if>
-        <if test="emailId != null">
+        <if test="emailId != null and emailId != ''">
             and parse.id=#{emailId}
         </if>
         ) a

service-deploy/src/main/resources/application.yml

@@ -62,7 +62,6 @@ spring:
           auth: true
           ssl:
             enable: true
-
 email-rule:
   nav: "净值,业绩报酬,规模"
   valuation: "估值表"
@@ -101,25 +100,3 @@ simuwang:
     - path: "/**"
       filters: anon
 
-quartz:
-  #quartz相关属性配置
-  org:
-    quartz:
-      scheduler:
-        instanceName: DefaultQuartzScheduler #调度器的实例名
-        instanceId: AUTO #调度器编号自动生成
-      jobStore:
-        class: org.quartz.impl.jdbcjobstore.JobStoreTX
-        driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate
-        tablePrefix: database.QRTZ_  #数据库表名前缀
-        isClustered: true #开启分布式部署
-        clusterCheckinInterval: 10000 #分布式节点有效性检查时间间隔,单位:秒
-        useProperties: false #是否读取自己写的配置文件
-        dataSource: qzds
-      dataSource:
-        qzds:
-          driver: dm.jdbc.driver.DmDriver #达梦数据库驱动
-          url: jdbc:dm://192.168.1.39:5236/PPW_EMAIL?zeroDateTimeBehavior=convertToNull&useUnicode=true&characterEncoding=utf-8&clobAsString=true
-          user: PPW_EMAIL
-          password: Dmppw2024
-          validationQuery: select 0 from dual

service-manage/src/main/java/com/simuwang/manage/api/navAsset/FundNavAssetController.java

@@ -70,7 +70,7 @@ public class FundNavAssetController {
     }
 
     /**
-     * 上传分红信息
+     * 上传规模信息
      * @param file
      * @param response
      * @param request

service-manage/src/main/java/com/simuwang/manage/init/QuartzInitConfig.java

@@ -1,84 +1,84 @@
-package com.simuwang.manage.init;
-
-import cn.hutool.crypto.asymmetric.KeyType;
-import cn.hutool.crypto.asymmetric.RSA;
-import com.alibaba.fastjson2.JSON;
-import com.simuwang.base.common.enums.OpenStatusType;
-import com.simuwang.base.common.util.DateUtils;
-import com.simuwang.base.common.util.QuartzUtils;
-import com.simuwang.base.config.DaqProperties;
-import com.simuwang.base.pojo.dos.MailboxInfoDO;
-import com.simuwang.base.pojo.dto.MailboxInfoDTO;
-import com.simuwang.base.pojo.dto.QuartzBean;
-import com.simuwang.manage.service.EmailConfigService;
-import org.quartz.Scheduler;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.boot.ApplicationArguments;
-import org.springframework.boot.ApplicationRunner;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.core.annotation.Order;
-
-import java.util.List;
-
-/**
- * 定时任务启动类
- * Author:   chenjianhua
- * Date:     2024/9/17 11:55
- * Description: ${DESCRIPTION}
- */
-@Configuration
-@Order(2)
-public class QuartzInitConfig implements ApplicationRunner {
-    @Autowired
-    private Scheduler scheduler;
-    private final Logger logger = LoggerFactory.getLogger(this.getClass());
-
-    @Autowired
-    private EmailConfigService emailConfigService;
-
-    @Value("${spring.task.groupName}")
-    private String groupName;
-
-    @Autowired
-    private DaqProperties properties;
-    private String JOB_CLASS="com.simuwang.manage.task.ParseSchedulerTask";
-
-    @Override
-    public void run(ApplicationArguments args){
-        List<MailboxInfoDO> mailboxInfoDOS = emailConfigService.getAll();
-        for(MailboxInfoDO mailboxInfoDO : mailboxInfoDOS){
-            try{
-                QuartzBean quartzBean = new QuartzBean();
-                quartzBean.setCronExpression(mailboxInfoDO.getCron());
-                quartzBean.setStatus(mailboxInfoDO.getOpenStatus());
-                quartzBean.setGroupName(groupName);
-                quartzBean.setJobName(mailboxInfoDO.getEmail());
-                quartzBean.setJobClass(JOB_CLASS);
-                //请求参数
-                MailboxInfoDTO paramDTO = new MailboxInfoDTO();
-                paramDTO.setAccount(mailboxInfoDO.getEmail());
-                String password = mailboxInfoDO.getPassword();
-                try{
-                    String privateKey = this.properties.getSecurityRsa().getPrivateKey();
-                    password = new RSA(privateKey, null).decryptStr(password, KeyType.PrivateKey);
-                }catch (Exception e){
-                    logger.error(e.getMessage(),e);
-                }
-                paramDTO.setPassword(password);
-                paramDTO.setPort(mailboxInfoDO.getPort());
-                paramDTO.setHost(mailboxInfoDO.getServer());
-                paramDTO.setProtocol(mailboxInfoDO.getProtocol());
-                quartzBean.setJobParam(JSON.toJSONString(paramDTO));
-                if(mailboxInfoDO.getOpenStatus() != null && mailboxInfoDO.getOpenStatus().equals(OpenStatusType.YES.getCode())){
-                    logger.info(quartzBean.getJobName()+":创建定时任务,cron:"+quartzBean.getCronExpression()+"--"+ DateUtils.getNowDate());
-                    QuartzUtils.createScheduleJob(scheduler,quartzBean);
-                }
-            }catch (Exception e) {
-                logger.error(e.getMessage(),e);
-            }
-        }
-    }
-}
+//package com.simuwang.manage.init;
+//
+//import cn.hutool.crypto.asymmetric.KeyType;
+//import cn.hutool.crypto.asymmetric.RSA;
+//import com.alibaba.fastjson2.JSON;
+//import com.simuwang.base.common.enums.OpenStatusType;
+//import com.simuwang.base.common.util.DateUtils;
+//import com.simuwang.base.common.util.QuartzUtils;
+//import com.simuwang.base.config.DaqProperties;
+//import com.simuwang.base.pojo.dos.MailboxInfoDO;
+//import com.simuwang.base.pojo.dto.MailboxInfoDTO;
+//import com.simuwang.base.pojo.dto.QuartzBean;
+//import com.simuwang.manage.service.EmailConfigService;
+//import org.quartz.Scheduler;
+//import org.slf4j.Logger;
+//import org.slf4j.LoggerFactory;
+//import org.springframework.beans.factory.annotation.Autowired;
+//import org.springframework.beans.factory.annotation.Value;
+//import org.springframework.boot.ApplicationArguments;
+//import org.springframework.boot.ApplicationRunner;
+//import org.springframework.context.annotation.Configuration;
+//import org.springframework.core.annotation.Order;
+//
+//import java.util.List;
+//
+///**
+// * 定时任务启动类
+// * Author:   chenjianhua
+// * Date:     2024/9/17 11:55
+// * Description: ${DESCRIPTION}
+// */
+//@Configuration
+//@Order(2)
+//public class QuartzInitConfig implements ApplicationRunner {
+//    @Autowired
+//    private Scheduler scheduler;
+//    private final Logger logger = LoggerFactory.getLogger(this.getClass());
+//
+//    @Autowired
+//    private EmailConfigService emailConfigService;
+//
+//    @Value("${spring.task.groupName}")
+//    private String groupName;
+//
+//    @Autowired
+//    private DaqProperties properties;
+//    private String JOB_CLASS="com.simuwang.manage.task.ParseSchedulerTask";
+//
+//    @Override
+//    public void run(ApplicationArguments args){
+//        List<MailboxInfoDO> mailboxInfoDOS = emailConfigService.getAll();
+//        for(MailboxInfoDO mailboxInfoDO : mailboxInfoDOS){
+//            try{
+//                QuartzBean quartzBean = new QuartzBean();
+//                quartzBean.setCronExpression(mailboxInfoDO.getCron());
+//                quartzBean.setStatus(mailboxInfoDO.getOpenStatus());
+//                quartzBean.setGroupName(groupName);
+//                quartzBean.setJobName(mailboxInfoDO.getEmail());
+//                quartzBean.setJobClass(JOB_CLASS);
+//                //请求参数
+//                MailboxInfoDTO paramDTO = new MailboxInfoDTO();
+//                paramDTO.setAccount(mailboxInfoDO.getEmail());
+//                String password = mailboxInfoDO.getPassword();
+//                try{
+//                    String privateKey = this.properties.getSecurityRsa().getPrivateKey();
+//                    password = new RSA(privateKey, null).decryptStr(password, KeyType.PrivateKey);
+//                }catch (Exception e){
+//                    logger.error(e.getMessage(),e);
+//                }
+//                paramDTO.setPassword(password);
+//                paramDTO.setPort(mailboxInfoDO.getPort());
+//                paramDTO.setHost(mailboxInfoDO.getServer());
+//                paramDTO.setProtocol(mailboxInfoDO.getProtocol());
+//                quartzBean.setJobParam(JSON.toJSONString(paramDTO));
+//                if(mailboxInfoDO.getOpenStatus() != null && mailboxInfoDO.getOpenStatus().equals(OpenStatusType.YES.getCode())){
+//                    logger.info(quartzBean.getJobName()+":创建定时任务,cron:"+quartzBean.getCronExpression()+"--"+ DateUtils.getNowDate());
+//                    QuartzUtils.createScheduleJob(scheduler,quartzBean);
+//                }
+//            }catch (Exception e) {
+//                logger.error(e.getMessage(),e);
+//            }
+//        }
+//    }
+//}

service-manage/src/main/java/com/simuwang/manage/init/ScheduleConfig.java

@@ -0,0 +1,53 @@
+package com.simuwang.manage.init;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.quartz.SchedulerFactoryBean;
+import javax.sql.DataSource;
+import java.util.Properties;
+
+/**
+ * 定时任务配置
+ * FileName: SchedulerConfig
+ * Author:   chenjianhua
+ * Date:     2024/9/22 14:39
+ * Description: ${DESCRIPTION}
+ */
+@Configuration
+public class ScheduleConfig {
+    @Bean
+    public SchedulerFactoryBean schedulerFactoryBean(DataSource dataSource) {
+        SchedulerFactoryBean factory = new SchedulerFactoryBean();
+        factory.setDataSource(dataSource);
+        // quartz参数
+        Properties prop = new Properties();
+        prop.put("org.quartz.scheduler.instanceName", "scheduler");
+        prop.put("org.quartz.scheduler.instanceId", "AUTO");
+        // 线程池配置
+        prop.put("org.quartz.threadPool.class", "org.quartz.simpl.SimpleThreadPool");
+        prop.put("org.quartz.threadPool.threadCount", "20");
+        prop.put("org.quartz.threadPool.threadPriority", "5");
+        // JobStore配置
+        prop.put("org.quartz.jobStore.class", "org.springframework.scheduling.quartz.LocalDataSourceJobStore");
+        // 集群配置
+        prop.put("org.quartz.jobStore.isClustered", "true");
+        prop.put("org.quartz.jobStore.clusterCheckinInterval", "15000");
+        prop.put("org.quartz.jobStore.maxMisfiresToHandleAtATime", "1");
+        prop.put("org.quartz.jobStore.txIsolationLevelSerializable", "true");
+        prop.put("org.quartz.jobStore.misfireThreshold", "12000");
+        prop.put("org.quartz.jobStore.tablePrefix", "QRTZ_");
+        factory.setQuartzProperties(prop);
+
+        factory.setSchedulerName("Scheduler");
+        // 延时启动
+        factory.setStartupDelay(1);
+        factory.setApplicationContextSchedulerContextKey("applicationContextKey");
+        // 可选，QuartzScheduler
+        // 启动时更新己存在的Job，这样就不用每次修改targetObject后删除qrtz_job_details表对应记录了
+        factory.setOverwriteExistingJobs(true);
+        // 设置自动启动，默认为true
+        factory.setAutoStartup(true);
+        return factory;
+    }
+}
+

service-manage/src/main/java/com/simuwang/manage/task/FundDeletionTask.java

@@ -47,7 +47,7 @@ public class FundDeletionTask {
 
     @Autowired
     private DistributionMapper distributionMapper;
-    @Scheduled(cron = "0 35 5,17,18 * * ?")
+    @Scheduled(cron = "0 30 5,17,19 * * ?")
     public void computeDeletion(){
         List<String> fundIdList = navMapper.getAllFundId();
         for(String fundId : fundIdList){
@@ -164,7 +164,7 @@ public class FundDeletionTask {
                     tradeDate = DateUtils.format(tradeDateDOS.get(tradeDateDOS.size()-1).getTradeDate(),DateUtils.YYYY_MM_DD);
                 }
                 //写入缺失信息表
-                saveDeletionInfoDO(fundId,tradeDate,DeletionType.NAV_DELETION.getCode());
+                saveDeletionInfoDO(fundId,tradeDate,DeletionType.ASSET_DELETION.getCode());
             }
         }
         if(Frequency.MONTH == Frequency.getFrequencyByCode(fundReportFrequencyDO.getAssetFrequency())){

service-manage/src/main/java/com/simuwang/manage/task/ParseSchedulerTask.java

@@ -3,6 +3,7 @@ package com.simuwang.manage.task;
 import ch.qos.logback.core.util.StringUtil;
 import cn.hutool.crypto.asymmetric.KeyType;
 import cn.hutool.crypto.asymmetric.RSA;
+import cn.hutool.extra.spring.SpringUtil;
 import com.alibaba.fastjson2.JSON;
 import com.simuwang.base.common.enums.EmailCron;
 import com.simuwang.base.common.util.DateUtils;
@@ -31,15 +32,18 @@ import java.util.Date;
  * Date:     2024/9/17 10:48
  * Description: ${DESCRIPTION}
  */
+@Component
 public class ParseSchedulerTask  extends QuartzJobBean {
 
-    @Autowired
+
     private EmailParseApiService emailParseApiService;
-    @Autowired
+
     private EmailConfigService emailConfigService;
     private static final Logger log = LoggerFactory.getLogger(ParseSchedulerTask.class);
     @Override
     protected void executeInternal(JobExecutionContext context) throws JobExecutionException {
+        emailParseApiService = SpringUtil.getBean(EmailParseApiService.class);
+        emailConfigService = SpringUtil.getBean(EmailConfigService.class);
         log.info("ParseSchedulerTask {}", context.getJobDetail().getKey());JobDataMap jobDataMap = context.getJobDetail().getJobDataMap();
         log.info("ParseSchedulerTask jobDataMap {}", JSON.toJSONString(jobDataMap));
         String account = jobDataMap.get("account").toString();
@@ -60,7 +64,7 @@ public class ParseSchedulerTask  extends QuartzJobBean {
             if(EmailCron.getEmailCronByCron(mailboxInfoDO.getCron()) == EmailCron.EVERY_HOUR){
                 endDate = DateUtils.getNowDate();
                 startDate = DateUtils.addHours(DateUtils.getNowDate(),-1);
-            }else if (EmailCron.getEmailCronByCron(mailboxInfoDO.getCron()) == EmailCron.ONE_AM){
+            }else{
                 endDate = DateUtils.getNowDate();
                 startDate = DateUtils.addHours(DateUtils.getNowDate(),-24);
             }