Merge branch 'develop' of http://112.74.196.215:3000/Tech2/data-daq into develop

service-base/src/main/java/com/simuwang/base/common/conts/ApplicationRuleFileConst.java

@@ -0,0 +1,16 @@
+package com.simuwang.base.common.conts;
+
+public class ApplicationRuleFileConst {
+    /**
+     * 邮件标题
+     */
+    public final static Integer EMAIL_TITLE_FILE = 1;
+    /**
+     * 附件名称
+     */
+    public final static Integer EMAIL_FILE_NAME_FILE = 2;
+    /**
+     * 表格内容
+     */
+    public final static Integer EMAIL_EXCEL_CONTENT_FILE = 3;
+}

service-base/src/main/java/com/simuwang/base/common/util/ExcelUtil.java

@@ -2,6 +2,7 @@ package com.simuwang.base.common.util;
 
 import cn.hutool.core.collection.CollUtil;
 import cn.hutool.core.date.DateUtil;
+import cn.hutool.core.exceptions.ExceptionUtil;
 import cn.hutool.core.util.StrUtil;
 import com.simuwang.base.common.conts.DateConst;
 import org.apache.poi.hssf.usermodel.*;
@@ -324,4 +325,13 @@ public class ExcelUtil {
         return wb;
     }
 
+    public static Sheet getFirstSheet(String filePath) {
+        try {
+            File file = new File(filePath);
+            return ExcelUtil.getSheet(file, 0);
+        }catch (Exception e) {
+            logger.error("获取文件的sheet错误 -> 文件路径:{},堆栈信息为：{}",filePath, ExceptionUtil.stacktraceToString(e));
+        }
+        return null;
+    }
 }

service-base/src/main/java/com/simuwang/base/mapper/EmailTemplateApplicationRuleMapper.java

@@ -0,0 +1,13 @@
+package com.simuwang.base.mapper;
+
+import com.simuwang.base.pojo.dos.EmailTemplateApplicationRuleDO;
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
+
+@Mapper
+public interface EmailTemplateApplicationRuleMapper {
+
+    List<EmailTemplateApplicationRuleDO> queryByTemplateId(@Param("templateIdList") List<Integer> templateIdList);
+}

service-base/src/main/java/com/simuwang/base/mapper/EmailTemplateDataRuleMapper.java

@@ -0,0 +1,14 @@
+package com.simuwang.base.mapper;
+
+import com.simuwang.base.pojo.dos.EmailTemplateDataRuleDO;
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
+
+@Mapper
+public interface EmailTemplateDataRuleMapper {
+
+    List<EmailTemplateDataRuleDO> queryByTemplateId(@Param("templateIdList") List<Integer> templateIdList);
+
+}

service-base/src/main/java/com/simuwang/base/mapper/EmailTemplateInfoMapper.java

@@ -0,0 +1,8 @@
+package com.simuwang.base.mapper;
+
+import org.apache.ibatis.annotations.Mapper;
+
+@Mapper
+public interface EmailTemplateInfoMapper {
+
+}

service-base/src/main/java/com/simuwang/base/mapper/EmailTemplateMappingMapper.java

@@ -0,0 +1,14 @@
+package com.simuwang.base.mapper;
+
+import com.simuwang.base.pojo.dos.EmailTemplateInfoDO;
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
+
+@Mapper
+public interface EmailTemplateMappingMapper {
+
+    List<EmailTemplateInfoDO> queryByEmail(@Param("email") String email);
+
+}

service-base/src/main/java/com/simuwang/base/pojo/dos/EmailTemplateApplicationRuleDO.java

@@ -0,0 +1,73 @@
+package com.simuwang.base.pojo.dos;
+
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+import java.util.Date;
+
+@Data
+@TableName("email_template_application_rule")
+public class EmailTemplateApplicationRuleDO {
+    /**
+     * 主键Id
+     */
+    @TableId(value = "id")
+    private Integer id;
+    /**
+     * 模版id(email_template_info.id)
+     */
+    @TableId(value = "template_id")
+    private Integer templateId;
+    /**
+     * 字段类型：1-邮件标题，2-附件名称，3-表格内容
+     */
+    @TableId(value = "type")
+    private Integer type;
+    /**
+     * 表格行
+     */
+    @TableId(value = "row")
+    private Integer row;
+    /**
+     * 表格列
+     */
+    @TableId(value = "column")
+    private String column;
+    /**
+     * 包含关键字(多个以英文逗号隔开)
+     */
+    @TableId(value = "contain_keyword")
+    private String containKeyword;
+    /**
+     * 不包含关键字(多个以英文逗号隔开)
+     */
+    @TableId(value = "not_contain_keyword")
+    private String notContainKeyword;
+    /**
+     * 是否有效：0-无效，1-有效
+     */
+    @TableField(value = "isvalid")
+    private Integer isvalid;
+    /**
+     * 创建者Id
+     */
+    @TableField(value = "creatorid")
+    private Integer creatorId;
+    /**
+     * 创建时间
+     */
+    @TableField(value = "createtime")
+    private Date createTime;
+    /**
+     * 修改者Id
+     */
+    @TableField(value = "updaterid")
+    private Integer updaterId;
+    /**
+     * 更新时间
+     */
+    @TableField(value = "updatetime")
+    private Date updateTime;
+}

service-base/src/main/java/com/simuwang/base/pojo/dos/EmailTemplateDataRuleDO.java

@@ -0,0 +1,68 @@
+package com.simuwang.base.pojo.dos;
+
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+import java.util.Date;
+
+@Data
+@TableName("email_template_data_rule")
+public class EmailTemplateDataRuleDO {
+    /**
+     * 主键Id
+     */
+    @TableId(value = "id")
+    private Integer id;
+    /**
+     * 模版id(email_template_info.id)
+     */
+    @TableId(value = "template_id")
+    private Integer templateId;
+    /**
+     * 字段名称：1-备案编码，2-基金名称，3-净值日期，4-单位净值，5-累计净值，6-资产份额，7-资产净值
+     */
+    @TableId(value = "field_name")
+    private Integer fieldName;
+    /**
+     * 表格行
+     */
+    @TableId(value = "row")
+    private Integer row;
+    /**
+     * 表格列
+     */
+    @TableId(value = "column")
+    private String column;
+    /**
+     * 提取规则
+     */
+    @TableId(value = "field_rule")
+    private Integer fieldRule;
+    /**
+     * 是否有效：0-无效，1-有效
+     */
+    @TableField(value = "isvalid")
+    private Integer isvalid;
+    /**
+     * 创建者Id
+     */
+    @TableField(value = "creatorid")
+    private Integer creatorId;
+    /**
+     * 创建时间
+     */
+    @TableField(value = "createtime")
+    private Date createTime;
+    /**
+     * 修改者Id
+     */
+    @TableField(value = "updaterid")
+    private Integer updaterId;
+    /**
+     * 更新时间
+     */
+    @TableField(value = "updatetime")
+    private Date updateTime;
+}

service-base/src/main/java/com/simuwang/base/pojo/dos/EmailTemplateInfoDO.java

@@ -0,0 +1,63 @@
+package com.simuwang.base.pojo.dos;
+
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+import java.util.Date;
+
+@Data
+@TableName("email_template_info")
+public class EmailTemplateInfoDO {
+    /**
+     * 主键Id
+     */
+    @TableId(value = "id")
+    private Integer id;
+    /**
+     * 模板名称
+     */
+    @TableId(value = "name")
+    private String name;
+    /**
+     * 模版方向：1-行，2-列
+     */
+    @TableId(value = "direction")
+    private Integer direction;
+    /**
+     * 模版备注
+     */
+    @TableId(value = "description")
+    private String description;
+    /**
+     * 状态：0-关闭，1-开启
+     */
+    @TableId(value = "status")
+    private Integer status;
+    /**
+     * 是否有效：0-无效，1-有效
+     */
+    @TableField(value = "isvalid")
+    private Integer isvalid;
+    /**
+     * 创建者Id
+     */
+    @TableField(value = "creatorid")
+    private Integer creatorId;
+    /**
+     * 创建时间
+     */
+    @TableField(value = "createtime")
+    private Date createTime;
+    /**
+     * 修改者Id
+     */
+    @TableField(value = "updaterid")
+    private Integer updaterId;
+    /**
+     * 更新时间
+     */
+    @TableField(value = "updatetime")
+    private Date updateTime;
+}

service-base/src/main/java/com/simuwang/base/pojo/dos/EmailTemplateMappingDO.java

@@ -0,0 +1,73 @@
+package com.simuwang.base.pojo.dos;
+
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+import java.util.Date;
+
+@Data
+@TableName("email_template_mapping")
+public class EmailTemplateMappingDO {
+    /**
+     * 主键Id
+     */
+    @TableId(value = "id")
+    private Integer id;
+    /**
+     * 模版id(email_template_info.id)
+     */
+    @TableId(value = "template_id")
+    private Integer templateId;
+    /**
+     * 标题名称
+     */
+    @TableId(value = "title")
+    private String title;
+    /**
+     * 邮箱地址
+     */
+    @TableId(value = "email")
+    private String email;
+    /**
+     * 公司名称
+     */
+    @TableId(value = "company_name")
+    private String companyName;
+    /**
+     * 备注
+     */
+    @TableId(value = "description")
+    private String description;
+    /**
+     * 状态：0-关闭，1-开启
+     */
+    @TableId(value = "status")
+    private Integer status;
+    /**
+     * 是否有效：0-无效，1-有效
+     */
+    @TableField(value = "isvalid")
+    private Integer isvalid;
+    /**
+     * 创建者Id
+     */
+    @TableField(value = "creatorid")
+    private Integer creatorId;
+    /**
+     * 创建时间
+     */
+    @TableField(value = "createtime")
+    private Date createTime;
+    /**
+     * 修改者Id
+     */
+    @TableField(value = "updaterid")
+    private Integer updaterId;
+    /**
+     * 更新时间
+     */
+    @TableField(value = "updatetime")
+    private Date updateTime;
+}

service-base/src/main/java/com/simuwang/base/pojo/dto/EmailContentInfoDTO.java

@@ -4,9 +4,8 @@ import lombok.Data;
 
 import java.io.Serializable;
 
-
 @Data
-public class EmailContentInfoDTO implements Serializable, Cloneable {
+public class EmailContentInfoDTO implements Serializable {
 
     private static final long serialVersionUID = 202104140906313753L;
 
@@ -62,6 +61,6 @@ public class EmailContentInfoDTO implements Serializable, Cloneable {
     /**
      * 文件id(重新解析邮件功能)
      */
-    private Integer fielId;
+    private Integer fileId;
 }
 

service-base/src/main/java/com/simuwang/base/pojo/dto/TemplateDetailDTO.java

@@ -0,0 +1,24 @@
+package com.simuwang.base.pojo.dto;
+
+import cn.hutool.core.lang.Pair;
+import lombok.Data;
+
+import java.util.Map;
+
+@Data
+public class TemplateDetailDTO {
+
+    /**
+     * 模板idi
+     */
+    private Integer templateId;
+    /**
+     * 循环方向：1-行，2-列
+     */
+    private Integer direction;
+
+    /**
+     * 字段所在表格中的位置
+     */
+    private Map<String, Pair<Integer, Integer>> fieldPositionMap;
+}

service-base/src/main/resources/mapper/EmailTemplateApplicationRuleMapper.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.simuwang.base.mapper.EmailTemplateApplicationRuleMapper">
+    <resultMap id="BaseResultMap" type="com.simuwang.base.pojo.dos.EmailTemplateApplicationRuleDO">
+        <id column="id" property="id"/>
+        <result column="template_id" property="templateId"/>
+        <result column="type" property="type"/>
+        <result column="row" property="row"/>
+        <result column="column" property="column"/>
+        <result column="contain_keyword" property="containKeyword"/>
+        <result column="not_contain_keyword" property="notContainKeyword"/>
+        <result column="isvalid" property="isvalid"/>
+        <result column="creatorid" property="creatorId"/>
+        <result column="createtime" property="createTime"/>
+        <result column="updaterid" property="updaterId"/>
+        <result column="updatetime" property="updateTime"/>
+    </resultMap>
+
+    <select id="queryByTemplateId" resultMap="BaseResultMap">
+        select *
+        from PPW_EMAIL.email_template_application_rule
+        where isvalid = 1
+        and template_id in
+        <foreach collection="templateIdList" index="index" item="templateId" open="(" separator="," close=")">
+            #{templateId}
+        </foreach>
+    </select>
+</mapper>

service-base/src/main/resources/mapper/EmailTemplateDataRuleMapper.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.simuwang.base.mapper.EmailTemplateDataRuleMapper">
+    <resultMap id="BaseResultMap" type="com.simuwang.base.pojo.dos.EmailTemplateDataRuleDO">
+        <id column="id" property="id"/>
+        <result column="template_id" property="templateId"/>
+        <result column="field_name" property="fieldName"/>
+        <result column="row" property="row"/>
+        <result column="column" property="column"/>
+        <result column="field_rule" property="fieldRule"/>
+        <result column="isvalid" property="isvalid"/>
+        <result column="creatorid" property="creatorId"/>
+        <result column="createtime" property="createTime"/>
+        <result column="updaterid" property="updaterId"/>
+        <result column="updatetime" property="updateTime"/>
+    </resultMap>
+
+    <select id="queryByTemplateId" resultMap="BaseResultMap">
+        select *
+        from PPW_EMAIL.email_template_data_rule
+        where isvalid = 1
+        and template_id in
+        <foreach collection="templateIdList" index="index" item="templateId" open="(" separator="," close=")">
+            #{templateId}
+        </foreach>
+    </select>
+</mapper>

service-base/src/main/resources/mapper/EmailTemplateInfoMapper.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.simuwang.base.mapper.EmailTemplateInfoMapper">
+    <resultMap id="BaseResultMap" type="com.simuwang.base.pojo.dos.EmailTemplateInfoDO">
+        <id column="id" property="id"/>
+        <result column="name" property="name"/>
+        <result column="direction" property="direction"/>
+        <result column="description" property="description"/>
+        <result column="status" property="status"/>
+        <result column="isvalid" property="isvalid"/>
+        <result column="creatorid" property="creatorId"/>
+        <result column="createtime" property="createTime"/>
+        <result column="updaterid" property="updaterId"/>
+        <result column="updatetime" property="updateTime"/>
+    </resultMap>
+
+
+</mapper>

service-base/src/main/resources/mapper/EmailTemplateMappingMapper.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.simuwang.base.mapper.EmailTemplateMappingMapper">
+    <resultMap id="BaseResultMap" type="com.simuwang.base.pojo.dos.EmailTemplateMappingDO">
+        <id column="id" property="id"/>
+        <result column="template_id" property="templateId"/>
+        <result column="title" property="title"/>
+        <result column="email" property="email"/>
+        <result column="company_name" property="companyName"/>
+        <result column="description" property="description"/>
+        <result column="status" property="status"/>
+        <result column="isvalid" property="isvalid"/>
+        <result column="creatorid" property="creatorId"/>
+        <result column="createtime" property="createTime"/>
+        <result column="updaterid" property="updaterId"/>
+        <result column="updatetime" property="updateTime"/>
+    </resultMap>
+
+    <select id="queryByEmail" resultType="com.simuwang.base.pojo.dos.EmailTemplateInfoDO">
+        select t1.template_id as id,
+               t2.direction as direction
+        from PPW_EMAIL.email_template_mapping t1
+            join PPW_EMAIL.email_template_info t2 on t2.id = t1.template_id and t2.isvalid = 1
+        where t1.email = #{email}
+          and t1.isvalid = 1
+          and t2.status = 1
+    </select>
+
+</mapper>

service-daq/src/main/java/com/simuwang/daq/service/EmailParseApiServiceImpl.java

@@ -116,12 +116,12 @@ public class EmailParseApiServiceImpl implements EmailParseApiService {
                 Map<EmailContentInfoDTO, List<EmailFundNavDTO>> fileNameNavMap = MapUtil.newHashMap();
                 for (EmailContentInfoDTO emailContentInfoDTO : emailContentInfoDTOList) {
                     try {
-                        log.info("开始重新解析文件 -> 文件id:{}", emailContentInfoDTO.getFielId());
+                        log.info("开始重新解析文件 -> 文件id:{}", emailContentInfoDTO.getFileId());
                         List<EmailFundNavDTO> fundNavDTOList = emailParseService.parseEmail(emailContentInfoDTO, MapUtil.newHashMap());
                         fileNameNavMap.put(emailContentInfoDTO, fundNavDTOList);
                         emailFundNavDTOList.addAll(fundNavDTOList);
                     } catch (Exception e) {
-                        log.error("重新解析文件失败,邮件id:{},文件id:{},堆栈信息:{}", emailId, emailContentInfoDTO.getFielId(), ExceptionUtil.stacktraceToString(e));
+                        log.error("重新解析文件失败,邮件id:{},文件id:{},堆栈信息:{}", emailId, emailContentInfoDTO.getFileId(), ExceptionUtil.stacktraceToString(e));
                     }
                 }
                 // 保存相关信息 -> 邮件信息表,邮件文件表,邮件净值表,邮件规模表,基金净值表
@@ -136,7 +136,7 @@ public class EmailParseApiServiceImpl implements EmailParseApiService {
         String parseDate = DateUtil.format(new Date(), DateConst.YYYY_MM_DD_HH_MM_SS);
         EmailContentInfoDTO contentInfoDTO = new EmailContentInfoDTO();
         contentInfoDTO.setEmailId(emailInfoDTO.getId());
-        contentInfoDTO.setFielId(emailInfoDTO.getFileId());
+        contentInfoDTO.setFileId(emailInfoDTO.getFileId());
         contentInfoDTO.setEmailAddress(emailInfoDTO.getEmail());
         contentInfoDTO.setEmailDate(emailDate);
         contentInfoDTO.setEmailTitle(emailInfoDTO.getEmailTitle());
@@ -154,7 +154,7 @@ public class EmailParseApiServiceImpl implements EmailParseApiService {
         for (EmailFileInfoDO fileInfoDO : emailFileInfoDOList) {
             EmailContentInfoDTO contentInfoDTO = new EmailContentInfoDTO();
             contentInfoDTO.setEmailId(emailId);
-            contentInfoDTO.setFielId(fileInfoDO.getId());
+            contentInfoDTO.setFileId(fileInfoDO.getId());
             contentInfoDTO.setEmailAddress(emailParseInfoDO.getEmail());
             contentInfoDTO.setEmailDate(emailDate);
             contentInfoDTO.setEmailTitle(emailParseInfoDO.getEmailTitle());

service-daq/src/main/java/com/simuwang/daq/service/EmailParseService.java

@@ -145,7 +145,7 @@ public class EmailParseService {
         for (Map.Entry<EmailContentInfoDTO, List<EmailFundNavDTO>> fileNameNavEntry : fileNameNavMap.entrySet()) {
             // 保存邮件文件表
             EmailContentInfoDTO emailContentInfoDTO = fileNameNavEntry.getKey();
-            Integer fileId = saveEmailFileInfo(emailId, emailContentInfoDTO.getFielId(), emailContentInfoDTO.getFileName(), emailContentInfoDTO.getFilePath(), parseDate);
+            Integer fileId = saveEmailFileInfo(emailId, emailContentInfoDTO.getFileId(), emailContentInfoDTO.getFileName(), emailContentInfoDTO.getFilePath(), parseDate);
 
             List<EmailFundNavDTO> fundNavDTOList = fileNameNavEntry.getValue();
             if (CollUtil.isEmpty(fundNavDTOList)) {
@@ -184,7 +184,7 @@ public class EmailParseService {
         List<EmailFundAssetDO> emailFundAssetDOList = fundNavDTOList.stream()
                 .map(e -> buildEmailFundAssetDo(fileId, e, parseDate)).filter(CollUtil::isNotEmpty).flatMap(List::stream).collect(Collectors.toList());
         if (CollUtil.isNotEmpty(emailFundAssetDOList)) {
-            // 先删除file_id下的规模数据(考虑到重新解析的需求,如果是首次解析,那么fiel_id下不存在规模数据)
+            // 先删除file_id下的规模数据(考虑到重新解析的需求,如果是首次解析,那么file_id下不存在规模数据)
             emailFundAssetMapper.deleteByFileId(fileId);
             emailFundAssetMapper.batchInsert(emailFundAssetDOList);
             List<AssetDO> assetDOList = emailFundAssetDOList.stream().filter(e -> StrUtil.isNotBlank(e.getFundId()))

service-daq/src/main/java/com/simuwang/daq/service/EmailTemplateService.java

@@ -0,0 +1,187 @@
+package com.simuwang.daq.service;
+
+import cn.hutool.core.collection.CollUtil;
+import cn.hutool.core.lang.Pair;
+import cn.hutool.core.util.StrUtil;
+import com.simuwang.base.common.conts.ApplicationRuleFileConst;
+import com.simuwang.base.common.conts.EmailFieldConst;
+import com.simuwang.base.common.util.ExcelUtil;
+import com.simuwang.base.mapper.EmailTemplateApplicationRuleMapper;
+import com.simuwang.base.mapper.EmailTemplateDataRuleMapper;
+import com.simuwang.base.mapper.EmailTemplateMappingMapper;
+import com.simuwang.base.pojo.dos.EmailTemplateApplicationRuleDO;
+import com.simuwang.base.pojo.dos.EmailTemplateDataRuleDO;
+import com.simuwang.base.pojo.dos.EmailTemplateInfoDO;
+import com.simuwang.base.pojo.dto.EmailContentInfoDTO;
+import com.simuwang.base.pojo.dto.TemplateDetailDTO;
+import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.Row;
+import org.apache.poi.ss.usermodel.Sheet;
+import org.springframework.stereotype.Service;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@Service
+public class EmailTemplateService {
+
+    private final EmailTemplateMappingMapper emailTemplateMappingMapper;
+    private final EmailTemplateApplicationRuleMapper emailTemplateApplicationRuleMapper;
+    private final EmailTemplateDataRuleMapper emailTemplateDataRuleMapper;
+    private final static Map<Integer, String> FILE_TYPE_MAP = new HashMap<>();
+
+    static {
+        // 1-备案编码，2-基金名称，3-净值日期，4-单位净值，5-累计净值，6-资产份额，7-资产净值
+        FILE_TYPE_MAP.put(1, EmailFieldConst.REGISTER_NUMBER);
+        FILE_TYPE_MAP.put(2, EmailFieldConst.FUND_NAME);
+        FILE_TYPE_MAP.put(3, EmailFieldConst.PRICE_DATE);
+        FILE_TYPE_MAP.put(4, EmailFieldConst.NAV);
+        FILE_TYPE_MAP.put(5, EmailFieldConst.CUMULATIVE_NAV_WITHDRAWAL);
+        FILE_TYPE_MAP.put(6, EmailFieldConst.ASSET_SHARE);
+        FILE_TYPE_MAP.put(7, EmailFieldConst.ASSET_NET);
+    }
+
+    public EmailTemplateService(EmailTemplateMappingMapper emailTemplateMappingMapper, EmailTemplateApplicationRuleMapper emailTemplateApplicationRuleMapper,
+                                EmailTemplateDataRuleMapper emailTemplateDataRuleMapper) {
+        this.emailTemplateMappingMapper = emailTemplateMappingMapper;
+        this.emailTemplateApplicationRuleMapper = emailTemplateApplicationRuleMapper;
+        this.emailTemplateDataRuleMapper = emailTemplateDataRuleMapper;
+    }
+
+    public List<TemplateDetailDTO> getTemplateDetail(EmailContentInfoDTO emailContentInfoDTO, String filePath) {
+        List<TemplateDetailDTO> templateDetailDTOList = CollUtil.newArrayList();
+        String senderEmail = emailContentInfoDTO.getSenderEmail();
+        if (StrUtil.isBlank(senderEmail) || StrUtil.isBlank(filePath)) {
+            return templateDetailDTOList;
+        }
+        // 查询邮箱配置的模板Id
+        List<EmailTemplateInfoDO> emailTemplateInfoDOList = emailTemplateMappingMapper.queryByEmail(senderEmail);
+        List<Integer> templateIdList = emailTemplateInfoDOList.stream().map(EmailTemplateInfoDO::getId).collect(Collectors.toList());
+        if (CollUtil.isEmpty(templateIdList)) {
+            return templateDetailDTOList;
+        }
+        Map<Integer, Integer> templateIdDirectionMap = emailTemplateInfoDOList.stream().collect(Collectors.toMap(EmailTemplateInfoDO::getId, EmailTemplateInfoDO::getDirection));
+        // 查询模版适用性规则 -> 判断邮件是否满足模板适用性规则
+        List<EmailTemplateApplicationRuleDO> templateApplicationRuleDOList = emailTemplateApplicationRuleMapper.queryByTemplateId(templateIdList);
+        if (CollUtil.isEmpty(templateApplicationRuleDOList)) {
+            return templateDetailDTOList;
+        }
+        Map<Integer, List<EmailTemplateApplicationRuleDO>> templateIdApplicationRuleMap = templateApplicationRuleDOList.stream().collect(Collectors.groupingBy(EmailTemplateApplicationRuleDO::getTemplateId));
+        // 查询模版数据规则
+        List<EmailTemplateDataRuleDO> templateDataRuleDOList = emailTemplateDataRuleMapper.queryByTemplateId(templateIdList);
+        Map<Integer, List<EmailTemplateDataRuleDO>> templateIdDataRuleMap = templateDataRuleDOList.stream().collect(Collectors.groupingBy(EmailTemplateDataRuleDO::getTemplateId));
+
+        for (Map.Entry<Integer, List<EmailTemplateApplicationRuleDO>> templateIdApplicationRule : templateIdApplicationRuleMap.entrySet()) {
+            Integer templateId = templateIdApplicationRule.getKey();
+            List<EmailTemplateApplicationRuleDO> applicationRuleDOList = templateIdApplicationRule.getValue();
+            // 判断是否满足模板适用性规则
+            boolean isMatchTemplate = isMatchTemplate(emailContentInfoDTO.getEmailTitle(), emailContentInfoDTO.getFileName(), filePath, applicationRuleDOList);
+            if (!isMatchTemplate) {
+                continue;
+            }
+            TemplateDetailDTO templateDetailDTO = new TemplateDetailDTO();
+            templateDetailDTO.setTemplateId(templateId);
+            // 模板的数据行方向：默认为行
+            Integer directionOrDefault = templateIdDirectionMap.getOrDefault(templateId, 1);
+            templateDetailDTO.setDirection(directionOrDefault);
+            Map<String, Pair<Integer, Integer>> fieldPositionMap = getFieldPosition(templateIdDataRuleMap.get(templateId));
+            templateDetailDTO.setFieldPositionMap(fieldPositionMap);
+            templateDetailDTOList.add(templateDetailDTO);
+        }
+        return templateDetailDTOList;
+    }
+
+    private Map<String, Pair<Integer, Integer>> getFieldPosition(List<EmailTemplateDataRuleDO> emailTemplateDataRuleDOList) {
+        Map<String, Pair<Integer, Integer>> fieldPositionMap = new HashMap<>();
+        for (EmailTemplateDataRuleDO templateDataRuleDO : emailTemplateDataRuleDOList) {
+            Integer fieldName = templateDataRuleDO.getFieldName();
+            String fieldValue = FILE_TYPE_MAP.get(fieldName);
+            int column = columnLetterToIndex(templateDataRuleDO.getColumn());
+            Pair<Integer, Integer> pair = new Pair<>(templateDataRuleDO.getRow() - 1, column);
+            fieldPositionMap.put(fieldValue, pair);
+        }
+        return fieldPositionMap;
+    }
+
+    private boolean isMatchTemplate(String emailTitle, String fileName, String filePath, List<EmailTemplateApplicationRuleDO> applicationRuleDOList) {
+        int notMatchCount = 0;
+        for (EmailTemplateApplicationRuleDO applicationRuleDO : applicationRuleDOList) {
+            if (!isMathApplicationRule(applicationRuleDO, emailTitle, fileName, filePath)) {
+                notMatchCount++;
+            }
+        }
+        return notMatchCount == 0;
+    }
+
+    private boolean isMathApplicationRule(EmailTemplateApplicationRuleDO applicationRuleDO, String emailTitle, String fileName, String filePath) {
+        boolean isMatch = true;
+        Integer type = applicationRuleDO.getType();
+        String containKeyword = applicationRuleDO.getContainKeyword();
+        String notContainKeyword = applicationRuleDO.getNotContainKeyword();
+        if (ApplicationRuleFileConst.EMAIL_TITLE_FILE.equals(type)) {
+            if (StrUtil.isNotBlank(containKeyword)) {
+                isMatch = Arrays.stream(containKeyword.split(",")).allMatch(emailTitle::contains);
+            }
+            if (StrUtil.isNotBlank(notContainKeyword)) {
+                isMatch = Arrays.stream(notContainKeyword.split(",")).noneMatch(emailTitle::contains) && isMatch;
+            }
+        }
+        if (ApplicationRuleFileConst.EMAIL_FILE_NAME_FILE.equals(type)) {
+            if (StrUtil.isNotBlank(containKeyword)) {
+                isMatch = Arrays.stream(containKeyword.split(",")).allMatch(fileName::contains);
+            }
+            if (StrUtil.isNotBlank(notContainKeyword)) {
+                isMatch = Arrays.stream(notContainKeyword.split(",")).noneMatch(fileName::contains) && isMatch;
+            }
+        }
+        if (ApplicationRuleFileConst.EMAIL_EXCEL_CONTENT_FILE.equals(type)) {
+            if (StrUtil.isNotBlank(fileName) && ExcelUtil.isExcel(fileName)) {
+                if (applicationRuleDO.getRow() == null || StrUtil.isBlank(applicationRuleDO.getColumn())) {
+                    return false;
+                }
+                Sheet sheet = ExcelUtil.getFirstSheet(filePath);
+                if (sheet == null) {
+                    return false;
+                }
+                Row row = sheet.getRow(applicationRuleDO.getRow());
+                if (row == null) {
+                    return false;
+                }
+                int column = columnLetterToIndex(applicationRuleDO.getColumn());
+                Cell cell = row.getCell(column);
+                if (cell == null) {
+                    return false;
+                }
+                String cellValue = ExcelUtil.getCellValue(cell);
+                if (StrUtil.isBlank(cellValue)) {
+                    return false;
+                }
+                if (StrUtil.isNotBlank(containKeyword)) {
+                    isMatch = Arrays.stream(containKeyword.split(",")).allMatch(cellValue::contains);
+                }
+                if (StrUtil.isNotBlank(notContainKeyword)) {
+                    isMatch = Arrays.stream(notContainKeyword.split(",")).noneMatch(cellValue::contains) && isMatch;
+                }
+            }
+        }
+        return isMatch;
+    }
+
+    public static int columnLetterToIndex(String columnName) {
+        int columnIndex = 0;
+        int columnLength = columnName.length();
+        // Excel列的基础是26
+        int base = 26;
+        for (int i = 0; i < columnLength; i++) {
+            char letter = columnName.charAt(i);
+            // 计算字母的位置（A=0, B=1, ..., Z=25）
+            int position = letter - 'A';
+            columnIndex += (int) (position * Math.pow(base, columnLength - i - 1));
+        }
+        return columnIndex;
+    }
+
+}

service-daq/src/main/java/com/simuwang/daq/service/NavEmailParser.java

@@ -14,6 +14,7 @@ import com.simuwang.base.common.util.StringUtil;
 import com.simuwang.base.pojo.dto.EmailContentInfoDTO;
 import com.simuwang.base.pojo.dto.EmailFundNavDTO;
 import com.simuwang.base.pojo.dto.FieldPositionDTO;
+import com.simuwang.base.pojo.dto.TemplateDetailDTO;
 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.poi.ss.usermodel.Cell;
@@ -53,14 +54,19 @@ public class NavEmailParser extends AbstractEmailParser {
     @Value("${email.file.path}")
     private String path;
 
+    @Value("${email.force-template.enable}")
+    private boolean forceTemplateEnable;
+
     private final ValuationEmailParser valuationEmailParser;
+    private final EmailTemplateService emailTemplateService;
 
     private static final Integer ROW_DIRECTION_TYPE = 1;
     private static final Integer COLUMN_DIRECTION_TYPE = 2;
     private static final int MAX_COLUMN = 20;
 
-    public NavEmailParser(ValuationEmailParser valuationEmailParser) {
+    public NavEmailParser(ValuationEmailParser valuationEmailParser, EmailTemplateService emailTemplateService) {
         this.valuationEmailParser = valuationEmailParser;
+        this.emailTemplateService = emailTemplateService;
     }
 
     @Override
@@ -94,10 +100,7 @@ public class NavEmailParser extends AbstractEmailParser {
             Optional.ofNullable(fundNavDTOList).ifPresent(emailFundNavDTOList::addAll);
         }
 
-        // 校验净值数据格式
-        if (CollUtil.isNotEmpty(emailFundNavDTOList)) {
-            emailFundNavDTOList = emailFundNavDTOList.stream().filter(super::dataFormat).collect(Collectors.toList());
-        }
+        // 兼容净值邮件,但附件是估值表的情况
         if (CollUtil.isEmpty(emailFundNavDTOList)) {
             // 判断文件名是否包含"估值表"
             String fileName = emailContentInfoDTO.getFileName();
@@ -105,11 +108,16 @@ public class NavEmailParser extends AbstractEmailParser {
                 emailFundNavDTOList = valuationEmailParser.parse(emailContentInfoDTO, emailFieldMap);
             }
         }
+
+        // 通用模版解析 -> 根据配置的模板进行解析
+        if (CollUtil.isEmpty(emailFundNavDTOList) || forceTemplateEnable) {
+            emailFundNavDTOList = parseUsingTemplate(emailContentInfoDTO);
+        }
         return emailFundNavDTOList;
     }
 
     private List<EmailFundNavDTO> parsePackageFile(EmailContentInfoDTO emailContentInfoDTO, String fileName, String filePath, Map<String, List<String>> emailFieldMap) {
-        String destPath = filePath.substring(0, filePath.indexOf(fileName)) + fileName.replaceAll(".zip", "").replaceAll(".ZIP", "");
+        String destPath = filePath.replaceAll(".zip", "").replaceAll(".ZIP", "");
         log.info("压缩包地址:{},解压后文件地址:{}", filePath, destPath);
         List<String> dir = ExcelUtil.extractCompressedFiles(filePath, destPath);
         List<EmailFundNavDTO> emailFundNavDTOList = CollUtil.newArrayList();
@@ -141,15 +149,16 @@ public class NavEmailParser extends AbstractEmailParser {
         return fundNavDTOList;
     }
 
-    /**
-     * 解析邮件pdf附件
-     *
-     * @param filePath      邮件pdf附件路径
-     * @param excelFilePath pdf转excel路径
-     * @param emailFieldMap 邮件字段识别规则映射表
-     * @return 解析到的净值数据
-     */
+
     private List<EmailFundNavDTO> parsePdfFile(String filePath, String excelFilePath, Map<String, List<String>> emailFieldMap) {
+        excelFilePath = pdfConvertToExcel(filePath, excelFilePath);
+        if (StrUtil.isBlank(excelFilePath)) {
+            return CollUtil.newArrayList();
+        }
+        return parseExcelFile(excelFilePath, emailFieldMap);
+    }
+
+    private String pdfConvertToExcel(String filePath, String excelFilePath) {
         File savefile = new File(excelFilePath);
         if (!savefile.exists()) {
             if (!savefile.getParentFile().exists()) {
@@ -184,8 +193,9 @@ public class NavEmailParser extends AbstractEmailParser {
             workbook.write(outputStream);
         } catch (Exception e) {
             log.error("解析邮件pdf附件报错 -> 堆栈信息:{}", ExceptionUtil.stacktraceToString(e));
+            return null;
         }
-        return parseExcelFile(excelFilePath, emailFieldMap);
+        return excelFilePath;
     }
 
     /**
@@ -196,8 +206,7 @@ public class NavEmailParser extends AbstractEmailParser {
      * @return 解析到的净值数据
      */
     private List<EmailFundNavDTO> parseExcelFile(String filePath, Map<String, List<String>> emailFieldMap) {
-        File file = new File(filePath);
-        Sheet sheet = ExcelUtil.getSheet(file, 0);
+        Sheet sheet = ExcelUtil.getFirstSheet(filePath);
         // 1.找到表头所在位置
         Map<String, Pair<Integer, Integer>> fieldPositionMap = getFieldPosition(sheet, emailFieldMap);
         if (MapUtil.isEmpty(fieldPositionMap)) {
@@ -205,7 +214,12 @@ public class NavEmailParser extends AbstractEmailParser {
             return CollUtil.newArrayList();
         }
         // 2.解析sheet中的净值数据
-        return parseSheetData(sheet, fieldPositionMap);
+        List<EmailFundNavDTO> emailFundNavDTOList = parseSheetData(sheet, fieldPositionMap, null);
+        // 3.校验净值数据格式
+        if (CollUtil.isNotEmpty(emailFundNavDTOList)) {
+            emailFundNavDTOList = emailFundNavDTOList.stream().filter(super::dataFormat).collect(Collectors.toList());
+        }
+        return emailFundNavDTOList;
     }
 
     /**
@@ -217,7 +231,15 @@ public class NavEmailParser extends AbstractEmailParser {
      * @return 解析到的净值数据
      */
     private List<EmailFundNavDTO> parseEmailContent(EmailContentInfoDTO emailContentInfoDTO, String emailContent, Map<String, List<String>> emailFieldMap) {
-        Document doc = Jsoup.parse(emailContent);
+        String excelFilePath = contentConvertToExcel(emailContentInfoDTO);
+        if (StrUtil.isBlank(excelFilePath)) {
+            return CollUtil.newArrayList();
+        }
+        return parseExcelFile(excelFilePath, emailFieldMap);
+    }
+
+    public String contentConvertToExcel(EmailContentInfoDTO emailContentInfoDTO) {
+        Document doc = Jsoup.parse(emailContentInfoDTO.getEmailContent());
         Element table = doc.select("table").first();
         Elements rows = table.select("tr");
         String excelFilePath = path + "/content/" + emailContentInfoDTO.getEmailAddress() + "/" + emailContentInfoDTO.getEmailDate().substring(0, 10).replaceAll("-", "") + "/"
@@ -238,14 +260,87 @@ public class NavEmailParser extends AbstractEmailParser {
             workbook.write(outputStream);
         } catch (Exception e) {
             log.error("解析正文报错 -> 邮件主题:{},邮件日期:{},堆栈信息:{}", emailContentInfoDTO.getEmailTitle(), emailContentInfoDTO.getEmailDate(), ExceptionUtil.stacktraceToString(e));
+            return null;
         }
-        return parseExcelFile(excelFilePath, emailFieldMap);
+        return excelFilePath;
     }
 
-    private List<EmailFundNavDTO> parseSheetData(Sheet sheet, Map<String, Pair<Integer, Integer>> fieldPositionMap) {
+    private List<EmailFundNavDTO> parseUsingTemplate(EmailContentInfoDTO emailContentInfoDTO) {
+        // 考虑文件为PDF，html,zip等情况 -> 将相关文件转成Excel
+        List<String> filePathList = getRealFilePath(emailContentInfoDTO.getFilePath(), emailContentInfoDTO);
+        if (CollUtil.isEmpty(filePathList)) {
+            return CollUtil.newArrayList();
+        }
+        List<EmailFundNavDTO> emailFundNavDTOList = CollUtil.newArrayList();
+        for (String filePath : filePathList) {
+            List<TemplateDetailDTO> templateDetailDTOList = emailTemplateService.getTemplateDetail(emailContentInfoDTO, filePath);
+            if (CollUtil.isEmpty(templateDetailDTOList) || ExcelUtil.getFirstSheet(filePath) == null) {
+                return CollUtil.newArrayList();
+            }
+            // 获取文件的sheet页
+            Sheet sheet = ExcelUtil.getFirstSheet(filePath);
+            // 按照模板分别进行解析 -> 只要有一个模板解析到数据就停止
+            for (TemplateDetailDTO templateDetailDTO : templateDetailDTOList) {
+                try {
+                    Integer direction = templateDetailDTO.getDirection();
+                    Map<String, Pair<Integer, Integer>> fieldPositionMap = templateDetailDTO.getFieldPositionMap();
+                    List<EmailFundNavDTO> fundNavDTOList = parseSheetData(sheet, fieldPositionMap, direction);
+                    fundNavDTOList = fundNavDTOList.stream().filter(super::dataFormat).collect(Collectors.toList());
+                    if (CollUtil.isNotEmpty(fundNavDTOList)) {
+                        emailFundNavDTOList.addAll(fundNavDTOList);
+                        break;
+                    }
+                } catch (Exception e) {
+                    log.error("净值模板解析报错 -> 模板id:{},文件:{},堆栈信息:{}", templateDetailDTO.getTemplateId(), filePath, ExceptionUtil.stacktraceToString(e));
+                }
+            }
+        }
+        return emailFundNavDTOList;
+    }
+
+    private List<String> getRealFilePath(String filePath, EmailContentInfoDTO emailContentInfoDTO) {
+        List<String> filePathList = CollUtil.newArrayList();
+        if (StrUtil.isBlank(filePath)) {
+            return filePathList;
+        }
+        if (ExcelUtil.isExcel(filePath)) {
+            filePathList.add(filePath);
+        } else if (ExcelUtil.isHTML(filePath)) {
+            String excelFilePath = contentConvertToExcel(emailContentInfoDTO);
+            Optional.ofNullable(excelFilePath).ifPresent(filePathList::add);
+        } else if (ExcelUtil.isPdf(filePath)) {
+            String excelFilePath = filePath.replace(".pdf", ".xlsx").replace(".PDF", ".xlsx");
+            excelFilePath = pdfConvertToExcel(filePath, excelFilePath);
+            Optional.ofNullable(excelFilePath).ifPresent(filePathList::add);
+        } else if (ExcelUtil.isZip(filePath)) {
+            String destPath = filePath.replaceAll(".zip", "").replaceAll(".ZIP", "");
+            List<String> dir = ExcelUtil.extractCompressedFiles(filePath, destPath);
+            for (String zipFilePath : dir) {
+                File file = new File(zipFilePath);
+                if (file.isDirectory()) {
+                    for (String navFilePath : Objects.requireNonNull(file.list())) {
+                        Optional.ofNullable(getRealFilePath(navFilePath, emailContentInfoDTO)).ifPresent(filePathList::addAll);
+                    }
+                } else {
+                    Optional.ofNullable(getRealFilePath(zipFilePath, emailContentInfoDTO)).ifPresent(filePathList::addAll);
+                }
+            }
+        }
+        return filePathList;
+    }
+
+    /**
+     * 根据字段所在表格的位置提取净值数据
+     *
+     * @param sheet            表格中的sheet页
+     * @param fieldPositionMap 字段所在表格中的位置
+     * @param direction        表格数据的形式：1-行，2-列
+     * @return 净值数据
+     */
+    private List<EmailFundNavDTO> parseSheetData(Sheet sheet, Map<String, Pair<Integer, Integer>> fieldPositionMap, Integer direction) {
         List<EmailFundNavDTO> fundNavDTOList = CollUtil.newArrayList();
         // 通过表头所在位置判断是行数据还是列数据
-        Integer dataDirectionType = detectDataDirection(fieldPositionMap);
+        Integer dataDirectionType = direction != null ? direction : detectDataDirection(fieldPositionMap);
         // 数据起始行,起始列
         int initRow = dataDirectionType.equals(ROW_DIRECTION_TYPE) ? fieldPositionMap.values().stream().map(Pair::getKey).max(Integer::compareTo).orElse(0)
                 : fieldPositionMap.values().stream().map(Pair::getKey).min(Integer::compareTo).orElse(0);

service-daq/src/main/java/com/simuwang/daq/service/ValuationEmailParser.java

@@ -55,6 +55,10 @@ public class ValuationEmailParser extends AbstractEmailParser {
             List<EmailFundNavDTO> fundNavDTOList = convertToFundNavDTO(parseSuccessList);
             Optional.ofNullable(fundNavDTOList).ifPresent(emailFundNavDTOList::addAll);
         }
+        // 校验净值数据格式
+        if (CollUtil.isNotEmpty(emailFundNavDTOList)) {
+            emailFundNavDTOList = emailFundNavDTOList.stream().filter(super::dataFormat).collect(Collectors.toList());
+        }
         return emailFundNavDTOList;
     }
 

service-deploy/src/main/resources/application.yml

@@ -70,6 +70,8 @@ email-rule:
 email:
   file:
     path: /data/file/nav
+  force-template:
+    enable: true
 
 # 配置
 simuwang: