Merge branch 'develop' of http://112.74.196.215:3000/Tech2/data-daq into develop

service-base/src/main/java/com/simuwang/base/common/enums/ReportType.java

@@ -2,15 +2,11 @@ package com.simuwang.base.common.enums;
 
 import lombok.Getter;
 
-import java.util.Arrays;
-import java.util.List;
-import java.util.stream.Collectors;
-
 @Getter
 public enum ReportType {
-    MONTHLY(0, "月报", new String[]{"月", "月度", "月报"}),
-    QUARTERLY(1, "季报", new String[]{"季", "季度", "季报"}),
-    ANNUALLY(2, "年报", new String[]{"年", "年度", "年报"});
+    MONTHLY(0, "月", new String[]{"月", "月度", "月报"}),
+    QUARTERLY(1, "季", new String[]{"季", "季度", "季报"}),
+    ANNUALLY(2, "年", new String[]{"年", "年度", "年报"});
 
     private final int type;
     private final String label;
@@ -21,12 +17,4 @@ public enum ReportType {
         this.label = label;
         this.patterns = patterns;
     }
-
-    public static String getAllPatterns() {
-        return String.join("|", patterns());
-    }
-
-    public static List<String> patterns() {
-        return Arrays.stream(ReportType.values()).flatMap(e -> Arrays.stream(e.getPatterns())).collect(Collectors.toList());
-    }
 }

service-base/src/main/java/com/simuwang/base/pojo/dto/report/BaseReportDTO.java

@@ -47,29 +47,29 @@ public abstract class BaseReportDTO<T extends BaseReportDO> {
         try {
             // 日期格式化，支持三种格式：yyyy年MM月dd日、yyyy-MM-dd和yyyy/MM/dd
             return DateUtil.parse(input.trim(),
-                    DatePattern.CHINESE_DATE_PATTERN, DatePattern.NORM_DATE_PATTERN, "yyyy/MM/dd");
+                    DatePattern.CHINESE_DATE_PATTERN, DatePattern.NORM_DATE_PATTERN, "yyyy/MM/dd", "yyyyMMdd");
         } catch (Exception ignored) {
         }
         return null;
     }
 
     /**
-     * 字符串转数字
+     * 字符串转数字，如果数据没有或者转换失败则用0填充
      *
      * @param input 待转换的字符串
      * @return /
      */
     protected BigDecimal toBigDecimal(String input) {
         if (StrUtil.isBlank(input)) {
-            return null;
+            return BigDecimal.ZERO;
         }
         try {
-            // 移除所有非数字和“.”字符
-            String cleanedInput = input.trim().replaceAll("[^\\d.]", "");
+            // 替换掉非正常的正负小数字符
+            String cleanedInput = input.trim().replaceAll("[^\\s" + "[-+]?\\d*\\.?\\d+" + "]", "");
             // 创建BigDecimal对象
             return new BigDecimal(cleanedInput);
         } catch (NumberFormatException ignored) {
         }
-        return null;
+        return BigDecimal.ZERO;
     }
 }

service-base/src/main/java/com/simuwang/base/pojo/dto/report/ReportParseStatus.java

@@ -4,11 +4,11 @@ import com.smppw.common.pojo.enums.status.StatusCode;
 
 public enum ReportParseStatus implements StatusCode {
     PARSE_FAIL(21000, "定期报告解析错误：{}"),
-    NOT_A_REPORT(21001, "不是定期报告"),
-    REPORT_IS_SCAN(21002, "报告为扫描件"),
-    NO_SUPPORT_TEMPLATE(21003, "不支持的报告文件格式"),
-    NOT_A_FIXED_FORMAT(21004, "不是基协统一格式"),
-    PARSE_FUND_INFO_FAIL(21010, "没有解析到报告中的基金基本信息"),
+    NOT_A_REPORT(21001, "[{}] 不是定期报告"),
+    REPORT_IS_SCAN(21002, "报告[{}] 为扫描件"),
+    NO_SUPPORT_TEMPLATE(21003, "报告[{}] 是不支持的文件格式"),
+    NOT_A_FIXED_FORMAT(21004, "报告[{}] 不是基协统一格式"),
+    PARSE_FUND_INFO_FAIL(21010, "报告[{}] 没有解析到基金基本信息"),
     ;
     private final int code;
     private final String msg;

service-daq/src/main/java/com/simuwang/daq/components/report/parser/pdf/AbstractPDReportParser.java

@@ -22,9 +22,9 @@ import technology.tabula.extractors.SpreadsheetExtractionAlgorithm;
 
 import java.io.IOException;
 import java.util.Calendar;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.function.Function;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -56,6 +56,7 @@ public abstract class AbstractPDReportParser<T extends ReportData> extends Abstr
         this.textList = null;
         // 初始化
         this.init();
+        String filename = params.getFilename();
         // 解析报告和表格
         try (PDDocument document = Loader.loadPDF(new RandomAccessReadBufferedFile(params.getFilepath()))) {
             // 识别所有文字（去水印后的）
@@ -65,11 +66,7 @@ public abstract class AbstractPDReportParser<T extends ReportData> extends Abstr
             this.textList = StrUtil.split(text, System.lineSeparator());
             this.textList.removeIf(StrUtil::isBlank);
             if (this.textList.isEmpty()) {
-                throw new ReportParseException(ReportParseStatus.REPORT_IS_SCAN);
-            }
-            // 报告名称和类型一般在第一第二行
-            if (this.matchReportType(this.textList.get(0)) == null && this.matchReportType(this.textList.get(1)) == null) {
-                throw new ReportParseException(ReportParseStatus.NOT_A_REPORT);
+                throw new ReportParseException(ReportParseStatus.REPORT_IS_SCAN, filename);
             }
             // 解析所有表格（单元格字符去水印）
             List<Table> tables = ListUtil.list(true);
@@ -81,7 +78,7 @@ public abstract class AbstractPDReportParser<T extends ReportData> extends Abstr
                 tables.addAll(extractionAlgorithm.extract(page));
             }
             if (tables.isEmpty()) {
-                throw new ReportParseException(ReportParseStatus.REPORT_IS_SCAN);
+                throw new ReportParseException(ReportParseStatus.REPORT_IS_SCAN, filename);
             }
             this.initTableInfo(tables);
         }
@@ -99,7 +96,7 @@ public abstract class AbstractPDReportParser<T extends ReportData> extends Abstr
             throw e;
         } catch (Exception e) {
             this.logger.warn("报告解析错误：{}", ExceptionUtil.stacktraceToString(e));
-            throw new ReportParseException(ReportParseStatus.NOT_A_FIXED_FORMAT);
+            throw new ReportParseException(ReportParseStatus.NOT_A_FIXED_FORMAT, filename);
         }
     }
 
@@ -119,10 +116,11 @@ public abstract class AbstractPDReportParser<T extends ReportData> extends Abstr
     protected ReportFundInfoDTO buildFundInfo(ReportParserParams params) {
         Table fundInfoTable = this.fundInfoTable;
         if (fundInfoTable == null) {
-            throw new ReportParseException(ReportParseStatus.PARSE_FUND_INFO_FAIL);
+            throw new ReportParseException(ReportParseStatus.PARSE_FUND_INFO_FAIL, params.getFilename());
         }
         // 基金基本信息映射
-        return this.buildDto(params.getFileId(), fundInfoTable, ReportFundInfoDTO.class, this::parseFundInfo);
+        Map<String, Object> extInfoMap = this.parseFundInfo(fundInfoTable);
+        return this.buildDto(params.getFileId(), ReportFundInfoDTO.class, extInfoMap);
     }
 
     /**
@@ -164,7 +162,7 @@ public abstract class AbstractPDReportParser<T extends ReportData> extends Abstr
     }
 
     /**
-     * 构建只有两列表格的dto数据对象，如果有分级基金时
+     * 构建只有两列表格的dto数据对象，如果有分级基金时（并且一个表格可能跨页）
      *
      * @param <DTO>    泛型对象
      * @param fileId   文件id
@@ -175,17 +173,37 @@ public abstract class AbstractPDReportParser<T extends ReportData> extends Abstr
      */
     protected <DTO extends BaseReportLevelDTO<?>> List<DTO> buildLevelDto(Integer fileId, List<Table> tables, Class<DTO> clazz,
                                                                           Function<Table, Map<String, Object>> function) {
-        // 映射转换
-        List<DTO> dtos = tables.stream().filter(Objects::nonNull)
-                .map(e -> this.buildDto(fileId, e, clazz, function)).collect(Collectors.toList());
+        List<DTO> dtos = ListUtil.list(true);
+        // 信息表格字段和值映射
+        List<Map<String, Object>> infos = ListUtil.list(true);
+        Map<String, Object> infoMap = null;
+        for (Table table : tables) {
+            Map<String, Object> temp = function.apply(table);
+            for (String key : temp.keySet()) {
+                // 如果infoMap为null，先声明然后放在infos中
+                if (infoMap == null) {
+                    infoMap = MapUtil.newHashMap(16);
+                    infos.add(infoMap);
+                }
+                // 如果infoMap中包含了该key时，先放infos中然后重新声明新map对象
+                if (infoMap.containsKey(key)) {
+                    infos.add(new HashMap<>(infoMap));
+                    infoMap = MapUtil.newHashMap(16);
+                } else {
+                    infoMap.put(key, temp.get(key));
+                }
+            }
+        }
         // 分级基金匹配
         List<String> levels = this.matchTieredFund(String.join(",", this.textList));
         levels.add(0, "母基金");
-        for (int i = 0; i < dtos.size(); i++) {
-            if (levels.size() <= i) {
+        for (int i = 0; i < infos.size(); i++) {
+            DTO dto = this.buildDto(fileId, clazz, infos.get(i));
+            if (dto == null) {
                 continue;
             }
-            dtos.get(i).setLevel(levels.get(i));
+            dto.setLevel(levels.get(i));
+            dtos.add(dto);
         }
         return dtos;
     }
@@ -193,20 +211,17 @@ public abstract class AbstractPDReportParser<T extends ReportData> extends Abstr
     /**
      * 构建只有两列表格的dto数据对象
      *
-     * @param <DTO>    泛型对象
-     * @param fileId   文件id
-     * @param table    表格
-     * @param clazz    泛型对象
-     * @param function 表格转换的函数
+     * @param <DTO>   泛型对象
+     * @param fileId  文件id
+     * @param clazz   泛型对象
+     * @param infoMap 表格转换的函数
      * @return /
      */
-    private <DTO extends BaseReportDTO<?>> DTO buildDto(Integer fileId, Table table, Class<DTO> clazz,
-                                                        Function<Table, Map<String, Object>> function) {
+    private <DTO extends BaseReportDTO<?>> DTO buildDto(Integer fileId, Class<DTO> clazz, Map<String, Object> infoMap) {
         try {
-            Map<String, Object> extInfoMap = function == null ? MapUtil.empty() : function.apply(table);
             DTO dto = clazz.getDeclaredConstructor().newInstance();
             dto.setFileId(fileId);
-            this.buildInfo(extInfoMap, dto);
+            this.buildInfo(infoMap, dto);
             return dto;
         } catch (Exception ignored) {
         }
@@ -303,18 +318,14 @@ public abstract class AbstractPDReportParser<T extends ReportData> extends Abstr
         if (string == null) {
             return null;
         }
-        // 所有报告的正则识别方式
-        String patterns = ReportType.getAllPatterns();
-        // 编译正则表达式模式
-        Pattern pattern = Pattern.compile(patterns);
-        // 创建Matcher对象
-        Matcher matcher = pattern.matcher(string);
-        // 尝试匹配
-        if (matcher.find()) {
-            return matcher.group();
-        } else {
-            return null;
+        // 类型识别---先识别季度报告，没有季度再识别年度报告，最后识别月报
+        ReportType reportType = ReportType.MONTHLY;
+        if (StrUtil.containsAny(string, ReportType.QUARTERLY.getPatterns())) {
+            reportType = ReportType.QUARTERLY;
+        } else if (StrUtil.containsAny(string, ReportType.ANNUALLY.getPatterns())) {
+            reportType = ReportType.ANNUALLY;
         }
+        return reportType.getLabel();
     }
 
     private int getLastDayOfMonth(int year, int month) {

service-daq/src/main/java/com/simuwang/daq/components/report/parser/pdf/PDAnnuallyReportParser.java

@@ -20,18 +20,6 @@ import java.util.function.Function;
  */
 @Component(ReportParserConstant.PARSER_PDF_ANNUALLY)
 public class PDAnnuallyReportParser extends PDQuarterlyReportParser<AnnuallyReportData> {
-    private static final List<String> FINANCIAL_INDICATORS_COLUMN_NAMES = ListUtil.list(false);
-
-    static {
-        FINANCIAL_INDICATORS_COLUMN_NAMES.add("期末基金净资产");
-        FINANCIAL_INDICATORS_COLUMN_NAMES.add("报告期期末单位净值");
-        FINANCIAL_INDICATORS_COLUMN_NAMES.add("本期利润");
-        FINANCIAL_INDICATORS_COLUMN_NAMES.add("本期已实现收益");
-        FINANCIAL_INDICATORS_COLUMN_NAMES.add("期末可供分配利润");
-        FINANCIAL_INDICATORS_COLUMN_NAMES.add("期末可供分配基金份额利润");
-        FINANCIAL_INDICATORS_COLUMN_NAMES.add("基金份额累计净值增长率");
-    }
-
     private List<Table> fundInfoTables;
 
     public PDAnnuallyReportParser(EmailFieldMappingMapper fieldMappingMapper) {
@@ -103,14 +91,9 @@ public class PDAnnuallyReportParser extends PDQuarterlyReportParser<AnnuallyRepo
     }
 
     @Override
-    protected AnnuallyReportData buildExtData(ReportBaseInfoDTO reportInfo, ReportFundInfoDTO fundInfo,
-                                              List<ReportShareChangeDTO> shareChanges,
-                                              List<ReportAssetAllocationDTO> assetAllocations,
-                                              List<ReportInvestmentIndustryDTO> investmentIndustries,
-                                              Function<Table, Map<String, Object>> function) {
-        // 处理财务指标
-        List<ReportFinancialIndicatorsDTO> financialIndicators = this.buildFinancialIndicatorsInfo(reportInfo.getFileId());
-        // 返回数据构建
+    protected AnnuallyReportData buildReportData(ReportBaseInfoDTO reportInfo, ReportFundInfoDTO fundInfo,
+                                                 List<ReportShareChangeDTO> shareChanges, List<ReportFinancialIndicatorsDTO> financialIndicators,
+                                                 List<ReportAssetAllocationDTO> assetAllocations, List<ReportInvestmentIndustryDTO> investmentIndustries) {
         AnnuallyReportData reportData = new AnnuallyReportData(reportInfo, fundInfo);
         reportData.setShareChange(shareChanges);
         reportData.setFinancialIndicators(financialIndicators);
@@ -124,7 +107,7 @@ public class PDAnnuallyReportParser extends PDQuarterlyReportParser<AnnuallyRepo
         // todo 数据清洗
     }
 
-    private List<ReportFinancialIndicatorsDTO> buildFinancialIndicatorsInfo(Integer fileId) {
+    protected List<ReportFinancialIndicatorsDTO> buildFinancialIndicatorsInfo(Integer fileId, Function<Table, Map<String, Object>> function) {
         List<ReportFinancialIndicatorsDTO> dtos = ListUtil.list(false);
         // 分级基金
         List<String> levels = this.matchTieredFund(String.join(",", this.textList));

service-daq/src/main/java/com/simuwang/daq/components/report/parser/pdf/PDMonthlyReportParser.java

@@ -22,7 +22,7 @@ import java.util.Map;
  */
 @Component(ReportParserConstant.PARSER_PDF_MONTHLY)
 public class PDMonthlyReportParser extends AbstractPDReportParser<MonthlyReportData> {
-    private final List<Table> extNavTables = ListUtil.list(true);
+    private List<Table> extNavTables;
 
     public PDMonthlyReportParser(EmailFieldMappingMapper fieldMappingMapper) {
         super(fieldMappingMapper);
@@ -35,6 +35,8 @@ public class PDMonthlyReportParser extends AbstractPDReportParser<MonthlyReportD
 
     @Override
     protected void initTableInfo(List<Table> tables) {
+        // 这里初始化
+        this.extNavTables = ListUtil.list(true);
         // 一般月报是固定的模板，4列表格是基金基本信息，其他5列的表格是月净值
         for (Table table : tables) {
             int colCount = table.getColCount();

service-daq/src/main/java/com/simuwang/daq/components/report/parser/pdf/PDQuarterlyReportParser.java

@@ -15,6 +15,7 @@ import java.awt.geom.Rectangle2D;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.function.Function;
 
 /**
@@ -26,8 +27,18 @@ import java.util.function.Function;
 public class PDQuarterlyReportParser<T extends QuarterlyReportData> extends AbstractPDReportParser<T> {
     protected static final List<String> INDUSTRY_COLUMN_NAMES = ListUtil.list(false);
     protected static final List<String> SHARE_CHANGE_COLUMN_NAMES = ListUtil.list(false);
+    protected static final List<String> FINANCIAL_INDICATORS_COLUMN_NAMES = ListUtil.list(false);
 
     static {
+        // 财务指标
+        FINANCIAL_INDICATORS_COLUMN_NAMES.add("期末基金净资产");
+        FINANCIAL_INDICATORS_COLUMN_NAMES.add("报告期期末单位净值");
+        FINANCIAL_INDICATORS_COLUMN_NAMES.add("本期利润");
+        FINANCIAL_INDICATORS_COLUMN_NAMES.add("本期已实现收益");
+        FINANCIAL_INDICATORS_COLUMN_NAMES.add("期末可供分配利润");
+        FINANCIAL_INDICATORS_COLUMN_NAMES.add("期末可供分配基金份额利润");
+        FINANCIAL_INDICATORS_COLUMN_NAMES.add("基金份额累计净值增长率");
+
         // 中国证监会行业标准
         INDUSTRY_COLUMN_NAMES.add("农、林、牧、渔业");
         INDUSTRY_COLUMN_NAMES.add("采矿业");
@@ -106,7 +117,7 @@ public class PDQuarterlyReportParser<T extends QuarterlyReportData> extends Abst
                 // 主要财务指标或份额变动
                 if (CollUtil.containsAny(texts, SHARE_CHANGE_COLUMN_NAMES)) {
                     this.shareChangeTables.add(table);
-                } else {
+                } else if (CollUtil.containsAny(texts, FINANCIAL_INDICATORS_COLUMN_NAMES)) {
                     this.financialIndicatorsTables.add(table);
                 }
             } else if (colCount == 4) {
@@ -155,30 +166,42 @@ public class PDQuarterlyReportParser<T extends QuarterlyReportData> extends Abst
         List<ReportShareChangeDTO> shareChanges = this.buildLevelDto(fileId, this.shareChangeTables,
                 ReportShareChangeDTO.class, function);
         // 主要财务指标
-        List<ReportFinancialIndicatorsDTO> financialIndicators = this.buildLevelDto(fileId, this.financialIndicatorsTables,
-                ReportFinancialIndicatorsDTO.class, function);
+        List<ReportFinancialIndicatorsDTO> financialIndicators = this.buildFinancialIndicatorsInfo(fileId, function);
         // 资产配置
         List<ReportAssetAllocationDTO> assetAllocations = this.buildAssetAllocationInfo(fileId);
         // 行业配置
         List<ReportInvestmentIndustryDTO> investmentIndustries = this.buildInvestmentIndustryInfo(fileId);
         // 返回数据构建
-        QuarterlyReportData reportData = new QuarterlyReportData(reportInfo, fundInfo);
-        reportData.setShareChange(shareChanges);
-        reportData.setFinancialIndicators(financialIndicators);
-        reportData.setAssetAllocation(assetAllocations);
-        reportData.setInvestmentIndustry(investmentIndustries);
-        return this.buildExtData(reportInfo, fundInfo, shareChanges, assetAllocations, investmentIndustries, function);
+        return this.buildReportData(reportInfo, fundInfo, shareChanges, financialIndicators, assetAllocations, investmentIndustries);
     }
 
-    protected T buildExtData(ReportBaseInfoDTO reportInfo, ReportFundInfoDTO fundInfo,
-                             List<ReportShareChangeDTO> shareChanges,
-                             List<ReportAssetAllocationDTO> assetAllocations,
-                             List<ReportInvestmentIndustryDTO> investmentIndustries,
-                             Function<Table, Map<String, Object>> function) {
-        Integer fileId = reportInfo.getFileId();
-        // 主要财务指标
-        List<ReportFinancialIndicatorsDTO> financialIndicators = this.buildLevelDto(fileId, this.financialIndicatorsTables,
-                ReportFinancialIndicatorsDTO.class, function);
+    /**
+     * 主要财务指标数据构建（包括分级基金，并且一个表格可能跨页）
+     *
+     * @param fileId   文件id
+     * @param function 字段映射关系
+     * @return /
+     */
+    protected List<ReportFinancialIndicatorsDTO> buildFinancialIndicatorsInfo(Integer fileId, Function<Table, Map<String, Object>> function) {
+        return this.buildLevelDto(fileId, this.financialIndicatorsTables, ReportFinancialIndicatorsDTO.class, function);
+    }
+
+    /**
+     * 子类重写，放在cast异常
+     *
+     * @param reportInfo           报告基本信息
+     * @param fundInfo             基金基本信息
+     * @param shareChanges         份额变动
+     * @param financialIndicators  基本财务指标
+     * @param assetAllocations     资产配置
+     * @param investmentIndustries 行业配置
+     * @return /
+     */
+    protected T buildReportData(ReportBaseInfoDTO reportInfo, ReportFundInfoDTO fundInfo,
+                                List<ReportShareChangeDTO> shareChanges,
+                                List<ReportFinancialIndicatorsDTO> financialIndicators,
+                                List<ReportAssetAllocationDTO> assetAllocations,
+                                List<ReportInvestmentIndustryDTO> investmentIndustries) {
         QuarterlyReportData reportData = new QuarterlyReportData(reportInfo, fundInfo);
         reportData.setShareChange(shareChanges);
         reportData.setFinancialIndicators(financialIndicators);
@@ -212,9 +235,13 @@ public class PDQuarterlyReportParser<T extends QuarterlyReportData> extends Abst
                 if (StrUtil.containsAny(text, "序号", "行业类别")) {
                     continue;
                 }
+                String industryName = this.cleaningValue(table.getCell(i, j).getText());
+                if (StrUtil.isBlank(industryName) || Objects.equals("合计", industryName)) {
+                    continue;
+                }
                 ReportInvestmentIndustryDTO dto = new ReportInvestmentIndustryDTO(fileId);
                 dto.setInvestType(investType);
-                dto.setIndustryName(this.cleaningValue(table.getCell(i, j).getText()));
+                dto.setIndustryName(industryName);
                 dto.setMarketValue(this.cleaningValue(table.getCell(i, j + 1).getText()));
                 dto.setRatio(this.cleaningValue(table.getCell(i, j + 2).getText()));
                 dtos.add(dto);
@@ -244,7 +271,7 @@ public class PDQuarterlyReportParser<T extends QuarterlyReportData> extends Abst
                 }
                 // 金额、市值，有时是 “备注#金额”的格式
                 String marketValueAndRemark = this.cleaningValue(row.get(2).getText());
-                if (StrUtil.isBlank(marketValueAndRemark) || StrUtil.isBlank(assetType)) {
+                if (StrUtil.isBlank(assetType)) {
                     continue;
                 }
                 // 资产明细

service-deploy/src/test/java/com/simuwang/ApplicationTest.java

@@ -45,8 +45,8 @@ public class ApplicationTest {
     @Test
     public void reportTest() {
         MailboxInfoDTO emailInfoDTO = this.buildMailbox();
-        Date startDate = DateUtil.parse("2024-10-11 08:30:30", DateConst.YYYY_MM_DD_HH_MM_SS);
-        Date endDate = DateUtil.parse("2024-10-11 09:59:30", DateConst.YYYY_MM_DD_HH_MM_SS);
+        Date startDate = DateUtil.parse("2024-10-12 17:40:30", DateConst.YYYY_MM_DD_HH_MM_SS);
+        Date endDate = DateUtil.parse("2024-10-12 17:50:30", DateConst.YYYY_MM_DD_HH_MM_SS);
         try {
             emailParseService.parseEmail(emailInfoDTO, startDate, endDate);
         } catch (Exception e) {