feat:报告解析临时保存

service-base/src/main/java/com/simuwang/base/common/enums/ReportType.java

@@ -4,15 +4,17 @@ import lombok.Getter;
 
 @Getter
 public enum ReportType {
-    MONTHLY(0, "月报"),
-    QUARTERLY(1, "季报"),
-    ANNUALLY(2, "年报");
+    MONTHLY(0, "月报", "月"),
+    QUARTERLY(1, "季报", "季度"),
+    ANNUALLY(2, "年报", "年度");
 
     private final int type;
     private final String label;
+    private final String pattern;
 
-    ReportType(int type, String label) {
+    ReportType(int type, String label, String pattern) {
         this.type = type;
         this.label = label;
+        this.pattern = pattern;
     }
 }

service-daq/src/main/java/com/simuwang/daq/components/report/parser/AbstractReportParser.java

@@ -1,6 +1,9 @@
 package com.simuwang.daq.components.report.parser;
 
 import cn.hutool.core.collection.CollUtil;
+import cn.hutool.core.util.ReflectUtil;
+import cn.hutool.core.util.StrUtil;
+import com.simuwang.base.common.conts.Constants;
 import com.simuwang.base.mapper.EmailFieldMappingMapper;
 import com.simuwang.base.pojo.dos.EmailFieldMappingDO;
 import com.simuwang.base.pojo.dto.report.ReportData;
@@ -9,6 +12,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 public abstract class AbstractReportParser<T extends ReportData> implements ReportParser<T> {
@@ -33,4 +37,46 @@ public abstract class AbstractReportParser<T extends ReportData> implements Repo
         }
         this.fieldMapper = emailFieldMapping.stream().map(e -> new ValueLabelVO(e.getCode(), e.getName())).collect(Collectors.toList());
     }
+
+    /**
+     * 对象字段设置
+     *
+     * @param extInfoMap 名称与值的对应关系
+     * @param info       待设置的对象
+     */
+    protected void buildInfo(Map<String, Object> extInfoMap, Object info) {
+        for (Map.Entry<String, Object> entry : extInfoMap.entrySet()) {
+            String k = entry.getKey();
+            Object v = entry.getValue();
+            String fieldValue = StrUtil.toStringOrNull(v);
+            if (fieldValue.startsWith("-") || fieldValue.endsWith("-")) {
+                fieldValue = null;
+            }
+            if (fieldValue != null) {
+                fieldValue = fieldValue.replace("\r", Constants.EMPTY);
+            }
+            for (ValueLabelVO vo : this.fieldMapper) {
+                String fieldName = vo.getValue();
+                List<String> labels = StrUtil.split(vo.getLabel(), ",");
+                if (labels.contains(k)) {
+                    try {
+                        ReflectUtil.setFieldValue(info, fieldName, fieldValue);
+                    } catch (Exception e) {
+                        this.logger.warn("{} 字段值设置错误：{}", fieldName, e.getMessage());
+                    }
+                    break;
+                }
+                for (String label : labels) {
+                    if (k.contains(label)) {
+                        try {
+                            ReflectUtil.setFieldValue(info, fieldName, fieldValue);
+                        } catch (Exception e) {
+                            this.logger.warn("{} 字段值设置错误：{}", fieldName, e.getMessage());
+                        }
+                        break;
+                    }
+                }
+            }
+        }
+    }
 }

service-daq/src/main/java/com/simuwang/daq/components/report/parser/pdf/AbstractPDReportParser.java

@@ -25,7 +25,6 @@ import java.util.Calendar;
 import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import java.util.stream.Collectors;
 
 /**
  * @author wangzaijun
@@ -60,7 +59,6 @@ public abstract class AbstractPDReportParser<T extends ReportData> extends Abstr
             while (pageIterator.hasNext()) {
                 Page page = pageIterator.next();
                 List<Table> tables = extractionAlgorithm.extract(page);
-                tables = tables.stream().distinct().collect(Collectors.toList());
                 this.initTableInfo(tables);
             }
         }

service-daq/src/main/java/com/simuwang/daq/components/report/parser/pdf/PDMonthlyReportParser.java

@@ -2,14 +2,13 @@ package com.simuwang.daq.components.report.parser.pdf;
 
 import cn.hutool.core.collection.ListUtil;
 import cn.hutool.core.map.MapUtil;
-import cn.hutool.core.util.ReflectUtil;
-import cn.hutool.core.util.StrUtil;
-import com.simuwang.base.common.conts.Constants;
 import com.simuwang.base.common.exception.APIException;
 import com.simuwang.base.mapper.EmailFieldMappingMapper;
-import com.simuwang.base.pojo.dto.report.*;
+import com.simuwang.base.pojo.dto.report.MonthlyReportData;
+import com.simuwang.base.pojo.dto.report.ReportFundInfoDTO;
+import com.simuwang.base.pojo.dto.report.ReportNetReportDTO;
+import com.simuwang.base.pojo.dto.report.ReportParserParams;
 import com.simuwang.daq.components.report.parser.ReportParserConstant;
-import com.smppw.common.pojo.ValueLabelVO;
 import org.springframework.stereotype.Component;
 import technology.tabula.RectangularTextContainer;
 import technology.tabula.Table;
@@ -56,6 +55,7 @@ public class PDMonthlyReportParser extends AbstractPDReportParser<MonthlyReportD
         }
         Map<String, Object> baseInfoMap = MapUtil.newHashMap(32);
         for (int i = 0; i < fundInfoTable.getRows().size(); i++) {
+            @SuppressWarnings("all")
             List<RectangularTextContainer> cols = fundInfoTable.getRows().get(i);
             for (int j = 0; j < 2; j++) {
                 baseInfoMap.put(cols.get(j * 2).getText(), cols.get(j * 2 + 1).getText());
@@ -89,40 +89,4 @@ public class PDMonthlyReportParser extends AbstractPDReportParser<MonthlyReportD
         reportData.setNetReport(exts);
         return reportData;
     }
-
-    private void buildInfo(Map<String, Object> extInfoMap, Object info) {
-        for (Map.Entry<String, Object> entry : extInfoMap.entrySet()) {
-            String k = entry.getKey();
-            Object v = entry.getValue();
-            String fieldValue = StrUtil.toStringOrNull(v);
-            if (fieldValue.startsWith("-") || fieldValue.endsWith("-")) {
-                fieldValue = null;
-            }
-            if (fieldValue != null) {
-                fieldValue = fieldValue.replace("\r", Constants.EMPTY);
-            }
-            for (ValueLabelVO vo : this.fieldMapper) {
-                String fieldName = vo.getValue();
-                List<String> labels = StrUtil.split(vo.getLabel(), ",");
-                if (labels.contains(k)) {
-                    try {
-                        ReflectUtil.setFieldValue(info, fieldName, fieldValue);
-                    } catch (Exception e) {
-                        this.logger.warn("{} 字段值设置错误：{}", fieldName, e.getMessage());
-                    }
-                    break;
-                }
-                for (String label : labels) {
-                    if (k.contains(label)) {
-                        try {
-                            ReflectUtil.setFieldValue(info, fieldName, fieldValue);
-                        } catch (Exception e) {
-                            this.logger.warn("{} 字段值设置错误：{}", fieldName, e.getMessage());
-                        }
-                        break;
-                    }
-                }
-            }
-        }
-    }
 }

service-daq/src/main/java/com/simuwang/daq/components/report/parser/pdf/PDQuarterlyReportParser.java

@@ -10,6 +10,11 @@ import technology.tabula.Table;
 
 import java.util.List;
 
+/**
+ * @author wangzaijun
+ * @date 2024/9/29 17:53
+ * @description pdf格式的季报解析逻辑
+ */
 @Component(ReportParserConstant.PARSER_PDF_QUARTERLY)
 public class PDQuarterlyReportParser extends AbstractPDReportParser<QuarterlyReportData> {
     public PDQuarterlyReportParser(EmailFieldMappingMapper fieldMappingMapper) {

service-daq/src/main/java/com/simuwang/daq/service/EmailParseService.java

@@ -355,12 +355,14 @@ public class EmailParseService {
         if (matcher.find()) {
             registerNumber = matcher.group();
         }
+        // 类型识别---先识别季度报告，没有季度再识别年度报告，最后识别月报
         ReportType reportType = ReportType.MONTHLY;
-        if (fileName.contains(ReportType.QUARTERLY.getLabel())) {
+        if (fileName.contains(ReportType.QUARTERLY.getPattern())) {
             reportType = ReportType.QUARTERLY;
-        } else if (fileName.contains(ReportType.ANNUALLY.getLabel())) {
+        } else if (fileName.contains(ReportType.ANNUALLY.getPattern())) {
             reportType = ReportType.ANNUALLY;
         }
+        // 解析器--如果开启python解析则直接调用python接口，否则根据文件后缀获取对应解析器
         ReportParserFileType fileType;
         if (Objects.equals(Boolean.TRUE, this.properties.getEnablePyParser())) {
             fileType = ReportParserFileType.PYTHON;