fix:定期报告解析

mo-daq/src/main/java/com/smppw/modaq/application/components/report/parser/pdf/AbstractPDReportParser.java

@@ -8,15 +8,13 @@ import cn.hutool.http.HttpUtil;
 import cn.hutool.json.JSONObject;
 import cn.hutool.json.JSONUtil;
 import com.smppw.modaq.application.components.CustomPDFTextStripper;
+import com.smppw.modaq.application.components.ReportParseUtils;
 import com.smppw.modaq.application.components.report.parser.AbstractReportParser;
 import com.smppw.modaq.common.conts.Constants;
 import com.smppw.modaq.common.enums.ReportParseStatus;
 import com.smppw.modaq.common.enums.ReportType;
 import com.smppw.modaq.common.exception.ReportParseException;
-import com.smppw.modaq.domain.dto.report.ReportBaseInfoDTO;
-import com.smppw.modaq.domain.dto.report.ReportData;
-import com.smppw.modaq.domain.dto.report.ReportFundInfoDTO;
-import com.smppw.modaq.domain.dto.report.ReportParserParams;
+import com.smppw.modaq.domain.dto.report.*;
 import com.smppw.modaq.domain.mapper.EmailFieldMappingMapper;
 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.io.RandomAccessReadBufferedFile;
@@ -29,8 +27,10 @@ import technology.tabula.Table;
 import technology.tabula.extractors.SpreadsheetExtractionAlgorithm;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 
 /**
  * @author wangzaijun
@@ -172,4 +172,50 @@ public abstract class AbstractPDReportParser<T extends ReportData> extends Abstr
         this.aiParserContent = null;
         this.aiParse = false;
     }
+
+    /**
+     * 构建只有两列表格的dto数据对象，如果有分级基金时（并且一个表格可能跨页）
+     *
+     * @param <DTO>    泛型对象
+     * @param fileId   文件id
+     * @param tables   表格
+     * @param clazz    泛型对象
+     * @param function 表格转换的函数
+     * @return /
+     */
+    protected <DTO extends BaseReportLevelDTO<?>> List<DTO> buildLevelDto(Integer fileId, List<Table> tables, Class<DTO> clazz,
+                                                                          Function<Table, Map<String, Object>> function) {
+        List<DTO> dtos = ListUtil.list(true);
+        // 信息表格字段和值映射
+        List<Map<String, Object>> infos = ListUtil.list(true);
+        Map<String, Object> infoMap = null;
+        for (Table table : tables) {
+            Map<String, Object> temp = function.apply(table);
+            for (String key : temp.keySet()) {
+                // 如果infoMap为null，先声明然后放在infos中
+                if (infoMap == null) {
+                    infoMap = MapUtil.newHashMap(16);
+                    infos.add(infoMap);
+                }
+                // 如果infoMap中包含了该key时，先放infos中然后重新声明新map对象
+                if (infoMap.containsKey(key)) {
+                    infos.add(new HashMap<>(infoMap));
+                    infoMap = MapUtil.newHashMap(16);
+                } else {
+                    infoMap.put(key, temp.get(key));
+                }
+            }
+        }
+        // 分级基金匹配
+        List<String> levels = ReportParseUtils.matchTieredFund(String.join(",", this.textList));
+        for (int i = 0; i < infos.size(); i++) {
+            DTO dto = this.buildDto(fileId, clazz, infos.get(i));
+            if (dto == null) {
+                continue;
+            }
+            dto.setLevel(levels.get(i));
+            dtos.add(dto);
+        }
+        return dtos;
+    }
 }

mo-daq/src/main/java/com/smppw/modaq/application/components/report/parser/pdf/PDMonthlyReportParser.java

@@ -1,15 +1,16 @@
 package com.smppw.modaq.application.components.report.parser.pdf;
 
+import cn.hutool.core.collection.ListUtil;
+import cn.hutool.core.map.MapUtil;
 import com.smppw.modaq.application.components.report.parser.ReportParserConstant;
-import com.smppw.modaq.domain.dto.report.MonthlyReportData;
-import com.smppw.modaq.domain.dto.report.ReportBaseInfoDTO;
-import com.smppw.modaq.domain.dto.report.ReportFundInfoDTO;
-import com.smppw.modaq.domain.dto.report.ReportParserParams;
+import com.smppw.modaq.domain.dto.report.*;
 import com.smppw.modaq.domain.mapper.EmailFieldMappingMapper;
 import org.springframework.stereotype.Component;
+import technology.tabula.RectangularTextContainer;
 import technology.tabula.Table;
 
 import java.util.List;
+import java.util.Map;
 
 /**
  * @author wangzaijun
@@ -18,7 +19,8 @@ import java.util.List;
  */
 @Component(ReportParserConstant.PARSER_PDF_MONTHLY)
 public class PDMonthlyReportParser extends AbstractPDReportParser<MonthlyReportData> {
-//    private List<Table> extNavTables;
+    private Table fundInfoTable;
+    private List<Table> extNavTables;
 
     public PDMonthlyReportParser(EmailFieldMappingMapper fieldMappingMapper) {
         super(fieldMappingMapper);
@@ -30,64 +32,60 @@ public class PDMonthlyReportParser extends AbstractPDReportParser<MonthlyReportD
     }
 
     @Override
+    protected void init() {
+        super.init();
+        // 这里初始化
+        this.extNavTables = ListUtil.list(true);
+    }
+
+    @Override
     protected void initTableInfo(List<Table> tables) {
-//        // 这里初始化
-//        this.extNavTables = ListUtil.list(true);
-//        // 一般月报是固定的模板，4列表格是基金基本信息，其他5列的表格是月净值
-//        for (Table table : tables) {
-//            int colCount = table.getColCount();
-//            int rowCount = table.getRowCount();
-//            if (colCount == 0 && rowCount == 0) {
-//                continue;
-//            }
-//            if (colCount == 4) {
-//                this.fundInfoTable = table;
-//            } else if (colCount >= 5) {
-//                this.extNavTables.add(table);
-//            }
-//        }
+        // 一般月报是固定的模板，4列表格是基金基本信息，其他5列的表格是月净值
+        for (Table table : tables) {
+            int colCount = table.getColCount();
+            int rowCount = table.getRowCount();
+            if (colCount == 0 && rowCount == 0) {
+                continue;
+            }
+            if (colCount == 4) {
+                this.fundInfoTable = table;
+            } else if (colCount >= 5) {
+                this.extNavTables.add(table);
+            }
+        }
     }
 
     @Override
     protected ReportFundInfoDTO buildFundInfo(ReportParserParams params) {
-        return null;
+        // 月报的基金基本信息是四列的表格
+        Map<String, Object> baseInfoMap = MapUtil.newHashMap(32);
+        for (int i = 0; i < fundInfoTable.getRows().size(); i++) {
+            @SuppressWarnings("all")
+            List<RectangularTextContainer> cols = fundInfoTable.getRows().get(i);
+            for (int j = 0; j < 2; j++) {
+                baseInfoMap.put(cols.get(j * 2).getText(), cols.get(j * 2 + 1).getText());
+            }
+        }
+        return this.buildDto(params.getFileId(), ReportFundInfoDTO.class, baseInfoMap);
     }
 
     @Override
     protected MonthlyReportData parseExtInfoAndSetData(ReportBaseInfoDTO reportInfo, ReportFundInfoDTO fundInfo) {
-        return null;
+        MonthlyReportData reportData = new MonthlyReportData(reportInfo, fundInfo);
+        // 母基金和分级基金的净值
+        List<ReportNetReportDTO> dtos = this.buildLevelDto(reportInfo.getFileId(), this.extNavTables,
+                ReportNetReportDTO.class, t -> {
+                    Map<String, Object> extInfoMap = MapUtil.newHashMap(16);
+                    for (int i = 0; i < t.getColCount(); i++) {
+                        String key = t.getCell(0, i).getText();
+                        String value = t.getCell(1, i).getText();
+                        extInfoMap.put(key, value);
+                    }
+                    return extInfoMap;
+                });
+        reportData.setNetReport(dtos);
+        return reportData;
     }
-//    @Override
-//    protected Map<String, Object> parseFundInfo(Table fundInfoTable) {
-//        // 月报的基金基本信息是四列的表格
-//        Map<String, Object> baseInfoMap = MapUtil.newHashMap(32);
-//        for (int i = 0; i < fundInfoTable.getRows().size(); i++) {
-//            @SuppressWarnings("all")
-//            List<RectangularTextContainer> cols = fundInfoTable.getRows().get(i);
-//            for (int j = 0; j < 2; j++) {
-//                baseInfoMap.put(cols.get(j * 2).getText(), cols.get(j * 2 + 1).getText());
-//            }
-//        }
-//        return baseInfoMap;
-//    }
-//
-//    @Override
-//    protected MonthlyReportData parseExtInfoAndSetData(ReportBaseInfoDTO reportInfo, ReportFundInfoDTO fundInfo) {
-//        MonthlyReportData reportData = new MonthlyReportData(reportInfo, fundInfo);
-////        // 母基金和分级基金的净值
-////        List<ReportNetReportDTO> dtos = this.buildLevelDto(reportInfo.getFileId(), this.extNavTables,
-////                ReportNetReportDTO.class, t -> {
-////                    Map<String, Object> extInfoMap = MapUtil.newHashMap(16);
-////                    for (int i = 0; i < t.getColCount(); i++) {
-////                        String key = t.getCell(0, i).getText();
-////                        String value = t.getCell(1, i).getText();
-////                        extInfoMap.put(key, value);
-////                    }
-////                    return extInfoMap;
-////                });
-////        reportData.setNetReport(dtos);
-//        return reportData;
-//    }
 
     @Override
     protected void cleaningReportData(MonthlyReportData reportData) {

mo-daq/src/main/java/com/smppw/modaq/application/components/report/parser/pdf/PDQuarterlyReportParser.java

@@ -15,7 +15,10 @@ import technology.tabula.RectangularTextContainer;
 import technology.tabula.Table;
 
 import java.awt.geom.Rectangle2D;
-import java.util.*;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.function.Function;
 
 /**
@@ -278,50 +281,4 @@ public class PDQuarterlyReportParser<T extends QuarterlyReportData> extends Abst
         }
         return details;
     }
-
-    /**
-     * 构建只有两列表格的dto数据对象，如果有分级基金时（并且一个表格可能跨页）
-     *
-     * @param <DTO>    泛型对象
-     * @param fileId   文件id
-     * @param tables   表格
-     * @param clazz    泛型对象
-     * @param function 表格转换的函数
-     * @return /
-     */
-    protected <DTO extends BaseReportLevelDTO<?>> List<DTO> buildLevelDto(Integer fileId, List<Table> tables, Class<DTO> clazz,
-                                                                          Function<Table, Map<String, Object>> function) {
-        List<DTO> dtos = ListUtil.list(true);
-        // 信息表格字段和值映射
-        List<Map<String, Object>> infos = ListUtil.list(true);
-        Map<String, Object> infoMap = null;
-        for (Table table : tables) {
-            Map<String, Object> temp = function.apply(table);
-            for (String key : temp.keySet()) {
-                // 如果infoMap为null，先声明然后放在infos中
-                if (infoMap == null) {
-                    infoMap = MapUtil.newHashMap(16);
-                    infos.add(infoMap);
-                }
-                // 如果infoMap中包含了该key时，先放infos中然后重新声明新map对象
-                if (infoMap.containsKey(key)) {
-                    infos.add(new HashMap<>(infoMap));
-                    infoMap = MapUtil.newHashMap(16);
-                } else {
-                    infoMap.put(key, temp.get(key));
-                }
-            }
-        }
-        // 分级基金匹配
-        List<String> levels = ReportParseUtils.matchTieredFund(String.join(",", this.textList));
-        for (int i = 0; i < infos.size(); i++) {
-            DTO dto = this.buildDto(fileId, clazz, infos.get(i));
-            if (dto == null) {
-                continue;
-            }
-            dto.setLevel(levels.get(i));
-            dtos.add(dto);
-        }
-        return dtos;
-    }
 }

mo-daq/src/main/java/com/smppw/modaq/application/components/report/writer/MonthlyReportWriter.java

@@ -1,25 +1,35 @@
 package com.smppw.modaq.application.components.report.writer;
 
+import cn.hutool.core.collection.CollUtil;
 import com.smppw.modaq.domain.dto.report.MonthlyReportData;
+import com.smppw.modaq.domain.dto.report.ReportNetReportDTO;
+import com.smppw.modaq.domain.entity.report.ReportNetReportDO;
 import com.smppw.modaq.domain.mapper.report.ReportBaseInfoMapper;
 import com.smppw.modaq.domain.mapper.report.ReportFundInfoMapper;
+import com.smppw.modaq.domain.mapper.report.ReportNetReportMapper;
 import org.springframework.stereotype.Component;
 
+import java.util.List;
+import java.util.stream.Collectors;
+
 @Component(ReportWriterConstant.WRITER_MONTHLY)
 public class MonthlyReportWriter extends AbstractReportWriter<MonthlyReportData> {
+    private final ReportNetReportMapper netReportMapper;
 
     public MonthlyReportWriter(ReportBaseInfoMapper baseInfoMapper,
-                               ReportFundInfoMapper fundInfoMapper) {
+                               ReportFundInfoMapper fundInfoMapper,
+                               ReportNetReportMapper netReportMapper) {
         super(baseInfoMapper, fundInfoMapper);
+        this.netReportMapper = netReportMapper;
     }
 
     @Override
     protected void writeExtData(MonthlyReportData reportData) {
-//        List<ReportNetReportDTO> netReport = reportData.getNetReport();
-//        if (CollUtil.isNotEmpty(netReport)) {
-//            List<ReportNetReportDO> entityList = netReport.stream()
-//                    .map(ReportNetReportDTO::toEntity).collect(Collectors.toList());
-//            this.netReportMapper.insert(entityList);
-//        }
+        List<ReportNetReportDTO> netReport = reportData.getNetReport();
+        if (CollUtil.isNotEmpty(netReport)) {
+            List<ReportNetReportDO> entityList = netReport.stream()
+                    .map(ReportNetReportDTO::toEntity).collect(Collectors.toList());
+            this.netReportMapper.insert(entityList);
+        }
     }
 }

mo-daq/src/main/java/com/smppw/modaq/domain/dto/report/MonthlyReportData.java

@@ -4,10 +4,12 @@ import com.smppw.modaq.common.enums.ReportType;
 import lombok.Getter;
 import lombok.Setter;
 
+import java.util.List;
+
 @Setter
 @Getter
 public class MonthlyReportData extends ReportData {
-//    private List<ReportNetReportDTO> netReport;
+    private List<ReportNetReportDTO> netReport;
 
     public MonthlyReportData(ReportBaseInfoDTO baseInfo, ReportFundInfoDTO fundInfo) {
         super(baseInfo, fundInfo);
@@ -20,6 +22,9 @@ public class MonthlyReportData extends ReportData {
 
     @Override
     public String toString() {
-        return super.toString();
+        return "{" +
+                super.toString() +
+                ", netReport=" + netReport +
+                '}';
     }
 }