package com.simuwang.daq.service; import cn.hutool.core.bean.BeanUtil; import cn.hutool.core.collection.CollUtil; import cn.hutool.core.collection.CollectionUtil; import cn.hutool.core.collection.ListUtil; import cn.hutool.core.date.DateUtil; import cn.hutool.core.exceptions.ExceptionUtil; import cn.hutool.core.map.MapUtil; import cn.hutool.core.util.StrUtil; import com.simuwang.base.common.conts.*; import com.simuwang.base.common.enums.ReportParserFileType; import com.simuwang.base.common.enums.ReportType; import com.simuwang.base.common.exception.ReportParseException; import com.simuwang.base.common.util.*; import com.simuwang.base.config.DaqProperties; import com.simuwang.base.config.EmailRuleConfig; import com.simuwang.base.mapper.*; import com.simuwang.base.pojo.dos.*; import com.simuwang.base.pojo.dto.EmailContentInfoDTO; import com.simuwang.base.pojo.dto.EmailFundNavDTO; import com.simuwang.base.pojo.dto.MailboxInfoDTO; import com.simuwang.base.pojo.dto.query.DataboardQuery; import com.simuwang.base.pojo.dto.report.ParseResult; import com.simuwang.base.pojo.dto.report.ReportData; import com.simuwang.base.pojo.dto.report.ReportParseStatus; import com.simuwang.base.pojo.dto.report.ReportParserParams; import com.simuwang.base.pojo.valuation.CmValuationTableAttribute; import com.simuwang.base.pojo.vo.*; import com.simuwang.daq.components.report.parser.ReportParser; import com.simuwang.daq.components.report.parser.ReportParserFactory; import com.simuwang.daq.components.report.writer.ReportWriter; import com.simuwang.daq.components.report.writer.ReportWriterFactory; import jakarta.mail.*; import jakarta.mail.internet.MimeMessage; import jakarta.mail.internet.MimeMultipart; import jakarta.mail.search.ComparisonTerm; import jakarta.mail.search.ReceivedDateTerm; import jakarta.mail.search.SearchTerm; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.util.StopWatch; import java.io.File; import java.math.BigDecimal; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; /** * @author mozuwen * @date 2024-09-04 * @description 邮件解析服务 */ @Service public class EmailParseService { public static final int stepSize = 10000; private static final Logger log = LoggerFactory.getLogger(EmailParseService.class); private final EmailTypeRuleMapper emailTypeRuleMapper; private final EmailRuleConfig emailRuleConfig; private final EmailFieldMappingMapper emailFieldMapper; private final EmailParserFactory emailParserFactory; private final EmailParseInfoMapper emailParseInfoMapper; private final EmailFileInfoMapper emailFileInfoMapper; private final EmailFundNavMapper emailFundNavMapper; private final EmailFundAssetMapper emailFundAssetMapper; private final AssetMapper assetMapper; private final NavMapper navMapper; private final FundService fundService; private final FundAliasMapper fundAliasMapper; private final ValuationTableMapper valuationTableMapper; private final ValuationTableAttributeMapper valuationTableAttributeMapper; private final FundPositionDetailMapper fundPositionDetailMapper; private final DistributionMapper distributionMapper; private final CompanyInformationMapper companyInformationMapper; @Autowired private FundInfoMapper fundInfoMapper; @Value("${email.file.path}") private String path; @Autowired private DaqProperties properties; /* 报告解析和入库的方法 */ @Autowired private ReportParserFactory reportParserFactory; @Autowired private ReportWriterFactory reportWriterFactory; public EmailParseService(EmailTypeRuleMapper emailTypeRuleMapper, EmailRuleConfig emailRuleConfig, EmailFieldMappingMapper emailFieldMapper, EmailParserFactory emailParserFactory, EmailParseInfoMapper emailParseInfoMapper, EmailFileInfoMapper emailFileInfoMapper, EmailFundNavMapper emailFundNavMapper, EmailFundAssetMapper emailFundAssetMapper, AssetMapper assetMapper, NavMapper navMapper, FundService fundService, FundAliasMapper fundAliasMapper, ValuationTableMapper valuationTableMapper, ValuationTableAttributeMapper valuationTableAttributeMapper, FundPositionDetailMapper fundPositionDetailMapper, DistributionMapper distributionMapper, CompanyInformationMapper companyInformationMapper) { this.emailTypeRuleMapper = emailTypeRuleMapper; this.emailRuleConfig = emailRuleConfig; this.emailFieldMapper = emailFieldMapper; this.emailParserFactory = emailParserFactory; this.emailParseInfoMapper = emailParseInfoMapper; this.emailFileInfoMapper = emailFileInfoMapper; this.emailFundNavMapper = emailFundNavMapper; this.emailFundAssetMapper = emailFundAssetMapper; this.assetMapper = assetMapper; this.navMapper = navMapper; this.fundService = fundService; this.fundAliasMapper = fundAliasMapper; this.valuationTableMapper = valuationTableMapper; this.valuationTableAttributeMapper = valuationTableAttributeMapper; this.fundPositionDetailMapper = fundPositionDetailMapper; this.distributionMapper = distributionMapper; this.companyInformationMapper = companyInformationMapper; } /** * 解析指定邮箱指定时间范围内的邮件 * * @param mailboxInfoDTO 邮箱配置信息 * @param startDate 邮件起始日期(yyyy-MM-dd HH:mm:ss) * @param endDate 邮件截止日期(yyyy-MM-dd HH:mm:ss, 为null,将解析邮件日期小于等于startDate的当天邮件) */ public void parseEmail(MailboxInfoDTO mailboxInfoDTO, Date startDate, Date endDate) { log.info("开始邮件解析 -> 邮箱信息:{},开始时间:{},结束时间:{}", mailboxInfoDTO, DateUtil.format(startDate, DateConst.YYYY_MM_DD_HH_MM_SS), DateUtil.format(endDate, DateConst.YYYY_MM_DD_HH_MM_SS)); // 邮件类型配置 Map> emailTypeMap = getEmailType(); // 邮件字段识别映射表 Map> emailFieldMap = getEmailFieldMapping(); Map> emailContentMap; try { emailContentMap = realEmail(mailboxInfoDTO, emailTypeMap, startDate, endDate); } catch (Exception e) { log.info("采集邮件失败 -> 邮箱配置信息:{},堆栈信息:{}", mailboxInfoDTO, ExceptionUtil.stacktraceToString(e)); return; } if (MapUtil.isEmpty(emailContentMap)) { log.info("未采集到邮件 -> 邮箱配置信息:{},开始时间:{},结束时间:{}", mailboxInfoDTO, DateUtil.format(startDate, DateConst.YYYY_MM_DD_HH_MM_SS), DateUtil.format(endDate, DateConst.YYYY_MM_DD_HH_MM_SS)); return; } for (Map.Entry> emailEntry : emailContentMap.entrySet()) { List emailContentInfoDTOList = emailEntry.getValue(); if (CollUtil.isEmpty(emailContentInfoDTOList)) { log.warn("未采集到正文或附件"); continue; } log.info("开始解析邮件数据 -> 邮件主题:{},邮件日期:{}", emailContentInfoDTOList.get(0).getEmailTitle(), emailContentInfoDTOList.get(0).getEmailDate()); List emailFundNavDTOList = CollUtil.newArrayList(); Map> fileNameNavMap = MapUtil.newHashMap(); for (EmailContentInfoDTO emailContentInfoDTO : emailContentInfoDTOList) { try { List fundNavDTOList = parseEmail(emailContentInfoDTO, emailFieldMap); fileNameNavMap.put(emailContentInfoDTO, fundNavDTOList); emailFundNavDTOList.addAll(fundNavDTOList); } catch (Exception e) { log.error("堆栈信息:{}", ExceptionUtil.stacktraceToString(e)); } } // 保存相关信息 -> 邮件信息表,邮件文件表,邮件净值表,邮件规模表,基金净值表 saveRelatedTable(mailboxInfoDTO.getAccount(), emailContentInfoDTOList, fileNameNavMap); log.info("结束邮件解析 -> 邮箱信息:{},开始时间:{},结束时间:{}", mailboxInfoDTO, DateUtil.format(startDate, DateConst.YYYY_MM_DD_HH_MM_SS), DateUtil.format(endDate, DateConst.YYYY_MM_DD_HH_MM_SS)); } } public List parseEmail(EmailContentInfoDTO emailContentInfoDTO, Map> emailFieldMap) { Integer emailType = emailContentInfoDTO.getEmailType(); AbstractEmailParser emailParser = emailParserFactory.getInstance(emailType); return emailParser.parse(emailContentInfoDTO, emailFieldMap); } public void saveRelatedTable(String emailAddress, List emailContentInfoDTOList, Map> fileNameNavMap) { String emailTitle = CollUtil.isNotEmpty(emailContentInfoDTOList) ? emailContentInfoDTOList.get(0).getEmailTitle() : null; String emailDate = CollUtil.isNotEmpty(emailContentInfoDTOList) ? emailContentInfoDTOList.get(0).getEmailDate() : null; Integer emailType = CollUtil.isNotEmpty(emailContentInfoDTOList) ? emailContentInfoDTOList.get(0).getEmailType() : null; Integer emailId = CollUtil.isNotEmpty(emailContentInfoDTOList) ? emailContentInfoDTOList.get(0).getEmailId() : null; String senderEmail = CollUtil.isNotEmpty(emailContentInfoDTOList) ? emailContentInfoDTOList.get(0).getSenderEmail() : null; Date parseDate = new Date(); int emailParseStatus = EmailParseStatusConst.SUCCESS; EmailParseInfoDO emailParseInfoDO = buildEmailParseInfo(emailId, emailAddress, senderEmail, emailDate, emailTitle, emailType, emailParseStatus, parseDate); emailId = saveEmailParseInfo(emailParseInfoDO); // python 报告解析接口结果 List> dataList = ListUtil.list(false); for (Map.Entry> fileNameNavEntry : fileNameNavMap.entrySet()) { // 保存邮件文件表 EmailContentInfoDTO emailContentInfoDTO = fileNameNavEntry.getKey(); String fileName = emailContentInfoDTO.getFileName(); if (Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType) && fileName.endsWith(".html")) { continue; } Integer fileId = saveEmailFileInfo(emailId, emailContentInfoDTO.getFileId(), fileName, emailContentInfoDTO.getFilePath(), parseDate); List fundNavDTOList = fileNameNavEntry.getValue(); if (CollUtil.isNotEmpty(fundNavDTOList)) { // 过滤出解析成功的数据 fundNavDTOList = fundNavDTOList.stream().filter(e -> e != null && StrUtil.isBlank(e.getFailReason())).toList(); } if (CollUtil.isEmpty(fundNavDTOList) && !Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType)) { continue; } if (Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType)) { // 解析结果（可以从python获取或者自行解析）并保存报告 ParseResult parseResult = this.parseReportAndHandleResult(fileId, emailContentInfoDTO); dataList.add(parseResult); } for (EmailFundNavDTO fundNavDTO : fundNavDTOList) { // 设置净值数据的解析状态 setNavParseStatus(fundNavDTO, emailTitle); } // 保存净值表和规模表 saveNavAndAssetNet(fileId, fundNavDTOList, parseDate); saveValuationInfo(fileId, fundNavDTOList); } // 更新邮件解析结果 -> 当【净值日期】和【备案编码/基金名称】能正常解读，即识别为【成功】 long successNavCount = fileNameNavMap.values().stream().flatMap(List::stream).filter(e -> e != null && StrUtil.isBlank(e.getFailReason())).count(); emailParseStatus = successNavCount >= 1 ? EmailParseStatusConst.SUCCESS : EmailParseStatusConst.FAIL; String failReason = null; if (emailParseStatus == EmailParseStatusConst.FAIL) { // 邮件解析失败时 -> 保存失败原因 int hasPdfFile = emailContentInfoDTOList.stream().map(EmailContentInfoDTO::getFilePath).anyMatch(ExcelUtil::isPdf) ? 1 : 0; List navDTOList = fileNameNavMap.values().stream().flatMap(List::stream).toList(); failReason = hasPdfFile == 1 && CollUtil.isEmpty(navDTOList) ? "无法从PDF文件中获取到数据" : navDTOList.stream().map(EmailFundNavDTO::getFailReason).distinct().collect(Collectors.joining("/")); if (StrUtil.isBlank(failReason)) { failReason = "模板不支持"; } } // 报告邮件有一条失败就表示整个邮件解析失败 if (Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType) && CollUtil.isNotEmpty(dataList)) { long sucNum = dataList.stream().filter(e -> Objects.equals(1, e.getStatus())).count(); if (sucNum > 0) { emailParseStatus = EmailParseStatusConst.SUCCESS; } else { emailParseStatus = EmailParseStatusConst.FAIL; failReason = dataList.stream().map(ParseResult::getMsg).collect(Collectors.joining("/")); } } emailParseInfoMapper.updateParseStatus(emailId, emailParseStatus, failReason); } private void saveValuationInfo(Integer fileId, List fundNavDTOList) { if (CollUtil.isEmpty(fundNavDTOList)) { return; } valuationTableMapper.unValid(fileId); for (EmailFundNavDTO fundNavDTO : fundNavDTOList) { ValuationTableDO valuationTableDO = fundNavDTO.getValuationTableDO(); if (valuationTableDO == null) { continue; } List fundIdList = fundNavDTO.getFundIdList(); valuationTableDO.setFileId(fileId); List fundPositionDetailDOList = fundNavDTO.getFundPositionDetailDOList(); List valuationTableAttributeList = fundNavDTO.getValuationTableAttributeList(); if (CollUtil.isEmpty(fundIdList)) { valuationTableMapper.insert(valuationTableDO); int valuationId = valuationTableDO.getId(); saveValuationTableAttribute(valuationId, valuationTableAttributeList); fundPositionDetailDOList.forEach(e -> e.setValuationId(valuationId)); // 不匹配基金的情况下 -> 不写fund_position_detail // saveFundPositionDetail(fundPositionDetailDOList, null, fundNavDTO.getPriceDate()); continue; } for (String fundId : fundIdList) { valuationTableDO.setFundId(fundId); valuationTableMapper.insert(valuationTableDO); int valuationId = valuationTableDO.getId(); fundPositionDetailDOList.forEach(e -> { e.setFundId(fundId); e.setValuationId(valuationId); }); saveValuationTableAttribute(valuationId, valuationTableAttributeList); saveFundPositionDetail(fundPositionDetailDOList, fundId, fundNavDTO.getPriceDate()); } } } public void saveFundPositionDetail(List fundPositionDetails, String fundId, String valuationDate) { int subBegin = 0; int subEnd = stepSize; int insertNum = 0; int hasStock = 0; int hasBond = 0; int hasFuture = 0; if (CollectionUtil.isNotEmpty(fundPositionDetails)) { //插入持仓数据 cm_fund_position_detail（记录数较多，得分批） // 先删除原先的数据 然后写入数据 fundPositionDetailMapper.deleteUnUsed(fundId, valuationDate); // 将最终结果写入 cm_fund_position_detail while (subBegin < fundPositionDetails.size()) { List segment = fundPositionDetails.subList(subBegin, Math.min(subEnd, fundPositionDetails.size())); for (FundPositionDetailDO fundPositionDetail : segment) { //实收信托、实收资本对应的编码为107 if (StrUtil.isNotBlank(fundPositionDetail.getSecuritiesName()) && "实收信托".equals(fundPositionDetail.getSecuritiesName().trim())) { fundPositionDetail.setSubjectCode("107"); fundPositionDetail.setSecuritiesCode("107"); } //设置创建者id,在拿取最近修改人时有用 fundPositionDetail.setCreatorId(0); if (fundPositionDetail.getLevel() != null && fundPositionDetail.getLevel() >= 4) { Integer secType = fundPositionDetail.getSecType(); Integer subType = fundPositionDetail.getSubType(); if (secType != null && subType != null) { if (secType == HoldingType.Stock.getId() && subType == HoldingType.Stock.getId()) { hasStock = 1; } // 正逆回购不属于债券 String subjectCode = fundPositionDetail.getSubjectCode(); String[] SALE_CODES = {"2202", "1202"}; if (secType == HoldingType.Bond.getId() && !StrUtil.containsAny(subjectCode, SALE_CODES)) { hasBond = 1; } if (secType == HoldingType.Future.getId() || secType == HoldingType.Option.getId()) { hasFuture = 1; } } } } insertNum += fundPositionDetailMapper.insertMulti(segment); subBegin = subEnd; subEnd += stepSize; } } // 更新 cm_user_valuation_table if (insertNum > 0) { valuationTableMapper.updateUpdateAnalyzed(fundId, valuationDate, hasStock, hasBond, hasFuture); } } private void saveValuationTableAttribute(Integer valuationId, List valuationTableAttributes) { if (valuationId == null || CollUtil.isEmpty(valuationTableAttributes)) { return; } List valuationTableAttributeDOList = buildValuationTableAttributeDO(valuationId, valuationTableAttributes); valuationTableAttributeMapper.deleteByValuationId(valuationId); if (CollUtil.isNotEmpty(valuationTableAttributeDOList)) { valuationTableAttributeMapper.batchInsert(valuationTableAttributeDOList); } } private List buildValuationTableAttributeDO(Integer valuationId, List valuationTableAttributes) { if (CollUtil.isEmpty(valuationTableAttributes)) { return CollUtil.newArrayList(); } return valuationTableAttributes.stream().map(e -> { ValuationTableAttributeDO tableAttributeDO = new ValuationTableAttributeDO(); tableAttributeDO.setValuationId(valuationId); tableAttributeDO.setSubjectCode(e.getOriginalSubjectCode()); tableAttributeDO.setSubjectName(e.getSubjectName()); tableAttributeDO.setCurrency(e.getCurrency()); tableAttributeDO.setExchangeRate(e.getExchangeRate()); tableAttributeDO.setSecuritiesAmount(e.getSecuritiesAmount()); tableAttributeDO.setUnitCost(e.getUnitCost()); tableAttributeDO.setOriCurrencyCost(e.getOCurrencyCost()); tableAttributeDO.setNetCost(e.getNetCost()); tableAttributeDO.setNetCostRatio(e.getNetCostRatio()); tableAttributeDO.setMarketPrice(e.getMarketPrice()); tableAttributeDO.setOriCurrencyMarketValue(e.getOCurrencyMarketValue()); tableAttributeDO.setMarketValue(e.getMarketValue()); tableAttributeDO.setMarketValueRatio(e.getMarketValueRatio()); tableAttributeDO.setOriCurrencyIncrement(e.getOCurrencyMarketValue()); tableAttributeDO.setIncrement(e.getIncrement()); tableAttributeDO.setHaltInfo(e.getHaltInfo()); tableAttributeDO.setRightsInterestsInfo(e.getRightsInterestsInfo()); tableAttributeDO.setIsvalid(1); tableAttributeDO.setCreatorId(0); tableAttributeDO.setUpdaterId(0); tableAttributeDO.setCreateTime(new Date()); tableAttributeDO.setUpdateTime(new Date()); return tableAttributeDO; }).collect(Collectors.toList()); } private ParseResult parseReportAndHandleResult(int fileId, EmailContentInfoDTO emailContentInfoDTO) { ParseResult result = new ParseResult<>(); String fileName = emailContentInfoDTO.getFileName(); Integer emailType = emailContentInfoDTO.getEmailType(); if (!Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType) || StrUtil.isBlank(fileName)) { result.setStatus(ReportParseStatus.NOT_A_REPORT.getCode()); result.setMsg(ReportParseStatus.NOT_A_REPORT.getMsg()); return result; } Pattern pattern = Pattern.compile("S(?:[A-Z]{0}[0-9]{5}|[A-Z][0-9]{4}|[A-Z]{2}[0-9]{3}|[A-Z]{3}[0-9]{2})"); Matcher matcher = pattern.matcher(fileName); String registerNumber = null; if (matcher.find()) { registerNumber = matcher.group(); } // 类型识别---先识别季度报告，没有季度再识别年度报告，最后识别月报 ReportType reportType = null; if (StrUtil.containsAny(fileName, ReportType.QUARTERLY.getPatterns())) { reportType = ReportType.QUARTERLY; } else if (StrUtil.containsAny(fileName, ReportType.ANNUALLY.getPatterns())) { reportType = ReportType.ANNUALLY; } else if (StrUtil.containsAny(fileName, ReportType.MONTHLY.getPatterns())) { reportType = ReportType.MONTHLY; } // 解析器--如果开启python解析则直接调用python接口，否则根据文件后缀获取对应解析器 ReportParserFileType fileType; if (Objects.equals(Boolean.TRUE, this.properties.getEnablePyParser())) { fileType = ReportParserFileType.PYTHON; } else { String fileSuffix = StrUtil.subAfter(fileName, ".", true); fileType = ReportParserFileType.getBySuffix(fileSuffix); } // 不支持的格式 if (fileType == null) { result.setStatus(ReportParseStatus.NO_SUPPORT_TEMPLATE.getCode()); result.setMsg(StrUtil.format(ReportParseStatus.NO_SUPPORT_TEMPLATE.getMsg(), fileName)); return result; } // 不是定期报告的判断逻辑放在不支持的格式下面 if (reportType == null) { result.setStatus(ReportParseStatus.NOT_A_REPORT.getCode()); result.setMsg(StrUtil.format(ReportParseStatus.NOT_A_REPORT.getMsg(), fileName)); return result; } // 解析报告 ReportData reportData = null; StopWatch parserWatch = new StopWatch(); parserWatch.start(); try { ReportParserParams params = ReportParserParams.builder().fileId(fileId).filename(fileName) .filepath(emailContentInfoDTO.getFilePath()).registerNumber(registerNumber).build(); ReportParser instance = this.reportParserFactory.getInstance(reportType, fileType); reportData = instance.parse(params); result.setStatus(1); result.setMsg("报告解析成功"); result.setData(reportData); } catch (ReportParseException e) { log.error("解析失败\n{}", e.getMsg()); result.setStatus(e.getCode()); result.setMsg(e.getMsg()); } catch (Exception e) { log.error("解析错误\n{}", ExceptionUtil.stacktraceToString(e)); result.setStatus(ReportParseStatus.PARSE_FAIL.getCode()); result.setMsg(StrUtil.format(ReportParseStatus.PARSE_FAIL.getMsg(), e.getMessage())); } finally { parserWatch.stop(); if (log.isInfoEnabled()) { log.info("报告{}解析结果为{}，耗时{}ms", fileName, reportData, parserWatch.getTotalTimeMillis()); } } // 保存报告解析结果 if (reportData != null) { StopWatch writeWatch = new StopWatch(); writeWatch.start(); try { ReportWriter instance = this.reportWriterFactory.getInstance(reportType); instance.write(reportData); } catch (Exception e) { log.error("报告{}结果保存失败\n{}", fileName, ExceptionUtil.stacktraceToString(e)); } finally { writeWatch.stop(); if (log.isInfoEnabled()) { log.info("报告{}解析结果保存完成，耗时{}ms", fileName, writeWatch.getTotalTimeMillis()); } } } return result; } private void saveNavAndAssetNet(Integer fileId, List fundNavDTOList, Date parseDate) { if (CollUtil.isEmpty(fundNavDTOList)) { return; } // 净值数据 List emailFundNavDOList = fundNavDTOList.stream() .map(e -> buildEmailFundNavDo(fileId, e, parseDate)).filter(CollUtil::isNotEmpty).flatMap(List::stream).collect(Collectors.toList()); if (CollUtil.isNotEmpty(emailFundNavDOList)) { // 先删除文件id下的净值数据(考虑到重新解析的需求,如果是首次解析,那么file_id下不存在净值数据) emailFundNavMapper.deleteByFileId(fileId); emailFundNavMapper.batchInsert(emailFundNavDOList); List navDOList = emailFundNavDOList.stream().filter(e -> StrUtil.isNotBlank(e.getFundId())) .map(e -> BeanUtil.copyProperties(e, NavDO.class)).collect(Collectors.toList()); saveNavDo(navDOList); } // 保存规模数据 List emailFundAssetDOList = fundNavDTOList.stream() .map(e -> buildEmailFundAssetDo(fileId, e, parseDate)).filter(CollUtil::isNotEmpty).flatMap(List::stream).collect(Collectors.toList()); if (CollUtil.isNotEmpty(emailFundAssetDOList)) { // 先删除file_id下的规模数据(考虑到重新解析的需求,如果是首次解析,那么file_id下不存在规模数据) emailFundAssetMapper.deleteByFileId(fileId); emailFundAssetMapper.batchInsert(emailFundAssetDOList); List assetDOList = emailFundAssetDOList.stream().filter(e -> StrUtil.isNotBlank(e.getFundId())) .map(e -> BeanUtil.copyProperties(e, AssetDO.class)).collect(Collectors.toList()); saveAssetDo(assetDOList); } } public void saveNavDo(List navDOList) { if (CollUtil.isEmpty(navDOList)) { return; } Map> fundIdNavMap = navDOList.stream().collect(Collectors.groupingBy(NavDO::getFundId)); for (Map.Entry> entry : fundIdNavMap.entrySet()) { List navDOS = entry.getValue(); List priceDateList = navDOS.stream().map(NavDO::getPriceDate).map(e -> DateUtil.format(e, DateConst.YYYY_MM_DD)).collect(Collectors.toList()); List dateList = navMapper.queryFundNavByDate(entry.getKey(), priceDateList); List updateNavDoList = navDOS.stream().filter(e -> dateList.contains(DateUtil.format(e.getPriceDate(), DateConst.YYYY_MM_DD))).collect(Collectors.toList()); List insertNavDoList = navDOS.stream().filter(e -> !dateList.contains(DateUtil.format(e.getPriceDate(), DateConst.YYYY_MM_DD))).collect(Collectors.toList()); if (CollUtil.isNotEmpty(insertNavDoList)) { Map> priceDateNavDoListMap = insertNavDoList.stream().collect(Collectors.groupingBy(NavDO::getPriceDate)); boolean hasDuplicationDateData = priceDateNavDoListMap.values().stream().map(List::size).anyMatch(e -> e > 1); if (!hasDuplicationDateData) { navMapper.batchInsert(insertNavDoList); } // 要插入的数据中存在相同日期的数据 -> 只能一条一条的插入数据了 insertNavDoList.forEach(e -> saveNavDo(ListUtil.toList(e))); } if (CollUtil.isNotEmpty(updateNavDoList)) { navMapper.batchUpdate(updateNavDoList); } } } public void saveAssetDo(List assetDOList) { if (CollUtil.isEmpty(assetDOList)) { return; } Map> fundIdNavMap = assetDOList.stream().collect(Collectors.groupingBy(AssetDO::getFundId)); for (Map.Entry> entry : fundIdNavMap.entrySet()) { List assetDOS = entry.getValue(); List priceDateList = assetDOS.stream().map(AssetDO::getPriceDate).map(e -> DateUtil.format(e, DateConst.YYYY_MM_DD)).collect(Collectors.toList()); List dateList = assetMapper.queryFundNavByDate(entry.getKey(), priceDateList); List updateAssetDoList = assetDOS.stream().filter(e -> dateList.contains(DateUtil.format(e.getPriceDate(), DateConst.YYYY_MM_DD))).collect(Collectors.toList()); List insertAssetDoList = assetDOS.stream().filter(e -> !dateList.contains(DateUtil.format(e.getPriceDate(), DateConst.YYYY_MM_DD))).collect(Collectors.toList()); if (CollUtil.isNotEmpty(insertAssetDoList)) { Map> priceDateAssetDoListMap = insertAssetDoList.stream().collect(Collectors.groupingBy(AssetDO::getPriceDate)); boolean hasDuplicationDateData = priceDateAssetDoListMap.values().stream().map(List::size).anyMatch(e -> e > 1); if (!hasDuplicationDateData) { assetMapper.batchInsert(insertAssetDoList); } // 要插入的数据中存在相同日期的数据 -> 只能一条一条的插入数据了 insertAssetDoList.forEach(e -> saveAssetDo(ListUtil.toList(e))); } if (CollUtil.isNotEmpty(updateAssetDoList)) { assetMapper.batchUpdate(updateAssetDoList); } } } private List buildEmailFundAssetDo(Integer fileId, EmailFundNavDTO fundNavDTO, Date parseDate) { List fundAssetDOList = CollUtil.newArrayList(); BigDecimal assetNet = StrUtil.isNotBlank(fundNavDTO.getAssetNet()) ? new BigDecimal(fundNavDTO.getAssetNet()) : null; BigDecimal assetShare = StrUtil.isNotBlank(fundNavDTO.getAssetShare()) ? new BigDecimal(fundNavDTO.getAssetShare()) : null; if (assetNet == null) { return fundAssetDOList; } Integer isStored = fundNavDTO.getParseStatus() != null && (fundNavDTO.getParseStatus().equals(NavParseStatusConst.ASSET_NET_NEGATIVE) || fundNavDTO.getParseStatus().equals(NavParseStatusConst.SUCCESS)) ? 1 : 0; Date priceDate = DateUtil.parse(fundNavDTO.getPriceDate(), DateConst.YYYY_MM_DD); if (CollUtil.isNotEmpty(fundNavDTO.getFundIdList())) { for (String fundId : fundNavDTO.getFundIdList()) { EmailFundAssetDO emailFundAssetDO = new EmailFundAssetDO(); emailFundAssetDO.setFileId(fileId); emailFundAssetDO.setPriceDate(priceDate); emailFundAssetDO.setFundId(fundId); emailFundAssetDO.setFundName(fundNavDTO.getFundName()); emailFundAssetDO.setRegisterNumber(fundNavDTO.getRegisterNumber()); emailFundAssetDO.setAssetNet(assetNet); emailFundAssetDO.setAssetShare(assetShare); emailFundAssetDO.setIsStored(isStored); emailFundAssetDO.setExceptionStatus(fundNavDTO.getParseStatus()); emailFundAssetDO.setIsvalid(1); emailFundAssetDO.setCreatorId(0); emailFundAssetDO.setCreateTime(parseDate); emailFundAssetDO.setUpdaterId(0); emailFundAssetDO.setUpdateTime(parseDate); fundAssetDOList.add(emailFundAssetDO); } } else { EmailFundAssetDO emailFundAssetDO = new EmailFundAssetDO(); emailFundAssetDO.setFileId(fileId); emailFundAssetDO.setPriceDate(priceDate); emailFundAssetDO.setFundName(fundNavDTO.getFundName()); emailFundAssetDO.setRegisterNumber(fundNavDTO.getRegisterNumber()); emailFundAssetDO.setAssetNet(assetNet); emailFundAssetDO.setAssetShare(assetShare); emailFundAssetDO.setIsStored(isStored); emailFundAssetDO.setExceptionStatus(fundNavDTO.getParseStatus()); emailFundAssetDO.setIsvalid(1); emailFundAssetDO.setCreatorId(0); emailFundAssetDO.setCreateTime(parseDate); emailFundAssetDO.setUpdaterId(0); emailFundAssetDO.setUpdateTime(parseDate); fundAssetDOList.add(emailFundAssetDO); } return fundAssetDOList; } private List buildEmailFundNavDo(Integer fileId, EmailFundNavDTO fundNavDTO, Date parseDate) { List fundNavDOList = CollUtil.newArrayList(); Date priceDate = DateUtil.parse(fundNavDTO.getPriceDate(), DateConst.YYYY_MM_DD); BigDecimal nav = StrUtil.isNotBlank(fundNavDTO.getNav()) ? new BigDecimal(fundNavDTO.getNav()) : null; BigDecimal cumulativeNavWithdrawal = StrUtil.isNotBlank(fundNavDTO.getCumulativeNavWithdrawal()) ? new BigDecimal(fundNavDTO.getCumulativeNavWithdrawal()) : null; Integer isStored = fundNavDTO.getParseStatus() != null && !fundNavDTO.getParseStatus().equals(NavParseStatusConst.NAV_DEFICIENCY) && !fundNavDTO.getParseStatus().equals(NavParseStatusConst.NOT_MATCH) ? 1 : 0; if (CollUtil.isNotEmpty(fundNavDTO.getFundIdList())) { for (String fundId : fundNavDTO.getFundIdList()) { EmailFundNavDO emailFundNavDO = new EmailFundNavDO(); emailFundNavDO.setFileId(fileId); emailFundNavDO.setIsStored(isStored); emailFundNavDO.setPriceDate(priceDate); emailFundNavDO.setNav(nav); emailFundNavDO.setFundId(fundId); emailFundNavDO.setCumulativeNavWithdrawal(cumulativeNavWithdrawal); emailFundNavDO.setFundName(fundNavDTO.getFundName()); emailFundNavDO.setRegisterNumber(fundNavDTO.getRegisterNumber()); emailFundNavDO.setExceptionStatus(fundNavDTO.getParseStatus()); emailFundNavDO.setTemplateId(fundNavDTO.getTemplateId()); emailFundNavDO.setIsvalid(1); emailFundNavDO.setCreatorId(0); emailFundNavDO.setCreateTime(parseDate); emailFundNavDO.setUpdaterId(0); emailFundNavDO.setUpdateTime(parseDate); fundNavDOList.add(emailFundNavDO); } } else { EmailFundNavDO emailFundNavDO = new EmailFundNavDO(); emailFundNavDO.setFileId(fileId); emailFundNavDO.setPriceDate(priceDate); emailFundNavDO.setNav(nav); emailFundNavDO.setCumulativeNavWithdrawal(cumulativeNavWithdrawal); emailFundNavDO.setFundName(fundNavDTO.getFundName()); emailFundNavDO.setRegisterNumber(fundNavDTO.getRegisterNumber()); emailFundNavDO.setExceptionStatus(fundNavDTO.getParseStatus()); emailFundNavDO.setTemplateId(fundNavDTO.getTemplateId()); emailFundNavDO.setIsStored(isStored); emailFundNavDO.setIsvalid(1); emailFundNavDO.setCreatorId(0); emailFundNavDO.setCreateTime(parseDate); emailFundNavDO.setUpdaterId(0); emailFundNavDO.setUpdateTime(parseDate); fundNavDOList.add(emailFundNavDO); } return fundNavDOList; } private Integer saveEmailFileInfo(Integer emailId, Integer fileId, String fileName, String filePath, Date parseDate) { EmailFileInfoDO emailFileInfoDO = buildEmailFileInfoDO(emailId, fileId, fileName, filePath, parseDate); if (emailFileInfoDO.getId() != null) { emailFileInfoMapper.updateTimeById(fileId, parseDate); return emailFileInfoDO.getId(); } emailFileInfoMapper.insert(emailFileInfoDO); return emailFileInfoDO.getId(); } private EmailFileInfoDO buildEmailFileInfoDO(Integer emailId, Integer fileId, String fileName, String filePath, Date parseDate) { EmailFileInfoDO emailFileInfoDO = new EmailFileInfoDO(); emailFileInfoDO.setId(fileId); emailFileInfoDO.setEmailId(emailId); emailFileInfoDO.setFileName(fileName); emailFileInfoDO.setFilePath(filePath); emailFileInfoDO.setIsvalid(1); emailFileInfoDO.setCreatorId(0); emailFileInfoDO.setCreateTime(parseDate); emailFileInfoDO.setUpdaterId(0); emailFileInfoDO.setUpdateTime(parseDate); return emailFileInfoDO; } private void setNavParseStatus(EmailFundNavDTO fundNavDTO, String emailTitle) { // 1.单位净值或累计净值缺失 if (StrUtil.isBlank(fundNavDTO.getNav()) || StrUtil.isBlank(fundNavDTO.getCumulativeNavWithdrawal())) { fundNavDTO.setParseStatus(NavParseStatusConst.NAV_DEFICIENCY); return; } // 考虑单独规模文件时 -> 无单位净值和累计净值 // 2.单位净值或累计净值不大于0 if (!emailTitle.contains("规模")) { if (StrUtil.isBlank(fundNavDTO.getNav()) || StrUtil.isBlank(fundNavDTO.getCumulativeNavWithdrawal()) || (fundNavDTO.getNav().compareTo("0") <= 0 || fundNavDTO.getCumulativeNavWithdrawal().compareTo("0") <= 0)) { fundNavDTO.setParseStatus(NavParseStatusConst.NAV_NEGATIVE); return; } } // 3.资产净值不大于0 if (StrUtil.isNotBlank(fundNavDTO.getAssetNet()) && fundNavDTO.getAssetNet().compareTo("0") <= 0) { fundNavDTO.setParseStatus(NavParseStatusConst.ASSET_NET_NEGATIVE); return; } // 4.匹配基金(考虑到解析估值表时已经匹配上基金的情况) List fundIdList = fundNavDTO.getFundIdList(); if (CollUtil.isEmpty(fundIdList)) { fundIdList = fundService.getFundIdByNamesAndCode(fundNavDTO.getFundName(), fundNavDTO.getRegisterNumber()); if (CollUtil.isEmpty(fundIdList)) { fundNavDTO.setParseStatus(NavParseStatusConst.NOT_MATCH); } } fundNavDTO.setFundIdList(fundIdList); // 写入别名管理表fund_alias saveFundAlias(fundNavDTO.getFundName(), fundNavDTO.getRegisterNumber(), fundIdList); if (CollUtil.isEmpty(fundIdList)) { return; } fundNavDTO.setParseStatus(NavParseStatusConst.SUCCESS); } private void saveFundAlias(String fundName, String registerNumber, List fundIdList) { // 未识别到基金名称和备案编码的数据不写入别名管理 if (StrUtil.isBlank(fundName) && StrUtil.isBlank(registerNumber)) { return; } List fundAliasDOList = CollUtil.newArrayList(); if (StrUtil.isNotBlank(fundName) && StrUtil.isNotBlank(registerNumber)) { fundAliasDOList = fundAliasMapper.queryFundIdByNameAndRegisterNumber(fundName, registerNumber); } if (StrUtil.isBlank(fundName) && StrUtil.isNotBlank(registerNumber)) { fundAliasDOList = fundAliasMapper.queryFundIdByRegisterNumber(registerNumber); } if (StrUtil.isNotBlank(fundName) && StrUtil.isBlank(registerNumber)) { fundAliasDOList = fundAliasMapper.queryFundIdByName(fundName); } // 未匹配基金且已写入别名表 if (CollUtil.isEmpty(fundIdList) && CollUtil.isNotEmpty(fundAliasDOList)) { return; } // 未匹配基金且未写入别名表 if (CollUtil.isEmpty(fundIdList) && CollUtil.isEmpty(fundAliasDOList)) { fundAliasMapper.batchInsert(ListUtil.toList(buildFundAliasDO(fundName, registerNumber, null))); log.info("写入别名表 -> 基金名称:{},备案编码:{},基金id:{}", fundName, registerNumber, fundIdList); return; } // 匹配上基金 -> 需要判断此时基金id是否已经在别名管理表 if (CollUtil.isNotEmpty(fundAliasDOList)) { List collect = fundAliasDOList.stream().filter(Objects::nonNull).map(FundAliasDO::getTargetFundId).filter(Objects::nonNull).distinct().toList(); fundIdList = fundIdList.stream().filter(e -> !collect.contains(e)).toList(); } List fundAliasDOS = CollUtil.isNotEmpty(fundIdList) ? fundIdList.stream().map(e -> buildFundAliasDO(fundName, registerNumber, e)).toList() : null; if (CollUtil.isNotEmpty(fundAliasDOS)) { fundAliasMapper.batchInsert(fundAliasDOS); log.info("写入别名表 -> 基金名称:{},备案编码:{},基金id:{}", fundName, registerNumber, fundIdList); } } public FundAliasDO buildFundAliasDO(String fundName, String registerNumber, String fundId) { FundAliasDO fundAliasDO = new FundAliasDO(); fundAliasDO.setTargetFundId(fundId); fundAliasDO.setSourceFundName(fundName); fundAliasDO.setSourceRegisterNumber(registerNumber); fundAliasDO.setIsvalid(1); fundAliasDO.setCreatorId(0); fundAliasDO.setCreateTime(new Date()); fundAliasDO.setUpdateTime(new Date()); fundAliasDO.setUpdaterId(0); return fundAliasDO; } private Integer saveEmailParseInfo(EmailParseInfoDO emailParseInfoDO) { if (emailParseInfoDO == null) { return null; } // 重新邮件功能 -> 修改解析时间和更新时间 if (emailParseInfoDO.getId() != null) { emailParseInfoMapper.updateParseTime(emailParseInfoDO.getId(), emailParseInfoDO.getParseDate()); return emailParseInfoDO.getId(); } emailParseInfoMapper.insert(emailParseInfoDO); return emailParseInfoDO.getId(); } private EmailParseInfoDO buildEmailParseInfo(Integer emailId, String emailAddress, String senderEmail, String emailDate, String emailTitle, Integer emailType, Integer parseStatus, Date parseDate) { EmailParseInfoDO emailParseInfoDO = new EmailParseInfoDO(); emailParseInfoDO.setId(emailId); emailParseInfoDO.setSenderEmail(senderEmail); emailParseInfoDO.setEmail(emailAddress); emailParseInfoDO.setEmailDate(DateUtil.parse(emailDate, DateConst.YYYY_MM_DD_HH_MM_SS)); emailParseInfoDO.setParseDate(parseDate); emailParseInfoDO.setEmailTitle(emailTitle); emailParseInfoDO.setEmailType(emailType); emailParseInfoDO.setParseStatus(parseStatus); emailParseInfoDO.setIsvalid(1); emailParseInfoDO.setCreatorId(0); emailParseInfoDO.setCreateTime(parseDate); emailParseInfoDO.setUpdaterId(0); emailParseInfoDO.setUpdateTime(parseDate); return emailParseInfoDO; } public Map> getEmailFieldMapping() { List emailFieldMappingDOList = emailFieldMapper.getEmailFieldMapping(1); return emailFieldMappingDOList.stream() .collect(Collectors.toMap(EmailFieldMappingDO::getCode, v -> Arrays.stream(v.getName().split(",")).toList())); } public Map> getEmailType() { Map> emailTypeMap = MapUtil.newHashMap(3, true); EmailTypeRuleDO emailTypeRuleDO = emailTypeRuleMapper.getEmailTypeRule(); String nav = emailTypeRuleDO != null && StrUtil.isNotBlank(emailTypeRuleDO.getNav()) ? emailTypeRuleDO.getNav() : emailRuleConfig.getNav(); String valuation = emailTypeRuleDO != null && StrUtil.isNotBlank(emailTypeRuleDO.getValuation()) ? emailTypeRuleDO.getValuation() : emailRuleConfig.getValuation(); String report = emailTypeRuleDO != null && StrUtil.isNotBlank(emailTypeRuleDO.getReport()) ? emailTypeRuleDO.getReport() : emailRuleConfig.getReport(); emailTypeMap.put(EmailTypeConst.VALUATION_EMAIL_TYPE, Arrays.stream(valuation.split(",")).toList()); emailTypeMap.put(EmailTypeConst.REPORT_EMAIL_TYPE, Arrays.stream(report.split(",")).toList()); emailTypeMap.put(EmailTypeConst.NAV_EMAIL_TYPE, Arrays.stream(nav.split(",")).toList()); return emailTypeMap; } /** * 读取邮件 * * @param mailboxInfoDTO 邮箱配置信息 * @param emailTypeMap 邮件类型识别规则映射表 * @param startDate 邮件起始日期 * @param endDate 邮件截止日期(为null,将解析邮件日期小于等于startDate的当天邮件) * @return 读取到的邮件信息 * @throws Exception 异常信息 */ private Map> realEmail(MailboxInfoDTO mailboxInfoDTO, Map> emailTypeMap, Date startDate, Date endDate) throws Exception { Store store = EmailUtil.getStoreNew(mailboxInfoDTO); if (store == null) { return MapUtil.newHashMap(); } // 默认读取收件箱的邮件 Folder folder = store.getFolder("INBOX"); folder.open(Folder.READ_ONLY); Message[] messages = getEmailMessage(folder, mailboxInfoDTO.getProtocol(), startDate); if (messages == null || messages.length == 0) { log.info("获取不到邮件 -> 邮箱信息:{},开始时间:{},结束时间:{}", mailboxInfoDTO, startDate, endDate); return MapUtil.newHashMap(); } Map> emailMessageMap = MapUtil.newHashMap(); for (Message message1 : messages) { MimeMessage message = (MimeMessage) message1; List emailContentInfoDTOList = CollUtil.newArrayList(); String uuidKey = UUID.randomUUID().toString().replaceAll("-", ""); Integer emailType; String senderEmail; try { Date emailDate = message.getSentDate(); boolean isNotParseConditionSatisfied = emailDate == null || (endDate != null && emailDate.compareTo(endDate) > 0) || (startDate != null && emailDate.compareTo(startDate) < 0); if (isNotParseConditionSatisfied) { continue; } senderEmail = getSenderEmail(message); emailType = EmailUtil.getEmailTypeBySubject(message.getSubject(), emailTypeMap); String emailDateStr = DateUtil.format(emailDate, DateConst.YYYY_MM_DD_HH_MM_SS); if (emailType == null) { log.info("邮件不满足解析条件 -> 邮件主题:{},邮件日期:{}", message.getSubject(), emailDateStr); continue; } log.info("邮件采集成功 -> 邮件主题:{},邮件日期:{}", message.getSubject(), emailDateStr); Object content = message.getContent(); // 1.邮件为MIME多部分消息体:可能既有邮件又有正文 if (content instanceof MimeMultipart) { emailContentInfoDTOList = EmailUtil.collectMimeMultipart(message, mailboxInfoDTO.getAccount(), path); } // 2.邮件只有正文 if (content instanceof String) { EmailContentInfoDTO emailContentInfoDTO = new EmailContentInfoDTO(); emailContentInfoDTO.setEmailContent(content.toString()); emailContentInfoDTO.setEmailDate(emailDateStr); emailContentInfoDTO.setEmailTitle(message.getSubject()); String fileName = message.getSubject() + DateUtil.format(emailDate, DateConst.YYYYMMDDHHMMSS24); String filePath = path + mailboxInfoDTO.getAccount() + "/" + DateUtil.format(emailDate, DateConst.YYYY_MM_DD) + "/" + fileName + ".html"; File saveFile = new File(filePath); saveFile.setReadable(true); if (!saveFile.exists()) { if (!saveFile.getParentFile().exists()) { saveFile.getParentFile().mkdirs(); saveFile.getParentFile().setExecutable(true); } } FileUtil.writeFile(filePath, content.toString()); emailContentInfoDTO.setFilePath(filePath); emailContentInfoDTO.setFileName(fileName); emailContentInfoDTOList.add(emailContentInfoDTO); } if (CollUtil.isNotEmpty(emailContentInfoDTOList)) { // 估值表或定期报告邮件不展示正文html文件 if (emailType.equals(EmailTypeConst.VALUATION_EMAIL_TYPE) || emailType.equals(EmailTypeConst.REPORT_EMAIL_TYPE)) { emailContentInfoDTOList = emailContentInfoDTOList.stream().filter(e -> !ExcelUtil.isHTML(e.getFilePath())).toList(); } emailContentInfoDTOList.forEach(e -> { e.setEmailType(emailType); e.setSenderEmail(senderEmail); }); emailMessageMap.put(uuidKey, emailContentInfoDTOList); } } catch (Exception e) { log.error("获取邮箱的邮件报错,堆栈信息:{}", ExceptionUtil.stacktraceToString(e)); } } folder.close(false); store.close(); return emailMessageMap; } private String getSenderEmail(MimeMessage message) { Address[] senderAddress = null; try { senderAddress = message.getFrom(); if (senderAddress == null || senderAddress.length == 0) { log.info("发件人获取失败============================="); return null; } // 此时的address是含有编码(MIME编码方式)后的文本和实际的邮件地址 String address = ""; for(Address from : senderAddress){ if(StringUtil.isNotEmpty(from.toString())){ address = from.toString(); break; } } log.info("发件人地址："+address+"========================senderAddress size:"+senderAddress.length); // 正则表达式匹配邮件地址 Pattern pattern = Pattern.compile("<(\\S+)>"); Matcher matcher = pattern.matcher(address); if (matcher.find()) { return matcher.group(1); } //说明匹配不到，直接获取sender Address sender = message.getSender(); if(sender == null){ return address; } String senderEmail = sender.toString(); log.info("senderEmail:"+senderEmail+"===================="); if(senderEmail.contains("<") && senderEmail.contains(">") && senderEmail.indexOf("<") < senderEmail.indexOf(">")){ senderEmail = senderEmail.substring(senderEmail.indexOf("<")+1,senderEmail.length()-1); } return senderEmail; } catch (MessagingException e) { log.error(e.getMessage(),e); } return null; } public List searchEmailCount(DataboardQuery databoardQuery) { if(StringUtil.isNotEmpty(databoardQuery.getEndDate())){ databoardQuery.setEndDate(com.smppw.utils.DateUtil.getAroundDate(DateUtils.parse(databoardQuery.getEndDate(),DateUtils.YYYY_MM_DD),1)); } List> dataList = emailParseInfoMapper.searchEmailDataBoard(databoardQuery); List result = new ArrayList<>(); Long total = 0L; for (Map data : dataList) { NameValueVO vo = new NameValueVO(); if (1 == ((Integer) data.get("parse_status")).intValue()) { vo.setValue((Long) data.get("total")); vo.setName("解析成功"); } else { vo.setValue((Long) data.get("total")); vo.setName("解析失败"); } total += (Long) data.get("total"); result.add(vo); } NameValueVO vo = new NameValueVO(); vo.setName("解析总数"); vo.setValue(total); result.add(vo); return result; } public List searchEmailTypeCount(DataboardQuery databoardQuery) { if(StringUtil.isNotEmpty(databoardQuery.getEndDate())){ databoardQuery.setEndDate(com.smppw.utils.DateUtil.getAroundDate(DateUtils.parse(databoardQuery.getEndDate(),DateUtils.YYYY_MM_DD),1)); } //邮件类型，1-净值，2-估值表，3-定期报告 List> dataList = emailParseInfoMapper.searchEmailTypeCount(databoardQuery); List result = new ArrayList<>(); Integer total = 0; for (Map data : dataList) { NameValueVO vo = new NameValueVO(); Integer emailType = (Integer) data.get("email_type"); Long totalType = (Long) data.get("total"); if (1 == emailType) { vo.setName("净值规模"); vo.setValue(totalType); } else if (2 == emailType) { vo.setName("估值表"); vo.setValue(totalType); } else { vo.setName("定期报告"); vo.setValue(totalType); } result.add(vo); } return result; } public EmailParseFailAnalysisVO parseFailAnalysis(DataboardQuery databoardQuery) { if(StringUtil.isNotEmpty(databoardQuery.getEndDate())){ databoardQuery.setEndDate(com.smppw.utils.DateUtil.getAroundDate(DateUtils.parse(databoardQuery.getEndDate(),DateUtils.YYYY_MM_DD),1)); } EmailParseFailAnalysisVO result = new EmailParseFailAnalysisVO(); //净值规模 List navNameValueVOS = new ArrayList<>(); NameValueVO pdfNoDataVO = new NameValueVO(); databoardQuery.setEmailType(1); Long pdfNoData = emailParseInfoMapper.countpdfNoData(databoardQuery, "无法从PDF文件中获取到数据"); pdfNoDataVO.setName("无法从PDF文件中获取到数据"); pdfNoDataVO.setValue(pdfNoData); navNameValueVOS.add(pdfNoDataVO); NameValueVO priceDateMissVO = new NameValueVO(); Long priceDateMiss = emailParseInfoMapper.countpdfNoData(databoardQuery, "缺少净值日期"); priceDateMissVO.setValue(priceDateMiss); priceDateMissVO.setName("缺少净值日期"); navNameValueVOS.add(priceDateMissVO); NameValueVO navMissVO = new NameValueVO(); Long navMiss = emailParseInfoMapper.countpdfNoData(databoardQuery, "单位净值和累计净值和资产净值均缺失"); navMissVO.setName("单位净值和累计净值和资产净值均缺失"); navMissVO.setValue(navMiss); navNameValueVOS.add(navMissVO); NameValueVO fundNameNumberMissVO = new NameValueVO(); Long fundNameNumberMiss = emailParseInfoMapper.countpdfNoData(databoardQuery, "基金名称和备案编码均缺失"); fundNameNumberMissVO.setName("基金名称和备案编码均缺失"); fundNameNumberMissVO.setValue(fundNameNumberMiss); navNameValueVOS.add(fundNameNumberMissVO); result.setNavFailAnalysisVO(navNameValueVOS); NameValueVO templateUnSupportVO = new NameValueVO(); Long templateUnSupport = emailParseInfoMapper.countpdfNoData(databoardQuery, "模板不支持"); templateUnSupportVO.setName("模板不支持"); templateUnSupportVO.setValue(templateUnSupport); navNameValueVOS.add(templateUnSupportVO); result.setNavFailAnalysisVO(navNameValueVOS); //估值表规模 databoardQuery.setEmailType(2); List valuationNameValueVOS = new ArrayList<>(); NameValueVO fileTypeErrorVO = new NameValueVO(); Long fileTypeError = emailParseInfoMapper.countpdfNoData(databoardQuery, "文件格式错误"); fileTypeErrorVO.setName("文件格式错误"); fileTypeErrorVO.setValue(fileTypeError); valuationNameValueVOS.add(fileTypeErrorVO); NameValueVO columnMissVO = new NameValueVO(); Long columnMiss = emailParseInfoMapper.countpdfNoData(databoardQuery, "无市值列或无数量列"); columnMissVO.setName("无市值列或无数量列"); columnMissVO.setValue(columnMiss); valuationNameValueVOS.add(columnMissVO); NameValueVO numbericMissVO = new NameValueVO(); Long numbericMiss = emailParseInfoMapper.countpdfNoData(databoardQuery, "非数值数据"); numbericMissVO.setName("非数值数据"); numbericMissVO.setValue(numbericMiss); valuationNameValueVOS.add(numbericMissVO); NameValueVO noDataVO = new NameValueVO(); Long noData = emailParseInfoMapper.countpdfNoData(databoardQuery, "无数据"); noDataVO.setValue(noData); noDataVO.setName("无数据"); valuationNameValueVOS.add(noDataVO); NameValueVO templateErrorVO = new NameValueVO(); Long templateError = emailParseInfoMapper.countpdfNoData(databoardQuery, "模板不支持"); templateErrorVO.setValue(templateError); templateErrorVO.setName("模板不支持"); valuationNameValueVOS.add(templateErrorVO); result.setValuationFailAnalysisVO(valuationNameValueVOS); //定期报告 /** * PARSE_FAIL(21000, "定期报告解析错误：{}"), * NOT_A_REPORT(21001, "[{}]不是定期报告"), * REPORT_IS_SCAN(21002, "报告[{}]为扫描件"), * NO_SUPPORT_TEMPLATE(21003, "报告[{}]是不支持的文件格式"), * NOT_A_FIXED_FORMAT(21004, "报告[{}]不是基协统一格式"), * * PARSE_FUND_INFO_FAIL(21010, "报告[{}]没有解析到基金基本信息"), * PARSE_NAV_INFO_FAIL(21011, "报告[{}]没有解析到基金净值信息"), * PARSE_FINANCIAL_INFO_FAIL(21012, "报告[{}]没有解析到基金财务指标信息"), * PARSE_INDUSTRY_INFO_FAIL(21013, "报告[{}]没有解析到基金行业配置信息"), * PARSE_ASSET_INFO_FAIL(21014, "报告[{}]没有解析到基金资产配置信息"), * PARSE_SHARE_INFO_FAIL(21015, "报告[{}]没有解析到基金份额变动信息"), */ databoardQuery.setEmailType(3); List reportNameValueVOS = new ArrayList<>(); NameValueVO scannedFileVO = new NameValueVO(); Long scannedFile = emailParseInfoMapper.countpdfNoData(databoardQuery, "为扫描件"); scannedFileVO.setName("报告为扫描件"); scannedFileVO.setValue(scannedFile); reportNameValueVOS.add(scannedFileVO); NameValueVO errorAmacFileTypeVO = new NameValueVO(); Long errorAmacFileType = emailParseInfoMapper.countpdfNoData(databoardQuery, "不是基协统一格式"); errorAmacFileTypeVO.setName("报告不是基协统一格式"); errorAmacFileTypeVO.setValue(errorAmacFileType); reportNameValueVOS.add(errorAmacFileTypeVO); NameValueVO watermarkFileErrorVO = new NameValueVO(); Long watermarkFileError = emailParseInfoMapper.countpdfNoData(databoardQuery, "没有解析到"); watermarkFileErrorVO.setName("报告水印干扰导致部分没有解析"); watermarkFileErrorVO.setValue(watermarkFileError); reportNameValueVOS.add(watermarkFileErrorVO); NameValueVO noReportVO = new NameValueVO(); Long noReport = emailParseInfoMapper.countpdfNoData(databoardQuery, "不是定期报告"); noReportVO.setName("报告不是定期报告"); noReportVO.setValue(noReport); reportNameValueVOS.add(noReportVO); result.setReportFailAnalysisVO(reportNameValueVOS); return result; } public EmailParseDataViewVO dataOverview(DataboardQuery databoardQuery) { EmailParseDataViewVO dataViewVO = new EmailParseDataViewVO(); dataViewVO.setEmailNum(emailParseInfoMapper.countEmailTotal(null)); dataViewVO.setNavEmailNum(emailParseInfoMapper.countEmailTotal(1)); dataViewVO.setValuationEmailNum(emailParseInfoMapper.countEmailTotal(2)); dataViewVO.setReportEmailNum(emailParseInfoMapper.countEmailTotal(3)); dataViewVO.setParseNavNum(emailFundNavMapper.countEmailNavTotal()); dataViewVO.setParseAssetNum(emailFundAssetMapper.countEmailAssetTotal()); dataViewVO.setNavNum(navMapper.countNavTotal()); dataViewVO.setAssetNum(assetMapper.countAssetTotal()); dataViewVO.setDistribute(distributionMapper.countDistributionTotal()); dataViewVO.setFundNum(fundInfoMapper.countFundTotal()); dataViewVO.setCompanyNum(companyInformationMapper.countCompanyTotal()); return dataViewVO; } private Message[] getEmailMessage(Folder folder, String protocol, Date startDate) { try { if (protocol.contains("imap")) { // 获取邮件日期大于等于startDate的邮件(搜索条件只支持按天) SearchTerm startDateTerm = new ReceivedDateTerm(ComparisonTerm.GE, startDate); return folder.search(startDateTerm); } else { return folder.getMessages(); } } catch (MessagingException e) { throw new RuntimeException(e); } } private static class PythonData { private Integer fileId; private Integer status; private String msg; private String register_number; public Integer getFileId() { return fileId; } public void setFileId(Integer fileId) { this.fileId = fileId; } public Integer getStatus() { return status; } public void setStatus(Integer status) { this.status = status; } public String getMsg() { return msg; } public void setMsg(String msg) { this.msg = msg; } public String getRegister_number() { return register_number; } public void setRegister_number(String register_number) { this.register_number = register_number; } } }