EmailParseService.java 47 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890
  1. package com.simuwang.daq.service;
  2. import cn.hutool.core.bean.BeanUtil;
  3. import cn.hutool.core.collection.CollUtil;
  4. import cn.hutool.core.collection.CollectionUtil;
  5. import cn.hutool.core.collection.ListUtil;
  6. import cn.hutool.core.date.DateUtil;
  7. import cn.hutool.core.exceptions.ExceptionUtil;
  8. import cn.hutool.core.map.MapUtil;
  9. import cn.hutool.core.util.StrUtil;
  10. import cn.hutool.http.HttpUtil;
  11. import cn.hutool.json.JSONUtil;
  12. import com.simuwang.base.common.conts.*;
  13. import com.simuwang.base.common.util.EmailUtil;
  14. import com.simuwang.base.common.util.FileUtil;
  15. import com.simuwang.base.config.DaqProperties;
  16. import com.simuwang.base.config.EmailRuleConfig;
  17. import com.simuwang.base.mapper.*;
  18. import com.simuwang.base.pojo.dos.*;
  19. import com.simuwang.base.pojo.dto.EmailContentInfoDTO;
  20. import com.simuwang.base.pojo.dto.EmailFundNavDTO;
  21. import com.simuwang.base.pojo.dto.MailboxInfoDTO;
  22. import com.simuwang.base.pojo.dto.report.PythonResult;
  23. import com.simuwang.base.pojo.dto.report.ReportData;
  24. import com.simuwang.base.pojo.valuation.CmValuationTableAttribute;
  25. import com.simuwang.daq.components.PythonReportConverter;
  26. import com.simuwang.daq.components.writer.ReportWriterFactory;
  27. import jakarta.mail.*;
  28. import jakarta.mail.internet.MimeMultipart;
  29. import jakarta.mail.search.ComparisonTerm;
  30. import jakarta.mail.search.ReceivedDateTerm;
  31. import jakarta.mail.search.SearchTerm;
  32. import org.slf4j.Logger;
  33. import org.slf4j.LoggerFactory;
  34. import org.springframework.beans.factory.annotation.Autowired;
  35. import org.springframework.beans.factory.annotation.Value;
  36. import org.springframework.stereotype.Service;
  37. import java.io.File;
  38. import java.math.BigDecimal;
  39. import java.util.*;
  40. import java.util.regex.Matcher;
  41. import java.util.regex.Pattern;
  42. import java.util.stream.Collectors;
  43. /**
  44. * @author mozuwen
  45. * @date 2024-09-04
  46. * @description 邮件解析服务
  47. */
  48. @Service
  49. public class EmailParseService {
  50. private static final Logger log = LoggerFactory.getLogger(EmailParseService.class);
  51. private final String pyBaseUrl;
  52. private final EmailTypeRuleMapper emailTypeRuleMapper;
  53. private final EmailRuleConfig emailRuleConfig;
  54. private final EmailFieldMappingMapper emailFieldMapper;
  55. private final EmailParserFactory emailParserFactory;
  56. private final EmailParseInfoMapper emailParseInfoMapper;
  57. private final EmailFileInfoMapper emailFileInfoMapper;
  58. private final EmailFundNavMapper emailFundNavMapper;
  59. private final EmailFundAssetMapper emailFundAssetMapper;
  60. private final AssetMapper assetMapper;
  61. private final NavMapper navMapper;
  62. private final FundService fundService;
  63. private final FundAliasMapper fundAliasMapper;
  64. private final ValuationTableMapper valuationTableMapper;
  65. private final ValuationTableAttributeMapper valuationTableAttributeMapper;
  66. private final FundPositionDetailMapper fundPositionDetailMapper;
  67. @Value("${email.file.path}")
  68. private String path;
  69. @Autowired
  70. private FundInfoMapper fundInfoMapper;
  71. @Autowired
  72. private ReportWriterFactory reportWriterFactory;
  73. public static final int stepSize = 10000;
  74. public EmailParseService(EmailTypeRuleMapper emailTypeRuleMapper, EmailRuleConfig emailRuleConfig,
  75. EmailFieldMappingMapper emailFieldMapper, EmailParserFactory emailParserFactory,
  76. EmailParseInfoMapper emailParseInfoMapper, EmailFileInfoMapper emailFileInfoMapper,
  77. EmailFundNavMapper emailFundNavMapper, EmailFundAssetMapper emailFundAssetMapper,
  78. AssetMapper assetMapper, NavMapper navMapper, FundService fundService,
  79. FundAliasMapper fundAliasMapper, DaqProperties properties,
  80. ValuationTableMapper valuationTableMapper, ValuationTableAttributeMapper valuationTableAttributeMapper,
  81. FundPositionDetailMapper fundPositionDetailMapper) {
  82. this.emailTypeRuleMapper = emailTypeRuleMapper;
  83. this.emailRuleConfig = emailRuleConfig;
  84. this.emailFieldMapper = emailFieldMapper;
  85. this.emailParserFactory = emailParserFactory;
  86. this.emailParseInfoMapper = emailParseInfoMapper;
  87. this.emailFileInfoMapper = emailFileInfoMapper;
  88. this.emailFundNavMapper = emailFundNavMapper;
  89. this.emailFundAssetMapper = emailFundAssetMapper;
  90. this.assetMapper = assetMapper;
  91. this.navMapper = navMapper;
  92. this.fundService = fundService;
  93. this.fundAliasMapper = fundAliasMapper;
  94. this.pyBaseUrl = properties.getPyBaseUrl();
  95. this.valuationTableMapper = valuationTableMapper;
  96. this.valuationTableAttributeMapper = valuationTableAttributeMapper;
  97. this.fundPositionDetailMapper = fundPositionDetailMapper;
  98. }
  99. /**
  100. * 解析指定邮箱指定时间范围内的邮件
  101. *
  102. * @param mailboxInfoDTO 邮箱配置信息
  103. * @param startDate 邮件起始日期(yyyy-MM-dd HH:mm:ss)
  104. * @param endDate 邮件截止日期(yyyy-MM-dd HH:mm:ss, 为null,将解析邮件日期小于等于startDate的当天邮件)
  105. */
  106. public void parseEmail(MailboxInfoDTO mailboxInfoDTO, Date startDate, Date endDate) {
  107. log.info("开始邮件解析 -> 邮箱信息:{},开始时间:{},结束时间:{}", mailboxInfoDTO, DateUtil.format(startDate, DateConst.YYYY_MM_DD_HH_MM_SS), DateUtil.format(endDate, DateConst.YYYY_MM_DD_HH_MM_SS));
  108. // 邮件类型配置
  109. Map<Integer, List<String>> emailTypeMap = getEmailType();
  110. // 邮件字段识别映射表
  111. Map<String, List<String>> emailFieldMap = getEmailFieldMapping();
  112. Map<String, List<EmailContentInfoDTO>> emailContentMap;
  113. try {
  114. emailContentMap = realEmail(mailboxInfoDTO, emailTypeMap, startDate, endDate);
  115. } catch (Exception e) {
  116. log.info("采集邮件失败 -> 邮箱配置信息:{},堆栈信息:{}", mailboxInfoDTO, ExceptionUtil.stacktraceToString(e));
  117. return;
  118. }
  119. for (Map.Entry<String, List<EmailContentInfoDTO>> emailEntry : emailContentMap.entrySet()) {
  120. List<EmailContentInfoDTO> emailContentInfoDTOList = emailEntry.getValue();
  121. if (CollUtil.isEmpty(emailContentInfoDTOList)) {
  122. log.warn("未采集到正文或附件");
  123. continue;
  124. }
  125. log.info("开始解析邮件数据 -> 邮件主题:{},邮件日期:{}", emailContentInfoDTOList.get(0).getEmailTitle(), emailContentInfoDTOList.get(0).getEmailDate());
  126. List<EmailFundNavDTO> emailFundNavDTOList = CollUtil.newArrayList();
  127. Map<EmailContentInfoDTO, List<EmailFundNavDTO>> fileNameNavMap = MapUtil.newHashMap();
  128. for (EmailContentInfoDTO emailContentInfoDTO : emailContentInfoDTOList) {
  129. try {
  130. List<EmailFundNavDTO> fundNavDTOList = parseEmail(emailContentInfoDTO, emailFieldMap);
  131. fileNameNavMap.put(emailContentInfoDTO, fundNavDTOList);
  132. emailFundNavDTOList.addAll(fundNavDTOList);
  133. } catch (Exception e) {
  134. log.error("堆栈信息:{}", ExceptionUtil.stacktraceToString(e));
  135. }
  136. }
  137. // 保存相关信息 -> 邮件信息表,邮件文件表,邮件净值表,邮件规模表,基金净值表
  138. saveRelatedTable(mailboxInfoDTO.getAccount(), emailContentInfoDTOList, fileNameNavMap);
  139. log.info("结束邮件解析 -> 邮箱信息:{},开始时间:{},结束时间:{}", mailboxInfoDTO, DateUtil.format(startDate, DateConst.YYYY_MM_DD_HH_MM_SS), DateUtil.format(endDate, DateConst.YYYY_MM_DD_HH_MM_SS));
  140. }
  141. }
  142. public List<EmailFundNavDTO> parseEmail(EmailContentInfoDTO emailContentInfoDTO, Map<String, List<String>> emailFieldMap) {
  143. Integer emailType = emailContentInfoDTO.getEmailType();
  144. AbstractEmailParser emailParser = emailParserFactory.getInstance(emailType);
  145. return emailParser.parse(emailContentInfoDTO, emailFieldMap);
  146. }
  147. public void saveRelatedTable(String emailAddress, List<EmailContentInfoDTO> emailContentInfoDTOList, Map<EmailContentInfoDTO, List<EmailFundNavDTO>> fileNameNavMap) {
  148. String emailTitle = CollUtil.isNotEmpty(emailContentInfoDTOList) ? emailContentInfoDTOList.get(0).getEmailTitle() : null;
  149. String emailDate = CollUtil.isNotEmpty(emailContentInfoDTOList) ? emailContentInfoDTOList.get(0).getEmailDate() : null;
  150. Integer emailType = CollUtil.isNotEmpty(emailContentInfoDTOList) ? emailContentInfoDTOList.get(0).getEmailType() : null;
  151. Integer emailId = CollUtil.isNotEmpty(emailContentInfoDTOList) ? emailContentInfoDTOList.get(0).getEmailId() : null;
  152. String senderEmail = CollUtil.isNotEmpty(emailContentInfoDTOList) ? emailContentInfoDTOList.get(0).getSenderEmail() : null;
  153. Date parseDate = new Date();
  154. int emailParseStatus = EmailParseStatusConst.SUCCESS;
  155. EmailParseInfoDO emailParseInfoDO = buildEmailParseInfo(emailId, emailAddress, senderEmail, emailDate, emailTitle, emailType, emailParseStatus, parseDate);
  156. emailId = saveEmailParseInfo(emailParseInfoDO);
  157. // python 报告解析接口结果
  158. List<ReportData> dataList = ListUtil.list(false);
  159. for (Map.Entry<EmailContentInfoDTO, List<EmailFundNavDTO>> fileNameNavEntry : fileNameNavMap.entrySet()) {
  160. // 保存邮件文件表
  161. EmailContentInfoDTO emailContentInfoDTO = fileNameNavEntry.getKey();
  162. String fileName = emailContentInfoDTO.getFileName();
  163. if (Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType) && fileName.endsWith(".html")) {
  164. continue;
  165. }
  166. Integer fileId = saveEmailFileInfo(emailId, emailContentInfoDTO.getFileId(), fileName, emailContentInfoDTO.getFilePath(), parseDate);
  167. List<EmailFundNavDTO> fundNavDTOList = fileNameNavEntry.getValue();
  168. if (CollUtil.isEmpty(fundNavDTOList) && !Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType)) {
  169. continue;
  170. }
  171. // python接口解析结果
  172. ReportData data = this.requestPyAndResult(fileId, emailContentInfoDTO);
  173. if (data != null) {
  174. // 保存报告解析数据
  175. this.reportWriterFactory.getInstance(data.getReportType()).write(data);
  176. dataList.add(data);
  177. }
  178. for (EmailFundNavDTO fundNavDTO : fundNavDTOList) {
  179. // 设置净值数据的解析状态
  180. setNavParseStatus(fundNavDTO, emailTitle);
  181. }
  182. // 保存净值表和规模表
  183. saveNavAndAssetNet(fileId, fundNavDTOList, parseDate);
  184. saveValuationInfo(fileId, fundNavDTOList);
  185. }
  186. // 更新邮件解析结果 -> 当【净值日期】和【备案编码/基金名称】能正常解读,即识别为【成功】
  187. long successNavCount = fileNameNavMap.values().stream().flatMap(List::stream).filter(Objects::nonNull).count();
  188. emailParseStatus = successNavCount >= 1 ? EmailParseStatusConst.SUCCESS : EmailParseStatusConst.FAIL;
  189. // 报告邮件有一条成功就表示整体成功
  190. if (Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType) && CollUtil.isNotEmpty(dataList)) {
  191. long count = dataList.size();
  192. emailParseStatus = count >= 1 ? EmailParseStatusConst.SUCCESS : EmailParseStatusConst.FAIL;
  193. }
  194. emailParseInfoMapper.updateParseStatus(emailId, emailParseStatus);
  195. }
  196. private void saveValuationInfo(Integer fileId, List<EmailFundNavDTO> fundNavDTOList) {
  197. if (CollUtil.isEmpty(fundNavDTOList)) {
  198. return;
  199. }
  200. for (EmailFundNavDTO fundNavDTO : fundNavDTOList) {
  201. ValuationTableDO valuationTableDO = fundNavDTO.getValuationTableDO();
  202. if (valuationTableDO == null) {
  203. continue;
  204. }
  205. List<String> fundIdList = fundNavDTO.getFundIdList();
  206. valuationTableDO.setFileId(fileId);
  207. List<FundPositionDetailDO> fundPositionDetailDOList = fundNavDTO.getFundPositionDetailDOList();
  208. List<CmValuationTableAttribute> valuationTableAttributeList = fundNavDTO.getValuationTableAttributeList();
  209. if (CollUtil.isEmpty(fundIdList)) {
  210. int valuationId = valuationTableMapper.insert(valuationTableDO);
  211. saveValuationTableAttribute(valuationId, valuationTableAttributeList);
  212. fundPositionDetailDOList.forEach(e -> e.setValuationId(valuationId));
  213. saveFundPositionDetail(fundPositionDetailDOList, null, fundNavDTO.getPriceDate());
  214. continue;
  215. }
  216. for (String fundId : fundIdList) {
  217. valuationTableDO.setFundId(fundId);
  218. int valuationId = valuationTableMapper.insert(valuationTableDO);
  219. fundPositionDetailDOList.forEach(e -> {
  220. e.setFundId(fundId);
  221. e.setValuationId(valuationId);
  222. });
  223. saveValuationTableAttribute(valuationId, valuationTableAttributeList);
  224. saveFundPositionDetail(fundPositionDetailDOList, null, fundNavDTO.getPriceDate());
  225. }
  226. }
  227. }
  228. public void saveFundPositionDetail(List<FundPositionDetailDO> fundPositionDetails, String fundId, String valuationDate) {
  229. int subBegin = 0;
  230. int subEnd = stepSize;
  231. int insertNum = 0;
  232. int hasStock = 0;
  233. int hasBond = 0;
  234. int hasFuture = 0;
  235. if (CollectionUtil.isNotEmpty(fundPositionDetails)) {
  236. //插入持仓数据 cm_fund_position_detail(记录数较多,得分批)
  237. // 先删除原先的数据 然后写入数据
  238. fundPositionDetailMapper.deleteUnUsed(fundId, valuationDate);
  239. // 将最终结果写入 cm_fund_position_detail
  240. while (subBegin < fundPositionDetails.size()) {
  241. List<FundPositionDetailDO> segment = fundPositionDetails.subList(subBegin, Math.min(subEnd, fundPositionDetails.size()));
  242. for (FundPositionDetailDO fundPositionDetail : segment) {
  243. //实收信托、实收资本对应的编码为107
  244. if (StrUtil.isNotBlank(fundPositionDetail.getSecuritiesName()) && "实收信托".equals(fundPositionDetail.getSecuritiesName().trim())) {
  245. fundPositionDetail.setSubjectCode("107");
  246. fundPositionDetail.setSecuritiesCode("107");
  247. }
  248. //设置创建者id,在拿取最近修改人时有用
  249. fundPositionDetail.setCreatorId(0);
  250. if (fundPositionDetail.getLevel() != null && fundPositionDetail.getLevel() >= 4) {
  251. Integer secType = fundPositionDetail.getSecType();
  252. Integer subType = fundPositionDetail.getSubType();
  253. if (secType != null && subType != null) {
  254. if (secType == HoldingType.Stock.getId() && subType == HoldingType.Stock.getId()) {
  255. hasStock = 1;
  256. }
  257. // 正逆回购不属于债券
  258. String subjectCode = fundPositionDetail.getSubjectCode();
  259. String[] SALE_CODES = {"2202", "1202"};
  260. if (secType == HoldingType.Bond.getId() && !StrUtil.containsAny(subjectCode, SALE_CODES)) {
  261. hasBond = 1;
  262. }
  263. if (secType == HoldingType.Future.getId() || secType == HoldingType.Option.getId()) {
  264. hasFuture = 1;
  265. }
  266. }
  267. }
  268. }
  269. insertNum += fundPositionDetailMapper.insertMulti(segment);
  270. subBegin = subEnd;
  271. subEnd += stepSize;
  272. }
  273. }
  274. // 更新 cm_user_valuation_table
  275. if (insertNum > 0) {
  276. valuationTableMapper.updateUpdateAnalyzed(fundId, valuationDate, hasStock, hasBond, hasFuture);
  277. }
  278. }
  279. private void saveValuationTableAttribute(Integer valuationId, List<CmValuationTableAttribute> valuationTableAttributes) {
  280. if (valuationId == null || CollUtil.isEmpty(valuationTableAttributes)) {
  281. return;
  282. }
  283. List<ValuationTableAttributeDO> valuationTableAttributeDOList = buildValuationTableAttributeDO(valuationId, valuationTableAttributes);
  284. valuationTableAttributeMapper.deleteByValuationId(valuationId);
  285. if (CollUtil.isNotEmpty(valuationTableAttributeDOList)) {
  286. valuationTableAttributeMapper.batchInsert(valuationTableAttributeDOList);
  287. }
  288. }
  289. private List<ValuationTableAttributeDO> buildValuationTableAttributeDO(Integer valuationId, List<CmValuationTableAttribute> valuationTableAttributes) {
  290. if (CollUtil.isEmpty(valuationTableAttributes)) {
  291. return CollUtil.newArrayList();
  292. }
  293. return valuationTableAttributes.stream().map(e -> {
  294. ValuationTableAttributeDO tableAttributeDO = new ValuationTableAttributeDO();
  295. tableAttributeDO.setValuationId(valuationId);
  296. tableAttributeDO.setSubjectCode(e.getOriginalSubjectCode());
  297. tableAttributeDO.setSubjectName(e.getSubjectName());
  298. tableAttributeDO.setCurrency(e.getCurrency());
  299. tableAttributeDO.setExchangeRate(e.getExchangeRate());
  300. tableAttributeDO.setSecuritiesAmount(e.getSecuritiesAmount());
  301. tableAttributeDO.setUnitCost(e.getUnitCost());
  302. tableAttributeDO.setOriCurrencyCost(e.getOCurrencyCost());
  303. tableAttributeDO.setNetCost(e.getNetCost());
  304. tableAttributeDO.setNetCostRatio(e.getNetCostRatio());
  305. tableAttributeDO.setMarketPrice(e.getMarketPrice());
  306. tableAttributeDO.setOriCurrencyMarketValue(e.getOCurrencyMarketValue());
  307. tableAttributeDO.setMarketValue(e.getMarketValue());
  308. tableAttributeDO.setMarketValueRatio(e.getMarketValueRatio());
  309. tableAttributeDO.setOriCurrencyIncrement(e.getOCurrencyMarketValue());
  310. tableAttributeDO.setIncrement(e.getIncrement());
  311. tableAttributeDO.setHaltInfo(e.getHaltInfo());
  312. tableAttributeDO.setRightsInterestsInfo(e.getRightsInterestsInfo());
  313. tableAttributeDO.setIsvalid(1);
  314. tableAttributeDO.setCreatorId(0);
  315. tableAttributeDO.setUpdaterId(0);
  316. tableAttributeDO.setCreateTime(new Date());
  317. tableAttributeDO.setUpdateTime(new Date());
  318. return tableAttributeDO;
  319. }).collect(Collectors.toList());
  320. }
  321. private ReportData requestPyAndResult(int fileId, EmailContentInfoDTO emailContentInfoDTO) {
  322. String fileName = emailContentInfoDTO.getFileName();
  323. Integer emailType = emailContentInfoDTO.getEmailType();
  324. ReportData reportData = null;
  325. if (Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType)) {
  326. if (StrUtil.isBlank(fileName)) {
  327. return null;
  328. }
  329. Pattern pattern = Pattern.compile("S(?:[A-Z]{0}[0-9]{5}|[A-Z][0-9]{4}|[A-Z]{2}[0-9]{3}|[A-Z]{3}[0-9]{2})");
  330. Matcher matcher = pattern.matcher(fileName);
  331. String registerNumber = null;
  332. if (matcher.find()) {
  333. registerNumber = matcher.group();
  334. }
  335. int type = 0;
  336. if (fileName.contains("季报")) {
  337. type = 1;
  338. } else if (fileName.contains("年报")) {
  339. type = 2;
  340. }
  341. String api = "/api/v1/parse/amac_report";
  342. Map<String, Object> params = MapUtil.newHashMap(16);
  343. params.put("file_id", fileId);
  344. params.put("file_path", emailContentInfoDTO.getFilePath());
  345. params.put("register_number", registerNumber);
  346. params.put("file_type", type);
  347. params.put("file_name", fileName);
  348. if (StrUtil.isNotBlank(registerNumber)) {
  349. FundAndCompanyInfoDO info = this.fundInfoMapper.queryFundAndTrustByRegisterNumber(registerNumber);
  350. if (info != null) {
  351. params.put("fund_name", info.getFundName());
  352. params.put("trust_name", info.getCompanyName());
  353. }
  354. }
  355. long millis = System.currentTimeMillis();
  356. try {
  357. String body = HttpUtil.post(this.pyBaseUrl + api, JSONUtil.toJsonStr(params));
  358. PythonResult<?> result = PythonReportConverter.convert(JSONUtil.parseObj(body), type);
  359. reportData = result.getData();
  360. if (log.isInfoEnabled()) {
  361. log.info("报告{}结果为:\n{}", params, reportData);
  362. }
  363. } catch (Exception e) {
  364. log.error("请求python的报告解析接口报错\n{}", ExceptionUtil.stacktraceToString(e));
  365. } finally {
  366. if (log.isInfoEnabled()) {
  367. log.info("当前报告{}解析完成,总计耗时{}ms", params, (System.currentTimeMillis() - millis));
  368. }
  369. }
  370. }
  371. return reportData;
  372. }
  373. private void saveNavAndAssetNet(Integer fileId, List<EmailFundNavDTO> fundNavDTOList, Date parseDate) {
  374. if (CollUtil.isEmpty(fundNavDTOList)) {
  375. return;
  376. }
  377. // 净值数据
  378. List<EmailFundNavDO> emailFundNavDOList = fundNavDTOList.stream()
  379. .map(e -> buildEmailFundNavDo(fileId, e, parseDate)).filter(CollUtil::isNotEmpty).flatMap(List::stream).collect(Collectors.toList());
  380. if (CollUtil.isNotEmpty(emailFundNavDOList)) {
  381. // 先删除文件id下的净值数据(考虑到重新解析的需求,如果是首次解析,那么file_id下不存在净值数据)
  382. emailFundNavMapper.deleteByFileId(fileId);
  383. emailFundNavMapper.batchInsert(emailFundNavDOList);
  384. List<NavDO> navDOList = emailFundNavDOList.stream().filter(e -> StrUtil.isNotBlank(e.getFundId()))
  385. .map(e -> BeanUtil.copyProperties(e, NavDO.class)).collect(Collectors.toList());
  386. saveNavDo(navDOList);
  387. }
  388. // 保存规模数据
  389. List<EmailFundAssetDO> emailFundAssetDOList = fundNavDTOList.stream()
  390. .map(e -> buildEmailFundAssetDo(fileId, e, parseDate)).filter(CollUtil::isNotEmpty).flatMap(List::stream).collect(Collectors.toList());
  391. if (CollUtil.isNotEmpty(emailFundAssetDOList)) {
  392. // 先删除file_id下的规模数据(考虑到重新解析的需求,如果是首次解析,那么file_id下不存在规模数据)
  393. emailFundAssetMapper.deleteByFileId(fileId);
  394. emailFundAssetMapper.batchInsert(emailFundAssetDOList);
  395. List<AssetDO> assetDOList = emailFundAssetDOList.stream().filter(e -> StrUtil.isNotBlank(e.getFundId()))
  396. .map(e -> BeanUtil.copyProperties(e, AssetDO.class)).collect(Collectors.toList());
  397. saveAssetDo(assetDOList);
  398. }
  399. }
  400. public void saveNavDo(List<NavDO> navDOList) {
  401. if (CollUtil.isEmpty(navDOList)) {
  402. return;
  403. }
  404. Map<String, List<NavDO>> fundIdNavMap = navDOList.stream().collect(Collectors.groupingBy(NavDO::getFundId));
  405. for (Map.Entry<String, List<NavDO>> entry : fundIdNavMap.entrySet()) {
  406. List<NavDO> navDOS = entry.getValue();
  407. List<String> priceDateList = navDOS.stream().map(NavDO::getPriceDate).map(e -> DateUtil.format(e, DateConst.YYYY_MM_DD)).collect(Collectors.toList());
  408. List<String> dateList = navMapper.queryFundNavByDate(entry.getKey(), priceDateList);
  409. List<NavDO> updateNavDoList = navDOS.stream().filter(e -> dateList.contains(DateUtil.format(e.getPriceDate(), DateConst.YYYY_MM_DD))).collect(Collectors.toList());
  410. List<NavDO> insertNavDoList = navDOS.stream().filter(e -> !dateList.contains(DateUtil.format(e.getPriceDate(), DateConst.YYYY_MM_DD))).collect(Collectors.toList());
  411. if (CollUtil.isNotEmpty(insertNavDoList)) {
  412. Map<Date, List<NavDO>> priceDateNavDoListMap = insertNavDoList.stream().collect(Collectors.groupingBy(NavDO::getPriceDate));
  413. boolean hasDuplicationDateData = priceDateNavDoListMap.values().stream().map(List::size).anyMatch(e -> e > 1);
  414. if (!hasDuplicationDateData) {
  415. navMapper.batchInsert(insertNavDoList);
  416. }
  417. // 要插入的数据中存在相同日期的数据 -> 只能一条一条的插入数据了
  418. insertNavDoList.forEach(e -> saveNavDo(ListUtil.toList(e)));
  419. }
  420. if (CollUtil.isNotEmpty(updateNavDoList)) {
  421. navMapper.batchUpdate(updateNavDoList);
  422. }
  423. }
  424. }
  425. public void saveAssetDo(List<AssetDO> assetDOList) {
  426. if (CollUtil.isEmpty(assetDOList)) {
  427. return;
  428. }
  429. Map<String, List<AssetDO>> fundIdNavMap = assetDOList.stream().collect(Collectors.groupingBy(AssetDO::getFundId));
  430. for (Map.Entry<String, List<AssetDO>> entry : fundIdNavMap.entrySet()) {
  431. List<AssetDO> assetDOS = entry.getValue();
  432. List<String> priceDateList = assetDOS.stream().map(AssetDO::getPriceDate).map(e -> DateUtil.format(e, DateConst.YYYY_MM_DD)).collect(Collectors.toList());
  433. List<String> dateList = assetMapper.queryFundNavByDate(entry.getKey(), priceDateList);
  434. List<AssetDO> updateAssetDoList = assetDOS.stream().filter(e -> dateList.contains(DateUtil.format(e.getPriceDate(), DateConst.YYYY_MM_DD))).collect(Collectors.toList());
  435. List<AssetDO> insertAssetDoList = assetDOS.stream().filter(e -> !dateList.contains(DateUtil.format(e.getPriceDate(), DateConst.YYYY_MM_DD))).collect(Collectors.toList());
  436. if (CollUtil.isNotEmpty(insertAssetDoList)) {
  437. Map<Date, List<AssetDO>> priceDateAssetDoListMap = insertAssetDoList.stream().collect(Collectors.groupingBy(AssetDO::getPriceDate));
  438. boolean hasDuplicationDateData = priceDateAssetDoListMap.values().stream().map(List::size).anyMatch(e -> e > 1);
  439. if (!hasDuplicationDateData) {
  440. assetMapper.batchInsert(insertAssetDoList);
  441. }
  442. // 要插入的数据中存在相同日期的数据 -> 只能一条一条的插入数据了
  443. insertAssetDoList.forEach(e -> saveAssetDo(ListUtil.toList(e)));
  444. }
  445. if (CollUtil.isNotEmpty(updateAssetDoList)) {
  446. assetMapper.batchUpdate(updateAssetDoList);
  447. }
  448. }
  449. }
  450. private List<EmailFundAssetDO> buildEmailFundAssetDo(Integer fileId, EmailFundNavDTO fundNavDTO, Date parseDate) {
  451. List<EmailFundAssetDO> fundAssetDOList = CollUtil.newArrayList();
  452. BigDecimal assetNet = StrUtil.isNotBlank(fundNavDTO.getAssetNet()) ? new BigDecimal(fundNavDTO.getAssetNet()) : null;
  453. BigDecimal assetShare = StrUtil.isNotBlank(fundNavDTO.getAssetShare()) ? new BigDecimal(fundNavDTO.getAssetShare()) : null;
  454. if (assetNet == null) {
  455. return fundAssetDOList;
  456. }
  457. Integer isStored = fundNavDTO.getParseStatus() != null
  458. && (fundNavDTO.getParseStatus().equals(NavParseStatusConst.ASSET_NET_NEGATIVE) || fundNavDTO.getParseStatus().equals(NavParseStatusConst.SUCCESS)) ? 1 : 0;
  459. Date priceDate = DateUtil.parse(fundNavDTO.getPriceDate(), DateConst.YYYY_MM_DD);
  460. if (CollUtil.isNotEmpty(fundNavDTO.getFundIdList())) {
  461. for (String fundId : fundNavDTO.getFundIdList()) {
  462. EmailFundAssetDO emailFundAssetDO = new EmailFundAssetDO();
  463. emailFundAssetDO.setFileId(fileId);
  464. emailFundAssetDO.setPriceDate(priceDate);
  465. emailFundAssetDO.setFundId(fundId);
  466. emailFundAssetDO.setFundName(fundNavDTO.getFundName());
  467. emailFundAssetDO.setRegisterNumber(fundNavDTO.getRegisterNumber());
  468. emailFundAssetDO.setAssetNet(assetNet);
  469. emailFundAssetDO.setAssetShare(assetShare);
  470. emailFundAssetDO.setIsStored(isStored);
  471. emailFundAssetDO.setExceptionStatus(fundNavDTO.getParseStatus());
  472. emailFundAssetDO.setIsvalid(1);
  473. emailFundAssetDO.setCreatorId(0);
  474. emailFundAssetDO.setCreateTime(parseDate);
  475. emailFundAssetDO.setUpdaterId(0);
  476. emailFundAssetDO.setUpdateTime(parseDate);
  477. fundAssetDOList.add(emailFundAssetDO);
  478. }
  479. } else {
  480. EmailFundAssetDO emailFundAssetDO = new EmailFundAssetDO();
  481. emailFundAssetDO.setFileId(fileId);
  482. emailFundAssetDO.setPriceDate(priceDate);
  483. emailFundAssetDO.setFundName(fundNavDTO.getFundName());
  484. emailFundAssetDO.setRegisterNumber(fundNavDTO.getRegisterNumber());
  485. emailFundAssetDO.setAssetNet(assetNet);
  486. emailFundAssetDO.setAssetShare(assetShare);
  487. emailFundAssetDO.setIsStored(isStored);
  488. emailFundAssetDO.setExceptionStatus(fundNavDTO.getParseStatus());
  489. emailFundAssetDO.setIsvalid(1);
  490. emailFundAssetDO.setCreatorId(0);
  491. emailFundAssetDO.setCreateTime(parseDate);
  492. emailFundAssetDO.setUpdaterId(0);
  493. emailFundAssetDO.setUpdateTime(parseDate);
  494. fundAssetDOList.add(emailFundAssetDO);
  495. }
  496. return fundAssetDOList;
  497. }
  498. private List<EmailFundNavDO> buildEmailFundNavDo(Integer fileId, EmailFundNavDTO fundNavDTO, Date parseDate) {
  499. List<EmailFundNavDO> fundNavDOList = CollUtil.newArrayList();
  500. Date priceDate = DateUtil.parse(fundNavDTO.getPriceDate(), DateConst.YYYY_MM_DD);
  501. BigDecimal nav = StrUtil.isNotBlank(fundNavDTO.getNav()) ? new BigDecimal(fundNavDTO.getNav()) : null;
  502. BigDecimal cumulativeNavWithdrawal = StrUtil.isNotBlank(fundNavDTO.getCumulativeNavWithdrawal()) ? new BigDecimal(fundNavDTO.getCumulativeNavWithdrawal()) : null;
  503. Integer isStored = fundNavDTO.getParseStatus() != null && !fundNavDTO.getParseStatus().equals(NavParseStatusConst.NAV_DEFICIENCY)
  504. && !fundNavDTO.getParseStatus().equals(NavParseStatusConst.NOT_MATCH) ? 1 : 0;
  505. if (CollUtil.isNotEmpty(fundNavDTO.getFundIdList())) {
  506. for (String fundId : fundNavDTO.getFundIdList()) {
  507. EmailFundNavDO emailFundNavDO = new EmailFundNavDO();
  508. emailFundNavDO.setFileId(fileId);
  509. emailFundNavDO.setIsStored(isStored);
  510. emailFundNavDO.setPriceDate(priceDate);
  511. emailFundNavDO.setNav(nav);
  512. emailFundNavDO.setFundId(fundId);
  513. emailFundNavDO.setCumulativeNavWithdrawal(cumulativeNavWithdrawal);
  514. emailFundNavDO.setFundName(fundNavDTO.getFundName());
  515. emailFundNavDO.setRegisterNumber(fundNavDTO.getRegisterNumber());
  516. emailFundNavDO.setExceptionStatus(fundNavDTO.getParseStatus());
  517. emailFundNavDO.setTemplateId(fundNavDTO.getTemplateId());
  518. emailFundNavDO.setIsvalid(1);
  519. emailFundNavDO.setCreatorId(0);
  520. emailFundNavDO.setCreateTime(parseDate);
  521. emailFundNavDO.setUpdaterId(0);
  522. emailFundNavDO.setUpdateTime(parseDate);
  523. fundNavDOList.add(emailFundNavDO);
  524. }
  525. } else {
  526. EmailFundNavDO emailFundNavDO = new EmailFundNavDO();
  527. emailFundNavDO.setFileId(fileId);
  528. emailFundNavDO.setPriceDate(priceDate);
  529. emailFundNavDO.setNav(nav);
  530. emailFundNavDO.setCumulativeNavWithdrawal(cumulativeNavWithdrawal);
  531. emailFundNavDO.setFundName(fundNavDTO.getFundName());
  532. emailFundNavDO.setRegisterNumber(fundNavDTO.getRegisterNumber());
  533. emailFundNavDO.setExceptionStatus(fundNavDTO.getParseStatus());
  534. emailFundNavDO.setTemplateId(fundNavDTO.getTemplateId());
  535. emailFundNavDO.setIsStored(isStored);
  536. emailFundNavDO.setIsvalid(1);
  537. emailFundNavDO.setCreatorId(0);
  538. emailFundNavDO.setCreateTime(parseDate);
  539. emailFundNavDO.setUpdaterId(0);
  540. emailFundNavDO.setUpdateTime(parseDate);
  541. fundNavDOList.add(emailFundNavDO);
  542. }
  543. return fundNavDOList;
  544. }
  545. private Integer saveEmailFileInfo(Integer emailId, Integer fileId, String fileName, String filePath, Date parseDate) {
  546. EmailFileInfoDO emailFileInfoDO = buildEmailFileInfoDO(emailId, fileId, fileName, filePath, parseDate);
  547. if (emailFileInfoDO.getId() != null) {
  548. emailFileInfoMapper.updateTimeById(fileId, parseDate);
  549. return emailFileInfoDO.getId();
  550. }
  551. emailFileInfoMapper.insert(emailFileInfoDO);
  552. return emailFileInfoDO.getId();
  553. }
  554. private EmailFileInfoDO buildEmailFileInfoDO(Integer emailId, Integer fileId, String fileName, String filePath, Date parseDate) {
  555. EmailFileInfoDO emailFileInfoDO = new EmailFileInfoDO();
  556. emailFileInfoDO.setId(fileId);
  557. emailFileInfoDO.setEmailId(emailId);
  558. emailFileInfoDO.setFileName(fileName);
  559. emailFileInfoDO.setFilePath(filePath);
  560. emailFileInfoDO.setIsvalid(1);
  561. emailFileInfoDO.setCreatorId(0);
  562. emailFileInfoDO.setCreateTime(parseDate);
  563. emailFileInfoDO.setUpdaterId(0);
  564. emailFileInfoDO.setUpdateTime(parseDate);
  565. return emailFileInfoDO;
  566. }
  567. private void setNavParseStatus(EmailFundNavDTO fundNavDTO, String emailTitle) {
  568. // 1.单位净值或累计净值缺失
  569. if (StrUtil.isBlank(fundNavDTO.getNav()) || StrUtil.isBlank(fundNavDTO.getCumulativeNavWithdrawal())) {
  570. fundNavDTO.setParseStatus(NavParseStatusConst.NAV_DEFICIENCY);
  571. return;
  572. }
  573. // 考虑单独规模文件时 -> 无单位净值和累计净值
  574. // 2.单位净值或累计净值不大于0
  575. if (!emailTitle.contains("规模")) {
  576. if (StrUtil.isBlank(fundNavDTO.getNav()) || StrUtil.isBlank(fundNavDTO.getCumulativeNavWithdrawal())
  577. || (fundNavDTO.getNav().compareTo("0") <= 0 || fundNavDTO.getCumulativeNavWithdrawal().compareTo("0") <= 0)) {
  578. fundNavDTO.setParseStatus(NavParseStatusConst.NAV_NEGATIVE);
  579. return;
  580. }
  581. }
  582. // 3.资产净值不大于0
  583. if (StrUtil.isNotBlank(fundNavDTO.getAssetNet()) && fundNavDTO.getAssetNet().compareTo("0") <= 0) {
  584. fundNavDTO.setParseStatus(NavParseStatusConst.ASSET_NET_NEGATIVE);
  585. return;
  586. }
  587. // 4.匹配基金(考虑到解析估值表时已经匹配上基金的情况)
  588. List<String> fundIdList = fundNavDTO.getFundIdList();
  589. if (CollUtil.isEmpty(fundIdList)) {
  590. fundIdList = fundService.getFundIdByNamesAndCode(fundNavDTO.getFundName(), fundNavDTO.getRegisterNumber());
  591. if (CollUtil.isEmpty(fundIdList)) {
  592. fundNavDTO.setParseStatus(NavParseStatusConst.NOT_MATCH);
  593. }
  594. }
  595. fundNavDTO.setFundIdList(fundIdList);
  596. // 写入别名管理表fund_alias
  597. saveFundAlias(fundNavDTO.getFundName(), fundNavDTO.getRegisterNumber(), fundIdList);
  598. if (CollUtil.isEmpty(fundIdList)) {
  599. return;
  600. }
  601. fundNavDTO.setParseStatus(NavParseStatusConst.SUCCESS);
  602. }
  603. private void saveFundAlias(String fundName, String registerNumber, List<String> fundIdList) {
  604. // 未识别到基金名称和备案编码的数据不写入别名管理
  605. if (StrUtil.isBlank(fundName) && StrUtil.isBlank(registerNumber)) {
  606. return;
  607. }
  608. List<FundAliasDO> fundAliasDOList = CollUtil.newArrayList();
  609. if (StrUtil.isNotBlank(fundName) && StrUtil.isNotBlank(registerNumber)) {
  610. fundAliasDOList = fundAliasMapper.queryFundIdByNameAndRegisterNumber(fundName, registerNumber);
  611. }
  612. if (StrUtil.isBlank(fundName) && StrUtil.isNotBlank(registerNumber)) {
  613. fundAliasDOList = fundAliasMapper.queryFundIdByRegisterNumber(registerNumber);
  614. }
  615. if (StrUtil.isNotBlank(fundName) && StrUtil.isBlank(registerNumber)) {
  616. fundAliasDOList = fundAliasMapper.queryFundIdByName(fundName);
  617. }
  618. // 未匹配基金且已写入别名表
  619. if (CollUtil.isEmpty(fundIdList) && CollUtil.isNotEmpty(fundAliasDOList)) {
  620. return;
  621. }
  622. // 未匹配基金且未写入别名表
  623. if (CollUtil.isEmpty(fundIdList) && CollUtil.isEmpty(fundAliasDOList)) {
  624. fundAliasMapper.batchInsert(ListUtil.toList(buildFundAliasDO(fundName, registerNumber, null)));
  625. log.info("写入别名表 -> 基金名称:{},备案编码:{},基金id:{}", fundName, registerNumber, fundIdList);
  626. return;
  627. }
  628. // 匹配上基金 -> 需要判断此时基金id是否已经在别名管理表
  629. if (CollUtil.isNotEmpty(fundAliasDOList)) {
  630. List<String> collect = fundAliasDOList.stream().filter(Objects::nonNull).map(FundAliasDO::getTargetFundId).filter(Objects::nonNull).distinct().toList();
  631. fundIdList = fundIdList.stream().filter(e -> !collect.contains(e)).toList();
  632. }
  633. List<FundAliasDO> fundAliasDOS = CollUtil.isNotEmpty(fundIdList) ? fundIdList.stream().map(e -> buildFundAliasDO(fundName, registerNumber, e)).toList() : null;
  634. if (CollUtil.isNotEmpty(fundAliasDOS)) {
  635. fundAliasMapper.batchInsert(fundAliasDOS);
  636. log.info("写入别名表 -> 基金名称:{},备案编码:{},基金id:{}", fundName, registerNumber, fundIdList);
  637. }
  638. }
  639. public FundAliasDO buildFundAliasDO(String fundName, String registerNumber, String fundId) {
  640. FundAliasDO fundAliasDO = new FundAliasDO();
  641. fundAliasDO.setTargetFundId(fundId);
  642. fundAliasDO.setSourceFundName(fundName);
  643. fundAliasDO.setSourceRegisterNumber(registerNumber);
  644. fundAliasDO.setIsvalid(1);
  645. fundAliasDO.setCreatorId(0);
  646. fundAliasDO.setCreateTime(new Date());
  647. fundAliasDO.setUpdateTime(new Date());
  648. fundAliasDO.setUpdaterId(0);
  649. return fundAliasDO;
  650. }
  651. private Integer saveEmailParseInfo(EmailParseInfoDO emailParseInfoDO) {
  652. if (emailParseInfoDO == null) {
  653. return null;
  654. }
  655. // 重新邮件功能 -> 修改解析时间和更新时间
  656. if (emailParseInfoDO.getId() != null) {
  657. emailParseInfoMapper.updateParseTime(emailParseInfoDO.getId(), emailParseInfoDO.getParseDate());
  658. return emailParseInfoDO.getId();
  659. }
  660. emailParseInfoMapper.insert(emailParseInfoDO);
  661. return emailParseInfoDO.getId();
  662. }
  663. private EmailParseInfoDO buildEmailParseInfo(Integer emailId, String emailAddress, String senderEmail, String emailDate,
  664. String emailTitle, Integer emailType, Integer parseStatus, Date parseDate) {
  665. EmailParseInfoDO emailParseInfoDO = new EmailParseInfoDO();
  666. emailParseInfoDO.setId(emailId);
  667. emailParseInfoDO.setSenderEmail(senderEmail);
  668. emailParseInfoDO.setEmail(emailAddress);
  669. emailParseInfoDO.setEmailDate(DateUtil.parse(emailDate, DateConst.YYYY_MM_DD_HH_MM_SS));
  670. emailParseInfoDO.setParseDate(parseDate);
  671. emailParseInfoDO.setEmailTitle(emailTitle);
  672. emailParseInfoDO.setEmailType(emailType);
  673. emailParseInfoDO.setParseStatus(parseStatus);
  674. emailParseInfoDO.setIsvalid(1);
  675. emailParseInfoDO.setCreatorId(0);
  676. emailParseInfoDO.setCreateTime(parseDate);
  677. emailParseInfoDO.setUpdaterId(0);
  678. emailParseInfoDO.setUpdateTime(parseDate);
  679. return emailParseInfoDO;
  680. }
  681. public Map<String, List<String>> getEmailFieldMapping() {
  682. List<EmailFieldMappingDO> emailFieldMappingDOList = emailFieldMapper.getEmailFieldMapping();
  683. return emailFieldMappingDOList.stream()
  684. .collect(Collectors.toMap(EmailFieldMappingDO::getCode, v -> Arrays.stream(v.getName().split(",")).toList()));
  685. }
  686. public Map<Integer, List<String>> getEmailType() {
  687. Map<Integer, List<String>> emailTypeMap = MapUtil.newHashMap(3, true);
  688. EmailTypeRuleDO emailTypeRuleDO = emailTypeRuleMapper.getEmailTypeRule();
  689. String nav = emailTypeRuleDO != null && StrUtil.isNotBlank(emailTypeRuleDO.getNav()) ? emailTypeRuleDO.getNav() : emailRuleConfig.getNav();
  690. String valuation = emailTypeRuleDO != null && StrUtil.isNotBlank(emailTypeRuleDO.getValuation()) ? emailTypeRuleDO.getValuation() : emailRuleConfig.getValuation();
  691. String report = emailTypeRuleDO != null && StrUtil.isNotBlank(emailTypeRuleDO.getReport()) ? emailTypeRuleDO.getReport() : emailRuleConfig.getReport();
  692. emailTypeMap.put(EmailTypeConst.VALUATION_EMAIL_TYPE, Arrays.stream(valuation.split(",")).toList());
  693. emailTypeMap.put(EmailTypeConst.REPORT_EMAIL_TYPE, Arrays.stream(report.split(",")).toList());
  694. emailTypeMap.put(EmailTypeConst.NAV_EMAIL_TYPE, Arrays.stream(nav.split(",")).toList());
  695. return emailTypeMap;
  696. }
  697. /**
  698. * 读取邮件
  699. *
  700. * @param mailboxInfoDTO 邮箱配置信息
  701. * @param emailTypeMap 邮件类型识别规则映射表
  702. * @param startDate 邮件起始日期
  703. * @param endDate 邮件截止日期(为null,将解析邮件日期小于等于startDate的当天邮件)
  704. * @return 读取到的邮件信息
  705. * @throws Exception 异常信息
  706. */
  707. private Map<String, List<EmailContentInfoDTO>> realEmail(MailboxInfoDTO mailboxInfoDTO, Map<Integer, List<String>> emailTypeMap, Date startDate, Date endDate) throws Exception {
  708. Store store = EmailUtil.getStoreNew(mailboxInfoDTO);
  709. if (store == null) {
  710. return MapUtil.newHashMap();
  711. }
  712. // 默认读取收件箱的邮件
  713. Folder folder = store.getFolder("INBOX");
  714. folder.open(Folder.READ_ONLY);
  715. Message[] messages = getEmailMessage(folder, mailboxInfoDTO.getProtocol(), startDate);
  716. if (messages == null || messages.length == 0) {
  717. log.info("获取不到邮件 -> 邮箱信息:{},开始时间:{},结束时间:{}", mailboxInfoDTO, startDate, endDate);
  718. return MapUtil.newHashMap();
  719. }
  720. Map<String, List<EmailContentInfoDTO>> emailMessageMap = MapUtil.newHashMap();
  721. for (Message message : messages) {
  722. List<EmailContentInfoDTO> emailContentInfoDTOList = CollUtil.newArrayList();
  723. String uuidKey = UUID.randomUUID().toString().replaceAll("-", "");
  724. Integer emailType;
  725. String senderEmail;
  726. try {
  727. Date emailDate = message.getSentDate();
  728. boolean isNotParseConditionSatisfied = emailDate == null || (endDate != null && emailDate.compareTo(endDate) > 0) || (startDate != null && emailDate.compareTo(startDate) < 0);
  729. if (isNotParseConditionSatisfied) {
  730. continue;
  731. }
  732. senderEmail = getSenderEmail(message.getFrom());
  733. emailType = EmailUtil.getEmailTypeBySubject(message.getSubject(), emailTypeMap);
  734. String emailDateStr = DateUtil.format(emailDate, DateConst.YYYY_MM_DD_HH_MM_SS);
  735. if (emailType == null) {
  736. log.info("邮件不满足解析条件 -> 邮件主题:{},邮件日期:{}", message.getSubject(), emailDateStr);
  737. continue;
  738. }
  739. log.info("邮件采集成功 -> 邮件主题:{},邮件日期:{}", message.getSubject(), emailDateStr);
  740. Object content = message.getContent();
  741. // 1.邮件为MIME多部分消息体:可能既有邮件又有正文
  742. if (content instanceof MimeMultipart) {
  743. emailContentInfoDTOList = EmailUtil.collectMimeMultipart(message, mailboxInfoDTO.getAccount(), path);
  744. }
  745. // 2.邮件只有正文
  746. if (content instanceof String) {
  747. EmailContentInfoDTO emailContentInfoDTO = new EmailContentInfoDTO();
  748. emailContentInfoDTO.setEmailContent(content.toString());
  749. emailContentInfoDTO.setEmailDate(emailDateStr);
  750. emailContentInfoDTO.setEmailTitle(message.getSubject());
  751. String fileName = message.getSubject() + DateUtil.format(emailDate, DateConst.YYYYMMDDHHMMSS24);
  752. String filePath = path + mailboxInfoDTO.getAccount() + "/" + DateUtil.format(emailDate, DateConst.YYYY_MM_DD) + "/" + fileName + ".html";
  753. File saveFile = new File(filePath);
  754. saveFile.setReadable(true);
  755. if (!saveFile.exists()) {
  756. if (!saveFile.getParentFile().exists()) {
  757. saveFile.getParentFile().mkdirs();
  758. saveFile.getParentFile().setExecutable(true);
  759. }
  760. }
  761. FileUtil.writeFile(filePath, content.toString());
  762. emailContentInfoDTO.setFilePath(filePath);
  763. emailContentInfoDTO.setFileName(fileName);
  764. emailContentInfoDTOList.add(emailContentInfoDTO);
  765. }
  766. if (CollUtil.isNotEmpty(emailContentInfoDTOList)) {
  767. emailContentInfoDTOList.forEach(e -> {
  768. e.setEmailType(emailType);
  769. e.setSenderEmail(senderEmail);
  770. });
  771. emailMessageMap.put(uuidKey, emailContentInfoDTOList);
  772. }
  773. } catch (Exception e) {
  774. log.error("获取邮箱的邮件报错,堆栈信息:{}", ExceptionUtil.stacktraceToString(e));
  775. }
  776. }
  777. folder.close(false);
  778. store.close();
  779. return emailMessageMap;
  780. }
  781. private String getSenderEmail(Address[] senderAddress) {
  782. if (senderAddress == null || senderAddress.length == 0) {
  783. return null;
  784. }
  785. // 此时的address是含有编码(MIME编码方式)后的文本和实际的邮件地址
  786. String address = senderAddress[0].toString();
  787. // 正则表达式匹配邮件地址
  788. Pattern pattern = Pattern.compile("<(\\S+)>");
  789. Matcher matcher = pattern.matcher(address);
  790. if (matcher.find()) {
  791. return matcher.group(1);
  792. }
  793. return null;
  794. }
  795. private Message[] getEmailMessage(Folder folder, String protocol, Date startDate) {
  796. try {
  797. if (protocol.contains("imap")) {
  798. // 获取邮件日期大于等于startDate的邮件(搜索条件只支持按天)
  799. SearchTerm startDateTerm = new ReceivedDateTerm(ComparisonTerm.GE, startDate);
  800. return folder.search(startDateTerm);
  801. } else {
  802. return folder.getMessages();
  803. }
  804. } catch (MessagingException e) {
  805. throw new RuntimeException(e);
  806. }
  807. }
  808. private static class PythonData {
  809. private Integer fileId;
  810. private Integer status;
  811. private String msg;
  812. private String register_number;
  813. public Integer getFileId() {
  814. return fileId;
  815. }
  816. public void setFileId(Integer fileId) {
  817. this.fileId = fileId;
  818. }
  819. public Integer getStatus() {
  820. return status;
  821. }
  822. public void setStatus(Integer status) {
  823. this.status = status;
  824. }
  825. public String getMsg() {
  826. return msg;
  827. }
  828. public void setMsg(String msg) {
  829. this.msg = msg;
  830. }
  831. public String getRegister_number() {
  832. return register_number;
  833. }
  834. public void setRegister_number(String register_number) {
  835. this.register_number = register_number;
  836. }
  837. }
  838. }