EmailParseService.java 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558
  1. package com.smppw.modaq.domain.service;
  2. import cn.hutool.core.collection.CollUtil;
  3. import cn.hutool.core.collection.ListUtil;
  4. import cn.hutool.core.date.DateUtil;
  5. import cn.hutool.core.exceptions.ExceptionUtil;
  6. import cn.hutool.core.map.MapUtil;
  7. import cn.hutool.core.util.StrUtil;
  8. import com.smppw.modaq.application.components.ReportParseUtils;
  9. import com.smppw.modaq.application.components.report.parser.ReportParser;
  10. import com.smppw.modaq.application.components.report.parser.ReportParserFactory;
  11. import com.smppw.modaq.application.components.report.writer.ReportWriter;
  12. import com.smppw.modaq.application.components.report.writer.ReportWriterFactory;
  13. import com.smppw.modaq.application.util.EmailUtil;
  14. import com.smppw.modaq.common.conts.DateConst;
  15. import com.smppw.modaq.common.conts.EmailParseStatusConst;
  16. import com.smppw.modaq.common.conts.EmailTypeConst;
  17. import com.smppw.modaq.common.enums.ReportParseStatus;
  18. import com.smppw.modaq.common.enums.ReportParserFileType;
  19. import com.smppw.modaq.common.enums.ReportType;
  20. import com.smppw.modaq.common.exception.ReportParseException;
  21. import com.smppw.modaq.domain.dto.EmailContentInfoDTO;
  22. import com.smppw.modaq.domain.dto.EmailZipFileDTO;
  23. import com.smppw.modaq.domain.dto.MailboxInfoDTO;
  24. import com.smppw.modaq.domain.dto.report.ParseResult;
  25. import com.smppw.modaq.domain.dto.report.ReportData;
  26. import com.smppw.modaq.domain.dto.report.ReportParserParams;
  27. import com.smppw.modaq.domain.entity.EmailFileInfoDO;
  28. import com.smppw.modaq.domain.entity.EmailParseInfoDO;
  29. import com.smppw.modaq.domain.mapper.EmailFileInfoMapper;
  30. import com.smppw.modaq.domain.mapper.EmailParseInfoMapper;
  31. import com.smppw.modaq.infrastructure.util.ExcelUtil;
  32. import jakarta.mail.*;
  33. import jakarta.mail.internet.MimeMessage;
  34. import jakarta.mail.internet.MimeMultipart;
  35. import jakarta.mail.search.ComparisonTerm;
  36. import jakarta.mail.search.ReceivedDateTerm;
  37. import jakarta.mail.search.SearchTerm;
  38. import org.apache.commons.compress.archivers.ArchiveException;
  39. import org.slf4j.Logger;
  40. import org.slf4j.LoggerFactory;
  41. import org.springframework.beans.factory.annotation.Value;
  42. import org.springframework.stereotype.Service;
  43. import org.springframework.util.StopWatch;
  44. import java.io.File;
  45. import java.io.FileNotFoundException;
  46. import java.io.IOException;
  47. import java.nio.file.Paths;
  48. import java.util.*;
  49. import java.util.regex.Matcher;
  50. import java.util.regex.Pattern;
  51. import java.util.stream.Collectors;
  52. /**
  53. * @author mozuwen
  54. * @date 2024-09-04
  55. * @description 邮件解析服务
  56. */
  57. @Service
  58. public class EmailParseService {
  59. // public static final int stepSize = 10000;
  60. private static final Logger log = LoggerFactory.getLogger(EmailParseService.class);
  61. // private final EmailFieldMappingMapper emailFieldMapper;
  62. private final EmailParseInfoMapper emailParseInfoMapper;
  63. private final EmailFileInfoMapper emailFileInfoMapper;
  64. /* 报告解析和入库的方法 */
  65. private final ReportParserFactory reportParserFactory;
  66. private final ReportWriterFactory reportWriterFactory;
  67. @Value("${email.file.path}")
  68. private String path;
  69. public EmailParseService(EmailParseInfoMapper emailParseInfoMapper,
  70. EmailFileInfoMapper emailFileInfoMapper,
  71. ReportParserFactory reportParserFactory,
  72. ReportWriterFactory reportWriterFactory) {
  73. // this.emailFieldMapper = emailFieldMapper;
  74. this.emailParseInfoMapper = emailParseInfoMapper;
  75. this.emailFileInfoMapper = emailFileInfoMapper;
  76. this.reportParserFactory = reportParserFactory;
  77. this.reportWriterFactory = reportWriterFactory;
  78. }
  79. /**
  80. * 解析指定邮箱指定时间范围内的邮件
  81. *
  82. * @param mailboxInfoDTO 邮箱配置信息
  83. * @param startDate 邮件起始日期(yyyy-MM-dd HH:mm:ss)
  84. * @param endDate 邮件截止日期(yyyy-MM-dd HH:mm:ss, 为null,将解析邮件日期小于等于startDate的当天邮件)
  85. */
  86. public void parseEmail(MailboxInfoDTO mailboxInfoDTO, Date startDate, Date endDate) {
  87. log.info("开始邮件解析 -> 邮箱信息:{},开始时间:{},结束时间:{}", mailboxInfoDTO, DateUtil.format(startDate,
  88. DateConst.YYYY_MM_DD_HH_MM_SS), DateUtil.format(endDate, DateConst.YYYY_MM_DD_HH_MM_SS));
  89. // 邮件类型配置
  90. Map<Integer, List<String>> emailTypeMap = getEmailType();
  91. // 邮件字段识别映射表
  92. // Map<String, List<String>> emailFieldMap = getEmailFieldMapping();
  93. Map<String, List<EmailContentInfoDTO>> emailContentMap;
  94. try {
  95. emailContentMap = realEmail(mailboxInfoDTO, emailTypeMap, startDate, endDate);
  96. } catch (Exception e) {
  97. log.info("采集邮件失败 -> 邮箱配置信息:{},堆栈信息:{}", mailboxInfoDTO, ExceptionUtil.stacktraceToString(e));
  98. return;
  99. }
  100. if (MapUtil.isEmpty(emailContentMap)) {
  101. log.info("未采集到邮件 -> 邮箱配置信息:{},开始时间:{},结束时间:{}", mailboxInfoDTO,
  102. DateUtil.format(startDate, DateConst.YYYY_MM_DD_HH_MM_SS), DateUtil.format(endDate, DateConst.YYYY_MM_DD_HH_MM_SS));
  103. return;
  104. }
  105. for (Map.Entry<String, List<EmailContentInfoDTO>> emailEntry : emailContentMap.entrySet()) {
  106. List<EmailContentInfoDTO> emailContentInfoDTOList = emailEntry.getValue();
  107. if (CollUtil.isEmpty(emailContentInfoDTOList)) {
  108. log.warn("未采集到正文或附件");
  109. continue;
  110. }
  111. log.info("开始解析邮件数据 -> 邮件主题:{},邮件日期:{}", emailContentInfoDTOList.get(0).getEmailTitle(), emailContentInfoDTOList.get(0).getEmailDate());
  112. Map<EmailContentInfoDTO, List<EmailZipFileDTO>> emailZipFileMap = MapUtil.newHashMap();
  113. Iterator<EmailContentInfoDTO> iterator = emailContentInfoDTOList.iterator();
  114. while (iterator.hasNext()) {
  115. EmailContentInfoDTO emailContentInfoDTO = iterator.next();
  116. try {
  117. List<EmailZipFileDTO> fundNavDTOList = parseZipEmail(emailContentInfoDTO);
  118. emailZipFileMap.put(emailContentInfoDTO, fundNavDTOList);
  119. } catch (IOException | ArchiveException e) {
  120. log.error("压缩包解压失败:{}", ExceptionUtil.stacktraceToString(e));
  121. EmailParseInfoDO fail = buildEmailParseInfo(null, mailboxInfoDTO.getAccount(), emailContentInfoDTO);
  122. fail.setFailReason("压缩包解压失败");
  123. fail.setParseStatus(EmailParseStatusConst.FAIL);
  124. fail.setEmailKey(emailEntry.getKey());
  125. this.emailParseInfoMapper.insert(fail);
  126. iterator.remove();
  127. } catch (Exception e) {
  128. log.error("堆栈信息:{}", ExceptionUtil.stacktraceToString(e));
  129. }
  130. }
  131. // 保存相关信息 -> 邮件信息表,邮件文件表,邮件净值表,邮件规模表,基金净值表
  132. saveRelatedTable(emailEntry.getKey(), mailboxInfoDTO.getAccount(), emailZipFileMap);
  133. log.info("结束邮件解析 -> 邮箱信息:{},开始时间:{},结束时间:{}", mailboxInfoDTO,
  134. DateUtil.format(startDate, DateConst.YYYY_MM_DD_HH_MM_SS), DateUtil.format(endDate, DateConst.YYYY_MM_DD_HH_MM_SS));
  135. }
  136. }
  137. public List<EmailZipFileDTO> parseZipEmail(EmailContentInfoDTO emailContentInfoDTO) throws ArchiveException, IOException {
  138. List<EmailZipFileDTO> resultList = ListUtil.list(false);
  139. Integer emailType = emailContentInfoDTO.getEmailType();
  140. String filepath = emailContentInfoDTO.getFilePath();
  141. if (ExcelUtil.isZip(filepath)) {
  142. handleCompressedFiles(filepath, ".zip", emailType, resultList);
  143. } else if (ExcelUtil.isRAR(filepath)) {
  144. handleCompressedFiles(filepath, ".rar", emailType, resultList);
  145. }
  146. return resultList;
  147. }
  148. private void handleCompressedFiles(String filepath, String extension, Integer emailType, List<EmailZipFileDTO> resultList) throws IOException, ArchiveException {
  149. String destPath = getDestinationPath(filepath, extension);
  150. log.info("压缩包地址:{}, 解压后文件地址:{}", filepath, destPath);
  151. File destFile = new File(destPath);
  152. if (!destFile.exists()) {
  153. if (!destFile.mkdirs()) {
  154. throw new IOException("无法创建目标目录: " + destPath);
  155. }
  156. }
  157. List<String> extractedDirs;
  158. if (ExcelUtil.isZip(filepath)) {
  159. extractedDirs = ExcelUtil.extractCompressedFiles(filepath, destPath);
  160. } else if (ExcelUtil.isRAR(filepath)) {
  161. extractedDirs = ExcelUtil.extractRar(filepath, destPath);
  162. } else {
  163. return;
  164. }
  165. for (String dir : extractedDirs) {
  166. File file = new File(dir);
  167. if (file.isDirectory()) {
  168. String[] subDirs = file.list();
  169. if (subDirs != null) {
  170. for (String subDir : subDirs) {
  171. resultList.add(new EmailZipFileDTO(subDir, emailType));
  172. }
  173. } else {
  174. log.warn("目录 {} 下无文件", dir);
  175. }
  176. } else {
  177. resultList.add(new EmailZipFileDTO(dir, emailType));
  178. }
  179. }
  180. }
  181. private String getDestinationPath(String filepath, String extension) {
  182. String fileName = Paths.get(filepath).getFileName().toString();
  183. String baseName = fileName.substring(0, fileName.length() - extension.length());
  184. return Paths.get(filepath).getParent().resolve(baseName).toString();
  185. }
  186. public void saveRelatedTable(String emailKey, String emailAddress,
  187. Map<EmailContentInfoDTO, List<EmailZipFileDTO>> emailZipFileMap) {
  188. // python 报告解析接口结果
  189. List<ParseResult<ReportData>> dataList = ListUtil.list(false);
  190. for (Map.Entry<EmailContentInfoDTO, List<EmailZipFileDTO>> entry : emailZipFileMap.entrySet()) {
  191. EmailContentInfoDTO emailContentInfoDTO = entry.getKey();
  192. Integer emailType = emailContentInfoDTO.getEmailType();
  193. Integer emailId = emailContentInfoDTO.getEmailId();
  194. EmailParseInfoDO emailParseInfoDO = buildEmailParseInfo(emailId, emailAddress, emailContentInfoDTO);
  195. emailParseInfoDO.setEmailKey(emailKey);
  196. emailId = saveEmailParseInfo(emailParseInfoDO);
  197. if (emailId == null) {
  198. continue;
  199. }
  200. List<EmailZipFileDTO> zipFiles = entry.getValue();
  201. if (CollUtil.isNotEmpty(zipFiles)) {
  202. for (EmailZipFileDTO zipFile : zipFiles) {
  203. EmailFileInfoDO emailFile = saveEmailFileInfo(emailId, null, zipFile.getFilename(), zipFile.getFilepath(), null);
  204. // 解析结果(可以从python获取或者自行解析)并保存报告
  205. ParseResult<ReportData> parseResult = this.parseReportAndHandleResult(emailFile.getId(), zipFile.getFilename(),
  206. zipFile.getFilepath(), emailType, emailFile.getAiFileId());
  207. dataList.add(parseResult);
  208. }
  209. } else {
  210. String fileName = emailContentInfoDTO.getFileName();
  211. EmailFileInfoDO emailFile = saveEmailFileInfo(emailId, emailContentInfoDTO.getFileId(), fileName,
  212. emailContentInfoDTO.getFilePath(), emailContentInfoDTO.getAiFileId());
  213. // 解析结果(可以从python获取或者自行解析)并保存报告
  214. ParseResult<ReportData> parseResult = this.parseReportAndHandleResult(emailFile.getId(), fileName,
  215. emailContentInfoDTO.getFilePath(), emailType, emailFile.getAiFileId());
  216. dataList.add(parseResult);
  217. }
  218. String failReason = null;
  219. int emailParseStatus = EmailParseStatusConst.SUCCESS;
  220. // 报告邮件有一条失败就表示整个邮件解析失败
  221. if (CollUtil.isNotEmpty(dataList)) {
  222. // ai解析结果
  223. List<ReportData> aiParaseList = dataList.stream().map(ParseResult::getData)
  224. .filter(Objects::nonNull).filter(e -> Objects.equals(true, e.getAiParse())).toList();
  225. if (CollUtil.isNotEmpty(aiParaseList)) {
  226. for (ReportData data : aiParaseList) {
  227. this.emailFileInfoMapper.updateAiParseByFileId(data.getBaseInfo().getFileId(), data.getAiParse(), data.getAiFileId());
  228. }
  229. }
  230. long failNum = dataList.stream().filter(e -> !Objects.equals(EmailParseStatusConst.SUCCESS, e.getStatus())).count();
  231. if (failNum > 0) {
  232. emailParseStatus = EmailParseStatusConst.FAIL;
  233. failReason = dataList.stream().map(ParseResult::getMsg).collect(Collectors.joining(";"));
  234. }
  235. }
  236. emailParseInfoMapper.updateParseStatus(emailId, emailParseStatus, failReason);
  237. }
  238. }
  239. private ParseResult<ReportData> parseReportAndHandleResult(int fileId, String fileName,
  240. String filepath, Integer emailType, String aiFileId) {
  241. ParseResult<ReportData> result = new ParseResult<>();
  242. if ((!Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType)
  243. && !Objects.equals(EmailTypeConst.REPORT_LETTER_EMAIL_TYPE, emailType))
  244. || StrUtil.isBlank(fileName)) {
  245. result.setStatus(ReportParseStatus.NOT_A_REPORT.getCode());
  246. result.setMsg(ReportParseStatus.NOT_A_REPORT.getMsg());
  247. return result;
  248. }
  249. Pattern pattern = Pattern.compile("[A-Z0-9]{6}");
  250. Matcher matcher = pattern.matcher(fileName);
  251. String registerNumber = null;
  252. if (matcher.find()) {
  253. registerNumber = matcher.group();
  254. }
  255. // 类型识别---先识别季度报告,没有季度再识别年度报告,最后识别月报
  256. ReportType reportType = ReportParseUtils.matchReportType(fileName);
  257. // 解析器--如果开启python解析则直接调用python接口,否则根据文件后缀获取对应解析器
  258. ReportParserFileType fileType;
  259. String fileSuffix = StrUtil.subAfter(fileName, ".", true);
  260. fileType = ReportParserFileType.getBySuffix(fileSuffix);
  261. // 不支持的格式
  262. if (fileType == null) {
  263. result.setStatus(ReportParseStatus.NO_SUPPORT_TEMPLATE.getCode());
  264. result.setMsg(StrUtil.format(ReportParseStatus.NO_SUPPORT_TEMPLATE.getMsg(), fileName));
  265. return result;
  266. }
  267. // 不是定期报告的判断逻辑放在不支持的格式下面
  268. if (reportType == null) {
  269. result.setStatus(ReportParseStatus.NOT_A_REPORT.getCode());
  270. result.setMsg(StrUtil.format(ReportParseStatus.NOT_A_REPORT.getMsg(), fileName));
  271. return result;
  272. }
  273. // 解析报告
  274. ReportData reportData = null;
  275. StopWatch parserWatch = new StopWatch();
  276. parserWatch.start();
  277. try {
  278. ReportParserParams params = ReportParserParams.builder().fileId(fileId).filename(fileName)
  279. .filepath(filepath).registerNumber(registerNumber).reportType(reportType).aiFileId(aiFileId).build();
  280. ReportParser<ReportData> instance = this.reportParserFactory.getInstance(reportType, fileType);
  281. reportData = instance.parse(params);
  282. result.setStatus(1);
  283. result.setMsg("报告解析成功");
  284. result.setData(reportData);
  285. } catch (ReportParseException e) {
  286. log.error("解析失败\n{}", StrUtil.format(e.getMsg(), filepath));
  287. result.setStatus(e.getCode());
  288. result.setMsg(StrUtil.format(e.getMsg(), filepath));
  289. } catch (Exception e) {
  290. log.error("解析错误\n{}", ExceptionUtil.stacktraceToString(e));
  291. result.setStatus(ReportParseStatus.PARSE_FAIL.getCode());
  292. result.setMsg(StrUtil.format(ReportParseStatus.PARSE_FAIL.getMsg(), e.getMessage()));
  293. } finally {
  294. parserWatch.stop();
  295. if (log.isInfoEnabled()) {
  296. log.info("报告{}解析结果为{},耗时{}ms", fileName, reportData, parserWatch.getTotalTimeMillis());
  297. }
  298. }
  299. // 保存报告解析结果
  300. if (reportData != null) {
  301. StopWatch writeWatch = new StopWatch();
  302. writeWatch.start();
  303. try {
  304. ReportWriter<ReportData> instance = this.reportWriterFactory.getInstance(reportType);
  305. instance.write(reportData);
  306. } catch (Exception e) {
  307. log.error("报告{}结果保存失败\n{}", fileName, ExceptionUtil.stacktraceToString(e));
  308. } finally {
  309. writeWatch.stop();
  310. if (log.isInfoEnabled()) {
  311. log.info("报告{}解析结果保存完成,耗时{}ms", fileName, writeWatch.getTotalTimeMillis());
  312. }
  313. }
  314. }
  315. return result;
  316. }
  317. private EmailFileInfoDO saveEmailFileInfo(Integer emailId, Integer fileId, String fileName, String filePath, String aiFileId) {
  318. EmailFileInfoDO emailFileInfoDO = buildEmailFileInfoDO(emailId, fileId, fileName, filePath);
  319. emailFileInfoDO.setAiFileId(aiFileId);
  320. if (emailFileInfoDO.getId() != null) {
  321. emailFileInfoMapper.updateTimeById(fileId, new Date());
  322. return emailFileInfoDO;
  323. }
  324. emailFileInfoMapper.insert(emailFileInfoDO);
  325. return emailFileInfoDO;
  326. }
  327. private EmailFileInfoDO buildEmailFileInfoDO(Integer emailId, Integer fileId, String fileName, String filePath) {
  328. EmailFileInfoDO emailFileInfoDO = new EmailFileInfoDO();
  329. emailFileInfoDO.setId(fileId);
  330. emailFileInfoDO.setEmailId(emailId);
  331. emailFileInfoDO.setFileName(fileName);
  332. emailFileInfoDO.setFilePath(filePath);
  333. emailFileInfoDO.setIsvalid(1);
  334. emailFileInfoDO.setCreatorId(0);
  335. emailFileInfoDO.setCreateTime(new Date());
  336. emailFileInfoDO.setUpdaterId(0);
  337. emailFileInfoDO.setUpdateTime(new Date());
  338. return emailFileInfoDO;
  339. }
  340. private Integer saveEmailParseInfo(EmailParseInfoDO emailParseInfoDO) {
  341. if (emailParseInfoDO == null) {
  342. return null;
  343. }
  344. // 重新邮件功能 -> 修改解析时间和更新时间
  345. if (emailParseInfoDO.getId() != null) {
  346. emailParseInfoMapper.updateParseTime(emailParseInfoDO.getId(), emailParseInfoDO.getParseDate());
  347. return emailParseInfoDO.getId();
  348. }
  349. // 根据邮件发送人、邮件地址、邮箱日期、主题找到是否已经存在的记录(不管是否成功),已存在就不解析了
  350. EmailParseInfoDO temp = this.emailParseInfoMapper.searchEmail(emailParseInfoDO);
  351. if (temp != null) {
  352. return null;
  353. }
  354. emailParseInfoMapper.insert(emailParseInfoDO);
  355. return emailParseInfoDO.getId();
  356. }
  357. private EmailParseInfoDO buildEmailParseInfo(Integer emailId, String emailAddress, EmailContentInfoDTO emailContentInfoDTO) {
  358. EmailParseInfoDO emailParseInfoDO = new EmailParseInfoDO();
  359. emailParseInfoDO.setId(emailId);
  360. emailParseInfoDO.setSenderEmail(emailContentInfoDTO.getSenderEmail());
  361. emailParseInfoDO.setEmail(emailAddress);
  362. emailParseInfoDO.setEmailDate(DateUtil.parse(emailContentInfoDTO.getEmailDate(), DateConst.YYYY_MM_DD_HH_MM_SS));
  363. emailParseInfoDO.setParseDate(emailContentInfoDTO.getParseDate() == null ? null : DateUtil.parseDate(emailContentInfoDTO.getParseDate()));
  364. emailParseInfoDO.setEmailTitle(emailContentInfoDTO.getEmailTitle());
  365. emailParseInfoDO.setEmailType(emailContentInfoDTO.getEmailType());
  366. emailParseInfoDO.setParseStatus(EmailParseStatusConst.SUCCESS);
  367. emailParseInfoDO.setIsvalid(1);
  368. emailParseInfoDO.setCreatorId(0);
  369. emailParseInfoDO.setCreateTime(new Date());
  370. emailParseInfoDO.setUpdaterId(0);
  371. emailParseInfoDO.setUpdateTime(new Date());
  372. return emailParseInfoDO;
  373. }
  374. public Map<Integer, List<String>> getEmailType() {
  375. Map<Integer, List<String>> emailTypeMap = MapUtil.newHashMap(3, true);
  376. // EmailTypeRuleDO emailTypeRuleDO = emailTypeRuleMapper.getEmailTypeRule();
  377. // String nav = emailTypeRuleDO != null && StrUtil.isNotBlank(emailTypeRuleDO.getNav()) ? emailTypeRuleDO.getNav() : emailRuleConfig.getNav();
  378. // String valuation = emailTypeRuleDO != null && StrUtil.isNotBlank(emailTypeRuleDO.getValuation()) ? emailTypeRuleDO.getValuation() : emailRuleConfig.getValuation();
  379. // String report = emailTypeRuleDO != null && StrUtil.isNotBlank(emailTypeRuleDO.getReport()) ? emailTypeRuleDO.getReport() : emailRuleConfig.getReport();
  380. // emailTypeMap.put(EmailTypeConst.VALUATION_EMAIL_TYPE, Arrays.stream(valuation.split(",")).toList());
  381. // emailTypeMap.put(EmailTypeConst.NAV_EMAIL_TYPE, Arrays.stream(nav.split(",")).toList());
  382. emailTypeMap.put(EmailTypeConst.REPORT_EMAIL_TYPE, ListUtil.toList("月报", "周报", "月度报告"));
  383. emailTypeMap.put(EmailTypeConst.REPORT_LETTER_EMAIL_TYPE, ListUtil.toList("确认单", "确认函", "确认"));
  384. return emailTypeMap;
  385. }
  386. /**
  387. * 读取邮件
  388. *
  389. * @param mailboxInfoDTO 邮箱配置信息
  390. * @param emailTypeMap 邮件类型识别规则映射表
  391. * @param startDate 邮件起始日期
  392. * @param endDate 邮件截止日期(为null,将解析邮件日期小于等于startDate的当天邮件)
  393. * @return 读取到的邮件信息
  394. * @throws Exception 异常信息
  395. */
  396. private Map<String, List<EmailContentInfoDTO>> realEmail(MailboxInfoDTO mailboxInfoDTO,
  397. Map<Integer, List<String>> emailTypeMap, Date startDate, Date endDate) throws Exception {
  398. Store store = EmailUtil.getStoreNew(mailboxInfoDTO);
  399. if (store == null) {
  400. return MapUtil.newHashMap();
  401. }
  402. // 默认读取收件箱的邮件
  403. Folder folder = store.getFolder("INBOX");
  404. folder.open(Folder.READ_ONLY);
  405. Message[] messages = getEmailMessage(folder, mailboxInfoDTO.getProtocol(), startDate);
  406. if (messages == null || messages.length == 0) {
  407. log.info("获取不到邮件 -> 邮箱信息:{},开始时间:{},结束时间:{}", mailboxInfoDTO, startDate, endDate);
  408. return MapUtil.newHashMap();
  409. }
  410. Map<String, List<EmailContentInfoDTO>> emailMessageMap = MapUtil.newHashMap();
  411. for (Message message1 : messages) {
  412. MimeMessage message = (MimeMessage) message1;
  413. List<EmailContentInfoDTO> emailContentInfoDTOList = CollUtil.newArrayList();
  414. String uuidKey = UUID.randomUUID().toString().replaceAll("-", "");
  415. Integer emailType;
  416. String senderEmail;
  417. try {
  418. Date emailDate = message.getSentDate();
  419. boolean isNotParseConditionSatisfied = emailDate == null || (endDate != null && emailDate.compareTo(endDate) > 0) || (startDate != null && emailDate.compareTo(startDate) < 0);
  420. if (isNotParseConditionSatisfied) {
  421. continue;
  422. }
  423. senderEmail = getSenderEmail(message);
  424. emailType = EmailUtil.getEmailTypeBySubject(message.getSubject(), emailTypeMap);
  425. String emailDateStr = DateUtil.format(emailDate, DateConst.YYYY_MM_DD_HH_MM_SS);
  426. if (emailType == null) {
  427. log.info("邮件不满足解析条件 -> 邮件主题:{},邮件日期:{}", message.getSubject(), emailDateStr);
  428. continue;
  429. }
  430. log.info("邮件采集成功 -> 邮件主题:{},邮件日期:{}", message.getSubject(), emailDateStr);
  431. Object content = message.getContent();
  432. // 1.邮件为MIME多部分消息体:可能既有邮件又有正文
  433. if (content instanceof MimeMultipart) {
  434. emailContentInfoDTOList = EmailUtil.collectMimeMultipart(message, mailboxInfoDTO.getAccount(), path);
  435. }
  436. // // 2.邮件只有正文
  437. // if (content instanceof String) {
  438. // EmailContentInfoDTO emailContentInfoDTO = new EmailContentInfoDTO();
  439. // emailContentInfoDTO.setEmailContent(content.toString());
  440. // emailContentInfoDTO.setEmailDate(emailDateStr);
  441. // emailContentInfoDTO.setEmailTitle(message.getSubject());
  442. // String fileName = message.getSubject() + DateUtil.format(emailDate, DateConst.YYYYMMDDHHMMSS24);
  443. // String filePath = path + mailboxInfoDTO.getAccount() + "/" + DateUtil.format(emailDate, DateConst.YYYY_MM_DD) + "/" + fileName + ".html";
  444. // File saveFile = new File(filePath);
  445. // saveFile.setReadable(true);
  446. // if (!saveFile.exists()) {
  447. // if (!saveFile.getParentFile().exists()) {
  448. // saveFile.getParentFile().mkdirs();
  449. // saveFile.getParentFile().setExecutable(true);
  450. // }
  451. // }
  452. // FileUtil.writeFile(filePath, content.toString());
  453. // emailContentInfoDTO.setFilePath(filePath);
  454. // emailContentInfoDTO.setFileName(fileName);
  455. // emailContentInfoDTOList.add(emailContentInfoDTO);
  456. // }
  457. if (CollUtil.isNotEmpty(emailContentInfoDTOList)) {
  458. // 估值表或定期报告邮件不展示正文html文件
  459. if (emailType.equals(EmailTypeConst.VALUATION_EMAIL_TYPE) || emailType.equals(EmailTypeConst.REPORT_EMAIL_TYPE)) {
  460. emailContentInfoDTOList = emailContentInfoDTOList.stream().filter(e -> !ExcelUtil.isHTML(e.getFilePath())).toList();
  461. }
  462. emailContentInfoDTOList.forEach(e -> {
  463. e.setEmailType(emailType);
  464. e.setSenderEmail(senderEmail);
  465. });
  466. emailMessageMap.put(uuidKey, emailContentInfoDTOList);
  467. }
  468. } catch (Exception e) {
  469. log.error("获取邮箱的邮件报错,堆栈信息:{}", ExceptionUtil.stacktraceToString(e));
  470. }
  471. }
  472. folder.close(false);
  473. store.close();
  474. return emailMessageMap;
  475. }
  476. private String getSenderEmail(MimeMessage message) {
  477. Address[] senderAddress = null;
  478. try {
  479. senderAddress = message.getFrom();
  480. if (senderAddress == null || senderAddress.length == 0) {
  481. log.info("发件人获取失败=============================");
  482. return null;
  483. }
  484. // 此时的address是含有编码(MIME编码方式)后的文本和实际的邮件地址
  485. String address = "";
  486. for (Address from : senderAddress) {
  487. if (StrUtil.isNotBlank(from.toString())) {
  488. address = from.toString();
  489. break;
  490. }
  491. }
  492. log.info("发件人地址:" + address + "========================senderAddress size:" + senderAddress.length);
  493. // 正则表达式匹配邮件地址
  494. Pattern pattern = Pattern.compile("<(\\S+)>");
  495. Matcher matcher = pattern.matcher(address);
  496. if (matcher.find()) {
  497. return matcher.group(1);
  498. }
  499. //说明匹配不到,直接获取sender
  500. Address sender = message.getSender();
  501. if (sender == null) {
  502. return address;
  503. }
  504. String senderEmail = sender.toString();
  505. log.info("senderEmail:" + senderEmail + "====================");
  506. if (senderEmail.contains("<") && senderEmail.contains(">") && senderEmail.indexOf("<") < senderEmail.indexOf(">")) {
  507. senderEmail = senderEmail.substring(senderEmail.indexOf("<") + 1, senderEmail.length() - 1);
  508. }
  509. return senderEmail;
  510. } catch (MessagingException e) {
  511. log.error(e.getMessage(), e);
  512. }
  513. return null;
  514. }
  515. private Message[] getEmailMessage(Folder folder, String protocol, Date startDate) {
  516. try {
  517. if (protocol.contains("imap")) {
  518. // 获取邮件日期大于等于startDate的邮件(搜索条件只支持按天)
  519. SearchTerm startDateTerm = new ReceivedDateTerm(ComparisonTerm.GE, startDate);
  520. return folder.search(startDateTerm);
  521. } else {
  522. return folder.getMessages();
  523. }
  524. } catch (MessagingException e) {
  525. throw new RuntimeException(e);
  526. }
  527. }
  528. }