EmailParseService.java 52 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090
  1. package com.smppw.modaq.domain.service;
  2. import cn.hutool.core.collection.CollUtil;
  3. import cn.hutool.core.collection.ListUtil;
  4. import cn.hutool.core.date.DateUtil;
  5. import cn.hutool.core.exceptions.ExceptionUtil;
  6. import cn.hutool.core.io.FileUtil;
  7. import cn.hutool.core.map.MapUtil;
  8. import cn.hutool.core.util.IdUtil;
  9. import cn.hutool.core.util.StrUtil;
  10. import com.smppw.modaq.application.components.OCRReportParser;
  11. import com.smppw.modaq.application.components.ReportParseUtils;
  12. import com.smppw.modaq.application.components.report.parser.ReportParser;
  13. import com.smppw.modaq.application.components.report.parser.ReportParserFactory;
  14. import com.smppw.modaq.application.components.report.writer.ReportWriter;
  15. import com.smppw.modaq.application.components.report.writer.ReportWriterFactory;
  16. import com.smppw.modaq.application.util.EmailUtil;
  17. import com.smppw.modaq.common.conts.Constants;
  18. import com.smppw.modaq.common.conts.DateConst;
  19. import com.smppw.modaq.common.conts.EmailParseStatusConst;
  20. import com.smppw.modaq.common.conts.EmailTypeConst;
  21. import com.smppw.modaq.common.enums.ReportMonthlyType;
  22. import com.smppw.modaq.common.enums.ReportParseStatus;
  23. import com.smppw.modaq.common.enums.ReportParserFileType;
  24. import com.smppw.modaq.common.enums.ReportType;
  25. import com.smppw.modaq.common.exception.NotSupportReportException;
  26. import com.smppw.modaq.common.exception.ReportParseException;
  27. import com.smppw.modaq.domain.dto.*;
  28. import com.smppw.modaq.domain.dto.report.OCRParseData;
  29. import com.smppw.modaq.domain.dto.report.ParseResult;
  30. import com.smppw.modaq.domain.dto.report.ReportData;
  31. import com.smppw.modaq.domain.dto.report.ReportParserParams;
  32. import com.smppw.modaq.domain.entity.EmailFileInfoDO;
  33. import com.smppw.modaq.domain.entity.EmailParseInfoDO;
  34. import com.smppw.modaq.domain.mapper.EmailFileInfoMapper;
  35. import com.smppw.modaq.domain.mapper.EmailParseInfoMapper;
  36. import com.smppw.modaq.infrastructure.util.ArchiveUtil;
  37. import com.smppw.modaq.infrastructure.util.PdfUtil;
  38. import jakarta.mail.*;
  39. import jakarta.mail.internet.MimeUtility;
  40. import jakarta.mail.search.ComparisonTerm;
  41. import jakarta.mail.search.ReceivedDateTerm;
  42. import jakarta.mail.search.SearchTerm;
  43. import org.slf4j.Logger;
  44. import org.slf4j.LoggerFactory;
  45. import org.springframework.beans.factory.annotation.Value;
  46. import org.springframework.stereotype.Service;
  47. import org.springframework.util.StopWatch;
  48. import java.io.File;
  49. import java.io.IOException;
  50. import java.io.InputStream;
  51. import java.nio.file.Files;
  52. import java.util.*;
  53. import java.util.regex.Matcher;
  54. import java.util.regex.Pattern;
  55. import java.util.stream.Collectors;
  56. /**
  57. * @author mozuwen
  58. * @date 2024-09-04
  59. * @description 邮件解析服务
  60. */
  61. @Service
  62. public class EmailParseService {
  63. // public static final int stepSize = 10000;
  64. private static final Logger log = LoggerFactory.getLogger(EmailParseService.class);
  65. // 常量定义:统一管理关键词
  66. private static final Set<String> AMAC_KEYWORDS = Set.of("协会", "信披");
  67. private static final Set<String> EXCLUDE_PATH_KEYWORDS = Set.of("公司及协会版", "公司和协会版");
  68. // 扩展支持的 MIME 类型
  69. private static final Set<String> attachmentMimePrefixes = Set.of(
  70. "application/pdf",
  71. "application/zip",
  72. "application/x-zip-compressed",
  73. "application/rar",
  74. "application/x-rar-compressed",
  75. "application/octet-stream"
  76. // 按需添加其他类型...
  77. );
  78. private final EmailParseInfoMapper emailParseInfoMapper;
  79. private final EmailFileInfoMapper emailFileInfoMapper;
  80. /* 报告解析和入库的方法 */
  81. private final ReportParserFactory reportParserFactory;
  82. private final ReportWriterFactory reportWriterFactory;
  83. @Value("${email.file.path}")
  84. private String path;
  85. @Value("${email.report.ocr-parser-url}")
  86. private String ocrParserUrl;
  87. @Value("${email.read-write-seen:true}")
  88. private boolean readWriteSeen;
  89. public EmailParseService(EmailParseInfoMapper emailParseInfoMapper,
  90. EmailFileInfoMapper emailFileInfoMapper,
  91. ReportParserFactory reportParserFactory,
  92. ReportWriterFactory reportWriterFactory) {
  93. this.emailParseInfoMapper = emailParseInfoMapper;
  94. this.emailFileInfoMapper = emailFileInfoMapper;
  95. this.reportParserFactory = reportParserFactory;
  96. this.reportWriterFactory = reportWriterFactory;
  97. }
  98. /**
  99. * 解析指定邮箱指定时间范围内的邮件
  100. *
  101. * @param mailboxInfoDTO 邮箱配置信息
  102. * @param startDate 邮件起始日期(yyyy-MM-dd HH:mm:ss)
  103. * @param endDate 邮件截止日期(yyyy-MM-dd HH:mm:ss, 为null,将解析邮件日期小于等于startDate的当天邮件)
  104. * @param emailTypes 当前任务支持的邮件类型,默认支持确认单
  105. */
  106. public void parseEmail(MailboxInfoDTO mailboxInfoDTO,
  107. Date startDate, Date endDate,
  108. List<String> folderNames, List<Integer> emailTypes) {
  109. if (CollUtil.isEmpty(emailTypes)) {
  110. emailTypes = ListUtil.of(EmailTypeConst.REPORT_LETTER_EMAIL_TYPE);
  111. }
  112. if (log.isInfoEnabled()) {
  113. log.info("开始邮件解析 -> 邮箱信息:{},开始时间:{},结束时间:{}", mailboxInfoDTO, DateUtil.format(startDate,
  114. DateConst.YYYY_MM_DD_HH_MM_SS), DateUtil.format(endDate, DateConst.YYYY_MM_DD_HH_MM_SS));
  115. }
  116. Map<String, List<EmailContentInfoDTO>> emailContentMap;
  117. try {
  118. emailContentMap = this.realEmail(mailboxInfoDTO, startDate, endDate, folderNames);
  119. } catch (Exception e) {
  120. log.error("采集邮件失败 -> 邮箱配置信息:{},堆栈信息:{}", mailboxInfoDTO, ExceptionUtil.stacktraceToString(e));
  121. return;
  122. }
  123. if (MapUtil.isEmpty(emailContentMap)) {
  124. log.warn("未采集到邮件 -> 邮箱配置信息:{},开始时间:{},结束时间:{}", mailboxInfoDTO,
  125. DateUtil.format(startDate, DateConst.YYYY_MM_DD_HH_MM_SS), DateUtil.format(endDate, DateConst.YYYY_MM_DD_HH_MM_SS));
  126. return;
  127. }
  128. for (Map.Entry<String, List<EmailContentInfoDTO>> emailEntry : emailContentMap.entrySet()) {
  129. List<EmailContentInfoDTO> emailContentInfoDTOList = emailEntry.getValue();
  130. if (CollUtil.isEmpty(emailContentInfoDTOList)) {
  131. log.warn("未采集到正文或附件");
  132. continue;
  133. }
  134. EmailContentInfoDTO dto = emailContentInfoDTOList.get(0);
  135. String emailTitle = dto.getEmailTitle();
  136. log.info("开始解析邮件数据 -> 邮件主题:{},邮件日期:{}", emailTitle, dto.getEmailDate());
  137. List<EmailZipFileDTO> emailFileList = ListUtil.list(false);
  138. EmailInfoDTO emailInfo = new EmailInfoDTO(dto, emailFileList);
  139. for (EmailContentInfoDTO emailDto : emailContentInfoDTOList) {
  140. // 正文不用解压附件
  141. if (emailDto.getFileName() != null && emailDto.getFileName().endsWith(Constants.FILE_HTML)) {
  142. continue;
  143. }
  144. try {
  145. emailFileList.addAll(this.parseZipEmail(emailDto));
  146. } catch (IOException e) {
  147. log.error("压缩包解压失败:{}", ExceptionUtil.stacktraceToString(e));
  148. EmailParseInfoDO fail = buildEmailParseInfo(mailboxInfoDTO.getAccount(),
  149. dto.getEmailType(), emailInfo, emailDto.getFileSize());
  150. fail.setFailReason("压缩包解压失败");
  151. fail.setParseStatus(EmailParseStatusConst.FAIL);
  152. fail.setEmailKey(emailEntry.getKey());
  153. this.emailParseInfoMapper.insert(fail);
  154. } catch (Exception e) {
  155. log.error("堆栈信息:{}", ExceptionUtil.stacktraceToString(e));
  156. }
  157. }
  158. // 重新判断类型
  159. for (EmailZipFileDTO emailFile : emailFileList) {
  160. if (EmailTypeConst.SUPPORT_NO_OTHER_TYPES.contains(emailFile.getEmailType())) {
  161. continue;
  162. }
  163. Integer type = EmailUtil.getEmailTypeBySubject(emailTitle + emailFile.getFilename());
  164. // 特殊月报
  165. if ((Objects.equals(EmailTypeConst.NAV_EMAIL_TYPE, type)
  166. || Objects.equals(EmailTypeConst.REPORT_OTHER_TYPE, type))
  167. && (ReportParseUtils.containsAny(emailTitle, ReportParseUtils.MANAGER_KEYWORDS)
  168. || emailTitle.contains("定期报告"))) {
  169. type = EmailTypeConst.REPORT_EMAIL_TYPE;
  170. }
  171. // 其他报告
  172. if (Objects.equals(EmailTypeConst.NAV_EMAIL_TYPE, type)) {
  173. type = EmailTypeConst.REPORT_OTHER_TYPE;
  174. }
  175. emailFile.setEmailType(type);
  176. }
  177. Iterator<EmailZipFileDTO> entryIterator = emailFileList.iterator();
  178. while (entryIterator.hasNext()) {
  179. EmailZipFileDTO entry = entryIterator.next();
  180. if (!emailTypes.contains(entry.getEmailType())) {
  181. log.warn("当前邮件{} 文件{} 的类型{} 不在支持的任务类型{} 中,不用执行解析逻辑。",
  182. entry.getEmailTitle(), entry.getFilepath(), entry.getEmailType(), emailTypes);
  183. entryIterator.remove();
  184. }
  185. }
  186. // 保存相关信息 -> 邮件信息表,邮件文件表,邮件净值表,邮件规模表,基金净值表
  187. saveRelatedTable(emailEntry.getKey(), mailboxInfoDTO.getAccount(), emailInfo);
  188. log.info("结束邮件解析 -> 邮箱信息:{},开始时间:{},结束时间:{}", emailEntry.getValue(),
  189. DateUtil.format(startDate, DateConst.YYYY_MM_DD_HH_MM_SS), DateUtil.format(endDate, DateConst.YYYY_MM_DD_HH_MM_SS));
  190. }
  191. }
  192. /**
  193. * 解压压缩包,如果不是压缩包需转换
  194. *
  195. * @param emailContentInfoDTO 邮件信息
  196. * @return 解压后的文件列表
  197. * @throws IOException /
  198. */
  199. public List<EmailZipFileDTO> parseZipEmail(EmailContentInfoDTO emailContentInfoDTO) throws IOException {
  200. List<EmailZipFileDTO> resultList = ListUtil.list(false);
  201. Integer emailType = emailContentInfoDTO.getEmailType();
  202. String filepath = emailContentInfoDTO.getFilePath();
  203. String emailTitle = emailContentInfoDTO.getEmailTitle();
  204. if (ArchiveUtil.isArchive(filepath)) {
  205. this.handleCompressedFiles(emailTitle, filepath, emailType, resultList);
  206. } else {
  207. // 不是压缩包时
  208. EmailZipFileDTO dto = new EmailZipFileDTO(emailTitle, emailContentInfoDTO);
  209. resultList.add(dto);
  210. }
  211. // 文件中的类型判断
  212. if (emailType == null || !EmailTypeConst.SUPPORT_NO_OTHER_TYPES.contains(emailType)) {
  213. emailType = EmailUtil.getEmailTypeBySubject(emailContentInfoDTO.getFileName());
  214. emailContentInfoDTO.setEmailType(emailType);
  215. }
  216. if (CollUtil.isNotEmpty(resultList)) {
  217. for (EmailZipFileDTO dto : resultList) {
  218. dto.setEmailType(emailType);
  219. }
  220. }
  221. return resultList;
  222. }
  223. /**
  224. * 解压压缩包并把压缩包里面的所有文件放在resultList中
  225. *
  226. * @param emailTitle 邮件主题
  227. * @param filepath 压缩包路径
  228. * @param emailType 邮件解析类型
  229. * @param resultList 解压结果列表
  230. * @throws IOException /
  231. */
  232. private void handleCompressedFiles(String emailTitle,
  233. String filepath,
  234. Integer emailType,
  235. List<EmailZipFileDTO> resultList) throws IOException {
  236. String parent = FileUtil.getParent(filepath, 2);
  237. String destPath = parent + File.separator + "archive" + File.separator + FileUtil.mainName(filepath);
  238. File destFile = new File(destPath);
  239. if (!destFile.exists()) {
  240. if (!destFile.mkdirs()) {
  241. throw new IOException("无法创建目标目录: " + destPath);
  242. }
  243. }
  244. List<String> extractedDirs;
  245. if (ArchiveUtil.isZip(filepath)) {
  246. extractedDirs = ArchiveUtil.extractCompressedFiles(filepath, destPath);
  247. } else if (ArchiveUtil.isRAR(filepath) || ArchiveUtil.is7z(filepath)) {
  248. // 7z和rar压缩包解压
  249. extractedDirs = ArchiveUtil.extractRar5(filepath, destPath);
  250. } else {
  251. return;
  252. }
  253. for (String dir : extractedDirs) {
  254. // 如果邮件类型不满足解析条件则重新根据文件名判断
  255. if (emailType == null || !EmailTypeConst.SUPPORT_EMAIL_TYPES.contains(emailType)) {
  256. emailType = EmailUtil.getEmailTypeBySubject(dir);
  257. }
  258. File file = new File(dir);
  259. if (file.isDirectory()) {
  260. String[] subDirs = file.list();
  261. if (subDirs != null) {
  262. for (String subDir : subDirs) {
  263. resultList.add(new EmailZipFileDTO(emailTitle, subDir, emailType));
  264. }
  265. } else {
  266. log.warn("目录 {} 下无文件", dir);
  267. }
  268. } else {
  269. resultList.add(new EmailZipFileDTO(emailTitle, dir, emailType));
  270. }
  271. }
  272. }
  273. /**
  274. * 邮件附件解析并保存结果数据
  275. *
  276. * @param emailKey 没封邮件的uuid
  277. * @param emailAddress 发送人地址
  278. * @param emailInfo 邮件信息
  279. */
  280. public void saveRelatedTable(String emailKey, String emailAddress, EmailInfoDTO emailInfo) {
  281. // 附件文件检查
  282. Long totalSize = this.checkEmailFileInfo(emailInfo);
  283. if (totalSize == null) {
  284. return;
  285. }
  286. // 解析并保存数据
  287. List<ParseResult<ReportData>> dataList = ListUtil.list(true);
  288. Integer emailId = this.parseResults(null, emailKey, emailAddress, totalSize, emailInfo, dataList);
  289. String failReason = null;
  290. int emailParseStatus = EmailParseStatusConst.SUCCESS;
  291. // 报告邮件有一条失败就表示整个邮件解析失败
  292. if (CollUtil.isNotEmpty(dataList)) {
  293. // ai解析结果
  294. List<ReportData> aiParaseList = dataList.stream().map(ParseResult::getData)
  295. .filter(Objects::nonNull).filter(e -> Objects.equals(true, e.getAiParse())).toList();
  296. if (CollUtil.isNotEmpty(aiParaseList)) {
  297. for (ReportData data : aiParaseList) {
  298. this.emailFileInfoMapper.updateAiParseByFileId(data.getBaseInfo().getFileId(),
  299. data.getAiParse(), data.getAiFileId());
  300. }
  301. }
  302. long failNum = dataList.stream().filter(e -> !Objects.equals(EmailParseStatusConst.SUCCESS, e.getStatus())).count();
  303. if (failNum > 0) {
  304. emailParseStatus = EmailParseStatusConst.FAIL;
  305. failReason = dataList.stream().map(ParseResult::getMsg).collect(Collectors.joining(";"));
  306. }
  307. }
  308. this.emailParseInfoMapper.updateParseStatus(emailId, emailParseStatus, failReason);
  309. }
  310. /**
  311. * 上传文件解析并返回解析状态
  312. *
  313. * @param params 上传文件路径
  314. * @return /
  315. */
  316. public List<UploadReportResult> uploadReportResults(UploadReportParams params) {
  317. List<ParseResult<ReportData>> dataList = ListUtil.list(false);
  318. List<UploadReportParams.ReportInfo> reportInfos = params.getReportInfos();
  319. List<EmailZipFileDTO> dtos = ListUtil.list(false);
  320. for (UploadReportParams.ReportInfo e : reportInfos) {
  321. String reportPath = e.getReportPath();
  322. if (ArchiveUtil.isArchive(reportPath)) {
  323. try {
  324. this.handleCompressedFiles(params.getTitle(), reportPath, e.getReportType(), dtos);
  325. } catch (Exception ex) {
  326. log.warn("报告{} 压缩包解压失败:{}", reportPath, ExceptionUtil.stacktraceToString(ex));
  327. ReportData reportData = new ReportData.DefaultReportData();
  328. reportData.setReportPath(reportPath);
  329. dataList.add(new ParseResult<>(ReportParseStatus.ARCHIVE_FAIL, reportData));
  330. }
  331. } else {
  332. dtos.add(new EmailZipFileDTO(params.getTitle(), reportPath, e.getReportType()));
  333. }
  334. }
  335. EmailInfoDTO emailInfo = new EmailInfoDTO(params.getTitle(), dtos);
  336. // 附件文件检查
  337. Long totalSize = this.checkEmailFileInfo(emailInfo);
  338. if (totalSize == null) {
  339. return null;
  340. }
  341. this.parseResults(-1, null, null, totalSize, emailInfo, dataList);
  342. List<UploadReportResult> resultList = ListUtil.list(false);
  343. for (ParseResult<ReportData> result : dataList) {
  344. ReportData data = result.getData();
  345. resultList.add(new UploadReportResult(data.getReportPath(), result.getStatus(), result.getMsg()));
  346. }
  347. return resultList;
  348. }
  349. /**
  350. * 邮件信息前置处理,在解析操作执行之前的过滤逻辑和校验逻辑。返回所有附件大小汇总
  351. *
  352. * @param emailInfo 邮件信息(包含所有解压后的文件)
  353. * @return 所有附件大小汇总,为null说明没有文件需要上传
  354. */
  355. private Long checkEmailFileInfo(EmailInfoDTO emailInfo) {
  356. String emailTitle = emailInfo.getEmailTitle();
  357. List<EmailZipFileDTO> dtos = emailInfo.getEmailFileList();
  358. // 如果压缩包里面既有pdf又有其他格式的文件,说明其他格式的文件是不需要解析的
  359. List<String> exts = dtos.stream().map(EmailZipFileDTO::getExtName).distinct().toList();
  360. if (exts.contains(Constants.FILE_PDF) && exts.size() > 1) {
  361. dtos.removeIf(e -> !Objects.equals(Constants.FILE_PDF, e.getExtName()));
  362. }
  363. // 移除逻辑
  364. Iterator<EmailZipFileDTO> removeIterator = dtos.iterator();
  365. while (removeIterator.hasNext()) {
  366. EmailZipFileDTO dto = removeIterator.next();
  367. String filename = dto.getFilename();
  368. // 删除复核函或基金合同
  369. if (filename.contains("复核函") || (filename.contains("基金合同") && !filename.contains("合同变更"))) {
  370. log.warn("邮件{} 中的报告{} 是复核函或基金合同,不用解析上传。", emailTitle, filename);
  371. removeIterator.remove();
  372. }
  373. // 不支持的类型
  374. Integer type = dto.getEmailType();
  375. if (!EmailTypeConst.SUPPORT_EMAIL_TYPES.contains(type)) {
  376. log.info("邮件{} 类型{} 不支持解析。", emailTitle, type);
  377. removeIterator.remove();
  378. }
  379. }
  380. // 数据库已存在的数据过滤(邮件主题+报告名称+附件大小,压缩包文件大小汇总)
  381. long totalSize = dtos.stream().map(EmailZipFileDTO::getFileSize).reduce(0L, Long::sum);
  382. Iterator<EmailZipFileDTO> iterator = dtos.iterator();
  383. while (iterator.hasNext()) {
  384. EmailZipFileDTO dto = iterator.next();
  385. String filename = dto.getFilename();
  386. Integer type = dto.getEmailType();
  387. int count = 0;
  388. if (Objects.equals(type, EmailTypeConst.REPORT_LETTER_EMAIL_TYPE)) {
  389. // 确认单
  390. count = this.emailFileInfoMapper.getLetterFilenameSuccessCount(emailTitle, filename);
  391. } else if (Objects.equals(type, EmailTypeConst.REPORT_EMAIL_TYPE)) {
  392. // 定期报告
  393. count = this.emailFileInfoMapper.getAmacFilenameSuccessCount(emailTitle, filename, totalSize);
  394. } else if (Objects.equals(type, EmailTypeConst.REPORT_WEEKLY_TYPE)) {
  395. // 管理人周报
  396. count = this.emailFileInfoMapper.getWeeklyFilenameSuccessCount(emailTitle, filename, totalSize);
  397. } else if (Objects.equals(type, EmailTypeConst.REPORT_OTHER_TYPE)) {
  398. // 其他报告
  399. count = this.emailFileInfoMapper.getOtherFilenameSuccessCount(emailTitle, filename, totalSize);
  400. }
  401. if (count > 0) {
  402. iterator.remove();
  403. log.info("邮件{} 报告{} 已存在解析成功的记录,不用重新解析。", emailTitle, filename);
  404. }
  405. }
  406. if (CollUtil.isEmpty(dtos)) {
  407. log.info("邮件{} 所有文件都已经解析成功过,不能重复解析了", emailTitle);
  408. return null;
  409. }
  410. if (log.isInfoEnabled()) {
  411. log.info("邮件{} 还有报告待解析:\n{}", emailTitle, dtos);
  412. }
  413. return totalSize;
  414. }
  415. /**
  416. * 邮件信息保存+附件解析
  417. *
  418. * @param emailId 邮件ID,上传解析时一定是-1
  419. * @param emailKey 邮件uuid(邮箱下载解析时)
  420. * @param emailAddress 接收人地址(邮箱下载解析时)
  421. * @param totalSize 所有附件大小汇总
  422. * @param emailInfo 邮件信息,包含附件
  423. * @param resultList 解析结果
  424. * @return 邮件数据ID
  425. */
  426. private Integer parseResults(Integer emailId,
  427. String emailKey,
  428. String emailAddress,
  429. long totalSize,
  430. EmailInfoDTO emailInfo,
  431. List<ParseResult<ReportData>> resultList) {
  432. String emailTitle = emailInfo.getEmailTitle();
  433. List<EmailZipFileDTO> dtos = emailInfo.getEmailFileList();
  434. if (emailId == null) {
  435. // 保存邮件信息
  436. Integer emailType = dtos.get(0).getEmailType();
  437. EmailParseInfoDO emailParseInfoDO = this.buildEmailParseInfo(emailAddress, emailType, emailInfo, totalSize);
  438. emailParseInfoDO.setEmailKey(emailKey);
  439. emailId = this.saveEmailParseInfo(emailParseInfoDO);
  440. }
  441. // 解析邮件报告
  442. for (EmailZipFileDTO zipFile : dtos) {
  443. EmailFileInfoDO emailFile = this.saveEmailFileInfo(emailId, zipFile.getFilename(), zipFile.getFilepath());
  444. // 解析并保存报告
  445. ParseResult<ReportData> parseResult = this.parseReportAndHandleResult(emailTitle, emailFile.getId(), zipFile);
  446. if (!Objects.equals(1, parseResult.getStatus())) {
  447. log.error(parseResult.getMsg());
  448. }
  449. if (parseResult.getData() == null) {
  450. parseResult.setData(new ReportData.DefaultReportData());
  451. }
  452. parseResult.getData().setReportPath(zipFile.getFilepath());
  453. resultList.add(parseResult);
  454. }
  455. return emailId;
  456. }
  457. /**
  458. * 解析报告并保存解析结果
  459. *
  460. * @param emailTitle 邮件主题
  461. * @param fileId 当前文件数据库ID
  462. * @param zipFile 当前报告的路径信息
  463. * @return /
  464. */
  465. private ParseResult<ReportData> parseReportAndHandleResult(String emailTitle,
  466. Integer fileId,
  467. EmailZipFileDTO zipFile) {
  468. Integer emailType = zipFile.getEmailType();
  469. String fileName = zipFile.getFilename();
  470. String filepath = zipFile.getFilepath();
  471. ParseResult<ReportData> result = new ParseResult<>();
  472. boolean reportFlag = emailType == null || !EmailTypeConst.SUPPORT_EMAIL_TYPES.contains(emailType);
  473. if (reportFlag || StrUtil.isBlank(fileName) || fileName.endsWith(Constants.FILE_HTML)) {
  474. return new ParseResult<>(ReportParseStatus.NOT_A_REPORT, null, fileName);
  475. }
  476. // 类型识别---先识别季度报告,没有季度再识别年度报告,最后识别月报
  477. ReportType reportType = ReportParseUtils.matchReportType(emailType, fileName);
  478. if (reportType == null) {
  479. reportType = ReportParseUtils.matchReportType(emailType, emailTitle);
  480. if (log.isDebugEnabled()) {
  481. log.debug("报告{} 根据邮件主题{} 重新识别的类型是:{}", fileName, emailTitle, reportType);
  482. }
  483. }
  484. // 解析器--根据文件后缀获取对应解析器,解析不了就用AI来解析
  485. ReportParserFileType fileType = ReportParserFileType.getBySuffix(zipFile.getExtName());
  486. // 不支持的格式
  487. if (fileType == null) {
  488. return new ParseResult<>(ReportParseStatus.NO_SUPPORT_TEMPLATE, null, fileName);
  489. }
  490. // 不是定期报告的判断逻辑放在不支持的格式下面
  491. if (reportType == null) {
  492. return new ParseResult<>(ReportParseStatus.NOT_A_REPORT, null, fileName);
  493. }
  494. // docx转pdf
  495. if (Objects.equals(ReportParserFileType.WORD, fileType)) {
  496. try {
  497. String outputFile = FileUtil.getParent(filepath, 1) + File.separator + FileUtil.mainName(fileName) + ".pdf";
  498. PdfUtil.convertDocxToPdf(filepath, outputFile);
  499. filepath = outputFile;
  500. } catch (Exception e) {
  501. log.warn("报告{} 转换为pdf失败:{}", fileName, ExceptionUtil.stacktraceToString(e));
  502. }
  503. }
  504. // 首页和尾页转为png图片,首页用来识别基金名称和基金代码、尾页用来识别印章和联系人
  505. List<String> images = ListUtil.list(true);
  506. if (Objects.equals(ReportParserFileType.PDF, fileType)) {
  507. try {
  508. String output = filepath.replaceAll("archive|original", "image");
  509. File outputFile = FileUtil.file(FileUtil.getParent(output, 1));
  510. images = PdfUtil.convertFirstAndLastPagesToPng(filepath, outputFile, 300);
  511. if (log.isDebugEnabled()) {
  512. log.debug("报告{} 生成的图片地址是:\n{}", fileName, images);
  513. }
  514. } catch (Exception e) {
  515. log.warn("报告{} 生成图片失败:{}", fileName, ExceptionUtil.stacktraceToString(e));
  516. }
  517. } else if (Objects.equals(ReportParserFileType.IMG, fileType)) {
  518. try {
  519. String outputFile = PdfUtil.compressAndSave(filepath);
  520. images.add(outputFile);
  521. } catch (IOException e) {
  522. log.error("报告{} 图片压缩失败,{}", fileName, ExceptionUtil.stacktraceToString(e));
  523. }
  524. }
  525. // ocr识别月报是否管理人版或协会版
  526. ReportMonthlyType monthlyType = ReportMonthlyType.NO_NEED;
  527. if (ReportType.MONTHLY == reportType) {
  528. monthlyType = this.determineReportType(emailTitle, fileName, filepath, images);
  529. }
  530. boolean isAmac = reportType == ReportType.ANNUALLY || reportType == ReportType.QUARTERLY
  531. || (reportType == ReportType.MONTHLY && ReportMonthlyType.AMAC == monthlyType);
  532. // 不支持解析的格式文件
  533. boolean notSupportFile = false;
  534. // 解析报告
  535. ReportData reportData = null;
  536. ReportParserParams params = new ReportParserParams(fileId, fileName, filepath, reportType);
  537. long start = System.currentTimeMillis();
  538. try {
  539. if (isAmac || reportType == ReportType.LETTER) {
  540. ReportParser<ReportData> instance = this.reportParserFactory.getInstance(reportType, fileType);
  541. reportData = instance.parse(params);
  542. result = new ParseResult<>(1, "报告解析成功", reportData);
  543. }
  544. } catch (ReportParseException e) {
  545. result = new ParseResult<>(e.getCode(), StrUtil.format(e.getMsg(), fileName), null);
  546. log.warn("解析失败:{}", result.getMsg());
  547. if (e instanceof NotSupportReportException) {
  548. notSupportFile = true;
  549. }
  550. } catch (Exception e) {
  551. log.warn("解析错误:{}", ExceptionUtil.stacktraceToString(e));
  552. result = new ParseResult<>(ReportParseStatus.PARSE_FAIL, null, e.getMessage());
  553. } finally {
  554. // 如果解析结果是空的就用AI工具解析一次
  555. if (reportData == null && !notSupportFile) {
  556. if (log.isInfoEnabled()) {
  557. log.info("报告{} 是周报或管理人月报或其他类型或解析失败,用AI解析器解析", fileName);
  558. }
  559. try {
  560. if (!isAmac && CollUtil.isNotEmpty(images)) {
  561. filepath = images.get(0);
  562. }
  563. params = new ReportParserParams(fileId, fileName, filepath, reportType);
  564. ReportParser<ReportData> instance = this.reportParserFactory.getInstance(reportType, ReportParserFileType.AI);
  565. reportData = instance.parse(params);
  566. result = new ParseResult<>(1, "报告解析成功--AI", reportData);
  567. } catch (ReportParseException e) {
  568. result = new ParseResult<>(e.getCode(), StrUtil.format(e.getMsg(), fileName), null);
  569. log.warn("AI解析失败:{}", result.getMsg());
  570. } catch (Exception e) {
  571. log.warn("AI解析错误:{}", ExceptionUtil.stacktraceToString(e));
  572. result = new ParseResult<>(ReportParseStatus.PARSE_FAIL, null, e.getMessage());
  573. }
  574. }
  575. if (log.isInfoEnabled()) {
  576. log.info("报告{} 用ocr补充解析结果。补充前的结果是:\n{}", fileName, reportData);
  577. }
  578. // ocr信息提取(印章、联系人、基金名称和产品代码)
  579. this.ocrReportData(reportType, reportData, fileName, images);
  580. // 设置月报类型
  581. if (reportData != null && reportData.getBaseInfo() != null) {
  582. reportData.getBaseInfo().setMonthlyType(monthlyType.getType());
  583. }
  584. if (log.isInfoEnabled()) {
  585. log.info("报告{} 解析耗时{}ms,结果是:\n{}", fileName, (System.currentTimeMillis() - start), reportData);
  586. }
  587. }
  588. // 保存报告解析结果
  589. this.saveReportData(reportData, reportType, fileName);
  590. return result;
  591. }
  592. /**
  593. * 判断月报类型(管理人版还是协会版)
  594. *
  595. * @param emailTitle 邮件主题
  596. * @param fileName 报告名称
  597. * @param filepath 报告路径
  598. * @param images 报告的第一页和尾页图片地址(主要用于ocr提取关键信息)
  599. */
  600. public ReportMonthlyType determineReportType(String emailTitle, String fileName,
  601. String filepath, List<String> images) {
  602. // 1. 优先根据文件名判断
  603. if (ReportParseUtils.containsAny(fileName, AMAC_KEYWORDS)) {
  604. return ReportMonthlyType.AMAC;
  605. }
  606. if (ReportParseUtils.containsAny(fileName, ReportParseUtils.MANAGER_KEYWORDS)) {
  607. return ReportMonthlyType.MANAGER;
  608. }
  609. // if (StrUtil.isNotBlank(ReportParseUtils.matchFundCode(fileName))) {
  610. // return ReportMonthlyType.AMAC;
  611. // }
  612. // 2. 根据文件路径判断
  613. List<String> pathSegments = StrUtil.split(filepath, File.separator);
  614. for (String segment : pathSegments) {
  615. boolean isExcluded = ReportParseUtils.containsAny(segment, EXCLUDE_PATH_KEYWORDS);
  616. if (!isExcluded && ReportParseUtils.containsAny(segment, AMAC_KEYWORDS)) {
  617. return ReportMonthlyType.AMAC;
  618. }
  619. if (!isExcluded && ReportParseUtils.containsAny(segment, ReportParseUtils.MANAGER_KEYWORDS)) {
  620. return ReportMonthlyType.MANAGER;
  621. }
  622. }
  623. // 3. 根据邮件主题判断
  624. boolean isAmacEmail = ReportParseUtils.containsAny(emailTitle, AMAC_KEYWORDS)
  625. && !emailTitle.contains("公司及协会版");
  626. if (isAmacEmail) {
  627. return ReportMonthlyType.AMAC;
  628. }
  629. if (ReportParseUtils.containsAny(emailTitle, ReportParseUtils.MANAGER_KEYWORDS)) {
  630. return ReportMonthlyType.MANAGER;
  631. }
  632. // 4.ocr 提取“曲线”、“基金份额”等关键字,如果有曲线则是管理人,如果有估值日期则是协会
  633. if (CollUtil.isNotEmpty(images)) {
  634. try {
  635. return new OCRReportParser().parseMonthlyType(fileName, this.ocrParserUrl, images.get(0));
  636. } catch (Exception ignored) {
  637. return ReportMonthlyType.FAILED;
  638. }
  639. }
  640. return ReportMonthlyType.FAILED;
  641. }
  642. /**
  643. * ocr 提取信息(包括首页的基金名称或报告日期,尾页的印章或联系人等信息)
  644. *
  645. * @param reportData 报告解析结果
  646. * @param fileName 报告名称
  647. * @param images 报告的收益和尾页png图片
  648. */
  649. private void ocrReportData(ReportType reportType,
  650. ReportData reportData,
  651. String fileName,
  652. List<String> images) {
  653. if (reportData == null || CollUtil.isEmpty(images)) {
  654. return;
  655. }
  656. OCRParseData parseRes = null;
  657. // 报告才识别尾页的印章和联系人,确认单不识别尾页
  658. if (ReportType.LETTER != reportType) {
  659. try {
  660. // 首页和尾页相等时只读首页
  661. String imageUrl = images.size() == 1 ? images.get(0) : images.get(1);
  662. parseRes = new OCRReportParser().parse(fileName, this.ocrParserUrl, imageUrl);
  663. } catch (Exception e) {
  664. log.error("报告{} OCR识别印章和联系人出错:{}", fileName, e.getMessage());
  665. }
  666. // ocr识别尾页是否包含印章和联系人信息
  667. if (parseRes != null) {
  668. if (reportData.getBaseInfo() != null) {
  669. reportData.getBaseInfo().setWithSeals(parseRes.getWithSeals());
  670. reportData.getBaseInfo().setWithContacts(parseRes.getWithContacts());
  671. if (fileName.contains("用印") && !Objects.equals(true, reportData.getBaseInfo().getWithSeals())) {
  672. reportData.getBaseInfo().setWithSeals(true);
  673. }
  674. }
  675. }
  676. }
  677. // 首页和尾页不相等时解析首页的数据
  678. if (images.size() != 1) {
  679. try {
  680. parseRes = new OCRReportParser().parse(fileName, this.ocrParserUrl, images.get(0));
  681. } catch (Exception e) {
  682. log.error("报告{} OCR识别首页基金名称和报告日期出错:{}", fileName, e.getMessage());
  683. }
  684. }
  685. // 用首页识别基金名称、产品代码和基金管理人
  686. if (reportData.getFundInfo() != null && parseRes != null) {
  687. if (StrUtil.isBlank(reportData.getFundInfo().getFundName())) {
  688. reportData.getFundInfo().setFundName(parseRes.getFundName());
  689. }
  690. if (StrUtil.isBlank(reportData.getFundInfo().getFundCode())) {
  691. reportData.getFundInfo().setFundCode(parseRes.getFundCode());
  692. }
  693. if (StrUtil.isBlank(reportData.getFundInfo().getCompanyName())
  694. || !reportData.getFundInfo().getCompanyName().contains("有限公司")) {
  695. reportData.getFundInfo().setCompanyName(parseRes.getCompanyName());
  696. }
  697. }
  698. }
  699. /**
  700. * 保存报告解析结果
  701. *
  702. * @param reportData 报告解析结果
  703. * @param reportType 报告类型
  704. * @param fileName 报告名称
  705. */
  706. private void saveReportData(ReportData reportData, ReportType reportType, String fileName) {
  707. if (reportData == null) {
  708. return;
  709. }
  710. StopWatch writeWatch = new StopWatch();
  711. writeWatch.start();
  712. try {
  713. ReportWriter<ReportData> instance = this.reportWriterFactory.getInstance(reportType);
  714. instance.write(reportData);
  715. } catch (Exception e) {
  716. log.error("报告{} 结果保存失败 {}", fileName, ExceptionUtil.stacktraceToString(e));
  717. } finally {
  718. writeWatch.stop();
  719. if (log.isInfoEnabled()) {
  720. log.info("报告{}解析结果保存完成,耗时{}ms", fileName, writeWatch.getTotalTimeMillis());
  721. }
  722. }
  723. }
  724. private EmailFileInfoDO saveEmailFileInfo(Integer emailId, String fileName, String filePath) {
  725. EmailFileInfoDO emailFileInfoDO = buildEmailFileInfoDO(emailId, fileName, filePath);
  726. emailFileInfoDO.setAiFileId(null);
  727. if (emailFileInfoDO.getId() != null) {
  728. emailFileInfoMapper.updateTimeById(null, new Date());
  729. return emailFileInfoDO;
  730. }
  731. emailFileInfoMapper.insert(emailFileInfoDO);
  732. return emailFileInfoDO;
  733. }
  734. private EmailFileInfoDO buildEmailFileInfoDO(Integer emailId, String fileName, String filePath) {
  735. EmailFileInfoDO emailFileInfoDO = new EmailFileInfoDO();
  736. emailFileInfoDO.setId(null);
  737. emailFileInfoDO.setEmailId(emailId);
  738. emailFileInfoDO.setFileName(fileName);
  739. emailFileInfoDO.setFilePath(filePath);
  740. emailFileInfoDO.setIsvalid(1);
  741. emailFileInfoDO.setCreatorId(0);
  742. emailFileInfoDO.setCreateTime(new Date());
  743. emailFileInfoDO.setUpdaterId(0);
  744. emailFileInfoDO.setUpdateTime(new Date());
  745. return emailFileInfoDO;
  746. }
  747. private Integer saveEmailParseInfo(EmailParseInfoDO emailParseInfoDO) {
  748. if (emailParseInfoDO == null) {
  749. return null;
  750. }
  751. // 重新邮件功能 -> 修改解析时间和更新时间
  752. if (emailParseInfoDO.getId() != null) {
  753. emailParseInfoMapper.updateParseTime(emailParseInfoDO.getId(), emailParseInfoDO.getParseDate());
  754. return emailParseInfoDO.getId();
  755. }
  756. emailParseInfoMapper.insert(emailParseInfoDO);
  757. return emailParseInfoDO.getId();
  758. }
  759. private EmailParseInfoDO buildEmailParseInfo(String emailAddress, Integer emailType,
  760. EmailInfoDTO emailInfo, long totalSize) {
  761. EmailParseInfoDO emailParseInfoDO = new EmailParseInfoDO();
  762. emailParseInfoDO.setId(null);
  763. emailParseInfoDO.setSenderEmail(emailInfo.getSenderEmail());
  764. emailParseInfoDO.setEmail(emailAddress);
  765. emailParseInfoDO.setEmailDate(DateUtil.parse(emailInfo.getEmailDate(), DateConst.YYYY_MM_DD_HH_MM_SS));
  766. emailParseInfoDO.setParseDate(new Date());
  767. emailParseInfoDO.setEmailTitle(emailInfo.getEmailTitle());
  768. emailParseInfoDO.setEmailType(emailType);
  769. emailParseInfoDO.setParseStatus(EmailParseStatusConst.SUCCESS);
  770. emailParseInfoDO.setAttrSize(totalSize);
  771. emailParseInfoDO.setIsvalid(1);
  772. emailParseInfoDO.setCreatorId(0);
  773. emailParseInfoDO.setCreateTime(new Date());
  774. emailParseInfoDO.setUpdaterId(0);
  775. emailParseInfoDO.setUpdateTime(new Date());
  776. return emailParseInfoDO;
  777. }
  778. /**
  779. * 读取邮件
  780. *
  781. * @param mailboxInfoDTO 邮箱配置信息
  782. * @param startDate 邮件起始日期
  783. * @param endDate 邮件截止日期(为null,将解析邮件日期小于等于startDate的当天邮件)
  784. * @return 读取到的邮件信息
  785. * @throws Exception 异常信息
  786. */
  787. private Map<String, List<EmailContentInfoDTO>> realEmail(MailboxInfoDTO mailboxInfoDTO,
  788. Date startDate, Date endDate,
  789. List<String> folderNames) throws Exception {
  790. if (CollUtil.isEmpty(folderNames)) {
  791. folderNames = ListUtil.toList("INBOX");
  792. }
  793. Store store = EmailUtil.getStoreNew(mailboxInfoDTO);
  794. if (store == null) {
  795. return MapUtil.newHashMap(4);
  796. }
  797. Map<String, List<EmailContentInfoDTO>> result = MapUtil.newHashMap(128);
  798. try {
  799. if (log.isDebugEnabled()) {
  800. Folder[] list = store.getDefaultFolder().list("*");
  801. List<String> names = Arrays.stream(list).map(Folder::getFullName).toList();
  802. log.debug("获取所有邮箱文件夹:{}", names);
  803. }
  804. for (String folderName : folderNames) {
  805. try {
  806. Map<String, List<EmailContentInfoDTO>> temp = this.getFolderEmail(mailboxInfoDTO,
  807. startDate, endDate, store, folderName);
  808. if (MapUtil.isNotEmpty(temp)) {
  809. result.putAll(temp);
  810. }
  811. } catch (Exception e) {
  812. log.warn("文件夹{} 邮件获取失败:{}", folderName, ExceptionUtil.stacktraceToString(e));
  813. }
  814. }
  815. } catch (Exception e) {
  816. log.error("邮件获取失败:{}", ExceptionUtil.stacktraceToString(e));
  817. } finally {
  818. store.close();
  819. }
  820. return result;
  821. }
  822. private Map<String, List<EmailContentInfoDTO>> getFolderEmail(MailboxInfoDTO mailboxInfoDTO,
  823. Date startDate, Date endDate,
  824. Store store, String folderName) throws MessagingException {
  825. // 默认读取收件箱的邮件
  826. Folder folder = store.getFolder(folderName);
  827. folder.open(this.readWriteSeen ? Folder.READ_WRITE : Folder.READ_ONLY);
  828. Message[] messages = getEmailMessage(folder, mailboxInfoDTO.getProtocol(), startDate);
  829. if (messages == null || messages.length == 0) {
  830. log.warn("{} 获取不到邮件 -> 邮箱信息:{},开始时间:{},结束时间:{}", folderName, mailboxInfoDTO, startDate, endDate);
  831. return MapUtil.newHashMap();
  832. }
  833. String emailAddress = mailboxInfoDTO.getAccount();
  834. Map<String, List<EmailContentInfoDTO>> emailMessageMap = MapUtil.newHashMap();
  835. for (Message message : messages) {
  836. long start = System.currentTimeMillis();
  837. List<EmailContentInfoDTO> dtos = CollUtil.newArrayList();
  838. String emailTitle = message.getSubject();
  839. if (this.readWriteSeen && isMessageRead(message)) {
  840. log.warn("{} 邮件{} 已读,不用重新下载解析!", folderName, emailTitle);
  841. continue;
  842. }
  843. try {
  844. Date emailDate = message.getSentDate();
  845. String emailDateStr = DateUtil.format(emailDate, DateConst.YYYY_MM_DD_HH_MM_SS);
  846. if (log.isInfoEnabled()) {
  847. log.info("{} 邮件{} 数据获取中,邮件时间:{}", folderName, emailTitle, emailDateStr);
  848. }
  849. boolean isNotParseConditionSatisfied = emailDate == null
  850. || (endDate != null && emailDate.compareTo(endDate) > 0)
  851. || (startDate != null && emailDate.compareTo(startDate) < 0);
  852. if (isNotParseConditionSatisfied) {
  853. String st = DateUtil.formatDateTime(startDate);
  854. String ed = DateUtil.formatDateTime(endDate);
  855. log.warn("{} 邮件{} 发送时间{}不在区间内【{} ~ {}】", folderName, emailTitle, emailDateStr, st, ed);
  856. continue;
  857. }
  858. String senderEmail = getSenderEmail(message);
  859. Integer emailType = EmailUtil.getEmailTypeBySubject(emailTitle);
  860. if (emailType == null) {
  861. log.warn("{} 邮件不满足解析条件 -> 邮件主题:{},邮件日期:{}", folderName, emailTitle, emailDateStr);
  862. continue;
  863. }
  864. // // 成功解析的邮件不用重复下载
  865. // Integer okNum = this.emailParseInfoMapper.countEmailByInfoAndStatus(emailTitle, senderEmail, emailAddress, emailDateStr);
  866. // if (okNum > 0) {
  867. // if (log.isInfoEnabled()) {
  868. // log.info("{} 邮件{} 已经存在解析完成的记录,不要重复下载了。", folderName, emailTitle);
  869. // }
  870. // continue;
  871. // }
  872. if (log.isInfoEnabled()) {
  873. log.info("{} 邮件{} 基本信息获取完成,开始下载附件!邮件日期:{}", folderName, emailTitle, emailDateStr);
  874. }
  875. Object content = message.getContent();
  876. if (content instanceof Multipart multipart) {
  877. this.reMultipart(emailAddress, emailTitle, emailDate, multipart, dtos);
  878. } else if (content instanceof Part part) {
  879. this.rePart(emailAddress, emailTitle, emailDate, part, dtos);
  880. } else {
  881. log.warn("{} 邮件{} 获取不了附件", folderName, emailTitle);
  882. }
  883. if (CollUtil.isEmpty(dtos)) {
  884. log.warn("{} 邮件{} 没有获取到附件", folderName, emailTitle);
  885. continue;
  886. }
  887. dtos.forEach(e -> {
  888. e.setEmailType(emailType);
  889. e.setSenderEmail(senderEmail);
  890. });
  891. emailMessageMap.put(IdUtil.simpleUUID(), dtos);
  892. } catch (Exception e) {
  893. log.error("{} 邮件{} 下载报错 {}", folderName, emailTitle, ExceptionUtil.stacktraceToString(e));
  894. } finally {
  895. if (CollUtil.isNotEmpty(dtos) && log.isInfoEnabled()) {
  896. log.info("{} 邮件{} 下载完成,总计耗时{} ms,文件内容如下\n {}", folderName,
  897. emailTitle, System.currentTimeMillis() - start, dtos);
  898. }
  899. }
  900. }
  901. if (this.readWriteSeen) {
  902. // 设置已读标志
  903. folder.setFlags(messages, new Flags(Flags.Flag.SEEN), true);
  904. }
  905. folder.close(false);
  906. return emailMessageMap;
  907. }
  908. private void rePart(String account, String subject, Date sendDate, Part part,
  909. List<EmailContentInfoDTO> emailContentInfoDTOList) throws Exception {
  910. String fileName = EmailUtil.decodeFileName(part);
  911. if (StrUtil.isBlank(fileName)) {
  912. return;
  913. }
  914. if (fileName.contains("\"") || fileName.contains("\n")) {
  915. fileName = fileName.replaceAll("\"", "").replaceAll("\n", "");
  916. }
  917. if (fileName.contains("=?")) {
  918. fileName = MimeUtility.decodeText(fileName);
  919. }
  920. String disposition = part.getDisposition();
  921. String contentType = part.getContentType();
  922. String[] att_files = new String[]{Constants.ARCHIVE_7Z, Constants.ARCHIVE_RAR, Constants.ARCHIVE_ZIP,
  923. Constants.FILE_PDF, Constants.FILE_DOCX, Constants.FILE_JPG, Constants.FILE_PNG};
  924. boolean attachmentFlag = StrUtil.endWithAny(fileName, att_files);
  925. boolean isAttachment = attachmentFlag
  926. || Part.ATTACHMENT.equalsIgnoreCase(disposition)
  927. || (contentType != null && attachmentMimePrefixes.stream().anyMatch(prefix ->
  928. StrUtil.startWithIgnoreCase(contentType, prefix)
  929. ));
  930. if (!isAttachment) {
  931. log.warn("邮件{} 未检测到{}类型的附件 (fileName={}, disposition={}, contentType={})",
  932. subject, att_files, fileName, disposition, contentType);
  933. return;
  934. }
  935. File saveFile = this.generateSavePath(account, sendDate, fileName);
  936. if (!saveFile.exists()) {
  937. if (!saveFile.getParentFile().exists()) {
  938. boolean mkdirs = saveFile.getParentFile().mkdirs();
  939. if (!mkdirs) {
  940. log.warn("file path mkdir failed.");
  941. }
  942. }
  943. try (InputStream is = part.getInputStream()) {
  944. Files.copy(is, saveFile.toPath());
  945. }
  946. } else {
  947. if (log.isInfoEnabled()) {
  948. log.info("邮件{} 已下载过附件:{},不用重新下载了。", subject, saveFile.toPath());
  949. }
  950. }
  951. EmailContentInfoDTO emailContentInfoDTO = new EmailContentInfoDTO();
  952. emailContentInfoDTO.setFileName(fileName);
  953. emailContentInfoDTO.setFileSize(part.getSize());
  954. emailContentInfoDTO.setFilePath(saveFile.getAbsolutePath());
  955. emailContentInfoDTO.setEmailAddress(account);
  956. emailContentInfoDTO.setEmailTitle(subject);
  957. emailContentInfoDTO.setEmailDate(DateUtil.format(sendDate, DateConst.YYYY_MM_DD_HH_MM_SS));
  958. emailContentInfoDTOList.add(emailContentInfoDTO);
  959. }
  960. public File generateSavePath(String account, Date sendDate, String fileName) {
  961. String emailDateStr = DateUtil.format(sendDate, DateConst.YYYYMMDD);
  962. String filePath = this.path + File.separator + account + File.separator +
  963. emailDateStr + File.separator + "original" + File.separator;
  964. // 压缩包重名时的后面的压缩包会覆盖前面压缩包的问题(不考虑普通文件)
  965. String emailDate = DateUtil.format(sendDate, DateConst.YYYYMMDDHHMMSS24);
  966. String realName = ArchiveUtil.isArchive(fileName) ? emailDate + fileName : fileName;
  967. return FileUtil.file(filePath + realName);
  968. }
  969. private void reMultipart(String account, String subject, Date emailDate, Multipart multipart,
  970. List<EmailContentInfoDTO> emailContentInfoDTOList) throws Exception {
  971. for (int i = 0; i < multipart.getCount(); i++) {
  972. Part bodyPart = multipart.getBodyPart(i);
  973. Object content = bodyPart.getContent();
  974. if (content instanceof String) {
  975. if (log.isDebugEnabled()) {
  976. log.debug("邮件{} 获取的正文不做解析,内容是 {}", subject, content);
  977. }
  978. continue;
  979. }
  980. if (content instanceof Multipart mp) {
  981. this.reMultipart(account, subject, emailDate, mp, emailContentInfoDTOList);
  982. } else {
  983. this.rePart(account, subject, emailDate, bodyPart, emailContentInfoDTOList);
  984. }
  985. }
  986. }
  987. private String getSenderEmail(Message message) {
  988. Address[] senderAddress;
  989. try {
  990. senderAddress = message.getFrom();
  991. if (senderAddress == null || senderAddress.length == 0) {
  992. return null;
  993. }
  994. // 此时的address是含有编码(MIME编码方式)后的文本和实际的邮件地址
  995. String address = "";
  996. for (Address from : senderAddress) {
  997. if (StrUtil.isNotBlank(from.toString())) {
  998. address = from.toString();
  999. break;
  1000. }
  1001. }
  1002. // 正则表达式匹配邮件地址
  1003. Pattern pattern = Pattern.compile("<(\\S+)>");
  1004. Matcher matcher = pattern.matcher(address);
  1005. if (matcher.find()) {
  1006. return matcher.group(1);
  1007. }
  1008. } catch (MessagingException e) {
  1009. log.error(e.getMessage(), e);
  1010. }
  1011. return null;
  1012. }
  1013. private Message[] getEmailMessage(Folder folder, String protocol, Date startDate) {
  1014. try {
  1015. if (protocol.contains("imap")) {
  1016. // 获取邮件日期大于等于startDate的邮件(搜索条件只支持按天)
  1017. SearchTerm startDateTerm = new ReceivedDateTerm(ComparisonTerm.GE, startDate);
  1018. return folder.search(startDateTerm);
  1019. } else {
  1020. return folder.getMessages();
  1021. }
  1022. } catch (MessagingException e) {
  1023. throw new RuntimeException(e);
  1024. }
  1025. }
  1026. /**
  1027. * 检查邮件是否已读
  1028. *
  1029. * @param message 邮件对象
  1030. * @return true表示已读,false表示未读
  1031. * @throws MessagingException 如果访问邮件标志时出错
  1032. */
  1033. private boolean isMessageRead(Message message) throws MessagingException {
  1034. // 获取邮件的所有标志
  1035. Flags flags = message.getFlags();
  1036. // 检查是否包含 SEEN 标志
  1037. return flags.contains(Flags.Flag.SEEN);
  1038. }
  1039. }