feat: 邮箱账单支持正文链接取件（适配微信账单）
邮箱账单发件人规则新增「取件方式」与「链接匹配规则」，支持从邮件正文提取账单下载链接并抓取文件，解决微信等来源不附附件、账单链接位于正文的取件问题；同时补齐附件文件名超长兜底与前端 i18n 特殊字符转义。 变更内容： - 新增 EmailBillFetchMode 枚举（ATTACHMENT/BODY_LINK），发件人规则实体、DTO、VO、前端表单与列表同步支持取件方式及链接匹配正则，保存时校验正则合法性。 - 新增 EmailBodyExtractor，基于 jsoup 解析 HTML 与纯文本正文提取候选链接，按出现顺序去重并限制数量。 - 新增 EmailBillLinkDownloader，仅允许 http/https 且拒绝回环/私网地址（防 SSRF），限制 20MB 与 HTML 响应，按 Content-Disposition、URL 路径与文件魔数推断文件名。 - EmailBillProcessor 按取件方式分流处理，未配置链接正则时优先选取受支持扩展名链接、否则依次尝试前若干链接，单条下载失败不影响其余。 - SchemaMigrator 幂等为 t_email_bill_sender_rule 补充 fetch_mode、link_pattern 列，并同步 MySQL/SQLite 建表脚本。 - 默认发件人规则中微信账单改为正文链接取件并补充发件人匹配串，其余来源显式指定附件取件。 - EmailBillStorage 对百分号编码后的文件名做长度截断并保留扩展名，避免超过文件系统 255 字节上限。 - 前端示例与提示文案对 `|`、`@` 使用 vue-i18n 字面量转义，修复渲染异常。 - build.gradle 引入 jsoup 1.18.1 用于正文 HTML 解析。
This commit is contained in:
@@ -68,6 +68,8 @@ dependencies {
|
||||
implementation 'net.lingala.zip4j:zip4j:2.11.5'
|
||||
// PDF 账单解析（招商银行「交易流水」PDF 为文本型，按字符坐标还原表格）
|
||||
implementation 'org.apache.pdfbox:pdfbox:3.0.3'
|
||||
// 邮件正文 HTML 解析：从中提取账单下载链接（部分来源如微信不发附件、账单在正文链接后）
|
||||
implementation 'org.jsoup:jsoup:1.18.1'
|
||||
|
||||
implementation 'io.jsonwebtoken:jjwt-api:0.12.6'
|
||||
runtimeOnly 'io.jsonwebtoken:jjwt-impl:0.12.6'
|
||||
|
||||
@@ -34,6 +34,8 @@ export interface EmailBillSenderRule {
|
||||
sortOrder: number
|
||||
senderPatterns: string[]
|
||||
billType: BillType
|
||||
fetchMode: string
|
||||
linkPattern: string | null
|
||||
archivePassword: string | null
|
||||
expenseCategoryId: number | null
|
||||
incomeCategoryId: number | null
|
||||
@@ -81,6 +83,8 @@ export interface EmailBillSenderRulePayload {
|
||||
sortOrder: number
|
||||
senderPatterns: string[]
|
||||
billType: BillType
|
||||
fetchMode?: string
|
||||
linkPattern?: string | null
|
||||
archivePassword?: string | null
|
||||
expenseCategoryId?: number | null
|
||||
incomeCategoryId?: number | null
|
||||
|
||||
@@ -1652,9 +1652,9 @@ export default {
|
||||
ja: '自然言語で検索（例：先月の食費はいくら）',
|
||||
},
|
||||
examples: {
|
||||
zh: '上个月餐饮|超过500的支出|待报销的支出|微信支付',
|
||||
en: 'Dining last month|Expenses over 500|Pending reimbursement|Paid by WeChat',
|
||||
ja: '先月の食費|500を超える支出|精算待ちの支出|WeChat 払い',
|
||||
zh: "上个月餐饮{'|'}超过500的支出{'|'}待报销的支出{'|'}微信支付",
|
||||
en: "Dining last month{'|'}Expenses over 500{'|'}Pending reimbursement{'|'}Paid by WeChat",
|
||||
ja: "先月の食費{'|'}500を超える支出{'|'}精算待ちの支出{'|'}WeChat 払い",
|
||||
},
|
||||
button: { zh: 'AI 查询', en: 'AI Query', ja: 'AI 検索' },
|
||||
entry: { zh: 'AI 搜索', en: 'AI Search', ja: 'AI 検索' },
|
||||
@@ -1706,9 +1706,9 @@ export default {
|
||||
ja: '例：先月の支出合計はいくら？',
|
||||
},
|
||||
examples: {
|
||||
zh: '上个月一共花了多少钱|按分类统计今年的支出|今年收入最高的一个月|各渠道的支出占比|本月支出最高的5笔',
|
||||
en: 'How much did I spend last month?|Expenses by category this year|Month with highest income this year|Spending share by channel|Top 5 expenses this month',
|
||||
ja: '先月の支出合計|今年のカテゴリ別支出|今年の収入が最も多い月|チャネル別支出の割合|今月の支出トップ5',
|
||||
zh: "上个月一共花了多少钱{'|'}按分类统计今年的支出{'|'}今年收入最高的一个月{'|'}各渠道的支出占比{'|'}本月支出最高的5笔",
|
||||
en: "How much did I spend last month?{'|'}Expenses by category this year{'|'}Month with highest income this year{'|'}Spending share by channel{'|'}Top 5 expenses this month",
|
||||
ja: "先月の支出合計{'|'}今年のカテゴリ別支出{'|'}今年の収入が最も多い月{'|'}チャネル別支出の割合{'|'}今月の支出トップ5",
|
||||
},
|
||||
submit: { zh: '查询', en: 'Query', ja: '検索' },
|
||||
submitting: { zh: '正在查询，请稍候…', en: 'Querying, please wait…', ja: '検索中です。しばらくお待ちください…' },
|
||||
@@ -1980,11 +1980,20 @@ export default {
|
||||
name: { zh: '规则名称', en: 'Rule Name', ja: 'ルール名' },
|
||||
patterns: { zh: '发件人匹配串', en: 'Sender Patterns', ja: '送信者パターン' },
|
||||
patternsHint: {
|
||||
zh: '每行一个，支持完整地址、@域名或关键字，包含匹配且忽略大小写',
|
||||
en: 'One per line; full address, @domain or keyword, case-insensitive contains',
|
||||
ja: '1行に1つ。完全なアドレス、@ドメイン、キーワードに対応（大文字小文字を区別しない部分一致）',
|
||||
zh: "每行一个，支持完整地址、{'@'}域名或关键字，包含匹配且忽略大小写",
|
||||
en: "One per line; full address, {'@'}domain or keyword, case-insensitive contains",
|
||||
ja: "1行に1つ。完全なアドレス、{'@'}ドメイン、キーワードに対応（大文字小文字を区別しない部分一致）",
|
||||
},
|
||||
billType: { zh: '账单类型', en: 'Bill Type', ja: '請求タイプ' },
|
||||
fetchMode: { zh: '取件方式', en: 'Fetch Mode', ja: '取得方法' },
|
||||
fetchModeAttachment: { zh: '邮件附件', en: 'Email Attachment', ja: 'メール添付' },
|
||||
fetchModeBodyLink: { zh: '正文链接', en: 'Link in Body', ja: '本文リンク' },
|
||||
linkPattern: { zh: '链接匹配规则', en: 'Link Pattern', ja: 'リンクパターン' },
|
||||
linkPatternPlaceholder: {
|
||||
zh: '可选，正则表达式（忽略大小写）；留空时自动选取正文链接',
|
||||
en: 'Optional regex (case-insensitive); auto-select links in body if empty',
|
||||
ja: '任意。正規表現（大文字小文字を区別しない）。空欄時は本文リンクを自動選択',
|
||||
},
|
||||
archivePassword: { zh: '解压密码', en: 'Archive Password', ja: '解凍パスワード' },
|
||||
archivePasswordPlaceholder: {
|
||||
zh: '压缩包加密时填写',
|
||||
|
||||
@@ -516,6 +516,8 @@ function emptyRuleForm() {
|
||||
sortOrder: 0,
|
||||
patternsText: '',
|
||||
billType: defaultBillType(),
|
||||
fetchMode: 'ATTACHMENT',
|
||||
linkPattern: '',
|
||||
archivePassword: '',
|
||||
expenseCategoryId: null,
|
||||
incomeCategoryId: null,
|
||||
@@ -523,6 +525,18 @@ function emptyRuleForm() {
|
||||
}
|
||||
}
|
||||
|
||||
/** 账单取件方式选项：邮件附件 / 正文链接 */
|
||||
const fetchModeOptions = computed(() => [
|
||||
{ label: t('emailBill.senderRule.fetchModeAttachment'), value: 'ATTACHMENT' },
|
||||
{ label: t('emailBill.senderRule.fetchModeBodyLink'), value: 'BODY_LINK' },
|
||||
])
|
||||
|
||||
/** 取件方式编码转当前语言标签 */
|
||||
function fetchModeLabel(value: string | null | undefined) {
|
||||
const option = fetchModeOptions.value.find((item) => item.value === (value || 'ATTACHMENT'))
|
||||
return option ? option.label : t('emailBill.senderRule.fetchModeAttachment')
|
||||
}
|
||||
|
||||
const ruleColumns = computed<DataTableColumns<EmailBillSenderRule>>(() => [
|
||||
{ title: t('emailBill.senderRule.name'), key: 'name', width: 140, ellipsis: { tooltip: true } },
|
||||
{
|
||||
@@ -538,6 +552,12 @@ const ruleColumns = computed<DataTableColumns<EmailBillSenderRule>>(() => [
|
||||
width: 100,
|
||||
render: (row) => h(NTag, { size: 'small', type: 'info' }, { default: () => billTypeLabel(row.billType) }),
|
||||
},
|
||||
{
|
||||
title: t('emailBill.senderRule.fetchMode'),
|
||||
key: 'fetchMode',
|
||||
width: 100,
|
||||
render: (row) => fetchModeLabel(row.fetchMode),
|
||||
},
|
||||
{ title: t('emailBill.senderRule.sortOrder'), key: 'sortOrder', width: 80 },
|
||||
{
|
||||
title: t('emailBill.common.enabled'),
|
||||
@@ -611,6 +631,8 @@ function openRuleModal(row?: EmailBillSenderRule) {
|
||||
sortOrder: row.sortOrder,
|
||||
patternsText: (row.senderPatterns || []).join('\n'),
|
||||
billType: row.billType,
|
||||
fetchMode: row.fetchMode || 'ATTACHMENT',
|
||||
linkPattern: row.linkPattern || '',
|
||||
archivePassword: row.archivePassword || '',
|
||||
expenseCategoryId: row.expenseCategoryId,
|
||||
incomeCategoryId: row.incomeCategoryId,
|
||||
@@ -638,6 +660,8 @@ async function saveRule() {
|
||||
sortOrder: ruleForm.value.sortOrder,
|
||||
senderPatterns: patterns,
|
||||
billType: ruleForm.value.billType,
|
||||
fetchMode: ruleForm.value.fetchMode,
|
||||
linkPattern: ruleForm.value.linkPattern || null,
|
||||
archivePassword: ruleForm.value.archivePassword,
|
||||
expenseCategoryId: ruleForm.value.expenseCategoryId,
|
||||
incomeCategoryId: ruleForm.value.incomeCategoryId,
|
||||
@@ -989,7 +1013,7 @@ onUnmounted(() => {
|
||||
:row-key="(r: EmailBillSenderRule) => r.id"
|
||||
size="small"
|
||||
:bordered="false"
|
||||
:scroll-x="780"
|
||||
:scroll-x="880"
|
||||
:pagination="rulePagination"
|
||||
@update:page="handleRulePageChange"
|
||||
@update:page-size="handleRulePageSizeChange"
|
||||
@@ -1177,6 +1201,15 @@ onUnmounted(() => {
|
||||
<NFormItem :label="t('emailBill.senderRule.billType')">
|
||||
<NSelect v-model:value="ruleForm.billType" :options="billTypeOptions" style="width: 100%" />
|
||||
</NFormItem>
|
||||
<NFormItem :label="t('emailBill.senderRule.fetchMode')">
|
||||
<NSelect v-model:value="ruleForm.fetchMode" :options="fetchModeOptions" style="width: 100%" />
|
||||
</NFormItem>
|
||||
<NFormItem v-if="ruleForm.fetchMode === 'BODY_LINK'" :label="t('emailBill.senderRule.linkPattern')">
|
||||
<NInput
|
||||
v-model:value="ruleForm.linkPattern"
|
||||
:placeholder="t('emailBill.senderRule.linkPatternPlaceholder')"
|
||||
/>
|
||||
</NFormItem>
|
||||
<NFormItem :label="t('emailBill.senderRule.archivePassword')">
|
||||
<NInput
|
||||
v-model:value="ruleForm.archivePassword"
|
||||
|
||||
@@ -41,6 +41,15 @@ public class SchemaMigrator {
|
||||
/** 交易记录表外部交易号列 */
|
||||
private static final String EXTERNAL_NO_COLUMN = "external_no";
|
||||
|
||||
/** 邮箱账单发件人规则表名 */
|
||||
private static final String SENDER_RULE_TABLE = "t_email_bill_sender_rule";
|
||||
|
||||
/** 发件人规则新增列：账单获取方式 */
|
||||
private static final String FETCH_MODE_COLUMN = "fetch_mode";
|
||||
|
||||
/** 发件人规则新增列：正文链接匹配规则 */
|
||||
private static final String LINK_PATTERN_COLUMN = "link_pattern";
|
||||
|
||||
/** 外部交易号列的目标长度（新版支付宝长订单号可达 80+ 字符，原 64 会截断报错） */
|
||||
private static final int EXTERNAL_NO_LENGTH = 255;
|
||||
|
||||
@@ -68,6 +77,7 @@ public class SchemaMigrator {
|
||||
renameChannelColumn(connection, type, progress);
|
||||
addVisionModelColumn(connection, type, progress);
|
||||
widenExternalNoColumn(connection, type, progress);
|
||||
addSenderRuleFetchColumns(connection, type, progress);
|
||||
} catch (SQLException e) {
|
||||
throw new IllegalStateException("数据库结构迁移失败：" + e.getMessage(), e);
|
||||
}
|
||||
@@ -98,6 +108,37 @@ public class SchemaMigrator {
|
||||
notifyProgress(progress, "已为 " + AI_CONFIG_TABLE + " 添加列 " + VISION_MODEL_COLUMN);
|
||||
}
|
||||
|
||||
/**
|
||||
* 为 {@code t_email_bill_sender_rule} 补充 {@code fetch_mode} 与 {@code link_pattern} 列。
|
||||
*
|
||||
* <p>用于支持「正文链接」方式取件（如微信账单邮件不含附件，账单下载链接在正文中）。
|
||||
* 仅当列不存在时执行，保证幂等。</p>
|
||||
*
|
||||
* @param connection 数据库连接
|
||||
* @param type 数据库类型
|
||||
* @param progress 进度回调，可为空
|
||||
* @throws SQLException 迁移失败
|
||||
*/
|
||||
private void addSenderRuleFetchColumns(Connection connection, DataSourceType type, Consumer<String> progress)
|
||||
throws SQLException {
|
||||
if (!columnExists(connection, type, SENDER_RULE_TABLE, FETCH_MODE_COLUMN)) {
|
||||
try (Statement statement = connection.createStatement()) {
|
||||
statement.execute("ALTER TABLE " + SENDER_RULE_TABLE + " ADD COLUMN " + FETCH_MODE_COLUMN
|
||||
+ " VARCHAR(20) NOT NULL DEFAULT 'ATTACHMENT'");
|
||||
}
|
||||
log.info("已为 {} 添加列 {}", SENDER_RULE_TABLE, FETCH_MODE_COLUMN);
|
||||
notifyProgress(progress, "已为 " + SENDER_RULE_TABLE + " 添加列 " + FETCH_MODE_COLUMN);
|
||||
}
|
||||
if (!columnExists(connection, type, SENDER_RULE_TABLE, LINK_PATTERN_COLUMN)) {
|
||||
try (Statement statement = connection.createStatement()) {
|
||||
statement.execute("ALTER TABLE " + SENDER_RULE_TABLE + " ADD COLUMN " + LINK_PATTERN_COLUMN
|
||||
+ " VARCHAR(500)");
|
||||
}
|
||||
log.info("已为 {} 添加列 {}", SENDER_RULE_TABLE, LINK_PATTERN_COLUMN);
|
||||
notifyProgress(progress, "已为 " + SENDER_RULE_TABLE + " 添加列 " + LINK_PATTERN_COLUMN);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 扩展 {@code t_transaction.external_no} 列长度。
|
||||
*
|
||||
|
||||
@@ -30,6 +30,12 @@ public class EmailBillSenderRuleRequest {
|
||||
@NotBlank(message = "请选择账单类型")
|
||||
private String billType;
|
||||
|
||||
/** 账单获取方式：ATTACHMENT-邮件附件，BODY_LINK-正文链接，可选，默认附件 */
|
||||
private String fetchMode;
|
||||
|
||||
/** 正文链接匹配规则（正则，大小写不敏感），仅 BODY_LINK 方式使用，可选 */
|
||||
private String linkPattern;
|
||||
|
||||
/** 该来源账单压缩包解压密码，可选 */
|
||||
private String archivePassword;
|
||||
|
||||
|
||||
@@ -0,0 +1,376 @@
|
||||
package cn.rosercode.ledgerly.email;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.InetAddress;
|
||||
import java.net.URI;
|
||||
import java.net.URLDecoder;
|
||||
import java.net.UnknownHostException;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import cn.rosercode.ledgerly.common.BusinessException;
|
||||
import cn.rosercode.ledgerly.enums.BillType;
|
||||
|
||||
/**
|
||||
* 邮件正文账单链接下载器。
|
||||
*
|
||||
* <p>对正文中提取出的直链发起 GET 请求（自动跟随重定向），下载账单文件字节并推断文件名。
|
||||
* 出于安全考虑，仅允许 http/https，拒绝指向回环/私网地址的链接（防 SSRF），
|
||||
* 并限制单文件体积、拒绝 HTML 等非文件响应。</p>
|
||||
*
|
||||
* @date 2026-10-16 10:00:00
|
||||
*/
|
||||
@Component
|
||||
@Slf4j
|
||||
public class EmailBillLinkDownloader {
|
||||
|
||||
/** 单文件最大字节数（20MB），与附件提取保持一致 */
|
||||
private static final long MAX_SIZE = 20L * 1024 * 1024;
|
||||
|
||||
/** 连接超时 */
|
||||
private static final Duration CONNECT_TIMEOUT = Duration.ofSeconds(10);
|
||||
|
||||
/** 单次请求超时 */
|
||||
private static final Duration REQUEST_TIMEOUT = Duration.ofSeconds(30);
|
||||
|
||||
/** 浏览器 User-Agent，部分下载链接会校验 */
|
||||
private static final String USER_AGENT =
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36";
|
||||
|
||||
/** 已知可用的账单文件扩展名（各来源并集 + 压缩包） */
|
||||
private static final Set<String> KNOWN_EXTENSIONS =
|
||||
Set.of(".csv", ".xlsx", ".xls", ".pdf", ".zip");
|
||||
|
||||
/** ZIP 本地文件头魔数 */
|
||||
private static final byte[] ZIP_MAGIC = {0x50, 0x4B, 0x03, 0x04};
|
||||
|
||||
/** OLE2 复合文档（老版 xls）魔数 */
|
||||
private static final byte[] OLE2_MAGIC = {(byte) 0xD0, (byte) 0xCF, 0x11, (byte) 0xE0,
|
||||
(byte) 0xA1, (byte) 0xB1, 0x1A, (byte) 0xE1};
|
||||
|
||||
/** Content-Disposition 中的 filename*（RFC 5987） */
|
||||
private static final Pattern DISPOSITION_FILENAME_STAR =
|
||||
Pattern.compile("filename\\*\\s*=\\s*([^;]+)", Pattern.CASE_INSENSITIVE);
|
||||
|
||||
/** Content-Disposition 中的 filename */
|
||||
private static final Pattern DISPOSITION_FILENAME =
|
||||
Pattern.compile("filename\\s*=\\s*\"?([^\";]+)\"?", Pattern.CASE_INSENSITIVE);
|
||||
|
||||
private final HttpClient httpClient;
|
||||
|
||||
public EmailBillLinkDownloader() {
|
||||
this.httpClient = HttpClient.newBuilder()
|
||||
.connectTimeout(CONNECT_TIMEOUT)
|
||||
.followRedirects(HttpClient.Redirect.NORMAL)
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 下载链接指向的账单文件。
|
||||
*
|
||||
* @param url 下载链接
|
||||
* @param billType 命中的账单类型，用于在无法识别时兜底推断扩展名
|
||||
* @return 下载得到的文件（文件名 + 字节内容）
|
||||
*/
|
||||
public EmailAttachment download(String url, BillType billType) {
|
||||
URI uri = parseAndValidate(url);
|
||||
HttpResponse<InputStream> response;
|
||||
try {
|
||||
HttpRequest request = HttpRequest.newBuilder(uri)
|
||||
.timeout(REQUEST_TIMEOUT)
|
||||
.header("User-Agent", USER_AGENT)
|
||||
.header("Accept", "*/*")
|
||||
.GET()
|
||||
.build();
|
||||
response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
|
||||
} catch (IOException e) {
|
||||
throw new BusinessException("下载账单链接失败：" + e.getMessage());
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new BusinessException("下载账单链接被中断");
|
||||
}
|
||||
if (response.statusCode() < 200 || response.statusCode() >= 300) {
|
||||
throw new BusinessException("下载账单链接返回状态码 " + response.statusCode());
|
||||
}
|
||||
String contentType = response.headers().firstValue("Content-Type").orElse("");
|
||||
if (contentType.toLowerCase(Locale.ROOT).startsWith("text/html")) {
|
||||
throw new BusinessException("下载链接返回的是网页而非账单文件");
|
||||
}
|
||||
byte[] content;
|
||||
try (InputStream in = response.body()) {
|
||||
content = readLimited(in);
|
||||
} catch (IOException e) {
|
||||
throw new BusinessException("读取账单文件失败：" + e.getMessage());
|
||||
}
|
||||
if (looksLikeHtml(content)) {
|
||||
throw new BusinessException("下载链接返回的是网页而非账单文件");
|
||||
}
|
||||
String disposition = response.headers().firstValue("Content-Disposition").orElse(null);
|
||||
return new EmailAttachment(resolveFileName(uri, disposition, content, billType), content);
|
||||
}
|
||||
|
||||
/**
|
||||
* 构造用于日志/进度的脱敏地址：仅保留 scheme、host 与路径，去除可能含签名的查询串。
|
||||
*
|
||||
* @param url 下载链接
|
||||
* @return 脱敏地址
|
||||
*/
|
||||
public String maskUrl(String url) {
|
||||
try {
|
||||
URI uri = URI.create(url.trim());
|
||||
String path = uri.getPath() == null ? "" : uri.getPath();
|
||||
return uri.getScheme() + "://" + uri.getHost() + path;
|
||||
} catch (IllegalArgumentException e) {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析并校验链接：必须是 http/https 且目标主机为公网地址。
|
||||
*
|
||||
* @param url 下载链接
|
||||
* @return 解析后的 URI
|
||||
*/
|
||||
private URI parseAndValidate(String url) {
|
||||
URI uri;
|
||||
try {
|
||||
uri = URI.create(url.trim());
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new BusinessException("非法的账单链接");
|
||||
}
|
||||
String scheme = uri.getScheme() == null ? "" : uri.getScheme().toLowerCase(Locale.ROOT);
|
||||
if (!"http".equals(scheme) && !"https".equals(scheme)) {
|
||||
throw new BusinessException("不支持的链接协议：" + scheme);
|
||||
}
|
||||
String host = uri.getHost();
|
||||
if (host == null || host.isBlank()) {
|
||||
throw new BusinessException("账单链接缺少主机名");
|
||||
}
|
||||
assertPublicHost(host);
|
||||
return uri;
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验主机解析到公网地址，拒绝回环/私网/链路本地等地址，防止 SSRF。
|
||||
*
|
||||
* @param host 主机名
|
||||
*/
|
||||
private void assertPublicHost(String host) {
|
||||
InetAddress[] addresses;
|
||||
try {
|
||||
addresses = InetAddress.getAllByName(host);
|
||||
} catch (UnknownHostException e) {
|
||||
throw new BusinessException("无法解析账单链接主机：" + host);
|
||||
}
|
||||
for (InetAddress address : addresses) {
|
||||
if (address.isLoopbackAddress() || address.isAnyLocalAddress()
|
||||
|| address.isLinkLocalAddress() || address.isSiteLocalAddress()
|
||||
|| address.isMulticastAddress() || isUniqueLocalIpv6(address)) {
|
||||
throw new BusinessException("账单链接指向内网地址，已拒绝");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断是否为 IPv6 唯一本地地址（fc00::/7）。
|
||||
*
|
||||
* @param address 地址
|
||||
* @return 是唯一本地地址返回 true
|
||||
*/
|
||||
private boolean isUniqueLocalIpv6(InetAddress address) {
|
||||
byte[] bytes = address.getAddress();
|
||||
return bytes.length == 16 && (bytes[0] & 0xFE) == 0xFC;
|
||||
}
|
||||
|
||||
/**
|
||||
* 推断文件名：优先 Content-Disposition，其次 URL 路径末段，最后按类型合成；
|
||||
* 扩展名缺失或不可识别时按内容魔数或账单类型补齐。
|
||||
*
|
||||
* @param uri 下载链接
|
||||
* @param disposition Content-Disposition 响应头，可为空
|
||||
* @param content 文件字节内容
|
||||
* @param billType 账单类型
|
||||
* @return 文件名
|
||||
*/
|
||||
String resolveFileName(URI uri, String disposition, byte[] content, BillType billType) {
|
||||
String baseName = null;
|
||||
String rawName = disposition == null ? null : fileNameFromDisposition(disposition);
|
||||
if (rawName == null || rawName.isBlank()) {
|
||||
rawName = lastNameFromPath(uri.getPath());
|
||||
}
|
||||
String extension = rawName == null ? null : extensionOf(rawName);
|
||||
if (rawName != null && !rawName.isBlank()) {
|
||||
baseName = extension == null ? rawName : rawName.substring(0, rawName.length() - extension.length());
|
||||
}
|
||||
if (extension == null || !KNOWN_EXTENSIONS.contains(extension)) {
|
||||
extension = detectExtension(content, billType);
|
||||
}
|
||||
if (baseName == null || baseName.isBlank()) {
|
||||
baseName = billType.name().toLowerCase(Locale.ROOT) + "-" + System.currentTimeMillis();
|
||||
}
|
||||
return baseName + extension;
|
||||
}
|
||||
|
||||
/**
|
||||
* 从 Content-Disposition 解析文件名，兼容 RFC 5987 的 {@code filename*}。
|
||||
*
|
||||
* @param disposition 响应头
|
||||
* @return 文件名，未提供时返回 null
|
||||
*/
|
||||
private String fileNameFromDisposition(String disposition) {
|
||||
Matcher star = DISPOSITION_FILENAME_STAR.matcher(disposition);
|
||||
if (star.find()) {
|
||||
String value = star.group(1).trim();
|
||||
int index = value.indexOf("''");
|
||||
if (index >= 0) {
|
||||
value = value.substring(index + 2);
|
||||
}
|
||||
try {
|
||||
return URLDecoder.decode(value, StandardCharsets.UTF_8);
|
||||
} catch (IllegalArgumentException e) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
Matcher plain = DISPOSITION_FILENAME.matcher(disposition);
|
||||
if (plain.find()) {
|
||||
return plain.group(1).trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 取 URL 路径的最后一段作为文件名。
|
||||
*
|
||||
* @param path URL 路径
|
||||
* @return 文件名，无有效末段时返回 null
|
||||
*/
|
||||
private String lastNameFromPath(String path) {
|
||||
if (path == null || path.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
int slash = path.lastIndexOf('/');
|
||||
String name = slash >= 0 ? path.substring(slash + 1) : path;
|
||||
return name.isBlank() ? null : URLDecoder.decode(name, StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
/**
|
||||
* 取文件名的小写扩展名（含点）。
|
||||
*
|
||||
* @param name 文件名
|
||||
* @return 扩展名，无扩展名时返回 null
|
||||
*/
|
||||
private String extensionOf(String name) {
|
||||
int dot = name.lastIndexOf('.');
|
||||
if (dot < 0 || dot == name.length() - 1) {
|
||||
return null;
|
||||
}
|
||||
return name.substring(dot).toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
/**
|
||||
* 按内容魔数推断扩展名，无法识别时回退账单类型的首选扩展名。
|
||||
*
|
||||
* @param content 文件字节内容
|
||||
* @param billType 账单类型
|
||||
* @return 扩展名（含点）
|
||||
*/
|
||||
String detectExtension(byte[] content, BillType billType) {
|
||||
if (startsWith(content, ZIP_MAGIC)) {
|
||||
return containsAscii(content, "xl/") ? ".xlsx" : ".zip";
|
||||
}
|
||||
if (startsWith(content, OLE2_MAGIC)) {
|
||||
return ".xls";
|
||||
}
|
||||
return billType.getExtensions().get(0);
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断字节内容是否以指定魔数开头。
|
||||
*
|
||||
* @param content 字节内容
|
||||
* @param magic 魔数
|
||||
* @return 匹配返回 true
|
||||
*/
|
||||
private boolean startsWith(byte[] content, byte[] magic) {
|
||||
if (content.length < magic.length) {
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < magic.length; i++) {
|
||||
if (content[i] != magic[i]) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* 在字节内容的前 4KB 内查找 ASCII 子串（用于识别 xlsx 内部结构）。
|
||||
*
|
||||
* @param content 字节内容
|
||||
* @param ascii 待查找的 ASCII 子串
|
||||
* @return 存在返回 true
|
||||
*/
|
||||
private boolean containsAscii(byte[] content, String ascii) {
|
||||
byte[] needle = ascii.getBytes(StandardCharsets.US_ASCII);
|
||||
int limit = Math.min(content.length - needle.length, 4096);
|
||||
for (int i = 0; i <= limit; i++) {
|
||||
boolean matched = true;
|
||||
for (int j = 0; j < needle.length; j++) {
|
||||
if (content[i + j] != needle[j]) {
|
||||
matched = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (matched) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断字节内容是否疑似 HTML 页面（错误页/登录页）。
|
||||
*
|
||||
* @param content 字节内容
|
||||
* @return 疑似 HTML 返回 true
|
||||
*/
|
||||
boolean looksLikeHtml(byte[] content) {
|
||||
int limit = Math.min(content.length, 512);
|
||||
String head = new String(content, 0, limit, StandardCharsets.UTF_8).trim().toLowerCase(Locale.ROOT);
|
||||
return head.startsWith("<!doctype html") || head.startsWith("<html");
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取输入流并限制最大字节数。
|
||||
*
|
||||
* @param in 输入流
|
||||
* @return 字节内容
|
||||
* @throws IOException 读取失败或超过上限
|
||||
*/
|
||||
private byte[] readLimited(InputStream in) throws IOException {
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
byte[] buffer = new byte[8192];
|
||||
long total = 0;
|
||||
int read;
|
||||
while ((read = in.read(buffer)) != -1) {
|
||||
total += read;
|
||||
if (total > MAX_SIZE) {
|
||||
throw new BusinessException("账单文件超过大小上限，已拒绝下载");
|
||||
}
|
||||
out.write(buffer, 0, read);
|
||||
}
|
||||
return out.toByteArray();
|
||||
}
|
||||
}
|
||||
@@ -3,11 +3,13 @@ package cn.rosercode.ledgerly.email;
|
||||
import java.io.IOException;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneId;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import jakarta.mail.Address;
|
||||
@@ -21,6 +23,7 @@ import cn.rosercode.ledgerly.entity.EmailBillConfig;
|
||||
import cn.rosercode.ledgerly.entity.EmailBillMessage;
|
||||
import cn.rosercode.ledgerly.entity.EmailBillSenderRule;
|
||||
import cn.rosercode.ledgerly.enums.BillType;
|
||||
import cn.rosercode.ledgerly.enums.EmailBillFetchMode;
|
||||
import cn.rosercode.ledgerly.enums.EmailBillStatus;
|
||||
import cn.rosercode.ledgerly.mapper.EmailBillMessageMapper;
|
||||
import cn.rosercode.ledgerly.service.EmailBillSenderRuleService;
|
||||
@@ -40,6 +43,9 @@ public class EmailBillProcessor {
|
||||
/** 支持的账单附件扩展名：各账单来源扩展名的并集，另含压缩包 */
|
||||
private static final Set<String> SUPPORTED_EXTENSIONS = buildSupportedExtensions();
|
||||
|
||||
/** 无链接匹配规则时，自动尝试下载的链接数上限 */
|
||||
private static final int MAX_AUTO_LINKS = 5;
|
||||
|
||||
/**
|
||||
* 由账单来源枚举汇总支持的附件扩展名，并补充压缩包扩展名。
|
||||
*
|
||||
@@ -58,17 +64,23 @@ public class EmailBillProcessor {
|
||||
private final EmailBillSenderRuleService senderRuleService;
|
||||
private final EmailBillSenderRuleMatcher senderRuleMatcher;
|
||||
private final EmailAttachmentExtractor attachmentExtractor;
|
||||
private final EmailBodyExtractor bodyExtractor;
|
||||
private final EmailBillLinkDownloader linkDownloader;
|
||||
private final EmailBillStorage storage;
|
||||
|
||||
public EmailBillProcessor(EmailBillMessageMapper messageMapper,
|
||||
EmailBillSenderRuleService senderRuleService,
|
||||
EmailBillSenderRuleMatcher senderRuleMatcher,
|
||||
EmailAttachmentExtractor attachmentExtractor,
|
||||
EmailBodyExtractor bodyExtractor,
|
||||
EmailBillLinkDownloader linkDownloader,
|
||||
EmailBillStorage storage) {
|
||||
this.messageMapper = messageMapper;
|
||||
this.senderRuleService = senderRuleService;
|
||||
this.senderRuleMatcher = senderRuleMatcher;
|
||||
this.attachmentExtractor = attachmentExtractor;
|
||||
this.bodyExtractor = bodyExtractor;
|
||||
this.linkDownloader = linkDownloader;
|
||||
this.storage = storage;
|
||||
}
|
||||
|
||||
@@ -104,14 +116,15 @@ public class EmailBillProcessor {
|
||||
progress.accept(" → 发件人未命中规则，忽略");
|
||||
return 0;
|
||||
}
|
||||
progress.accept(" → 命中规则「" + rule.getName() + "」，账单类型 " + rule.getBillType());
|
||||
progress.accept(" → 命中规则「" + rule.getName() + "」，账单类型 " + rule.getBillType()
|
||||
+ "，取件方式 " + EmailBillFetchMode.resolve(rule.getFetchMode()).getLabel());
|
||||
|
||||
List<EmailAttachment> attachments;
|
||||
try {
|
||||
attachments = attachmentExtractor.extract(message);
|
||||
attachments = fetchAttachments(config, message, uid, rule, progress);
|
||||
} catch (MessagingException | IOException e) {
|
||||
log.warn("提取邮件附件失败: configId={}, uid={}, error={}", config.getId(), uid, e.getMessage());
|
||||
progress.accept(" → 提取附件失败：" + e.getMessage());
|
||||
log.warn("提取邮件账单失败: configId={}, uid={}, error={}", config.getId(), uid, e.getMessage());
|
||||
progress.accept(" → 提取账单失败：" + e.getMessage());
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -127,13 +140,95 @@ public class EmailBillProcessor {
|
||||
saved++;
|
||||
}
|
||||
if (saved > 0) {
|
||||
log.info("邮箱账单附件已保存: configId={}, uid={}, from={}, billType={}, 附件数={}",
|
||||
config.getId(), uid, from, rule.getBillType(), saved);
|
||||
log.info("邮箱账单已保存: configId={}, uid={}, from={}, billType={}, fetchMode={}, 文件数={}",
|
||||
config.getId(), uid, from, rule.getBillType(), rule.getFetchMode(), saved);
|
||||
}
|
||||
progress.accept(" → 保存 " + saved + " 个附件");
|
||||
progress.accept(" → 保存 " + saved + " 个账单文件");
|
||||
return saved;
|
||||
}
|
||||
|
||||
/**
|
||||
* 按规则的取件方式获取账单文件。
|
||||
*
|
||||
* <p>附件方式递归提取 MIME 附件；正文链接方式解析正文链接并下载。单条链接下载失败
|
||||
* 不影响其余链接，仅记录日志并上报进度。</p>
|
||||
*
|
||||
* @param config 邮箱配置
|
||||
* @param message 邮件消息
|
||||
* @param uid 邮件 UID
|
||||
* @param rule 命中的发件人规则
|
||||
* @param progress 进度回调
|
||||
* @return 获取到的账单文件列表
|
||||
* @throws MessagingException 邮件解析失败
|
||||
* @throws IOException 内容读取失败
|
||||
*/
|
||||
private List<EmailAttachment> fetchAttachments(EmailBillConfig config, Message message, long uid,
|
||||
EmailBillSenderRule rule, Consumer<String> progress)
|
||||
throws MessagingException, IOException {
|
||||
if (EmailBillFetchMode.resolve(rule.getFetchMode()) != EmailBillFetchMode.BODY_LINK) {
|
||||
return attachmentExtractor.extract(message);
|
||||
}
|
||||
BillType billType = BillType.valueOf(rule.getBillType().trim().toUpperCase());
|
||||
List<String> links = bodyExtractor.extractLinks(message);
|
||||
if (links.isEmpty()) {
|
||||
progress.accept(" → 正文中未找到链接");
|
||||
return List.of();
|
||||
}
|
||||
List<String> selected = selectLinks(links, rule);
|
||||
progress.accept(" → 正文链接命中 " + selected.size() + " 个");
|
||||
List<EmailAttachment> files = new ArrayList<>();
|
||||
for (String url : selected) {
|
||||
try {
|
||||
EmailAttachment file = linkDownloader.download(url, billType);
|
||||
files.add(file);
|
||||
progress.accept(" ↓ " + linkDownloader.maskUrl(url) + " → " + file.getFileName());
|
||||
} catch (RuntimeException e) {
|
||||
log.warn("下载正文账单链接失败: configId={}, uid={}, url={}, error={}",
|
||||
config.getId(), uid, linkDownloader.maskUrl(url), e.getMessage());
|
||||
progress.accept(" ✗ " + linkDownloader.maskUrl(url) + "：" + e.getMessage());
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
/**
|
||||
* 从正文候选链接中筛选待下载的链接。
|
||||
*
|
||||
* <p>配置了链接匹配规则时按正则（大小写不敏感）筛选；未配置时优先选择 URL 以受支持
|
||||
* 扩展名结尾的链接，若均不匹配则取前若干个链接依次尝试。</p>
|
||||
*
|
||||
* @param links 候选链接
|
||||
* @param rule 发件人规则
|
||||
* @return 待下载的链接列表
|
||||
*/
|
||||
private List<String> selectLinks(List<String> links, EmailBillSenderRule rule) {
|
||||
if (rule.getLinkPattern() != null && !rule.getLinkPattern().isBlank()) {
|
||||
Pattern pattern = Pattern.compile(rule.getLinkPattern().trim(), Pattern.CASE_INSENSITIVE);
|
||||
return links.stream().filter(link -> pattern.matcher(link).find()).toList();
|
||||
}
|
||||
List<String> byExtension = links.stream().filter(this::endsWithSupportedExtension).toList();
|
||||
if (!byExtension.isEmpty()) {
|
||||
return byExtension;
|
||||
}
|
||||
return links.stream().limit(MAX_AUTO_LINKS).toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断链接路径是否以受支持的账单扩展名结尾。
|
||||
*
|
||||
* @param link 链接
|
||||
* @return 匹配返回 true
|
||||
*/
|
||||
private boolean endsWithSupportedExtension(String link) {
|
||||
String path = link;
|
||||
int query = path.indexOf('?');
|
||||
if (query >= 0) {
|
||||
path = path.substring(0, query);
|
||||
}
|
||||
String lower = path.toLowerCase();
|
||||
return SUPPORTED_EXTENSIONS.stream().anyMatch(lower::endsWith);
|
||||
}
|
||||
|
||||
/**
|
||||
* 构造待解析消息实体。
|
||||
*
|
||||
|
||||
@@ -27,6 +27,12 @@ public class EmailBillStorage {
|
||||
/** 附件存储根目录（相对应用基准目录） */
|
||||
private static final Path ROOT = AppHome.resolve("data", "email-bill");
|
||||
|
||||
/** 落盘文件名最大长度（ASCII 字符数=字节数），留出 index 前缀与文件系统 255 字节上限余量 */
|
||||
private static final int MAX_NAME_LENGTH = 200;
|
||||
|
||||
/** 截断时保留的扩展名最大长度，超出视为非扩展名 */
|
||||
private static final int MAX_EXTENSION_LENGTH = 10;
|
||||
|
||||
/**
|
||||
* 保存附件到本地，返回相对存储根目录的路径。
|
||||
*
|
||||
@@ -122,6 +128,32 @@ public class EmailBillStorage {
|
||||
if (!StandardCharsets.US_ASCII.newEncoder().canEncode(name)) {
|
||||
name = URLEncoder.encode(name, StandardCharsets.UTF_8).replace("+", "%20");
|
||||
}
|
||||
return limitLength(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* 限制文件名长度以避免超出文件系统单文件名上限（过长时保留扩展名截断）。
|
||||
*
|
||||
* <p>百分号编码会把每个非 ASCII 字符展开为 9 个 ASCII 字符，长中文名编码后极易超过
|
||||
* 255 字节上限而抛 {@code File name too long}，故在此统一兜底截断。</p>
|
||||
*
|
||||
* @param name 已清洗（含百分号编码）的文件名
|
||||
* @return 长度受限的文件名
|
||||
*/
|
||||
private String limitLength(String name) {
|
||||
if (name.length() <= MAX_NAME_LENGTH) {
|
||||
return name;
|
||||
}
|
||||
String extension = "";
|
||||
int dot = name.lastIndexOf('.');
|
||||
if (dot > 0 && name.length() - dot <= MAX_EXTENSION_LENGTH) {
|
||||
extension = name.substring(dot);
|
||||
}
|
||||
int baseLength = MAX_NAME_LENGTH - extension.length();
|
||||
if (baseLength < 1) {
|
||||
extension = "";
|
||||
baseLength = MAX_NAME_LENGTH;
|
||||
}
|
||||
return name.substring(0, baseLength) + extension;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
package cn.rosercode.ledgerly.email;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import jakarta.mail.Message;
|
||||
import jakarta.mail.MessagingException;
|
||||
import jakarta.mail.Multipart;
|
||||
import jakarta.mail.Part;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 邮件正文链接提取器，从 MIME 邮件的正文（HTML 与纯文本）中提取候选链接。
|
||||
*
|
||||
* <p>部分账单来源（如微信）不以附件下发账单，而是在邮件正文放置下载链接，
|
||||
* 本类负责把正文里的链接集中提取出来，供 {@link EmailBillProcessor} 下载。
|
||||
* HTML 用 jsoup 解析 {@code <a href>}（含相对链接转绝对链接），纯文本用正则匹配裸链接。</p>
|
||||
*
|
||||
* @date 2026-10-16 10:00:00
|
||||
*/
|
||||
@Component
|
||||
@Slf4j
|
||||
public class EmailBodyExtractor {
|
||||
|
||||
/** 纯文本中的裸链接匹配规则 */
|
||||
private static final Pattern PLAIN_URL = Pattern.compile("https?://[^\\s\"'<>()\\[\\]]+");
|
||||
|
||||
/** 单封邮件提取的正文链接上限，避免异常邮件导致链接过多 */
|
||||
private static final int MAX_LINKS = 50;
|
||||
|
||||
/**
|
||||
* 提取邮件正文中的候选链接，按出现顺序去重。
|
||||
*
|
||||
* @param message 邮件消息
|
||||
* @return 候选链接列表，无链接时返回空列表
|
||||
* @throws MessagingException 邮件解析失败
|
||||
* @throws IOException 正文读取失败
|
||||
*/
|
||||
public List<String> extractLinks(Message message) throws MessagingException, IOException {
|
||||
Set<String> links = new LinkedHashSet<>();
|
||||
collect(message, links);
|
||||
return new ArrayList<>(links);
|
||||
}
|
||||
|
||||
/**
|
||||
* 递归遍历 MIME 部件，收集正文中的链接。
|
||||
*
|
||||
* @param part 当前部件
|
||||
* @param links 链接收集集合
|
||||
* @throws MessagingException 邮件解析失败
|
||||
* @throws IOException 正文读取失败
|
||||
*/
|
||||
private void collect(Part part, Set<String> links) throws MessagingException, IOException {
|
||||
if (links.size() >= MAX_LINKS) {
|
||||
return;
|
||||
}
|
||||
if (part.isMimeType("multipart/*")) {
|
||||
Multipart multipart = (Multipart) part.getContent();
|
||||
for (int i = 0; i < multipart.getCount(); i++) {
|
||||
collect(multipart.getBodyPart(i), links);
|
||||
}
|
||||
return;
|
||||
}
|
||||
// 跳过显式附件（如以附件形式发送的 HTML 文件），仅扫描正文
|
||||
if (Part.ATTACHMENT.equalsIgnoreCase(part.getDisposition())) {
|
||||
return;
|
||||
}
|
||||
if (part.isMimeType("text/html")) {
|
||||
extractFromHtml(part.getContent(), links);
|
||||
return;
|
||||
}
|
||||
if (part.isMimeType("text/plain")) {
|
||||
extractFromPlain(part.getContent(), links);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从 HTML 正文中提取 {@code <a href>} 链接。
|
||||
*
|
||||
* @param content 正文内容
|
||||
* @param links 链接收集集合
|
||||
*/
|
||||
private void extractFromHtml(Object content, Set<String> links) {
|
||||
if (!(content instanceof String html) || html.isBlank()) {
|
||||
return;
|
||||
}
|
||||
Document document = Jsoup.parse(html);
|
||||
for (Element anchor : document.select("a[href]")) {
|
||||
String url = anchor.attr("abs:href");
|
||||
if (url.isBlank()) {
|
||||
url = anchor.attr("href");
|
||||
}
|
||||
addLink(url, links);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从纯文本正文中提取裸链接。
|
||||
*
|
||||
* @param content 正文内容
|
||||
* @param links 链接收集集合
|
||||
*/
|
||||
private void extractFromPlain(Object content, Set<String> links) {
|
||||
if (!(content instanceof String text) || text.isBlank()) {
|
||||
return;
|
||||
}
|
||||
Matcher matcher = PLAIN_URL.matcher(text);
|
||||
while (matcher.find() && links.size() < MAX_LINKS) {
|
||||
addLink(matcher.group(), links);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 加入链接：去除首尾空白与常见尾随标点，仅保留 http/https。
|
||||
*
|
||||
* @param raw 原始链接
|
||||
* @param links 链接收集集合
|
||||
*/
|
||||
private void addLink(String raw, Set<String> links) {
|
||||
if (raw == null || links.size() >= MAX_LINKS) {
|
||||
return;
|
||||
}
|
||||
String url = raw.trim();
|
||||
while (!url.isEmpty() && ".,;)>]}'\"".indexOf(url.charAt(url.length() - 1)) >= 0) {
|
||||
url = url.substring(0, url.length() - 1);
|
||||
}
|
||||
String lower = url.toLowerCase();
|
||||
if (lower.startsWith("http://") || lower.startsWith("https://")) {
|
||||
links.add(url);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -47,6 +47,12 @@ public class EmailBillSenderRule {
|
||||
/** 命中的账单类型：ALIPAY/WECHAT/MEITUAN/CMB/BOCOM */
|
||||
private String billType;
|
||||
|
||||
/** 账单获取方式：ATTACHMENT-邮件附件，BODY_LINK-正文链接 */
|
||||
private String fetchMode;
|
||||
|
||||
/** 正文链接匹配规则（正则，大小写不敏感），仅 BODY_LINK 方式使用，可为空表示自动选取 */
|
||||
private String linkPattern;
|
||||
|
||||
/** 该来源账单压缩包解压密码，可为空 */
|
||||
private String archivePassword;
|
||||
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package cn.rosercode.ledgerly.enums;
|
||||
|
||||
/**
|
||||
* 邮箱账单取件方式枚举。
|
||||
*
|
||||
* <p>多数来源（支付宝、银行等）以邮件附件形式下发账单，取件方式为 {@link #ATTACHMENT}；
|
||||
* 微信等来源不附附件，账单下载链接位于邮件正文，取件方式为 {@link #BODY_LINK}，
|
||||
* 由系统解析正文链接并下载。</p>
|
||||
*
|
||||
* @date 2026-10-16 10:00:00
|
||||
*/
|
||||
public enum EmailBillFetchMode {
|
||||
|
||||
/** 邮件附件 */
|
||||
ATTACHMENT("邮件附件"),
|
||||
|
||||
/** 正文链接 */
|
||||
BODY_LINK("正文链接");
|
||||
|
||||
/** 取件方式的中文名称 */
|
||||
private final String label;
|
||||
|
||||
EmailBillFetchMode(String label) {
|
||||
this.label = label;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取取件方式的中文名称
|
||||
*
|
||||
* @return 中文名称
|
||||
*/
|
||||
public String getLabel() {
|
||||
return label;
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析取件方式编码，非法或为空时回退为邮件附件。
|
||||
*
|
||||
* @param value 取件方式编码
|
||||
* @return 取件方式枚举
|
||||
*/
|
||||
public static EmailBillFetchMode resolve(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return ATTACHMENT;
|
||||
}
|
||||
try {
|
||||
return valueOf(value.trim().toUpperCase());
|
||||
} catch (IllegalArgumentException e) {
|
||||
return ATTACHMENT;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import org.springframework.stereotype.Component;
|
||||
import cn.rosercode.ledgerly.entity.Category;
|
||||
import cn.rosercode.ledgerly.entity.EmailBillSenderRule;
|
||||
import cn.rosercode.ledgerly.entity.Tag;
|
||||
import cn.rosercode.ledgerly.enums.EmailBillFetchMode;
|
||||
import cn.rosercode.ledgerly.enums.TransactionType;
|
||||
import cn.rosercode.ledgerly.mapper.CategoryMapper;
|
||||
import cn.rosercode.ledgerly.mapper.EmailBillSenderRuleMapper;
|
||||
@@ -64,18 +65,23 @@ public class DefaultDataInitializer {
|
||||
new TagSeed("宠物", "#a16207", 120),
|
||||
new TagSeed("电影", "#14b8a6", 130));
|
||||
|
||||
/** 默认邮箱账单发件人规则（名称、发件人匹配串、账单类型、排序） */
|
||||
/** 默认邮箱账单发件人规则（名称、发件人匹配串、账单类型、取件方式、排序） */
|
||||
private static final List<SenderRuleSeed> DEFAULT_SENDER_RULES = List.of(
|
||||
new SenderRuleSeed("支付宝账单",
|
||||
List.of("service@mail.alipay.com", "@alipay.com"), "ALIPAY", 10),
|
||||
List.of("service@mail.alipay.com", "@alipay.com"), "ALIPAY",
|
||||
EmailBillFetchMode.ATTACHMENT, 10),
|
||||
new SenderRuleSeed("微信支付账单",
|
||||
List.of("@wechatpay.com", "@tenpay.com"), "WECHAT", 20),
|
||||
List.of("wechatpay@tencent.com", "@wechatpay.com", "@tenpay.com"), "WECHAT",
|
||||
EmailBillFetchMode.BODY_LINK, 20),
|
||||
new SenderRuleSeed("美团账单",
|
||||
List.of("it_paybill@meituan.com", "@meituan.com"), "MEITUAN", 30),
|
||||
List.of("it_paybill@meituan.com", "@meituan.com"), "MEITUAN",
|
||||
EmailBillFetchMode.ATTACHMENT, 30),
|
||||
new SenderRuleSeed("招商银行账单",
|
||||
List.of("95555@message.cmbchina.com", "@message.cmbchina.com"), "CMB", 40),
|
||||
List.of("95555@message.cmbchina.com", "@message.cmbchina.com"), "CMB",
|
||||
EmailBillFetchMode.ATTACHMENT, 40),
|
||||
new SenderRuleSeed("交通银行账单",
|
||||
List.of("@bankcomm.com"), "BOCOM", 50));
|
||||
List.of("@bankcomm.com"), "BOCOM",
|
||||
EmailBillFetchMode.ATTACHMENT, 50));
|
||||
|
||||
private final CategoryMapper categoryMapper;
|
||||
private final TagMapper tagMapper;
|
||||
@@ -120,6 +126,7 @@ public class DefaultDataInitializer {
|
||||
rule.setSortOrder(seed.sortOrder());
|
||||
rule.setSenderPatterns(writePatterns(seed.patterns()));
|
||||
rule.setBillType(seed.billType());
|
||||
rule.setFetchMode(seed.fetchMode().name());
|
||||
emailBillSenderRuleMapper.insert(rule);
|
||||
}
|
||||
}
|
||||
@@ -181,8 +188,10 @@ public class DefaultDataInitializer {
|
||||
* @param name 规则名称
|
||||
* @param patterns 发件人匹配串列表
|
||||
* @param billType 账单类型
|
||||
* @param fetchMode 取件方式
|
||||
* @param sortOrder 排序序号
|
||||
*/
|
||||
private record SenderRuleSeed(String name, List<String> patterns, String billType, int sortOrder) {
|
||||
private record SenderRuleSeed(String name, List<String> patterns, String billType,
|
||||
EmailBillFetchMode fetchMode, int sortOrder) {
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,8 @@ package cn.rosercode.ledgerly.service.impl;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.regex.PatternSyntaxException;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
@@ -21,6 +23,7 @@ import cn.rosercode.ledgerly.entity.Category;
|
||||
import cn.rosercode.ledgerly.entity.EmailBillSenderRule;
|
||||
import cn.rosercode.ledgerly.entity.Tag;
|
||||
import cn.rosercode.ledgerly.enums.BillType;
|
||||
import cn.rosercode.ledgerly.enums.EmailBillFetchMode;
|
||||
import cn.rosercode.ledgerly.enums.TransactionType;
|
||||
import cn.rosercode.ledgerly.mapper.CategoryMapper;
|
||||
import cn.rosercode.ledgerly.mapper.EmailBillSenderRuleMapper;
|
||||
@@ -179,6 +182,9 @@ public class EmailBillSenderRuleServiceImpl implements EmailBillSenderRuleServic
|
||||
rule.setSortOrder(request.getSortOrder() != null ? request.getSortOrder() : 0);
|
||||
rule.setSenderPatterns(writeJson(cleanPatterns(request.getSenderPatterns())));
|
||||
rule.setBillType(request.getBillType().trim().toUpperCase());
|
||||
rule.setFetchMode(EmailBillFetchMode.resolve(request.getFetchMode()).name());
|
||||
rule.setLinkPattern(StringUtils.hasText(request.getLinkPattern())
|
||||
? request.getLinkPattern().trim() : null);
|
||||
rule.setArchivePassword(StringUtils.hasText(request.getArchivePassword())
|
||||
? request.getArchivePassword().trim() : null);
|
||||
rule.setExpenseCategoryId(request.getExpenseCategoryId());
|
||||
@@ -204,6 +210,14 @@ public class EmailBillSenderRuleServiceImpl implements EmailBillSenderRuleServic
|
||||
if (cleanPatterns(request.getSenderPatterns()).isEmpty()) {
|
||||
throw new BusinessException("请至少填写一个发件人匹配串");
|
||||
}
|
||||
// 正文链接匹配规则必须为合法正则
|
||||
if (StringUtils.hasText(request.getLinkPattern())) {
|
||||
try {
|
||||
Pattern.compile(request.getLinkPattern().trim());
|
||||
} catch (PatternSyntaxException e) {
|
||||
throw new BusinessException("链接匹配规则不是合法的正则表达式：" + e.getDescription());
|
||||
}
|
||||
}
|
||||
// 默认分类归属与类型一致性
|
||||
validateCategory(userId, request.getExpenseCategoryId(), TransactionType.EXPENSE.name());
|
||||
validateCategory(userId, request.getIncomeCategoryId(), TransactionType.INCOME.name());
|
||||
|
||||
@@ -31,6 +31,12 @@ public class EmailBillSenderRuleVO {
|
||||
/** 命中的账单类型 */
|
||||
private String billType;
|
||||
|
||||
/** 账单获取方式：ATTACHMENT-邮件附件，BODY_LINK-正文链接 */
|
||||
private String fetchMode;
|
||||
|
||||
/** 正文链接匹配规则（正则），仅 BODY_LINK 方式使用 */
|
||||
private String linkPattern;
|
||||
|
||||
/** 该来源账单压缩包解压密码 */
|
||||
private String archivePassword;
|
||||
|
||||
|
||||
@@ -291,6 +291,8 @@ CREATE TABLE IF NOT EXISTS t_email_bill_sender_rule (
|
||||
sort_order INT NOT NULL DEFAULT 0 COMMENT '优先级，数值越小越优先',
|
||||
sender_patterns TEXT COMMENT '发件人匹配串列表（JSON数组），包含匹配且忽略大小写',
|
||||
bill_type VARCHAR(20) NOT NULL COMMENT '命中的账单类型：ALIPAY/WECHAT/MEITUAN/CMB/BOCOM',
|
||||
fetch_mode VARCHAR(20) NOT NULL DEFAULT 'ATTACHMENT' COMMENT '账单获取方式：ATTACHMENT-邮件附件，BODY_LINK-正文链接',
|
||||
link_pattern VARCHAR(500) COMMENT '正文链接匹配规则（正则，大小写不敏感），空表示自动选取',
|
||||
archive_password VARCHAR(100) COMMENT '该来源账单压缩包解压密码，可为空',
|
||||
expense_category_id BIGINT COMMENT '默认支出分类ID，可为空',
|
||||
income_category_id BIGINT COMMENT '默认收入分类ID，可为空',
|
||||
|
||||
@@ -339,6 +339,8 @@ CREATE TABLE IF NOT EXISTS t_email_bill_sender_rule (
|
||||
sort_order INTEGER NOT NULL DEFAULT 0,
|
||||
sender_patterns TEXT,
|
||||
bill_type VARCHAR(20) NOT NULL,
|
||||
fetch_mode VARCHAR(20) NOT NULL DEFAULT 'ATTACHMENT',
|
||||
link_pattern VARCHAR(500),
|
||||
archive_password VARCHAR(100),
|
||||
expense_category_id INTEGER,
|
||||
income_category_id INTEGER,
|
||||
|
||||
Reference in New Issue
Block a user