feat: AI 对话附件上下文解析 + 历史会话归档 + 多页面 UI 重构
- 后端: 新增 AttachmentContextBuilder 解析图片/PDF 摘要并拼入 LLM 上下文; ai_chat_endpoints 扩展附件接口; 新增 ReportAnalysisService - 前端: 新增历史会话页与 conversation_history_provider; chat 链路支持附件展示与回放 - UI: 重构 medication_checkin / notification_center / profile / health_drawer 等多页面 - 配置: api_client baseUrl 适配当前 WiFi IP
This commit is contained in:
179
backend/src/Health.Infrastructure/AI/AttachmentContextBuilder.cs
Normal file
179
backend/src/Health.Infrastructure/AI/AttachmentContextBuilder.cs
Normal file
@@ -0,0 +1,179 @@
|
||||
using System.Text.Json;
|
||||
using Health.Application.AI;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using UglyToad.PdfPig;
|
||||
|
||||
namespace Health.Infrastructure.AI;
|
||||
|
||||
public sealed class AttachmentContextBuilder(
|
||||
VisionClient vision,
|
||||
ILogger<AttachmentContextBuilder> logger) : IAttachmentContextBuilder
|
||||
{
|
||||
private const int MaxPdfChars = 6000;
|
||||
private static readonly JsonSerializerOptions JsonOpts = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
PropertyNameCaseInsensitive = true,
|
||||
};
|
||||
|
||||
private readonly VisionClient _vision = vision;
|
||||
private readonly ILogger<AttachmentContextBuilder> _logger = logger;
|
||||
|
||||
public async Task<AttachmentContext?> BuildAsync(string? imageUrl, string? pdfUrl, CancellationToken ct)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(imageUrl))
|
||||
return await BuildImageAsync(imageUrl!, ct);
|
||||
if (!string.IsNullOrWhiteSpace(pdfUrl))
|
||||
return await BuildPdfAsync(pdfUrl!, ct);
|
||||
return null;
|
||||
}
|
||||
|
||||
// ── 图片:调 VLM 输出结构化 JSON ──
|
||||
private async Task<AttachmentContext?> BuildImageAsync(string imageUrl, CancellationToken ct)
|
||||
{
|
||||
var filePath = ResolveLocalPath(imageUrl);
|
||||
if (filePath == null || !File.Exists(filePath))
|
||||
{
|
||||
_logger.LogWarning("Image file not found for {Url}", imageUrl);
|
||||
return new AttachmentContext("image", null, null, "图片暂时无法读取", "[图片附件读取失败,请描述图片内容]");
|
||||
}
|
||||
|
||||
var bytes = await File.ReadAllBytesAsync(filePath, ct);
|
||||
var mime = Path.GetExtension(filePath).ToLowerInvariant() switch
|
||||
{
|
||||
".png" => "image/png",
|
||||
".webp" => "image/webp",
|
||||
".heic" => "image/heic",
|
||||
_ => "image/jpeg",
|
||||
};
|
||||
var dataUrl = $"data:{mime};base64,{Convert.ToBase64String(bytes)}";
|
||||
|
||||
var prompt = """
|
||||
识别图片内容,输出 JSON:
|
||||
{"category":"food|report|wound|drug|chart|other","summary":"1-2句话描述图片","details":["关键信息1","关键信息2"]}
|
||||
|
||||
分类标准:
|
||||
- food:任何食物、饮品、餐食
|
||||
- report:医学检查报告、化验单、影像报告(X光/CT/MRI/B超截图等)
|
||||
- wound:伤口、术后切口、皮肤异常
|
||||
- drug:药品包装、药品说明书
|
||||
- chart:心电图、监护仪截图等
|
||||
- other:其他
|
||||
|
||||
details 给出最有价值的细节(食物种类、报告关键值、伤口部位等)。
|
||||
只输出 JSON 本身,不要任何前后缀。
|
||||
""";
|
||||
|
||||
string? raw = null;
|
||||
try
|
||||
{
|
||||
var resp = await _vision.VisionAsync(prompt, [dataUrl], userText: null, maxTokens: 1024, ct: ct);
|
||||
raw = resp.Choices?.FirstOrDefault()?.Message?.Content;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "VLM call failed for image {Url}", imageUrl);
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(raw))
|
||||
return new AttachmentContext("image", null, null, "图片识别失败", "[图片附件识别失败,请简要描述图片内容]");
|
||||
|
||||
try
|
||||
{
|
||||
var cleaned = StripCodeFence(raw.Trim());
|
||||
using var doc = JsonDocument.Parse(cleaned);
|
||||
var root = doc.RootElement;
|
||||
var category = GetString(root, "category") ?? "other";
|
||||
var summary = GetString(root, "summary") ?? "图片内容";
|
||||
var details = root.TryGetProperty("details", out var d) && d.ValueKind == JsonValueKind.Array
|
||||
? string.Join("、", d.EnumerateArray().Select(x => x.GetString()).Where(s => !string.IsNullOrWhiteSpace(s)))
|
||||
: "";
|
||||
|
||||
var compact = string.IsNullOrEmpty(details)
|
||||
? $"图片识别:{summary}"
|
||||
: $"图片识别:{summary}({details})";
|
||||
var llmContent = $"[图片识别(类别 {category}):{summary}{(string.IsNullOrEmpty(details) ? "" : $",包含 {details}")}]";
|
||||
|
||||
return new AttachmentContext("image", category, null, compact, llmContent);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to parse VLM JSON: {Raw}", raw);
|
||||
// 兜底:原文也能用
|
||||
return new AttachmentContext("image", null, null, $"图片识别:{raw[..Math.Min(raw.Length, 80)]}", $"[图片识别:{raw}]");
|
||||
}
|
||||
}
|
||||
|
||||
// ── PDF:PdfPig 抽取文本 ──
|
||||
private Task<AttachmentContext?> BuildPdfAsync(string pdfUrl, CancellationToken ct)
|
||||
{
|
||||
var filePath = ResolveLocalPath(pdfUrl);
|
||||
var fileName = Path.GetFileName(pdfUrl);
|
||||
if (filePath == null || !File.Exists(filePath))
|
||||
{
|
||||
_logger.LogWarning("PDF file not found for {Url}", pdfUrl);
|
||||
return Task.FromResult<AttachmentContext?>(new AttachmentContext(
|
||||
"pdf", null, fileName, "PDF 文件读取失败",
|
||||
"[PDF 附件读取失败,请描述文档内容]"));
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using var pdf = PdfDocument.Open(filePath);
|
||||
var sb = new System.Text.StringBuilder();
|
||||
foreach (var page in pdf.GetPages())
|
||||
{
|
||||
sb.AppendLine(page.Text);
|
||||
if (sb.Length > MaxPdfChars) break;
|
||||
}
|
||||
var text = sb.ToString().Trim();
|
||||
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
return Task.FromResult<AttachmentContext?>(new AttachmentContext(
|
||||
"pdf", null, fileName, "PDF 内容为空或扫描件",
|
||||
"[PDF 看起来是扫描件或图片版,无法提取文字。请提示用户改用拍照上传,或简要描述报告内容]"));
|
||||
}
|
||||
|
||||
var truncated = text.Length > MaxPdfChars ? text[..MaxPdfChars] + "...(已截断)" : text;
|
||||
var summary = $"PDF:{fileName}";
|
||||
var llmContent = $"[用户上传 PDF 「{fileName}」,文档内容如下]\n{truncated}";
|
||||
|
||||
return Task.FromResult<AttachmentContext?>(new AttachmentContext("pdf", null, fileName, summary, llmContent));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to parse PDF {Url}", pdfUrl);
|
||||
return Task.FromResult<AttachmentContext?>(new AttachmentContext(
|
||||
"pdf", null, fileName, "PDF 解析异常",
|
||||
"[PDF 解析失败,请提示用户文件可能损坏或加密]"));
|
||||
}
|
||||
}
|
||||
|
||||
private static string? ResolveLocalPath(string url)
|
||||
{
|
||||
// url 形如 "/uploads/{guid}.{ext}"。处理 base URL 前缀也兼容。
|
||||
var idx = url.IndexOf("/uploads/", StringComparison.Ordinal);
|
||||
if (idx < 0) return null;
|
||||
var relative = url[(idx + "/uploads/".Length)..];
|
||||
// 去掉可能的 query string
|
||||
var q = relative.IndexOf('?');
|
||||
if (q >= 0) relative = relative[..q];
|
||||
return Path.Combine(Directory.GetCurrentDirectory(), "uploads", relative);
|
||||
}
|
||||
|
||||
private static string StripCodeFence(string raw)
|
||||
{
|
||||
var t = raw.Trim();
|
||||
if (t.StartsWith("```"))
|
||||
{
|
||||
var firstNewline = t.IndexOf('\n');
|
||||
if (firstNewline > 0) t = t[(firstNewline + 1)..];
|
||||
if (t.EndsWith("```")) t = t[..^3];
|
||||
}
|
||||
return t.Trim();
|
||||
}
|
||||
|
||||
private static string? GetString(JsonElement el, string name) =>
|
||||
el.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.String ? v.GetString() : null;
|
||||
}
|
||||
@@ -23,9 +23,21 @@ public sealed class PromptManager
|
||||
_ => DefaultPrompt
|
||||
};
|
||||
|
||||
return $"{prompt}\n\n{MedicalBoundaryRules}";
|
||||
return $"{prompt}\n\n{MedicalBoundaryRules}\n\n{SmartLinkRules}";
|
||||
}
|
||||
|
||||
private const string SmartLinkRules = """
|
||||
智能跳转链接(按需嵌入):
|
||||
- 当用户明确询问食物的 热量/卡路里 时,在回答的自然位置嵌入 Markdown 链接 [热量分析](app://diet),引导用户使用专门的饮食拍照分析功能。
|
||||
- 当用户上传医学检查报告、化验单、影像报告、出院小结等需要专业解读时,在回答末尾嵌入 [报告分析](app://report),引导上传到报告分析功能获得详细解读。
|
||||
- 当用户询问血压、血糖、血氧等需要长期追踪的设备测量场景时,可嵌入 [蓝牙设备](app://device)。
|
||||
|
||||
硬性约束:
|
||||
- 每条回复最多嵌入一个跳转链接,且必须在合适语境下自然提及,不要堆砌。
|
||||
- 仅"问热量/卡路里"才用 app://diet;"营养、能不能吃、是否健康"类问题正常回答,禁止插入饮食跳转链接。
|
||||
- 链接的 Markdown 文本只能用上面列出的 4 个固定文案,不要自创其他文案。
|
||||
""";
|
||||
|
||||
private const string MedicalBoundaryRules = """
|
||||
医疗边界(必须遵守):
|
||||
- 你的定位是患者端 AI 健康解释与预问诊助手,不是医生,不能替代医生诊断、处方或治疗决策。
|
||||
|
||||
@@ -31,9 +31,38 @@ public sealed class DietImageAnalyzer(VisionClient vision) : IDietImageAnalyzer
|
||||
}
|
||||
|
||||
var prompt = """
|
||||
识别图片中的食物和饮品,返回JSON数组:
|
||||
[{"name":"名称","portion":"份量","calories":热量}]
|
||||
只返回JSON,不要其他内容。
|
||||
你是营养师助手。请识别图片中所有食物和饮品,估算每项的具体克数或毫升数,输出 JSON 数组。
|
||||
|
||||
## 无食物判定(必读)
|
||||
如果图片中完全没有可食用的食物或饮品(例如是风景、人物、宠物、文档、屏幕截图、纯背景、空盘子等),**直接输出空数组 []**,不要硬编造食物。
|
||||
|
||||
## 输出格式
|
||||
[{"name":"食物名","portion":"具体克数","calories":数字}]
|
||||
|
||||
## 份量估算依据(容器与参照物对照表)
|
||||
- 标准饭碗(口径11cm):满碗米饭≈200g、满碗面条≈250g、满碗粥≈250g、半碗≈100g
|
||||
- 标准汤碗(口径14cm):满碗汤≈400ml、半碗≈200ml
|
||||
- 标准盘子(口径23cm):满盘菜≈300g、半盘≈150g
|
||||
- 普通玻璃杯:满杯水/饮料≈250ml、纸杯≈200ml
|
||||
- 鸡蛋1个≈50g、馒头1个≈100g、包子1个≈80g、饺子1个≈15g
|
||||
- 香蕉1根≈100g、苹果1个≈200g、橘子1个≈120g
|
||||
- 肉块巴掌大小≈80-120g、鸡腿1个≈100g、鱼1条(中等)≈300g
|
||||
|
||||
## 输出示例(务必模仿此风格)
|
||||
正确:{"name":"米饭","portion":"约200g","calories":230}
|
||||
正确:{"name":"红烧肉","portion":"约150g","calories":420}
|
||||
正确:{"name":"豆浆","portion":"约250ml","calories":80}
|
||||
错误:{"name":"米饭","portion":"一碗","calories":230} ← portion 缺克数
|
||||
错误:{"name":"汤","portion":"一份","calories":80} ← portion 缺毫升数
|
||||
错误:{"name":"水果","portion":"少许","calories":50} ← portion 是模糊词
|
||||
|
||||
## 硬性要求
|
||||
1. 图片若无食物,输出 [] 即可。不要硬凑食物。
|
||||
2. portion 字段必须包含阿拉伯数字 + 单位(g 或 ml)。不允许任何只有量词没有克数的回答。
|
||||
3. 即使图片不够清楚,也要根据可见容器/参照物给出估算,可加"约"或区间("约80-120g")。
|
||||
4. calories 是数字(千卡),不带单位、不带引号。
|
||||
|
||||
只输出 JSON 数组本身,不要任何前缀、后缀、代码块标记或说明文字。
|
||||
""";
|
||||
var response = await _vision.VisionAsync(prompt, imageUrls, userText: null, maxTokens: 8192, ct: ct);
|
||||
var result = response.Choices?.FirstOrDefault()?.Message?.Content ?? "[]";
|
||||
|
||||
@@ -7,11 +7,13 @@
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="10.0.8" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="10.0.8" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="10.0.8" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.8" />
|
||||
<PackageReference Include="Microsoft.IdentityModel.Tokens" Version="8.18.0" />
|
||||
<PackageReference Include="Minio" Version="7.0.0" />
|
||||
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="10.0.2" />
|
||||
<PackageReference Include="PdfPig" Version="0.1.16-alpha-20260629-34229" />
|
||||
<PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="8.18.0" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ using Health.Application.Reports;
|
||||
using Health.Application.Notifications;
|
||||
using Health.Infrastructure.AI;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using UglyToad.PdfPig;
|
||||
|
||||
namespace Health.Infrastructure.Reports;
|
||||
|
||||
@@ -17,6 +18,7 @@ public sealed class ReportAnalysisService(
|
||||
private readonly DeepSeekClient _llm = llm;
|
||||
private readonly IUserNotificationProducer _notifications = notifications;
|
||||
private readonly ILogger<ReportAnalysisService> _logger = logger;
|
||||
private const int MaxPdfChars = 8000;
|
||||
|
||||
public async Task AnalyzeAsync(ReportAnalysisJob job, CancellationToken ct)
|
||||
{
|
||||
@@ -72,6 +74,9 @@ public sealed class ReportAnalysisService(
|
||||
|
||||
private async Task<JsonElement> ExtractIndicatorsAsync(string filePath, CancellationToken ct)
|
||||
{
|
||||
if (Path.GetExtension(filePath).Equals(".pdf", StringComparison.OrdinalIgnoreCase))
|
||||
return await ExtractIndicatorsFromPdfAsync(filePath, ct);
|
||||
|
||||
var imageUrl = await GetLocalImageUrl(filePath, ct);
|
||||
if (imageUrl == null)
|
||||
throw new InvalidOperationException("报告图片文件不存在或无法读取");
|
||||
@@ -112,6 +117,66 @@ public sealed class ReportAnalysisService(
|
||||
return json ?? throw new InvalidOperationException("报告识别结果格式异常");
|
||||
}
|
||||
|
||||
private async Task<JsonElement> ExtractIndicatorsFromPdfAsync(string filePath, CancellationToken ct)
|
||||
{
|
||||
var text = ExtractPdfText(filePath);
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
return JsonDocument.Parse("""
|
||||
{"isReport": false, "reason": "这份 PDF 可能是扫描件或图片版,当前无法直接提取文字。请上传清晰图片,或后续启用 PDF 页转图片后再用 VLM 识别。"}
|
||||
""").RootElement;
|
||||
}
|
||||
|
||||
var prompt = $$"""
|
||||
你是一名医学检验报告分析专家。下面是用户上传 PDF 中提取出的文字。
|
||||
|
||||
PDF 文本:
|
||||
{{text}}
|
||||
|
||||
第一步:判断这到底是不是一份医学检验报告。
|
||||
如果不是,直接返回:
|
||||
{"isReport": false, "reason": "这不是医学报告,原因:..."}
|
||||
|
||||
如果是医学报告,提取所有检测指标,返回严格 JSON 格式(不要 markdown 包裹):
|
||||
{
|
||||
"isReport": true,
|
||||
"reportType": "BloodTest/Biochemistry/Ecg/Ultrasound/Discharge/Other",
|
||||
"indicators": [
|
||||
{"name": "指标名称", "value": "数值", "unit": "单位", "referenceRange": "参考范围", "status": "normal/high/low"}
|
||||
]
|
||||
}
|
||||
|
||||
注意:
|
||||
- 只提取报告里真实出现的指标
|
||||
- status 判断:数值在参考范围内=normal,超出上限=high,低于下限=low
|
||||
- 不确定的参考范围不要编造,可留空
|
||||
""";
|
||||
|
||||
var messages = new List<ChatMessage>
|
||||
{
|
||||
new() { Role = "system", Content = "你是医学报告结构化抽取助手,只输出严格 JSON。" },
|
||||
new() { Role = "user", Content = prompt }
|
||||
};
|
||||
|
||||
var response = await _llm.ChatAsync(messages, maxTokens: 1800, temperature: 0.1f, ct: ct);
|
||||
var content = response.Choices?.FirstOrDefault()?.Message?.Content ?? "";
|
||||
var json = TryParseJson(content);
|
||||
return json ?? throw new InvalidOperationException("PDF 报告解析结果格式异常");
|
||||
}
|
||||
|
||||
private static string ExtractPdfText(string filePath)
|
||||
{
|
||||
using var pdf = PdfDocument.Open(filePath);
|
||||
var sb = new System.Text.StringBuilder();
|
||||
foreach (var page in pdf.GetPages())
|
||||
{
|
||||
sb.AppendLine(page.Text);
|
||||
if (sb.Length > MaxPdfChars) break;
|
||||
}
|
||||
var text = sb.ToString().Trim();
|
||||
return text.Length > MaxPdfChars ? text[..MaxPdfChars] : text;
|
||||
}
|
||||
|
||||
private async Task<string> GenerateSummaryAsync(string indicatorsJson, CancellationToken ct)
|
||||
{
|
||||
var prompt = $"""
|
||||
|
||||
Reference in New Issue
Block a user