feat: AI 对话附件上下文解析 + 历史会话归档 + 多页面 UI 重构
- 后端: 新增 AttachmentContextBuilder 解析图片/PDF 摘要并拼入 LLM 上下文; ai_chat_endpoints 扩展附件接口; 新增 ReportAnalysisService - 前端: 新增历史会话页与 conversation_history_provider; chat 链路支持附件展示与回放 - UI: 重构 medication_checkin / notification_center / profile / health_drawer 等多页面 - 配置: api_client baseUrl 适配当前 WiFi IP
This commit is contained in:
MingNian
179
backend/src/Health.Infrastructure/AI/AttachmentContextBuilder.cs
Normal file
179
backend/src/Health.Infrastructure/AI/AttachmentContextBuilder.cs
Normal file
@@ -0,0 +1,179 @@
|
||||
using System.Text.Json;
|
||||
using Health.Application.AI;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using UglyToad.PdfPig;
|
||||
|
||||
namespace Health.Infrastructure.AI;
|
||||
|
||||
public sealed class AttachmentContextBuilder(
|
||||
VisionClient vision,
|
||||
ILogger<AttachmentContextBuilder> logger) : IAttachmentContextBuilder
|
||||
{
|
||||
private const int MaxPdfChars = 6000;
|
||||
private static readonly JsonSerializerOptions JsonOpts = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
PropertyNameCaseInsensitive = true,
|
||||
};
|
||||
|
||||
private readonly VisionClient _vision = vision;
|
||||
private readonly ILogger<AttachmentContextBuilder> _logger = logger;
|
||||
|
||||
public async Task<AttachmentContext?> BuildAsync(string? imageUrl, string? pdfUrl, CancellationToken ct)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(imageUrl))
|
||||
return await BuildImageAsync(imageUrl!, ct);
|
||||
if (!string.IsNullOrWhiteSpace(pdfUrl))
|
||||
return await BuildPdfAsync(pdfUrl!, ct);
|
||||
return null;
|
||||
}
|
||||
|
||||
// ── 图片：调 VLM 输出结构化 JSON ──
|
||||
private async Task<AttachmentContext?> BuildImageAsync(string imageUrl, CancellationToken ct)
|
||||
{
|
||||
var filePath = ResolveLocalPath(imageUrl);
|
||||
if (filePath == null || !File.Exists(filePath))
|
||||
{
|
||||
_logger.LogWarning("Image file not found for {Url}", imageUrl);
|
||||
return new AttachmentContext("image", null, null, "图片暂时无法读取", "[图片附件读取失败，请描述图片内容]");
|
||||
}
|
||||
|
||||
var bytes = await File.ReadAllBytesAsync(filePath, ct);
|
||||
var mime = Path.GetExtension(filePath).ToLowerInvariant() switch
|
||||
{
|
||||
".png" => "image/png",
|
||||
".webp" => "image/webp",
|
||||
".heic" => "image/heic",
|
||||
_ => "image/jpeg",
|
||||
};
|
||||
var dataUrl = $"data:{mime};base64,{Convert.ToBase64String(bytes)}";
|
||||
|
||||
var prompt = """
|
||||
识别图片内容，输出 JSON：
|
||||
{"category":"food|report|wound|drug|chart|other","summary":"1-2句话描述图片","details":["关键信息1","关键信息2"]}
|
||||
|
||||
分类标准：
|
||||
- food：任何食物、饮品、餐食
|
||||
- report：医学检查报告、化验单、影像报告（X光/CT/MRI/B超截图等）
|
||||
- wound：伤口、术后切口、皮肤异常
|
||||
- drug：药品包装、药品说明书
|
||||
- chart：心电图、监护仪截图等
|
||||
- other：其他
|
||||
|
||||
details 给出最有价值的细节（食物种类、报告关键值、伤口部位等）。
|
||||
只输出 JSON 本身，不要任何前后缀。
|
||||
""";
|
||||
|
||||
string? raw = null;
|
||||
try
|
||||
{
|
||||
var resp = await _vision.VisionAsync(prompt, [dataUrl], userText: null, maxTokens: 1024, ct: ct);
|
||||
raw = resp.Choices?.FirstOrDefault()?.Message?.Content;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "VLM call failed for image {Url}", imageUrl);
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(raw))
|
||||
return new AttachmentContext("image", null, null, "图片识别失败", "[图片附件识别失败，请简要描述图片内容]");
|
||||
|
||||
try
|
||||
{
|
||||
var cleaned = StripCodeFence(raw.Trim());
|
||||
using var doc = JsonDocument.Parse(cleaned);
|
||||
var root = doc.RootElement;
|
||||
var category = GetString(root, "category") ?? "other";
|
||||
var summary = GetString(root, "summary") ?? "图片内容";
|
||||
var details = root.TryGetProperty("details", out var d) && d.ValueKind == JsonValueKind.Array
|
||||
? string.Join("、", d.EnumerateArray().Select(x => x.GetString()).Where(s => !string.IsNullOrWhiteSpace(s)))
|
||||
: "";
|
||||
|
||||
var compact = string.IsNullOrEmpty(details)
|
||||
? $"图片识别：{summary}"
|
||||
: $"图片识别：{summary}（{details}）";
|
||||
var llmContent = $"[图片识别（类别 {category}）：{summary}{(string.IsNullOrEmpty(details) ? "" : $"，包含 {details}")}]";
|
||||
|
||||
return new AttachmentContext("image", category, null, compact, llmContent);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to parse VLM JSON: {Raw}", raw);
|
||||
// 兜底：原文也能用
|
||||
return new AttachmentContext("image", null, null, $"图片识别：{raw[..Math.Min(raw.Length, 80)]}", $"[图片识别：{raw}]");
|
||||
}
|
||||
}
|
||||
|
||||
// ── PDF：PdfPig 抽取文本 ──
|
||||
private Task<AttachmentContext?> BuildPdfAsync(string pdfUrl, CancellationToken ct)
|
||||
{
|
||||
var filePath = ResolveLocalPath(pdfUrl);
|
||||
var fileName = Path.GetFileName(pdfUrl);
|
||||
if (filePath == null || !File.Exists(filePath))
|
||||
{
|
||||
_logger.LogWarning("PDF file not found for {Url}", pdfUrl);
|
||||
return Task.FromResult<AttachmentContext?>(new AttachmentContext(
|
||||
"pdf", null, fileName, "PDF 文件读取失败",
|
||||
"[PDF 附件读取失败，请描述文档内容]"));
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using var pdf = PdfDocument.Open(filePath);
|
||||
var sb = new System.Text.StringBuilder();
|
||||
foreach (var page in pdf.GetPages())
|
||||
{
|
||||
sb.AppendLine(page.Text);
|
||||
if (sb.Length > MaxPdfChars) break;
|
||||
}
|
||||
var text = sb.ToString().Trim();
|
||||
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
return Task.FromResult<AttachmentContext?>(new AttachmentContext(
|
||||
"pdf", null, fileName, "PDF 内容为空或扫描件",
|
||||
"[PDF 看起来是扫描件或图片版，无法提取文字。请提示用户改用拍照上传，或简要描述报告内容]"));
|
||||
}
|
||||
|
||||
var truncated = text.Length > MaxPdfChars ? text[..MaxPdfChars] + "...（已截断）" : text;
|
||||
var summary = $"PDF：{fileName}";
|
||||
var llmContent = $"[用户上传 PDF 「{fileName}」，文档内容如下]\n{truncated}";
|
||||
|
||||
return Task.FromResult<AttachmentContext?>(new AttachmentContext("pdf", null, fileName, summary, llmContent));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to parse PDF {Url}", pdfUrl);
|
||||
return Task.FromResult<AttachmentContext?>(new AttachmentContext(
|
||||
"pdf", null, fileName, "PDF 解析异常",
|
||||
"[PDF 解析失败，请提示用户文件可能损坏或加密]"));
|
||||
}
|
||||
}
|
||||
|
||||
private static string? ResolveLocalPath(string url)
|
||||
{
|
||||
// url 形如 "/uploads/{guid}.{ext}"。处理 base URL 前缀也兼容。
|
||||
var idx = url.IndexOf("/uploads/", StringComparison.Ordinal);
|
||||
if (idx < 0) return null;
|
||||
var relative = url[(idx + "/uploads/".Length)..];
|
||||
// 去掉可能的 query string
|
||||
var q = relative.IndexOf('?');
|
||||
if (q >= 0) relative = relative[..q];
|
||||
return Path.Combine(Directory.GetCurrentDirectory(), "uploads", relative);
|
||||
}
|
||||
|
||||
private static string StripCodeFence(string raw)
|
||||
{
|
||||
var t = raw.Trim();
|
||||
if (t.StartsWith("```"))
|
||||
{
|
||||
var firstNewline = t.IndexOf('\n');
|
||||
if (firstNewline > 0) t = t[(firstNewline + 1)..];
|
||||
if (t.EndsWith("```")) t = t[..^3];
|
||||
}
|
||||
return t.Trim();
|
||||
}
|
||||
|
||||
private static string? GetString(JsonElement el, string name) =>
|
||||
el.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.String ? v.GetString() : null;
|
||||
}
|
||||
@@ -23,9 +23,21 @@ public sealed class PromptManager
|
||||
_ => DefaultPrompt
|
||||
};
|
||||
|
||||
return $"{prompt}\n\n{MedicalBoundaryRules}";
|
||||
return $"{prompt}\n\n{MedicalBoundaryRules}\n\n{SmartLinkRules}";
|
||||
}
|
||||
|
||||
private const string SmartLinkRules = """
|
||||
智能跳转链接（按需嵌入）：
|
||||
- 当用户明确询问食物的 热量/卡路里 时，在回答的自然位置嵌入 Markdown 链接 [热量分析](app://diet)，引导用户使用专门的饮食拍照分析功能。
|
||||
- 当用户上传医学检查报告、化验单、影像报告、出院小结等需要专业解读时，在回答末尾嵌入 [报告分析](app://report)，引导上传到报告分析功能获得详细解读。
|
||||
- 当用户询问血压、血糖、血氧等需要长期追踪的设备测量场景时，可嵌入 [蓝牙设备](app://device)。
|
||||
|
||||
硬性约束：
|
||||
- 每条回复最多嵌入一个跳转链接，且必须在合适语境下自然提及，不要堆砌。
|
||||
- 仅"问热量/卡路里"才用 app://diet；"营养、能不能吃、是否健康"类问题正常回答，禁止插入饮食跳转链接。
|
||||
- 链接的 Markdown 文本只能用上面列出的 4 个固定文案，不要自创其他文案。
|
||||
""";
|
||||
|
||||
private const string MedicalBoundaryRules = """
|
||||
医疗边界（必须遵守）：
|
||||
- 你的定位是患者端 AI 健康解释与预问诊助手，不是医生，不能替代医生诊断、处方或治疗决策。
|
||||
|
||||
Reference in New Issue
Block a user