feat: AI 对话附件上下文解析 + 历史会话归档 + 多页面 UI 重构

- 后端: 新增 AttachmentContextBuilder 解析图片/PDF 摘要并拼入 LLM 上下文; ai_chat_endpoints 扩展附件接口; 新增 ReportAnalysisService
- 前端: 新增历史会话页与 conversation_history_provider; chat 链路支持附件展示与回放
- UI: 重构 medication_checkin / notification_center / profile / health_drawer 等多页面
- 配置: api_client baseUrl 适配当前 WiFi IP
This commit is contained in:
MingNian
2026-07-06 12:44:59 +08:00
parent 4507083f3f
commit 7a93237069
38 changed files with 4165 additions and 1157 deletions

View File

@@ -0,0 +1,179 @@
using System.Text.Json;
using Health.Application.AI;
using Microsoft.Extensions.Logging;
using UglyToad.PdfPig;
namespace Health.Infrastructure.AI;
public sealed class AttachmentContextBuilder(
VisionClient vision,
ILogger<AttachmentContextBuilder> logger) : IAttachmentContextBuilder
{
private const int MaxPdfChars = 6000;
private static readonly JsonSerializerOptions JsonOpts = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
PropertyNameCaseInsensitive = true,
};
private readonly VisionClient _vision = vision;
private readonly ILogger<AttachmentContextBuilder> _logger = logger;
public async Task<AttachmentContext?> BuildAsync(string? imageUrl, string? pdfUrl, CancellationToken ct)
{
if (!string.IsNullOrWhiteSpace(imageUrl))
return await BuildImageAsync(imageUrl!, ct);
if (!string.IsNullOrWhiteSpace(pdfUrl))
return await BuildPdfAsync(pdfUrl!, ct);
return null;
}
// ── 图片:调 VLM 输出结构化 JSON ──
private async Task<AttachmentContext?> BuildImageAsync(string imageUrl, CancellationToken ct)
{
var filePath = ResolveLocalPath(imageUrl);
if (filePath == null || !File.Exists(filePath))
{
_logger.LogWarning("Image file not found for {Url}", imageUrl);
return new AttachmentContext("image", null, null, "图片暂时无法读取", "[图片附件读取失败,请描述图片内容]");
}
var bytes = await File.ReadAllBytesAsync(filePath, ct);
var mime = Path.GetExtension(filePath).ToLowerInvariant() switch
{
".png" => "image/png",
".webp" => "image/webp",
".heic" => "image/heic",
_ => "image/jpeg",
};
var dataUrl = $"data:{mime};base64,{Convert.ToBase64String(bytes)}";
var prompt = """
JSON
{"category":"food|report|wound|drug|chart|other","summary":"1-2句话描述图片","details":["关键信息1","关键信息2"]}
- food
- reportX光/CT/MRI/B超截图等
- wound
- drug
- chart
- other
details
JSON
""";
string? raw = null;
try
{
var resp = await _vision.VisionAsync(prompt, [dataUrl], userText: null, maxTokens: 1024, ct: ct);
raw = resp.Choices?.FirstOrDefault()?.Message?.Content;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "VLM call failed for image {Url}", imageUrl);
}
if (string.IsNullOrWhiteSpace(raw))
return new AttachmentContext("image", null, null, "图片识别失败", "[图片附件识别失败,请简要描述图片内容]");
try
{
var cleaned = StripCodeFence(raw.Trim());
using var doc = JsonDocument.Parse(cleaned);
var root = doc.RootElement;
var category = GetString(root, "category") ?? "other";
var summary = GetString(root, "summary") ?? "图片内容";
var details = root.TryGetProperty("details", out var d) && d.ValueKind == JsonValueKind.Array
? string.Join("、", d.EnumerateArray().Select(x => x.GetString()).Where(s => !string.IsNullOrWhiteSpace(s)))
: "";
var compact = string.IsNullOrEmpty(details)
? $"图片识别:{summary}"
: $"图片识别:{summary}{details}";
var llmContent = $"[图片识别(类别 {category}{summary}{(string.IsNullOrEmpty(details) ? "" : $" {details}")}]";
return new AttachmentContext("image", category, null, compact, llmContent);
}
catch (JsonException ex)
{
_logger.LogWarning(ex, "Failed to parse VLM JSON: {Raw}", raw);
// 兜底:原文也能用
return new AttachmentContext("image", null, null, $"图片识别:{raw[..Math.Min(raw.Length, 80)]}", $"[图片识别:{raw}]");
}
}
// ── PDFPdfPig 抽取文本 ──
private Task<AttachmentContext?> BuildPdfAsync(string pdfUrl, CancellationToken ct)
{
var filePath = ResolveLocalPath(pdfUrl);
var fileName = Path.GetFileName(pdfUrl);
if (filePath == null || !File.Exists(filePath))
{
_logger.LogWarning("PDF file not found for {Url}", pdfUrl);
return Task.FromResult<AttachmentContext?>(new AttachmentContext(
"pdf", null, fileName, "PDF 文件读取失败",
"[PDF 附件读取失败,请描述文档内容]"));
}
try
{
using var pdf = PdfDocument.Open(filePath);
var sb = new System.Text.StringBuilder();
foreach (var page in pdf.GetPages())
{
sb.AppendLine(page.Text);
if (sb.Length > MaxPdfChars) break;
}
var text = sb.ToString().Trim();
if (string.IsNullOrWhiteSpace(text))
{
return Task.FromResult<AttachmentContext?>(new AttachmentContext(
"pdf", null, fileName, "PDF 内容为空或扫描件",
"[PDF 看起来是扫描件或图片版,无法提取文字。请提示用户改用拍照上传,或简要描述报告内容]"));
}
var truncated = text.Length > MaxPdfChars ? text[..MaxPdfChars] + "...(已截断)" : text;
var summary = $"PDF{fileName}";
var llmContent = $"[用户上传 PDF 「{fileName}」,文档内容如下]\n{truncated}";
return Task.FromResult<AttachmentContext?>(new AttachmentContext("pdf", null, fileName, summary, llmContent));
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to parse PDF {Url}", pdfUrl);
return Task.FromResult<AttachmentContext?>(new AttachmentContext(
"pdf", null, fileName, "PDF 解析异常",
"[PDF 解析失败,请提示用户文件可能损坏或加密]"));
}
}
private static string? ResolveLocalPath(string url)
{
// url 形如 "/uploads/{guid}.{ext}"。处理 base URL 前缀也兼容。
var idx = url.IndexOf("/uploads/", StringComparison.Ordinal);
if (idx < 0) return null;
var relative = url[(idx + "/uploads/".Length)..];
// 去掉可能的 query string
var q = relative.IndexOf('?');
if (q >= 0) relative = relative[..q];
return Path.Combine(Directory.GetCurrentDirectory(), "uploads", relative);
}
private static string StripCodeFence(string raw)
{
var t = raw.Trim();
if (t.StartsWith("```"))
{
var firstNewline = t.IndexOf('\n');
if (firstNewline > 0) t = t[(firstNewline + 1)..];
if (t.EndsWith("```")) t = t[..^3];
}
return t.Trim();
}
private static string? GetString(JsonElement el, string name) =>
el.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.String ? v.GetString() : null;
}

View File

@@ -23,9 +23,21 @@ public sealed class PromptManager
_ => DefaultPrompt
};
return $"{prompt}\n\n{MedicalBoundaryRules}";
return $"{prompt}\n\n{MedicalBoundaryRules}\n\n{SmartLinkRules}";
}
private const string SmartLinkRules = """
- / Markdown [](app://diet),引导用户使用专门的饮食拍照分析功能。
- [](app://report),引导上传到报告分析功能获得详细解读。
- [](app://device)。
-
- "问热量/卡路里" app://diet"营养、能不能吃、是否健康"类问题正常回答,禁止插入饮食跳转链接。
- Markdown 4
""";
private const string MedicalBoundaryRules = """
- AI

View File

@@ -31,9 +31,38 @@ public sealed class DietImageAnalyzer(VisionClient vision) : IDietImageAnalyzer
}
var prompt = """
JSON数
[{"name":"名称","portion":"份量","calories":热量}]
JSON
JSON
##
** []**
##
[{"name":"食物名","portion":"具体克数","calories":数字}]
##
- (11cm)200g250g250g100g
- (14cm)400ml200ml
- (23cm)300g150g
- /250ml200ml
- 150g1100g180g115g
- 1100g1200g1120g
- 80-120g1100g1()300g
## 仿
{"name":"米饭","portion":"约200g","calories":230}
{"name":"红烧肉","portion":"约150g","calories":420}
{"name":"豆浆","portion":"约250ml","calories":80}
{"name":"米饭","portion":"一碗","calories":230} portion
{"name":"汤","portion":"一份","calories":80} portion
{"name":"水果","portion":"少许","calories":50} portion
##
1. []
2. portion + (g ml)
3. 使/"约"("约80-120g")
4. calories
JSON
""";
var response = await _vision.VisionAsync(prompt, imageUrls, userText: null, maxTokens: 8192, ct: ct);
var result = response.Choices?.FirstOrDefault()?.Message?.Content ?? "[]";

View File

@@ -7,11 +7,13 @@
<ItemGroup>
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="10.0.8" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="10.0.8" />
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="10.0.8" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.8" />
<PackageReference Include="Microsoft.IdentityModel.Tokens" Version="8.18.0" />
<PackageReference Include="Minio" Version="7.0.0" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="10.0.2" />
<PackageReference Include="PdfPig" Version="0.1.16-alpha-20260629-34229" />
<PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="8.18.0" />
</ItemGroup>

View File

@@ -2,6 +2,7 @@ using Health.Application.Reports;
using Health.Application.Notifications;
using Health.Infrastructure.AI;
using Microsoft.Extensions.Logging;
using UglyToad.PdfPig;
namespace Health.Infrastructure.Reports;
@@ -17,6 +18,7 @@ public sealed class ReportAnalysisService(
private readonly DeepSeekClient _llm = llm;
private readonly IUserNotificationProducer _notifications = notifications;
private readonly ILogger<ReportAnalysisService> _logger = logger;
private const int MaxPdfChars = 8000;
public async Task AnalyzeAsync(ReportAnalysisJob job, CancellationToken ct)
{
@@ -72,6 +74,9 @@ public sealed class ReportAnalysisService(
private async Task<JsonElement> ExtractIndicatorsAsync(string filePath, CancellationToken ct)
{
if (Path.GetExtension(filePath).Equals(".pdf", StringComparison.OrdinalIgnoreCase))
return await ExtractIndicatorsFromPdfAsync(filePath, ct);
var imageUrl = await GetLocalImageUrl(filePath, ct);
if (imageUrl == null)
throw new InvalidOperationException("报告图片文件不存在或无法读取");
@@ -112,6 +117,66 @@ public sealed class ReportAnalysisService(
return json ?? throw new InvalidOperationException("报告识别结果格式异常");
}
private async Task<JsonElement> ExtractIndicatorsFromPdfAsync(string filePath, CancellationToken ct)
{
var text = ExtractPdfText(filePath);
if (string.IsNullOrWhiteSpace(text))
{
return JsonDocument.Parse("""
{"isReport": false, "reason": "这份 PDF 可能是扫描件或图片版,当前无法直接提取文字。请上传清晰图片,或后续启用 PDF 页转图片后再用 VLM 识别。"}
""").RootElement;
}
var prompt = $$"""
你是一名医学检验报告分析专家。下面是用户上传 PDF 中提取出的文字。
PDF 文本:
{{text}}
第一步:判断这到底是不是一份医学检验报告。
如果不是,直接返回:
{"isReport": false, "reason": "..."}
JSON markdown
{
"isReport": true,
"reportType": "BloodTest/Biochemistry/Ecg/Ultrasound/Discharge/Other",
"indicators": [
{"name": "指标名称", "value": "数值", "unit": "单位", "referenceRange": "参考范围", "status": "normal/high/low"}
]
}
-
- status =normal=high=low
-
""";
var messages = new List<ChatMessage>
{
new() { Role = "system", Content = "你是医学报告结构化抽取助手,只输出严格 JSON。" },
new() { Role = "user", Content = prompt }
};
var response = await _llm.ChatAsync(messages, maxTokens: 1800, temperature: 0.1f, ct: ct);
var content = response.Choices?.FirstOrDefault()?.Message?.Content ?? "";
var json = TryParseJson(content);
return json ?? throw new InvalidOperationException("PDF 报告解析结果格式异常");
}
private static string ExtractPdfText(string filePath)
{
using var pdf = PdfDocument.Open(filePath);
var sb = new System.Text.StringBuilder();
foreach (var page in pdf.GetPages())
{
sb.AppendLine(page.Text);
if (sb.Length > MaxPdfChars) break;
}
var text = sb.ToString().Trim();
return text.Length > MaxPdfChars ? text[..MaxPdfChars] : text;
}
private async Task<string> GenerateSummaryAsync(string indicatorsJson, CancellationToken ct)
{
var prompt = $"""