feat: AI 对话附件上下文解析 + 历史会话归档 + 多页面 UI 重构
- 后端: 新增 AttachmentContextBuilder 解析图片/PDF 摘要并拼入 LLM 上下文; ai_chat_endpoints 扩展附件接口; 新增 ReportAnalysisService - 前端: 新增历史会话页与 conversation_history_provider; chat 链路支持附件展示与回放 - UI: 重构 medication_checkin / notification_center / profile / health_drawer 等多页面 - 配置: api_client baseUrl 适配当前 WiFi IP
This commit is contained in:
@@ -28,12 +28,15 @@ public sealed record OpenConversationResult(
|
||||
public interface IAiConversationService
|
||||
{
|
||||
Task<OpenConversationResult> OpenAsync(Guid userId, Guid? conversationId, AgentType agentType, string firstMessage, CancellationToken ct);
|
||||
Task AddUserMessageAsync(Guid conversationId, string content, CancellationToken ct);
|
||||
Task AddUserMessageAsync(Guid conversationId, string content, string? metadataJson, CancellationToken ct);
|
||||
Task AddAssistantMessageAsync(Guid conversationId, string content, CancellationToken ct);
|
||||
Task AddAssistantMessageAsync(Guid conversationId, string content, string? metadataJson, CancellationToken ct);
|
||||
Task<IReadOnlyList<AiConversationMessageDto>> GetRecentMessagesAsync(Guid conversationId, int limit, CancellationToken ct);
|
||||
Task<IReadOnlyList<AiConversationDto>> ListAsync(Guid userId, CancellationToken ct);
|
||||
Task<IReadOnlyList<AiConversationMessageDto>> GetMessagesAsync(Guid userId, Guid conversationId, CancellationToken ct);
|
||||
Task UpdateSummaryAsync(Guid conversationId, string summary, CancellationToken ct);
|
||||
Task DeleteAsync(Guid userId, Guid conversationId, CancellationToken ct);
|
||||
Task<int> DeleteAllAsync(Guid userId, CancellationToken ct);
|
||||
}
|
||||
|
||||
public interface IAiConversationRepository
|
||||
|
||||
@@ -5,6 +5,7 @@ namespace Health.Application.AI;
|
||||
|
||||
public sealed class AiConversationService(IAiConversationRepository conversations) : IAiConversationService
|
||||
{
|
||||
private const int HistoryConversationLimit = 7;
|
||||
private readonly IAiConversationRepository _conversations = conversations;
|
||||
|
||||
public async Task<OpenConversationResult> OpenAsync(
|
||||
@@ -33,14 +34,18 @@ public sealed class AiConversationService(IAiConversationRepository conversation
|
||||
};
|
||||
await _conversations.AddConversationAsync(conversation, ct);
|
||||
await _conversations.SaveChangesAsync(ct);
|
||||
await PruneHistoryAsync(userId, ct);
|
||||
return new OpenConversationResult(true, true, conversation.Id);
|
||||
}
|
||||
|
||||
public async Task AddUserMessageAsync(Guid conversationId, string content, CancellationToken ct) =>
|
||||
await AddMessageAsync(conversationId, MessageRole.User, content, updateSummary: false, ct);
|
||||
public async Task AddUserMessageAsync(Guid conversationId, string content, string? metadataJson, CancellationToken ct) =>
|
||||
await AddMessageAsync(conversationId, MessageRole.User, content, updateSummary: false, metadataJson, ct);
|
||||
|
||||
public async Task AddAssistantMessageAsync(Guid conversationId, string content, CancellationToken ct) =>
|
||||
await AddMessageAsync(conversationId, MessageRole.Assistant, content, updateSummary: true, ct);
|
||||
await AddMessageAsync(conversationId, MessageRole.Assistant, content, updateSummary: true, metadataJson: null, ct);
|
||||
|
||||
public async Task AddAssistantMessageAsync(Guid conversationId, string content, string? metadataJson, CancellationToken ct) =>
|
||||
await AddMessageAsync(conversationId, MessageRole.Assistant, content, updateSummary: true, metadataJson, ct);
|
||||
|
||||
public async Task<IReadOnlyList<AiConversationMessageDto>> GetRecentMessagesAsync(Guid conversationId, int limit, CancellationToken ct)
|
||||
{
|
||||
@@ -51,7 +56,7 @@ public sealed class AiConversationService(IAiConversationRepository conversation
|
||||
public async Task<IReadOnlyList<AiConversationDto>> ListAsync(Guid userId, CancellationToken ct)
|
||||
{
|
||||
var conversations = await _conversations.ListAsync(userId, ct);
|
||||
return conversations.Select(ToDto).ToList();
|
||||
return conversations.Take(HistoryConversationLimit).Select(ToDto).ToList();
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<AiConversationMessageDto>> GetMessagesAsync(Guid userId, Guid conversationId, CancellationToken ct)
|
||||
@@ -63,6 +68,17 @@ public sealed class AiConversationService(IAiConversationRepository conversation
|
||||
return messages.Select(ToMessageDto).ToList();
|
||||
}
|
||||
|
||||
public async Task UpdateSummaryAsync(Guid conversationId, string summary, CancellationToken ct)
|
||||
{
|
||||
var conversation = await _conversations.GetAsync(conversationId, ct)
|
||||
?? throw new InvalidOperationException("会话不存在");
|
||||
var normalized = summary.Trim();
|
||||
if (string.IsNullOrWhiteSpace(normalized)) return;
|
||||
|
||||
conversation.Summary = normalized.Length > 24 ? normalized[..24] : normalized;
|
||||
await _conversations.SaveChangesAsync(ct);
|
||||
}
|
||||
|
||||
public async Task DeleteAsync(Guid userId, Guid conversationId, CancellationToken ct)
|
||||
{
|
||||
var conversation = await _conversations.GetOwnedAsync(userId, conversationId, ct);
|
||||
@@ -72,7 +88,19 @@ public sealed class AiConversationService(IAiConversationRepository conversation
|
||||
await _conversations.SaveChangesAsync(ct);
|
||||
}
|
||||
|
||||
private async Task AddMessageAsync(Guid conversationId, MessageRole role, string content, bool updateSummary, CancellationToken ct)
|
||||
public async Task<int> DeleteAllAsync(Guid userId, CancellationToken ct)
|
||||
{
|
||||
var list = await _conversations.ListAsync(userId, ct);
|
||||
foreach (var conv in list)
|
||||
{
|
||||
_conversations.Delete(conv);
|
||||
}
|
||||
if (list.Count > 0)
|
||||
await _conversations.SaveChangesAsync(ct);
|
||||
return list.Count;
|
||||
}
|
||||
|
||||
private async Task AddMessageAsync(Guid conversationId, MessageRole role, string content, bool updateSummary, string? metadataJson, CancellationToken ct)
|
||||
{
|
||||
var conversation = await _conversations.GetAsync(conversationId, ct)
|
||||
?? throw new InvalidOperationException("会话不存在");
|
||||
@@ -82,6 +110,7 @@ public sealed class AiConversationService(IAiConversationRepository conversation
|
||||
ConversationId = conversationId,
|
||||
Role = role,
|
||||
Content = content,
|
||||
MetadataJson = metadataJson,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
}, ct);
|
||||
|
||||
@@ -90,6 +119,18 @@ public sealed class AiConversationService(IAiConversationRepository conversation
|
||||
if (updateSummary)
|
||||
conversation.Summary = content.Length > 100 ? content[..100] : content;
|
||||
await _conversations.SaveChangesAsync(ct);
|
||||
await PruneHistoryAsync(conversation.UserId, ct);
|
||||
}
|
||||
|
||||
private async Task PruneHistoryAsync(Guid userId, CancellationToken ct)
|
||||
{
|
||||
var list = await _conversations.ListAsync(userId, ct);
|
||||
var expired = list.Skip(HistoryConversationLimit).ToList();
|
||||
if (expired.Count == 0) return;
|
||||
|
||||
foreach (var conversation in expired)
|
||||
_conversations.Delete(conversation);
|
||||
await _conversations.SaveChangesAsync(ct);
|
||||
}
|
||||
|
||||
private static AiConversationDto ToDto(Conversation conversation) => new(
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
namespace Health.Application.AI;
|
||||
|
||||
/// <summary>
|
||||
/// 用户消息附带的图片/PDF 解析结果。
|
||||
/// 既用作 LLM 上下文拼装,也持久化到 ConversationMessage.MetadataJson。
|
||||
/// </summary>
|
||||
public sealed record AttachmentContext(
|
||||
string Kind, // "image" 或 "pdf"
|
||||
string? Category, // 仅图片:food / report / wound / drug / chart / other
|
||||
string? FileName, // PDF 文件原名
|
||||
string Summary, // 简短摘要,用于持久化和历史回顾
|
||||
string LlmContent); // 完整文本,拼到当前轮 LLM user content 前
|
||||
|
||||
public interface IAttachmentContextBuilder
|
||||
{
|
||||
/// <summary>
|
||||
/// 根据 imageUrl 或 pdfUrl 构建附件上下文。两个都为空返回 null。
|
||||
/// </summary>
|
||||
Task<AttachmentContext?> BuildAsync(string? imageUrl, string? pdfUrl, CancellationToken ct);
|
||||
}
|
||||
@@ -97,6 +97,10 @@ public sealed class ExerciseService(IExerciseRepository exercises) : IExerciseSe
|
||||
{
|
||||
var item = await _exercises.GetOwnedItemAsync(userId, itemId, ct);
|
||||
if (item == null) return null;
|
||||
if (item.ScheduledDate != BeijingToday())
|
||||
throw new ValidationException("只能打卡今天的运动任务");
|
||||
if (item.IsRestDay)
|
||||
throw new ValidationException("休息日无需打卡");
|
||||
item.IsCompleted = !item.IsCompleted;
|
||||
item.CompletedAt = item.IsCompleted ? DateTime.UtcNow : null;
|
||||
item.Plan.UpdatedAt = DateTime.UtcNow;
|
||||
@@ -108,6 +112,10 @@ public sealed class ExerciseService(IExerciseRepository exercises) : IExerciseSe
|
||||
{
|
||||
var item = await _exercises.GetOwnedItemAsync(userId, itemId, ct);
|
||||
if (item == null) return false;
|
||||
if (item.ScheduledDate != BeijingToday())
|
||||
throw new ValidationException("只能打卡今天的运动任务");
|
||||
if (item.IsRestDay)
|
||||
throw new ValidationException("休息日无需打卡");
|
||||
item.IsCompleted = true;
|
||||
item.CompletedAt = DateTime.UtcNow;
|
||||
item.Plan.UpdatedAt = DateTime.UtcNow;
|
||||
|
||||
@@ -8,10 +8,10 @@ public sealed class ReportService(
|
||||
IReportFileStorage fileStorage,
|
||||
IReportAnalysisQueue queue) : IReportService
|
||||
{
|
||||
private const long MaxReportImageBytes = 10 * 1024 * 1024;
|
||||
private const long MaxReportFileBytes = 20 * 1024 * 1024;
|
||||
private static readonly HashSet<string> AllowedExtensions = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
".jpg", ".jpeg", ".png", ".webp"
|
||||
".jpg", ".jpeg", ".png", ".webp", ".pdf"
|
||||
};
|
||||
|
||||
private readonly IReportRepository _reports = reports;
|
||||
@@ -35,12 +35,12 @@ public sealed class ReportService(
|
||||
if (file.Length <= 0)
|
||||
return Fail(400, "未上传文件");
|
||||
|
||||
if (file.Length > MaxReportImageBytes)
|
||||
return Fail(400, "报告图片不能超过 10MB,请压缩后重新上传");
|
||||
if (file.Length > MaxReportFileBytes)
|
||||
return Fail(400, "报告文件不能超过 20MB,请压缩后重新上传");
|
||||
|
||||
var ext = Path.GetExtension(file.FileName).ToLowerInvariant();
|
||||
if (!AllowedExtensions.Contains(ext))
|
||||
return Fail(400, "目前仅支持上传 JPG、PNG、WEBP 格式的报告图片");
|
||||
return Fail(400, "目前支持上传 JPG、PNG、WEBP 图片或 PDF 报告");
|
||||
|
||||
var storedFile = await _fileStorage.SaveAsync(file, ext, ct);
|
||||
var report = new Report
|
||||
@@ -48,7 +48,7 @@ public sealed class ReportService(
|
||||
Id = Guid.NewGuid(),
|
||||
UserId = userId,
|
||||
FileUrl = storedFile.FileUrl,
|
||||
FileType = ReportFileType.Image,
|
||||
FileType = ext == ".pdf" ? ReportFileType.Pdf : ReportFileType.Image,
|
||||
Category = ReportCategory.Other,
|
||||
Status = ReportStatus.Analyzing,
|
||||
AiSummary = null,
|
||||
|
||||
179
backend/src/Health.Infrastructure/AI/AttachmentContextBuilder.cs
Normal file
179
backend/src/Health.Infrastructure/AI/AttachmentContextBuilder.cs
Normal file
@@ -0,0 +1,179 @@
|
||||
using System.Text.Json;
|
||||
using Health.Application.AI;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using UglyToad.PdfPig;
|
||||
|
||||
namespace Health.Infrastructure.AI;
|
||||
|
||||
public sealed class AttachmentContextBuilder(
|
||||
VisionClient vision,
|
||||
ILogger<AttachmentContextBuilder> logger) : IAttachmentContextBuilder
|
||||
{
|
||||
private const int MaxPdfChars = 6000;
|
||||
private static readonly JsonSerializerOptions JsonOpts = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
PropertyNameCaseInsensitive = true,
|
||||
};
|
||||
|
||||
private readonly VisionClient _vision = vision;
|
||||
private readonly ILogger<AttachmentContextBuilder> _logger = logger;
|
||||
|
||||
public async Task<AttachmentContext?> BuildAsync(string? imageUrl, string? pdfUrl, CancellationToken ct)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(imageUrl))
|
||||
return await BuildImageAsync(imageUrl!, ct);
|
||||
if (!string.IsNullOrWhiteSpace(pdfUrl))
|
||||
return await BuildPdfAsync(pdfUrl!, ct);
|
||||
return null;
|
||||
}
|
||||
|
||||
// ── 图片:调 VLM 输出结构化 JSON ──
|
||||
private async Task<AttachmentContext?> BuildImageAsync(string imageUrl, CancellationToken ct)
|
||||
{
|
||||
var filePath = ResolveLocalPath(imageUrl);
|
||||
if (filePath == null || !File.Exists(filePath))
|
||||
{
|
||||
_logger.LogWarning("Image file not found for {Url}", imageUrl);
|
||||
return new AttachmentContext("image", null, null, "图片暂时无法读取", "[图片附件读取失败,请描述图片内容]");
|
||||
}
|
||||
|
||||
var bytes = await File.ReadAllBytesAsync(filePath, ct);
|
||||
var mime = Path.GetExtension(filePath).ToLowerInvariant() switch
|
||||
{
|
||||
".png" => "image/png",
|
||||
".webp" => "image/webp",
|
||||
".heic" => "image/heic",
|
||||
_ => "image/jpeg",
|
||||
};
|
||||
var dataUrl = $"data:{mime};base64,{Convert.ToBase64String(bytes)}";
|
||||
|
||||
var prompt = """
|
||||
识别图片内容,输出 JSON:
|
||||
{"category":"food|report|wound|drug|chart|other","summary":"1-2句话描述图片","details":["关键信息1","关键信息2"]}
|
||||
|
||||
分类标准:
|
||||
- food:任何食物、饮品、餐食
|
||||
- report:医学检查报告、化验单、影像报告(X光/CT/MRI/B超截图等)
|
||||
- wound:伤口、术后切口、皮肤异常
|
||||
- drug:药品包装、药品说明书
|
||||
- chart:心电图、监护仪截图等
|
||||
- other:其他
|
||||
|
||||
details 给出最有价值的细节(食物种类、报告关键值、伤口部位等)。
|
||||
只输出 JSON 本身,不要任何前后缀。
|
||||
""";
|
||||
|
||||
string? raw = null;
|
||||
try
|
||||
{
|
||||
var resp = await _vision.VisionAsync(prompt, [dataUrl], userText: null, maxTokens: 1024, ct: ct);
|
||||
raw = resp.Choices?.FirstOrDefault()?.Message?.Content;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "VLM call failed for image {Url}", imageUrl);
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(raw))
|
||||
return new AttachmentContext("image", null, null, "图片识别失败", "[图片附件识别失败,请简要描述图片内容]");
|
||||
|
||||
try
|
||||
{
|
||||
var cleaned = StripCodeFence(raw.Trim());
|
||||
using var doc = JsonDocument.Parse(cleaned);
|
||||
var root = doc.RootElement;
|
||||
var category = GetString(root, "category") ?? "other";
|
||||
var summary = GetString(root, "summary") ?? "图片内容";
|
||||
var details = root.TryGetProperty("details", out var d) && d.ValueKind == JsonValueKind.Array
|
||||
? string.Join("、", d.EnumerateArray().Select(x => x.GetString()).Where(s => !string.IsNullOrWhiteSpace(s)))
|
||||
: "";
|
||||
|
||||
var compact = string.IsNullOrEmpty(details)
|
||||
? $"图片识别:{summary}"
|
||||
: $"图片识别:{summary}({details})";
|
||||
var llmContent = $"[图片识别(类别 {category}):{summary}{(string.IsNullOrEmpty(details) ? "" : $",包含 {details}")}]";
|
||||
|
||||
return new AttachmentContext("image", category, null, compact, llmContent);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to parse VLM JSON: {Raw}", raw);
|
||||
// 兜底:原文也能用
|
||||
return new AttachmentContext("image", null, null, $"图片识别:{raw[..Math.Min(raw.Length, 80)]}", $"[图片识别:{raw}]");
|
||||
}
|
||||
}
|
||||
|
||||
// ── PDF:PdfPig 抽取文本 ──
|
||||
private Task<AttachmentContext?> BuildPdfAsync(string pdfUrl, CancellationToken ct)
|
||||
{
|
||||
var filePath = ResolveLocalPath(pdfUrl);
|
||||
var fileName = Path.GetFileName(pdfUrl);
|
||||
if (filePath == null || !File.Exists(filePath))
|
||||
{
|
||||
_logger.LogWarning("PDF file not found for {Url}", pdfUrl);
|
||||
return Task.FromResult<AttachmentContext?>(new AttachmentContext(
|
||||
"pdf", null, fileName, "PDF 文件读取失败",
|
||||
"[PDF 附件读取失败,请描述文档内容]"));
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using var pdf = PdfDocument.Open(filePath);
|
||||
var sb = new System.Text.StringBuilder();
|
||||
foreach (var page in pdf.GetPages())
|
||||
{
|
||||
sb.AppendLine(page.Text);
|
||||
if (sb.Length > MaxPdfChars) break;
|
||||
}
|
||||
var text = sb.ToString().Trim();
|
||||
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
return Task.FromResult<AttachmentContext?>(new AttachmentContext(
|
||||
"pdf", null, fileName, "PDF 内容为空或扫描件",
|
||||
"[PDF 看起来是扫描件或图片版,无法提取文字。请提示用户改用拍照上传,或简要描述报告内容]"));
|
||||
}
|
||||
|
||||
var truncated = text.Length > MaxPdfChars ? text[..MaxPdfChars] + "...(已截断)" : text;
|
||||
var summary = $"PDF:{fileName}";
|
||||
var llmContent = $"[用户上传 PDF 「{fileName}」,文档内容如下]\n{truncated}";
|
||||
|
||||
return Task.FromResult<AttachmentContext?>(new AttachmentContext("pdf", null, fileName, summary, llmContent));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to parse PDF {Url}", pdfUrl);
|
||||
return Task.FromResult<AttachmentContext?>(new AttachmentContext(
|
||||
"pdf", null, fileName, "PDF 解析异常",
|
||||
"[PDF 解析失败,请提示用户文件可能损坏或加密]"));
|
||||
}
|
||||
}
|
||||
|
||||
private static string? ResolveLocalPath(string url)
|
||||
{
|
||||
// url 形如 "/uploads/{guid}.{ext}"。处理 base URL 前缀也兼容。
|
||||
var idx = url.IndexOf("/uploads/", StringComparison.Ordinal);
|
||||
if (idx < 0) return null;
|
||||
var relative = url[(idx + "/uploads/".Length)..];
|
||||
// 去掉可能的 query string
|
||||
var q = relative.IndexOf('?');
|
||||
if (q >= 0) relative = relative[..q];
|
||||
return Path.Combine(Directory.GetCurrentDirectory(), "uploads", relative);
|
||||
}
|
||||
|
||||
private static string StripCodeFence(string raw)
|
||||
{
|
||||
var t = raw.Trim();
|
||||
if (t.StartsWith("```"))
|
||||
{
|
||||
var firstNewline = t.IndexOf('\n');
|
||||
if (firstNewline > 0) t = t[(firstNewline + 1)..];
|
||||
if (t.EndsWith("```")) t = t[..^3];
|
||||
}
|
||||
return t.Trim();
|
||||
}
|
||||
|
||||
private static string? GetString(JsonElement el, string name) =>
|
||||
el.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.String ? v.GetString() : null;
|
||||
}
|
||||
@@ -23,9 +23,21 @@ public sealed class PromptManager
|
||||
_ => DefaultPrompt
|
||||
};
|
||||
|
||||
return $"{prompt}\n\n{MedicalBoundaryRules}";
|
||||
return $"{prompt}\n\n{MedicalBoundaryRules}\n\n{SmartLinkRules}";
|
||||
}
|
||||
|
||||
private const string SmartLinkRules = """
|
||||
智能跳转链接(按需嵌入):
|
||||
- 当用户明确询问食物的 热量/卡路里 时,在回答的自然位置嵌入 Markdown 链接 [热量分析](app://diet),引导用户使用专门的饮食拍照分析功能。
|
||||
- 当用户上传医学检查报告、化验单、影像报告、出院小结等需要专业解读时,在回答末尾嵌入 [报告分析](app://report),引导上传到报告分析功能获得详细解读。
|
||||
- 当用户询问血压、血糖、血氧等需要长期追踪的设备测量场景时,可嵌入 [蓝牙设备](app://device)。
|
||||
|
||||
硬性约束:
|
||||
- 每条回复最多嵌入一个跳转链接,且必须在合适语境下自然提及,不要堆砌。
|
||||
- 仅"问热量/卡路里"才用 app://diet;"营养、能不能吃、是否健康"类问题正常回答,禁止插入饮食跳转链接。
|
||||
- 链接的 Markdown 文本只能用上面列出的 4 个固定文案,不要自创其他文案。
|
||||
""";
|
||||
|
||||
private const string MedicalBoundaryRules = """
|
||||
医疗边界(必须遵守):
|
||||
- 你的定位是患者端 AI 健康解释与预问诊助手,不是医生,不能替代医生诊断、处方或治疗决策。
|
||||
|
||||
@@ -31,9 +31,38 @@ public sealed class DietImageAnalyzer(VisionClient vision) : IDietImageAnalyzer
|
||||
}
|
||||
|
||||
var prompt = """
|
||||
识别图片中的食物和饮品,返回JSON数组:
|
||||
[{"name":"名称","portion":"份量","calories":热量}]
|
||||
只返回JSON,不要其他内容。
|
||||
你是营养师助手。请识别图片中所有食物和饮品,估算每项的具体克数或毫升数,输出 JSON 数组。
|
||||
|
||||
## 无食物判定(必读)
|
||||
如果图片中完全没有可食用的食物或饮品(例如是风景、人物、宠物、文档、屏幕截图、纯背景、空盘子等),**直接输出空数组 []**,不要硬编造食物。
|
||||
|
||||
## 输出格式
|
||||
[{"name":"食物名","portion":"具体克数","calories":数字}]
|
||||
|
||||
## 份量估算依据(容器与参照物对照表)
|
||||
- 标准饭碗(口径11cm):满碗米饭≈200g、满碗面条≈250g、满碗粥≈250g、半碗≈100g
|
||||
- 标准汤碗(口径14cm):满碗汤≈400ml、半碗≈200ml
|
||||
- 标准盘子(口径23cm):满盘菜≈300g、半盘≈150g
|
||||
- 普通玻璃杯:满杯水/饮料≈250ml、纸杯≈200ml
|
||||
- 鸡蛋1个≈50g、馒头1个≈100g、包子1个≈80g、饺子1个≈15g
|
||||
- 香蕉1根≈100g、苹果1个≈200g、橘子1个≈120g
|
||||
- 肉块巴掌大小≈80-120g、鸡腿1个≈100g、鱼1条(中等)≈300g
|
||||
|
||||
## 输出示例(务必模仿此风格)
|
||||
正确:{"name":"米饭","portion":"约200g","calories":230}
|
||||
正确:{"name":"红烧肉","portion":"约150g","calories":420}
|
||||
正确:{"name":"豆浆","portion":"约250ml","calories":80}
|
||||
错误:{"name":"米饭","portion":"一碗","calories":230} ← portion 缺克数
|
||||
错误:{"name":"汤","portion":"一份","calories":80} ← portion 缺毫升数
|
||||
错误:{"name":"水果","portion":"少许","calories":50} ← portion 是模糊词
|
||||
|
||||
## 硬性要求
|
||||
1. 图片若无食物,输出 [] 即可。不要硬凑食物。
|
||||
2. portion 字段必须包含阿拉伯数字 + 单位(g 或 ml)。不允许任何只有量词没有克数的回答。
|
||||
3. 即使图片不够清楚,也要根据可见容器/参照物给出估算,可加"约"或区间("约80-120g")。
|
||||
4. calories 是数字(千卡),不带单位、不带引号。
|
||||
|
||||
只输出 JSON 数组本身,不要任何前缀、后缀、代码块标记或说明文字。
|
||||
""";
|
||||
var response = await _vision.VisionAsync(prompt, imageUrls, userText: null, maxTokens: 8192, ct: ct);
|
||||
var result = response.Choices?.FirstOrDefault()?.Message?.Content ?? "[]";
|
||||
|
||||
@@ -7,11 +7,13 @@
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="10.0.8" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="10.0.8" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="10.0.8" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.8" />
|
||||
<PackageReference Include="Microsoft.IdentityModel.Tokens" Version="8.18.0" />
|
||||
<PackageReference Include="Minio" Version="7.0.0" />
|
||||
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="10.0.2" />
|
||||
<PackageReference Include="PdfPig" Version="0.1.16-alpha-20260629-34229" />
|
||||
<PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="8.18.0" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ using Health.Application.Reports;
|
||||
using Health.Application.Notifications;
|
||||
using Health.Infrastructure.AI;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using UglyToad.PdfPig;
|
||||
|
||||
namespace Health.Infrastructure.Reports;
|
||||
|
||||
@@ -17,6 +18,7 @@ public sealed class ReportAnalysisService(
|
||||
private readonly DeepSeekClient _llm = llm;
|
||||
private readonly IUserNotificationProducer _notifications = notifications;
|
||||
private readonly ILogger<ReportAnalysisService> _logger = logger;
|
||||
private const int MaxPdfChars = 8000;
|
||||
|
||||
public async Task AnalyzeAsync(ReportAnalysisJob job, CancellationToken ct)
|
||||
{
|
||||
@@ -72,6 +74,9 @@ public sealed class ReportAnalysisService(
|
||||
|
||||
private async Task<JsonElement> ExtractIndicatorsAsync(string filePath, CancellationToken ct)
|
||||
{
|
||||
if (Path.GetExtension(filePath).Equals(".pdf", StringComparison.OrdinalIgnoreCase))
|
||||
return await ExtractIndicatorsFromPdfAsync(filePath, ct);
|
||||
|
||||
var imageUrl = await GetLocalImageUrl(filePath, ct);
|
||||
if (imageUrl == null)
|
||||
throw new InvalidOperationException("报告图片文件不存在或无法读取");
|
||||
@@ -112,6 +117,66 @@ public sealed class ReportAnalysisService(
|
||||
return json ?? throw new InvalidOperationException("报告识别结果格式异常");
|
||||
}
|
||||
|
||||
private async Task<JsonElement> ExtractIndicatorsFromPdfAsync(string filePath, CancellationToken ct)
|
||||
{
|
||||
var text = ExtractPdfText(filePath);
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
return JsonDocument.Parse("""
|
||||
{"isReport": false, "reason": "这份 PDF 可能是扫描件或图片版,当前无法直接提取文字。请上传清晰图片,或后续启用 PDF 页转图片后再用 VLM 识别。"}
|
||||
""").RootElement;
|
||||
}
|
||||
|
||||
var prompt = $$"""
|
||||
你是一名医学检验报告分析专家。下面是用户上传 PDF 中提取出的文字。
|
||||
|
||||
PDF 文本:
|
||||
{{text}}
|
||||
|
||||
第一步:判断这到底是不是一份医学检验报告。
|
||||
如果不是,直接返回:
|
||||
{"isReport": false, "reason": "这不是医学报告,原因:..."}
|
||||
|
||||
如果是医学报告,提取所有检测指标,返回严格 JSON 格式(不要 markdown 包裹):
|
||||
{
|
||||
"isReport": true,
|
||||
"reportType": "BloodTest/Biochemistry/Ecg/Ultrasound/Discharge/Other",
|
||||
"indicators": [
|
||||
{"name": "指标名称", "value": "数值", "unit": "单位", "referenceRange": "参考范围", "status": "normal/high/low"}
|
||||
]
|
||||
}
|
||||
|
||||
注意:
|
||||
- 只提取报告里真实出现的指标
|
||||
- status 判断:数值在参考范围内=normal,超出上限=high,低于下限=low
|
||||
- 不确定的参考范围不要编造,可留空
|
||||
""";
|
||||
|
||||
var messages = new List<ChatMessage>
|
||||
{
|
||||
new() { Role = "system", Content = "你是医学报告结构化抽取助手,只输出严格 JSON。" },
|
||||
new() { Role = "user", Content = prompt }
|
||||
};
|
||||
|
||||
var response = await _llm.ChatAsync(messages, maxTokens: 1800, temperature: 0.1f, ct: ct);
|
||||
var content = response.Choices?.FirstOrDefault()?.Message?.Content ?? "";
|
||||
var json = TryParseJson(content);
|
||||
return json ?? throw new InvalidOperationException("PDF 报告解析结果格式异常");
|
||||
}
|
||||
|
||||
private static string ExtractPdfText(string filePath)
|
||||
{
|
||||
using var pdf = PdfDocument.Open(filePath);
|
||||
var sb = new System.Text.StringBuilder();
|
||||
foreach (var page in pdf.GetPages())
|
||||
{
|
||||
sb.AppendLine(page.Text);
|
||||
if (sb.Length > MaxPdfChars) break;
|
||||
}
|
||||
var text = sb.ToString().Trim();
|
||||
return text.Length > MaxPdfChars ? text[..MaxPdfChars] : text;
|
||||
}
|
||||
|
||||
private async Task<string> GenerateSummaryAsync(string indicatorsJson, CancellationToken ct)
|
||||
{
|
||||
var prompt = $"""
|
||||
|
||||
@@ -28,6 +28,8 @@ public static class AiChatEndpoints
|
||||
app.MapGet("/api/ai/{agentType}/chat", async (
|
||||
string message,
|
||||
string? conversationId,
|
||||
string? imageUrl,
|
||||
string? pdfUrl,
|
||||
string token,
|
||||
string agentType,
|
||||
HttpContext http,
|
||||
@@ -37,6 +39,7 @@ public static class AiChatEndpoints
|
||||
IAiToolExecutionService toolExecution,
|
||||
IAiWriteConfirmationStore confirmations,
|
||||
IAiConversationService conversations,
|
||||
IAttachmentContextBuilder attachments,
|
||||
IPatientContextService patientContexts,
|
||||
CancellationToken ct) =>
|
||||
{
|
||||
@@ -85,7 +88,24 @@ public static class AiChatEndpoints
|
||||
if (opened.Created)
|
||||
await SseWriteAsync(http, new { action = "conversation_id", data = activeConversationId.ToString() }, ct);
|
||||
|
||||
await conversations.AddUserMessageAsync(activeConversationId, message, ct);
|
||||
// 附件解析(图片走 VLM、PDF 走 PdfPig),结果同时拼 LLM 上下文 + 持久化到 user message metadata
|
||||
var attachment = await attachments.BuildAsync(imageUrl, pdfUrl, ct);
|
||||
string? userMessageMetadataJson = null;
|
||||
if (attachment != null)
|
||||
{
|
||||
userMessageMetadataJson = JsonSerializer.Serialize(new
|
||||
{
|
||||
kind = attachment.Kind,
|
||||
category = attachment.Category,
|
||||
fileName = attachment.FileName,
|
||||
summary = attachment.Summary,
|
||||
imageUrl,
|
||||
pdfUrl,
|
||||
}, JsonOpts);
|
||||
await SseWriteAsync(http, new { action = "attachment", data = new { attachment.Kind, attachment.Category, attachment.Summary } }, ct);
|
||||
}
|
||||
|
||||
await conversations.AddUserMessageAsync(activeConversationId, message, userMessageMetadataJson, ct);
|
||||
|
||||
var urgentWarning = DetectUrgentRisk(message);
|
||||
if (!string.IsNullOrEmpty(urgentWarning))
|
||||
@@ -99,6 +119,7 @@ public static class AiChatEndpoints
|
||||
""";
|
||||
|
||||
await conversations.AddAssistantMessageAsync(activeConversationId, urgentResponse, ct);
|
||||
await TryUpdateConversationSummaryAsync(conversations, llmClient, userId.Value, activeConversationId, ct);
|
||||
|
||||
await SseWriteAsync(http, new { action = "notice", message = "检测到可能的危险信号,优先给出就医提醒" }, ct);
|
||||
await SseWriteAsync(http, new { action = "answer", data = urgentResponse, type = "text" }, ct);
|
||||
@@ -131,16 +152,46 @@ public static class AiChatEndpoints
|
||||
new() { Role = "system", Content = enhancedSystem },
|
||||
};
|
||||
|
||||
// 加载历史对话(最近 10 条)
|
||||
// 加载历史对话(最近 12 条)
|
||||
var history = await conversations.GetRecentMessagesAsync(activeConversationId, 12, ct);
|
||||
|
||||
foreach (var h in history)
|
||||
{
|
||||
messages.Add(new ChatMessage
|
||||
var role = h.Role == MessageRole.User.ToString() ? "user" : "assistant";
|
||||
var content = h.Content;
|
||||
|
||||
// 历史 user message 如果带过附件,把附件摘要拼回 content,让 AI 能回忆起来
|
||||
if (role == "user" && !string.IsNullOrWhiteSpace(h.MetadataJson))
|
||||
{
|
||||
Role = h.Role == MessageRole.User.ToString() ? "user" : "assistant",
|
||||
Content = h.Content,
|
||||
});
|
||||
try
|
||||
{
|
||||
using var meta = JsonDocument.Parse(h.MetadataJson);
|
||||
if (meta.RootElement.TryGetProperty("summary", out var sumEl) &&
|
||||
sumEl.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
var sum = sumEl.GetString();
|
||||
if (!string.IsNullOrWhiteSpace(sum) &&
|
||||
// 跳过本轮:本轮 user message 会在下面单独拼 LlmContent,避免重复
|
||||
h.Content != message)
|
||||
{
|
||||
content = $"[历史附件回忆:{sum}]\n{content}";
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (JsonException) { /* 忽略损坏的 metadata */ }
|
||||
}
|
||||
|
||||
messages.Add(new ChatMessage { Role = role, Content = content });
|
||||
}
|
||||
|
||||
// 本轮如果带了附件,把完整识别结果拼到最后一条 user message 前
|
||||
if (attachment != null && messages.Count > 0 && messages[^1].Role == "user")
|
||||
{
|
||||
messages[^1] = new ChatMessage
|
||||
{
|
||||
Role = "user",
|
||||
Content = $"{attachment.LlmContent}\n{messages[^1].Content}",
|
||||
};
|
||||
}
|
||||
|
||||
// Tool Calling 循环
|
||||
@@ -213,7 +264,16 @@ public static class AiChatEndpoints
|
||||
|
||||
// 保存 AI 回复
|
||||
if (!string.IsNullOrEmpty(fullResponse))
|
||||
await conversations.AddAssistantMessageAsync(activeConversationId, fullResponse, ct);
|
||||
{
|
||||
string? assistantMetadataJson = null;
|
||||
if (metadata.Count > 0 || messageType != "text")
|
||||
{
|
||||
metadata["messageType"] = messageType;
|
||||
assistantMetadataJson = JsonSerializer.Serialize(metadata, JsonOpts);
|
||||
}
|
||||
await conversations.AddAssistantMessageAsync(activeConversationId, fullResponse, assistantMetadataJson, ct);
|
||||
await TryUpdateConversationSummaryAsync(conversations, llmClient, userId.Value, activeConversationId, ct);
|
||||
}
|
||||
|
||||
await SseWriteAsync(http, new { action = "status", data = completedNormally ? "done" : "error" }, ct);
|
||||
await http.Response.WriteAsync("data: [DONE]\n\n", ct);
|
||||
@@ -265,6 +325,16 @@ public static class AiChatEndpoints
|
||||
return Results.Ok(new { code = 0, data = new { success = true }, message = (string?)null });
|
||||
});
|
||||
|
||||
// 一键清空当前用户的全部对话
|
||||
app.MapDelete("/api/ai/conversations", async (HttpContext http, IAiConversationService conversations, CancellationToken ct) =>
|
||||
{
|
||||
var userId = GetUserId(http);
|
||||
if (userId == null) return Results.Json(new { code = 40002 }, statusCode: 401);
|
||||
|
||||
var count = await conversations.DeleteAllAsync(userId.Value, ct);
|
||||
return Results.Ok(new { code = 0, data = new { deleted = count }, message = (string?)null });
|
||||
});
|
||||
|
||||
app.MapPost("/api/ai/analyze-food-image", async (
|
||||
HttpRequest httpRequest,
|
||||
HttpContext http,
|
||||
@@ -317,6 +387,116 @@ public static class AiChatEndpoints
|
||||
catch (Exception) { return null; }
|
||||
}
|
||||
|
||||
private static async Task TryUpdateConversationSummaryAsync(
|
||||
IAiConversationService conversations,
|
||||
DeepSeekClient llmClient,
|
||||
Guid userId,
|
||||
Guid conversationId,
|
||||
CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
var messages = await conversations.GetMessagesAsync(userId, conversationId, ct);
|
||||
var transcript = BuildSummaryTranscript(messages);
|
||||
if (string.IsNullOrWhiteSpace(transcript)) return;
|
||||
|
||||
var prompt = $"""
|
||||
请把下面这次健康助手会话压缩成一个“对话记录标题”。
|
||||
|
||||
要求:
|
||||
- 只输出标题,不要解释
|
||||
- 8到18个汉字
|
||||
- 优先保留用户真正要解决的问题、对象、最终动作或结论
|
||||
- 像列表标题,不要像完整句子
|
||||
- 不要写“本次会话、用户咨询、健康建议、进行分析、相关问题”
|
||||
- 如果是上传图片/PDF,标题要体现附件内容和目的
|
||||
|
||||
示例:
|
||||
- 早餐热量识别
|
||||
- 血压偏高记录
|
||||
- 服药时间调整
|
||||
- 报告指标解读
|
||||
- 跑步计划打卡
|
||||
|
||||
会话内容:
|
||||
{transcript}
|
||||
""";
|
||||
|
||||
var response = await llmClient.ChatAsync(
|
||||
[
|
||||
new ChatMessage { Role = "system", Content = "你是对话标题生成助手,只输出短标题。" },
|
||||
new ChatMessage { Role = "user", Content = prompt },
|
||||
],
|
||||
maxTokens: 120,
|
||||
temperature: 0.2f,
|
||||
ct: ct);
|
||||
|
||||
var summary = response.Choices?.FirstOrDefault()?.Message?.Content?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(summary)) return;
|
||||
|
||||
summary = Regex.Replace(summary, @"\s+", "");
|
||||
summary = summary.Trim(' ', '。', '.', '"', '"', '\'', '“', '”', ':', ':');
|
||||
summary = TrimSummaryPrefix(summary);
|
||||
await conversations.UpdateSummaryAsync(conversationId, summary, ct);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// 摘要只是历史列表体验增强,失败不能影响主对话。
|
||||
}
|
||||
}
|
||||
|
||||
private static string BuildSummaryTranscript(IReadOnlyList<AiConversationMessageDto> messages)
|
||||
{
|
||||
var sb = new System.Text.StringBuilder();
|
||||
foreach (var message in messages.TakeLast(24))
|
||||
{
|
||||
var role = message.Role == MessageRole.User.ToString() ? "用户" : "AI";
|
||||
var content = message.Content?.Trim() ?? "";
|
||||
|
||||
if (message.Role == MessageRole.User.ToString() &&
|
||||
!string.IsNullOrWhiteSpace(message.MetadataJson))
|
||||
{
|
||||
try
|
||||
{
|
||||
using var meta = JsonDocument.Parse(message.MetadataJson);
|
||||
if (meta.RootElement.TryGetProperty("summary", out var sumEl) &&
|
||||
sumEl.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
var attachmentSummary = sumEl.GetString();
|
||||
if (!string.IsNullOrWhiteSpace(attachmentSummary))
|
||||
content = $"[附件:{attachmentSummary}] {content}";
|
||||
}
|
||||
}
|
||||
catch (JsonException) { }
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(content)) continue;
|
||||
if (content.Length > 800) content = content[..800] + "...";
|
||||
sb.Append(role).Append(":").AppendLine(content);
|
||||
if (sb.Length > 6000) break;
|
||||
}
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
private static string TrimSummaryPrefix(string summary)
|
||||
{
|
||||
var prefixes = new[]
|
||||
{
|
||||
"本次会话",
|
||||
"用户咨询",
|
||||
"用户询问",
|
||||
"健康建议",
|
||||
"相关问题",
|
||||
};
|
||||
foreach (var prefix in prefixes)
|
||||
{
|
||||
if (summary.StartsWith(prefix, StringComparison.Ordinal))
|
||||
return summary[prefix.Length..].Trim(' ', ':', ':', ',', ',');
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
|
||||
// ── Agent / Tool 调度 ──
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -123,6 +123,7 @@ builder.Services.AddScoped<IDietRepository, EfDietRepository>();
|
||||
builder.Services.AddScoped<IDietImageAnalysisCoordinator, DietImageAnalysisCoordinator>();
|
||||
builder.Services.AddScoped<IDietImageAnalyzer, DietImageAnalyzer>();
|
||||
builder.Services.AddScoped<IDietImageAnalysisQueue, DietImageAnalysisQueue>();
|
||||
builder.Services.AddScoped<IAttachmentContextBuilder, AttachmentContextBuilder>();
|
||||
builder.Services.AddScoped<IExerciseService, ExerciseService>();
|
||||
builder.Services.AddScoped<IExerciseRepository, EfExerciseRepository>();
|
||||
builder.Services.AddScoped<IExerciseReminderProducer, ExerciseReminderProducer>();
|
||||
|
||||
@@ -3,6 +3,7 @@ using Health.Application.Calendars;
|
||||
using Health.Application.HealthArchives;
|
||||
using Health.Application.HealthRecords;
|
||||
using Health.Application.Exercises;
|
||||
using Health.Domain;
|
||||
using Health.Domain.Entities;
|
||||
using Health.Domain.Enums;
|
||||
|
||||
@@ -167,6 +168,26 @@ public sealed class ApplicationServiceTests
|
||||
Assert.Equal(2, current.Items.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExercisePlan_CheckInRejectsNonTodayItem()
|
||||
{
|
||||
var today = DateOnly.FromDateTime(DateTime.UtcNow.AddHours(8));
|
||||
var repository = new FakeExerciseRepository();
|
||||
var plan = new ExercisePlan
|
||||
{
|
||||
Id = Guid.NewGuid(), UserId = Guid.NewGuid(), StartDate = today.AddDays(-1), EndDate = today, ReminderTime = new TimeOnly(19, 0),
|
||||
};
|
||||
var item = new ExercisePlanItem
|
||||
{
|
||||
Id = Guid.NewGuid(), Plan = plan, ScheduledDate = today.AddDays(-1), ExerciseType = "散步", DurationMinutes = 30,
|
||||
};
|
||||
plan.Items.Add(item);
|
||||
repository.SetPlan(plan);
|
||||
var service = new ExerciseService(repository);
|
||||
|
||||
await Assert.ThrowsAsync<ValidationException>(() => service.ToggleCheckInAsync(plan.UserId, item.Id, CancellationToken.None));
|
||||
}
|
||||
|
||||
private sealed class FakeHealthArchiveRepository(HealthArchive? archive) : IHealthArchiveRepository
|
||||
{
|
||||
public HealthArchive? Archive { get; private set; } = archive;
|
||||
|
||||
Reference in New Issue
Block a user