0e0e8ce72b
- 问诊对话页: 新建consultation_provider, 完整AI分身聊天UI - 首次建档引导: OnboardingPrompt+ManageArchiveTool+前端卡片 - 报告模块: POST端点 + report_agent_handler接VLM - 健康日历: GET /api/calendar端点 + 前端接API - 趋势页重写: 删除Mock数据 + 真实API + 手动录入 - 饮食保存: submit按钮接后端API - 侧边栏: 健康概览横排 + 统一紫色配色 - 运动计划: 修复JSON反序列化(record→手动解析) - VLM: prompt优化 + 模型切qwen3-vl-plus + 1280px压缩 - UI颜色: 加深主色 8B9CF7→6C5CE7 - 个人信息页精简 + 健康档案可编辑重设计 - 保洁: 删除无用agent_bar + 删除意见反馈/关于我们 - 新增AI提示词文档
452 lines
20 KiB
C#
452 lines
20 KiB
C#
using System.Drawing;
|
||
using System.Drawing.Imaging;
|
||
using Health.Infrastructure.AI;
|
||
using Health.Infrastructure.AI.AgentHandlers;
|
||
|
||
namespace Health.WebApi.Endpoints;
|
||
|
||
/// <summary>
|
||
/// AI 对话 SSE 端点——支持 7 个 Agent
|
||
/// </summary>
|
||
public static class AiChatEndpoints
|
||
{
|
||
private static readonly JsonSerializerOptions JsonOpts = new()
|
||
{
|
||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||
PropertyNameCaseInsensitive = true,
|
||
};
|
||
|
||
public static void MapAiChatEndpoints(this WebApplication app)
|
||
{
|
||
// SSE 流式对话（GET 方式，token 通过 query string 传递）
|
||
app.MapGet("/api/ai/{agentType}/chat", async (
|
||
string message,
|
||
string? conversationId,
|
||
string token,
|
||
string agentType,
|
||
HttpContext http,
|
||
AppDbContext db,
|
||
DeepSeekClient llmClient,
|
||
PromptManager promptManager,
|
||
VisionClient visionClient,
|
||
CancellationToken ct) =>
|
||
{
|
||
// 支持 token 通过 query string（浏览器 EventSource）或 header 传递
|
||
var userId = GetUserId(http) ?? GetUserIdFromToken(token);
|
||
if (userId == null)
|
||
{
|
||
http.Response.StatusCode = 401;
|
||
http.Response.ContentType = "application/json";
|
||
await http.Response.WriteAsync(JsonSerializer.Serialize(new { code = 40002, data = (object?)null, message = "未登录" }), ct);
|
||
return;
|
||
}
|
||
|
||
if (!Enum.TryParse<AgentType>(agentType, ignoreCase: true, out var parsedType))
|
||
parsedType = AgentType.Default;
|
||
|
||
// SSE 响应头
|
||
http.Response.ContentType = "text/event-stream";
|
||
http.Response.Headers.CacheControl = "no-cache";
|
||
http.Response.Headers.Connection = "keep-alive";
|
||
http.Response.Headers["X-Accel-Buffering"] = "no";
|
||
|
||
// 创建或获取对话
|
||
Conversation? conversation = null;
|
||
if (!string.IsNullOrEmpty(conversationId) && Guid.TryParse(conversationId, out var convId))
|
||
conversation = await db.Conversations.FindAsync([convId], ct);
|
||
|
||
if (conversation == null)
|
||
{
|
||
conversation = new Conversation
|
||
{
|
||
Id = Guid.NewGuid(), UserId = userId.Value, AgentType = parsedType,
|
||
Title = message.Length > 30 ? message[..30] : message,
|
||
CreatedAt = DateTime.UtcNow, UpdatedAt = DateTime.UtcNow,
|
||
};
|
||
db.Conversations.Add(conversation);
|
||
await db.SaveChangesAsync(ct);
|
||
await SseWriteAsync(http, new { action = "conversation_id", data = conversation.Id.ToString() }, ct);
|
||
}
|
||
|
||
// 保存用户消息
|
||
var userMsg = new ConversationMessage
|
||
{
|
||
Id = Guid.NewGuid(), ConversationId = conversation.Id, Role = MessageRole.User,
|
||
Content = message, CreatedAt = DateTime.UtcNow,
|
||
};
|
||
db.ConversationMessages.Add(userMsg);
|
||
conversation.MessageCount++;
|
||
conversation.UpdatedAt = DateTime.UtcNow;
|
||
await db.SaveChangesAsync(ct);
|
||
|
||
// 加载上下文
|
||
var systemPrompt = promptManager.GetSystemPrompt(parsedType);
|
||
var patientContext = await BuildPatientContext(db, userId.Value, ct);
|
||
|
||
var messages = new List<ChatMessage>
|
||
{
|
||
new() { Role = "system", Content = systemPrompt + "\n\n当前患者信息：\n" + patientContext },
|
||
};
|
||
|
||
// 加载历史对话（最近 10 条）
|
||
var history = await db.ConversationMessages
|
||
.Where(m => m.ConversationId == conversation.Id)
|
||
.OrderByDescending(m => m.CreatedAt)
|
||
.Take(12)
|
||
.ToListAsync(ct);
|
||
|
||
foreach (var h in history.Reverse<ConversationMessage>())
|
||
{
|
||
messages.Add(new ChatMessage
|
||
{
|
||
Role = h.Role == MessageRole.User ? "user" : "assistant",
|
||
Content = h.Content,
|
||
});
|
||
}
|
||
|
||
// Tool Calling 循环
|
||
var tools = GetToolsForAgent(parsedType);
|
||
var maxIterations = 5;
|
||
var fullResponse = "";
|
||
var completedNormally = false;
|
||
var messageType = "text";
|
||
var metadata = new Dictionary<string, object>();
|
||
|
||
for (int i = 0; i < maxIterations; i++)
|
||
{
|
||
await SseWriteAsync(http, new { action = "notice", message = i == 0 ? "正在分析..." : "正在处理..." }, ct);
|
||
|
||
var response = await llmClient.ChatAsync(messages, tools: tools.Count > 0 ? tools : null, ct: ct);
|
||
|
||
var choice = response.Choices?.FirstOrDefault();
|
||
if (choice == null) break;
|
||
|
||
if (choice.FinishReason == "stop")
|
||
{
|
||
await foreach (var chunk in llmClient.ChatStreamAsync(messages, tools: null, ct: ct))
|
||
{
|
||
try
|
||
{
|
||
var delta = JsonSerializer.Deserialize<ChatCompletionResponse>(chunk, JsonOpts);
|
||
var content = delta?.Choices?.FirstOrDefault()?.Delta?.Content;
|
||
if (!string.IsNullOrEmpty(content))
|
||
{
|
||
fullResponse += content;
|
||
await SseWriteAsync(http, new { action = "answer", data = content, type = messageType }, ct);
|
||
}
|
||
}
|
||
catch (JsonException) { /* 跳过解析失败的 chunk */ }
|
||
}
|
||
completedNormally = true;
|
||
break;
|
||
}
|
||
else if (choice.FinishReason == "tool_calls" && choice.Message?.ToolCalls != null)
|
||
{
|
||
messages.Add(new ChatMessage
|
||
{
|
||
Role = "assistant",
|
||
Content = choice.Message.Content ?? "",
|
||
ToolCalls = choice.Message.ToolCalls,
|
||
});
|
||
|
||
foreach (var tc in choice.Message.ToolCalls)
|
||
{
|
||
object toolResult;
|
||
try
|
||
{
|
||
toolResult = await ExecuteToolCall(tc.Function.Name, tc.Function.Arguments, db, userId.Value, visionClient);
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
toolResult = new { success = false, message = $"工具执行异常: {ex.Message}" };
|
||
}
|
||
await SseWriteAsync(http, new { action = "tool_result", tool = tc.Function.Name, data = toolResult }, ct);
|
||
|
||
_UpdateMessageTypeAndMetadata(tc.Function.Name, toolResult, ref messageType, ref metadata);
|
||
|
||
messages.Add(new ChatMessage { Role = "tool", Content = JsonSerializer.Serialize(toolResult, JsonOpts), ToolCallId = tc.Id });
|
||
}
|
||
}
|
||
else break;
|
||
}
|
||
|
||
// 保存 AI 回复
|
||
if (!string.IsNullOrEmpty(fullResponse))
|
||
{
|
||
db.ConversationMessages.Add(new ConversationMessage
|
||
{
|
||
Id = Guid.NewGuid(), ConversationId = conversation.Id, Role = MessageRole.Assistant,
|
||
Content = fullResponse, CreatedAt = DateTime.UtcNow,
|
||
});
|
||
conversation.MessageCount++;
|
||
conversation.Summary = fullResponse.Length > 100 ? fullResponse[..100] : fullResponse;
|
||
conversation.UpdatedAt = DateTime.UtcNow;
|
||
await db.SaveChangesAsync(ct);
|
||
}
|
||
|
||
await SseWriteAsync(http, new { action = "status", data = completedNormally ? "done" : "error" }, ct);
|
||
await http.Response.WriteAsync("data: [DONE]\n\n", ct);
|
||
});
|
||
|
||
// 获取对话列表
|
||
app.MapGet("/api/ai/conversations", async (HttpContext http, AppDbContext db, CancellationToken ct) =>
|
||
{
|
||
var userId = GetUserId(http);
|
||
if (userId == null) return Results.Json(new { code = 40002, data = (object?)null, message = "未登录" }, statusCode: 401);
|
||
|
||
var conversations = await db.Conversations
|
||
.Where(c => c.UserId == userId.Value)
|
||
.OrderByDescending(c => c.UpdatedAt)
|
||
.Select(c => new { c.Id, AgentType = c.AgentType.ToString(), c.Title, c.Summary, c.MessageCount, c.CreatedAt, c.UpdatedAt })
|
||
.ToListAsync(ct);
|
||
|
||
return Results.Ok(new { code = 0, data = conversations, message = (string?)null });
|
||
});
|
||
|
||
// 获取对话历史
|
||
app.MapGet("/api/ai/conversations/{id:guid}", async (Guid id, HttpContext http, AppDbContext db, CancellationToken ct) =>
|
||
{
|
||
var userId = GetUserId(http);
|
||
if (userId == null) return Results.Json(new { code = 40002 }, statusCode: 401);
|
||
|
||
var messages = await db.ConversationMessages
|
||
.Where(m => m.ConversationId == id && m.Conversation.UserId == userId.Value)
|
||
.OrderBy(m => m.CreatedAt)
|
||
.Select(m => new { m.Id, Role = m.Role.ToString(), m.Content, m.Intent, m.MetadataJson, m.CreatedAt })
|
||
.ToListAsync(ct);
|
||
|
||
return Results.Ok(new { code = 0, data = messages, message = (string?)null });
|
||
});
|
||
|
||
// 删除对话
|
||
app.MapDelete("/api/ai/conversations/{id:guid}", async (Guid id, HttpContext http, AppDbContext db, CancellationToken ct) =>
|
||
{
|
||
var userId = GetUserId(http);
|
||
if (userId == null) return Results.Json(new { code = 40002 }, statusCode: 401);
|
||
|
||
var conv = await db.Conversations.FirstOrDefaultAsync(c => c.Id == id && c.UserId == userId.Value, ct);
|
||
if (conv != null)
|
||
{
|
||
db.Conversations.Remove(conv);
|
||
await db.SaveChangesAsync(ct);
|
||
}
|
||
return Results.Ok(new { code = 0, data = new { success = true }, message = (string?)null });
|
||
});
|
||
|
||
// VLM 食物识别
|
||
app.MapPost("/api/ai/analyze-food-image", async (
|
||
HttpRequest httpRequest, HttpContext http,
|
||
VisionClient visionClient, AppDbContext db,
|
||
CancellationToken ct) =>
|
||
{
|
||
var userId = GetUserId(http);
|
||
if (userId == null) return Results.Json(new { code = 40002 }, statusCode: 401);
|
||
|
||
var form = await httpRequest.ReadFormAsync(ct);
|
||
var files = form.Files.GetFiles("images");
|
||
if (files == null || files.Count == 0)
|
||
return Results.Ok(new { code = 40001, data = (object?)null, message = "请上传至少一张图片" });
|
||
|
||
var imageUrls = new List<string>();
|
||
var uploadsDir = Path.Combine(Directory.GetCurrentDirectory(), "uploads");
|
||
Directory.CreateDirectory(uploadsDir);
|
||
|
||
foreach (var file in files)
|
||
{
|
||
if (file.Length > 20 * 1024 * 1024)
|
||
return Results.Ok(new { code = 40001, data = (object?)null, message = "文件大小超过 20MB 限制" });
|
||
|
||
var ext = Path.GetExtension(file.FileName).ToLowerInvariant();
|
||
if (ext is not ".jpg" and not ".jpeg" and not ".png" and not ".heic")
|
||
return Results.Ok(new { code = 40001, data = (object?)null, message = "不支持的图片格式，仅支持 JPG/PNG/HEIC" });
|
||
|
||
var safeName = $"{Guid.NewGuid()}_{Path.GetFileName(file.FileName)}";
|
||
var filePath = Path.Combine(uploadsDir, safeName);
|
||
using (var stream = new FileStream(filePath, FileMode.Create))
|
||
await file.CopyToAsync(stream, ct);
|
||
|
||
var compressedPath = Path.Combine(uploadsDir, $"compressed_{safeName}");
|
||
CompressImage(filePath, compressedPath, maxWidth: 1280, quality: 85L);
|
||
var compressedBytes = await File.ReadAllBytesAsync(compressedPath, ct);
|
||
var base64 = Convert.ToBase64String(compressedBytes);
|
||
imageUrls.Add($"data:image/jpeg;base64,{base64}");
|
||
}
|
||
|
||
var prompt = """
|
||
Describe everything you see in this image in detail. What objects are present? What are their colors, shapes, labels?
|
||
If there are any food items, drinks, or beverages, identify them specifically by name.
|
||
Then, for each food/drink item, estimate the portion size and calories (kcal).
|
||
Return ONLY a JSON array: [{"name":"item name","portion":"estimated portion","calories":estimated_calories}]
|
||
If you cannot identify something, say so in the name field.
|
||
""";
|
||
|
||
try
|
||
{
|
||
var response = await visionClient.VisionAsync(prompt, imageUrls, userText: "请看图识别食物", maxTokens: 8192, ct: ct);
|
||
var result = response.Choices?.FirstOrDefault()?.Message?.Content ?? "{}";
|
||
return Results.Ok(new { code = 0, data = result, message = (string?)null });
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
return Results.Ok(new { code = 50001, data = (object?)null, message = $"食物识别失败：{ex.Message}" });
|
||
}
|
||
});
|
||
}
|
||
|
||
// ── SSE / 认证辅助 ──
|
||
|
||
private static async Task SseWriteAsync(HttpContext http, object data, CancellationToken ct)
|
||
{
|
||
var json = JsonSerializer.Serialize(data, JsonOpts);
|
||
await http.Response.WriteAsync($"data: {json}\n\n", ct);
|
||
await http.Response.Body.FlushAsync(ct);
|
||
}
|
||
|
||
private static Guid? GetUserId(HttpContext http) =>
|
||
Guid.TryParse(http.User.FindFirst(System.Security.Claims.ClaimTypes.NameIdentifier)?.Value, out var id) ? id : null;
|
||
|
||
private static Guid? GetUserIdFromToken(string? token)
|
||
{
|
||
if (string.IsNullOrEmpty(token)) return null;
|
||
try
|
||
{
|
||
var handler = new System.IdentityModel.Tokens.Jwt.JwtSecurityTokenHandler();
|
||
var jwt = handler.ReadJwtToken(token);
|
||
var sub = jwt.Claims.FirstOrDefault(c => c.Type == System.Security.Claims.ClaimTypes.NameIdentifier)?.Value;
|
||
return sub != null && Guid.TryParse(sub, out var id) ? id : null;
|
||
}
|
||
catch (Exception) { return null; }
|
||
}
|
||
|
||
// ── Agent / Tool 调度 ──
|
||
|
||
private static List<ToolDefinition> GetToolsForAgent(AgentType agentType) => agentType switch
|
||
{
|
||
AgentType.Health => HealthDataAgentHandler.Tools,
|
||
AgentType.Medication => MedicationAgentHandler.Tools,
|
||
AgentType.Diet => DietAgentHandler.Tools,
|
||
AgentType.Consultation => ConsultationAgentHandler.Tools,
|
||
AgentType.Report => ReportAgentHandler.Tools,
|
||
AgentType.Exercise => ExerciseAgentHandler.Tools,
|
||
AgentType.Onboarding => [CommonAgentHandler.ManageArchiveTool, CommonAgentHandler.CheckArchiveTool],
|
||
_ => CommonAgentHandler.Tools,
|
||
};
|
||
|
||
private static Task<object> ExecuteToolCall(string toolName, string arguments, AppDbContext db, Guid userId, VisionClient? visionClient = null)
|
||
{
|
||
using var jsonDoc = JsonDocument.Parse(arguments);
|
||
var root = jsonDoc.RootElement;
|
||
|
||
return toolName switch
|
||
{
|
||
"record_health_data" => HealthDataAgentHandler.Execute(toolName, root, db, userId),
|
||
"query_health_records" => CommonAgentHandler.Execute(toolName, root, db, userId),
|
||
"check_archive" => CommonAgentHandler.Execute(toolName, root, db, userId),
|
||
"manage_medication" => MedicationAgentHandler.Execute(toolName, root, db, userId),
|
||
"estimate_food_text" => DietAgentHandler.Execute(toolName, root, db, userId),
|
||
"analyze_report" => visionClient != null
|
||
? ReportAgentHandler.AnalyzeReport(db, userId, root, visionClient)
|
||
: Task.FromResult<object>(new { success = false, message = "VLM服务未配置" }),
|
||
"manage_exercise" => ExerciseAgentHandler.Execute(toolName, root, db, userId),
|
||
"manage_archive" => CommonAgentHandler.ExecuteManageArchive(db, userId, root),
|
||
"request_doctor" => ConsultationAgentHandler.Execute(toolName, root, db, userId),
|
||
_ => Task.FromResult<object>(new { success = false, message = $"未知工具: {toolName}" })
|
||
};
|
||
}
|
||
|
||
// ── 患者上下文构建 ──
|
||
|
||
private static async Task<string> BuildPatientContext(AppDbContext db, Guid userId, CancellationToken ct)
|
||
{
|
||
var archive = await db.HealthArchives.FirstOrDefaultAsync(a => a.UserId == userId, ct);
|
||
var recentRecords = await db.HealthRecords.Where(r => r.UserId == userId)
|
||
.OrderByDescending(r => r.RecordedAt).Take(10).ToListAsync(ct);
|
||
|
||
var sb = new System.Text.StringBuilder();
|
||
if (archive != null)
|
||
{
|
||
if (!string.IsNullOrEmpty(archive.Diagnosis)) sb.AppendLine($"诊断: {archive.Diagnosis}");
|
||
if (!string.IsNullOrEmpty(archive.SurgeryType)) sb.AppendLine($"手术: {archive.SurgeryType} ({archive.SurgeryDate})");
|
||
if (archive.Allergies.Count > 0) sb.AppendLine($"过敏: {string.Join(", ", archive.Allergies)}");
|
||
if (archive.DietRestrictions.Count > 0) sb.AppendLine($"饮食限制: {string.Join(", ", archive.DietRestrictions)}");
|
||
}
|
||
if (recentRecords.Count > 0)
|
||
{
|
||
sb.AppendLine("近期健康数据:");
|
||
foreach (var r in recentRecords)
|
||
sb.AppendLine($" {r.MetricType}: {RecordValue(r)} ({r.RecordedAt:MM-dd HH:mm})");
|
||
}
|
||
return sb.ToString();
|
||
}
|
||
|
||
private static string RecordValue(HealthRecord r) => r.MetricType switch
|
||
{
|
||
HealthMetricType.BloodPressure => $"{r.Systolic}/{r.Diastolic}",
|
||
HealthMetricType.HeartRate => $"{r.Value}次/分",
|
||
HealthMetricType.Glucose => $"{r.Value}",
|
||
HealthMetricType.SpO2 => $"{r.Value}%",
|
||
HealthMetricType.Weight => $"{r.Value}kg",
|
||
_ => "—"
|
||
};
|
||
|
||
// ── 消息类型判断 ──
|
||
|
||
private static void _UpdateMessageTypeAndMetadata(string toolName, object toolResult, ref string messageType, ref Dictionary<string, object> metadata)
|
||
{
|
||
switch (toolName)
|
||
{
|
||
case "record_health_data":
|
||
messageType = "data_confirm";
|
||
if (toolResult is IDictionary<string, object> resultDict)
|
||
{
|
||
if (resultDict.TryGetValue("type", out var type))
|
||
metadata["type"] = type.ToString();
|
||
if (resultDict.TryGetValue("success", out var success) && success is bool b && b)
|
||
metadata["success"] = true;
|
||
}
|
||
break;
|
||
case "manage_medication":
|
||
messageType = "medication_confirm";
|
||
if (toolResult is IDictionary<string, object> medDict)
|
||
{
|
||
if (medDict.TryGetValue("name", out var name))
|
||
metadata["name"] = name.ToString();
|
||
if (medDict.TryGetValue("dosage", out var dosage))
|
||
metadata["dosage"] = dosage.ToString();
|
||
}
|
||
break;
|
||
case "estimate_food_text":
|
||
messageType = "diet_analysis";
|
||
break;
|
||
case "analyze_report":
|
||
messageType = "report_analysis";
|
||
break;
|
||
}
|
||
}
|
||
|
||
// ── 图片处理 ──
|
||
|
||
private static void CompressImage(string inputPath, string outputPath, int maxWidth, long quality)
|
||
{
|
||
using var image = Image.FromFile(inputPath);
|
||
var width = image.Width;
|
||
var height = image.Height;
|
||
if (width > maxWidth)
|
||
{
|
||
height = (int)((double)height / width * maxWidth);
|
||
width = maxWidth;
|
||
}
|
||
using var bitmap = new Bitmap(width, height);
|
||
using var graphics = Graphics.FromImage(bitmap);
|
||
graphics.InterpolationMode = System.Drawing.Drawing2D.InterpolationMode.HighQualityBicubic;
|
||
graphics.DrawImage(image, 0, 0, width, height);
|
||
|
||
var jpegCodec = ImageCodecInfo.GetImageEncoders().First(c => c.MimeType == "image/jpeg");
|
||
var parameters = new EncoderParameters(1);
|
||
parameters.Param[0] = new EncoderParameter(Encoder.Quality, quality);
|
||
bitmap.Save(outputPath, jpegCodec, parameters);
|
||
}
|
||
}
|
||
|
||
public sealed record ChatRequest(string Message, string? ConversationId);
|