using System.Drawing; using System.Drawing.Imaging; using Health.Infrastructure.AI; using Health.Infrastructure.AI.AgentHandlers; namespace Health.WebApi.Endpoints; /// /// AI 对话 SSE 端点——支持 7 个 Agent /// public static class AiChatEndpoints { private static readonly JsonSerializerOptions JsonOpts = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, PropertyNameCaseInsensitive = true, }; public static void MapAiChatEndpoints(this WebApplication app) { // SSE 流式对话(GET 方式,token 通过 query string 传递) app.MapGet("/api/ai/{agentType}/chat", async ( string message, string? conversationId, string token, string agentType, HttpContext http, AppDbContext db, DeepSeekClient llmClient, PromptManager promptManager, VisionClient visionClient, CancellationToken ct) => { // 支持 token 通过 query string(浏览器 EventSource)或 header 传递 var userId = GetUserId(http) ?? GetUserIdFromToken(token); if (userId == null) { http.Response.StatusCode = 401; http.Response.ContentType = "application/json"; await http.Response.WriteAsync(JsonSerializer.Serialize(new { code = 40002, data = (object?)null, message = "未登录" }), ct); return; } if (!Enum.TryParse(agentType, ignoreCase: true, out var parsedType)) parsedType = AgentType.Default; // SSE 响应头 http.Response.ContentType = "text/event-stream"; http.Response.Headers.CacheControl = "no-cache"; http.Response.Headers.Connection = "keep-alive"; http.Response.Headers["X-Accel-Buffering"] = "no"; // 创建或获取对话 Conversation? conversation = null; if (!string.IsNullOrEmpty(conversationId) && Guid.TryParse(conversationId, out var convId)) conversation = await db.Conversations.FindAsync([convId], ct); if (conversation == null) { conversation = new Conversation { Id = Guid.NewGuid(), UserId = userId.Value, AgentType = parsedType, Title = message.Length > 30 ? message[..30] : message, CreatedAt = DateTime.UtcNow, UpdatedAt = DateTime.UtcNow, }; db.Conversations.Add(conversation); await db.SaveChangesAsync(ct); await SseWriteAsync(http, new { action = "conversation_id", data = conversation.Id.ToString() }, ct); } // 保存用户消息 var userMsg = new ConversationMessage { Id = Guid.NewGuid(), ConversationId = conversation.Id, Role = MessageRole.User, Content = message, CreatedAt = DateTime.UtcNow, }; db.ConversationMessages.Add(userMsg); conversation.MessageCount++; conversation.UpdatedAt = DateTime.UtcNow; await db.SaveChangesAsync(ct); // 加载上下文 var systemPrompt = promptManager.GetSystemPrompt(parsedType); var patientContext = await BuildPatientContext(db, userId.Value, ct); var messages = new List { new() { Role = "system", Content = systemPrompt + "\n\n当前患者信息:\n" + patientContext }, }; // 加载历史对话(最近 10 条) var history = await db.ConversationMessages .Where(m => m.ConversationId == conversation.Id) .OrderByDescending(m => m.CreatedAt) .Take(12) .ToListAsync(ct); foreach (var h in history.Reverse()) { messages.Add(new ChatMessage { Role = h.Role == MessageRole.User ? "user" : "assistant", Content = h.Content, }); } // Tool Calling 循环 var tools = GetToolsForAgent(parsedType); var maxIterations = 5; var fullResponse = ""; var completedNormally = false; var messageType = "text"; var metadata = new Dictionary(); for (int i = 0; i < maxIterations; i++) { await SseWriteAsync(http, new { action = "notice", message = i == 0 ? "正在分析..." : "正在处理..." }, ct); var response = await llmClient.ChatAsync(messages, tools: tools.Count > 0 ? tools : null, ct: ct); var choice = response.Choices?.FirstOrDefault(); if (choice == null) break; if (choice.FinishReason == "stop") { await foreach (var chunk in llmClient.ChatStreamAsync(messages, tools: null, ct: ct)) { try { var delta = JsonSerializer.Deserialize(chunk, JsonOpts); var content = delta?.Choices?.FirstOrDefault()?.Delta?.Content; if (!string.IsNullOrEmpty(content)) { fullResponse += content; await SseWriteAsync(http, new { action = "answer", data = content, type = messageType }, ct); } } catch (JsonException) { /* 跳过解析失败的 chunk */ } } completedNormally = true; break; } else if (choice.FinishReason == "tool_calls" && choice.Message?.ToolCalls != null) { messages.Add(new ChatMessage { Role = "assistant", Content = choice.Message.Content ?? "", ToolCalls = choice.Message.ToolCalls, }); foreach (var tc in choice.Message.ToolCalls) { object toolResult; try { toolResult = await ExecuteToolCall(tc.Function.Name, tc.Function.Arguments, db, userId.Value, visionClient); } catch (Exception ex) { toolResult = new { success = false, message = $"工具执行异常: {ex.Message}" }; } await SseWriteAsync(http, new { action = "tool_result", tool = tc.Function.Name, data = toolResult }, ct); _UpdateMessageTypeAndMetadata(tc.Function.Name, toolResult, ref messageType, ref metadata); messages.Add(new ChatMessage { Role = "tool", Content = JsonSerializer.Serialize(toolResult, JsonOpts), ToolCallId = tc.Id }); } } else break; } // 保存 AI 回复 if (!string.IsNullOrEmpty(fullResponse)) { db.ConversationMessages.Add(new ConversationMessage { Id = Guid.NewGuid(), ConversationId = conversation.Id, Role = MessageRole.Assistant, Content = fullResponse, CreatedAt = DateTime.UtcNow, }); conversation.MessageCount++; conversation.Summary = fullResponse.Length > 100 ? fullResponse[..100] : fullResponse; conversation.UpdatedAt = DateTime.UtcNow; await db.SaveChangesAsync(ct); } await SseWriteAsync(http, new { action = "status", data = completedNormally ? "done" : "error" }, ct); await http.Response.WriteAsync("data: [DONE]\n\n", ct); }); // 获取对话列表 app.MapGet("/api/ai/conversations", async (HttpContext http, AppDbContext db, CancellationToken ct) => { var userId = GetUserId(http); if (userId == null) return Results.Json(new { code = 40002, data = (object?)null, message = "未登录" }, statusCode: 401); var conversations = await db.Conversations .Where(c => c.UserId == userId.Value) .OrderByDescending(c => c.UpdatedAt) .Select(c => new { c.Id, AgentType = c.AgentType.ToString(), c.Title, c.Summary, c.MessageCount, c.CreatedAt, c.UpdatedAt }) .ToListAsync(ct); return Results.Ok(new { code = 0, data = conversations, message = (string?)null }); }); // 获取对话历史 app.MapGet("/api/ai/conversations/{id:guid}", async (Guid id, HttpContext http, AppDbContext db, CancellationToken ct) => { var userId = GetUserId(http); if (userId == null) return Results.Json(new { code = 40002 }, statusCode: 401); var messages = await db.ConversationMessages .Where(m => m.ConversationId == id && m.Conversation.UserId == userId.Value) .OrderBy(m => m.CreatedAt) .Select(m => new { m.Id, Role = m.Role.ToString(), m.Content, m.Intent, m.MetadataJson, m.CreatedAt }) .ToListAsync(ct); return Results.Ok(new { code = 0, data = messages, message = (string?)null }); }); // 删除对话 app.MapDelete("/api/ai/conversations/{id:guid}", async (Guid id, HttpContext http, AppDbContext db, CancellationToken ct) => { var userId = GetUserId(http); if (userId == null) return Results.Json(new { code = 40002 }, statusCode: 401); var conv = await db.Conversations.FirstOrDefaultAsync(c => c.Id == id && c.UserId == userId.Value, ct); if (conv != null) { db.Conversations.Remove(conv); await db.SaveChangesAsync(ct); } return Results.Ok(new { code = 0, data = new { success = true }, message = (string?)null }); }); // VLM 食物识别 app.MapPost("/api/ai/analyze-food-image", async ( HttpRequest httpRequest, HttpContext http, VisionClient visionClient, AppDbContext db, CancellationToken ct) => { var userId = GetUserId(http); if (userId == null) return Results.Json(new { code = 40002 }, statusCode: 401); var form = await httpRequest.ReadFormAsync(ct); var files = form.Files.GetFiles("images"); if (files == null || files.Count == 0) return Results.Ok(new { code = 40001, data = (object?)null, message = "请上传至少一张图片" }); var imageUrls = new List(); var uploadsDir = Path.Combine(Directory.GetCurrentDirectory(), "uploads"); Directory.CreateDirectory(uploadsDir); foreach (var file in files) { if (file.Length > 20 * 1024 * 1024) return Results.Ok(new { code = 40001, data = (object?)null, message = "文件大小超过 20MB 限制" }); var ext = Path.GetExtension(file.FileName).ToLowerInvariant(); if (ext is not ".jpg" and not ".jpeg" and not ".png" and not ".heic") return Results.Ok(new { code = 40001, data = (object?)null, message = "不支持的图片格式,仅支持 JPG/PNG/HEIC" }); var safeName = $"{Guid.NewGuid()}_{Path.GetFileName(file.FileName)}"; var filePath = Path.Combine(uploadsDir, safeName); using (var stream = new FileStream(filePath, FileMode.Create)) await file.CopyToAsync(stream, ct); var compressedPath = Path.Combine(uploadsDir, $"compressed_{safeName}"); CompressImage(filePath, compressedPath, maxWidth: 1280, quality: 85L); var compressedBytes = await File.ReadAllBytesAsync(compressedPath, ct); var base64 = Convert.ToBase64String(compressedBytes); imageUrls.Add($"data:image/jpeg;base64,{base64}"); } var prompt = """ Describe everything you see in this image in detail. What objects are present? What are their colors, shapes, labels? If there are any food items, drinks, or beverages, identify them specifically by name. Then, for each food/drink item, estimate the portion size and calories (kcal). Return ONLY a JSON array: [{"name":"item name","portion":"estimated portion","calories":estimated_calories}] If you cannot identify something, say so in the name field. """; try { var response = await visionClient.VisionAsync(prompt, imageUrls, userText: "请看图识别食物", maxTokens: 8192, ct: ct); var result = response.Choices?.FirstOrDefault()?.Message?.Content ?? "{}"; return Results.Ok(new { code = 0, data = result, message = (string?)null }); } catch (Exception ex) { return Results.Ok(new { code = 50001, data = (object?)null, message = $"食物识别失败:{ex.Message}" }); } }); } // ── SSE / 认证辅助 ── private static async Task SseWriteAsync(HttpContext http, object data, CancellationToken ct) { var json = JsonSerializer.Serialize(data, JsonOpts); await http.Response.WriteAsync($"data: {json}\n\n", ct); await http.Response.Body.FlushAsync(ct); } private static Guid? GetUserId(HttpContext http) => Guid.TryParse(http.User.FindFirst(System.Security.Claims.ClaimTypes.NameIdentifier)?.Value, out var id) ? id : null; private static Guid? GetUserIdFromToken(string? token) { if (string.IsNullOrEmpty(token)) return null; try { var handler = new System.IdentityModel.Tokens.Jwt.JwtSecurityTokenHandler(); var jwt = handler.ReadJwtToken(token); var sub = jwt.Claims.FirstOrDefault(c => c.Type == System.Security.Claims.ClaimTypes.NameIdentifier)?.Value; return sub != null && Guid.TryParse(sub, out var id) ? id : null; } catch (Exception) { return null; } } // ── Agent / Tool 调度 ── private static List GetToolsForAgent(AgentType agentType) => agentType switch { AgentType.Health => HealthDataAgentHandler.Tools, AgentType.Medication => MedicationAgentHandler.Tools, AgentType.Diet => DietAgentHandler.Tools, AgentType.Consultation => ConsultationAgentHandler.Tools, AgentType.Report => ReportAgentHandler.Tools, AgentType.Exercise => ExerciseAgentHandler.Tools, AgentType.Onboarding => [CommonAgentHandler.ManageArchiveTool, CommonAgentHandler.CheckArchiveTool], _ => CommonAgentHandler.Tools, }; private static Task ExecuteToolCall(string toolName, string arguments, AppDbContext db, Guid userId, VisionClient? visionClient = null) { using var jsonDoc = JsonDocument.Parse(arguments); var root = jsonDoc.RootElement; return toolName switch { "record_health_data" => HealthDataAgentHandler.Execute(toolName, root, db, userId), "query_health_records" => CommonAgentHandler.Execute(toolName, root, db, userId), "check_archive" => CommonAgentHandler.Execute(toolName, root, db, userId), "manage_medication" => MedicationAgentHandler.Execute(toolName, root, db, userId), "estimate_food_text" => DietAgentHandler.Execute(toolName, root, db, userId), "analyze_report" => visionClient != null ? ReportAgentHandler.AnalyzeReport(db, userId, root, visionClient) : Task.FromResult(new { success = false, message = "VLM服务未配置" }), "manage_exercise" => ExerciseAgentHandler.Execute(toolName, root, db, userId), "manage_archive" => CommonAgentHandler.ExecuteManageArchive(db, userId, root), "request_doctor" => ConsultationAgentHandler.Execute(toolName, root, db, userId), _ => Task.FromResult(new { success = false, message = $"未知工具: {toolName}" }) }; } // ── 患者上下文构建 ── private static async Task BuildPatientContext(AppDbContext db, Guid userId, CancellationToken ct) { var archive = await db.HealthArchives.FirstOrDefaultAsync(a => a.UserId == userId, ct); var recentRecords = await db.HealthRecords.Where(r => r.UserId == userId) .OrderByDescending(r => r.RecordedAt).Take(10).ToListAsync(ct); var sb = new System.Text.StringBuilder(); if (archive != null) { if (!string.IsNullOrEmpty(archive.Diagnosis)) sb.AppendLine($"诊断: {archive.Diagnosis}"); if (!string.IsNullOrEmpty(archive.SurgeryType)) sb.AppendLine($"手术: {archive.SurgeryType} ({archive.SurgeryDate})"); if (archive.Allergies.Count > 0) sb.AppendLine($"过敏: {string.Join(", ", archive.Allergies)}"); if (archive.DietRestrictions.Count > 0) sb.AppendLine($"饮食限制: {string.Join(", ", archive.DietRestrictions)}"); } if (recentRecords.Count > 0) { sb.AppendLine("近期健康数据:"); foreach (var r in recentRecords) sb.AppendLine($" {r.MetricType}: {RecordValue(r)} ({r.RecordedAt:MM-dd HH:mm})"); } return sb.ToString(); } private static string RecordValue(HealthRecord r) => r.MetricType switch { HealthMetricType.BloodPressure => $"{r.Systolic}/{r.Diastolic}", HealthMetricType.HeartRate => $"{r.Value}次/分", HealthMetricType.Glucose => $"{r.Value}", HealthMetricType.SpO2 => $"{r.Value}%", HealthMetricType.Weight => $"{r.Value}kg", _ => "—" }; // ── 消息类型判断 ── private static void _UpdateMessageTypeAndMetadata(string toolName, object toolResult, ref string messageType, ref Dictionary metadata) { switch (toolName) { case "record_health_data": messageType = "data_confirm"; if (toolResult is IDictionary resultDict) { if (resultDict.TryGetValue("type", out var type)) metadata["type"] = type.ToString(); if (resultDict.TryGetValue("success", out var success) && success is bool b && b) metadata["success"] = true; } break; case "manage_medication": messageType = "medication_confirm"; if (toolResult is IDictionary medDict) { if (medDict.TryGetValue("name", out var name)) metadata["name"] = name.ToString(); if (medDict.TryGetValue("dosage", out var dosage)) metadata["dosage"] = dosage.ToString(); } break; case "estimate_food_text": messageType = "diet_analysis"; break; case "analyze_report": messageType = "report_analysis"; break; } } // ── 图片处理 ── private static void CompressImage(string inputPath, string outputPath, int maxWidth, long quality) { using var image = Image.FromFile(inputPath); var width = image.Width; var height = image.Height; if (width > maxWidth) { height = (int)((double)height / width * maxWidth); width = maxWidth; } using var bitmap = new Bitmap(width, height); using var graphics = Graphics.FromImage(bitmap); graphics.InterpolationMode = System.Drawing.Drawing2D.InterpolationMode.HighQualityBicubic; graphics.DrawImage(image, 0, 0, width, height); var jpegCodec = ImageCodecInfo.GetImageEncoders().First(c => c.MimeType == "image/jpeg"); var parameters = new EncoderParameters(1); parameters.Param[0] = new EncoderParameter(Encoder.Quality, quality); bitmap.Save(outputPath, jpegCodec, parameters); } } public sealed record ChatRequest(string Message, string? ConversationId);