jiaolongzhen/AI-Health
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: 4层架构重构 + 饮食VLM接入 + 多项修复

Browse Source 
- 后端: remaining_endpoints拆分为6个独立文件
- 后端: AI Agent Handler从ai_chat_endpoints抽取为7个独立处理器
- 后端: 食物识别prompt改为输出结构化JSON
- 前端: 饮食识别从Mock替换为真实VLM API调用
- 前端: 首页图片上传URL修复(/api/upload→/api/files/upload)
- 前端: 拍饮食按钮导航到独立DietCapturePage
- 前端: 删除无用agent_bar.dart
- 前端: 修复widget_test.dart过期属性名
- 前端: 恢复ServicePackageCard和详情页
- 新增6份实施文档(情况/问诊/报告/建档/日历/视觉统一)
This commit is contained in:
MingNian
2026-06-03 23:17:37 +08:00
parent 5bd0155e17
commit c2399b952f
33 changed files with 3311 additions and 660 deletions
Download Patch File Download Diff File Expand all files Collapse all files

278
backend/src/Health.WebApi/Endpoints/ai_chat_endpoints.cs
View File

@@ -1,6 +1,7 @@
using System.Drawing;
using System.Drawing.Imaging;
using Health.Infrastructure.AI;
using Health.Infrastructure.AI.AgentHandlers;
namespace Health.WebApi.Endpoints;
@@ -121,7 +122,6 @@ public static class AiChatEndpoints
if (choice.FinishReason == "stop")
{
// 流式输出最终回复（带上完整的 tool call 历史，方便 LLM 利用工具结果生成回复）
await foreach (var chunk in llmClient.ChatStreamAsync(messages, tools: null, ct: ct))
{
try
@@ -141,7 +141,6 @@ public static class AiChatEndpoints
}
else if (choice.FinishReason == "tool_calls" && choice.Message?.ToolCalls != null)
{
// 一条 assistant 消息包含所有 tool calls（符合 OpenAI 协议）
messages.Add(new ChatMessage
{
Role = "assistant",
@@ -265,7 +264,6 @@ public static class AiChatEndpoints
using (var stream = new FileStream(filePath, FileMode.Create))
await file.CopyToAsync(stream, ct);
// 压缩图片后转 base64（VLM API 有请求体大小限制）
var compressedPath = Path.Combine(uploadsDir, $"compressed_{safeName}");
CompressImage(filePath, compressedPath, maxWidth: 1280, quality: 75L);
var compressedBytes = await File.ReadAllBytesAsync(compressedPath, ct);
@@ -273,7 +271,14 @@ public static class AiChatEndpoints
imageUrls.Add($"data:image/jpeg;base64,{base64}");
}
var prompt = "精准识别用户提供的食物图片，提取并返回详细信息，包括但不限于食物名称、具体份量及对应热量值。系统应确保识别结果的准确性和清晰度，以便为病人的饮食管理提供可靠数据支持。";
var prompt = """
识别图片中的食物，以 JSON 数组格式返回，每项包含：
- name: 食物名称
- portion: 份量描述（如"约1碗"、"约200g"）
- calories: 估算热量（千卡，整数）
只返回 JSON 数组，不要任何其他文字。
示例：[{"name":"米饭","portion":"约1碗","calories":150},{"name":"番茄炒蛋","portion":"约1份","calories":200},{...}]
""";
try
{
@@ -288,6 +293,8 @@ public static class AiChatEndpoints
});
}
// ── SSE / 认证辅助 ──
private static async Task SseWriteAsync(HttpContext http, object data, CancellationToken ct)
{
var json = JsonSerializer.Serialize(data, JsonOpts);
@@ -298,7 +305,6 @@ public static class AiChatEndpoints
private static Guid? GetUserId(HttpContext http) =>
Guid.TryParse(http.User.FindFirst(System.Security.Claims.ClaimTypes.NameIdentifier)?.Value, out var id) ? id : null;
/// 从 query string token 解析用户 ID（浏览器 EventSource 用）
private static Guid? GetUserIdFromToken(string? token)
{
if (string.IsNullOrEmpty(token)) return null;
@@ -312,202 +318,39 @@ public static class AiChatEndpoints
catch (Exception) { return null; }
}
// ── Agent / Tool 调度 ──
private static List<ToolDefinition> GetToolsForAgent(AgentType agentType) => agentType switch
{
AgentType.Health => [RecordHealthDataTool, QueryHealthRecordsTool],
AgentType.Medication => [ManageMedicationTool, CheckArchiveTool],
AgentType.Diet => [EstimateFoodTool, CheckArchiveTool],
AgentType.Consultation => [QueryHealthRecordsTool, CheckArchiveTool, RequestDoctorTool],
AgentType.Report => [AnalyzeReportTool, QueryHealthRecordsTool],
AgentType.Exercise => [ManageExerciseTool],
_ => [QueryHealthRecordsTool, CheckArchiveTool],
AgentType.Health => HealthDataAgentHandler.Tools,
AgentType.Medication => MedicationAgentHandler.Tools,
AgentType.Diet => DietAgentHandler.Tools,
AgentType.Consultation => ConsultationAgentHandler.Tools,
AgentType.Report => ReportAgentHandler.Tools,
AgentType.Exercise => ExerciseAgentHandler.Tools,
_ => CommonAgentHandler.Tools,
};
private static async Task<object> ExecuteToolCall(string toolName, string arguments, AppDbContext db, Guid userId)
private static Task<object> ExecuteToolCall(string toolName, string arguments, AppDbContext db, Guid userId)
{
using var jsonDoc = JsonDocument.Parse(arguments);
var root = jsonDoc.RootElement;
return toolName switch
{
"record_health_data" => await ExecuteRecordHealthData(db, userId, root),
"query_health_records" => await ExecuteQueryHealthRecords(db, userId, root),
"check_archive" => await ExecuteCheckArchive(db, userId),
"manage_medication" => await ExecuteManageMedication(db, userId, root),
"manage_exercise" => await ExecuteManageExercise(db, userId, root),
_ => new { success = false, message = $"未知工具: {toolName}" }
"record_health_data" => HealthDataAgentHandler.Execute(toolName, root, db, userId),
"query_health_records" => CommonAgentHandler.Execute(toolName, root, db, userId),
"check_archive" => CommonAgentHandler.Execute(toolName, root, db, userId),
"manage_medication" => MedicationAgentHandler.Execute(toolName, root, db, userId),
"estimate_food_text" => DietAgentHandler.Execute(toolName, root, db, userId),
"analyze_report" => ReportAgentHandler.Execute(toolName, root, db, userId),
"manage_exercise" => ExerciseAgentHandler.Execute(toolName, root, db, userId),
"request_doctor" => ConsultationAgentHandler.Execute(toolName, root, db, userId),
_ => Task.FromResult<object>(new { success = false, message = $"未知工具: {toolName}" })
};
}
private static async Task<object> ExecuteRecordHealthData(AppDbContext db, Guid userId, JsonElement args)
{
var type = args.TryGetProperty("type", out var t) ? t.GetString()! : "";
var record = new HealthRecord
{
Id = Guid.NewGuid(), UserId = userId, Source = HealthRecordSource.AiEntry,
RecordedAt = args.TryGetProperty("recorded_at", out var ra) && ra.TryGetDateTime(out var dt) ? dt : DateTime.UtcNow,
CreatedAt = DateTime.UtcNow,
};
switch (type)
{
case "blood_pressure":
record.MetricType = HealthMetricType.BloodPressure;
record.Systolic = args.TryGetProperty("systolic", out var s) ? s.GetInt32() : null;
record.Diastolic = args.TryGetProperty("diastolic", out var d) ? d.GetInt32() : null;
record.Unit = "mmHg";
record.IsAbnormal = record.Systolic >= 140 || record.Diastolic >= 90 || record.Systolic <= 89 || record.Diastolic <= 59;
break;
case "heart_rate":
record.MetricType = HealthMetricType.HeartRate;
record.Value = args.TryGetProperty("heart_rate", out var hr) ? hr.GetDecimal() : null;
record.Unit = "次/分";
record.IsAbnormal = record.Value > 100 || record.Value < 60;
break;
case "glucose":
record.MetricType = HealthMetricType.Glucose;
record.Value = args.TryGetProperty("glucose", out var g) ? g.GetDecimal() : null;
record.Unit = "mmol/L";
record.IsAbnormal = record.Value >= 7.0m || record.Value <= 3.8m;
break;
case "spo2":
record.MetricType = HealthMetricType.SpO2;
record.Value = args.TryGetProperty("spo2", out var o) ? o.GetDecimal() : null;
record.Unit = "%";
record.IsAbnormal = record.Value <= 94;
break;
case "weight":
record.MetricType = HealthMetricType.Weight;
record.Value = args.TryGetProperty("weight", out var w) ? w.GetDecimal() : null;
record.Unit = "kg";
break;
default:
return new { success = false, message = $"未知指标类型: {type}" };
}
db.HealthRecords.Add(record);
await db.SaveChangesAsync();
return new { success = true, record_id = record.Id, type = record.MetricType.ToString() };
}
private static async Task<object> ExecuteQueryHealthRecords(AppDbContext db, Guid userId, JsonElement args)
{
var type = args.TryGetProperty("type", out var t) ? t.GetString() : null;
var days = args.TryGetProperty("days", out var d) ? d.GetInt32() : 7;
var query = db.HealthRecords.Where(r => r.UserId == userId);
if (!string.IsNullOrEmpty(type) && Enum.TryParse<HealthMetricType>(type, ignoreCase: true, out var mt))
query = query.Where(r => r.MetricType == mt);
query = query.Where(r => r.RecordedAt >= DateTime.UtcNow.AddDays(-days));
var records = await query.OrderByDescending(r => r.RecordedAt).Take(30).Select(r => new
{
r.Id, Type = r.MetricType.ToString(), r.Systolic, r.Diastolic, r.Value, r.Unit, r.IsAbnormal, r.RecordedAt,
}).ToListAsync();
return new { count = records.Count, records };
}
private static async Task<object> ExecuteCheckArchive(AppDbContext db, Guid userId)
{
var archive = await db.HealthArchives.FirstOrDefaultAsync(a => a.UserId == userId);
if (archive == null) return new { found = false };
return new
{
found = true, archive.Diagnosis, archive.SurgeryType,
SurgeryDate = archive.SurgeryDate?.ToString("yyyy-MM-dd"),
archive.Allergies, archive.DietRestrictions, archive.ChronicDiseases, archive.FamilyHistory,
};
}
private static async Task<object> ExecuteManageMedication(AppDbContext db, Guid userId, JsonElement args)
{
var action = args.TryGetProperty("action", out var a) ? a.GetString()! : "query";
return action switch
{
"create" => await CreateMedication(db, userId, args),
"query" => await QueryMedications(db, userId),
"confirm" => await ConfirmMedication(db, userId, args),
_ => new { success = false, message = $"未知操作: {action}" }
};
}
private static async Task<object> CreateMedication(AppDbContext db, Guid userId, JsonElement args)
{
var med = new Medication
{
Id = Guid.NewGuid(), UserId = userId,
Name = args.TryGetProperty("name", out var n) ? n.GetString()! : "",
Dosage = args.TryGetProperty("dosage", out var dg) ? dg.GetString() : null,
Source = MedicationSource.AiEntry, IsActive = true,
};
db.Medications.Add(med);
await db.SaveChangesAsync();
return new { success = true, medication_id = med.Id, med.Name };
}
private static async Task<object> QueryMedications(AppDbContext db, Guid userId)
{
var meds = await db.Medications.Where(m => m.UserId == userId && m.IsActive)
.Select(m => new { m.Id, m.Name, m.Dosage, m.TimeOfDay }).ToListAsync();
return new { count = meds.Count, medications = meds };
}
private static async Task<object> ConfirmMedication(AppDbContext db, Guid userId, JsonElement args)
{
var medId = args.TryGetProperty("medication_id", out var mid) ? mid.GetGuid() : Guid.Empty;
db.MedicationLogs.Add(new MedicationLog
{
Id = Guid.NewGuid(), MedicationId = medId, UserId = userId,
Status = MedicationLogStatus.Taken, ScheduledTime = TimeOnly.FromDateTime(DateTime.Now), ConfirmedAt = DateTime.UtcNow,
});
await db.SaveChangesAsync();
return new { success = true };
}
private static async Task<object> ExecuteManageExercise(AppDbContext db, Guid userId, JsonElement args)
{
var action = args.TryGetProperty("action", out var a) ? a.GetString()! : "query";
switch (action)
{
case "create":
var weekStart = args.TryGetProperty("week_start_date", out var wsd) ? DateOnly.Parse(wsd.GetString()!) : DateOnly.FromDateTime(DateTime.Now);
var plan = new ExercisePlan { Id = Guid.NewGuid(), UserId = userId, WeekStartDate = weekStart };
if (args.TryGetProperty("items", out var items))
{
foreach (var item in items.EnumerateArray())
{
plan.Items.Add(new ExercisePlanItem
{
Id = Guid.NewGuid(), DayOfWeek = item.GetProperty("day_of_week").GetInt32(),
ExerciseType = item.GetProperty("exercise_type").GetString() ?? "散步",
DurationMinutes = item.GetProperty("duration_minutes").GetInt32(),
IsRestDay = item.TryGetProperty("is_rest_day", out var rd) && rd.GetBoolean(),
});
}
}
db.ExercisePlans.Add(plan);
await db.SaveChangesAsync();
return new { success = true, plan_id = plan.Id };
case "checkin":
var itemId = args.TryGetProperty("item_id", out var iid) ? iid.GetGuid() : Guid.Empty;
var exerciseItem = await db.ExercisePlanItems.FindAsync([itemId]);
if (exerciseItem == null) return new { success = false, message = "条目不存在" };
exerciseItem.IsCompleted = true;
exerciseItem.CompletedAt = DateTime.UtcNow;
await db.SaveChangesAsync();
return new { success = true };
default: // query
var existingPlan = await db.ExercisePlans.Where(p => p.UserId == userId)
.OrderByDescending(p => p.WeekStartDate).FirstOrDefaultAsync();
if (existingPlan == null) return new { found = false };
var exerciseItems = await db.ExercisePlanItems.Where(i => i.PlanId == existingPlan.Id).OrderBy(i => i.DayOfWeek).ToListAsync();
return new { found = true, plan_id = existingPlan.Id, items = exerciseItems.Select(i => new { i.Id, i.DayOfWeek, i.ExerciseType, i.DurationMinutes, i.IsCompleted }) };
}
}
// ── 患者上下文构建 ──
private static async Task<string> BuildPatientContext(AppDbContext db, Guid userId, CancellationToken ct)
{
@@ -542,61 +385,8 @@ public static class AiChatEndpoints
_ => "—"
};
// ---- Tool Definitions ----
private static readonly ToolDefinition RecordHealthDataTool = new()
{
Function = new()
{
Name = "record_health_data", Description = "记录健康数据（血压/心率/血糖/血氧/体重）",
Parameters = new { type = "object", properties = new { type = new { type = "string" }, systolic = new { type = "integer" }, diastolic = new { type = "integer" }, heart_rate = new { type = "number" }, glucose = new { type = "number" }, spo2 = new { type = "number" }, weight = new { type = "number" } }, required = new[] { "type" } }
}
};
private static readonly ToolDefinition QueryHealthRecordsTool = new()
{
Function = new()
{
Name = "query_health_records", Description = "查询近期健康数据",
Parameters = new { type = "object", properties = new { type = new { type = "string" }, days = new { type = "integer" } } }
}
};
private static readonly ToolDefinition CheckArchiveTool = new()
{
Function = new() { Name = "check_archive", Description = "查询患者健康档案", Parameters = new { type = "object", properties = new { } } }
};
private static readonly ToolDefinition ManageMedicationTool = new()
{
Function = new()
{
Name = "manage_medication", Description = "用药管理",
Parameters = new { type = "object", properties = new { action = new { type = "string" }, name = new { type = "string" }, dosage = new { type = "string" } }, required = new[] { "action" } }
}
};
private static readonly ToolDefinition ManageExerciseTool = new()
{
Function = new()
{
Name = "manage_exercise", Description = "运动计划管理",
Parameters = new { type = "object", properties = new { action = new { type = "string" } }, required = new[] { "action" } }
}
};
private static readonly ToolDefinition EstimateFoodTool = new()
{
Function = new() { Name = "estimate_food_text", Description = "根据文字描述估算食物份量和热量", Parameters = new { type = "object", properties = new { text = new { type = "string" } }, required = new[] { "text" } } }
};
private static readonly ToolDefinition AnalyzeReportTool = new()
{
Function = new() { Name = "analyze_report", Description = "分析报告图片", Parameters = new { type = "object", properties = new { image_url = new { type = "string" } }, required = new[] { "image_url" } } }
};
private static readonly ToolDefinition RequestDoctorTool = new()
{
Function = new()
{
Name = "request_doctor", Description = "请求转接真人医生",
Parameters = new { type = "object", properties = new { reason = new { type = "string" }, urgency_level = new { type = "string" } } }
}
};
// ── 消息类型判断 ──
/// <summary>根据工具调用结果更新消息类型和元数据</summary>
private static void _UpdateMessageTypeAndMetadata(string toolName, object toolResult, ref string messageType, ref Dictionary<string, object> metadata)
{
switch (toolName)
@@ -630,7 +420,8 @@ public static class AiChatEndpoints
}
}
/// <summary>压缩图片到合理大小供 VLM API 使用</summary>
// ── 图片处理 ──
private static void CompressImage(string inputPath, string outputPath, int maxWidth, long quality)
{
using var image = Image.FromFile(inputPath);
@@ -653,5 +444,4 @@ public static class AiChatEndpoints
}
}
/// <summary>AI 对话请求</summary>
public sealed record ChatRequest(string Message, string? ConversationId);