From 27cc920a4a2f05f3e8e0f37758a56132f4a7b7c9 Mon Sep 17 00:00:00 2001 From: MingNian <1281442923@qq.com> Date: Tue, 2 Jun 2026 13:44:14 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20VLM=20=E9=A3=9F=E7=89=A9=E8=AF=86?= =?UTF-8?q?=E5=88=AB=E5=85=A8=E9=93=BE=E8=B7=AF=E4=BF=AE=E5=A4=8D=20+=20?= =?UTF-8?q?=E7=94=A8=E8=8D=AF=20Agent=20Prompt=20=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 图片上传自动压缩（max 860px, JPEG Q65），解决大图超 API 129KB 限制 - 修复 VLM 传 file:// 本地路径 Bug，改为 base64 data URL - VLM Prompt 优化为中文食堂场景，附带常见中餐热量参考 - 千问 API 错误信息透传，方便调试 - 用药 Agent Prompt 加查询规则：先调 manage_medication 再回答 - 新增 System.Drawing.Common 依赖用于服务端图片压缩 --- .../Health.Infrastructure/AI/ai_clients.cs | 6 +- .../AI/prompt_manager.cs | 11 ++-- .../Endpoints/ai_chat_endpoints.cs | 56 +++++++++++++++---- .../src/Health.WebApi/Health.WebApi.csproj | 1 + 4 files changed, 57 insertions(+), 17 deletions(-) diff --git a/backend/src/Health.Infrastructure/AI/ai_clients.cs b/backend/src/Health.Infrastructure/AI/ai_clients.cs index dad8f3d..0337c4d 100644 --- a/backend/src/Health.Infrastructure/AI/ai_clients.cs +++ b/backend/src/Health.Infrastructure/AI/ai_clients.cs @@ -129,7 +129,11 @@ public sealed class QwenVisionClient(HttpClient http, IConfiguration config) var json = JsonSerializer.Serialize(request, _jsonOptions); var content = new StringContent(json, Encoding.UTF8, "application/json"); var response = await _http.PostAsync("chat/completions", content, ct); - response.EnsureSuccessStatusCode(); + if (!response.IsSuccessStatusCode) + { + var errorBody = await response.Content.ReadAsStringAsync(ct); + throw new HttpRequestException($"VLM API {response.StatusCode}: {errorBody}"); + } var body = await response.Content.ReadAsStringAsync(ct); return JsonSerializer.Deserialize(body, _jsonOptions)!; } diff --git a/backend/src/Health.Infrastructure/AI/prompt_manager.cs b/backend/src/Health.Infrastructure/AI/prompt_manager.cs index 26f7217..17f0e88 100644 --- a/backend/src/Health.Infrastructure/AI/prompt_manager.cs +++ b/backend/src/Health.Infrastructure/AI/prompt_manager.cs @@ -83,11 +83,12 @@ public sealed class PromptManager 你是一个用药管理专家。 规则： - 1. 解析用户口中的药品信息（药名/剂量/频次/时间） - 2. "早饭后"等模糊时间→追问具体几点 - 3. 解析完成后展示确认卡片 - 4. 处方拍照→提取药品信息→生成用药计划→让用户确认 - 5. 回答用药相关疑问 + 1. 用户询问当前用药时，必须先调用 manage_medication(action="query") 查询已有用药记录 + 2. 解析用户口中的药品信息（药名/剂量/频次/时间） + 3. "早饭后"等模糊时间→追问具体几点 + 4. 解析完成后展示确认卡片 + 5. 处方拍照→提取药品信息→生成用药计划→让用户确认 + 6. 回答用药相关疑问 """; private const string ReportPrompt = """ diff --git a/backend/src/Health.WebApi/Endpoints/ai_chat_endpoints.cs b/backend/src/Health.WebApi/Endpoints/ai_chat_endpoints.cs index fe45e24..1076960 100644 --- a/backend/src/Health.WebApi/Endpoints/ai_chat_endpoints.cs +++ b/backend/src/Health.WebApi/Endpoints/ai_chat_endpoints.cs @@ -1,3 +1,5 @@ +using System.Drawing; +using System.Drawing.Imaging; using Health.Infrastructure.AI; namespace Health.WebApi.Endpoints; @@ -256,20 +258,30 @@ public static class AiChatEndpoints var safeName = $"{Guid.NewGuid()}_{Path.GetFileName(file.FileName)}"; var filePath = Path.Combine(uploadsDir, safeName); - using var stream = new FileStream(filePath, FileMode.Create); - await file.CopyToAsync(stream, ct); - imageUrls.Add($"file://{filePath}"); + using (var stream = new FileStream(filePath, FileMode.Create)) + await file.CopyToAsync(stream, ct); + + // 压缩图片后转 base64（VLM API 有请求体大小限制） + var compressedPath = Path.Combine(uploadsDir, $"compressed_{safeName}"); + CompressImage(filePath, compressedPath, maxWidth: 860, quality: 65L); + var compressedBytes = await File.ReadAllBytesAsync(compressedPath, ct); + var base64 = Convert.ToBase64String(compressedBytes); + imageUrls.Add($"data:image/jpeg;base64,{base64}"); } var prompt = """ - 识别图片中的所有食物，返回 JSON 格式： + 你是一个中餐营养分析专家。请仔细识别图片中的所有食物。 + 注意：这是中国食堂的中式菜肴，请用中文名称。 + + 返回 JSON 格式： { - "foods": [{"name":"食物名","portion":"份量描述","calories":数字,"proteinGrams":数字,"carbsGrams":数字,"fatGrams":数字,"warning":null或警告文字}], - "totalCalories":总热量数字, - "warnings":["整体警告"], - "score":1-5评分 + "foods": [{"name":"食物名（中文）","portion":"份量","calories":数字,"proteinGrams":数字,"carbsGrams":数字,"fatGrams":数字,"warning":null}], + "totalCalories":总热量, + "warnings":["整体建议"], + "score":1-5 } - 请只返回 JSON，不要加任何其他文字。 + 常见中餐参考：米饭约200g/230卡，炒青菜约200g/80卡，青椒肉丝约200g/200卡，红烧肉约150g/400卡，番茄炒蛋约200g/180卡，麻婆豆腐约200g/220卡。 + 只返回JSON。 """; try @@ -278,9 +290,9 @@ public static class AiChatEndpoints var result = response.Choices?.FirstOrDefault()?.Message?.Content ?? "{}"; return Results.Ok(new { code = 0, data = result, message = (string?)null }); } - catch (Exception) + catch (Exception ex) { - return Results.Ok(new { code = 50001, data = (object?)null, message = $"食物识别失败，请重试" }); + return Results.Ok(new { code = 50001, data = (object?)null, message = $"食物识别失败：{ex.Message}" }); } }); } @@ -561,6 +573,28 @@ public static class AiChatEndpoints Parameters = new { type = "object", properties = new { reason = new { type = "string" }, urgency_level = new { type = "string" } } } } }; + + /// 压缩图片到合理大小供 VLM API 使用 + private static void CompressImage(string inputPath, string outputPath, int maxWidth, long quality) + { + using var image = Image.FromFile(inputPath); + var width = image.Width; + var height = image.Height; + if (width > maxWidth) + { + height = (int)((double)height / width * maxWidth); + width = maxWidth; + } + using var bitmap = new Bitmap(width, height); + using var graphics = Graphics.FromImage(bitmap); + graphics.InterpolationMode = System.Drawing.Drawing2D.InterpolationMode.HighQualityBicubic; + graphics.DrawImage(image, 0, 0, width, height); + + var jpegCodec = ImageCodecInfo.GetImageEncoders().First(c => c.MimeType == "image/jpeg"); + var parameters = new EncoderParameters(1); + parameters.Param[0] = new EncoderParameter(Encoder.Quality, quality); + bitmap.Save(outputPath, jpegCodec, parameters); + } } /// AI 对话请求 diff --git a/backend/src/Health.WebApi/Health.WebApi.csproj b/backend/src/Health.WebApi/Health.WebApi.csproj index d5e873c..87dd5a1 100644 --- a/backend/src/Health.WebApi/Health.WebApi.csproj +++ b/backend/src/Health.WebApi/Health.WebApi.csproj @@ -10,6 +10,7 @@ +