fix: VLM 食物识别全链路修复 + 用药 Agent Prompt 优化

- 图片上传自动压缩(max 860px, JPEG Q65),解决大图超 API 129KB 限制
- 修复 VLM 传 file:// 本地路径 Bug,改为 base64 data URL
- VLM Prompt 优化为中文食堂场景,附带常见中餐热量参考
- 千问 API 错误信息透传,方便调试
- 用药 Agent Prompt 加查询规则:先调 manage_medication 再回答
- 新增 System.Drawing.Common 依赖用于服务端图片压缩
This commit is contained in:
MingNian
2026-06-02 13:44:14 +08:00
parent 0e7f0a0b80
commit 27cc920a4a
4 changed files with 57 additions and 17 deletions

View File

@@ -1,3 +1,5 @@
using System.Drawing;
using System.Drawing.Imaging;
using Health.Infrastructure.AI;
namespace Health.WebApi.Endpoints;
@@ -256,20 +258,30 @@ public static class AiChatEndpoints
var safeName = $"{Guid.NewGuid()}_{Path.GetFileName(file.FileName)}";
var filePath = Path.Combine(uploadsDir, safeName);
using var stream = new FileStream(filePath, FileMode.Create);
await file.CopyToAsync(stream, ct);
imageUrls.Add($"file://{filePath}");
using (var stream = new FileStream(filePath, FileMode.Create))
await file.CopyToAsync(stream, ct);
// 压缩图片后转 base64VLM API 有请求体大小限制)
var compressedPath = Path.Combine(uploadsDir, $"compressed_{safeName}");
CompressImage(filePath, compressedPath, maxWidth: 860, quality: 65L);
var compressedBytes = await File.ReadAllBytesAsync(compressedPath, ct);
var base64 = Convert.ToBase64String(compressedBytes);
imageUrls.Add($"data:image/jpeg;base64,{base64}");
}
var prompt = """
JSON
JSON
{
"foods": [{"name":"食物名","portion":"份量描述","calories":,"proteinGrams":,"carbsGrams":,"fatGrams":,"warning":null或警告文字}],
"totalCalories":,
"warnings":["整体警告"],
"score":1-5
"foods": [{"name":"食物名(中文)","portion":"份量","calories":,"proteinGrams":,"carbsGrams":,"fatGrams":,"warning":null}],
"totalCalories":,
"warnings":["整体建议"],
"score":1-5
}
JSON
200g/230200g/80200g/200150g/400200g/180200g/220
JSON
""";
try
@@ -278,9 +290,9 @@ public static class AiChatEndpoints
var result = response.Choices?.FirstOrDefault()?.Message?.Content ?? "{}";
return Results.Ok(new { code = 0, data = result, message = (string?)null });
}
catch (Exception)
catch (Exception ex)
{
return Results.Ok(new { code = 50001, data = (object?)null, message = $"食物识别失败,请重试" });
return Results.Ok(new { code = 50001, data = (object?)null, message = $"食物识别失败{ex.Message}" });
}
});
}
@@ -561,6 +573,28 @@ public static class AiChatEndpoints
Parameters = new { type = "object", properties = new { reason = new { type = "string" }, urgency_level = new { type = "string" } } }
}
};
/// <summary>压缩图片到合理大小供 VLM API 使用</summary>
private static void CompressImage(string inputPath, string outputPath, int maxWidth, long quality)
{
using var image = Image.FromFile(inputPath);
var width = image.Width;
var height = image.Height;
if (width > maxWidth)
{
height = (int)((double)height / width * maxWidth);
width = maxWidth;
}
using var bitmap = new Bitmap(width, height);
using var graphics = Graphics.FromImage(bitmap);
graphics.InterpolationMode = System.Drawing.Drawing2D.InterpolationMode.HighQualityBicubic;
graphics.DrawImage(image, 0, 0, width, height);
var jpegCodec = ImageCodecInfo.GetImageEncoders().First(c => c.MimeType == "image/jpeg");
var parameters = new EncoderParameters(1);
parameters.Param[0] = new EncoderParameter(Encoder.Quality, quality);
bitmap.Save(outputPath, jpegCodec, parameters);
}
}
/// <summary>AI 对话请求</summary>