fix: VLM 食物识别全链路修复 + 用药 Agent Prompt 优化
- 图片上传自动压缩(max 860px, JPEG Q65),解决大图超 API 129KB 限制 - 修复 VLM 传 file:// 本地路径 Bug,改为 base64 data URL - VLM Prompt 优化为中文食堂场景,附带常见中餐热量参考 - 千问 API 错误信息透传,方便调试 - 用药 Agent Prompt 加查询规则:先调 manage_medication 再回答 - 新增 System.Drawing.Common 依赖用于服务端图片压缩
This commit is contained in:
@@ -1,3 +1,5 @@
|
||||
using System.Drawing;
|
||||
using System.Drawing.Imaging;
|
||||
using Health.Infrastructure.AI;
|
||||
|
||||
namespace Health.WebApi.Endpoints;
|
||||
@@ -256,20 +258,30 @@ public static class AiChatEndpoints
|
||||
|
||||
var safeName = $"{Guid.NewGuid()}_{Path.GetFileName(file.FileName)}";
|
||||
var filePath = Path.Combine(uploadsDir, safeName);
|
||||
using var stream = new FileStream(filePath, FileMode.Create);
|
||||
await file.CopyToAsync(stream, ct);
|
||||
imageUrls.Add($"file://{filePath}");
|
||||
using (var stream = new FileStream(filePath, FileMode.Create))
|
||||
await file.CopyToAsync(stream, ct);
|
||||
|
||||
// 压缩图片后转 base64(VLM API 有请求体大小限制)
|
||||
var compressedPath = Path.Combine(uploadsDir, $"compressed_{safeName}");
|
||||
CompressImage(filePath, compressedPath, maxWidth: 860, quality: 65L);
|
||||
var compressedBytes = await File.ReadAllBytesAsync(compressedPath, ct);
|
||||
var base64 = Convert.ToBase64String(compressedBytes);
|
||||
imageUrls.Add($"data:image/jpeg;base64,{base64}");
|
||||
}
|
||||
|
||||
var prompt = """
|
||||
识别图片中的所有食物,返回 JSON 格式:
|
||||
你是一个中餐营养分析专家。请仔细识别图片中的所有食物。
|
||||
注意:这是中国食堂的中式菜肴,请用中文名称。
|
||||
|
||||
返回 JSON 格式:
|
||||
{
|
||||
"foods": [{"name":"食物名","portion":"份量描述","calories":数字,"proteinGrams":数字,"carbsGrams":数字,"fatGrams":数字,"warning":null或警告文字}],
|
||||
"totalCalories":总热量数字,
|
||||
"warnings":["整体警告"],
|
||||
"score":1-5评分
|
||||
"foods": [{"name":"食物名(中文)","portion":"份量","calories":数字,"proteinGrams":数字,"carbsGrams":数字,"fatGrams":数字,"warning":null}],
|
||||
"totalCalories":总热量,
|
||||
"warnings":["整体建议"],
|
||||
"score":1-5
|
||||
}
|
||||
请只返回 JSON,不要加任何其他文字。
|
||||
常见中餐参考:米饭约200g/230卡,炒青菜约200g/80卡,青椒肉丝约200g/200卡,红烧肉约150g/400卡,番茄炒蛋约200g/180卡,麻婆豆腐约200g/220卡。
|
||||
只返回JSON。
|
||||
""";
|
||||
|
||||
try
|
||||
@@ -278,9 +290,9 @@ public static class AiChatEndpoints
|
||||
var result = response.Choices?.FirstOrDefault()?.Message?.Content ?? "{}";
|
||||
return Results.Ok(new { code = 0, data = result, message = (string?)null });
|
||||
}
|
||||
catch (Exception)
|
||||
catch (Exception ex)
|
||||
{
|
||||
return Results.Ok(new { code = 50001, data = (object?)null, message = $"食物识别失败,请重试" });
|
||||
return Results.Ok(new { code = 50001, data = (object?)null, message = $"食物识别失败:{ex.Message}" });
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -561,6 +573,28 @@ public static class AiChatEndpoints
|
||||
Parameters = new { type = "object", properties = new { reason = new { type = "string" }, urgency_level = new { type = "string" } } }
|
||||
}
|
||||
};
|
||||
|
||||
/// <summary>压缩图片到合理大小供 VLM API 使用</summary>
|
||||
private static void CompressImage(string inputPath, string outputPath, int maxWidth, long quality)
|
||||
{
|
||||
using var image = Image.FromFile(inputPath);
|
||||
var width = image.Width;
|
||||
var height = image.Height;
|
||||
if (width > maxWidth)
|
||||
{
|
||||
height = (int)((double)height / width * maxWidth);
|
||||
width = maxWidth;
|
||||
}
|
||||
using var bitmap = new Bitmap(width, height);
|
||||
using var graphics = Graphics.FromImage(bitmap);
|
||||
graphics.InterpolationMode = System.Drawing.Drawing2D.InterpolationMode.HighQualityBicubic;
|
||||
graphics.DrawImage(image, 0, 0, width, height);
|
||||
|
||||
var jpegCodec = ImageCodecInfo.GetImageEncoders().First(c => c.MimeType == "image/jpeg");
|
||||
var parameters = new EncoderParameters(1);
|
||||
parameters.Param[0] = new EncoderParameter(Encoder.Quality, quality);
|
||||
bitmap.Save(outputPath, jpegCodec, parameters);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>AI 对话请求</summary>
|
||||
|
||||
Reference in New Issue
Block a user