fix: VLM 食物识别全链路修复 + 用药 Agent Prompt 优化
- 图片上传自动压缩(max 860px, JPEG Q65),解决大图超 API 129KB 限制 - 修复 VLM 传 file:// 本地路径 Bug,改为 base64 data URL - VLM Prompt 优化为中文食堂场景,附带常见中餐热量参考 - 千问 API 错误信息透传,方便调试 - 用药 Agent Prompt 加查询规则:先调 manage_medication 再回答 - 新增 System.Drawing.Common 依赖用于服务端图片压缩
This commit is contained in:
@@ -129,7 +129,11 @@ public sealed class QwenVisionClient(HttpClient http, IConfiguration config)
|
|||||||
var json = JsonSerializer.Serialize(request, _jsonOptions);
|
var json = JsonSerializer.Serialize(request, _jsonOptions);
|
||||||
var content = new StringContent(json, Encoding.UTF8, "application/json");
|
var content = new StringContent(json, Encoding.UTF8, "application/json");
|
||||||
var response = await _http.PostAsync("chat/completions", content, ct);
|
var response = await _http.PostAsync("chat/completions", content, ct);
|
||||||
response.EnsureSuccessStatusCode();
|
if (!response.IsSuccessStatusCode)
|
||||||
|
{
|
||||||
|
var errorBody = await response.Content.ReadAsStringAsync(ct);
|
||||||
|
throw new HttpRequestException($"VLM API {response.StatusCode}: {errorBody}");
|
||||||
|
}
|
||||||
var body = await response.Content.ReadAsStringAsync(ct);
|
var body = await response.Content.ReadAsStringAsync(ct);
|
||||||
return JsonSerializer.Deserialize<ChatCompletionResponse>(body, _jsonOptions)!;
|
return JsonSerializer.Deserialize<ChatCompletionResponse>(body, _jsonOptions)!;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -83,11 +83,12 @@ public sealed class PromptManager
|
|||||||
你是一个用药管理专家。
|
你是一个用药管理专家。
|
||||||
|
|
||||||
规则:
|
规则:
|
||||||
1. 解析用户口中的药品信息(药名/剂量/频次/时间)
|
1. 用户询问当前用药时,必须先调用 manage_medication(action="query") 查询已有用药记录
|
||||||
2. "早饭后"等模糊时间→追问具体几点
|
2. 解析用户口中的药品信息(药名/剂量/频次/时间)
|
||||||
3. 解析完成后展示确认卡片
|
3. "早饭后"等模糊时间→追问具体几点
|
||||||
4. 处方拍照→提取药品信息→生成用药计划→让用户确认
|
4. 解析完成后展示确认卡片
|
||||||
5. 回答用药相关疑问
|
5. 处方拍照→提取药品信息→生成用药计划→让用户确认
|
||||||
|
6. 回答用药相关疑问
|
||||||
""";
|
""";
|
||||||
|
|
||||||
private const string ReportPrompt = """
|
private const string ReportPrompt = """
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
using System.Drawing;
|
||||||
|
using System.Drawing.Imaging;
|
||||||
using Health.Infrastructure.AI;
|
using Health.Infrastructure.AI;
|
||||||
|
|
||||||
namespace Health.WebApi.Endpoints;
|
namespace Health.WebApi.Endpoints;
|
||||||
@@ -256,20 +258,30 @@ public static class AiChatEndpoints
|
|||||||
|
|
||||||
var safeName = $"{Guid.NewGuid()}_{Path.GetFileName(file.FileName)}";
|
var safeName = $"{Guid.NewGuid()}_{Path.GetFileName(file.FileName)}";
|
||||||
var filePath = Path.Combine(uploadsDir, safeName);
|
var filePath = Path.Combine(uploadsDir, safeName);
|
||||||
using var stream = new FileStream(filePath, FileMode.Create);
|
using (var stream = new FileStream(filePath, FileMode.Create))
|
||||||
await file.CopyToAsync(stream, ct);
|
await file.CopyToAsync(stream, ct);
|
||||||
imageUrls.Add($"file://{filePath}");
|
|
||||||
|
// 压缩图片后转 base64(VLM API 有请求体大小限制)
|
||||||
|
var compressedPath = Path.Combine(uploadsDir, $"compressed_{safeName}");
|
||||||
|
CompressImage(filePath, compressedPath, maxWidth: 860, quality: 65L);
|
||||||
|
var compressedBytes = await File.ReadAllBytesAsync(compressedPath, ct);
|
||||||
|
var base64 = Convert.ToBase64String(compressedBytes);
|
||||||
|
imageUrls.Add($"data:image/jpeg;base64,{base64}");
|
||||||
}
|
}
|
||||||
|
|
||||||
var prompt = """
|
var prompt = """
|
||||||
识别图片中的所有食物,返回 JSON 格式:
|
你是一个中餐营养分析专家。请仔细识别图片中的所有食物。
|
||||||
|
注意:这是中国食堂的中式菜肴,请用中文名称。
|
||||||
|
|
||||||
|
返回 JSON 格式:
|
||||||
{
|
{
|
||||||
"foods": [{"name":"食物名","portion":"份量描述","calories":数字,"proteinGrams":数字,"carbsGrams":数字,"fatGrams":数字,"warning":null或警告文字}],
|
"foods": [{"name":"食物名(中文)","portion":"份量","calories":数字,"proteinGrams":数字,"carbsGrams":数字,"fatGrams":数字,"warning":null}],
|
||||||
"totalCalories":总热量数字,
|
"totalCalories":总热量,
|
||||||
"warnings":["整体警告"],
|
"warnings":["整体建议"],
|
||||||
"score":1-5评分
|
"score":1-5
|
||||||
}
|
}
|
||||||
请只返回 JSON,不要加任何其他文字。
|
常见中餐参考:米饭约200g/230卡,炒青菜约200g/80卡,青椒肉丝约200g/200卡,红烧肉约150g/400卡,番茄炒蛋约200g/180卡,麻婆豆腐约200g/220卡。
|
||||||
|
只返回JSON。
|
||||||
""";
|
""";
|
||||||
|
|
||||||
try
|
try
|
||||||
@@ -278,9 +290,9 @@ public static class AiChatEndpoints
|
|||||||
var result = response.Choices?.FirstOrDefault()?.Message?.Content ?? "{}";
|
var result = response.Choices?.FirstOrDefault()?.Message?.Content ?? "{}";
|
||||||
return Results.Ok(new { code = 0, data = result, message = (string?)null });
|
return Results.Ok(new { code = 0, data = result, message = (string?)null });
|
||||||
}
|
}
|
||||||
catch (Exception)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
return Results.Ok(new { code = 50001, data = (object?)null, message = $"食物识别失败,请重试" });
|
return Results.Ok(new { code = 50001, data = (object?)null, message = $"食物识别失败:{ex.Message}" });
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -561,6 +573,28 @@ public static class AiChatEndpoints
|
|||||||
Parameters = new { type = "object", properties = new { reason = new { type = "string" }, urgency_level = new { type = "string" } } }
|
Parameters = new { type = "object", properties = new { reason = new { type = "string" }, urgency_level = new { type = "string" } } }
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/// <summary>压缩图片到合理大小供 VLM API 使用</summary>
|
||||||
|
private static void CompressImage(string inputPath, string outputPath, int maxWidth, long quality)
|
||||||
|
{
|
||||||
|
using var image = Image.FromFile(inputPath);
|
||||||
|
var width = image.Width;
|
||||||
|
var height = image.Height;
|
||||||
|
if (width > maxWidth)
|
||||||
|
{
|
||||||
|
height = (int)((double)height / width * maxWidth);
|
||||||
|
width = maxWidth;
|
||||||
|
}
|
||||||
|
using var bitmap = new Bitmap(width, height);
|
||||||
|
using var graphics = Graphics.FromImage(bitmap);
|
||||||
|
graphics.InterpolationMode = System.Drawing.Drawing2D.InterpolationMode.HighQualityBicubic;
|
||||||
|
graphics.DrawImage(image, 0, 0, width, height);
|
||||||
|
|
||||||
|
var jpegCodec = ImageCodecInfo.GetImageEncoders().First(c => c.MimeType == "image/jpeg");
|
||||||
|
var parameters = new EncoderParameters(1);
|
||||||
|
parameters.Param[0] = new EncoderParameter(Encoder.Quality, quality);
|
||||||
|
bitmap.Save(outputPath, jpegCodec, parameters);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>AI 对话请求</summary>
|
/// <summary>AI 对话请求</summary>
|
||||||
|
|||||||
@@ -10,6 +10,7 @@
|
|||||||
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="10.0.8" />
|
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="10.0.8" />
|
||||||
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="10.0.8" />
|
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="10.0.8" />
|
||||||
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="10.0.8" />
|
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="10.0.8" />
|
||||||
|
<PackageReference Include="System.Drawing.Common" Version="10.0.8" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|||||||
Reference in New Issue
Block a user