From c44917b8e9691575ddbdf2387edf50e8090f1f6e Mon Sep 17 00:00:00 2001 From: MingNian <1281442923@qq.com> Date: Thu, 4 Jun 2026 15:02:12 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20VLM=E8=AF=86=E5=88=AB=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=20-=20=E5=8E=BBSystem=20Message=20+=20=E4=BD=8E=E6=B8=A9=20+?= =?UTF-8?q?=20=E7=B2=BE=E7=AE=80=E6=8F=90=E7=A4=BA=E8=AF=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/Health.Infrastructure/AI/ai_clients.cs | 15 +++++---------- .../AI/open_ai_compatible_client.cs | 2 ++ .../Health.WebApi/Endpoints/ai_chat_endpoints.cs | 14 ++++++++------ health_app/lib/pages/diet/diet_capture_page.dart | 4 +++- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/backend/src/Health.Infrastructure/AI/ai_clients.cs b/backend/src/Health.Infrastructure/AI/ai_clients.cs index 12d7afc..e7f3ac4 100644 --- a/backend/src/Health.Infrastructure/AI/ai_clients.cs +++ b/backend/src/Health.Infrastructure/AI/ai_clients.cs @@ -104,27 +104,23 @@ public sealed class VisionClient(HttpClient http, IConfiguration config) int maxTokens = 2048, CancellationToken ct = default) { - var messages = new List(); - if (!string.IsNullOrEmpty(systemPrompt)) - messages.Add(new ChatMessage { Role = "system", Content = systemPrompt }); - var contentParts = new List(); foreach (var url in imageUrls) contentParts.Add(new { type = "image_url", image_url = new { url } }); + if (!string.IsNullOrEmpty(systemPrompt)) + contentParts.Add(new { type = "text", text = systemPrompt }); if (!string.IsNullOrEmpty(userText)) contentParts.Add(new { type = "text", text = userText }); - var userMessage = new ChatMessage + var messages = new List { - Role = "user", - Content = JsonSerializer.Serialize(contentParts, _jsonOptions) + new() { Role = "user", Content = JsonSerializer.Serialize(contentParts, _jsonOptions) } }; - messages.Add(userMessage); var request = new ChatCompletionRequest { Model = _model, Messages = messages, MaxTokens = maxTokens, Stream = false, - Temperature = 0.7f, TopP = 0.8f, + Temperature = 0.1f, VlHighResolutionImages = true, }; var json = JsonSerializer.Serialize(request, _jsonOptions); @@ -136,7 +132,6 @@ public sealed class VisionClient(HttpClient http, IConfiguration config) throw new HttpRequestException($"VLM API {response.StatusCode}: {errorBody}"); } var body = await response.Content.ReadAsStringAsync(ct); - Console.WriteLine($"[VLM RAW] Model={_model}, BodyLen={body.Length}, Body={body[..Math.Min(body.Length, 500)]}"); return JsonSerializer.Deserialize(body, _jsonOptions)!; } } diff --git a/backend/src/Health.Infrastructure/AI/open_ai_compatible_client.cs b/backend/src/Health.Infrastructure/AI/open_ai_compatible_client.cs index 2e0191c..68df7fc 100644 --- a/backend/src/Health.Infrastructure/AI/open_ai_compatible_client.cs +++ b/backend/src/Health.Infrastructure/AI/open_ai_compatible_client.cs @@ -167,6 +167,8 @@ public sealed class ChatCompletionRequest public float? TopP { get; set; } public List? Tools { get; set; } public string? ToolChoice { get; set; } + [System.Text.Json.Serialization.JsonPropertyName("vl_high_resolution_images")] + public bool VlHighResolutionImages { get; set; } } public sealed class ChatMessage diff --git a/backend/src/Health.WebApi/Endpoints/ai_chat_endpoints.cs b/backend/src/Health.WebApi/Endpoints/ai_chat_endpoints.cs index c0c89be..30df224 100644 --- a/backend/src/Health.WebApi/Endpoints/ai_chat_endpoints.cs +++ b/backend/src/Health.WebApi/Endpoints/ai_chat_endpoints.cs @@ -266,24 +266,26 @@ public static class AiChatEndpoints await file.CopyToAsync(stream, ct); var compressedPath = Path.Combine(uploadsDir, $"compressed_{safeName}"); - CompressImage(filePath, compressedPath, maxWidth: 1280, quality: 85L); + CompressImage(filePath, compressedPath, maxWidth: 1024, quality: 90L); var compressedBytes = await File.ReadAllBytesAsync(compressedPath, ct); var base64 = Convert.ToBase64String(compressedBytes); imageUrls.Add($"data:image/jpeg;base64,{base64}"); } var prompt = """ - Describe everything you see in this image in detail. What objects are present? What are their colors, shapes, labels? - If there are any food items, drinks, or beverages, identify them specifically by name. - Then, for each food/drink item, estimate the portion size and calories (kcal). - Return ONLY a JSON array: [{"name":"item name","portion":"estimated portion","calories":estimated_calories}] - If you cannot identify something, say so in the name field. + 识别这张图片中最主要的食物或饮品（最多2个）。只返回JSON数组，格式： + [{"name":"名称","portion":"份量","calories":热量}] + 不要说图片里没有的东西。不要编造。 """; try { var response = await visionClient.VisionAsync(prompt, imageUrls, userText: "请看图识别食物", maxTokens: 8192, ct: ct); var result = response.Choices?.FirstOrDefault()?.Message?.Content ?? "{}"; + // 记录VLM原始返回用于排查 + var uploadsDir2 = Path.Combine(Directory.GetCurrentDirectory(), "uploads"); + var logPath = Path.Combine(uploadsDir2, $"vlm_log_{DateTime.Now:HHmmss}.txt"); + await File.WriteAllTextAsync(logPath, $"MODEL: {Environment.GetEnvironmentVariable("VLM_MODEL")}\nIMAGE_SIZE: {imageUrls.FirstOrDefault()?.Length ?? 0}\nRESPONSE:\n{result}", ct); return Results.Ok(new { code = 0, data = result, message = (string?)null }); } catch (Exception ex) diff --git a/health_app/lib/pages/diet/diet_capture_page.dart b/health_app/lib/pages/diet/diet_capture_page.dart index 0866db9..368d59c 100644 --- a/health_app/lib/pages/diet/diet_capture_page.dart +++ b/health_app/lib/pages/diet/diet_capture_page.dart @@ -73,7 +73,9 @@ class DietNotifier extends Notifier { state = state.copyWith(isAnalyzing: true, errorMessage: null); try { final api = ref.read(apiClientProvider); - final imageFile = File(state.imagePath!); + final path = state.imagePath!; + debugPrint('[DietDebug] analyzeImage using: $path'); + final imageFile = File(path); final formData = FormData.fromMap({ 'images': await MultipartFile.fromFile(