fix: VLM识别修复 - 去System Message + 低温 + 精简提示词
This commit is contained in:
@@ -104,27 +104,23 @@ public sealed class VisionClient(HttpClient http, IConfiguration config)
|
||||
int maxTokens = 2048,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
var messages = new List<ChatMessage>();
|
||||
if (!string.IsNullOrEmpty(systemPrompt))
|
||||
messages.Add(new ChatMessage { Role = "system", Content = systemPrompt });
|
||||
|
||||
var contentParts = new List<object>();
|
||||
foreach (var url in imageUrls)
|
||||
contentParts.Add(new { type = "image_url", image_url = new { url } });
|
||||
if (!string.IsNullOrEmpty(systemPrompt))
|
||||
contentParts.Add(new { type = "text", text = systemPrompt });
|
||||
if (!string.IsNullOrEmpty(userText))
|
||||
contentParts.Add(new { type = "text", text = userText });
|
||||
|
||||
var userMessage = new ChatMessage
|
||||
var messages = new List<ChatMessage>
|
||||
{
|
||||
Role = "user",
|
||||
Content = JsonSerializer.Serialize(contentParts, _jsonOptions)
|
||||
new() { Role = "user", Content = JsonSerializer.Serialize(contentParts, _jsonOptions) }
|
||||
};
|
||||
messages.Add(userMessage);
|
||||
|
||||
var request = new ChatCompletionRequest
|
||||
{
|
||||
Model = _model, Messages = messages, MaxTokens = maxTokens, Stream = false,
|
||||
Temperature = 0.7f, TopP = 0.8f,
|
||||
Temperature = 0.1f, VlHighResolutionImages = true,
|
||||
};
|
||||
|
||||
var json = JsonSerializer.Serialize(request, _jsonOptions);
|
||||
@@ -136,7 +132,6 @@ public sealed class VisionClient(HttpClient http, IConfiguration config)
|
||||
throw new HttpRequestException($"VLM API {response.StatusCode}: {errorBody}");
|
||||
}
|
||||
var body = await response.Content.ReadAsStringAsync(ct);
|
||||
Console.WriteLine($"[VLM RAW] Model={_model}, BodyLen={body.Length}, Body={body[..Math.Min(body.Length, 500)]}");
|
||||
return JsonSerializer.Deserialize<ChatCompletionResponse>(body, _jsonOptions)!;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -167,6 +167,8 @@ public sealed class ChatCompletionRequest
|
||||
public float? TopP { get; set; }
|
||||
public List<ToolDefinition>? Tools { get; set; }
|
||||
public string? ToolChoice { get; set; }
|
||||
[System.Text.Json.Serialization.JsonPropertyName("vl_high_resolution_images")]
|
||||
public bool VlHighResolutionImages { get; set; }
|
||||
}
|
||||
|
||||
public sealed class ChatMessage
|
||||
|
||||
@@ -266,24 +266,26 @@ public static class AiChatEndpoints
|
||||
await file.CopyToAsync(stream, ct);
|
||||
|
||||
var compressedPath = Path.Combine(uploadsDir, $"compressed_{safeName}");
|
||||
CompressImage(filePath, compressedPath, maxWidth: 1280, quality: 85L);
|
||||
CompressImage(filePath, compressedPath, maxWidth: 1024, quality: 90L);
|
||||
var compressedBytes = await File.ReadAllBytesAsync(compressedPath, ct);
|
||||
var base64 = Convert.ToBase64String(compressedBytes);
|
||||
imageUrls.Add($"data:image/jpeg;base64,{base64}");
|
||||
}
|
||||
|
||||
var prompt = """
|
||||
Describe everything you see in this image in detail. What objects are present? What are their colors, shapes, labels?
|
||||
If there are any food items, drinks, or beverages, identify them specifically by name.
|
||||
Then, for each food/drink item, estimate the portion size and calories (kcal).
|
||||
Return ONLY a JSON array: [{"name":"item name","portion":"estimated portion","calories":estimated_calories}]
|
||||
If you cannot identify something, say so in the name field.
|
||||
识别这张图片中最主要的食物或饮品(最多2个)。只返回JSON数组,格式:
|
||||
[{"name":"名称","portion":"份量","calories":热量}]
|
||||
不要说图片里没有的东西。不要编造。
|
||||
""";
|
||||
|
||||
try
|
||||
{
|
||||
var response = await visionClient.VisionAsync(prompt, imageUrls, userText: "请看图识别食物", maxTokens: 8192, ct: ct);
|
||||
var result = response.Choices?.FirstOrDefault()?.Message?.Content ?? "{}";
|
||||
// 记录VLM原始返回用于排查
|
||||
var uploadsDir2 = Path.Combine(Directory.GetCurrentDirectory(), "uploads");
|
||||
var logPath = Path.Combine(uploadsDir2, $"vlm_log_{DateTime.Now:HHmmss}.txt");
|
||||
await File.WriteAllTextAsync(logPath, $"MODEL: {Environment.GetEnvironmentVariable("VLM_MODEL")}\nIMAGE_SIZE: {imageUrls.FirstOrDefault()?.Length ?? 0}\nRESPONSE:\n{result}", ct);
|
||||
return Results.Ok(new { code = 0, data = result, message = (string?)null });
|
||||
}
|
||||
catch (Exception ex)
|
||||
|
||||
@@ -73,7 +73,9 @@ class DietNotifier extends Notifier<DietState> {
|
||||
state = state.copyWith(isAnalyzing: true, errorMessage: null);
|
||||
try {
|
||||
final api = ref.read(apiClientProvider);
|
||||
final imageFile = File(state.imagePath!);
|
||||
final path = state.imagePath!;
|
||||
debugPrint('[DietDebug] analyzeImage using: $path');
|
||||
final imageFile = File(path);
|
||||
|
||||
final formData = FormData.fromMap({
|
||||
'images': await MultipartFile.fromFile(
|
||||
|
||||
Reference in New Issue
Block a user