fix: VLM识别修复 - 去System Message + 低温 + 精简提示词
This commit is contained in:
@@ -104,27 +104,23 @@ public sealed class VisionClient(HttpClient http, IConfiguration config)
|
|||||||
int maxTokens = 2048,
|
int maxTokens = 2048,
|
||||||
CancellationToken ct = default)
|
CancellationToken ct = default)
|
||||||
{
|
{
|
||||||
var messages = new List<ChatMessage>();
|
|
||||||
if (!string.IsNullOrEmpty(systemPrompt))
|
|
||||||
messages.Add(new ChatMessage { Role = "system", Content = systemPrompt });
|
|
||||||
|
|
||||||
var contentParts = new List<object>();
|
var contentParts = new List<object>();
|
||||||
foreach (var url in imageUrls)
|
foreach (var url in imageUrls)
|
||||||
contentParts.Add(new { type = "image_url", image_url = new { url } });
|
contentParts.Add(new { type = "image_url", image_url = new { url } });
|
||||||
|
if (!string.IsNullOrEmpty(systemPrompt))
|
||||||
|
contentParts.Add(new { type = "text", text = systemPrompt });
|
||||||
if (!string.IsNullOrEmpty(userText))
|
if (!string.IsNullOrEmpty(userText))
|
||||||
contentParts.Add(new { type = "text", text = userText });
|
contentParts.Add(new { type = "text", text = userText });
|
||||||
|
|
||||||
var userMessage = new ChatMessage
|
var messages = new List<ChatMessage>
|
||||||
{
|
{
|
||||||
Role = "user",
|
new() { Role = "user", Content = JsonSerializer.Serialize(contentParts, _jsonOptions) }
|
||||||
Content = JsonSerializer.Serialize(contentParts, _jsonOptions)
|
|
||||||
};
|
};
|
||||||
messages.Add(userMessage);
|
|
||||||
|
|
||||||
var request = new ChatCompletionRequest
|
var request = new ChatCompletionRequest
|
||||||
{
|
{
|
||||||
Model = _model, Messages = messages, MaxTokens = maxTokens, Stream = false,
|
Model = _model, Messages = messages, MaxTokens = maxTokens, Stream = false,
|
||||||
Temperature = 0.7f, TopP = 0.8f,
|
Temperature = 0.1f, VlHighResolutionImages = true,
|
||||||
};
|
};
|
||||||
|
|
||||||
var json = JsonSerializer.Serialize(request, _jsonOptions);
|
var json = JsonSerializer.Serialize(request, _jsonOptions);
|
||||||
@@ -136,7 +132,6 @@ public sealed class VisionClient(HttpClient http, IConfiguration config)
|
|||||||
throw new HttpRequestException($"VLM API {response.StatusCode}: {errorBody}");
|
throw new HttpRequestException($"VLM API {response.StatusCode}: {errorBody}");
|
||||||
}
|
}
|
||||||
var body = await response.Content.ReadAsStringAsync(ct);
|
var body = await response.Content.ReadAsStringAsync(ct);
|
||||||
Console.WriteLine($"[VLM RAW] Model={_model}, BodyLen={body.Length}, Body={body[..Math.Min(body.Length, 500)]}");
|
|
||||||
return JsonSerializer.Deserialize<ChatCompletionResponse>(body, _jsonOptions)!;
|
return JsonSerializer.Deserialize<ChatCompletionResponse>(body, _jsonOptions)!;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -167,6 +167,8 @@ public sealed class ChatCompletionRequest
|
|||||||
public float? TopP { get; set; }
|
public float? TopP { get; set; }
|
||||||
public List<ToolDefinition>? Tools { get; set; }
|
public List<ToolDefinition>? Tools { get; set; }
|
||||||
public string? ToolChoice { get; set; }
|
public string? ToolChoice { get; set; }
|
||||||
|
[System.Text.Json.Serialization.JsonPropertyName("vl_high_resolution_images")]
|
||||||
|
public bool VlHighResolutionImages { get; set; }
|
||||||
}
|
}
|
||||||
|
|
||||||
public sealed class ChatMessage
|
public sealed class ChatMessage
|
||||||
|
|||||||
@@ -266,24 +266,26 @@ public static class AiChatEndpoints
|
|||||||
await file.CopyToAsync(stream, ct);
|
await file.CopyToAsync(stream, ct);
|
||||||
|
|
||||||
var compressedPath = Path.Combine(uploadsDir, $"compressed_{safeName}");
|
var compressedPath = Path.Combine(uploadsDir, $"compressed_{safeName}");
|
||||||
CompressImage(filePath, compressedPath, maxWidth: 1280, quality: 85L);
|
CompressImage(filePath, compressedPath, maxWidth: 1024, quality: 90L);
|
||||||
var compressedBytes = await File.ReadAllBytesAsync(compressedPath, ct);
|
var compressedBytes = await File.ReadAllBytesAsync(compressedPath, ct);
|
||||||
var base64 = Convert.ToBase64String(compressedBytes);
|
var base64 = Convert.ToBase64String(compressedBytes);
|
||||||
imageUrls.Add($"data:image/jpeg;base64,{base64}");
|
imageUrls.Add($"data:image/jpeg;base64,{base64}");
|
||||||
}
|
}
|
||||||
|
|
||||||
var prompt = """
|
var prompt = """
|
||||||
Describe everything you see in this image in detail. What objects are present? What are their colors, shapes, labels?
|
识别这张图片中最主要的食物或饮品(最多2个)。只返回JSON数组,格式:
|
||||||
If there are any food items, drinks, or beverages, identify them specifically by name.
|
[{"name":"名称","portion":"份量","calories":热量}]
|
||||||
Then, for each food/drink item, estimate the portion size and calories (kcal).
|
不要说图片里没有的东西。不要编造。
|
||||||
Return ONLY a JSON array: [{"name":"item name","portion":"estimated portion","calories":estimated_calories}]
|
|
||||||
If you cannot identify something, say so in the name field.
|
|
||||||
""";
|
""";
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
var response = await visionClient.VisionAsync(prompt, imageUrls, userText: "请看图识别食物", maxTokens: 8192, ct: ct);
|
var response = await visionClient.VisionAsync(prompt, imageUrls, userText: "请看图识别食物", maxTokens: 8192, ct: ct);
|
||||||
var result = response.Choices?.FirstOrDefault()?.Message?.Content ?? "{}";
|
var result = response.Choices?.FirstOrDefault()?.Message?.Content ?? "{}";
|
||||||
|
// 记录VLM原始返回用于排查
|
||||||
|
var uploadsDir2 = Path.Combine(Directory.GetCurrentDirectory(), "uploads");
|
||||||
|
var logPath = Path.Combine(uploadsDir2, $"vlm_log_{DateTime.Now:HHmmss}.txt");
|
||||||
|
await File.WriteAllTextAsync(logPath, $"MODEL: {Environment.GetEnvironmentVariable("VLM_MODEL")}\nIMAGE_SIZE: {imageUrls.FirstOrDefault()?.Length ?? 0}\nRESPONSE:\n{result}", ct);
|
||||||
return Results.Ok(new { code = 0, data = result, message = (string?)null });
|
return Results.Ok(new { code = 0, data = result, message = (string?)null });
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
|
|||||||
@@ -73,7 +73,9 @@ class DietNotifier extends Notifier<DietState> {
|
|||||||
state = state.copyWith(isAnalyzing: true, errorMessage: null);
|
state = state.copyWith(isAnalyzing: true, errorMessage: null);
|
||||||
try {
|
try {
|
||||||
final api = ref.read(apiClientProvider);
|
final api = ref.read(apiClientProvider);
|
||||||
final imageFile = File(state.imagePath!);
|
final path = state.imagePath!;
|
||||||
|
debugPrint('[DietDebug] analyzeImage using: $path');
|
||||||
|
final imageFile = File(path);
|
||||||
|
|
||||||
final formData = FormData.fromMap({
|
final formData = FormData.fromMap({
|
||||||
'images': await MultipartFile.fromFile(
|
'images': await MultipartFile.fromFile(
|
||||||
|
|||||||
Reference in New Issue
Block a user