feat: 长辈模式（大字大按钮 + 偏好按账号隔离）+ 语音输入（按住说话 + 实时转写 + 后端代理 DashScope ASR + 患者端专用）+ 健康仪表盘背景渐变

2026-07-21 10:53:16 +08:00
parent 0cb5b8e85a
commit 28f704c98e
25 changed files with 2214 additions and 131 deletions
--- a/backend/src/Health.Application/Speech/realtime_speech_contracts.cs
+++ b/backend/src/Health.Application/Speech/realtime_speech_contracts.cs
@@ -0,0 +1,10 @@
+using System.Net.WebSockets;
+
+namespace Health.Application.Speech;
+
+public interface IRealtimeSpeechRecognitionProxy
+{
+    bool IsConfigured { get; }
+
+    Task ProxyAsync(WebSocket clientSocket, CancellationToken cancellationToken);
+}
--- a/backend/src/Health.Infrastructure/Speech/dashscope_speech_recognition_proxy.cs
+++ b/backend/src/Health.Infrastructure/Speech/dashscope_speech_recognition_proxy.cs
@@ -0,0 +1,389 @@
+using System.Net.WebSockets;
+using System.Text;
+using System.Text.Json;
+using Health.Application.Speech;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
+
+namespace Health.Infrastructure.Speech;
+
+public sealed class DashScopeSpeechRecognitionProxy : IRealtimeSpeechRecognitionProxy
+{
+    private const int SampleRate = 16_000;
+    private const int BytesPerSecond = SampleRate * 2;
+    private readonly string _apiKey;
+    private readonly string _model;
+    private readonly string _workspaceId;
+    private readonly Uri? _endpoint;
+    private readonly int _maxDurationSeconds;
+    private readonly ILogger<DashScopeSpeechRecognitionProxy> _logger;
+
+    public DashScopeSpeechRecognitionProxy(
+        IConfiguration configuration,
+        ILogger<DashScopeSpeechRecognitionProxy> logger)
+    {
+        _apiKey = FirstConfigured(
+            configuration["DASHSCOPE_ASR_API_KEY"],
+            configuration["VLM_API_KEY"],
+            configuration["QWEN_API_KEY"]);
+        _model = configuration["DASHSCOPE_ASR_MODEL"]?.Trim() is { Length: > 0 } model
+            ? model
+            : "fun-asr-realtime";
+        _workspaceId = configuration["DASHSCOPE_ASR_WORKSPACE_ID"]?.Trim() ?? "";
+        var configuredMaxDuration = int.TryParse(
+            configuration["DASHSCOPE_ASR_MAX_DURATION_SECONDS"],
+            out var parsedMaxDuration)
+            ? parsedMaxDuration
+            : 60;
+        _maxDurationSeconds = Math.Clamp(configuredMaxDuration, 10, 180);
+
+        var configuredEndpoint = configuration["DASHSCOPE_ASR_WEBSOCKET_URL"]?.Trim();
+        var endpoint = !string.IsNullOrWhiteSpace(configuredEndpoint)
+            ? configuredEndpoint
+            : !string.IsNullOrWhiteSpace(_workspaceId)
+                ? $"wss://{_workspaceId}.cn-beijing.maas.aliyuncs.com/api-ws/v1/inference"
+                : "wss://dashscope.aliyuncs.com/api-ws/v1/inference";
+        if (Uri.TryCreate(endpoint, UriKind.Absolute, out var uri) && uri.Scheme == "wss")
+            _endpoint = uri;
+
+        _logger = logger;
+    }
+
+    public bool IsConfigured =>
+        !string.IsNullOrWhiteSpace(_apiKey) && _endpoint is not null;
+
+    private static string FirstConfigured(params string?[] values) =>
+        values.FirstOrDefault(value => !string.IsNullOrWhiteSpace(value))?.Trim() ?? "";
+
+    public async Task ProxyAsync(
+        WebSocket clientSocket,
+        CancellationToken cancellationToken)
+    {
+        if (!IsConfigured)
+            throw new InvalidOperationException("实时语音识别服务尚未配置");
+
+        using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(
+            cancellationToken);
+        timeoutCts.CancelAfter(TimeSpan.FromSeconds(_maxDurationSeconds + 15));
+        var ct = timeoutCts.Token;
+
+        using var cloudSocket = new ClientWebSocket();
+        cloudSocket.Options.SetRequestHeader("Authorization", $"Bearer {_apiKey}");
+        cloudSocket.Options.SetRequestHeader("User-Agent", "XiaomaiHealth/1.0");
+        if (!string.IsNullOrWhiteSpace(_workspaceId))
+            cloudSocket.Options.SetRequestHeader("X-DashScope-WorkSpace", _workspaceId);
+
+        await cloudSocket.ConnectAsync(_endpoint!, ct);
+
+        var taskId = Guid.NewGuid().ToString();
+        await SendJsonAsync(cloudSocket, new
+        {
+            header = new
+            {
+                action = "run-task",
+                task_id = taskId,
+                streaming = "duplex",
+            },
+            payload = new
+            {
+                task_group = "audio",
+                task = "asr",
+                function = "recognition",
+                model = _model,
+                parameters = new
+                {
+                    format = "pcm",
+                    sample_rate = SampleRate,
+                    language_hints = new[] { "zh" },
+                    semantic_punctuation_enabled = false,
+                    max_sentence_silence = 800,
+                    heartbeat = true,
+                },
+                input = new { },
+            },
+        }, ct);
+
+        await WaitForTaskStartedAsync(cloudSocket, taskId, ct);
+        await SendClientEventAsync(clientSocket, new { type = "ready" }, ct);
+
+        var clientPump = PumpClientAudioAsync(
+            clientSocket,
+            cloudSocket,
+            taskId,
+            _maxDurationSeconds,
+            ct);
+        var cloudPump = PumpCloudResultsAsync(cloudSocket, clientSocket, ct);
+
+        var completed = await Task.WhenAny(clientPump, cloudPump);
+        if (completed == cloudPump)
+        {
+            await cloudPump;
+            timeoutCts.Cancel();
+        }
+        else
+        {
+            var action = await clientPump;
+            if (action == ClientAction.Finish)
+            {
+                try
+                {
+                    await cloudPump.WaitAsync(TimeSpan.FromSeconds(12), ct);
+                }
+                catch (TimeoutException)
+                {
+                    await SendClientEventAsync(
+                        clientSocket,
+                        new { type = "error", message = "语音识别超时，请重试" },
+                        CancellationToken.None);
+                }
+                catch (OperationCanceledException)
+                {
+                    // 客户端断开或硬超时，无需再给客户端发消息
+                }
+            }
+            else
+            {
+                timeoutCts.Cancel();
+            }
+        }
+
+        await CloseSocketQuietlyAsync(cloudSocket);
+        await CloseSocketQuietlyAsync(clientSocket);
+    }
+
+    private static async Task WaitForTaskStartedAsync(
+        WebSocket cloudSocket,
+        string taskId,
+        CancellationToken ct)
+    {
+        while (true)
+        {
+            var message = await ReceiveMessageAsync(cloudSocket, ct);
+            if (message is null)
+                throw new InvalidOperationException("语音识别服务提前断开");
+            if (message.Value.Type != WebSocketMessageType.Text) continue;
+
+            using var json = JsonDocument.Parse(message.Value.Payload);
+            var header = json.RootElement.GetProperty("header");
+            var eventName = header.GetProperty("event").GetString();
+            var responseTaskId = header.TryGetProperty("task_id", out var id)
+                ? id.GetString()
+                : null;
+            if (eventName == "task-started" && responseTaskId == taskId) return;
+            if (eventName == "task-failed")
+            {
+                var error = header.TryGetProperty("error_message", out var errorMessage)
+                    ? errorMessage.GetString()
+                    : "语音识别启动失败";
+                throw new InvalidOperationException(error);
+            }
+        }
+    }
+
+    private static async Task<ClientAction> PumpClientAudioAsync(
+        WebSocket clientSocket,
+        WebSocket cloudSocket,
+        string taskId,
+        int maxDurationSeconds,
+        CancellationToken ct)
+    {
+        var maxBytes = (long)BytesPerSecond * maxDurationSeconds;
+        long audioBytes = 0;
+
+        while (!ct.IsCancellationRequested &&
+               clientSocket.State == WebSocketState.Open)
+        {
+            var message = await ReceiveMessageAsync(clientSocket, ct);
+            if (message is null) return ClientAction.Disconnect;
+
+            if (message.Value.Type == WebSocketMessageType.Binary)
+            {
+                audioBytes += message.Value.Payload.Length;
+                if (audioBytes > maxBytes)
+                {
+                    // 达到时长上限：主动结束并让阿里云返回已识别的结果，
+                    // 而不是直接断开。客户端松手后会拿到这段文字。
+                    await SendJsonAsync(cloudSocket, new
+                    {
+                        header = new
+                        {
+                            action = "finish-task",
+                            task_id = taskId,
+                            streaming = "duplex",
+                        },
+                        payload = new { input = new { } },
+                    }, ct);
+                    return ClientAction.Finish;
+                }
+                await cloudSocket.SendAsync(
+                    message.Value.Payload,
+                    WebSocketMessageType.Binary,
+                    true,
+                    ct);
+                continue;
+            }
+
+            if (message.Value.Type != WebSocketMessageType.Text) continue;
+            using var json = JsonDocument.Parse(message.Value.Payload);
+            var action = json.RootElement.TryGetProperty("action", out var actionElement)
+                ? actionElement.GetString()
+                : null;
+            if (action == "cancel") return ClientAction.Cancel;
+            if (action != "finish") continue;
+
+            await SendJsonAsync(cloudSocket, new
+            {
+                header = new
+                {
+                    action = "finish-task",
+                    task_id = taskId,
+                    streaming = "duplex",
+                },
+                payload = new { input = new { } },
+            }, ct);
+            return ClientAction.Finish;
+        }
+
+        return ClientAction.Disconnect;
+    }
+
+    private async Task PumpCloudResultsAsync(
+        WebSocket cloudSocket,
+        WebSocket clientSocket,
+        CancellationToken ct)
+    {
+        var committed = new StringBuilder();
+        var latest = "";
+
+        while (!ct.IsCancellationRequested &&
+               cloudSocket.State == WebSocketState.Open)
+        {
+            var message = await ReceiveMessageAsync(cloudSocket, ct);
+            if (message is null) return;
+            if (message.Value.Type != WebSocketMessageType.Text) continue;
+
+            using var json = JsonDocument.Parse(message.Value.Payload);
+            var root = json.RootElement;
+            if (!root.TryGetProperty("header", out var header)) continue;
+            var eventName = header.TryGetProperty("event", out var eventElement)
+                ? eventElement.GetString()
+                : null;
+
+            if (eventName == "result-generated")
+            {
+                if (!TryReadSentence(root, out var sentence, out var sentenceEnd))
+                    continue;
+                if (sentenceEnd && !string.IsNullOrWhiteSpace(sentence))
+                    committed.Append(sentence);
+                latest = sentenceEnd
+                    ? committed.ToString()
+                    : committed + sentence;
+                await SendClientEventAsync(clientSocket, new
+                {
+                    type = "partial",
+                    text = latest,
+                    sentenceEnd,
+                }, ct);
+                continue;
+            }
+
+            if (eventName == "task-finished")
+            {
+                await SendClientEventAsync(
+                    clientSocket,
+                    new { type = "done", text = latest.Trim() },
+                    ct);
+                return;
+            }
+
+            if (eventName == "task-failed")
+            {
+                var error = header.TryGetProperty("error_message", out var errorElement)
+                    ? errorElement.GetString()
+                    : "语音识别失败";
+                _logger.LogWarning("DashScope ASR task failed: {Error}", error);
+                await SendClientEventAsync(
+                    clientSocket,
+                    new { type = "error", message = "语音识别失败，请重试" },
+                    ct);
+                return;
+            }
+        }
+    }
+
+    private static bool TryReadSentence(
+        JsonElement root,
+        out string text,
+        out bool sentenceEnd)
+    {
+        text = "";
+        sentenceEnd = false;
+        if (!root.TryGetProperty("payload", out var payload) ||
+            !payload.TryGetProperty("output", out var output) ||
+            !output.TryGetProperty("sentence", out var sentence))
+            return false;
+
+        text = sentence.TryGetProperty("text", out var textElement)
+            ? textElement.GetString() ?? ""
+            : "";
+        sentenceEnd = sentence.TryGetProperty("sentence_end", out var endElement) &&
+                      endElement.ValueKind == JsonValueKind.True;
+        return true;
+    }
+
+    private static Task SendClientEventAsync(
+        WebSocket socket,
+        object value,
+        CancellationToken ct) => SendJsonAsync(socket, value, ct);
+
+    private static async Task SendJsonAsync(
+        WebSocket socket,
+        object value,
+        CancellationToken ct)
+    {
+        if (socket.State != WebSocketState.Open) return;
+        var bytes = JsonSerializer.SerializeToUtf8Bytes(value);
+        await socket.SendAsync(bytes, WebSocketMessageType.Text, true, ct);
+    }
+
+    private static async Task<(WebSocketMessageType Type, byte[] Payload)?>
+        ReceiveMessageAsync(WebSocket socket, CancellationToken ct)
+    {
+        var buffer = new byte[16 * 1024];
+        using var stream = new MemoryStream();
+        while (true)
+        {
+            var result = await socket.ReceiveAsync(buffer, ct);
+            if (result.MessageType == WebSocketMessageType.Close) return null;
+            stream.Write(buffer, 0, result.Count);
+            if (stream.Length > 2 * 1024 * 1024)
+                throw new InvalidOperationException("WebSocket 消息过大");
+            if (result.EndOfMessage)
+                return (result.MessageType, stream.ToArray());
+        }
+    }
+
+    private static async Task CloseSocketQuietlyAsync(WebSocket socket)
+    {
+        if (socket.State is not (WebSocketState.Open or WebSocketState.CloseReceived))
+            return;
+        try
+        {
+            await socket.CloseAsync(
+                WebSocketCloseStatus.NormalClosure,
+                "completed",
+                CancellationToken.None);
+        }
+        catch (WebSocketException)
+        {
+            // 对端可能已经主动断开。
+        }
+    }
+
+    private enum ClientAction
+    {
+        Finish,
+        Cancel,
+        Disconnect,
+    }
+}
--- a/backend/src/Health.WebApi/Endpoints/speech_endpoints.cs
+++ b/backend/src/Health.WebApi/Endpoints/speech_endpoints.cs
@@ -0,0 +1,69 @@
+using System.Net.WebSockets;
+using System.Text.Json;
+using Health.Application.Speech;
+
+namespace Health.WebApi.Endpoints;
+
+public static class SpeechEndpoints
+{
+    public static void MapSpeechEndpoints(this WebApplication app)
+    {
+        app.Map("/api/speech/realtime", async (
+            HttpContext http,
+            IRealtimeSpeechRecognitionProxy proxy,
+            ILoggerFactory loggerFactory) =>
+        {
+            if (!http.WebSockets.IsWebSocketRequest)
+            {
+                http.Response.StatusCode = StatusCodes.Status400BadRequest;
+                await http.Response.WriteAsJsonAsync(new
+                {
+                    code = 40001,
+                    data = (object?)null,
+                    message = "该接口仅支持 WebSocket 连接",
+                });
+                return;
+            }
+
+            if (!proxy.IsConfigured)
+            {
+                http.Response.StatusCode = StatusCodes.Status503ServiceUnavailable;
+                await http.Response.WriteAsJsonAsync(new
+                {
+                    code = 50301,
+                    data = (object?)null,
+                    message = "实时语音识别服务尚未配置",
+                });
+                return;
+            }
+
+            using var socket = await http.WebSockets.AcceptWebSocketAsync();
+            try
+            {
+                await proxy.ProxyAsync(socket, http.RequestAborted);
+            }
+            catch (OperationCanceledException) when (http.RequestAborted.IsCancellationRequested)
+            {
+                // 客户端离开页面或断开网络。
+            }
+            catch (Exception ex)
+            {
+                loggerFactory.CreateLogger("SpeechEndpoints")
+                    .LogError(ex, "Realtime speech proxy failed");
+                if (socket.State == WebSocketState.Open)
+                {
+                    var payload = JsonSerializer.SerializeToUtf8Bytes(new
+                    {
+                        type = "error",
+                        message = "语音服务暂时不可用，请稍后重试",
+                    });
+                    await socket.SendAsync(
+                        payload,
+                        WebSocketMessageType.Text,
+                        true,
+                        CancellationToken.None);
+                }
+            }
+        }).RequireAuthorization(policy => policy.RequireRole("User"));
+    }
+}
--- a/backend/src/Health.WebApi/Program.cs
+++ b/backend/src/Health.WebApi/Program.cs
@@ -11,6 +11,7 @@ using Health.Application.Medications;
 using Health.Application.Maintenance;
 using Health.Application.Notifications;
 using Health.Application.Reports;
+using Health.Application.Speech;
 using Health.Application.Users;
 using Health.Infrastructure.AI;
 using Health.Infrastructure.Admin;
@@ -25,6 +26,7 @@ using Health.Infrastructure.Medications;
 using Health.Infrastructure.Maintenance;
 using Health.Infrastructure.Notifications;
 using Health.Infrastructure.Reports;
+using Health.Infrastructure.Speech;
 using Health.Infrastructure.Services;
 using Health.Infrastructure.Users;
 using Health.WebApi.BackgroundServices;
@@ -150,6 +152,7 @@ builder.Services.AddScoped<IReportAnalysisService, ReportAnalysisService>();
 builder.Services.AddScoped<IReportAnalysisQueue, EfReportAnalysisQueue>();
 builder.Services.AddScoped<IUserService, UserService>();
 builder.Services.AddScoped<IUserRepository, EfUserRepository>();
+builder.Services.AddSingleton<IRealtimeSpeechRecognitionProxy, DashScopeSpeechRecognitionProxy>();
 builder.Services.AddScoped<IAccountFileCleanup>(_ => new LocalAccountFileCleanup(
    Path.Combine(Directory.GetCurrentDirectory(), "uploads")));

@@ -205,6 +208,10 @@ app.UseMiddleware<ExceptionMiddleware>();
 app.UseCors();
 app.UseAuthentication();
 app.UseAuthorization();
+app.UseWebSockets(new WebSocketOptions
+{
+    KeepAliveInterval = TimeSpan.FromSeconds(20),
+});

 // 默认 wwwroot 静态文件（法律文档 H5 页面等）
 app.UseDefaultFiles();
@@ -240,6 +247,7 @@ app.MapNotificationEndpoints();
 app.MapDoctorEndpoints();
 app.MapAdminEndpoints();
 app.MapFollowUpEndpoints();
+app.MapSpeechEndpoints();

 // SignalR Hub
 app.MapHub<Health.WebApi.Hubs.ConsultationHub>("/hubs/consultation");