mirror of
https://github.com/openclaw/openclaw.git
synced 2026-04-19 11:18:37 +00:00
iOS: wire node commands and incremental TTS
This commit is contained in:
committed by
Mariano Belinky
parent
b7aac92ac4
commit
532b9653be
25
apps/ios/Sources/Capabilities/NodeCapabilityRouter.swift
Normal file
25
apps/ios/Sources/Capabilities/NodeCapabilityRouter.swift
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
import Foundation
|
||||||
|
import OpenClawKit
|
||||||
|
|
||||||
|
@MainActor
|
||||||
|
final class NodeCapabilityRouter {
|
||||||
|
enum RouterError: Error {
|
||||||
|
case unknownCommand
|
||||||
|
case handlerUnavailable
|
||||||
|
}
|
||||||
|
|
||||||
|
typealias Handler = (BridgeInvokeRequest) async throws -> BridgeInvokeResponse
|
||||||
|
|
||||||
|
private let handlers: [String: Handler]
|
||||||
|
|
||||||
|
init(handlers: [String: Handler]) {
|
||||||
|
self.handlers = handlers
|
||||||
|
}
|
||||||
|
|
||||||
|
func handle(_ request: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
|
||||||
|
guard let handler = handlers[request.command] else {
|
||||||
|
throw RouterError.unknownCommand
|
||||||
|
}
|
||||||
|
return try await handler(request)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -214,8 +214,8 @@ final class GatewayConnectionController {
|
|||||||
guard let appModel else { return }
|
guard let appModel else { return }
|
||||||
let connectOptions = self.makeConnectOptions()
|
let connectOptions = self.makeConnectOptions()
|
||||||
|
|
||||||
Task { [weak self] in
|
Task { [weak appModel] in
|
||||||
guard let self else { return }
|
guard let appModel else { return }
|
||||||
await MainActor.run {
|
await MainActor.run {
|
||||||
appModel.gatewayStatusText = "Connecting…"
|
appModel.gatewayStatusText = "Connecting…"
|
||||||
}
|
}
|
||||||
@@ -353,6 +353,7 @@ final class GatewayConnectionController {
|
|||||||
OpenClawCanvasA2UICommand.reset.rawValue,
|
OpenClawCanvasA2UICommand.reset.rawValue,
|
||||||
OpenClawScreenCommand.record.rawValue,
|
OpenClawScreenCommand.record.rawValue,
|
||||||
OpenClawSystemCommand.notify.rawValue,
|
OpenClawSystemCommand.notify.rawValue,
|
||||||
|
OpenClawChatCommand.push.rawValue,
|
||||||
OpenClawTalkCommand.pttStart.rawValue,
|
OpenClawTalkCommand.pttStart.rawValue,
|
||||||
OpenClawTalkCommand.pttStop.rawValue,
|
OpenClawTalkCommand.pttStop.rawValue,
|
||||||
OpenClawTalkCommand.pttCancel.rawValue,
|
OpenClawTalkCommand.pttCancel.rawValue,
|
||||||
|
|||||||
85
apps/ios/Sources/Gateway/GatewayHealthMonitor.swift
Normal file
85
apps/ios/Sources/Gateway/GatewayHealthMonitor.swift
Normal file
@@ -0,0 +1,85 @@
|
|||||||
|
import Foundation
|
||||||
|
import OpenClawKit
|
||||||
|
|
||||||
|
@MainActor
|
||||||
|
final class GatewayHealthMonitor {
|
||||||
|
struct Config: Sendable {
|
||||||
|
var intervalSeconds: Double
|
||||||
|
var timeoutSeconds: Double
|
||||||
|
var maxFailures: Int
|
||||||
|
}
|
||||||
|
|
||||||
|
private let config: Config
|
||||||
|
private let sleep: @Sendable (UInt64) async -> Void
|
||||||
|
private var task: Task<Void, Never>?
|
||||||
|
|
||||||
|
init(
|
||||||
|
config: Config = Config(intervalSeconds: 15, timeoutSeconds: 5, maxFailures: 3),
|
||||||
|
sleep: @escaping @Sendable (UInt64) async -> Void = { nanoseconds in
|
||||||
|
try? await Task.sleep(nanoseconds: nanoseconds)
|
||||||
|
}
|
||||||
|
) {
|
||||||
|
self.config = config
|
||||||
|
self.sleep = sleep
|
||||||
|
}
|
||||||
|
|
||||||
|
func start(
|
||||||
|
check: @escaping @Sendable () async throws -> Bool,
|
||||||
|
onFailure: @escaping @Sendable (_ failureCount: Int) async -> Void)
|
||||||
|
{
|
||||||
|
self.stop()
|
||||||
|
let config = self.config
|
||||||
|
let sleep = self.sleep
|
||||||
|
self.task = Task { @MainActor in
|
||||||
|
var failures = 0
|
||||||
|
while !Task.isCancelled {
|
||||||
|
let ok = await Self.runCheck(check: check, timeoutSeconds: config.timeoutSeconds)
|
||||||
|
if ok {
|
||||||
|
failures = 0
|
||||||
|
} else {
|
||||||
|
failures += 1
|
||||||
|
if failures >= max(1, config.maxFailures) {
|
||||||
|
await onFailure(failures)
|
||||||
|
failures = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if Task.isCancelled { break }
|
||||||
|
let interval = max(0.0, config.intervalSeconds)
|
||||||
|
let nanos = UInt64(interval * 1_000_000_000)
|
||||||
|
if nanos > 0 {
|
||||||
|
await sleep(nanos)
|
||||||
|
} else {
|
||||||
|
await Task.yield()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func stop() {
|
||||||
|
self.task?.cancel()
|
||||||
|
self.task = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func runCheck(
|
||||||
|
check: @escaping @Sendable () async throws -> Bool,
|
||||||
|
timeoutSeconds: Double) async -> Bool
|
||||||
|
{
|
||||||
|
let timeout = max(0.0, timeoutSeconds)
|
||||||
|
if timeout == 0 {
|
||||||
|
return (try? await check()) ?? false
|
||||||
|
}
|
||||||
|
do {
|
||||||
|
let timeoutError = NSError(
|
||||||
|
domain: "GatewayHealthMonitor",
|
||||||
|
code: 1,
|
||||||
|
userInfo: [NSLocalizedDescriptionKey: "health check timed out"])
|
||||||
|
return try await AsyncTimeout.withTimeout(
|
||||||
|
seconds: timeout,
|
||||||
|
onTimeout: { timeoutError },
|
||||||
|
operation: check)
|
||||||
|
} catch {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import OpenClawChatUI
|
||||||
import OpenClawKit
|
import OpenClawKit
|
||||||
import Observation
|
import Observation
|
||||||
import SwiftUI
|
import SwiftUI
|
||||||
@@ -61,6 +62,8 @@ final class NodeAppModel {
|
|||||||
private var gatewayTask: Task<Void, Never>?
|
private var gatewayTask: Task<Void, Never>?
|
||||||
private var voiceWakeSyncTask: Task<Void, Never>?
|
private var voiceWakeSyncTask: Task<Void, Never>?
|
||||||
@ObservationIgnored private var cameraHUDDismissTask: Task<Void, Never>?
|
@ObservationIgnored private var cameraHUDDismissTask: Task<Void, Never>?
|
||||||
|
@ObservationIgnored private var capabilityRouter: NodeCapabilityRouter
|
||||||
|
private let gatewayHealthMonitor = GatewayHealthMonitor()
|
||||||
private let notificationCenter: NotificationCentering
|
private let notificationCenter: NotificationCentering
|
||||||
let voiceWake = VoiceWakeManager()
|
let voiceWake = VoiceWakeManager()
|
||||||
let talkMode: TalkModeManager
|
let talkMode: TalkModeManager
|
||||||
@@ -108,6 +111,8 @@ final class NodeAppModel {
|
|||||||
self.remindersService = remindersService
|
self.remindersService = remindersService
|
||||||
self.motionService = motionService
|
self.motionService = motionService
|
||||||
self.talkMode = talkMode
|
self.talkMode = talkMode
|
||||||
|
self.capabilityRouter = NodeCapabilityRouter(handlers: [:])
|
||||||
|
self.capabilityRouter = self.buildCapabilityRouter()
|
||||||
|
|
||||||
self.voiceWake.configure { [weak self] cmd in
|
self.voiceWake.configure { [weak self] cmd in
|
||||||
guard let self else { return }
|
guard let self else { return }
|
||||||
@@ -281,6 +286,7 @@ final class NodeAppModel {
|
|||||||
connectOptions: GatewayConnectOptions)
|
connectOptions: GatewayConnectOptions)
|
||||||
{
|
{
|
||||||
self.gatewayTask?.cancel()
|
self.gatewayTask?.cancel()
|
||||||
|
self.gatewayHealthMonitor.stop()
|
||||||
self.gatewayServerName = nil
|
self.gatewayServerName = nil
|
||||||
self.gatewayRemoteAddress = nil
|
self.gatewayRemoteAddress = nil
|
||||||
let id = gatewayStableID.trimmingCharacters(in: .whitespacesAndNewlines)
|
let id = gatewayStableID.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||||
@@ -325,6 +331,7 @@ final class NodeAppModel {
|
|||||||
}
|
}
|
||||||
await self.refreshBrandingFromGateway()
|
await self.refreshBrandingFromGateway()
|
||||||
await self.startVoiceWakeSync()
|
await self.startVoiceWakeSync()
|
||||||
|
await MainActor.run { self.startGatewayHealthMonitor() }
|
||||||
await self.showA2UIOnConnectIfNeeded()
|
await self.showA2UIOnConnectIfNeeded()
|
||||||
},
|
},
|
||||||
onDisconnected: { [weak self] reason in
|
onDisconnected: { [weak self] reason in
|
||||||
@@ -337,6 +344,7 @@ final class NodeAppModel {
|
|||||||
self.showLocalCanvasOnDisconnect()
|
self.showLocalCanvasOnDisconnect()
|
||||||
self.gatewayStatusText = "Disconnected: \(reason)"
|
self.gatewayStatusText = "Disconnected: \(reason)"
|
||||||
}
|
}
|
||||||
|
await MainActor.run { self.stopGatewayHealthMonitor() }
|
||||||
},
|
},
|
||||||
onInvoke: { [weak self] req in
|
onInvoke: { [weak self] req in
|
||||||
guard let self else {
|
guard let self else {
|
||||||
@@ -391,6 +399,7 @@ final class NodeAppModel {
|
|||||||
self.gatewayTask = nil
|
self.gatewayTask = nil
|
||||||
self.voiceWakeSyncTask?.cancel()
|
self.voiceWakeSyncTask?.cancel()
|
||||||
self.voiceWakeSyncTask = nil
|
self.voiceWakeSyncTask = nil
|
||||||
|
self.gatewayHealthMonitor.stop()
|
||||||
Task { await self.gateway.disconnect() }
|
Task { await self.gateway.disconnect() }
|
||||||
self.gatewayStatusText = "Offline"
|
self.gatewayStatusText = "Offline"
|
||||||
self.gatewayServerName = nil
|
self.gatewayServerName = nil
|
||||||
@@ -492,6 +501,27 @@ final class NodeAppModel {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private func startGatewayHealthMonitor() {
|
||||||
|
self.gatewayHealthMonitor.start(
|
||||||
|
check: { [weak self] in
|
||||||
|
guard let self else { return false }
|
||||||
|
do {
|
||||||
|
let data = try await self.gateway.request(method: "health", paramsJSON: nil, timeoutSeconds: 6)
|
||||||
|
return (try? JSONDecoder().decode(OpenClawGatewayHealthOK.self, from: data))?.ok ?? true
|
||||||
|
} catch {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
onFailure: { [weak self] _ in
|
||||||
|
guard let self else { return }
|
||||||
|
await self.gateway.disconnect()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
private func stopGatewayHealthMonitor() {
|
||||||
|
self.gatewayHealthMonitor.stop()
|
||||||
|
}
|
||||||
|
|
||||||
private func refreshWakeWordsFromGateway() async {
|
private func refreshWakeWordsFromGateway() async {
|
||||||
do {
|
do {
|
||||||
let data = try await self.gateway.request(method: "voicewake.get", paramsJSON: "{}", timeoutSeconds: 8)
|
let data = try await self.gateway.request(method: "voicewake.get", paramsJSON: "{}", timeoutSeconds: 8)
|
||||||
@@ -597,54 +627,19 @@ final class NodeAppModel {
|
|||||||
}
|
}
|
||||||
|
|
||||||
do {
|
do {
|
||||||
switch command {
|
return try await self.capabilityRouter.handle(req)
|
||||||
case OpenClawLocationCommand.get.rawValue:
|
} catch let error as NodeCapabilityRouter.RouterError {
|
||||||
return try await self.handleLocationInvoke(req)
|
switch error {
|
||||||
case OpenClawCanvasCommand.present.rawValue,
|
case .unknownCommand:
|
||||||
OpenClawCanvasCommand.hide.rawValue,
|
|
||||||
OpenClawCanvasCommand.navigate.rawValue,
|
|
||||||
OpenClawCanvasCommand.evalJS.rawValue,
|
|
||||||
OpenClawCanvasCommand.snapshot.rawValue:
|
|
||||||
return try await self.handleCanvasInvoke(req)
|
|
||||||
case OpenClawCanvasA2UICommand.reset.rawValue,
|
|
||||||
OpenClawCanvasA2UICommand.push.rawValue,
|
|
||||||
OpenClawCanvasA2UICommand.pushJSONL.rawValue:
|
|
||||||
return try await self.handleCanvasA2UIInvoke(req)
|
|
||||||
case OpenClawCameraCommand.list.rawValue,
|
|
||||||
OpenClawCameraCommand.snap.rawValue,
|
|
||||||
OpenClawCameraCommand.clip.rawValue:
|
|
||||||
return try await self.handleCameraInvoke(req)
|
|
||||||
case OpenClawScreenCommand.record.rawValue:
|
|
||||||
return try await self.handleScreenRecordInvoke(req)
|
|
||||||
case OpenClawSystemCommand.notify.rawValue:
|
|
||||||
return try await self.handleSystemNotify(req)
|
|
||||||
case OpenClawDeviceCommand.status.rawValue,
|
|
||||||
OpenClawDeviceCommand.info.rawValue:
|
|
||||||
return try await self.handleDeviceInvoke(req)
|
|
||||||
case OpenClawPhotosCommand.latest.rawValue:
|
|
||||||
return try await self.handlePhotosInvoke(req)
|
|
||||||
case OpenClawContactsCommand.search.rawValue,
|
|
||||||
OpenClawContactsCommand.add.rawValue:
|
|
||||||
return try await self.handleContactsInvoke(req)
|
|
||||||
case OpenClawCalendarCommand.events.rawValue,
|
|
||||||
OpenClawCalendarCommand.add.rawValue:
|
|
||||||
return try await self.handleCalendarInvoke(req)
|
|
||||||
case OpenClawRemindersCommand.list.rawValue,
|
|
||||||
OpenClawRemindersCommand.add.rawValue:
|
|
||||||
return try await self.handleRemindersInvoke(req)
|
|
||||||
case OpenClawMotionCommand.activity.rawValue,
|
|
||||||
OpenClawMotionCommand.pedometer.rawValue:
|
|
||||||
return try await self.handleMotionInvoke(req)
|
|
||||||
case OpenClawTalkCommand.pttStart.rawValue,
|
|
||||||
OpenClawTalkCommand.pttStop.rawValue,
|
|
||||||
OpenClawTalkCommand.pttCancel.rawValue,
|
|
||||||
OpenClawTalkCommand.pttOnce.rawValue:
|
|
||||||
return try await self.handleTalkInvoke(req)
|
|
||||||
default:
|
|
||||||
return BridgeInvokeResponse(
|
return BridgeInvokeResponse(
|
||||||
id: req.id,
|
id: req.id,
|
||||||
ok: false,
|
ok: false,
|
||||||
error: OpenClawNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
|
error: OpenClawNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
|
||||||
|
case .handlerUnavailable:
|
||||||
|
return BridgeInvokeResponse(
|
||||||
|
id: req.id,
|
||||||
|
ok: false,
|
||||||
|
error: OpenClawNodeError(code: .unavailable, message: "node handler unavailable"))
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
if command.hasPrefix("camera.") {
|
if command.hasPrefix("camera.") {
|
||||||
@@ -983,6 +978,22 @@ final class NodeAppModel {
|
|||||||
let content = UNMutableNotificationContent()
|
let content = UNMutableNotificationContent()
|
||||||
content.title = title
|
content.title = title
|
||||||
content.body = body
|
content.body = body
|
||||||
|
if #available(iOS 15.0, *) {
|
||||||
|
switch params.priority ?? .active {
|
||||||
|
case .passive:
|
||||||
|
content.interruptionLevel = .passive
|
||||||
|
case .timeSensitive:
|
||||||
|
content.interruptionLevel = .timeSensitive
|
||||||
|
case .active:
|
||||||
|
content.interruptionLevel = .active
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let soundValue = params.sound?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
|
||||||
|
if let soundValue, ["none", "silent", "off", "false", "0"].contains(soundValue) {
|
||||||
|
content.sound = nil
|
||||||
|
} else {
|
||||||
|
content.sound = .default
|
||||||
|
}
|
||||||
let request = UNNotificationRequest(
|
let request = UNNotificationRequest(
|
||||||
identifier: UUID().uuidString,
|
identifier: UUID().uuidString,
|
||||||
content: content,
|
content: content,
|
||||||
@@ -998,6 +1009,51 @@ final class NodeAppModel {
|
|||||||
return BridgeInvokeResponse(id: req.id, ok: true)
|
return BridgeInvokeResponse(id: req.id, ok: true)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private func handleChatPushInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
|
||||||
|
let params = try Self.decodeParams(OpenClawChatPushParams.self, from: req.paramsJSON)
|
||||||
|
let text = params.text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||||
|
guard !text.isEmpty else {
|
||||||
|
return BridgeInvokeResponse(
|
||||||
|
id: req.id,
|
||||||
|
ok: false,
|
||||||
|
error: OpenClawNodeError(code: .invalidRequest, message: "INVALID_REQUEST: empty chat.push text"))
|
||||||
|
}
|
||||||
|
|
||||||
|
let finalStatus = await self.requestNotificationAuthorizationIfNeeded()
|
||||||
|
let messageId = UUID().uuidString
|
||||||
|
if finalStatus == .authorized || finalStatus == .provisional || finalStatus == .ephemeral {
|
||||||
|
let addResult = await self.runNotificationCall(timeoutSeconds: 2.0) { [notificationCenter] in
|
||||||
|
let content = UNMutableNotificationContent()
|
||||||
|
content.title = "OpenClaw"
|
||||||
|
content.body = text
|
||||||
|
content.sound = .default
|
||||||
|
content.userInfo = ["messageId": messageId]
|
||||||
|
let request = UNNotificationRequest(
|
||||||
|
identifier: messageId,
|
||||||
|
content: content,
|
||||||
|
trigger: nil)
|
||||||
|
try await notificationCenter.add(request)
|
||||||
|
}
|
||||||
|
if case let .failure(error) = addResult {
|
||||||
|
return BridgeInvokeResponse(
|
||||||
|
id: req.id,
|
||||||
|
ok: false,
|
||||||
|
error: OpenClawNodeError(code: .unavailable, message: "NOTIFICATION_FAILED: \(error.message)"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if params.speak ?? true {
|
||||||
|
let toSpeak = text
|
||||||
|
Task { @MainActor in
|
||||||
|
try? await TalkSystemSpeechSynthesizer.shared.speak(text: toSpeak)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let payload = OpenClawChatPushPayload(messageId: messageId)
|
||||||
|
let json = try Self.encodePayload(payload)
|
||||||
|
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: json)
|
||||||
|
}
|
||||||
|
|
||||||
private func requestNotificationAuthorizationIfNeeded() async -> NotificationAuthorizationStatus {
|
private func requestNotificationAuthorizationIfNeeded() async -> NotificationAuthorizationStatus {
|
||||||
let status = await self.notificationAuthorizationStatus()
|
let status = await self.notificationAuthorizationStatus()
|
||||||
guard status == .notDetermined else { return status }
|
guard status == .notDetermined else { return status }
|
||||||
@@ -1203,6 +1259,123 @@ final class NodeAppModel {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private extension NodeAppModel {
|
private extension NodeAppModel {
|
||||||
|
// Central registry for node invoke routing to keep commands in one place.
|
||||||
|
func buildCapabilityRouter() -> NodeCapabilityRouter {
|
||||||
|
var handlers: [String: NodeCapabilityRouter.Handler] = [:]
|
||||||
|
|
||||||
|
func register(_ commands: [String], handler: @escaping NodeCapabilityRouter.Handler) {
|
||||||
|
for command in commands {
|
||||||
|
handlers[command] = handler
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
register([OpenClawLocationCommand.get.rawValue]) { [weak self] req in
|
||||||
|
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||||
|
return try await self.handleLocationInvoke(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
register([
|
||||||
|
OpenClawCanvasCommand.present.rawValue,
|
||||||
|
OpenClawCanvasCommand.hide.rawValue,
|
||||||
|
OpenClawCanvasCommand.navigate.rawValue,
|
||||||
|
OpenClawCanvasCommand.evalJS.rawValue,
|
||||||
|
OpenClawCanvasCommand.snapshot.rawValue,
|
||||||
|
]) { [weak self] req in
|
||||||
|
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||||
|
return try await self.handleCanvasInvoke(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
register([
|
||||||
|
OpenClawCanvasA2UICommand.reset.rawValue,
|
||||||
|
OpenClawCanvasA2UICommand.push.rawValue,
|
||||||
|
OpenClawCanvasA2UICommand.pushJSONL.rawValue,
|
||||||
|
]) { [weak self] req in
|
||||||
|
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||||
|
return try await self.handleCanvasA2UIInvoke(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
register([
|
||||||
|
OpenClawCameraCommand.list.rawValue,
|
||||||
|
OpenClawCameraCommand.snap.rawValue,
|
||||||
|
OpenClawCameraCommand.clip.rawValue,
|
||||||
|
]) { [weak self] req in
|
||||||
|
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||||
|
return try await self.handleCameraInvoke(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
register([OpenClawScreenCommand.record.rawValue]) { [weak self] req in
|
||||||
|
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||||
|
return try await self.handleScreenRecordInvoke(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
register([OpenClawSystemCommand.notify.rawValue]) { [weak self] req in
|
||||||
|
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||||
|
return try await self.handleSystemNotify(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
register([OpenClawChatCommand.push.rawValue]) { [weak self] req in
|
||||||
|
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||||
|
return try await self.handleChatPushInvoke(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
register([
|
||||||
|
OpenClawDeviceCommand.status.rawValue,
|
||||||
|
OpenClawDeviceCommand.info.rawValue,
|
||||||
|
]) { [weak self] req in
|
||||||
|
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||||
|
return try await self.handleDeviceInvoke(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
register([OpenClawPhotosCommand.latest.rawValue]) { [weak self] req in
|
||||||
|
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||||
|
return try await self.handlePhotosInvoke(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
register([
|
||||||
|
OpenClawContactsCommand.search.rawValue,
|
||||||
|
OpenClawContactsCommand.add.rawValue,
|
||||||
|
]) { [weak self] req in
|
||||||
|
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||||
|
return try await self.handleContactsInvoke(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
register([
|
||||||
|
OpenClawCalendarCommand.events.rawValue,
|
||||||
|
OpenClawCalendarCommand.add.rawValue,
|
||||||
|
]) { [weak self] req in
|
||||||
|
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||||
|
return try await self.handleCalendarInvoke(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
register([
|
||||||
|
OpenClawRemindersCommand.list.rawValue,
|
||||||
|
OpenClawRemindersCommand.add.rawValue,
|
||||||
|
]) { [weak self] req in
|
||||||
|
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||||
|
return try await self.handleRemindersInvoke(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
register([
|
||||||
|
OpenClawMotionCommand.activity.rawValue,
|
||||||
|
OpenClawMotionCommand.pedometer.rawValue,
|
||||||
|
]) { [weak self] req in
|
||||||
|
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||||
|
return try await self.handleMotionInvoke(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
register([
|
||||||
|
OpenClawTalkCommand.pttStart.rawValue,
|
||||||
|
OpenClawTalkCommand.pttStop.rawValue,
|
||||||
|
OpenClawTalkCommand.pttCancel.rawValue,
|
||||||
|
OpenClawTalkCommand.pttOnce.rawValue,
|
||||||
|
]) { [weak self] req in
|
||||||
|
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||||
|
return try await self.handleTalkInvoke(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
return NodeCapabilityRouter(handlers: handlers)
|
||||||
|
}
|
||||||
|
|
||||||
func locationMode() -> OpenClawLocationMode {
|
func locationMode() -> OpenClawLocationMode {
|
||||||
let raw = UserDefaults.standard.string(forKey: "location.enabledMode") ?? "off"
|
let raw = UserDefaults.standard.string(forKey: "location.enabledMode") ?? "off"
|
||||||
return OpenClawLocationMode(rawValue: raw) ?? .off
|
return OpenClawLocationMode(rawValue: raw) ?? .off
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import AVFAudio
|
import AVFAudio
|
||||||
|
import OpenClawChatUI
|
||||||
import OpenClawKit
|
import OpenClawKit
|
||||||
import OpenClawProtocol
|
import OpenClawProtocol
|
||||||
import Foundation
|
import Foundation
|
||||||
@@ -65,6 +66,14 @@ final class TalkModeManager: NSObject {
|
|||||||
private let silenceWindow: TimeInterval = 0.7
|
private let silenceWindow: TimeInterval = 0.7
|
||||||
|
|
||||||
private var chatSubscribedSessionKeys = Set<String>()
|
private var chatSubscribedSessionKeys = Set<String>()
|
||||||
|
private var incrementalSpeechQueue: [String] = []
|
||||||
|
private var incrementalSpeechTask: Task<Void, Never>?
|
||||||
|
private var incrementalSpeechActive = false
|
||||||
|
private var incrementalSpeechUsed = false
|
||||||
|
private var incrementalSpeechLanguage: String?
|
||||||
|
private var incrementalSpeechBuffer = IncrementalSpeechBuffer()
|
||||||
|
private var incrementalSpeechContext: IncrementalSpeechContext?
|
||||||
|
private var incrementalSpeechDirective: TalkDirective?
|
||||||
|
|
||||||
private let logger = Logger(subsystem: "bot.molt", category: "TalkMode")
|
private let logger = Logger(subsystem: "bot.molt", category: "TalkMode")
|
||||||
|
|
||||||
@@ -456,6 +465,14 @@ final class TalkModeManager: NSObject {
|
|||||||
}
|
}
|
||||||
if isFinal {
|
if isFinal {
|
||||||
self.lastTranscript = trimmed
|
self.lastTranscript = trimmed
|
||||||
|
guard !trimmed.isEmpty else { return }
|
||||||
|
if self.captureMode == .pushToTalk, self.pttAutoStopEnabled, self.isPushToTalkActive {
|
||||||
|
_ = await self.endPushToTalk()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if self.captureMode == .continuous, !self.isSpeaking {
|
||||||
|
await self.processTranscript(trimmed, restartAfter: true)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -539,6 +556,15 @@ final class TalkModeManager: NSObject {
|
|||||||
"chat.send start sessionKey=\(sessionKey, privacy: .public) chars=\(prompt.count, privacy: .public)")
|
"chat.send start sessionKey=\(sessionKey, privacy: .public) chars=\(prompt.count, privacy: .public)")
|
||||||
let runId = try await self.sendChat(prompt, gateway: gateway)
|
let runId = try await self.sendChat(prompt, gateway: gateway)
|
||||||
self.logger.info("chat.send ok runId=\(runId, privacy: .public)")
|
self.logger.info("chat.send ok runId=\(runId, privacy: .public)")
|
||||||
|
let shouldIncremental = self.shouldUseIncrementalTTS()
|
||||||
|
var streamingTask: Task<Void, Never>?
|
||||||
|
if shouldIncremental {
|
||||||
|
self.resetIncrementalSpeech()
|
||||||
|
streamingTask = Task { @MainActor [weak self] in
|
||||||
|
guard let self else { return }
|
||||||
|
await self.streamAssistant(runId: runId, gateway: gateway)
|
||||||
|
}
|
||||||
|
}
|
||||||
let completion = await self.waitForChatCompletion(runId: runId, gateway: gateway, timeoutSeconds: 120)
|
let completion = await self.waitForChatCompletion(runId: runId, gateway: gateway, timeoutSeconds: 120)
|
||||||
if completion == .timeout {
|
if completion == .timeout {
|
||||||
self.logger.warning(
|
self.logger.warning(
|
||||||
@@ -546,27 +572,44 @@ final class TalkModeManager: NSObject {
|
|||||||
} else if completion == .aborted {
|
} else if completion == .aborted {
|
||||||
self.statusText = "Aborted"
|
self.statusText = "Aborted"
|
||||||
self.logger.warning("chat completion aborted runId=\(runId, privacy: .public)")
|
self.logger.warning("chat completion aborted runId=\(runId, privacy: .public)")
|
||||||
|
streamingTask?.cancel()
|
||||||
|
await self.finishIncrementalSpeech()
|
||||||
await self.start()
|
await self.start()
|
||||||
return
|
return
|
||||||
} else if completion == .error {
|
} else if completion == .error {
|
||||||
self.statusText = "Chat error"
|
self.statusText = "Chat error"
|
||||||
self.logger.warning("chat completion error runId=\(runId, privacy: .public)")
|
self.logger.warning("chat completion error runId=\(runId, privacy: .public)")
|
||||||
|
streamingTask?.cancel()
|
||||||
|
await self.finishIncrementalSpeech()
|
||||||
await self.start()
|
await self.start()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
guard let assistantText = try await self.waitForAssistantText(
|
var assistantText = try await self.waitForAssistantText(
|
||||||
gateway: gateway,
|
gateway: gateway,
|
||||||
since: startedAt,
|
since: startedAt,
|
||||||
timeoutSeconds: completion == .final ? 12 : 25)
|
timeoutSeconds: completion == .final ? 12 : 25)
|
||||||
else {
|
if assistantText == nil, shouldIncremental {
|
||||||
|
let fallback = self.incrementalSpeechBuffer.latestText
|
||||||
|
if !fallback.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
|
||||||
|
assistantText = fallback
|
||||||
|
}
|
||||||
|
}
|
||||||
|
guard let assistantText else {
|
||||||
self.statusText = "No reply"
|
self.statusText = "No reply"
|
||||||
self.logger.warning("assistant text timeout runId=\(runId, privacy: .public)")
|
self.logger.warning("assistant text timeout runId=\(runId, privacy: .public)")
|
||||||
|
streamingTask?.cancel()
|
||||||
|
await self.finishIncrementalSpeech()
|
||||||
await self.start()
|
await self.start()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
self.logger.info("assistant text ok chars=\(assistantText.count, privacy: .public)")
|
self.logger.info("assistant text ok chars=\(assistantText.count, privacy: .public)")
|
||||||
await self.playAssistant(text: assistantText)
|
streamingTask?.cancel()
|
||||||
|
if shouldIncremental {
|
||||||
|
await self.handleIncrementalAssistantFinal(text: assistantText)
|
||||||
|
} else {
|
||||||
|
await self.playAssistant(text: assistantText)
|
||||||
|
}
|
||||||
} catch {
|
} catch {
|
||||||
self.statusText = "Talk failed: \(error.localizedDescription)"
|
self.statusText = "Talk failed: \(error.localizedDescription)"
|
||||||
self.logger.error("finalize failed: \(error.localizedDescription, privacy: .public)")
|
self.logger.error("finalize failed: \(error.localizedDescription, privacy: .public)")
|
||||||
@@ -720,24 +763,7 @@ final class TalkModeManager: NSObject {
|
|||||||
let directive = parsed.directive
|
let directive = parsed.directive
|
||||||
let cleaned = parsed.stripped.trimmingCharacters(in: .whitespacesAndNewlines)
|
let cleaned = parsed.stripped.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||||
guard !cleaned.isEmpty else { return }
|
guard !cleaned.isEmpty else { return }
|
||||||
|
self.applyDirective(directive)
|
||||||
let requestedVoice = directive?.voiceId?.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
||||||
let resolvedVoice = self.resolveVoiceAlias(requestedVoice)
|
|
||||||
if requestedVoice?.isEmpty == false, resolvedVoice == nil {
|
|
||||||
self.logger.warning("unknown voice alias \(requestedVoice ?? "?", privacy: .public)")
|
|
||||||
}
|
|
||||||
if let voice = resolvedVoice {
|
|
||||||
if directive?.once != true {
|
|
||||||
self.currentVoiceId = voice
|
|
||||||
self.voiceOverrideActive = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if let model = directive?.modelId {
|
|
||||||
if directive?.once != true {
|
|
||||||
self.currentModelId = model
|
|
||||||
self.modelOverrideActive = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
self.statusText = "Generating voice…"
|
self.statusText = "Generating voice…"
|
||||||
self.isSpeaking = true
|
self.isSpeaking = true
|
||||||
@@ -746,6 +772,11 @@ final class TalkModeManager: NSObject {
|
|||||||
do {
|
do {
|
||||||
let started = Date()
|
let started = Date()
|
||||||
let language = ElevenLabsTTSClient.validatedLanguage(directive?.language)
|
let language = ElevenLabsTTSClient.validatedLanguage(directive?.language)
|
||||||
|
let requestedVoice = directive?.voiceId?.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||||
|
let resolvedVoice = self.resolveVoiceAlias(requestedVoice)
|
||||||
|
if requestedVoice?.isEmpty == false, resolvedVoice == nil {
|
||||||
|
self.logger.warning("unknown voice alias \(requestedVoice ?? "?", privacy: .public)")
|
||||||
|
}
|
||||||
|
|
||||||
let resolvedKey =
|
let resolvedKey =
|
||||||
(self.apiKey?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false ? self.apiKey : nil) ??
|
(self.apiKey?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false ? self.apiKey : nil) ??
|
||||||
@@ -875,6 +906,7 @@ final class TalkModeManager: NSObject {
|
|||||||
? self.mp3Player.stop()
|
? self.mp3Player.stop()
|
||||||
: self.pcmPlayer.stop()
|
: self.pcmPlayer.stop()
|
||||||
TalkSystemSpeechSynthesizer.shared.stop()
|
TalkSystemSpeechSynthesizer.shared.stop()
|
||||||
|
self.cancelIncrementalSpeech()
|
||||||
self.isSpeaking = false
|
self.isSpeaking = false
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -887,6 +919,268 @@ final class TalkModeManager: NSObject {
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private func shouldUseIncrementalTTS() -> Bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
private func applyDirective(_ directive: TalkDirective?) {
|
||||||
|
let requestedVoice = directive?.voiceId?.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||||
|
let resolvedVoice = self.resolveVoiceAlias(requestedVoice)
|
||||||
|
if requestedVoice?.isEmpty == false, resolvedVoice == nil {
|
||||||
|
self.logger.warning("unknown voice alias \(requestedVoice ?? "?", privacy: .public)")
|
||||||
|
}
|
||||||
|
if let voice = resolvedVoice {
|
||||||
|
if directive?.once != true {
|
||||||
|
self.currentVoiceId = voice
|
||||||
|
self.voiceOverrideActive = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let model = directive?.modelId {
|
||||||
|
if directive?.once != true {
|
||||||
|
self.currentModelId = model
|
||||||
|
self.modelOverrideActive = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func resetIncrementalSpeech() {
|
||||||
|
self.incrementalSpeechQueue.removeAll()
|
||||||
|
self.incrementalSpeechTask?.cancel()
|
||||||
|
self.incrementalSpeechTask = nil
|
||||||
|
self.incrementalSpeechActive = true
|
||||||
|
self.incrementalSpeechUsed = false
|
||||||
|
self.incrementalSpeechLanguage = nil
|
||||||
|
self.incrementalSpeechBuffer = IncrementalSpeechBuffer()
|
||||||
|
self.incrementalSpeechContext = nil
|
||||||
|
self.incrementalSpeechDirective = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
private func cancelIncrementalSpeech() {
|
||||||
|
self.incrementalSpeechQueue.removeAll()
|
||||||
|
self.incrementalSpeechTask?.cancel()
|
||||||
|
self.incrementalSpeechTask = nil
|
||||||
|
self.incrementalSpeechActive = false
|
||||||
|
self.incrementalSpeechContext = nil
|
||||||
|
self.incrementalSpeechDirective = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
private func enqueueIncrementalSpeech(_ text: String) {
|
||||||
|
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||||
|
guard !trimmed.isEmpty else { return }
|
||||||
|
self.incrementalSpeechQueue.append(trimmed)
|
||||||
|
self.incrementalSpeechUsed = true
|
||||||
|
if self.incrementalSpeechTask == nil {
|
||||||
|
self.startIncrementalSpeechTask()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func startIncrementalSpeechTask() {
|
||||||
|
if self.interruptOnSpeech {
|
||||||
|
do {
|
||||||
|
try self.startRecognition()
|
||||||
|
} catch {
|
||||||
|
self.logger.warning(
|
||||||
|
"startRecognition during incremental speak failed: \(error.localizedDescription, privacy: .public)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
self.incrementalSpeechTask = Task { @MainActor [weak self] in
|
||||||
|
guard let self else { return }
|
||||||
|
while !Task.isCancelled {
|
||||||
|
guard !self.incrementalSpeechQueue.isEmpty else { break }
|
||||||
|
let segment = self.incrementalSpeechQueue.removeFirst()
|
||||||
|
self.statusText = "Speaking…"
|
||||||
|
self.isSpeaking = true
|
||||||
|
self.lastSpokenText = segment
|
||||||
|
await self.speakIncrementalSegment(segment)
|
||||||
|
}
|
||||||
|
self.isSpeaking = false
|
||||||
|
self.stopRecognition()
|
||||||
|
self.incrementalSpeechTask = nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func finishIncrementalSpeech() async {
|
||||||
|
guard self.incrementalSpeechActive else { return }
|
||||||
|
let leftover = self.incrementalSpeechBuffer.flush()
|
||||||
|
if let leftover {
|
||||||
|
self.enqueueIncrementalSpeech(leftover)
|
||||||
|
}
|
||||||
|
if let task = self.incrementalSpeechTask {
|
||||||
|
_ = await task.result
|
||||||
|
}
|
||||||
|
self.incrementalSpeechActive = false
|
||||||
|
}
|
||||||
|
|
||||||
|
private func handleIncrementalAssistantFinal(text: String) async {
|
||||||
|
let parsed = TalkDirectiveParser.parse(text)
|
||||||
|
self.applyDirective(parsed.directive)
|
||||||
|
if let lang = parsed.directive?.language {
|
||||||
|
self.incrementalSpeechLanguage = ElevenLabsTTSClient.validatedLanguage(lang)
|
||||||
|
}
|
||||||
|
await self.updateIncrementalContextIfNeeded()
|
||||||
|
let segments = self.incrementalSpeechBuffer.ingest(text: text, isFinal: true)
|
||||||
|
for segment in segments {
|
||||||
|
self.enqueueIncrementalSpeech(segment)
|
||||||
|
}
|
||||||
|
await self.finishIncrementalSpeech()
|
||||||
|
if !self.incrementalSpeechUsed {
|
||||||
|
await self.playAssistant(text: text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func streamAssistant(runId: String, gateway: GatewayNodeSession) async {
|
||||||
|
let stream = await gateway.subscribeServerEvents(bufferingNewest: 200)
|
||||||
|
for await evt in stream {
|
||||||
|
if Task.isCancelled { return }
|
||||||
|
guard evt.event == "agent", let payload = evt.payload else { continue }
|
||||||
|
guard let agentEvent = try? GatewayPayloadDecoding.decode(payload, as: OpenClawAgentEventPayload.self) else {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
guard agentEvent.runId == runId, agentEvent.stream == "assistant" else { continue }
|
||||||
|
guard let text = agentEvent.data["text"]?.value as? String else { continue }
|
||||||
|
let segments = self.incrementalSpeechBuffer.ingest(text: text, isFinal: false)
|
||||||
|
if let lang = self.incrementalSpeechBuffer.directive?.language {
|
||||||
|
self.incrementalSpeechLanguage = ElevenLabsTTSClient.validatedLanguage(lang)
|
||||||
|
}
|
||||||
|
await self.updateIncrementalContextIfNeeded()
|
||||||
|
for segment in segments {
|
||||||
|
self.enqueueIncrementalSpeech(segment)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func updateIncrementalContextIfNeeded() async {
|
||||||
|
let directive = self.incrementalSpeechBuffer.directive
|
||||||
|
if let existing = self.incrementalSpeechContext, directive == self.incrementalSpeechDirective {
|
||||||
|
if existing.language != self.incrementalSpeechLanguage {
|
||||||
|
self.incrementalSpeechContext = IncrementalSpeechContext(
|
||||||
|
apiKey: existing.apiKey,
|
||||||
|
voiceId: existing.voiceId,
|
||||||
|
modelId: existing.modelId,
|
||||||
|
outputFormat: existing.outputFormat,
|
||||||
|
language: self.incrementalSpeechLanguage,
|
||||||
|
directive: existing.directive,
|
||||||
|
canUseElevenLabs: existing.canUseElevenLabs)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
let context = await self.buildIncrementalSpeechContext(directive: directive)
|
||||||
|
self.incrementalSpeechContext = context
|
||||||
|
self.incrementalSpeechDirective = directive
|
||||||
|
}
|
||||||
|
|
||||||
|
private func buildIncrementalSpeechContext(directive: TalkDirective?) async -> IncrementalSpeechContext {
|
||||||
|
let requestedVoice = directive?.voiceId?.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||||
|
let resolvedVoice = self.resolveVoiceAlias(requestedVoice)
|
||||||
|
if requestedVoice?.isEmpty == false, resolvedVoice == nil {
|
||||||
|
self.logger.warning("unknown voice alias \(requestedVoice ?? "?", privacy: .public)")
|
||||||
|
}
|
||||||
|
let preferredVoice = resolvedVoice ?? self.currentVoiceId ?? self.defaultVoiceId
|
||||||
|
let modelId = directive?.modelId ?? self.currentModelId ?? self.defaultModelId
|
||||||
|
let desiredOutputFormat = (directive?.outputFormat ?? self.defaultOutputFormat)?
|
||||||
|
.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||||
|
let requestedOutputFormat = (desiredOutputFormat?.isEmpty == false) ? desiredOutputFormat : nil
|
||||||
|
let outputFormat = ElevenLabsTTSClient.validatedOutputFormat(requestedOutputFormat ?? "pcm_44100")
|
||||||
|
if outputFormat == nil, let requestedOutputFormat {
|
||||||
|
self.logger.warning(
|
||||||
|
"talk output_format unsupported for local playback: \(requestedOutputFormat, privacy: .public)")
|
||||||
|
}
|
||||||
|
|
||||||
|
let resolvedKey =
|
||||||
|
(self.apiKey?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false ? self.apiKey : nil) ??
|
||||||
|
ProcessInfo.processInfo.environment["ELEVENLABS_API_KEY"]
|
||||||
|
let apiKey = resolvedKey?.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||||
|
let voiceId: String? = if let apiKey, !apiKey.isEmpty {
|
||||||
|
await self.resolveVoiceId(preferred: preferredVoice, apiKey: apiKey)
|
||||||
|
} else {
|
||||||
|
nil
|
||||||
|
}
|
||||||
|
let canUseElevenLabs = (voiceId?.isEmpty == false) && (apiKey?.isEmpty == false)
|
||||||
|
return IncrementalSpeechContext(
|
||||||
|
apiKey: apiKey,
|
||||||
|
voiceId: voiceId,
|
||||||
|
modelId: modelId,
|
||||||
|
outputFormat: outputFormat,
|
||||||
|
language: self.incrementalSpeechLanguage,
|
||||||
|
directive: directive,
|
||||||
|
canUseElevenLabs: canUseElevenLabs)
|
||||||
|
}
|
||||||
|
|
||||||
|
private func speakIncrementalSegment(_ text: String) async {
|
||||||
|
await self.updateIncrementalContextIfNeeded()
|
||||||
|
guard let context = self.incrementalSpeechContext else {
|
||||||
|
try? await TalkSystemSpeechSynthesizer.shared.speak(
|
||||||
|
text: text,
|
||||||
|
language: self.incrementalSpeechLanguage)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if context.canUseElevenLabs, let apiKey = context.apiKey, let voiceId = context.voiceId {
|
||||||
|
let request = ElevenLabsTTSRequest(
|
||||||
|
text: text,
|
||||||
|
modelId: context.modelId,
|
||||||
|
outputFormat: context.outputFormat,
|
||||||
|
speed: TalkTTSValidation.resolveSpeed(
|
||||||
|
speed: context.directive?.speed,
|
||||||
|
rateWPM: context.directive?.rateWPM),
|
||||||
|
stability: TalkTTSValidation.validatedStability(
|
||||||
|
context.directive?.stability,
|
||||||
|
modelId: context.modelId),
|
||||||
|
similarity: TalkTTSValidation.validatedUnit(context.directive?.similarity),
|
||||||
|
style: TalkTTSValidation.validatedUnit(context.directive?.style),
|
||||||
|
speakerBoost: context.directive?.speakerBoost,
|
||||||
|
seed: TalkTTSValidation.validatedSeed(context.directive?.seed),
|
||||||
|
normalize: ElevenLabsTTSClient.validatedNormalize(context.directive?.normalize),
|
||||||
|
language: context.language,
|
||||||
|
latencyTier: TalkTTSValidation.validatedLatencyTier(context.directive?.latencyTier))
|
||||||
|
let client = ElevenLabsTTSClient(apiKey: apiKey)
|
||||||
|
let stream = client.streamSynthesize(voiceId: voiceId, request: request)
|
||||||
|
let sampleRate = TalkTTSValidation.pcmSampleRate(from: context.outputFormat)
|
||||||
|
let result: StreamingPlaybackResult
|
||||||
|
if let sampleRate {
|
||||||
|
self.lastPlaybackWasPCM = true
|
||||||
|
var playback = await self.pcmPlayer.play(stream: stream, sampleRate: sampleRate)
|
||||||
|
if !playback.finished, playback.interruptedAt == nil {
|
||||||
|
self.logger.warning("pcm playback failed; retrying mp3")
|
||||||
|
self.lastPlaybackWasPCM = false
|
||||||
|
let mp3Format = ElevenLabsTTSClient.validatedOutputFormat("mp3_44100")
|
||||||
|
let mp3Stream = client.streamSynthesize(
|
||||||
|
voiceId: voiceId,
|
||||||
|
request: ElevenLabsTTSRequest(
|
||||||
|
text: text,
|
||||||
|
modelId: context.modelId,
|
||||||
|
outputFormat: mp3Format,
|
||||||
|
speed: TalkTTSValidation.resolveSpeed(
|
||||||
|
speed: context.directive?.speed,
|
||||||
|
rateWPM: context.directive?.rateWPM),
|
||||||
|
stability: TalkTTSValidation.validatedStability(
|
||||||
|
context.directive?.stability,
|
||||||
|
modelId: context.modelId),
|
||||||
|
similarity: TalkTTSValidation.validatedUnit(context.directive?.similarity),
|
||||||
|
style: TalkTTSValidation.validatedUnit(context.directive?.style),
|
||||||
|
speakerBoost: context.directive?.speakerBoost,
|
||||||
|
seed: TalkTTSValidation.validatedSeed(context.directive?.seed),
|
||||||
|
normalize: ElevenLabsTTSClient.validatedNormalize(context.directive?.normalize),
|
||||||
|
language: context.language,
|
||||||
|
latencyTier: TalkTTSValidation.validatedLatencyTier(context.directive?.latencyTier)))
|
||||||
|
playback = await self.mp3Player.play(stream: mp3Stream)
|
||||||
|
}
|
||||||
|
result = playback
|
||||||
|
} else {
|
||||||
|
self.lastPlaybackWasPCM = false
|
||||||
|
result = await self.mp3Player.play(stream: stream)
|
||||||
|
}
|
||||||
|
if !result.finished, let interruptedAt = result.interruptedAt {
|
||||||
|
self.lastInterruptedAtSeconds = interruptedAt
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
try? await TalkSystemSpeechSynthesizer.shared.speak(
|
||||||
|
text: text,
|
||||||
|
language: self.incrementalSpeechLanguage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private func resolveVoiceAlias(_ value: String?) -> String? {
|
private func resolveVoiceAlias(_ value: String?) -> String? {
|
||||||
let trimmed = (value ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
|
let trimmed = (value ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
|
||||||
guard !trimmed.isEmpty else { return nil }
|
guard !trimmed.isEmpty else { return nil }
|
||||||
@@ -1010,6 +1304,121 @@ final class TalkModeManager: NSObject {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private struct IncrementalSpeechBuffer {
|
||||||
|
private(set) var latestText: String = ""
|
||||||
|
private(set) var directive: TalkDirective?
|
||||||
|
private var spokenOffset: Int = 0
|
||||||
|
private var inCodeBlock = false
|
||||||
|
private var directiveParsed = false
|
||||||
|
|
||||||
|
mutating func ingest(text: String, isFinal: Bool) -> [String] {
|
||||||
|
let normalized = text.replacingOccurrences(of: "\r\n", with: "\n")
|
||||||
|
guard let usable = self.stripDirectiveIfReady(from: normalized) else { return [] }
|
||||||
|
self.updateText(usable)
|
||||||
|
return self.extractSegments(isFinal: isFinal)
|
||||||
|
}
|
||||||
|
|
||||||
|
mutating func flush() -> String? {
|
||||||
|
guard !self.latestText.isEmpty else { return nil }
|
||||||
|
let segments = self.extractSegments(isFinal: true)
|
||||||
|
return segments.first
|
||||||
|
}
|
||||||
|
|
||||||
|
private mutating func stripDirectiveIfReady(from text: String) -> String? {
|
||||||
|
guard !self.directiveParsed else { return text }
|
||||||
|
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||||
|
guard !trimmed.isEmpty else { return nil }
|
||||||
|
if trimmed.hasPrefix("{") {
|
||||||
|
guard let newlineRange = text.range(of: "\n") else { return nil }
|
||||||
|
let firstLine = text[..<newlineRange.lowerBound]
|
||||||
|
let head = firstLine.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||||
|
guard head.hasSuffix("}") else { return nil }
|
||||||
|
let parsed = TalkDirectiveParser.parse(text)
|
||||||
|
if let directive = parsed.directive {
|
||||||
|
self.directive = directive
|
||||||
|
}
|
||||||
|
self.directiveParsed = true
|
||||||
|
return parsed.stripped
|
||||||
|
}
|
||||||
|
self.directiveParsed = true
|
||||||
|
return text
|
||||||
|
}
|
||||||
|
|
||||||
|
private mutating func updateText(_ newText: String) {
|
||||||
|
if newText.hasPrefix(self.latestText) {
|
||||||
|
self.latestText = newText
|
||||||
|
} else if self.latestText.hasPrefix(newText) {
|
||||||
|
// Keep the longer cached text.
|
||||||
|
} else {
|
||||||
|
self.latestText += newText
|
||||||
|
}
|
||||||
|
if self.spokenOffset > self.latestText.count {
|
||||||
|
self.spokenOffset = self.latestText.count
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private mutating func extractSegments(isFinal: Bool) -> [String] {
|
||||||
|
let chars = Array(self.latestText)
|
||||||
|
guard self.spokenOffset < chars.count else { return [] }
|
||||||
|
var idx = self.spokenOffset
|
||||||
|
var lastBoundary: Int?
|
||||||
|
var inCodeBlock = self.inCodeBlock
|
||||||
|
var buffer = ""
|
||||||
|
var bufferAtBoundary = ""
|
||||||
|
var inCodeBlockAtBoundary = inCodeBlock
|
||||||
|
|
||||||
|
while idx < chars.count {
|
||||||
|
if idx + 2 < chars.count,
|
||||||
|
chars[idx] == "`",
|
||||||
|
chars[idx + 1] == "`",
|
||||||
|
chars[idx + 2] == "`"
|
||||||
|
{
|
||||||
|
inCodeBlock.toggle()
|
||||||
|
idx += 3
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if !inCodeBlock {
|
||||||
|
buffer.append(chars[idx])
|
||||||
|
if Self.isBoundary(chars[idx]) {
|
||||||
|
lastBoundary = idx + 1
|
||||||
|
bufferAtBoundary = buffer
|
||||||
|
inCodeBlockAtBoundary = inCodeBlock
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
idx += 1
|
||||||
|
}
|
||||||
|
|
||||||
|
if let boundary = lastBoundary {
|
||||||
|
self.spokenOffset = boundary
|
||||||
|
self.inCodeBlock = inCodeBlockAtBoundary
|
||||||
|
let trimmed = bufferAtBoundary.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||||
|
return trimmed.isEmpty ? [] : [trimmed]
|
||||||
|
}
|
||||||
|
|
||||||
|
guard isFinal else { return [] }
|
||||||
|
self.spokenOffset = chars.count
|
||||||
|
self.inCodeBlock = inCodeBlock
|
||||||
|
let trimmed = buffer.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||||
|
return trimmed.isEmpty ? [] : [trimmed]
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func isBoundary(_ ch: Character) -> Bool {
|
||||||
|
ch == "." || ch == "!" || ch == "?" || ch == "\n"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private struct IncrementalSpeechContext {
|
||||||
|
let apiKey: String?
|
||||||
|
let voiceId: String?
|
||||||
|
let modelId: String?
|
||||||
|
let outputFormat: String?
|
||||||
|
let language: String?
|
||||||
|
let directive: TalkDirective?
|
||||||
|
let canUseElevenLabs: Bool
|
||||||
|
}
|
||||||
|
|
||||||
#if DEBUG
|
#if DEBUG
|
||||||
extension TalkModeManager {
|
extension TalkModeManager {
|
||||||
func _test_seedTranscript(_ transcript: String) {
|
func _test_seedTranscript(_ transcript: String) {
|
||||||
@@ -1017,6 +1426,10 @@ extension TalkModeManager {
|
|||||||
self.lastHeard = Date()
|
self.lastHeard = Date()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func _test_handleTranscript(_ transcript: String, isFinal: Bool) async {
|
||||||
|
await self.handleTranscript(transcript: transcript, isFinal: isFinal)
|
||||||
|
}
|
||||||
|
|
||||||
func _test_backdateLastHeard(seconds: TimeInterval) {
|
func _test_backdateLastHeard(seconds: TimeInterval) {
|
||||||
self.lastHeard = Date().addingTimeInterval(-seconds)
|
self.lastHeard = Date().addingTimeInterval(-seconds)
|
||||||
}
|
}
|
||||||
@@ -1024,5 +1437,13 @@ extension TalkModeManager {
|
|||||||
func _test_runSilenceCheck() async {
|
func _test_runSilenceCheck() async {
|
||||||
await self.checkSilence()
|
await self.checkSilence()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func _test_incrementalReset() {
|
||||||
|
self.incrementalSpeechBuffer = IncrementalSpeechBuffer()
|
||||||
|
}
|
||||||
|
|
||||||
|
func _test_incrementalIngest(_ text: String, isFinal: Bool) -> [String] {
|
||||||
|
self.incrementalSpeechBuffer.ingest(text: text, isFinal: isFinal)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@@ -2,7 +2,9 @@ Sources/Gateway/GatewayConnectionController.swift
|
|||||||
Sources/Gateway/GatewayDiscoveryDebugLogView.swift
|
Sources/Gateway/GatewayDiscoveryDebugLogView.swift
|
||||||
Sources/Gateway/GatewayDiscoveryModel.swift
|
Sources/Gateway/GatewayDiscoveryModel.swift
|
||||||
Sources/Gateway/GatewaySettingsStore.swift
|
Sources/Gateway/GatewaySettingsStore.swift
|
||||||
|
Sources/Gateway/GatewayHealthMonitor.swift
|
||||||
Sources/Gateway/KeychainStore.swift
|
Sources/Gateway/KeychainStore.swift
|
||||||
|
Sources/Capabilities/NodeCapabilityRouter.swift
|
||||||
Sources/Camera/CameraController.swift
|
Sources/Camera/CameraController.swift
|
||||||
Sources/Chat/ChatSheet.swift
|
Sources/Chat/ChatSheet.swift
|
||||||
Sources/Chat/IOSGatewayChatTransport.swift
|
Sources/Chat/IOSGatewayChatTransport.swift
|
||||||
@@ -58,6 +60,7 @@ Sources/Voice/VoiceWakePreferences.swift
|
|||||||
../shared/OpenClawKit/Sources/OpenClawKit/CanvasCommandParams.swift
|
../shared/OpenClawKit/Sources/OpenClawKit/CanvasCommandParams.swift
|
||||||
../shared/OpenClawKit/Sources/OpenClawKit/CanvasCommands.swift
|
../shared/OpenClawKit/Sources/OpenClawKit/CanvasCommands.swift
|
||||||
../shared/OpenClawKit/Sources/OpenClawKit/Capabilities.swift
|
../shared/OpenClawKit/Sources/OpenClawKit/Capabilities.swift
|
||||||
|
../shared/OpenClawKit/Sources/OpenClawKit/ChatCommands.swift
|
||||||
../shared/OpenClawKit/Sources/OpenClawKit/ContactsCommands.swift
|
../shared/OpenClawKit/Sources/OpenClawKit/ContactsCommands.swift
|
||||||
../shared/OpenClawKit/Sources/OpenClawKit/DeviceCommands.swift
|
../shared/OpenClawKit/Sources/OpenClawKit/DeviceCommands.swift
|
||||||
../shared/OpenClawKit/Sources/OpenClawKit/OpenClawKitResources.swift
|
../shared/OpenClawKit/Sources/OpenClawKit/OpenClawKitResources.swift
|
||||||
|
|||||||
@@ -92,6 +92,7 @@ private func withUserDefaults<T>(_ updates: [String: Any?], _ body: () throws ->
|
|||||||
let commands = Set(controller._test_currentCommands())
|
let commands = Set(controller._test_currentCommands())
|
||||||
|
|
||||||
#expect(commands.contains(OpenClawSystemCommand.notify.rawValue))
|
#expect(commands.contains(OpenClawSystemCommand.notify.rawValue))
|
||||||
|
#expect(commands.contains(OpenClawChatCommand.push.rawValue))
|
||||||
#expect(!commands.contains(OpenClawSystemCommand.run.rawValue))
|
#expect(!commands.contains(OpenClawSystemCommand.run.rawValue))
|
||||||
#expect(!commands.contains(OpenClawSystemCommand.which.rawValue))
|
#expect(!commands.contains(OpenClawSystemCommand.which.rawValue))
|
||||||
#expect(!commands.contains(OpenClawSystemCommand.execApprovalsGet.rawValue))
|
#expect(!commands.contains(OpenClawSystemCommand.execApprovalsGet.rawValue))
|
||||||
|
|||||||
60
apps/ios/Tests/GatewayHealthMonitorTests.swift
Normal file
60
apps/ios/Tests/GatewayHealthMonitorTests.swift
Normal file
@@ -0,0 +1,60 @@
|
|||||||
|
import Foundation
|
||||||
|
import Testing
|
||||||
|
@testable import OpenClaw
|
||||||
|
|
||||||
|
private actor Counter {
|
||||||
|
private var value = 0
|
||||||
|
|
||||||
|
func increment() {
|
||||||
|
value += 1
|
||||||
|
}
|
||||||
|
|
||||||
|
func get() -> Int {
|
||||||
|
value
|
||||||
|
}
|
||||||
|
|
||||||
|
func set(_ newValue: Int) {
|
||||||
|
value = newValue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Suite struct GatewayHealthMonitorTests {
|
||||||
|
@Test @MainActor func triggersFailureAfterThreshold() async {
|
||||||
|
let failureCount = Counter()
|
||||||
|
let monitor = GatewayHealthMonitor(
|
||||||
|
config: .init(intervalSeconds: 0.001, timeoutSeconds: 0.0, maxFailures: 2))
|
||||||
|
|
||||||
|
monitor.start(
|
||||||
|
check: { false },
|
||||||
|
onFailure: { _ in
|
||||||
|
await failureCount.increment()
|
||||||
|
await monitor.stop()
|
||||||
|
})
|
||||||
|
|
||||||
|
try? await Task.sleep(nanoseconds: 60_000_000)
|
||||||
|
#expect(await failureCount.get() == 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test @MainActor func resetsFailuresAfterSuccess() async {
|
||||||
|
let failureCount = Counter()
|
||||||
|
let calls = Counter()
|
||||||
|
let monitor = GatewayHealthMonitor(
|
||||||
|
config: .init(intervalSeconds: 0.001, timeoutSeconds: 0.0, maxFailures: 2))
|
||||||
|
|
||||||
|
monitor.start(
|
||||||
|
check: {
|
||||||
|
await calls.increment()
|
||||||
|
let callCount = await calls.get()
|
||||||
|
if callCount >= 6 {
|
||||||
|
await monitor.stop()
|
||||||
|
}
|
||||||
|
return callCount % 2 == 0
|
||||||
|
},
|
||||||
|
onFailure: { _ in
|
||||||
|
await failureCount.increment()
|
||||||
|
})
|
||||||
|
|
||||||
|
try? await Task.sleep(nanoseconds: 60_000_000)
|
||||||
|
#expect(await failureCount.get() == 0)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -448,6 +448,91 @@ private func decodePayload<T: Decodable>(_ json: String?, as type: T.Type) throw
|
|||||||
#expect(request.content.body == "World")
|
#expect(request.content.body == "World")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test @MainActor func handleInvokeChatPushCreatesNotification() async throws {
|
||||||
|
let notifier = TestNotificationCenter(status: .authorized)
|
||||||
|
let deviceStatus = TestDeviceStatusService(
|
||||||
|
statusPayload: OpenClawDeviceStatusPayload(
|
||||||
|
battery: OpenClawBatteryStatusPayload(level: 0.5, state: .charging, lowPowerModeEnabled: false),
|
||||||
|
thermal: OpenClawThermalStatusPayload(state: .nominal),
|
||||||
|
storage: OpenClawStorageStatusPayload(totalBytes: 100, freeBytes: 50, usedBytes: 50),
|
||||||
|
network: OpenClawNetworkStatusPayload(
|
||||||
|
status: .satisfied,
|
||||||
|
isExpensive: false,
|
||||||
|
isConstrained: false,
|
||||||
|
interfaces: [.wifi]),
|
||||||
|
uptimeSeconds: 10),
|
||||||
|
infoPayload: OpenClawDeviceInfoPayload(
|
||||||
|
deviceName: "Test",
|
||||||
|
modelIdentifier: "Test1,1",
|
||||||
|
systemName: "iOS",
|
||||||
|
systemVersion: "1.0",
|
||||||
|
appVersion: "dev",
|
||||||
|
appBuild: "0",
|
||||||
|
locale: "en-US"))
|
||||||
|
let emptyContact = OpenClawContactPayload(
|
||||||
|
identifier: "c0",
|
||||||
|
displayName: "",
|
||||||
|
givenName: "",
|
||||||
|
familyName: "",
|
||||||
|
organizationName: "",
|
||||||
|
phoneNumbers: [],
|
||||||
|
emails: [])
|
||||||
|
let emptyEvent = OpenClawCalendarEventPayload(
|
||||||
|
identifier: "e0",
|
||||||
|
title: "Test",
|
||||||
|
startISO: "2024-01-01T00:00:00Z",
|
||||||
|
endISO: "2024-01-01T00:30:00Z",
|
||||||
|
isAllDay: false,
|
||||||
|
location: nil,
|
||||||
|
calendarTitle: nil)
|
||||||
|
let emptyReminder = OpenClawReminderPayload(
|
||||||
|
identifier: "r0",
|
||||||
|
title: "Test",
|
||||||
|
dueISO: nil,
|
||||||
|
completed: false,
|
||||||
|
listName: nil)
|
||||||
|
let appModel = makeTestAppModel(
|
||||||
|
notificationCenter: notifier,
|
||||||
|
deviceStatusService: deviceStatus,
|
||||||
|
photosService: TestPhotosService(payload: OpenClawPhotosLatestPayload(photos: [])),
|
||||||
|
contactsService: TestContactsService(
|
||||||
|
searchPayload: OpenClawContactsSearchPayload(contacts: []),
|
||||||
|
addPayload: OpenClawContactsAddPayload(contact: emptyContact)),
|
||||||
|
calendarService: TestCalendarService(
|
||||||
|
eventsPayload: OpenClawCalendarEventsPayload(events: []),
|
||||||
|
addPayload: OpenClawCalendarAddPayload(event: emptyEvent)),
|
||||||
|
remindersService: TestRemindersService(
|
||||||
|
listPayload: OpenClawRemindersListPayload(reminders: []),
|
||||||
|
addPayload: OpenClawRemindersAddPayload(reminder: emptyReminder)),
|
||||||
|
motionService: TestMotionService(
|
||||||
|
activityPayload: OpenClawMotionActivityPayload(activities: []),
|
||||||
|
pedometerPayload: OpenClawPedometerPayload(
|
||||||
|
startISO: "2024-01-01T00:00:00Z",
|
||||||
|
endISO: "2024-01-01T01:00:00Z",
|
||||||
|
steps: nil,
|
||||||
|
distanceMeters: nil,
|
||||||
|
floorsAscended: nil,
|
||||||
|
floorsDescended: nil)))
|
||||||
|
|
||||||
|
let params = OpenClawChatPushParams(text: "Ping", speak: false)
|
||||||
|
let data = try JSONEncoder().encode(params)
|
||||||
|
let json = String(decoding: data, as: UTF8.self)
|
||||||
|
let req = BridgeInvokeRequest(
|
||||||
|
id: "chat-push",
|
||||||
|
command: OpenClawChatCommand.push.rawValue,
|
||||||
|
paramsJSON: json)
|
||||||
|
let res = await appModel._test_handleInvoke(req)
|
||||||
|
#expect(res.ok == true)
|
||||||
|
#expect(notifier.addedRequests.count == 1)
|
||||||
|
let request = try #require(notifier.addedRequests.first)
|
||||||
|
#expect(request.content.title == "OpenClaw")
|
||||||
|
#expect(request.content.body == "Ping")
|
||||||
|
let payloadJSON = try #require(res.payloadJSON)
|
||||||
|
let decoded = try JSONDecoder().decode(OpenClawChatPushPayload.self, from: Data(payloadJSON.utf8))
|
||||||
|
#expect((decoded.messageId ?? "").isEmpty == false)
|
||||||
|
#expect(request.identifier == decoded.messageId)
|
||||||
|
}
|
||||||
|
|
||||||
@Test @MainActor func handleInvokeDeviceAndDataCommandsReturnPayloads() async throws {
|
@Test @MainActor func handleInvokeDeviceAndDataCommandsReturnPayloads() async throws {
|
||||||
let deviceStatusPayload = OpenClawDeviceStatusPayload(
|
let deviceStatusPayload = OpenClawDeviceStatusPayload(
|
||||||
battery: OpenClawBatteryStatusPayload(level: 0.25, state: .unplugged, lowPowerModeEnabled: false),
|
battery: OpenClawBatteryStatusPayload(level: 0.25, state: .unplugged, lowPowerModeEnabled: false),
|
||||||
@@ -723,6 +808,28 @@ private func decodePayload<T: Decodable>(_ json: String?, as type: T.Type) throw
|
|||||||
#expect(oncePayload.status == "offline")
|
#expect(oncePayload.status == "offline")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test @MainActor func handleInvokePushToTalkOnceStopsOnFinalTranscript() async throws {
|
||||||
|
let talkMode = TalkModeManager(allowSimulatorCapture: true)
|
||||||
|
talkMode.updateGatewayConnected(false)
|
||||||
|
let appModel = makeTalkTestAppModel(talkMode: talkMode)
|
||||||
|
|
||||||
|
let onceReq = BridgeInvokeRequest(id: "ptt-once-final", command: OpenClawTalkCommand.pttOnce.rawValue)
|
||||||
|
let onceTask = Task { await appModel._test_handleInvoke(onceReq) }
|
||||||
|
|
||||||
|
for _ in 0..<5 where !talkMode.isPushToTalkActive {
|
||||||
|
await Task.yield()
|
||||||
|
}
|
||||||
|
#expect(talkMode.isPushToTalkActive == true)
|
||||||
|
|
||||||
|
await talkMode._test_handleTranscript("Hello final", isFinal: true)
|
||||||
|
|
||||||
|
let onceRes = await onceTask.value
|
||||||
|
#expect(onceRes.ok == true)
|
||||||
|
let oncePayload = try decodePayload(onceRes.payloadJSON, as: OpenClawTalkPTTStopPayload.self)
|
||||||
|
#expect(oncePayload.transcript == "Hello final")
|
||||||
|
#expect(oncePayload.status == "offline")
|
||||||
|
}
|
||||||
|
|
||||||
@Test @MainActor func handleDeepLinkSetsErrorWhenNotConnected() async {
|
@Test @MainActor func handleDeepLinkSetsErrorWhenNotConnected() async {
|
||||||
let appModel = NodeAppModel()
|
let appModel = NodeAppModel()
|
||||||
let url = URL(string: "openclaw://agent?message=hello")!
|
let url = URL(string: "openclaw://agent?message=hello")!
|
||||||
|
|||||||
33
apps/ios/Tests/TalkModeIncrementalTests.swift
Normal file
33
apps/ios/Tests/TalkModeIncrementalTests.swift
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
import Foundation
|
||||||
|
import Testing
|
||||||
|
@testable import OpenClaw
|
||||||
|
|
||||||
|
@Suite struct TalkModeIncrementalTests {
|
||||||
|
@Test @MainActor func incrementalSpeechSplitsOnBoundary() {
|
||||||
|
let talkMode = TalkModeManager(allowSimulatorCapture: true)
|
||||||
|
talkMode._test_incrementalReset()
|
||||||
|
let segments = talkMode._test_incrementalIngest("Hello world. Next", isFinal: false)
|
||||||
|
#expect(segments.count == 1)
|
||||||
|
#expect(segments.first == "Hello world.")
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test @MainActor func incrementalSpeechSkipsDirectiveLine() {
|
||||||
|
let talkMode = TalkModeManager(allowSimulatorCapture: true)
|
||||||
|
talkMode._test_incrementalReset()
|
||||||
|
let segments = talkMode._test_incrementalIngest("{\"voice\":\"abc\"}\nHello.", isFinal: false)
|
||||||
|
#expect(segments.count == 1)
|
||||||
|
#expect(segments.first == "Hello.")
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test @MainActor func incrementalSpeechIgnoresCodeBlocks() {
|
||||||
|
let talkMode = TalkModeManager(allowSimulatorCapture: true)
|
||||||
|
talkMode._test_incrementalReset()
|
||||||
|
let text = "Here is code:\n```js\nx=1\n```\nDone."
|
||||||
|
let segments = talkMode._test_incrementalIngest(text, isFinal: true)
|
||||||
|
#expect(segments.count == 1)
|
||||||
|
let value = segments.first ?? ""
|
||||||
|
#expect(value.contains("x=1") == false)
|
||||||
|
#expect(value.contains("Here is code") == true)
|
||||||
|
#expect(value.contains("Done.") == true)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
import Foundation
|
||||||
|
|
||||||
|
public enum OpenClawChatCommand: String, Codable, Sendable {
|
||||||
|
case push = "chat.push"
|
||||||
|
}
|
||||||
|
|
||||||
|
public struct OpenClawChatPushParams: Codable, Sendable, Equatable {
|
||||||
|
public var text: String
|
||||||
|
public var speak: Bool?
|
||||||
|
|
||||||
|
public init(text: String, speak: Bool? = nil) {
|
||||||
|
self.text = text
|
||||||
|
self.speak = speak
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public struct OpenClawChatPushPayload: Codable, Sendable, Equatable {
|
||||||
|
public var messageId: String?
|
||||||
|
|
||||||
|
public init(messageId: String? = nil) {
|
||||||
|
self.messageId = messageId
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -34,6 +34,8 @@ const MOTION_COMMANDS = ["motion.activity", "motion.pedometer"];
|
|||||||
|
|
||||||
const SYSTEM_NOTIFY_COMMANDS = ["system.notify"];
|
const SYSTEM_NOTIFY_COMMANDS = ["system.notify"];
|
||||||
|
|
||||||
|
const CHAT_COMMANDS = ["chat.push"];
|
||||||
|
|
||||||
const TALK_COMMANDS = ["talk.ptt.start", "talk.ptt.stop", "talk.ptt.cancel", "talk.ptt.once"];
|
const TALK_COMMANDS = ["talk.ptt.start", "talk.ptt.stop", "talk.ptt.cancel", "talk.ptt.once"];
|
||||||
|
|
||||||
const SYSTEM_COMMANDS = [
|
const SYSTEM_COMMANDS = [
|
||||||
@@ -52,6 +54,7 @@ const PLATFORM_DEFAULTS: Record<string, string[]> = {
|
|||||||
...SCREEN_COMMANDS,
|
...SCREEN_COMMANDS,
|
||||||
...LOCATION_COMMANDS,
|
...LOCATION_COMMANDS,
|
||||||
...SYSTEM_NOTIFY_COMMANDS,
|
...SYSTEM_NOTIFY_COMMANDS,
|
||||||
|
...CHAT_COMMANDS,
|
||||||
...DEVICE_COMMANDS,
|
...DEVICE_COMMANDS,
|
||||||
...PHOTOS_COMMANDS,
|
...PHOTOS_COMMANDS,
|
||||||
...CONTACTS_COMMANDS,
|
...CONTACTS_COMMANDS,
|
||||||
|
|||||||
Reference in New Issue
Block a user