Improve Watch Assist, silence detection and volume control (#2844)

<!-- Thank you for submitting a Pull Request and helping to improve Home
Assistant. Please complete the following sections to help the processing
and review of your changes. Please do not delete anything from this
template. -->

## Summary
<!-- Provide a brief summary of the changes you have made and most
importantly what they aim to achieve -->

## Screenshots
<!-- If this is a user-facing change not in the frontend, please include
screenshots in light and dark mode. -->

## Link to pull request in Documentation repository
<!-- Pull requests that add, change or remove functionality must have a
corresponding pull request in the Companion App Documentation repository
(https://github.com/home-assistant/companion.home-assistant). Please add
the number of this pull request after the "#" -->
Documentation: home-assistant/companion.home-assistant#

## Any other notes
<!-- If there is any other information of note, like if this Pull
Request is part of a bigger change, please include it here. -->

HomeAssistant.xcodeproj/project.pbxproj

@@ -582,6 +582,7 @@
  426740A92B17391000C1DD73 /* Data+Hexadecimal.swift in Sources */ = {isa = PBXBuildFile; fileRef = 426740A72B17390A00C1DD73 /* Data+Hexadecimal.swift */; };
  427692E32B98B82500F24321 /* SharedPush in Frameworks */ = {isa = PBXBuildFile; productRef = 427692E22B98B82500F24321 /* SharedPush */; };
  427692E52B98B83200F24321 /* SharedPush in Frameworks */ = {isa = PBXBuildFile; productRef = 427692E42B98B83200F24321 /* SharedPush */; };
+ 427756CB2C3ED5F700E11D0B /* VolumeView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 427756CA2C3ED5F700E11D0B /* VolumeView.swift */; };
  4278DFD22B45C7AE0087C9D7 /* Core.strings in Resources */ = {isa = PBXBuildFile; fileRef = 4278DFAF2B45C6680087C9D7 /* Core.strings */; };
  4278DFD32B45C7AE0087C9D7 /* Core.strings in Resources */ = {isa = PBXBuildFile; fileRef = 4278DFAF2B45C6680087C9D7 /* Core.strings */; };
  4279407F2B8369EC001D7E14 /* AppIntentVocabulary.plist in Resources */ = {isa = PBXBuildFile; fileRef = 42805A142B0226050095414C /* AppIntentVocabulary.plist */; };
@@ -1760,6 +1761,7 @@
  426490742C0F20FF002155CC /* WatchAssistView+Build.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "WatchAssistView+Build.swift"; sourceTree = "<group>"; };
  426490762C0F2403002155CC /* WatchAudioRecorder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WatchAudioRecorder.swift; sourceTree = "<group>"; };
  426740A72B17390A00C1DD73 /* Data+Hexadecimal.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Data+Hexadecimal.swift"; sourceTree = "<group>"; };
+ 427756CA2C3ED5F700E11D0B /* VolumeView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VolumeView.swift; sourceTree = "<group>"; };
  4278DFB02B45C6680087C9D7 /* ja */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = ja; path = ja.lproj/Core.strings; sourceTree = "<group>"; };
  4278DFB12B45C6680087C9D7 /* pt-BR */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = "pt-BR"; path = "pt-BR.lproj/Core.strings"; sourceTree = "<group>"; };
  4278DFB22B45C6680087C9D7 /* cs */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = cs; path = cs.lproj/Core.strings; sourceTree = "<group>"; };
@@ -3387,6 +3389,7 @@
  isa = PBXGroup;
  children = (
  423F44EF2C17238200766A99 /* ChatBubbleView.swift */,
+ 427756CA2C3ED5F700E11D0B /* VolumeView.swift */,
  );
  path = Views;
  sourceTree = "<group>";
@@ -6668,6 +6671,7 @@
  42C373B52BC53B1400898990 /* WatchHomeViewModel.swift in Sources */,
  42C373B32BC538A300898990 /* WatchHomeView.swift in Sources */,
  423F44FF2C186E4500766A99 /* WatchCommunicatorService.swift in Sources */,
+ 427756CB2C3ED5F700E11D0B /* VolumeView.swift in Sources */,
  11FA936A263FAA920015F1FC /* NotificationSubController.swift in Sources */,
  4264906E2C0F1B8B002155CC /* WatchAssistViewModel.swift in Sources */,
  );

Sources/App/Resources/en.lproj/Localizable.strings

@@ -63,6 +63,7 @@
 "assist.pipelines_picker.title" = "Assist Pipelines";
 "assist.watch.mic_button.title" = "Tap to ";
 "assist.watch.not_reachable.title" = "Assist requires iPhone connectivity. Your iPhone is currently unreachable.";
+"assist.watch.volume.title" = "Volume control";
 "cancel_label" = "Cancel";
 "carPlay.action.intro.item.body" = "Tap to continue on your iPhone";
 "carPlay.action.intro.item.title" = "Create your first action";
@@ -877,4 +878,4 @@ Home Assistant is free and open source home automation software with a focus on
 "widgets.open_page.description" = "Open a frontend page in Home Assistant.";
 "widgets.open_page.not_configured" = "No Pages Available";
 "widgets.open_page.title" = "Open Page";
-"yes_label" = "Yes";
+"yes_label" = "Yes";

Sources/Extensions/Watch/Assist/Settings/WatchAssistSettings.swift

@@ -1,3 +1,5 @@
+import AVFAudio
+import MediaPlayer
 import Shared
 import SwiftUI
 
@@ -11,7 +13,23 @@ struct WatchAssistSettings: View {
  var body: some View {
  ScrollView {
  VStack(spacing: Spaces.two) {
- if !assistService.servers.isEmpty, assistService.selectedServer != nil {
+ HStack {
+ Text(L10n.Assist.Watch.Volume.title)
+ .frame(maxWidth: .infinity, alignment: .leading)
+ VolumeView()
+ }
+ .padding()
+ .background(.gray.opacity(0.2))
+ .modify({ view in
+ if #available(watchOS 10, *) {
+ view.background(.ultraThinMaterial)
+ } else {
+ view
+ }
+ })
+ .clipShape(RoundedRectangle(cornerRadius: 35))
+ .padding(.top)
+ if !assistService.servers.isEmpty {
  VStack {
  Text(L10n.Settings.ConnectionSection.servers)
  Picker(selection: $assistService.selectedServer) {

Sources/Extensions/Watch/Assist/Views/VolumeView.swift

@@ -0,0 +1,20 @@
+import Foundation
+import Shared
+import SwiftUI
+import UIKit
+import WatchKit
+
+struct VolumeView: WKInterfaceObjectRepresentable {
+ typealias WKInterfaceObjectType = WKInterfaceVolumeControl
+
+ func makeWKInterfaceObject(context: Self.Context) -> WKInterfaceVolumeControl {
+ let view = WKInterfaceVolumeControl(origin: .local)
+ view.setTintColor(Asset.Colors.haPrimary.color)
+ return view
+ }
+
+ func updateWKInterfaceObject(
+ _ wkInterfaceObject: WKInterfaceVolumeControl,
+ context: WKInterfaceObjectRepresentableContext<VolumeView>
+ ) {}
+}

Sources/Extensions/Watch/Assist/WatchAssistService.swift

@@ -1,4 +1,3 @@
-import Combine
 import Communicator
 import Foundation
 import PromiseKit
@@ -25,6 +24,7 @@ final class WatchAssistService: ObservableObject {
 
  private let watchPreferredServerUserDefaultsKey = "watch-preferred-server-id"
  private var reachabilityObservation: Observation?
+ private var cancellable: Cancellable?
 
  init() {
  setupServers()
@@ -80,6 +80,7 @@ final class WatchAssistService: ObservableObject {
  }
 
  func assist(audioURL: URL, sampleRate: Double, completion: @escaping (Error?) -> Void) {
+ cancellable?.cancel()
  guard Communicator.shared.currentReachability == .immediatelyReachable else {
  completion(WatchSendError.notImmediate)
  return
@@ -110,7 +111,8 @@ final class WatchAssistService: ObservableObject {
  )
 
  Current.Log.verbose("Sending \(blob.identifier)")
- Communicator.shared.transfer(blob) { result in
+
+ cancellable = Communicator.shared.transfer(blob) { result in
  switch result {
  case .success:
  completion(nil)

Sources/Extensions/Watch/Assist/WatchAssistView.swift

@@ -4,11 +4,6 @@ import SwiftUI
 struct WatchAssistView: View {
  @StateObject private var viewModel: WatchAssistViewModel
 
- /// Used when there are multiple server
- @State private var showSettings = false
- /// Used when there are just one server for quicker access to pipeline selection
- @State private var showPipelinesPicker = false
-
  private let progressViewId = "progressViewId"
 
  init(
@@ -60,17 +55,12 @@ struct WatchAssistView: View {
  break
  }
  }
- .onChange(of: showSettings) { newValue in
- if newValue {
- viewModel.stopRecording()
- }
- }
- .onChange(of: showPipelinesPicker) { newValue in
+ .onChange(of: viewModel.showSettings) { newValue in
  if newValue {
  viewModel.stopRecording()
  }
  }
- .fullScreenCover(isPresented: $showSettings) {
+ .fullScreenCover(isPresented: $viewModel.showSettings) {
  WatchAssistSettings(assistService: viewModel.assistService)
  }
  .onReceive(NotificationCenter.default.publisher(for: AssistDefaultComplication.launchNotification)) { _ in
@@ -100,7 +90,7 @@ struct WatchAssistView: View {
  @ViewBuilder
  private var pipelineSelector: some View {
  Button {
- showSettings = true
+ viewModel.showSettings = true
  } label: {
  Image(systemName: "gear")
  }

Sources/Extensions/Watch/Assist/WatchAssistViewModel.swift

@@ -19,6 +19,7 @@ final class WatchAssistViewModel: ObservableObject {
  @Published var chatItems: [AssistChatItem] = []
  @Published var state: State = .idle
  @Published var showChatLoader = false
+ @Published var showSettings = false
 
  private let audioRecorder: any WatchAudioRecorderProtocol
  private let audioPlayer: any AudioPlayerProtocol
@@ -44,6 +45,7 @@ final class WatchAssistViewModel: ObservableObject {
  }
 
  func initialRoutine() {
+ appendChatItem(.init(content: "BETA", itemType: .info))
  state = .loading
  guard !assistService.selectedServer.isEmpty else {
  fatalError("Server can't be nil")
@@ -72,6 +74,11 @@ final class WatchAssistViewModel: ObservableObject {
  }
 
  func assist() {
+ guard !showSettings else {
+ state = .idle
+ stopRecording()
+ return
+ }
  if assistService.deviceReachable {
  // Extra message just to wake up iPhone from the background
  Communicator.shared.send(ImmediateMessage(identifier: "wakeup"))
@@ -160,6 +167,7 @@ extension WatchAssistViewModel: WatchAudioRecorderDelegate {
 
  func didFailRecording(error: any Error) {
  Current.Log.error("Failed to record Assist audio in watch App: \(error.localizedDescription)")
+ appendChatItem(.init(content: error.localizedDescription, itemType: .error))
  runInMainThread { [weak self] in
  self?.state = .idle
  }
@@ -178,5 +186,6 @@ extension WatchAssistViewModel: ImmediateCommunicatorServiceDelegate {
  func didReceiveError(code: String, message: String) {
  Current.Log.error("Watch Assist error: \(code)")
  appendChatItem(.init(content: message, itemType: .error))
+ stopRecording()
  }
 }

Sources/Extensions/Watch/Assist/WatchAudioRecorder.swift

@@ -20,6 +20,10 @@ final class WatchAudioRecorder: NSObject, WatchAudioRecorderProtocol {
  private var audioSampleRate: Double?
  weak var delegate: WatchAudioRecorderDelegate?
 
+ private var silenceTimer: Timer?
+ private let silenceThreshold: TimeInterval = 3.0
+ private let silenceLevel: Float = -50.0
+
  private var firstLaunch = true
 
  func startRecording() {
@@ -55,6 +59,9 @@ final class WatchAudioRecorder: NSObject, WatchAudioRecorderProtocol {
  audioRecorder?.record()
 
  delegate?.didStartRecording()
+ DispatchQueue.main.asyncAfter(deadline: .now() + 1) { [weak self] in
+ self?.startMonitoringAudioLevels()
+ }
  } catch {
  delegate?.didFailRecording(error: error)
  }
@@ -63,13 +70,34 @@ final class WatchAudioRecorder: NSObject, WatchAudioRecorderProtocol {
  func stopRecording() {
  audioRecorder?.stop()
  audioRecorder = nil
+ silenceTimer?.invalidate()
+ silenceTimer = nil
  delegate?.didStopRecording()
  }
 
  private func getAudioFileURL() -> URL {
  let sharedGroupContainerDirectory = Constants.AppGroupContainer
  return sharedGroupContainerDirectory.appendingPathComponent("assist.wav")
  }
+
+ private func startMonitoringAudioLevels() {
+ silenceTimer = Timer.scheduledTimer(withTimeInterval: 0.1, repeats: true) { [weak self] _ in
+ guard let self, let audioRecorder, audioRecorder.isRecording else { return }
+ audioRecorder.updateMeters()
+
+ let averagePower = audioRecorder.averagePower(forChannel: 0)
+ #if DEBUG
+ print("Average power channel 0: \(audioRecorder.averagePower(forChannel: 0))")
+ #endif
+ if averagePower < silenceLevel {
+ silenceTimer?.invalidate()
+ silenceTimer = Timer
+ .scheduledTimer(withTimeInterval: silenceThreshold, repeats: false) { [weak self] _ in
+ self?.stopRecording()
+ }
+ }
+ }
+ }
 }
 
 extension WatchAudioRecorder: AVAudioRecorderDelegate {
@@ -78,10 +106,15 @@ extension WatchAudioRecorder: AVAudioRecorderDelegate {
  delegate?.didFinishRecording(audioURL: getAudioFileURL(), audioSampleRate: audioSampleRate)
  } else {
  Current.Log.error("Finished recording without audio sample rate available")
+ delegate?.didFailRecording(error: WatchRecordingError.noAudioSampleRate)
  }
 
  #if DEBUG
  print(getAudioFileURL())
  #endif
  }
 }
+
+enum WatchRecordingError: Error {
+ case noAudioSampleRate
+}

Sources/Shared/Resources/Swiftgen/Strings.swift

@@ -300,6 +300,10 @@ public enum L10n {
  /// Assist requires iPhone connectivity. Your iPhone is currently unreachable.
  public static var title: String { return L10n.tr("Localizable", "assist.watch.not_reachable.title") }
  }
+ public enum Volume {
+ /// Volume control
+ public static var title: String { return L10n.tr("Localizable", "assist.watch.volume.title") }
+ }
  }
  }