Voice Assistant Widget

dependencies: [
    .package(url: "https://github.com/Liquid4All/leap-sdk.git", from: "0.10.7")
]

targets: [
    .target(
        name: "YourApp",
        dependencies: [
            .product(name: "LeapModelDownloader", package: "leap-sdk"),
            .product(name: "LeapUI",              package: "leap-sdk"),
        ]
    )
]

import LeapModelDownloader
import LeapUi

@MainActor
final class VoiceAssistantViewModel: ObservableObject {
    let store: VoiceAssistantStore
    private let downloader: ModelDownloader = {
        let caches = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first!.path
        let modelsDir = (caches as NSString).appendingPathComponent("leap_models")
        return ModelDownloader(config: LeapDownloaderConfig(saveDir: modelsDir))
    }()

    init() {
        // Defaults: AppleAudioRecorder, AppleAudioPlayer, MainScope, interruptToSpeak = true
        store = VoiceAssistantStore.makeForApple()
    }

    deinit { store.close() }

    func loadModel() async {
        do {
            let runner = try await downloader.loadModel(
                modelName: "LFM2.5-Audio-1.5B",
                quantizationType: "Q4_0",
                downloadProgress: { fraction, _ in
                    // `fraction` is `Double` from the Kotlin (Double, Long) -> Unit
                    // closure; `setModelProgress.fraction` is `Float`, so cast.
                    Task { @MainActor in
                        self.store.setModelProgress(
                            fraction: Float(fraction),
                            message: "Downloading (\(Int(fraction * 100))%)"
                        )
                    }
                }
            )
            let conversation = runner.createConversation(
                systemPrompt: "Respond with interleaved text and audio."
            )
            store.setConversation(conv: AppleVoiceConversation(conversation: conversation))
        } catch {
            store.setModelError(message: "✗ \(error.localizedDescription)")
        }
    }
}

let store = VoiceAssistantStore.makeForApple(
    recorder: myCustomRecorder,
    player: myCustomPlayer,
    smoothingAlpha: 0.3,
    playbackTimeoutMs: 10_000,
    interruptToSpeak: false  // Press during a response only cancels; doesn't re-record immediately
)

import LeapUi
import SwiftUI

struct VoiceAssistantScreen: View {
    @StateObject private var viewModel = VoiceAssistantViewModel()

    var body: some View {
        VoiceWidgetRepresentable(store: viewModel.store)
            .background(Color.black)
            .ignoresSafeArea()
            .task { await viewModel.loadModel() }
    }
}

private struct VoiceWidgetRepresentable: UIViewControllerRepresentable {
    let store: VoiceAssistantStore

    func makeUIViewController(context: Context) -> UIViewController {
        VoiceAssistantViewControllerKt.VoiceAssistantViewController(
            state: store.widgetStateHolder,
            onIntent: { intent in store.processIntent(intent: intent) },
            labels: VoiceWidgetLabels(
                idle: "Tap and hold to speak",
                listening: "Listening",
                responding: "Generating",
                micStartDescription: "Start recording",
                micStopDescription: "Stop recording",
                micCancelDescription: "Cancel recording"
            ),
            colors: VoiceWidgetColors.companion.Default,
            showPoweredBy: true
        )
    }

    func updateUIViewController(_ uiViewController: UIViewController, context: Context) {}
}

import LeapModelDownloader
import LeapSDK
import LeapUi

final class AppleVoiceConversation: VoiceConversation {
    private let conversation: Conversation

    init(conversation: Conversation) {
        self.conversation = conversation
    }

    // Note: this method is `__generateResponse` in the SKIE-generated overlay
    // because `LeapUI` and `LeapSDK` are separate frameworks with separate Kotlin
    // runtimes. The runtime-types-as-parameters force the underscore prefix.
    func generateResponse(
        audioSamples: LeapUi.KotlinFloatArray,
        sampleRate: Int32,
        onAudioChunk: @escaping (LeapUi.KotlinFloatArray, LeapUi.KotlinInt) -> Void
    ) async throws -> Leap_sdkGenerationStats? {
        // LeapUi.KotlinFloatArray -> Swift [Float] (for use inside this method body):
        let nsData = LeapUi.ArrayConversionsKt.floatArrayToNSData(array: audioSamples)
        let samples: [Float] = nsData.withUnsafeBytes { Array($0.bindMemory(to: Float.self)) }

        let audioContent = ChatMessageContent.fromFloatSamples(samples, sampleRate: Int(sampleRate))
        let userMessage = ChatMessage(
            role: .user,
            content: [audioContent as ChatMessageContent],
            reasoningContent: nil,
            functionCalls: nil
        )

        var stats: Leap_sdkGenerationStats?
        for try await response in conversation.generateResponse(message: userMessage) {
            switch onEnum(of: response) {
            case .audioSample(let chunk):
                // Bridge LeapSDK.KotlinFloatArray -> LeapUi.KotlinFloatArray via NSData.
                let data = LeapSDK.ArrayConversionsKt.floatArrayToNSData(array: chunk.samples)
                let uiSamples = LeapUi.ArrayConversionsKt.nsDataToFloatArray(data: data)
                onAudioChunk(uiSamples, LeapUi.KotlinInt(value: chunk.sampleRate))
            case .complete(let c):
                stats = c.stats
            case .chunk, .reasoningChunk, .functionCalls:
                break
            }
        }
        return stats
    }

    func reset() -> VoiceConversation {
        AppleVoiceConversation(
            conversation: conversation.modelRunner.createConversation(systemPrompt: nil)
        )
    }
}

import AVFoundation

let session = AVAudioSession.sharedInstance()
try session.setCategory(.playAndRecord, mode: .default, options: [.defaultToSpeaker])
try session.setActive(true)
session.requestRecordPermission { _ in }

let store = VoiceAssistantStore.makeForApple(interruptToSpeak: false)