mirror of
https://github.com/zenlm/ml-fastvlm.git
synced 2026-10-30 19:52:02 +00:00
Co-authored-by: David Koski <[email protected]> Co-authored-by: Christopher Webb <[email protected]>
191 lines
5.9 KiB
Swift
191 lines
5.9 KiB
Swift
//
|
|
// For licensing see accompanying LICENSE file.
|
|
// Copyright (C) 2025 Apple Inc. All Rights Reserved.
|
|
//
|
|
|
|
import CoreImage
|
|
import FastVLM
|
|
import Foundation
|
|
import MLX
|
|
import MLXLMCommon
|
|
import MLXRandom
|
|
import MLXVLM
|
|
|
|
@Observable
|
|
@MainActor
|
|
class FastVLMModel {
|
|
|
|
public var running = false
|
|
public var modelInfo = ""
|
|
public var output = ""
|
|
public var promptTime: String = ""
|
|
|
|
enum LoadState {
|
|
case idle
|
|
case loaded(ModelContainer)
|
|
}
|
|
|
|
private let modelConfiguration = FastVLM.modelConfiguration
|
|
|
|
/// parameters controlling the output
|
|
let generateParameters = GenerateParameters(temperature: 0.0)
|
|
let maxTokens = 240
|
|
|
|
/// update the display every N tokens -- 4 looks like it updates continuously
|
|
/// and is low overhead. observed ~15% reduction in tokens/s when updating
|
|
/// on every token
|
|
let displayEveryNTokens = 4
|
|
|
|
private var loadState = LoadState.idle
|
|
private var currentTask: Task<Void, Never>?
|
|
|
|
enum EvaluationState: String, CaseIterable {
|
|
case idle = "Idle"
|
|
case processingPrompt = "Processing Prompt"
|
|
case generatingResponse = "Generating Response"
|
|
}
|
|
|
|
public var evaluationState = EvaluationState.idle
|
|
|
|
public init() {
|
|
FastVLM.register(modelFactory: VLMModelFactory.shared)
|
|
}
|
|
|
|
private func _load() async throws -> ModelContainer {
|
|
switch loadState {
|
|
case .idle:
|
|
// limit the buffer cache
|
|
MLX.GPU.set(cacheLimit: 20 * 1024 * 1024)
|
|
|
|
let modelContainer = try await VLMModelFactory.shared.loadContainer(
|
|
configuration: modelConfiguration
|
|
) {
|
|
[modelConfiguration] progress in
|
|
Task { @MainActor in
|
|
self.modelInfo =
|
|
"Downloading \(modelConfiguration.name): \(Int(progress.fractionCompleted * 100))%"
|
|
}
|
|
}
|
|
self.modelInfo = "Loaded"
|
|
loadState = .loaded(modelContainer)
|
|
return modelContainer
|
|
|
|
case .loaded(let modelContainer):
|
|
return modelContainer
|
|
}
|
|
}
|
|
|
|
public func load() async {
|
|
do {
|
|
_ = try await _load()
|
|
} catch {
|
|
self.modelInfo = "Error loading model: \(error)"
|
|
}
|
|
}
|
|
|
|
public func generate(_ userInput: UserInput) async -> Task<Void, Never> {
|
|
if let currentTask, running {
|
|
return currentTask
|
|
}
|
|
|
|
running = true
|
|
|
|
// Cancel any existing task
|
|
currentTask?.cancel()
|
|
|
|
// Create new task and store reference
|
|
let task = Task {
|
|
do {
|
|
let modelContainer = try await _load()
|
|
|
|
// each time you generate you will get something new
|
|
MLXRandom.seed(UInt64(Date.timeIntervalSinceReferenceDate * 1000))
|
|
|
|
// Check if task was cancelled
|
|
if Task.isCancelled { return }
|
|
|
|
let result = try await modelContainer.perform { context in
|
|
// Measure the time it takes to prepare the input
|
|
|
|
Task { @MainActor in
|
|
evaluationState = .processingPrompt
|
|
}
|
|
|
|
let llmStart = Date()
|
|
let input = try await context.processor.prepare(input: userInput)
|
|
|
|
var seenFirstToken = false
|
|
|
|
// FastVLM generates the output
|
|
let result = try MLXLMCommon.generate(
|
|
input: input, parameters: generateParameters, context: context
|
|
) { tokens in
|
|
// Check if task was cancelled
|
|
if Task.isCancelled {
|
|
return .stop
|
|
}
|
|
|
|
if !seenFirstToken {
|
|
seenFirstToken = true
|
|
|
|
// produced first token, update the time to first token,
|
|
// the processing state and start displaying the text
|
|
let llmDuration = Date().timeIntervalSince(llmStart)
|
|
let text = context.tokenizer.decode(tokens: tokens)
|
|
Task { @MainActor in
|
|
evaluationState = .generatingResponse
|
|
self.output = text
|
|
self.promptTime = "\(Int(llmDuration * 1000)) ms"
|
|
}
|
|
}
|
|
|
|
// Show the text in the view as it generates
|
|
if tokens.count % displayEveryNTokens == 0 {
|
|
let text = context.tokenizer.decode(tokens: tokens)
|
|
Task { @MainActor in
|
|
self.output = text
|
|
}
|
|
}
|
|
|
|
if tokens.count >= maxTokens {
|
|
return .stop
|
|
} else {
|
|
return .more
|
|
}
|
|
}
|
|
|
|
// Return the duration of the LLM and the result
|
|
return result
|
|
}
|
|
|
|
// Check if task was cancelled before updating UI
|
|
if !Task.isCancelled {
|
|
self.output = result.output
|
|
}
|
|
|
|
} catch {
|
|
if !Task.isCancelled {
|
|
output = "Failed: \(error)"
|
|
}
|
|
}
|
|
|
|
if evaluationState == .generatingResponse {
|
|
evaluationState = .idle
|
|
}
|
|
|
|
running = false
|
|
}
|
|
|
|
currentTask = task
|
|
return task
|
|
}
|
|
|
|
public func cancel() {
|
|
currentTask?.cancel()
|
|
currentTask = nil
|
|
running = false
|
|
output = ""
|
|
promptTime = ""
|
|
}
|
|
}
|