Files
592b4add3c first commit
Co-authored-by: David Koski <[email protected]>
Co-authored-by: Christopher Webb <[email protected]>
2025-05-05 15:57:53 -07:00

454 lines
17 KiB
Swift

//
// For licensing see accompanying LICENSE file.
// Copyright (C) 2025 Apple Inc. All Rights Reserved.
//
import AVFoundation
import MLXLMCommon
import SwiftUI
import Video
// support swift 6
extension CVImageBuffer: @unchecked @retroactive Sendable {}
extension CMSampleBuffer: @unchecked @retroactive Sendable {}
// delay between frames -- controls the frame rate of the updates
let FRAME_DELAY = Duration.milliseconds(1)
struct ContentView: View {
@State private var camera = CameraController()
@State private var model = FastVLMModel()
/// stream of frames -> VideoFrameView, see distributeVideoFrames
@State private var framesToDisplay: AsyncStream<CVImageBuffer>?
@State private var prompt = "Describe the image in English."
@State private var promptSuffix = "Output should be brief, about 15 words or less."
@State private var isShowingInfo: Bool = false
@State private var selectedCameraType: CameraType = .continuous
@State private var isEditingPrompt: Bool = false
var toolbarItemPlacement: ToolbarItemPlacement {
var placement: ToolbarItemPlacement = .navigation
#if os(iOS)
placement = .topBarLeading
#endif
return placement
}
var statusTextColor : Color {
return model.evaluationState == .processingPrompt ? .black : .white
}
var statusBackgroundColor : Color {
switch model.evaluationState {
case .idle:
return .gray
case .generatingResponse:
return .green
case .processingPrompt:
return .yellow
}
}
var body: some View {
NavigationStack {
Form {
Section {
VStack(alignment: .leading, spacing: 10.0) {
Picker("Camera Type", selection: $selectedCameraType) {
ForEach(CameraType.allCases, id: \.self) { cameraType in
Text(cameraType.rawValue.capitalized).tag(cameraType)
}
}
// Prevent macOS from adding a text label for the picker
.labelsHidden()
.pickerStyle(.segmented)
.onChange(of: selectedCameraType) { _, _ in
// Cancel any in-flight requests when switching modes
model.cancel()
}
if let framesToDisplay {
VideoFrameView(
frames: framesToDisplay,
cameraType: selectedCameraType,
action: { frame in
processSingleFrame(frame)
})
// Because we're using the AVCaptureSession preset
// `.vga640x480`, we can assume this aspect ratio
.aspectRatio(4/3, contentMode: .fit)
#if os(macOS)
.frame(maxWidth: 750)
#endif
.overlay(alignment: .top) {
if !model.promptTime.isEmpty {
Text("TTFT \(model.promptTime)")
.font(.caption)
.foregroundStyle(.white)
.monospaced()
.padding(.vertical, 4.0)
.padding(.horizontal, 6.0)
.background(alignment: .center) {
RoundedRectangle(cornerRadius: 8)
.fill(Color.black.opacity(0.6))
}
.padding(.top)
}
}
#if !os(macOS)
.overlay(alignment: .topTrailing) {
CameraControlsView(
backCamera: $camera.backCamera,
device: $camera.device,
devices: $camera.devices)
.padding()
}
#endif
.overlay(alignment: .bottom) {
if selectedCameraType == .continuous {
Group {
if model.evaluationState == .processingPrompt {
HStack {
ProgressView()
.tint(self.statusTextColor)
.controlSize(.small)
Text(model.evaluationState.rawValue)
}
} else if model.evaluationState == .idle {
HStack(spacing: 6.0) {
Image(systemName: "clock.fill")
.font(.caption)
Text(model.evaluationState.rawValue)
}
}
else {
// I'm manually tweaking the spacing to
// better match the spacing with ProgressView
HStack(spacing: 6.0) {
Image(systemName: "lightbulb.fill")
.font(.caption)
Text(model.evaluationState.rawValue)
}
}
}
.foregroundStyle(self.statusTextColor)
.font(.caption)
.bold()
.padding(.vertical, 6.0)
.padding(.horizontal, 8.0)
.background(self.statusBackgroundColor)
.clipShape(.capsule)
.padding(.bottom)
}
}
#if os(macOS)
.frame(maxWidth: .infinity)
.frame(minWidth: 500)
.frame(minHeight: 375)
#endif
}
}
}
.listRowInsets(EdgeInsets())
.listRowBackground(Color.clear)
.listRowSeparator(.hidden)
promptSections
Section {
if model.output.isEmpty && model.running {
ProgressView()
.controlSize(.large)
.frame(maxWidth: .infinity)
} else {
ScrollView {
Text(model.output)
.foregroundStyle(isEditingPrompt ? .secondary : .primary)
.textSelection(.enabled)
#if os(macOS)
.font(.headline)
.fontWeight(.regular)
#endif
}
.frame(minHeight: 50.0, maxHeight: 200.0)
}
} header: {
Text("Response")
#if os(macOS)
.font(.headline)
.padding(.bottom, 2.0)
#endif
}
#if os(macOS)
Spacer()
#endif
}
#if os(iOS)
.listSectionSpacing(0)
#elseif os(macOS)
.padding()
#endif
.task {
camera.start()
}
.task {
await model.load()
}
#if !os(macOS)
.onAppear {
// Prevent the screen from dimming or sleeping due to inactivity
UIApplication.shared.isIdleTimerDisabled = true
}
.onDisappear {
// Resumes normal idle timer behavior
UIApplication.shared.isIdleTimerDisabled = false
}
#endif
// task to distribute video frames -- this will cancel
// and restart when the view is on/off screen. note: it is
// important that this is here (attached to the VideoFrameView)
// rather than the outer view because this has the correct lifecycle
.task {
if Task.isCancelled {
return
}
await distributeVideoFrames()
}
.navigationTitle("FastVLM")
#if os(iOS)
.navigationBarTitleDisplayMode(.inline)
#endif
.toolbar {
ToolbarItem(placement: toolbarItemPlacement) {
Button {
isShowingInfo.toggle()
}
label: {
Image(systemName: "info.circle")
}
}
ToolbarItem(placement: .primaryAction) {
if isEditingPrompt {
Button {
isEditingPrompt.toggle()
}
label: {
Text("Done")
.fontWeight(.bold)
}
}
else {
Menu {
Button("Describe image") {
prompt = "Describe the image in English."
promptSuffix = "Output should be brief, about 15 words or less."
}
Button("Facial expression") {
prompt = "What is this person's facial expression?"
promptSuffix = "Output only one or two words."
}
Button("Read text") {
prompt = "What is written in this image?"
promptSuffix = "Output only the text in the image."
}
#if !os(macOS)
Button("Customize...") {
isEditingPrompt.toggle()
}
#endif
} label: { Text("Prompts") }
}
}
}
.sheet(isPresented: $isShowingInfo) {
InfoView()
}
}
}
var promptSummary: some View {
Section("Prompt") {
VStack(alignment: .leading, spacing: 4.0) {
let trimmedPrompt = prompt.trimmingCharacters(in: .whitespacesAndNewlines)
if !trimmedPrompt.isEmpty {
Text(trimmedPrompt)
.foregroundStyle(.secondary)
}
let trimmedSuffix = promptSuffix.trimmingCharacters(in: .whitespacesAndNewlines)
if !trimmedSuffix.isEmpty {
Text(trimmedSuffix)
.font(.caption)
.foregroundStyle(.tertiary)
}
}
}
}
var promptForm: some View {
Group {
#if os(iOS)
Section("Prompt") {
TextEditor(text: $prompt)
.frame(minHeight: 38)
}
Section("Prompt Suffix") {
TextEditor(text: $promptSuffix)
.frame(minHeight: 38)
}
#elseif os(macOS)
Section {
HStack(alignment: .top) {
VStack(alignment: .leading) {
Text("Prompt")
.font(.headline)
TextEditor(text: $prompt)
.frame(height: 38)
.padding(.horizontal, 8.0)
.padding(.vertical, 10.0)
.background(Color(.textBackgroundColor))
.cornerRadius(10.0)
}
VStack(alignment: .leading) {
Text("Prompt Suffix")
.font(.headline)
TextEditor(text: $promptSuffix)
.frame(height: 38)
.padding(.horizontal, 8.0)
.padding(.vertical, 10.0)
.background(Color(.textBackgroundColor))
.cornerRadius(10.0)
}
}
}
.padding(.vertical)
#endif
}
}
var promptSections: some View {
Group {
#if os(iOS)
if isEditingPrompt {
promptForm
}
else {
promptSummary
}
#elseif os(macOS)
promptForm
#endif
}
}
func analyzeVideoFrames(_ frames: AsyncStream<CVImageBuffer>) async {
for await frame in frames {
let userInput = UserInput(
prompt: .text("\(prompt) \(promptSuffix)"),
images: [.ciImage(CIImage(cvPixelBuffer: frame))]
)
// generate output for a frame and wait for generation to complete
let t = await model.generate(userInput)
_ = await t.result
do {
try await Task.sleep(for: FRAME_DELAY)
} catch { return }
}
}
func distributeVideoFrames() async {
// attach a stream to the camera -- this code will read this
let frames = AsyncStream<CMSampleBuffer>(bufferingPolicy: .bufferingNewest(1)) {
camera.attach(continuation: $0)
}
let (framesToDisplay, framesToDisplayContinuation) = AsyncStream.makeStream(
of: CVImageBuffer.self,
bufferingPolicy: .bufferingNewest(1)
)
self.framesToDisplay = framesToDisplay
// Only create analysis stream if in continuous mode
let (framesToAnalyze, framesToAnalyzeContinuation) = AsyncStream.makeStream(
of: CVImageBuffer.self,
bufferingPolicy: .bufferingNewest(1)
)
// set up structured tasks (important -- this means the child tasks
// are cancelled when the parent is cancelled)
async let distributeFrames: () = {
for await sampleBuffer in frames {
if let frame = sampleBuffer.imageBuffer {
framesToDisplayContinuation.yield(frame)
// Only send frames for analysis in continuous mode
if await selectedCameraType == .continuous {
framesToAnalyzeContinuation.yield(frame)
}
}
}
// detach from the camera controller and feed to the video view
await MainActor.run {
self.framesToDisplay = nil
self.camera.detatch()
}
framesToDisplayContinuation.finish()
framesToAnalyzeContinuation.finish()
}()
// Only analyze frames if in continuous mode
if selectedCameraType == .continuous {
async let analyze: () = analyzeVideoFrames(framesToAnalyze)
await distributeFrames
await analyze
} else {
await distributeFrames
}
}
/// Perform FastVLM inference on a single frame.
/// - Parameter frame: The frame to analyze.
func processSingleFrame(_ frame: CVImageBuffer) {
// Reset Response UI (show spinner)
Task { @MainActor in
model.output = ""
}
// Construct request to model
let userInput = UserInput(
prompt: .text("\(prompt) \(promptSuffix)"),
images: [.ciImage(CIImage(cvPixelBuffer: frame))]
)
// Post request to FastVLM
Task {
await model.generate(userInput)
}
}
}
#Preview {
ContentView()
}