Files
592b4add3c first commit
Co-authored-by: David Koski <[email protected]>
Co-authored-by: Christopher Webb <[email protected]>
2025-05-05 15:57:53 -07:00

693 lines
24 KiB
Swift

//
// For licensing see accompanying LICENSE file.
// Copyright (C) 2025 Apple Inc. All Rights Reserved.
//
import CoreImage
import CoreML
import Foundation
import MLX
import MLXFast
import MLXLMCommon
import MLXNN
import MLXVLM
import Tokenizers
// FastVLM is Qwen2VL with a custom vision tower.
// MARK: - Common
/// Rotates half the hidden dims of the input
private func rotateHalf(_ x: MLXArray) -> MLXArray {
let index = x.dim(-1) / 2
let x1 = x[.ellipsis, 0 ..< index]
let x2 = x[.ellipsis, index...]
return concatenated([-x2, x1], axis: -1)
}
// MARK: - Language
private enum Language {
/// Applies Rotary Position Embedding with Multimodal Sections to the query and key tensors
static private func applyMultimodalRotaryPositionEmbedding(
q: MLXArray, k: MLXArray, cos: MLXArray, sin: MLXArray,
positionIds: MLXArray, mropeSection: [Int]
) -> (MLXArray, MLXArray) {
var cos = cos[positionIds]
var sin = sin[positionIds]
cos =
concatenated(
// [m[i % 3] for i, m in enumerate(mx.split(cos, mrope_section, axis=-1))]
split(cos, indices: mropeSection, axis: -1).enumerated().map { i, m in m[i % 3] },
axis: -1
)[0..., .newAxis, 0..., 0...]
sin =
concatenated(
split(sin, indices: mropeSection, axis: -1).enumerated().map { i, m in m[i % 3] },
axis: -1
)[0..., .newAxis, 0..., 0...]
// Apply rotary embedding
let qEmbed = (q * cos) + (rotateHalf(q) * sin)
let kEmbed = (k * cos) + (rotateHalf(k) * sin)
return (qEmbed, kEmbed)
}
fileprivate class Attention: Module {
let heads: Int
let kvHeads: Int
let headDim: Int
let scale: Float
let mropeSection: [Int]
@ModuleInfo(key: "q_proj") var wq: Linear
@ModuleInfo(key: "k_proj") var wk: Linear
@ModuleInfo(key: "v_proj") var wv: Linear
@ModuleInfo(key: "o_proj") var wo: Linear
@ModuleInfo(key: "rotary_emb") var rotaryEmbedding: RoPE
public init(_ args: FastVLMConfiguration.TextConfiguration) {
let dim = args.hiddenSize
self.heads = args.attentionHeads
self.kvHeads = args.kvHeads
self.headDim = dim / heads
self.scale = pow(Float(headDim), -0.5)
self._wq.wrappedValue = Linear(dim, heads * headDim, bias: true)
self._wk.wrappedValue = Linear(dim, kvHeads * headDim, bias: true)
self._wv.wrappedValue = Linear(dim, kvHeads * headDim, bias: true)
self._wo.wrappedValue = Linear(heads * headDim, dim, bias: false)
if let v = args.ropeScaling?["mrope_section"], let array = v.asInts() {
// mrope_section = np.cumsum(mrope_section * 2)[:-1].tolist()
self.mropeSection = sequence(state: (0, array.makeIterator())) { state in
if let v = state.1.next() {
// note the *2
state.0 += v * 2
return state.0
} else {
return nil
}
}.dropLast()
} else {
fatalError("rope_scaling['mrope_section'] must be an array of integers")
}
self._rotaryEmbedding.wrappedValue = RoPE(
dimensions: headDim, traditional: args.ropeTraditional, base: args.ropeTheta)
}
public func callAsFunction(
_ x: MLXArray, mask: MLXArray? = nil, cache: KVCache?
) -> MLXArray {
let (B, L) = (x.dim(0), x.dim(1))
var queries = wq(x)
var keys = wk(x)
var values = wv(x)
// prepare the queries, keys and values for the attention computation
queries = queries.reshaped(B, L, heads, headDim).transposed(0, 2, 1, 3)
keys = keys.reshaped(B, L, kvHeads, headDim).transposed(0, 2, 1, 3)
values = values.reshaped(B, L, kvHeads, headDim).transposed(0, 2, 1, 3)
let offset = cache?.offset ?? 0
let mask = mask?[0..., 0 ..< keys.dim(-2)]
queries = rotaryEmbedding(queries, offset: offset)
keys = rotaryEmbedding(keys, offset: offset)
if let cache {
(keys, values) = cache.update(keys: keys, values: values)
}
let output = MLXFast.scaledDotProductAttention(
queries: queries, keys: keys, values: values, scale: scale, mask: mask
)
.transposed(0, 2, 1, 3)
.reshaped(B, L, -1)
return wo(output)
}
}
fileprivate class MLP: Module, UnaryLayer {
@ModuleInfo(key: "gate_proj") var gate: Linear
@ModuleInfo(key: "down_proj") var down: Linear
@ModuleInfo(key: "up_proj") var up: Linear
public init(dimensions: Int, hiddenDimensions: Int) {
self._gate.wrappedValue = Linear(dimensions, hiddenDimensions, bias: false)
self._down.wrappedValue = Linear(hiddenDimensions, dimensions, bias: false)
self._up.wrappedValue = Linear(dimensions, hiddenDimensions, bias: false)
}
public func callAsFunction(_ x: MLXArray) -> MLXArray {
down(silu(gate(x)) * up(x))
}
}
fileprivate class FastVLMDecoderLayer: Module {
@ModuleInfo(key: "self_attn") var attention: Attention
let mlp: MLP
@ModuleInfo(key: "input_layernorm") var inputLayerNorm: RMSNorm
@ModuleInfo(key: "post_attention_layernorm") var postAttentionLayerNorm: RMSNorm
public init(_ args: FastVLMConfiguration.TextConfiguration) {
self._attention.wrappedValue = Attention(args)
self.mlp = MLP(dimensions: args.hiddenSize, hiddenDimensions: args.intermediateSize)
self._inputLayerNorm.wrappedValue = RMSNorm(
dimensions: args.hiddenSize, eps: args.rmsNormEps)
self._postAttentionLayerNorm.wrappedValue = RMSNorm(
dimensions: args.hiddenSize, eps: args.rmsNormEps)
}
public func callAsFunction(
_ x: MLXArray, mask: MLXArray? = nil, cache: KVCache?
) -> MLXArray {
var r = attention(inputLayerNorm(x), mask: mask, cache: cache)
let h = x + r
r = mlp(postAttentionLayerNorm(h))
let out = h + r
return out
}
}
fileprivate class Qwen2Model: Module {
@ModuleInfo(key: "embed_tokens") var embedTokens: Embedding
fileprivate let layers: [FastVLMDecoderLayer]
fileprivate let norm: RMSNorm
public init(_ args: FastVLMConfiguration.TextConfiguration) {
precondition(args.vocabularySize > 0)
self._embedTokens.wrappedValue = Embedding(
embeddingCount: args.vocabularySize, dimensions: args.hiddenSize)
self.layers = (0 ..< args.hiddenLayers)
.map { _ in
FastVLMDecoderLayer(args)
}
self.norm = RMSNorm(dimensions: args.hiddenSize, eps: args.rmsNormEps)
}
public func callAsFunction(
_ inputs: MLXArray?, cache: [KVCache]? = nil, inputEmbedding: MLXArray? = nil
) -> MLXArray {
var h: MLXArray
if let inputEmbedding {
h = inputEmbedding
} else if let inputs {
h = embedTokens(inputs)
} else {
fatalError("one of inputs or inputEmbedding must be non-nil")
}
let mask = createAttentionMask(h: h, cache: cache)
for (i, layer) in layers.enumerated() {
h = layer(h, mask: mask, cache: cache?[i])
}
return norm(h)
}
}
fileprivate class LanguageModel: Module, KVCacheDimensionProvider {
@ModuleInfo var model: Qwen2Model
@ModuleInfo(key: "lm_head") var lmHead: Linear?
var kvHeads: [Int]
public init(_ args: FastVLMConfiguration.TextConfiguration) {
self.model = Qwen2Model(args)
if !args.tieWordEmbeddings {
_lmHead.wrappedValue = Linear(args.hiddenSize, args.vocabularySize, bias: false)
}
self.kvHeads = (0 ..< args.hiddenLayers).map { _ in args.kvHeads }
}
public func callAsFunction(
_ inputs: MLXArray?, cache: [KVCache]? = nil, inputEmbedding: MLXArray? = nil
) -> LMOutput {
var out = model(inputs, cache: cache, inputEmbedding: inputEmbedding)
if let lmHead {
out = lmHead(out)
} else {
out = model.embedTokens.asLinear(out)
}
return LMOutput(logits: out)
}
}
}
// MARK: - Vision
private enum Vision {
fileprivate class VisionModelCoreML {
let lock = NSLock()
var _model: fastvithd?
init() {
}
func load() throws -> fastvithd {
try lock.withLock {
if let model = _model { return model }
let model = try fastvithd()
_model = model
return model
}
}
public func model() -> fastvithd {
try! load()
}
public func encode(_ image: MLXArray) -> MLXArray {
// MLMultiArray requires mutable input data
var (data, strides) = {
let arrayData = image.asType(.float32).asData(access: .noCopyIfContiguous)
return (arrayData.data, arrayData.strides)
}()
precondition(image.ndim == 4)
precondition(image.dim(0) == 1)
precondition(image.dim(1) == 3)
let h = NSNumber(value: image.dim(2))
let w = NSNumber(value: image.dim(3))
return data.withUnsafeMutableBytes { (ptr: UnsafeMutableRawBufferPointer) in
// wrap the backing of the MLXArray
let array = try! MLMultiArray(
dataPointer: ptr.baseAddress!, shape: [1, 3, h, w], dataType: .float32,
strides: strides.map { .init(value: $0) })
// inference
let output = try! model().prediction(images: array)
precondition(output.image_features.shape == [1, 256, 3072])
precondition(output.image_features.dataType == .float32)
return output.image_features.withUnsafeBytes { ptr in
MLXArray(ptr, [1, 256, 3072], type: Float32.self)
}
}
}
}
fileprivate class VisionModel: Module {
let model = VisionModelCoreML()
public override init() {}
public func callAsFunction(_ hiddenStates: MLXArray, gridThw: [THW]) -> MLXArray {
model.encode(hiddenStates)
}
}
}
// MARK: - Processor
/// FastVLM `UserInputProcessor`.
///
/// This is meant to be used with ``FastVLM`` and is typically created by ``VLMModelFactory``.
public class FastVLMProcessor: UserInputProcessor {
private let config: FastVLMProcessorConfiguration
private let imageProcessingConfig: FastVLMPreProcessorConfiguration
private let tokenizer: any Tokenizer
public init(_ config: FastVLMPreProcessorConfiguration, tokenizer: any Tokenizer) {
self.config = FastVLMProcessorConfiguration()
self.imageProcessingConfig = config
self.tokenizer = tokenizer
}
public func preprocess(image: CIImage, processing: UserInput.Processing?) throws -> (
MLXArray, THW
) {
// first apply the user requested resizing, etc. if any
var image = MediaProcessingExtensions.apply(image, processing: processing)
// image_processing_clip.py
let size = MediaProcessingExtensions.fitIn(
image.extent.size, shortestEdge: imageProcessingConfig.size.shortestEdge)
image = MediaProcessingExtensions.resampleBicubic(image, to: size)
image = MediaProcessingExtensions.centerCrop(
image, size: imageProcessingConfig.cropSize.size)
image = MediaProcessing.normalize(
image, mean: imageProcessingConfig.imageMeanTuple,
std: imageProcessingConfig.imageStdTuple)
let array = MediaProcessingExtensions.asPlanarMLXArray(image)
return (array, .init(0, array.dim(2), array.dim(3)))
}
public func prepare(prompt: UserInput.Prompt, imageTHW: THW?) -> String {
var messages = prompt.asMessages()
if messages[0]["role"] != "system" {
messages.insert(["role": "system", "content": "You are a helpful assistant."], at: 0)
}
let lastIndex = messages.count - 1
var lastMessage = messages[lastIndex]["content"] ?? ""
// processing_llava.py
if let imageTHW {
let height = imageTHW.h
let width = imageTHW.w
let patchSize = config.patchSize
var numImageTokens =
(height / patchSize) * (width / patchSize) + config.numAdditionalImageTokens
if config.visionFeatureSelectStrategy == .default {
numImageTokens -= 1
}
lastMessage += Array(repeating: config.imageToken, count: numImageTokens)
.joined()
}
messages[lastIndex]["content"] = lastMessage
return
messages
.map {
"<|im_start|>\($0["role"] ?? "user")\n\($0["content"] ?? "")<|im_end|>"
}
.joined(separator: "\n")
+ "\n<|im_start|>assistant\n"
}
public func prepare(input: UserInput) throws -> LMInput {
if input.images.isEmpty {
// just a straight text prompt
let prompt = prepare(prompt: input.prompt, imageTHW: nil)
let promptTokens = tokenizer.encode(text: prompt)
return LMInput(tokens: MLXArray(promptTokens))
}
if input.images.count > 1 {
throw VLMError.singleImageAllowed
}
let (pixels, thw) = try preprocess(
image: input.images[0].asCIImage(), processing: input.processing)
let image = LMInput.ProcessedImage(pixels: pixels, imageGridThw: [thw])
let prompt = prepare(prompt: input.prompt, imageTHW: thw)
let promptTokens = tokenizer.encode(text: prompt)
let promptArray = MLXArray(promptTokens).expandedDimensions(axis: 0)
let mask = ones(like: promptArray).asType(.int8)
return LMInput(text: .init(tokens: promptArray, mask: mask), image: image)
}
}
// MARK: - Model
private class FastVLMMultiModalProjector: Module, UnaryLayer {
@ModuleInfo(key: "linear_0") var linear0: Linear
@ModuleInfo(key: "gelu") var gelu: GELU
@ModuleInfo(key: "linear_2") var linear2: Linear
public init(_ config: FastVLMConfiguration) {
self._linear0.wrappedValue = Linear(
config.visionConfiguration.hiddenSize,
config.textConfiguration.hiddenSize,
bias: true)
self._gelu.wrappedValue = GELU()
self._linear2.wrappedValue = Linear(
config.textConfiguration.hiddenSize,
config.textConfiguration.hiddenSize,
bias: true)
}
public func callAsFunction(_ x: MLXArray) -> MLXArray {
var x = linear0(x)
x = gelu(x)
x = linear2(x)
return x
}
}
/// FastVLM
///
/// This is typically created by ``VLMModelFactory``.
public class FastVLM: Module, VLMModel, KVCacheDimensionProvider {
static public var modelConfiguration: ModelConfiguration {
let bundle = Bundle(for: FastVLM.self)
let url = bundle.url(forResource: "config", withExtension: "json")!
.resolvingSymlinksInPath()
.deletingLastPathComponent()
return ModelConfiguration(directory: url)
}
static public func register(modelFactory: VLMModelFactory) {
modelFactory.typeRegistry.registerModelType("llava_qwen2") { url in
let configuration = try JSONDecoder().decode(
FastVLMConfiguration.self, from: Data(contentsOf: url))
return FastVLM(configuration)
}
modelFactory.processorRegistry.registerProcessorType("LlavaProcessor") { url, tokenizer in
let configuration = try JSONDecoder().decode(
FastVLMPreProcessorConfiguration.self, from: Data(contentsOf: url))
return FastVLMProcessor(configuration, tokenizer: tokenizer)
}
}
@ModuleInfo(key: "vision_tower") private var visionModel: Vision.VisionModel
@ModuleInfo(key: "language_model") private var languageModel: Language.LanguageModel
@ModuleInfo(key: "multi_modal_projector") private var multiModalProjector:
FastVLMMultiModalProjector
public let config: FastVLMConfiguration
public var vocabularySize: Int { config.baseConfiguration.vocabularySize }
public var kvHeads: [Int] { languageModel.kvHeads }
public func loraLinearLayers() -> MLXLMCommon.LoRALinearLayers {
languageModel.model.layers.map { ($0.attention, ["q_proj", "v_proj"]) }
}
public init(_ config: FastVLMConfiguration) {
self.config = config
self._visionModel.wrappedValue = Vision.VisionModel()
self._languageModel.wrappedValue = Language.LanguageModel(config.textConfiguration)
self._multiModalProjector.wrappedValue = FastVLMMultiModalProjector(config)
}
private func inputEmbeddings(inputIds: MLXArray, pixelValues: MLXArray?, gridThw: [THW]?)
-> MLXArray
{
guard let pixelValues, let gridThw else {
return languageModel(inputIds).logits
}
// Get the input embeddings from the language model
let inputEmbeds = languageModel.model.embedTokens(inputIds)
// Get the ouptut hidden states from the vision model
let imageFeaturesCoreML = self.visionModel(pixelValues, gridThw: gridThw)
let imageFeatures = multiModalProjector(imageFeaturesCoreML)
// Insert special image tokens in the input_ids
return mergeInputIdsWithImageFeatures(
inputIds: inputIds, inputEmbeds: inputEmbeds, imageFeatures: imageFeatures)
}
private func mergeInputIdsWithImageFeatures(
inputIds: MLXArray, inputEmbeds: MLXArray, imageFeatures: MLXArray
) -> MLXArray {
let imageTokenIndex = config.baseConfiguration.imageTokenId
var imageIndices = [Int]()
for (i, v) in inputIds.asArray(Int.self).enumerated() {
if v == imageTokenIndex {
imageIndices.append(i)
}
}
inputEmbeds[0..., MLXArray(imageIndices), 0...] = imageFeatures
return inputEmbeds
}
public func prepare(_ input: LMInput, cache: [any KVCache], windowSize: Int?) throws
-> PrepareResult
{
let gridThw = input.image?.imageGridThw
let dtype = DType.float32
let pixels = input.image?.pixels.asType(dtype)
let inputEmbeddings = self.inputEmbeddings(
inputIds: input.text.tokens, pixelValues: pixels, gridThw: gridThw)
let result = languageModel(nil, cache: cache, inputEmbedding: inputEmbeddings)
return .logits(result)
}
public func callAsFunction(_ inputs: MLXArray, cache: [any KVCache]?) -> MLXArray {
languageModel(inputs, cache: cache).logits
}
public func sanitize(weights: [String: MLXArray]) -> [String: MLXArray] {
_ = try? visionModel.model.load()
return weights
}
}
// MARK: - Configuration
/// Configuration for ``FastVLM``
public struct FastVLMConfiguration: Codable, Sendable {
public struct VisionConfiguration: Codable, Sendable {
public let hiddenSize: Int
enum CodingKeys: String, CodingKey {
case hiddenSize = "mm_hidden_size"
}
}
public struct TextConfiguration: Codable, Sendable {
public let modelType: String
public let hiddenSize: Int
public let hiddenLayers: Int
public let intermediateSize: Int
public let attentionHeads: Int
private let _rmsNormEps: Float?
public var rmsNormEps: Float { _rmsNormEps ?? 1e-6 }
public let vocabularySize: Int
public let kvHeads: Int
private let _maxPositionEmbeddings: Int?
public var maxpPositionEmbeddings: Int { _maxPositionEmbeddings ?? 32768 }
private let _ropeTheta: Float?
public var ropeTheta: Float { _ropeTheta ?? 1_000_000 }
private let _ropeTraditional: Bool?
public var ropeTraditional: Bool { _ropeTraditional ?? false }
public let _ropeScaling: [String: StringOrNumber]?
public var ropeScaling: [String: StringOrNumber]? {
_ropeScaling ?? ["mrope_section": .ints([2, 1, 1])]
}
private let _tieWordEmbeddings: Bool?
public var tieWordEmbeddings: Bool { _tieWordEmbeddings ?? true }
enum CodingKeys: String, CodingKey {
case modelType = "model_type"
case hiddenSize = "hidden_size"
case hiddenLayers = "num_hidden_layers"
case intermediateSize = "intermediate_size"
case attentionHeads = "num_attention_heads"
case _rmsNormEps = "rms_norm_eps"
case vocabularySize = "vocab_size"
case kvHeads = "num_key_value_heads"
case _maxPositionEmbeddings = "max_position_embeddings"
case _ropeTheta = "rope_theta"
case _ropeTraditional = "rope_traditional"
case _ropeScaling = "rope_scaling"
case _tieWordEmbeddings = "tie_word_embeddings"
}
}
public struct BaseConfiguration: Codable, Sendable {
public let modelType: String
public let vocabularySize: Int
public let imageTokenId: Int
public let hiddenSize: Int
enum CodingKeys: String, CodingKey {
case modelType = "model_type"
case vocabularySize = "vocab_size"
case imageTokenId = "image_token_index"
case hiddenSize = "hidden_size"
}
}
public let visionConfiguration: VisionConfiguration
public let textConfiguration: TextConfiguration
public let baseConfiguration: BaseConfiguration
public init(from decoder: any Swift.Decoder) throws {
// these are overlaid in the top level
self.visionConfiguration = try VisionConfiguration(from: decoder)
self.textConfiguration = try TextConfiguration(from: decoder)
self.baseConfiguration = try BaseConfiguration(from: decoder)
}
}
/// Configuration for ``FastVLMProcessor``
public struct FastVLMPreProcessorConfiguration: Codable, Sendable {
public struct CropSize: Codable, Sendable {
let width: Int
let height: Int
var size: CGSize { .init(width: CGFloat(width), height: CGFloat(height)) }
}
public struct Size: Codable, Sendable {
let shortestEdge: Int
enum CodingKeys: String, CodingKey {
case shortestEdge = "shortest_edge"
}
}
public var imageMean: [CGFloat]
public var imageStd: [CGFloat]
public var size: Size
public var cropSize: CropSize
public var imageMeanTuple: (CGFloat, CGFloat, CGFloat) {
(imageMean[0], imageMean[1], imageMean[2])
}
public var imageStdTuple: (CGFloat, CGFloat, CGFloat) {
(imageStd[0], imageStd[1], imageStd[2])
}
enum CodingKeys: String, CodingKey {
case imageMean = "image_mean"
case imageStd = "image_std"
case size
case cropSize = "crop_size"
}
}
public struct FastVLMProcessorConfiguration: Codable, Sendable {
public enum Strategy: Codable, Sendable {
case `default`
}
public var imageToken = "<image>"
public var numAdditionalImageTokens = 0
public var patchSize = 64
public var visionFeatureSelectStrategy: Strategy?
}