Files
zeekayandHanzo Dev 6a363f7ee3 merge: upstream main ddf1b879 and the aarch64 repack branch 35d7ae7c
Three-way per file through the rename: base 39355c6c91, theirs the upstream
merge of main with 35d7ae7c (via 7ef6a694d), ours origin/main.

Lands the barrier thread pool, runtime CPU feature dispatch, aarch64 and x86
repacked quantized kernels (indexed_gemv, gemv_fused_shared_lhs), quantized
embedding on CPU/Metal/CUDA, paged flash-attn kernels, cutile fused MoE, the
in-place op aliasing API, parking_lot storage locks, Metal buffer builders and
debug labels, safetensors 0.8, hf-hub 1.0, tokenizers 0.23.

Ours keeps ROCm, Vulkan, wgpu, the IQ/MXFP4/NVFP4/TQ dtypes, and the i-quant
weight/activation block split in matmul. Our backends ride InplaceOpN and the
quantized embedding; Metal get_rows gains the IQ kernels. ug leaves, as upstream.

Co-authored-by: Hanzo Dev <[email protected]>
2026-09-15 15:52:28 -07:00

1202 lines
40 KiB
Rust
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! PaddleOCR-VL: Vision-Language Model for Document Parsing.
//!
//! PaddleOCR-VL is a compact vision-language model (0.9B parameters) that combines
//! a NaViT-style visual encoder with ERNIE-4.5-0.3B for document understanding.
//!
//! Supports:
//! - Text recognition (OCR)
//! - Table recognition
//! - Formula recognition
//! - Chart recognition
//! - Multi-image processing (e.g., multi-page documents)
//! - Video processing with temporal position encoding
//!
//! ```bash
//! # Basic OCR
//! cargo run --example paddleocr-vl --release -- \
//! --image document.png
//!
//! # Table recognition
//! cargo run --example paddleocr-vl --release -- \
//! --image table.png \
//! --task table
//!
//! # Formula recognition
//! cargo run --example paddleocr-vl --release -- \
//! --image formula.png \
//! --task formula
//!
//! # Chart recognition
//! cargo run --example paddleocr-vl --release -- \
//! --image chart.png \
//! --task chart
//!
//! # Multi-page document OCR (2 pages)
//! cargo run --example paddleocr-vl --release -- \
//! --image page1.png --image page2.png
//!
//! # Batch mode - process multiple images sequentially without reloading model
//! cargo run --example paddleocr-vl --release -- \
//! --batch doc1.png doc2.png doc3.png
//!
//! # Batch mode with glob pattern (shell expansion)
//! cargo run --example paddleocr-vl --release -- \
//! --batch ./documents/*.png
//!
//! # Video OCR (requires ffmpeg)
//! cargo run --example paddleocr-vl --release -- \
//! --video clip.mp4 \
//! --fps 1.0 \
//! --max-frames 16
//! ```
#[cfg(feature = "mkl")]
extern crate intel_mkl_src;
#[cfg(feature = "accelerate")]
extern crate accelerate_src;
use anyhow::{Error as E, Result};
use clap::{Parser, ValueEnum};
use hanzo_ml::{DType, Device, Tensor};
use hanzo_nn::VarBuilder;
use hanzo_transformers::models::paddleocr_vl::{Config, PaddleOCRVLModel};
use tokenizers::Tokenizer;
const DEFAULT_MODEL_ID: &str = "PaddlePaddle/PaddleOCR-VL";
#[derive(Debug, Clone, Copy, ValueEnum, PartialEq)]
enum Task {
/// Text recognition (OCR)
Ocr,
/// Table recognition
Table,
/// Formula recognition
Formula,
/// Chart recognition
Chart,
/// Video mode - process all frames as a single video sequence (experimental)
Video,
}
impl Task {
fn prompt(&self) -> &'static str {
match self {
Task::Ocr => "OCR:",
Task::Table => "Table Recognition:",
Task::Formula => "Formula Recognition:",
Task::Chart => "Chart Recognition:",
Task::Video => "OCR:", // Video uses same prompt as OCR
}
}
}
#[derive(Parser, Debug)]
#[command(author, version, about, long_about = None)]
struct Args {
/// Path to document image(s). Can specify multiple times for multi-image processing.
#[arg(long, num_args = 1..)]
image: Vec<String>,
/// Batch mode: process multiple images sequentially without reloading model.
/// Each image is processed independently with separate output.
/// Unlike --image which combines multiple images into one prompt,
/// --batch processes each image as a separate inference run.
#[arg(long, num_args = 1..)]
batch: Vec<String>,
/// Path to video file. Mutually exclusive with --image.
#[arg(long)]
video: Option<String>,
/// Frames per second to extract from video (default: 1.0)
#[arg(long, default_value = "1.0")]
fps: f32,
/// Maximum number of frames to extract from video (default: 16)
#[arg(long, default_value = "16")]
max_frames: usize,
/// Similarity threshold for deduplication in video processing (0.0-1.0, default: 0.85)
/// Text with similarity above this threshold to the previous frame is considered duplicate.
#[arg(long, default_value = "0.85")]
similarity_threshold: f32,
/// Task type
#[arg(long, value_enum, default_value = "ocr")]
task: Task,
/// Model repository or path
#[arg(long, default_value = DEFAULT_MODEL_ID)]
model_id: String,
/// Model revision
#[arg(long, default_value = "main")]
revision: String,
/// Run on CPU rather than GPU
#[arg(long)]
cpu: bool,
/// Maximum generation length
#[arg(long, default_value = "1024")]
max_length: usize,
/// Use bfloat16 precision
#[arg(long)]
bf16: bool,
}
/// Compute Levenshtein distance between two strings.
///
/// Returns the minimum number of single-character edits (insertions, deletions,
/// substitutions) required to transform one string into the other.
fn levenshtein_distance(a: &str, b: &str) -> usize {
let a_chars: Vec<char> = a.chars().collect();
let b_chars: Vec<char> = b.chars().collect();
let m = a_chars.len();
let n = b_chars.len();
if m == 0 {
return n;
}
if n == 0 {
return m;
}
// Use two rows instead of full matrix for space efficiency
let mut prev_row: Vec<usize> = (0..=n).collect();
let mut curr_row: Vec<usize> = vec![0; n + 1];
for i in 1..=m {
curr_row[0] = i;
for j in 1..=n {
let cost = if a_chars[i - 1] == b_chars[j - 1] {
0
} else {
1
};
curr_row[j] = (prev_row[j] + 1) // deletion
.min(curr_row[j - 1] + 1) // insertion
.min(prev_row[j - 1] + cost); // substitution
}
std::mem::swap(&mut prev_row, &mut curr_row);
}
prev_row[n]
}
/// Compute normalized similarity between two strings (0.0 to 1.0).
///
/// Returns 1.0 for identical strings, 0.0 for completely different strings.
/// Uses Levenshtein distance normalized by the length of the longer string.
fn string_similarity(a: &str, b: &str) -> f32 {
if a.is_empty() && b.is_empty() {
return 1.0;
}
let max_len = a.chars().count().max(b.chars().count());
if max_len == 0 {
return 1.0;
}
let distance = levenshtein_distance(a, b);
1.0 - (distance as f32 / max_len as f32)
}
/// Result from frame-by-frame OCR processing.
#[derive(Debug, Clone)]
struct FrameOcrResult {
/// Frame index (0-based)
frame_index: usize,
/// Timestamp in seconds
timestamp: f32,
/// Recognized text
text: String,
}
/// Check if text is a known hallucination pattern.
///
/// Models often produce these phrases when there's no actual text to recognize
/// (e.g., empty frames, black screens, or images without text).
fn is_hallucination(text: &str) -> bool {
let normalized = text.to_lowercase();
// Common hallucination patterns (lowercase for comparison)
let patterns = ["the quick brown fox jumps over the lazy dog"];
for pattern in patterns {
if normalized.contains(pattern) {
return true;
}
}
false
}
/// Smart resize algorithm matching PyTorch's PaddleOCRVLImageProcessor.
///
/// Rescales the image so that:
/// 1. Both dimensions are divisible by `factor` (patch_size × merge_size = 28)
/// 2. Total pixels are within [min_pixels, max_pixels] range
/// 3. Aspect ratio is maintained as closely as possible
fn smart_resize(
height: usize,
width: usize,
factor: usize,
min_pixels: usize,
max_pixels: usize,
) -> Result<(usize, usize)> {
let mut h = height;
let mut w = width;
// Handle tiny images by scaling up to minimum factor
if h < factor {
w = (w * factor + h / 2) / h;
h = factor;
}
if w < factor {
h = (h * factor + w / 2) / w;
w = factor;
}
// Check aspect ratio constraint
let aspect = if h > w {
h as f64 / w as f64
} else {
w as f64 / h as f64
};
if aspect > 200.0 {
return Err(E::msg(format!(
"Aspect ratio {:.1} exceeds maximum of 200",
aspect
)));
}
// Round to nearest multiple of factor
let mut h_bar = ((h + factor / 2) / factor) * factor;
let mut w_bar = ((w + factor / 2) / factor) * factor;
let total_pixels = h_bar * w_bar;
if total_pixels > max_pixels {
// Scale down to fit within max_pixels
let beta = ((h * w) as f64 / max_pixels as f64).sqrt();
h_bar = ((h as f64 / beta / factor as f64).floor() as usize) * factor;
w_bar = ((w as f64 / beta / factor as f64).floor() as usize) * factor;
} else if total_pixels < min_pixels {
// Scale up to meet min_pixels
let beta = (min_pixels as f64 / (h * w) as f64).sqrt();
h_bar = ((h as f64 * beta / factor as f64).ceil() as usize) * factor;
w_bar = ((w as f64 * beta / factor as f64).ceil() as usize) * factor;
}
Ok((h_bar, w_bar))
}
/// Load and preprocess image for PaddleOCR-VL.
fn load_image(path: &str, device: &Device, dtype: DType) -> Result<(Tensor, Tensor)> {
let img = image::ImageReader::open(path)?
.decode()
.map_err(|e| E::msg(format!("Failed to decode image: {}", e)))?;
let img = img.to_rgb8();
let (width, height) = (img.width() as usize, img.height() as usize);
// PaddleOCR-VL uses dynamic resolution with patch size 14
// Resize to be divisible by factor (patch_size * spatial_merge = 28)
// Use smart_resize to match PyTorch processor's preprocessing exactly
let patch_size = 14;
let spatial_merge = 2;
let factor = patch_size * spatial_merge; // 28
let min_pixels = 147384; // from preprocessor_config.json
let max_pixels = 2822400; // from preprocessor_config.json
// Use smart_resize to match PyTorch's preprocessing exactly
let (new_height, new_width) = smart_resize(height, width, factor, min_pixels, max_pixels)?;
// Note: PyTorch uses PIL's BICUBIC resampling which differs slightly from
// Rust's CatmullRom. This causes minor pixel differences which may cascade
// through transformer layers, but the model output remains correct.
// CatmullRom is the closest match to PIL's BICUBIC among available filters.
let resized = image::imageops::resize(
&img,
new_width as u32,
new_height as u32,
image::imageops::FilterType::CatmullRom,
);
// Normalize to [-1, 1] range (matching PyTorch processor output)
// Note: PyTorch processor outputs values in [-1, 1] range despite using CLIP mean/std
// This simpler normalization appears to match the actual output
let mut normalized = vec![0f32; 3 * new_height * new_width];
for c in 0..3 {
for y in 0..new_height {
for x in 0..new_width {
let pixel = resized.get_pixel(x as u32, y as u32);
let idx = c * new_height * new_width + y * new_width + x;
// Simple [-1, 1] normalization: 2 * (x/255) - 1
normalized[idx] = pixel[c] as f32 / 255.0 * 2.0 - 1.0;
}
}
}
// Create tensor: (1, 3, H, W)
let pixel_values =
Tensor::from_vec(normalized, (1, 3, new_height, new_width), device)?.to_dtype(dtype)?;
// Grid THW: (temporal, height_patches, width_patches)
let h_patches = (new_height / patch_size) as u32;
let w_patches = (new_width / patch_size) as u32;
let grid_thw = Tensor::new(&[[1u32, h_patches, w_patches]], device)?;
println!(
"Image: {}x{} -> {}x{} ({} x {} patches)",
width, height, new_width, new_height, h_patches, w_patches
);
Ok((pixel_values, grid_thw))
}
/// Load and preprocess video frames for PaddleOCR-VL.
///
/// Extracts frames from a video file at the specified fps and preprocesses them
/// for the vision encoder. All frames are resized to the same resolution.
///
/// # Arguments
/// * `path` - Path to video file
/// * `fps` - Target frames per second to extract
/// * `max_frames` - Maximum number of frames to extract
/// * `device` - Device for tensors
/// * `dtype` - Data type for tensors
///
/// # Returns
/// Tuple of (pixel_values, video_grid_thw) where:
/// - pixel_values: (num_patches, hidden) flattened vision patches
/// - video_grid_thw: (1, 3) = [num_frames, height_patches, width_patches]
fn load_video_frames(
path: &str,
fps: f32,
max_frames: usize,
device: &Device,
dtype: DType,
) -> Result<(Tensor, Tensor)> {
use std::process::Command;
// Create temporary directory for frames
let temp_dir = std::env::temp_dir().join(format!("paddleocr_vl_frames_{}", std::process::id()));
std::fs::create_dir_all(&temp_dir)?;
// Use ffmpeg to extract frames
let output = Command::new("ffmpeg")
.args([
"-i",
path,
"-vf",
&format!("fps={}", fps),
"-frames:v",
&max_frames.to_string(),
"-y",
&temp_dir.join("frame_%04d.png").to_string_lossy(),
])
.output()
.map_err(|e| {
E::msg(format!(
"Failed to run ffmpeg: {}. Make sure ffmpeg is installed.",
e
))
})?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
// Clean up temp directory
let _ = std::fs::remove_dir_all(&temp_dir);
return Err(E::msg(format!("ffmpeg failed: {}", stderr)));
}
// Find all extracted frames
let mut frame_paths: Vec<_> = std::fs::read_dir(&temp_dir)?
.filter_map(|e| e.ok())
.filter(|e| e.path().extension().is_some_and(|ext| ext == "png"))
.map(|e| e.path())
.collect();
frame_paths.sort();
if frame_paths.is_empty() {
let _ = std::fs::remove_dir_all(&temp_dir);
return Err(E::msg("No frames extracted from video"));
}
let num_frames = frame_paths.len();
println!("Extracted {} frames from video at {} fps", num_frames, fps);
let patch_size = 14;
let spatial_merge = 2;
let factor = patch_size * spatial_merge; // 28
let min_pixels = 147384; // from preprocessor_config.json
let max_pixels = 2822400; // from preprocessor_config.json
// Load first frame to determine dimensions
let first_img = image::ImageReader::open(&frame_paths[0])?
.decode()
.map_err(|e| E::msg(format!("Failed to decode frame: {}", e)))?;
let first_img = first_img.to_rgb8();
let (width, height) = (first_img.width() as usize, first_img.height() as usize);
// Use smart_resize to match PyTorch's preprocessing (same for all frames)
let (new_height, new_width) = smart_resize(height, width, factor, min_pixels, max_pixels)?;
let h_patches = new_height / patch_size;
let w_patches = new_width / patch_size;
println!(
"Video frames: {}x{} -> {}x{} ({} x {} patches, {} frames)",
width, height, new_width, new_height, h_patches, w_patches, num_frames
);
// Process all frames
let mut all_normalized = Vec::with_capacity(num_frames * 3 * new_height * new_width);
for (i, frame_path) in frame_paths.iter().enumerate() {
let img = image::ImageReader::open(frame_path)?
.decode()
.map_err(|e| E::msg(format!("Failed to decode frame {}: {}", i, e)))?;
let img = img.to_rgb8();
let resized = image::imageops::resize(
&img,
new_width as u32,
new_height as u32,
image::imageops::FilterType::CatmullRom,
);
// Normalize to [-1, 1] range
for c in 0..3 {
for y in 0..new_height {
for x in 0..new_width {
let pixel = resized.get_pixel(x as u32, y as u32);
all_normalized.push(pixel[c] as f32 / 255.0 * 2.0 - 1.0);
}
}
}
}
// Clean up temp directory
let _ = std::fs::remove_dir_all(&temp_dir);
// Create tensor: (num_frames, 3, H, W)
let pixel_values = Tensor::from_vec(
all_normalized,
(num_frames, 3, new_height, new_width),
device,
)?
.to_dtype(dtype)?;
// Video grid THW: (1, 3) = [temporal, height_patches, width_patches]
let video_grid_thw = Tensor::new(
&[[num_frames as u32, h_patches as u32, w_patches as u32]],
device,
)?;
Ok((pixel_values, video_grid_thw))
}
/// Build input tokens for video with proper chat format.
///
/// Format: <BOS>User: <VIDEO_START><VIDEO>×N<VIDEO_END>[task]\nAssistant:
fn build_video_input_tokens(
tokenizer: &Tokenizer,
task: Task,
num_video_tokens: usize,
video_token_id: u32,
vision_start_token_id: u32,
vision_end_token_id: u32,
device: &Device,
) -> Result<Tensor> {
// Get BOS token
let bos_token_id = tokenizer.token_to_id("<|begin_of_sentence|>").unwrap_or(1);
// Build prompt parts
let user_prefix = "User: ";
let task_text = task.prompt();
let assistant_prefix = "\nAssistant: ";
// Tokenize parts
let user_encoding = tokenizer
.encode(user_prefix, false)
.map_err(|e| E::msg(format!("Tokenization error: {}", e)))?;
let task_encoding = tokenizer
.encode(task_text, false)
.map_err(|e| E::msg(format!("Tokenization error: {}", e)))?;
let assistant_encoding = tokenizer
.encode(assistant_prefix, false)
.map_err(|e| E::msg(format!("Tokenization error: {}", e)))?;
// Build full input with VIDEO tokens
let mut input_ids: Vec<u32> = vec![bos_token_id];
input_ids.extend(user_encoding.get_ids());
input_ids.push(vision_start_token_id);
input_ids.extend(vec![video_token_id; num_video_tokens]);
input_ids.push(vision_end_token_id);
input_ids.extend(task_encoding.get_ids());
input_ids.extend(assistant_encoding.get_ids());
let tensor = Tensor::new(input_ids.as_slice(), device)?.unsqueeze(0)?;
Ok(tensor)
}
/// Build input tokens with proper chat format.
/// Format: <|begin_of_sentence|>User: <|IMAGE_START|><|IMAGE_PLACEHOLDER|>...<|IMAGE_END|>[task]\nAssistant:
fn build_input_tokens(
tokenizer: &Tokenizer,
task: Task,
num_image_tokens: usize,
image_token_id: u32,
vision_start_token_id: u32,
vision_end_token_id: u32,
device: &Device,
) -> Result<Tensor> {
// Get BOS token
let bos_token_id = tokenizer.token_to_id("<|begin_of_sentence|>").unwrap_or(1); // Default BOS
// Build prompt parts
let user_prefix = "User: ";
let task_text = task.prompt();
let assistant_prefix = "\nAssistant: ";
// Tokenize parts
let user_encoding = tokenizer
.encode(user_prefix, false)
.map_err(|e| E::msg(format!("Tokenization error: {}", e)))?;
let task_encoding = tokenizer
.encode(task_text, false)
.map_err(|e| E::msg(format!("Tokenization error: {}", e)))?;
let assistant_encoding = tokenizer
.encode(assistant_prefix, false)
.map_err(|e| E::msg(format!("Tokenization error: {}", e)))?;
// Build full input:
// <BOS> + "User: " + <IMAGE_START> + <IMAGE_PLACEHOLDER>... + <IMAGE_END> + task + "\nAssistant: "
let mut input_ids: Vec<u32> = vec![bos_token_id];
input_ids.extend(user_encoding.get_ids());
input_ids.push(vision_start_token_id);
input_ids.extend(vec![image_token_id; num_image_tokens]);
input_ids.push(vision_end_token_id);
input_ids.extend(task_encoding.get_ids());
input_ids.extend(assistant_encoding.get_ids());
let tensor = Tensor::new(input_ids.as_slice(), device)?.unsqueeze(0)?;
Ok(tensor)
}
fn main() -> Result<()> {
let args = Args::parse();
let device = hanzo_ml_examples::device(args.cpu)?;
let dtype = if args.bf16 { DType::BF16 } else { DType::F32 };
println!("Using device: {:?}, dtype: {:?}", device, dtype);
// Load model from HuggingFace
println!("Loading model from {}...", args.model_id);
let api = hanzo_ml_examples::hub::Api::new()?;
let repo = api
.model(args.model_id.clone())
.with_revision(args.revision.clone());
// Load config
let config_file = repo.get("config.json")?;
let config: Config = serde_json::from_str(&std::fs::read_to_string(&config_file)?)?;
println!(
"Vision: {}L {}H, Text: {}L {}H (GQA: {}KV)",
config.vision_config.num_hidden_layers,
config.vision_config.num_attention_heads,
config.num_hidden_layers,
config.num_attention_heads,
config.num_key_value_heads,
);
// Load tokenizer
let tokenizer_file = repo.get("tokenizer.json")?;
let tokenizer = Tokenizer::from_file(&tokenizer_file).map_err(E::msg)?;
// Load model weights
let model_file = match repo.get("model.safetensors") {
Ok(f) => f,
Err(_) => repo.get("pytorch_model.bin")?,
};
println!("Loading weights from {:?}...", model_file);
let vb = if model_file.extension().is_some_and(|ext| ext == "bin") {
VarBuilder::from_pth(&model_file, dtype, &device)?
} else {
unsafe { VarBuilder::from_mmaped_safetensors(&[&model_file], dtype, &device)? }
};
let mut model = PaddleOCRVLModel::new(&config, vb)?;
println!("Model loaded successfully");
// Validate input: either image(s), batch, or video - but not multiple
let is_video = args.video.is_some();
let is_batch = !args.batch.is_empty();
let is_image = !args.image.is_empty();
let input_count = is_video as u8 + is_batch as u8 + is_image as u8;
if input_count == 0 {
return Err(E::msg(
"Either --image, --batch, or --video must be specified",
));
}
if input_count > 1 {
return Err(E::msg(
"Cannot combine --image, --batch, and --video. Use only one input mode.",
));
}
// Handle video input separately
if is_video {
let video_path = args.video.as_ref().unwrap();
println!("Processing video: {}", video_path);
// Use frame-by-frame processing by default (works better for most use cases)
// Only use experimental video mode if --task video is specified
if args.task != Task::Video {
println!(
"Processing frames individually (similarity threshold: {})",
args.similarity_threshold
);
// Extract frames to temp directory
use std::process::Command;
let temp_dir =
std::env::temp_dir().join(format!("paddleocr_vl_frames_{}", std::process::id()));
std::fs::create_dir_all(&temp_dir)?;
let output = Command::new("ffmpeg")
.args([
"-i",
video_path,
"-vf",
&format!("fps={}", args.fps),
"-frames:v",
&args.max_frames.to_string(),
"-y",
&temp_dir.join("frame_%04d.png").to_string_lossy(),
])
.output()
.map_err(|e| {
E::msg(format!(
"Failed to run ffmpeg: {}. Make sure ffmpeg is installed.",
e
))
})?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
let _ = std::fs::remove_dir_all(&temp_dir);
return Err(E::msg(format!("ffmpeg failed: {}", stderr)));
}
// Find all extracted frames
let mut frame_paths: Vec<_> = std::fs::read_dir(&temp_dir)?
.filter_map(|e| e.ok())
.filter(|e| e.path().extension().is_some_and(|ext| ext == "png"))
.map(|e| e.path())
.collect();
frame_paths.sort();
if frame_paths.is_empty() {
let _ = std::fs::remove_dir_all(&temp_dir);
return Err(E::msg("No frames extracted from video"));
}
println!("Extracted {} frames at {} fps", frame_paths.len(), args.fps);
// Get EOS token ID
let eos_token_id = tokenizer
.token_to_id("</s>")
.or_else(|| tokenizer.token_to_id("<|end_of_sentence|>"))
.or_else(|| tokenizer.token_to_id("<|endoftext|>"))
.unwrap_or(2);
// Process each frame individually
let mut results: Vec<FrameOcrResult> = Vec::new();
let mut prev_text = String::new();
for (frame_idx, frame_path) in frame_paths.iter().enumerate() {
let timestamp = frame_idx as f32 / args.fps;
print!(
"\rProcessing frame {}/{} (t={:.1}s)...",
frame_idx + 1,
frame_paths.len(),
timestamp
);
std::io::Write::flush(&mut std::io::stdout())?;
// Load frame as single image
let frame_path_str = frame_path.to_string_lossy().to_string();
let (pixel_values, grid_thw) = load_image(&frame_path_str, &device, dtype)?;
// Build input tokens for this frame
let grid_thw_vec: Vec<Vec<u32>> = grid_thw.to_vec2()?;
let g = &grid_thw_vec[0];
let spatial_merge_size = 2;
let num_image_tokens =
(g[1] as usize / spatial_merge_size) * (g[2] as usize / spatial_merge_size);
let input_ids = build_input_tokens(
&tokenizer,
args.task,
num_image_tokens,
config.image_token_id,
config.vision_start_token_id,
config.vision_end_token_id,
&device,
)?;
// Clear KV cache for fresh generation
model.clear_kv_cache();
// Generate text for this frame
let generated_tokens = model.generate(
&input_ids,
&pixel_values,
&grid_thw,
args.max_length,
eos_token_id,
)?;
// Decode text
let output_tokens: Vec<u32> = generated_tokens
.into_iter()
.take_while(|&t| t != eos_token_id)
.collect();
let text = tokenizer.decode(&output_tokens, true).unwrap_or_default();
let text = text.trim().to_string();
// Skip empty text and hallucinations
if text.is_empty() || is_hallucination(&text) {
continue;
}
// Check similarity with previous text
let similarity = string_similarity(&text, &prev_text);
if similarity < args.similarity_threshold {
// Text is sufficiently different - record it
results.push(FrameOcrResult {
frame_index: frame_idx,
timestamp,
text: text.clone(),
});
prev_text = text;
}
}
// Clean up temp directory
let _ = std::fs::remove_dir_all(&temp_dir);
// Output results
println!("\n\n{:=<60}", "");
println!(
"Frame-by-Frame OCR Results ({} unique text segments):",
results.len()
);
println!("{:=<60}", "");
for result in &results {
println!(
"[{:.1}s] Frame {}: {}",
result.timestamp, result.frame_index, result.text
);
}
println!("{:=<60}\n", "");
// Also output combined text
if !results.is_empty() {
println!("Combined text:");
println!("{:-<60}", "");
for result in &results {
println!("{}", result.text);
}
println!("{:-<60}\n", "");
}
return Ok(());
}
// Experimental video mode (--task video)
// Processes all frames as a single video sequence with temporal position encoding
println!("Using experimental video mode (--task video)");
// Load video frames
let (pixel_values_video, video_grid_thw) =
load_video_frames(video_path, args.fps, args.max_frames, &device, dtype)?;
// Compute number of video tokens (after spatial merge)
let grid_thw_vec: Vec<Vec<u32>> = video_grid_thw.to_vec2()?;
let g = &grid_thw_vec[0];
let spatial_merge_size = 2;
let num_video_tokens = (g[0] as usize)
* (g[1] as usize / spatial_merge_size)
* (g[2] as usize / spatial_merge_size);
println!(
"Video tokens: {} ({}t x {}h x {}w after merge)",
num_video_tokens,
g[0],
g[1] as usize / spatial_merge_size,
g[2] as usize / spatial_merge_size
);
// Build input tokens for video
let input_ids = build_video_input_tokens(
&tokenizer,
args.task,
num_video_tokens,
config.video_token_id,
config.vision_start_token_id,
config.vision_end_token_id,
&device,
)?;
println!("Input sequence length: {}", input_ids.dim(1)?);
println!("Task: {:?}", args.task);
println!("\nGenerating (max {} tokens)...", args.max_length);
// Get EOS token ID (same as image generation path)
let eos_token_id = tokenizer
.token_to_id("</s>")
.or_else(|| tokenizer.token_to_id("<|end_of_sentence|>"))
.or_else(|| tokenizer.token_to_id("<|endoftext|>"))
.unwrap_or(2);
// Generate using video method
let generated_tokens = model.generate_video(
&input_ids,
&pixel_values_video,
&video_grid_thw,
args.fps,
args.max_length,
eos_token_id,
)?;
// Debug: print generated tokens
println!("Generated {} tokens:", generated_tokens.len());
for (i, &tok) in generated_tokens.iter().enumerate().take(50) {
let tok_str = tokenizer
.decode(&[tok], true)
.unwrap_or_else(|_| format!("<{}>", tok));
println!(" {}: {} = '{}'", i, tok, tok_str);
}
if generated_tokens.len() > 50 {
println!(" ... ({} more tokens)", generated_tokens.len() - 50);
}
// Filter out any trailing tokens after EOS (shouldn't happen, but safety check)
let output_tokens: Vec<u32> = generated_tokens
.into_iter()
.take_while(|&t| t != eos_token_id)
.collect();
let output_text = tokenizer.decode(&output_tokens, true).map_err(E::msg)?;
println!("\n{:=<60}", "");
println!("Video Recognition Result:");
println!("{:=<60}", "");
println!("{}", output_text);
println!("{:=<60}\n", "");
return Ok(());
}
// Handle batch mode - process multiple images sequentially
if is_batch {
println!(
"Batch mode: processing {} images sequentially...",
args.batch.len()
);
println!("{:=<60}\n", "");
// Get EOS token ID
let eos_token_id = tokenizer
.token_to_id("</s>")
.or_else(|| tokenizer.token_to_id("<|end_of_sentence|>"))
.or_else(|| tokenizer.token_to_id("<|endoftext|>"))
.unwrap_or(2);
let spatial_merge = config.vision_config.spatial_merge_size;
let total_start = std::time::Instant::now();
let mut total_tokens = 0usize;
let mut successful = 0usize;
let mut failed = 0usize;
for (idx, image_path) in args.batch.iter().enumerate() {
println!(
"[{}/{}] Processing: {}",
idx + 1,
args.batch.len(),
image_path
);
// Load and preprocess this image
let result = (|| -> Result<(String, usize, std::time::Duration)> {
let (pixel_values, grid_thw) = load_image(image_path, &device, dtype)?;
// Calculate number of image tokens after spatial merge
let grid_vec = grid_thw.to_vec2::<u32>()?;
let g = &grid_vec[0];
let h_patches = g[1] as usize;
let w_patches = g[2] as usize;
let num_image_tokens = (h_patches / spatial_merge) * (w_patches / spatial_merge);
// Build input tokens for this single image
let input_ids = build_input_tokens(
&tokenizer,
args.task,
num_image_tokens,
config.image_token_id,
config.vision_start_token_id,
config.vision_end_token_id,
&device,
)?;
// Clear KV cache for fresh generation
model.clear_kv_cache();
// Generate output
let start = std::time::Instant::now();
let generated_tokens = model.generate(
&input_ids,
&pixel_values,
&grid_thw,
args.max_length,
eos_token_id,
)?;
let elapsed = start.elapsed();
// Decode tokens
let output_text = tokenizer
.decode(&generated_tokens, true)
.map_err(|e| E::msg(format!("Decoding error: {}", e)))?;
Ok((
output_text.trim().to_string(),
generated_tokens.len(),
elapsed,
))
})();
match result {
Ok((text, tokens, elapsed)) => {
println!(" └─ {} tokens in {:.2}s", tokens, elapsed.as_secs_f32());
println!("{:-<60}", "");
println!("{}", text);
println!("{:-<60}\n", "");
total_tokens += tokens;
successful += 1;
}
Err(e) => {
println!(" └─ Error: {}", e);
println!();
failed += 1;
}
}
}
let total_elapsed = total_start.elapsed();
println!("{:=<60}", "");
println!("Batch Summary:");
println!(
" Images processed: {} successful, {} failed",
successful, failed
);
println!(
" Total tokens: {} in {:.2}s ({:.1} tokens/sec)",
total_tokens,
total_elapsed.as_secs_f32(),
total_tokens as f32 / total_elapsed.as_secs_f32()
);
println!("{:=<60}", "");
return Ok(());
}
// Image processing path
let is_multi_image = args.image.len() > 1;
// Get EOS token ID
let eos_token_id = tokenizer
.token_to_id("</s>")
.or_else(|| tokenizer.token_to_id("<|end_of_sentence|>"))
.or_else(|| tokenizer.token_to_id("<|endoftext|>"))
.unwrap_or(2);
let spatial_merge = config.vision_config.spatial_merge_size;
// Multi-image: Process each image sequentially (like official PaddleOCR-VL)
// The model's attention is optimized for single-image input, so we process
// each image independently and concatenate the text outputs.
if is_multi_image {
println!(
"Multi-page mode: Processing {} images sequentially...",
args.image.len()
);
println!("{:=<60}\n", "");
let total_start = std::time::Instant::now();
let mut all_results: Vec<String> = Vec::new();
let mut total_tokens = 0usize;
for (idx, image_path) in args.image.iter().enumerate() {
println!(
"[Page {}/{}] Processing: {}",
idx + 1,
args.image.len(),
image_path
);
// Load and preprocess this image
let (pixel_values, grid_thw) = load_image(image_path, &device, dtype)?;
// Calculate number of image tokens after spatial merge
let grid_vec = grid_thw.to_vec2::<u32>()?;
let g = &grid_vec[0];
let h_patches = g[1] as usize;
let w_patches = g[2] as usize;
let num_image_tokens = (h_patches / spatial_merge) * (w_patches / spatial_merge);
// Build input tokens for this single image
let input_ids = build_input_tokens(
&tokenizer,
args.task,
num_image_tokens,
config.image_token_id,
config.vision_start_token_id,
config.vision_end_token_id,
&device,
)?;
// Clear KV cache for fresh generation
model.clear_kv_cache();
// Generate output
let start = std::time::Instant::now();
let generated_tokens = model.generate(
&input_ids,
&pixel_values,
&grid_thw,
args.max_length,
eos_token_id,
)?;
let elapsed = start.elapsed();
// Decode tokens
let output_text = tokenizer
.decode(&generated_tokens, true)
.map_err(|e| E::msg(format!("Decoding error: {}", e)))?;
let text = output_text.trim().to_string();
println!(
" └─ {} tokens in {:.2}s",
generated_tokens.len(),
elapsed.as_secs_f32()
);
println!("{:-<60}", "");
println!("{}", text);
println!("{:-<60}\n", "");
all_results.push(text);
total_tokens += generated_tokens.len();
}
let total_elapsed = total_start.elapsed();
// Print combined output
println!("{:=<60}", "");
println!(
"Combined {} Output ({} pages):",
args.task.prompt(),
args.image.len()
);
println!("{:=<60}", "");
for (idx, result) in all_results.iter().enumerate() {
if idx > 0 {
println!("\n--- Page {} ---\n", idx + 1);
}
println!("{}", result);
}
println!("{:=<60}", "");
println!(
"Total: {} tokens in {:.2}s ({:.1} tokens/sec)",
total_tokens,
total_elapsed.as_secs_f32(),
total_tokens as f32 / total_elapsed.as_secs_f32()
);
return Ok(());
}
// Single image processing path
println!("Processing image: {}", args.image[0]);
let (pixel_values, grid_thw) = load_image(&args.image[0], &device, dtype)?;
// Calculate number of image tokens after spatial merge
let grid_vec = grid_thw.to_vec2::<u32>()?;
let g = &grid_vec[0];
let num_image_tokens = (g[1] as usize / spatial_merge) * (g[2] as usize / spatial_merge);
println!(
"Image tokens: {} (after {}x{} merge)",
num_image_tokens, spatial_merge, spatial_merge
);
// Build input tokens
let input_ids = build_input_tokens(
&tokenizer,
args.task,
num_image_tokens,
config.image_token_id,
config.vision_start_token_id,
config.vision_end_token_id,
&device,
)?;
println!("Input shape: {:?}", input_ids.dims());
// Generate output
println!(
"Generating {} output (max_length={})...",
args.task.prompt(),
args.max_length
);
let start = std::time::Instant::now();
let generated_tokens = model.generate(
&input_ids,
&pixel_values,
&grid_thw,
args.max_length,
eos_token_id,
)?;
let elapsed = start.elapsed();
// Decode tokens
let output_text = tokenizer
.decode(&generated_tokens, true)
.map_err(|e| E::msg(format!("Decoding error: {}", e)))?;
println!("\n{:=<60}", "");
println!("Task: {:?}", args.task);
println!("{:=<60}", "");
println!("{}", output_text.trim());
println!("{:=<60}", "");
println!(
"Generated {} tokens in {:.2}s ({:.1} tokens/sec)",
generated_tokens.len(),
elapsed.as_secs_f32(),
generated_tokens.len() as f32 / elapsed.as_secs_f32()
);
Ok(())
}