Patch: fix dependency/runtime fallbacks for OCR models

app.py

@@ -17,8 +17,18 @@ from PIL import Image
 
 try:
     from transformers import pipeline as hf_pipeline
+    from transformers import (
+        AutoModelForCausalLM,
+        AutoModelForImageTextToText,
+        AutoModelForVision2Seq,
+        AutoProcessor,
+    )
 except Exception:  # pragma: no cover
     hf_pipeline = None  # pragma: no cover
+    AutoModelForCausalLM = None  # pragma: no cover
+    AutoModelForImageTextToText = None  # pragma: no cover
+    AutoModelForVision2Seq = None  # pragma: no cover
+    AutoProcessor = None  # pragma: no cover
 
 try:
     import torch
@@ -128,6 +138,7 @@ ZERO_GPU_MIN_GUARDED_FREE_GB = 1.0
 ZERO_GPU_MODEL_FALLBACK_GB_SAFE = 2.6
 ZERO_GPU_MODEL_FALLBACK_GB_MAX = 2.0
 DEFAULT_MAX_TOKENS = 4096
+DIRECT_MODEL_CACHE_MAX = 1
 LOCAL_PIPELINE_CACHE_MAX = 1
 TESSERACT_EXECUTABLE_PATHS = ("/usr/bin/tesseract", "/usr/local/bin/tesseract", "/opt/conda/bin/tesseract", "/bin/tesseract")
 
@@ -155,6 +166,16 @@ TRUST_REMOTE_CODE_MODELS = {
 TRUST_REMOTE_CODE_MODELS = {m.lower() for m in TRUST_REMOTE_CODE_MODELS}
 
 PIPELINE_TASK_HINTS = {
+    "qwen/qwen3-vl-8b-instruct": "image-to-text",
+    "qwen/qwen3-vl-4b-instruct": "image-to-text",
+    "qwen/qwen3-vl-8b-thinking": "image-to-text",
+    "qwen/qwen3-vl-4b-thinking": "image-to-text",
+    "qwen/qwen3-vl-30b-a3b-instruct": "image-to-text",
+    "qwen/qwen3-vl-30b-a3b-thinking": "image-to-text",
+    "google/gemma-4-e4b-it": "image-to-text",
+    "google/gemma-4-12b-it": "image-to-text",
+    "google/gemma-4-26b-a4b-it": "image-to-text",
+    "google/gemma-4-31b-it": "image-to-text",
     "ronylicha/gigapdf-ocr-hebrew": "image-to-text",
     "liskcell/qunie-v7-mini": "image-to-text",
     "0cve0/openmlkitocr": "image-to-text",
@@ -180,7 +201,9 @@ GATED_MODELS_REQUIRING_TOKEN = {
 GATED_MODELS_REQUIRING_TOKEN = {m.lower() for m in GATED_MODELS_REQUIRING_TOKEN}
 
 _LOCAL_PIPELINE_CACHE: OrderedDict = OrderedDict()
+_LOCAL_DIRECT_CACHE: OrderedDict = OrderedDict()
 _LOCAL_PIPELINE_LOCK = threading.Lock()
+_LOCAL_DIRECT_LOCK = threading.Lock()
 
 
 def load_registry() -> List[dict]:
@@ -377,6 +400,110 @@ def _build_local_pipeline(model_id: str, hf_token: str, task: str, trust_remote_
     raise RuntimeError(f"Failed to load local pipeline for {model_id}: {last_error}")
 
 
+def _load_direct_components(model_id: str, hf_token: str, trust_remote_code: bool = False):
+    if AutoProcessor is None:
+        return None, None
+    token = _sanitize_hf_token(hf_token) or None
+    component = (model_id, trust_remote_code)
+    with _LOCAL_DIRECT_LOCK:
+        cached = _LOCAL_DIRECT_CACHE.get(component)
+        if cached is not None:
+            _LOCAL_DIRECT_CACHE.move_to_end(component)
+            return cached
+
+    processor = AutoProcessor.from_pretrained(model_id, token=token, trust_remote_code=trust_remote_code)
+
+    if torch is None:
+        raise RuntimeError("PyTorch is required for direct model loading.")
+
+    model_errors = []
+    model = None
+    for model_class in [AutoModelForImageTextToText, AutoModelForVision2Seq, AutoModelForCausalLM]:
+        if model_class is None:
+            continue
+        try:
+            model = model_class.from_pretrained(
+                model_id,
+                token=token,
+                trust_remote_code=trust_remote_code,
+                torch_dtype=getattr(torch, "float16", None),
+            )
+            break
+        except Exception as exc:  # pragma: no cover
+            model_errors.append(f"{getattr(model_class, '__name__', str(model_class))}: {exc}")
+
+    if model is None:
+        raise RuntimeError(f"Direct loading failed for {model_id}. " + " | ".join(model_errors))
+
+    if torch.cuda.is_available():
+        model = model.to("cuda")
+
+    with _LOCAL_DIRECT_LOCK:
+        _LOCAL_DIRECT_CACHE[component] = (model, processor)
+        if len(_LOCAL_DIRECT_CACHE) > DIRECT_MODEL_CACHE_MAX:
+            _LOCAL_DIRECT_CACHE.popitem(last=False)
+    return model, processor
+
+
+def _direct_infer(model_id: str, image_bytes: bytes, prompt: str, hf_token: str, params: dict, trust_remote_code: bool = False) -> str:
+    if torch is None:
+        raise RuntimeError("PyTorch is required for direct model inference.")
+    model, processor = _load_direct_components(model_id, hf_token, trust_remote_code=trust_remote_code)
+    image = _decode_image(image_bytes)
+
+    input_candidates = []
+    if prompt:
+        input_candidates.extend(
+            [
+                lambda: processor(text=prompt, images=image, return_tensors="pt"),
+                lambda: processor(images=image, text=prompt, return_tensors="pt"),
+                lambda: processor(prompt, image, return_tensors="pt"),
+                lambda: processor(prompt, return_tensors="pt"),
+            ]
+        )
+    input_candidates.append(lambda: processor(images=image, return_tensors="pt"))
+
+    prepared_inputs = None
+    prep_error = None
+    for builder in input_candidates:
+        try:
+            candidate = builder()
+            if isinstance(candidate, Mapping) and candidate:
+                prepared_inputs = {
+                    k: (v.to(model.device) if hasattr(v, "to") else v) for k, v in candidate.items()
+                }
+                break
+        except Exception as exc:  # pragma: no cover
+            prep_error = exc
+
+    if prepared_inputs is None:
+        raise RuntimeError(f"Could not prepare inputs for {model_id}: {prep_error}")
+
+    gen_kwargs = {}
+    if "max_new_tokens" in params:
+        gen_kwargs["max_new_tokens"] = params["max_new_tokens"]
+    if "temperature" in params:
+        gen_kwargs["temperature"] = params["temperature"]
+    if "top_p" in params:
+        gen_kwargs["top_p"] = params["top_p"]
+    if "top_k" in params:
+        gen_kwargs["top_k"] = params["top_k"]
+    if "do_sample" in params:
+        gen_kwargs["do_sample"] = params["do_sample"]
+    if "num_beams" in params:
+        gen_kwargs["num_beams"] = params["num_beams"]
+
+    with torch.no_grad():
+        generated = model.generate(**prepared_inputs, **gen_kwargs)
+
+    if isinstance(generated, torch.Tensor):
+        decoded = processor.batch_decode(generated, skip_special_tokens=True)
+        return parse_output(decoded)
+    if isinstance(generated, (list, tuple)):
+        return parse_output(generated)
+    return parse_output(str(generated))
+
+
 def _get_local_pipeline(model_id: str, hf_token: str, task: str, trust_remote_code: bool = False):
     device = "gpu" if (torch is not None and torch.cuda.is_available()) else "cpu"
     key = _pipeline_cache_key(model_id, task or "auto", device, trust_remote_code)
@@ -402,6 +529,7 @@ def _get_local_pipeline(model_id: str, hf_token: str, task: str, trust_remote_co
             if torch is not None and torch.cuda.is_available():
                 torch.cuda.empty_cache()
             gc.collect()
+    gc.collect()
     return pipeline
 
 
@@ -425,6 +553,21 @@ def _clear_local_pipeline_cache() -> None:
         torch.cuda.empty_cache()
 
 
+def _clear_direct_cache() -> None:
+    global _LOCAL_DIRECT_CACHE
+    with _LOCAL_DIRECT_LOCK:
+        directs = list(_LOCAL_DIRECT_CACHE.values())
+        _LOCAL_DIRECT_CACHE.clear()
+    for model, _ in directs:
+        try:
+            del model
+        except Exception:
+            pass
+    gc.collect()
+    if torch is not None and torch.cuda.is_available():
+        torch.cuda.empty_cache()
+
+
 def run_local_model(entry: dict, image_bytes: bytes, prompt: str, hf_token: str) -> str:
     if hf_pipeline is None:
         raise RuntimeError("transformers is not installed in this space.")
@@ -434,7 +577,13 @@ def run_local_model(entry: dict, image_bytes: bytes, prompt: str, hf_token: str)
     model_id = entry["model_id"]
     task = _model_pipeline_task(entry)
     trust_remote_code = _model_requires_trust_remote_code(entry)
-    pipeline = _get_local_pipeline(model_id, hf_token, task, trust_remote_code=trust_remote_code)
+    try:
+        pipeline = _get_local_pipeline(model_id, hf_token, task, trust_remote_code=trust_remote_code)
+    except Exception as exc:
+        pipeline = None
+        pipeline_error = exc
+    else:
+        pipeline_error = None
     params = sanitize_generation_params(entry.get("parameters", {}))
     image = _decode_image(image_bytes)
 
@@ -460,34 +609,53 @@ def run_local_model(entry: dict, image_bytes: bytes, prompt: str, hf_token: str)
                 if isinstance(key, str) and key in supported_body_keys:
                     params.setdefault(key, value)
 
-    calls = []
-    if prompt:
-        calls.extend(
-            [
-                lambda: pipeline({"image": image, "text": prompt}, **params),
-                lambda: pipeline({"image": image, "question": prompt}, **params),
-                lambda: pipeline({"text": prompt, "image": image}, **params),
-                lambda: pipeline({"images": image, "text": prompt}, **params),
-                lambda: pipeline(image, text=prompt, **params),
-                lambda: pipeline(image, question=prompt, **params),
-                lambda: pipeline(image, **params),
-                lambda: pipeline({"image": image}, **params),
-            ]
+    if pipeline is not None:
+        calls = []
+        if prompt:
+            calls.extend(
+                [
+                    lambda: pipeline({"image": image, "text": prompt}, **params),
+                    lambda: pipeline({"image": image, "question": prompt}, **params),
+                    lambda: pipeline({"text": prompt, "image": image}, **params),
+                    lambda: pipeline({"images": image, "text": prompt}, **params),
+                    lambda: pipeline(image, text=prompt, **params),
+                    lambda: pipeline(image, question=prompt, **params),
+                    lambda: pipeline(image, **params),
+                    lambda: pipeline({"image": image}, **params),
+                ]
+            )
+        else:
+            calls.extend([lambda: pipeline(image, **params), lambda: pipeline({"image": image}, **params)])
+
+        for call in calls:
+            try:
+                output = call()
+                parsed = parse_output(output)
+                if parsed:
+                    return parsed
+            except Exception as exc:  # pragma: no cover
+                pipeline_error = exc
+                continue
+
+    try:
+        direct_output = _direct_infer(
+            model_id,
+            image_bytes,
+            prompt,
+            hf_token,
+            params,
+            trust_remote_code=trust_remote_code,
         )
-    else:
-        calls.extend([lambda: pipeline(image, **params), lambda: pipeline({"image": image}, **params)])
+        parsed = parse_output(direct_output)
+        if parsed:
+            return parsed
+    except Exception as exc:  # pragma: no cover
+        if pipeline_error is None:
+            pipeline_error = exc
+        else:
+            pipeline_error = RuntimeError(f"{pipeline_error}; direct fallback failed: {exc}")
 
-    last_error = None
-    for call in calls:
-        try:
-            output = call()
-            parsed = parse_output(output)
-            if parsed:
-                return parsed
-        except Exception as exc:  # pragma: no cover
-            last_error = exc
-            continue
-    raise RuntimeError(f"Local model inference failed for {model_id}: {last_error}")
+    raise RuntimeError(f"Local model inference failed for {model_id}: {pipeline_error}")
 
 
 def run_tesseract(image_bytes: bytes, params: dict) -> str:
@@ -838,15 +1006,16 @@ def run_comparison(
                 continue
             futures_map[executor.submit(run_single_model, entry, image_bytes, "", hf_token)] = entry
 
-        for future in as_completed(futures_map):
-            result = future.result()
-            if ground_truth_text:
-                cer, wer = compute_cer_wer(ground_truth_text, result.output)
-                result.cer = cer
-                result.wer = wer
-            results.append(result)
+            for future in as_completed(futures_map):
+                result = future.result()
+                if ground_truth_text:
+                    cer, wer = compute_cer_wer(ground_truth_text, result.output)
+                    result.cer = cer
+                    result.wer = wer
+                results.append(result)
 
     _clear_local_pipeline_cache()
+    _clear_direct_cache()
 
     order_map = {model["id"]: idx for idx, model in enumerate(models)}
     results.sort(key=lambda r: order_map.get(r.model_id, 1_000_000))

apt.txt

@@ -1,3 +1,4 @@
 tesseract-ocr
 tesseract-ocr-eng
 tesseract-ocr-heb
+libgl1

model_registry.json

@@ -5,6 +5,7 @@
       "name": "Qwen3-VL-8B-Instruct",
       "model_id": "Qwen/Qwen3-VL-8B-Instruct",
       "provider": "local_transformer",
+      "pipeline_task": "image-to-text",
       "enabled": true,
       "prompt": "Transcribe every piece of printed and handwritten text from this document image. Keep paragraph/line layout when obvious. Return plain text only.",
       "parameters": {
@@ -20,6 +21,7 @@
       "name": "Qwen3-VL-4B-Instruct",
       "model_id": "Qwen/Qwen3-VL-4B-Instruct",
       "provider": "local_transformer",
+      "pipeline_task": "image-to-text",
       "enabled": true,
       "prompt": "Transcribe every piece of printed and handwritten text from this Hebrew document image. Keep line breaks and text order.",
       "parameters": {
@@ -35,6 +37,7 @@
       "name": "Qwen3-VL-8B-Thinking",
       "model_id": "Qwen/Qwen3-VL-8B-Thinking",
       "provider": "local_transformer",
+      "pipeline_task": "image-to-text",
       "enabled": true,
       "prompt": "Use deliberate reasoning to first recover layout and then output final OCR text exactly as visible. Hide reasoning and return only plain text.",
       "parameters": {
@@ -52,6 +55,7 @@
       "name": "Qwen3-VL-4B-Thinking",
       "model_id": "Qwen/Qwen3-VL-4B-Thinking",
       "provider": "local_transformer",
+      "pipeline_task": "image-to-text",
       "enabled": true,
       "prompt": "Use a deliberate OCR pass for printed and handwritten Hebrew text, then output only the final transcription.",
       "parameters": {
@@ -69,6 +73,7 @@
       "name": "Qwen3-VL-30B-A3B-Instruct",
       "model_id": "Qwen/Qwen3-VL-30B-A3B-Instruct",
       "provider": "local_transformer",
+      "pipeline_task": "image-to-text",
       "enabled": true,
       "prompt": "OCR this Hebrew document image and return plain text only, preserving order, lines and spacing.",
       "parameters": {
@@ -83,6 +88,7 @@
       "name": "Qwen3-VL-30B-A3B-Thinking",
       "model_id": "Qwen/Qwen3-VL-30B-A3B-Thinking",
       "provider": "local_transformer",
+      "pipeline_task": "image-to-text",
       "enabled": true,
       "prompt": "Think through layout and character ambiguity first, then output final OCR text only.",
       "parameters": {
@@ -238,6 +244,8 @@
       "name": "Chandra2 (datalab-to/chandra-ocr-2)",
       "model_id": "datalab-to/chandra-ocr-2",
       "provider": "local_transformer",
+      "pipeline_task": "image-to-text",
+      "trust_remote_code": true,
       "enabled": true,
       "prompt": "Output only the exact text from the uploaded document, including handwritten and printed parts.",
       "parameters": {
@@ -251,6 +259,8 @@
       "name": "ronylicha/gigapdf-ocr-hebrew",
       "model_id": "ronylicha/gigapdf-ocr-hebrew",
       "provider": "local_transformer",
+      "pipeline_task": "image-to-text",
+      "trust_remote_code": true,
       "enabled": true,
       "prompt": "Read the Hebrew document image and return only transcribed text.",
       "parameters": {
@@ -264,6 +274,8 @@
       "name": "Qunie-V7-mini",
       "model_id": "liskcell/Qunie-V7-mini",
       "provider": "local_transformer",
+      "pipeline_task": "image-to-text",
+      "trust_remote_code": true,
       "enabled": true,
       "prompt": "Strict OCR extraction. Output line-by-line text from the whole image.",
       "parameters": {
@@ -278,6 +290,8 @@
       "name": "OpenMLKitOCR",
       "model_id": "0cve0/OpenMLKitOCR",
       "provider": "local_transformer",
+      "pipeline_task": "image-to-text",
+      "trust_remote_code": true,
       "enabled": true,
       "prompt": "OCR the document and return readable text as-is.",
       "parameters": {
@@ -291,6 +305,8 @@
       "name": "Tzefa-Word-OCR-TrOCR",
       "model_id": "WARAJA/Tzefa-Word-OCR-TrOCR",
       "provider": "local_transformer",
+      "pipeline_task": "image-to-text",
+      "trust_remote_code": true,
       "enabled": true,
       "prompt": "Do OCR on this image. Return only transcribed words in reading order.",
       "parameters": {
@@ -304,6 +320,8 @@
       "name": "Qunie-V7-Pico",
       "model_id": "liskcell/Qunie-V7-Pico",
       "provider": "local_transformer",
+      "pipeline_task": "image-to-text",
+      "trust_remote_code": true,
       "enabled": true,
       "prompt": "Extract all readable text. Prioritize transcription correctness over grammar cleanup.",
       "parameters": {
@@ -318,6 +336,7 @@
       "name": "Aya Vision 32B",
       "model_id": "CohereLabs/aya-vision-32b",
       "provider": "local_transformer",
+      "pipeline_task": "image-to-text",
       "enabled": true,
       "prompt": "Return plain OCR output only from the image, including punctuation and line breaks.",
       "parameters": {
@@ -332,6 +351,7 @@
       "name": "Aya Vision 8B",
       "model_id": "CohereLabs/aya-vision-8b",
       "provider": "local_transformer",
+      "pipeline_task": "image-to-text",
       "enabled": true,
       "prompt": "Return only text from the document image; preserve paragraph breaks where clear.",
       "parameters": {
@@ -346,6 +366,8 @@
       "name": "Phi-4-Multimodal-Instruct",
       "model_id": "microsoft/Phi-4-multimodal-instruct",
       "provider": "local_transformer",
+      "pipeline_task": "image-text-to-text",
+      "trust_remote_code": true,
       "enabled": true,
       "prompt": "OCR-only transcription. Output exactly what is printed/handwritten in the image.",
       "parameters": {
@@ -360,6 +382,8 @@
       "name": "surya2 (datalab-to/surya-ocr-2)",
       "model_id": "datalab-to/surya-ocr-2",
       "provider": "local_transformer",
+      "pipeline_task": "image-to-text",
+      "trust_remote_code": true,
       "enabled": true,
       "prompt": "Return only OCR output from this document image, preserving line breaks and order.",
       "parameters": {
@@ -374,6 +398,8 @@
       "name": "PaddleOCR-VL",
       "model_id": "PaddlePaddle/PaddleOCR-VL",
       "provider": "local_transformer",
+      "pipeline_task": "image-to-text",
+      "trust_remote_code": true,
       "enabled": true,
       "prompt": "Transcribe all readable printed and handwritten text from this document image in order.",
       "parameters": {
@@ -387,6 +413,8 @@
       "name": "PaddleOCR-VL 1.5",
       "model_id": "PaddlePaddle/PaddleOCR-VL-1.5",
       "provider": "local_transformer",
+      "pipeline_task": "image-to-text",
+      "trust_remote_code": true,
       "enabled": true,
       "prompt": "Transcribe all readable printed and handwritten text from this document image in order.",
       "parameters": {
@@ -400,6 +428,8 @@
       "name": "PaddleOCR-VL 1.6",
       "model_id": "PaddlePaddle/PaddleOCR-VL-1.6",
       "provider": "local_transformer",
+      "pipeline_task": "image-to-text",
+      "trust_remote_code": true,
       "enabled": true,
       "prompt": "Transcribe all readable printed and handwritten text from this document image in order.",
       "parameters": {
@@ -413,6 +443,8 @@
       "name": "DeepSeek OCR-1",
       "model_id": "deepseek-ai/DeepSeek-OCR",
       "provider": "local_transformer",
+      "pipeline_task": "image-to-text",
+      "trust_remote_code": true,
       "enabled": true,
       "prompt": "Transcribe all printed and handwritten text from this Hebrew document image. Output only plain text.",
       "parameters": {
@@ -426,6 +458,8 @@
       "name": "DeepSeek OCR-2",
       "model_id": "deepseek-ai/DeepSeek-OCR-2",
       "provider": "local_transformer",
+      "pipeline_task": "image-to-text",
+      "trust_remote_code": true,
       "enabled": true,
       "prompt": "Transcribe all printed and handwritten text from this Hebrew document image. Preserve line breaks as visible.",
       "parameters": {
@@ -454,6 +488,8 @@
       "name": "cyttic/exp10-trocr-hebrew-matan-full",
       "model_id": "cyttic/exp10-trocr-hebrew-matan-full",
       "provider": "local_transformer",
+      "pipeline_task": "image-to-text",
+      "trust_remote_code": true,
       "enabled": true,
       "prompt": "Transcribe all readable printed and handwritten Hebrew text from this document image. Return plain text only, preserve line breaks and spacing.",
       "parameters": {
@@ -468,6 +504,8 @@
       "name": "cyttic/exp23-directfit-unfrozen",
       "model_id": "cyttic/exp23-directfit-unfrozen",
       "provider": "local_transformer",
+      "pipeline_task": "image-to-text",
+      "trust_remote_code": true,
       "enabled": true,
       "prompt": "OCR this document image and extract all printed and handwritten Hebrew text faithfully. Return only the plain transcription.",
       "parameters": {
@@ -482,6 +520,8 @@
       "name": "cyttic/heb-verifier17-connected",
       "model_id": "cyttic/heb-verifier17-connected",
       "provider": "local_transformer",
+      "pipeline_task": "image-to-text",
+      "trust_remote_code": true,
       "enabled": true,
       "prompt": "Return only the transcribed text from the uploaded document image (Hebrew document with printed and handwritten text). Preserve line order.",
       "parameters": {
@@ -496,6 +536,8 @@
       "name": "cyttic/exp26-composed1m",
       "model_id": "cyttic/exp26-composed1m",
       "provider": "local_transformer",
+      "pipeline_task": "image-to-text",
+      "trust_remote_code": true,
       "enabled": true,
       "prompt": "Transcribe exactly what is visible in the image, including Hebrew text lines and mixed-direction fragments.",
       "parameters": {

requirements.txt

@@ -3,7 +3,9 @@ huggingface-hub>=0.25.0,<1.0.0
 requests>=2.32.3
 Pillow>=10.4.0
 pytesseract>=0.3.13
-transformers>=4.57.0
+transformers>=5.14.1,<6.0.0
+addict
+torchvision
 qwen-vl-utils>=0.0.8
 tokenizers>=0.20.0
 sentencepiece>=0.1.99