Upload folder using huggingface_hub

.eval_results/olmocrbench.yaml

@@ -0,0 +1,80 @@
+- dataset:
+    id: allenai/olmOCR-bench
+    task_id: overall
+  value: 83.3
+  source:
+    url: https://huggingface.co/datalab-to/surya-ocr-2
+    name: Surya OCR 2 Model Card
+    user: vikp
+
+- dataset:
+    id: allenai/olmOCR-bench
+    task_id: arxiv_math
+  value: 88.3
+  source:
+    url: https://huggingface.co/datalab-to/surya-ocr-2
+    name: Surya OCR 2 Model Card
+    user: vikp
+
+- dataset:
+    id: allenai/olmOCR-bench
+    task_id: old_scans_math
+  value: 81.4
+  source:
+    url: https://huggingface.co/datalab-to/surya-ocr-2
+    name: Surya OCR 2 Model Card
+    user: vikp
+
+- dataset:
+    id: allenai/olmOCR-bench
+    task_id: table_tests
+  value: 86.6
+  source:
+    url: https://huggingface.co/datalab-to/surya-ocr-2
+    name: Surya OCR 2 Model Card
+    user: vikp
+
+- dataset:
+    id: allenai/olmOCR-bench
+    task_id: old_scans
+  value: 41.8
+  source:
+    url: https://huggingface.co/datalab-to/surya-ocr-2
+    name: Surya OCR 2 Model Card
+    user: vikp
+
+- dataset:
+    id: allenai/olmOCR-bench
+    task_id: headers_footers
+  value: 92.5
+  source:
+    url: https://huggingface.co/datalab-to/surya-ocr-2
+    name: Surya OCR 2 Model Card
+    user: vikp
+
+- dataset:
+    id: allenai/olmOCR-bench
+    task_id: multi_column
+  value: 82.4
+  source:
+    url: https://huggingface.co/datalab-to/surya-ocr-2
+    name: Surya OCR 2 Model Card
+    user: vikp
+
+- dataset:
+    id: allenai/olmOCR-bench
+    task_id: long_tiny_text
+  value: 93.7
+  source:
+    url: https://huggingface.co/datalab-to/surya-ocr-2
+    name: Surya OCR 2 Model Card
+    user: vikp
+
+- dataset:
+    id: allenai/olmOCR-bench
+    task_id: baseline
+  value: 99.7
+  source:
+    url: https://huggingface.co/datalab-to/surya-ocr-2
+    name: Surya OCR 2 Model Card
+    user: vikp

.gitattributes

@@ -33,3 +33,35 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+excerpt.png filter=lfs diff=lfs merge=lfs -text
+excerpt_text.png filter=lfs diff=lfs merge=lfs -text
+excerpt_layout.png filter=lfs diff=lfs merge=lfs -text
+scanned_tablerec.png filter=lfs diff=lfs merge=lfs -text
+olmocr_size_chart.png filter=lfs diff=lfs merge=lfs -text
+newspaper.png filter=lfs diff=lfs merge=lfs -text
+newspaper_text.png filter=lfs diff=lfs merge=lfs -text
+newspaper_layout.png filter=lfs diff=lfs merge=lfs -text
+newspaper_reading.png filter=lfs diff=lfs merge=lfs -text
+textbook.png filter=lfs diff=lfs merge=lfs -text
+textbook_text.png filter=lfs diff=lfs merge=lfs -text
+textbook_layout.png filter=lfs diff=lfs merge=lfs -text
+textbook_reading.png filter=lfs diff=lfs merge=lfs -text
+form.png filter=lfs diff=lfs merge=lfs -text
+form_text.png filter=lfs diff=lfs merge=lfs -text
+form_layout.png filter=lfs diff=lfs merge=lfs -text
+form_reading.png filter=lfs diff=lfs merge=lfs -text
+form_tablerec.png filter=lfs diff=lfs merge=lfs -text
+handwritten.png filter=lfs diff=lfs merge=lfs -text
+handwritten_text.png filter=lfs diff=lfs merge=lfs -text
+handwritten_layout.png filter=lfs diff=lfs merge=lfs -text
+handwritten_reading.png filter=lfs diff=lfs merge=lfs -text
+handwritten_tablerec.png filter=lfs diff=lfs merge=lfs -text
+corporate.png filter=lfs diff=lfs merge=lfs -text
+corporate_text.png filter=lfs diff=lfs merge=lfs -text
+corporate_layout.png filter=lfs diff=lfs merge=lfs -text
+corporate_reading.png filter=lfs diff=lfs merge=lfs -text
+corporate_tablerec.png filter=lfs diff=lfs merge=lfs -text
+cpu_fp16/text.onnx.data filter=lfs diff=lfs merge=lfs -text
+cpu_fp32/text.onnx.data filter=lfs diff=lfs merge=lfs -text
+cpu_int4/text.onnx.data filter=lfs diff=lfs merge=lfs -text
+cuda_fp16/text.onnx.data filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,59 @@
+                   AI PUBS OPEN RAIL-M LICENSE (MODIFIED)
+
+Version 0.1, March 2, 2023 (Modified)
+http://licenses.ai/
+
+PLEASE READ THESE TERMS CAREFULLY BEFORE USING THE MODEL OR A DERIVATIVE WORKS OF THE MODEL MADE AVAILABLE IN CONNECTION WITH THESE TERMS.  BY DOWNLOADING, REPRODUCING, DISTRIBUTING OR USING THE MODEL OR A DERIVATIVE WORK OF THE MODEL IN ANY MANNER, YOU (“YOU”) AGREE TO BE BOUND BY THESE TERMS (THE “AGREEMENT”) TO THE EXCLUSION OF ALL OTHER TERMS. YOU REPRESENT AND WARRANT THAT YOU HAVE THE AUTHORITY TO ENTER INTO THIS AGREEMENT; IF YOU ARE ENTERING INTO THIS AGREEMENT ON BEHALF OF AN ORGANIZATION OR ENTITY, REFERENCES TO AND “YOU” IN THIS AGREEMENT, REFER TO THAT ORGANIZATION OR ENTITY. IF YOU DO NOT AGREE TO ALL OF THE FOLLOWING, YOU MAY NOT DOWNLOAD, REPRODUCE, DISTRIBUTE OR USE THE MODEL OR A DERIVATIVE WORK OF THE MODEL IN ANY MANNER. 
+ Section  I:  PREAMBLE
+This OpenRAIL-M License, as modified, is generally applicable to any machine-learning Model.
+The “Open” nomenclature indicates that the licensed Model is be freely accessible to downstream and other users.  The “RAIL” nomenclature indicates that there are use restrictions prohibiting the use of the Model. These restrictions are intended to avoid potential misuse. This License specifies that the  use restrictions in the original License must apply to such derivatives.
+NOW THEREFORE, You and Licensor agree as follows:
+1. Definitions 
+(a) “Complementary Material” means the applicable source code and scripts used to define, run, load, benchmark or evaluate the Model, and used to prepare data for training or evaluation, if any. This includes any accompanying documentation, tutorials, examples, and any related information, if any. Complementary Material is not licensed under this License.
+(b) "Contribution" means any work, including the original version of the Model and any modifications or additions to that Model or Derivatives of the Model thereof, that is intentionally submitted to Licensor for inclusion in the Model by the rights owner or by an individual or legal entity authorized to submit on behalf of the rights owner. For the purposes of this definition, “submitted” means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Model, but excluding communication that is conspicuously marked or otherwise designated in writing by the rights owner as "Not a Contribution." 
+(c) "Contributor"  means Licensor and any individual or legal entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Model.
+(d) “Data” means a collection of information and/or content extracted from the dataset used with the Model, including to train, pretrain, or otherwise evaluate the Model. The Data is not licensed under this License. 
+(e) “Derivatives of the Model” means all modifications to the Model, works based on the Model, or any other model which is created or initialized by transfer of patterns of the weights, parameters, activations or output of the Model, to the other model, in order to cause the other model to perform similarly to the Model, including - but not limited to - distillation methods entailing the use of intermediate data representations or methods based on the generation of synthetic data by the Model for training the other model.
+(f) “Distribution” means any transmission, reproduction, publication, distribution, or other sharing of the Model or Derivatives of the Model to a third party, including providing the Model as a hosted service made available by electronic or other remote means, including but not limited to API-based or web access. 
+(g) “Harm” includes but is not limited to physical, mental, psychological, financial and reputational damage, pain, or loss
+(h) "License" means the terms and conditions for use, reproduction, and Distribution as defined in this document.
+(i) “Licensor” means the rights owner or entity authorized by the rights owner that is granting the License, including the persons or entities that may have rights in the Model and/or distributing the Model.
+(j) “Model” means any accompanying machine-learning based assemblies (including checkpoints), consisting of learnt weights, parameters (including optimizer states), corresponding to the model architecture as embodied in the Complementary Material, that have been trained or tuned, in whole or in part on the Data, using the Complementary Material. 
+(k) “Output” means the results of operating a Model as embodied in informational content resulting therefrom.
+(l) “Third Parties” means individuals or legal entities that are not under common control with Licensor or You.
+(m) "You" (or "Your")  means an individual or legal entity exercising permissions granted by this License and/or making use of the Model for whichever purpose and in any field of use, including usage of the Model in an end-use application, including but not limited to a chatbot, translator, or image generator.
+                                              Section II:   INTELLECTUAL PROPERTY RIGHTS
+Both copyright and patent grants may apply to the Model and Derivatives of the Model. The Model and Derivatives of the Model are subject to additional terms as described in Section III, which shall govern the use of the Model and Derivatives of the Model even in the event Section II is held unenforceable.
+2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare, publicly display, publicly perform, sublicense, and distribute the Model and Derivatives of the Model.
+3. Grant of Patent License. Subject to the terms and conditions of this License and where and as applicable, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this paragraph) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Model and/or Derivatives of the Model where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Model or Derivatives of the Model to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Model or Derivative of the Model and/or a Contribution incorporated within the Model or Derivative of the Model constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for the Model and/or Derivative of the Model shall terminate as of the date such litigation is asserted or filed.
+Section III: CONDITIONS OF USAGE, DISTRIBUTION AND REDISTRIBUTION
+4. Distribution and Redistribution. You may host the Model or Derivatives of the Model for remote access by Third Parties, including but not limited to  software-as-a-service, reproduce,  or Distribute copies of the Model or Derivatives of the Model thereof in any medium, with or without modifications, provided that You meet the conditions in this Section III:
+(a) Use-based restrictions in paragraph 5 MUST be included as an enforceable provision by You in any type of legal agreement (for example, a license) governing the use and/or distribution of the Model or Derivatives of the Model, and You shall give notice to subsequent users You Distribute to, that the Model and Derivatives of the Model are subject to paragraph 5; 
+(b) You must give any Third Party recipients of the Model or Derivatives of the Model a copy of this License; 
+(c) You must cause any modified files to carry prominent notices stating that You changed the files; and
+(d) You must retain all copyright, patent, trademark, and attribution notices excluding those notices that do not pertain to any part of the Model or  Derivatives of the Model.
+You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions, consistent with paragraph 4.a., for use, reproduction, or Distribution of Your modifications, or for any such Derivatives of the Model as a whole, provided Your use, reproduction, and Distribution of the Model otherwise complies with the conditions stated in this License.
+5. Use-based restrictions. The restrictions set forth in Attachment A are considered Use-based restrictions. Accordingly, You cannot use the Model or the Derivatives of the Model in violation of such restrictions. You may use the Model subject to this License, including only for lawful purposes and in accordance with the License. Use may include creating any content with, fine-tuning, updating, running, training, evaluating and/or re-parametrizing the Model. You shall require all of Your users who use the Model or a Derivative of the Model to comply with the terms of this paragraph 5. 
+6.  The Output You Generate. Except as set forth herein, Licensor claims no rights in the Output You generate using the Model. You are solely responsible for the Output you generate and its subsequent uses. No use of the Output can contravene any provision as stated in the License. 
+7.  Attribution.  In connection with any Output, or use of Distribution of any Model or Derivatives of the Model, You agree to give appropriate credit and attribution to Licensor, provide a link to the original Model or Derivatives of the Model, provide a copy of this License, and identify any changes You have made to the Model or Derivatives of the Model (collectively, the “Attribution”).  The Attribution must not suggest endorsement by any Licensor.  
+8.  Share-a-Like.  As a condition to the license and authorizations herein, You agree to apply this License (to the exclusion of all others) to any and all copies of the Model, Derivatives of the Model, any changes or improvements to the Model or Derivatives of the Model, and to the Output and any derivatives, changes or improvements to or of the Output.
+Section IV: OTHER PROVISIONS
+9. Updates and Runtime Restrictions. To the maximum extent permitted by law, Licensor reserves the right to restrict (remotely or otherwise) usage of the Model in violation of this License, update the Model through electronic means, or cause modification to the Output resulting from updates to the Model based. 
+10. Trademarks and related. Nothing in this License permits You to make use of Licensors’ trademarks, trade names, logos or to otherwise suggest endorsement or misrepresent the relationship between the parties; and any rights not expressly granted herein are reserved by the Licensors.
+11. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Model (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Model and Derivatives of the Model, and assume any risks associated with Your exercise of permissions under this License.
+12. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Model (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.
+13. Accepting Warranty or Additional Liability. While Distributing the Model or Derivatives of the Model, You may choose to charge a fee in exchange for support, warranty, indemnity, or other obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor or Licensor, and only if You agree to indemnify, defend, and hold each Contributor and the Licensor harmless for any liability incurred by, or claims asserted against, such Contributor or Licensor by reason of your accepting any such warranty or additional liability.
+14. If any provision of this License is held to be invalid, illegal or unenforceable, the remaining provisions shall be unaffected thereby and remain valid as if such provision had not been set forth herein.
+END OF TERMS AND CONDITIONS
+
+ Attachment A
+USE RESTRICTIONS
+As conditions to the Licenses set forth in this Agreement, You agree not to use, reproduce, modify, create or Distribute the Model, Derivatives of the Model, or Output (collectively, “Use”)  in any of the following ways:
+1. Legal: 
+(a) In any way that violates any applicable national, federal, state, local or international law or regulation; or
+(b) to directly or indirectly infringe or misappropriate any third party intellectual property rights (including those of Licensor or any Contributor)
+2. Commercial:
+(a) for any purpose if You (your employer, or the entity you are affiliated with) generated more than five million US Dollars ($5,000,000) in gross revenue in the prior year, except where Your Use is limited to personal use or research purposes; 
+(b) for any purpose if You (your employer, or the entity you are affiliated with) has raised more than five million US dollars ($5,000,000) in total equity or debt funding from any source, except where Your Use is limited to personal use or research purposes; or
+(c)  for any purpose if You (your employer, or the entity you are affiliated with) provides or otherwise makes available any product or service that competes with any product or service offered by or made available by Licensor or any of its affiliates.
+Commercial and broader use licenses may be available from Licensor at the following URL: https://www.datalab.to/

README.md

@@ -0,0 +1,465 @@
+---
+library_name: onnxruntime
+license: openrail
+license_link: LICENSE
+tags:
+  - ocr
+  - pdf
+  - markdown
+  - layout
+  - onnx
+  - onnxruntime
+base_model:
+- datalab-to/surya-ocr-2
+---
+
+<p align="center">
+  <img src="datalab-logo.png" alt="Datalab Logo" width="150"/>
+</p>
+
+<h1 align="center">Datalab</h1>
+<p align="center">
+  <strong>State of the Art models for Document Intelligence</strong>
+</p>
+<p align="center">
+  <a href="https://www.datalab.to"><img src="https://img.shields.io/badge/Homepage-datalab.to-blue" alt="Homepage"></a>
+  <a href="https://documentation.datalab.to"><img src="https://img.shields.io/badge/Docs-Read%20the%20docs-blue" alt="Docs"></a>
+  <a href="https://www.datalab.to/playground"><img src="https://img.shields.io/badge/Datalab Playground-Try%20it-orange" alt="Datalab Playground"></a>
+</p>
+
+<hr/>
+
+# Surya
+
+Surya is a 650M param OCR model with these features:
+
+- Accuracy - scores 83.3% on [olmOCR-bench](https://huggingface.co/datasets/allenai/olmOCR-bench) (top under 3B params)
+- Speed - throughput of 5 pages/s on an RTX 5090
+- Multilingual - scores 87.2% on an internal benchmark set of 91 languages (more [here](#multilingual))
+- Layout analysis (table, image, header, etc.) with reading order
+- Table recognition (rows + columns)
+
+It works on a range of documents (see [usage](#usage) and [benchmarks](#benchmarks)).
+
+## Try Datalab's Managed Platform
+
+Our managed platform runs both Surya, and variants of our highest accuracy model, [Chandra](https://github.com/datalab-to/chandra).
+
+Get started with **$5 in free credits** — [sign up](https://www.datalab.to/?utm_source=gh-surya) (takes under 30 seconds) or try our free [public playground](https://www.datalab.to/playground?utm_source=gh-surya).
+
+## Model Information
+
+<img src="assets/olmocr_size_chart.png" width="700"/>
+
+
+|                            Detection                             |                                   OCR                                   |
+|:----------------------------------------------------------------:|:-----------------------------------------------------------------------:|
+|  <img src="assets/excerpt.png" width="140"/>  |  <img src="assets/excerpt_text.png" width="140"/> |
+
+|                               Layout                               |                       Table Recognition                       |
+|:------------------------------------------------------------------:|:-------------------------------------------------------------:|
+| <img src="assets/excerpt_layout.png" width="140"/> | <img src="assets/scanned_tablerec.png" width="140"/> |
+
+
+Surya is named for the [Hindu sun god](https://en.wikipedia.org/wiki/Surya), who has universal vision.
+
+## Examples
+
+| Name              |              Detection              |                                       OCR |                                       Layout |                                          Order |                                       Table Rec |
+|-------------------|:-----------------------------------:|------------------------------------------:|---------------------------------------------:|------------------------------------------------:|------------------------------------------------:|
+| Newspaper         | [Image](assets/newspaper.png) | [Image](assets/newspaper_text.png) | [Image](assets/newspaper_layout.png) | [Image](assets/newspaper_reading.png) |                                                  |
+| Textbook          | [Image](assets/textbook.png)  | [Image](assets/textbook_text.png)  | [Image](assets/textbook_layout.png)  | [Image](assets/textbook_reading.png)  |                                                  |
+| Tax Form          | [Image](assets/form.png)      | [Image](assets/form_text.png)      | [Image](assets/form_layout.png)      | [Image](assets/form_reading.png)      | [Image](assets/form_tablerec.png)      |
+| Handwritten Notes | [Image](assets/handwritten.png) | [Image](assets/handwritten_text.png) | [Image](assets/handwritten_layout.png) | [Image](assets/handwritten_reading.png) | [Image](assets/handwritten_tablerec.png) |
+| Corporate Doc     | [Image](assets/corporate.png) | [Image](assets/corporate_text.png) | [Image](assets/corporate_layout.png) | [Image](assets/corporate_reading.png) | [Image](assets/corporate_tablerec.png) |
+
+# Commercial usage
+
+The Surya code is licensed under Apache 2.0. The model weights use a modified AI Pubs Open Rail-M license (free for research, personal use, and startups under $5M funding/revenue). For broader commercial licensing of the model weights, visit our pricing page [here](https://www.datalab.to/pricing?utm_source=gh-surya).
+
+# Installation
+
+Install with:
+
+```shell
+pip install surya-ocr
+```
+
+# Usage
+
+Surya 2 runs layout, OCR, and table recognition through a single VLM served
+by `vllm` (GPU) or `llama.cpp` (CPU / Apple Silicon). The inference manager
+will spawn one for you on first use; you can also point it at an existing
+server via `SURYA_INFERENCE_URL=http://host:port/v1`.
+
+- Inspect the settings in `surya/settings.py`.  You can override any setting via env var (e.g. `SURYA_INFERENCE_BACKEND=vllm`).
+- Text detection and OCR errors are separate models.
+
+## Interactive App
+
+I've included a streamlit app that lets you interactively try Surya on images or PDF files.  Run it with:
+
+```shell
+pip install streamlit pdftext
+surya_gui
+```
+
+## OCR (text recognition)
+
+This command will write out a json file with the detected text and bboxes:
+
+```shell
+surya_ocr DATA_PATH
+```
+
+- `DATA_PATH` can be an image, pdf, or folder of images/pdfs
+- `--images` will save images of the pages and detected blocks (optional)
+- `--output_dir` specifies the directory to save results to instead of the default
+- `--page_range` specifies the page range to process in the PDF, specified as a single number, a comma separated list, a range, or comma separated ranges - example: `0,5-10,20`.
+
+The `results.json` file contains a dict keyed by input filename (no extension). Each value is a list of page dicts. Each page dict contains:
+
+- `blocks` - per-block OCR results in reading order
+  - `label` - canonicalized layout label (e.g. `Text`, `SectionHeader`, `Table`, `Equation`, `Picture`, `Form`, `PageHeader`, ...). See `surya/layout/label.py:LAYOUT_PRED_RELABEL` for the full canonical-name set.
+  - `raw_label` - original label emitted by the model, before canonicalization
+  - `reading_order` - 0-indexed position in layout output
+  - `html` - block content as HTML (math wrapped in `<math>...</math>`, tables as `<table>...</table>`, etc.). `""` if the block was skipped
+  - `polygon` - 4-corner polygon in `[[x0,y0],[x1,y0],[x1,y1],[x0,y1]]` order
+  - `bbox` - axis-aligned `[x0, y0, x1, y1]` derived from the polygon
+  - `confidence` - mean per-token probability across the block's decode (0-1)
+  - `skipped` - true if the block was a visual label (e.g. Picture) and not OCR'd
+  - `error` - true if the block OCR call failed
+- `image_bbox` - `[0, 0, width, height]` for the page image
+
+**Performance tips**
+
+Throughput is governed by the inference backend, not a `RECOGNITION_BATCH_SIZE` env var. With `vllm`, raise `--max-num-seqs` / `--max-num-batched-tokens` (or `SURYA_INFERENCE_PARALLEL` on the client side) to keep more pages in flight. With `llama.cpp`, set `SURYA_INFERENCE_PARALLEL` to match `--parallel` on `llama-server`.
+
+### From python
+
+```python
+from PIL import Image
+from surya.inference import SuryaInferenceManager
+from surya.recognition import RecognitionPredictor
+
+manager = SuryaInferenceManager()
+recognition_predictor = RecognitionPredictor(manager)
+
+# Default: full-page OCR. One VLM call per page; returns layout + content as
+# HTML <div data-bbox=... data-label=...> blocks.
+predictions = recognition_predictor([Image.open(IMAGE_PATH)])
+
+# Block mode: pre-run layout, then per-block OCR. Auto-selected when
+# `layout_results` is passed.
+from surya.layout import LayoutPredictor
+layout = LayoutPredictor(manager)
+layouts = layout([Image.open(IMAGE_PATH)])
+predictions = recognition_predictor([Image.open(IMAGE_PATH)], layouts)
+```
+
+
+## Text line detection
+
+This command will write out a json file with the detected bboxes.
+
+```shell
+surya_detect DATA_PATH
+```
+
+- `DATA_PATH` can be an image, pdf, or folder of images/pdfs
+- `--images` will save images of the pages and detected text lines (optional)
+- `--output_dir` specifies the directory to save results to instead of the default
+- `--page_range` specifies the page range to process in the PDF, specified as a single number, a comma separated list, a range, or comma separated ranges - example: `0,5-10,20`.
+
+The `results.json` file will contain a json dictionary where the keys are the input filenames without extensions.  Each value will be a list of dictionaries, one per page of the input document.  Each page dictionary contains:
+
+- `bboxes` - detected bounding boxes for text
+  - `bbox` - the axis-aligned rectangle for the text line in (x1, y1, x2, y2) format.  (x1, y1) is the top left corner, and (x2, y2) is the bottom right corner.
+  - `polygon` - the polygon for the text line in (x1, y1), (x2, y2), (x3, y3), (x4, y4) format.  The points are in clockwise order from the top left.
+  - `confidence` - the confidence of the model in the detected text (0-1)
+- `vertical_lines` - vertical lines detected in the document
+  - `bbox` - the axis-aligned line coordinates.
+- `page` - the page number in the file
+- `image_bbox` - the bbox for the image in (x1, y1, x2, y2) format.  (x1, y1) is the top left corner, and (x2, y2) is the bottom right corner.  All line bboxes will be contained within this bbox.
+
+**Performance tips**
+
+Detection is a torch model. `DETECTOR_BATCH_SIZE` defaults to an auto-picked value at runtime; override the env var to control VRAM usage on GPU and raise it on larger cards.
+
+### From python
+
+```python
+from PIL import Image
+from surya.detection import DetectionPredictor
+
+det_predictor = DetectionPredictor()
+predictions = det_predictor([Image.open(IMAGE_PATH)])
+```
+
+## Layout and reading order
+
+This command will write out a json file with the detected layout and reading order.
+
+```shell
+surya_layout DATA_PATH
+```
+
+- `DATA_PATH` can be an image, pdf, or folder of images/pdfs
+- `--images` will save images of the pages and detected text lines (optional)
+- `--output_dir` specifies the directory to save results to instead of the default
+- `--page_range` specifies the page range to process in the PDF, specified as a single number, a comma separated list, a range, or comma separated ranges - example: `0,5-10,20`.
+
+The `results.json` file contains a dict keyed by input filename (no extension). Each value is a list of page dicts. Each page dict contains:
+
+- `bboxes` - layout boxes in reading order
+  - `polygon` - 4-corner polygon `[[x0,y0],[x1,y0],[x1,y1],[x0,y1]]`
+  - `bbox` - axis-aligned `[x0, y0, x1, y1]` derived from the polygon
+  - `label` - canonicalized label. One of `Caption`, `Footnote`, `Equation`, `ListGroup`, `PageHeader`, `PageFooter`, `Picture`, `SectionHeader`, `Table`, `Text`, `Figure`, `Code`, `Form`, `TableOfContents`, `ChemicalBlock`, `Diagram`, `Bibliography`, `BlankPage`
+  - `raw_label` - original label emitted by the model
+  - `position` - 0-indexed reading order
+  - `count` - model's token estimate for OCR'ing this block (rounded to multiples of 50; used to size the per-block decode budget)
+  - `confidence` - mean per-token probability across the layout decode (0-1)
+- `image_bbox` - `[0, 0, width, height]`
+- `raw` - raw JSON the layout model emitted, for debugging
+- `error` - true if the layout call failed
+
+**Performance tips**
+
+Layout runs through the shared inference backend. Throughput tuning is the same as OCR — see Performance tips above.
+
+### From python
+
+```python
+from PIL import Image
+from surya.inference import SuryaInferenceManager
+from surya.layout import LayoutPredictor
+
+layout_predictor = LayoutPredictor(SuryaInferenceManager())
+layout_predictions = layout_predictor([Image.open(IMAGE_PATH)])
+```
+
+## Table Recognition
+
+This command will write out a json file with the detected table cells and row/column ids, along with row/column bounding boxes.  If you want to get cell positions and text, along with nice formatting, check out the [marker](https://github.com/datalab-to/marker) repo.  You can use the `TableConverter` to detect and extract tables in images and PDFs.  It supports output in json (with bboxes), markdown, and html.
+
+```shell
+surya_table DATA_PATH
+```
+
+- `DATA_PATH` can be an image, pdf, or folder of images/pdfs
+- `--images` will save annotated row + column overlays alongside the json (optional)
+- `--output_dir` specifies the directory to save results to instead of the default
+- `--page_range` specifies the page range to process in the PDF, specified as a single number, a comma separated list, a range, or comma separated ranges - example: `0,5-10,20`.
+- `--skip_table_detection` tells table recognition not to detect tables first.  Use this if your image is already cropped to a table.
+
+The `results.json` file contains a dict keyed by input filename (no extension). Each value is a list of per-table dicts. Each table dict contains:
+
+- `rows` - detected table rows in reading order
+  - `polygon` / `bbox` - row geometry (same convention as everywhere else)
+  - `row_id` - 0-indexed row id
+- `cols` - detected table columns
+  - `polygon` / `bbox` - column geometry
+  - `col_id` - 0-indexed column id
+- `cells` - geometric row × column intersections (simple mode)
+  - `polygon` / `bbox` - cell geometry
+  - `row_id`, `col_id`, `cell_id`
+- `html` - full `<table>...</table>` HTML (only populated when `predict_full` is used; handles spanning cells / header rows). `null` in simple mode.
+- `mode` - `"simple"` or `"full"`
+- `image_bbox` - the table crop bbox
+- `error` - true if the table_rec call failed
+- `raw` - raw model output, for debugging
+
+**Performance tips**
+
+Table recognition routes through the shared VLM. Throughput tuning is the same as OCR.
+
+### From python
+
+```python
+from PIL import Image
+from surya.inference import SuryaInferenceManager
+from surya.table_rec import TableRecPredictor
+
+table_rec_predictor = TableRecPredictor(SuryaInferenceManager())
+
+# Default: rows + columns only, cells derived from intersections.
+table_predictions = table_rec_predictor([Image.open(IMAGE_PATH)])
+
+# Or full HTML output (better for spanning cells / headers):
+# table_predictions = table_rec_predictor.predict_full([image])
+```
+
+## Math / equations
+
+Surya 2 handles math inline as part of full-page OCR — recognized equations
+come back inside `<math>...</math>` tags in the same HTML output as
+surrounding prose, in KaTeX-compatible LaTeX. No separate LaTeX OCR pass.
+
+# Inference Backends
+
+Layout / OCR / table_rec all share one VLM, served either by `vllm` (GPU) or `llama.cpp` (CPU / Apple Silicon). The `SuryaInferenceManager` will spawn one automatically; you can also point at a pre-running server:
+
+```bash
+# Attach to an existing vllm
+export SURYA_INFERENCE_BACKEND=vllm
+export SURYA_INFERENCE_URL=http://localhost:8000/v1
+```
+
+| Setting                           | Default                           | Notes                                                  |
+|-----------------------------------|-----------------------------------|--------------------------------------------------------|
+| `SURYA_INFERENCE_BACKEND`         | auto (vllm if NVIDIA, else llamacpp) | `vllm` \| `llamacpp` \| unset (auto)                |
+| `SURYA_INFERENCE_URL`             | (auto-spawn)                      | Attach to a running OpenAI-compatible server          |
+| `SURYA_INFERENCE_PARALLEL`        | 8                                 | Client-side concurrency to the backend                |
+| `SURYA_GUIDED_LAYOUT`             | true                              | JSON-schema-constrained layout decode                 |
+
+# Limitations
+
+- This is specialized for document OCR. Performance on photos or natural scenes is not the goal.
+- Layout / OCR / table_rec all need a running inference backend (vllm or llama.cpp). Detection runs purely on torch and works without it.
+
+## Troubleshooting
+
+If OCR isn't working properly:
+
+- Try increasing resolution of the image so the text is bigger.  If the resolution is already very high, try decreasing it to no more than a `2048px` width.
+- Preprocessing the image (binarizing, deskewing, etc) can help with very old/blurry images.
+- You can adjust `DETECTOR_BLANK_THRESHOLD` and `DETECTOR_TEXT_THRESHOLD` if you don't get good results.  `DETECTOR_BLANK_THRESHOLD` controls the space between lines - any prediction below this number will be considered blank space.  `DETECTOR_TEXT_THRESHOLD` controls how text is joined - any number above this is considered text.  `DETECTOR_TEXT_THRESHOLD` should always be higher than `DETECTOR_BLANK_THRESHOLD`, and both should be in the 0-1 range.  Looking at the heatmap from the debug output of the detector can tell you how to adjust these (if you see faint things that look like boxes, lower the thresholds, and if you see bboxes being joined together, raise the thresholds).
+
+# Manual install
+
+If you want to develop surya, you can install it manually with [uv](https://docs.astral.sh/uv/):
+
+```bash
+git clone https://github.com/datalab-to/surya.git
+cd surya
+uv sync --group dev      # installs runtime + dev deps
+uv run surya_ocr ...     # or `uv shell` to enter the venv
+```
+
+# Benchmarks
+
+Surya 2 is a single VLM that handles layout analysis, OCR (full-page or
+per-block), and table recognition in one model. We evaluate end-to-end on
+[olmOCR-bench](https://huggingface.co/datasets/allenai/olmOCR-bench) — the
+standard quality benchmark for document parsers.
+
+## olmOCR-bench
+
+Pareto-optimal, and best in class under 3B params.
+
+| Model                       | Params    | Score    |
+|-----------------------------|----------:|---------:|
+| Infinity-Parser2-Pro        |     35.1B |     87.6 |
+| Chandra OCR 2 (Datalab)     |      5.3B |     85.9 |
+| dots.mocr                   |      3.0B |     83.9 |
+| **Surya OCR 2** (Datalab)   | **0.65B** | **83.3** |
+| LightOnOCR 2-1B \*          |      1.0B |     83.2 |
+| Chandra OCR 1 (Datalab)     |      9.0B |     83.1 |
+| olmOCR (anchored)           |      8.3B |     77.4 |
+| GOT OCR                     |      0.6B |     48.3 |
+
+\* **LightOnOCR 2-1B** uses a different benchmark methodology than the other entries (see their [release notes](https://huggingface.co/lightonai/LightOnOCR-2-1B)); the score is included for context but is not directly comparable.
+
+Comparison scores from the [olmOCR-bench dataset card](https://huggingface.co/datasets/allenai/olmOCR-bench).
+
+Surya 2, per-source pass rate on the `default` preset (8,413 tests total):
+
+| ArXiv | Base | Hdr/Ftr | TinyTxt | MultCol | OldScan | OldMath | Tables |
+|------:|-----:|--------:|--------:|--------:|--------:|--------:|-------:|
+|  88.3 | 99.7 |    92.5 |    93.7 |    82.4 |    41.8 |    81.4 |   86.6 |
+
+## Multilingual
+
+We also evaluate Surya 2 against a 91-language internal benchmark covering
+text accuracy, layout, tables, math, and reading order in documents drawn
+from each language.
+
+**Overall pass rate: 87.2% across 91 languages.** 38 of the
+91 languages score ≥ 90%; 76 score ≥ 80%.
+
+Top 15 widely-spoken languages:
+
+| Code | Language    | Score |
+|------|-------------|------:|
+| `ar` | Arabic      | 72.7% |
+| `bn` | Bengali     | 82.7% |
+| `zh` | Chinese     | 82.5% |
+| `en` | English     | 92.3% |
+| `fr` | French      | 89.3% |
+| `de` | German      | 89.7% |
+| `hi` | Hindi       | 82.2% |
+| `it` | Italian     | 93.0% |
+| `ja` | Japanese    | 86.2% |
+| `ko` | Korean      | 86.7% |
+| `fa` | Persian     | 82.3% |
+| `pt` | Portuguese  | 86.1% |
+| `ru` | Russian     | 88.8% |
+| `es` | Spanish     | 90.7% |
+| `vi` | Vietnamese  | 73.2% |
+
+See [https://github.com/datalab-to/surya/blob/master/static/docs/multilingual.md](https://github.com/datalab-to/surya/blob/master/static/docs/multilingual.md) for the full 91-language table.
+
+## Throughput
+
+Full-page OCR, 96 DPI input (~2,400 output tokens/page average), measured
+client-side against a running inference server.
+
+### RTX 5090 (vllm)
+
+`vllm/vllm-openai:v0.20.1`, single RTX 5090 (32 GB).
+
+| Concurrency | Pages/s |  Tokens/s | p50 (ms) | p95 (ms) | avg tok/page |
+|------------:|--------:|----------:|---:|---:|---:|
+|         128 |    5.35 |    12,884 | 18,915 | 42,538 | 2,410 |
+
+### Apple Silicon (llama.cpp / Metal)
+
+`llama-server` with Metal backend.
+
+| `--parallel` |  Pages/s | Tokens/s | p50 (ms) | p95 (ms) | avg tok/page | Power |
+|-------------:|---------:|---------:|---:|---:|---:|---:|
+|            8 |    0.108 |      254 | 59,313 | 129,173 | 2,360 | ~30 W |
+
+## Reproducing
+
+We score Surya 2 on olmOCR-bench by serving the model with `vllm` (or
+`llama.cpp`) and running the olmOCR-bench harness from
+[allenai/olmocr](https://github.com/allenai/olmocr), with some adjustments applied to account for our output HTML format.
+
+# Training
+
+Layout, OCR, and table recognition all share a single vision-language model
+(Qwen3.5-style architecture, ~650M params). It's trained on diverse document
+images to emit either a layout JSON or a full-page HTML output, depending on
+prompt. Text-line detection is a separate small torch model — a modified
+EfficientViT segformer trained from scratch on document line annotations.
+
+If you want help finetuning Surya on your own data, or to use our managed
+training stack, reach us at [email protected].
+
+# Thanks
+
+This work would not have been possible without amazing open source AI work:
+
+- [Qwen3-VL](https://huggingface.co/Qwen) from Alibaba
+- [vllm](https://github.com/vllm-project/vllm) and [llama.cpp](https://github.com/ggerganov/llama.cpp) for inference
+- [Segformer](https://arxiv.org/pdf/2105.15203.pdf) from NVIDIA
+- [EfficientViT](https://github.com/mit-han-lab/efficientvit) from MIT
+- [timm](https://github.com/huggingface/pytorch-image-models) from Ross Wightman
+- [transformers](https://github.com/huggingface/transformers) from huggingface
+- [CRAFT](https://github.com/clovaai/CRAFT-pytorch), a great scene text detection model
+
+Thank you to everyone who makes open source AI possible.
+
+# Citation
+
+If you use surya (or the associated models) in your work or research, please consider citing us using the following BibTeX entry:
+
+```bibtex
+@misc{paruchuri2025surya,
+  author       = {Vikas Paruchuri and Datalab Team},
+  title        = {Surya: A lightweight document OCR and analysis toolkit},
+  year         = {2025},
+  howpublished = {\url{https://github.com/datalab-to/surya}},
+  note         = {GitHub repository},
+}

cpu_fp16/chat_template.jinja

@@ -0,0 +1,66 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+    {%- if content is string %}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping %}
+        {%- for item in content %}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain images.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Picture ' ~ image_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+            {%- elif 'video' in item or item.type == 'video' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain videos.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Video ' ~ video_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+            {%- elif 'text' in item %}
+                {{- item.text }}
+            {%- else %}
+                {{- raise_exception('Unexpected item type in content.') }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif content is none or content is undefined %}
+        {{- '' }}
+    {%- else %}
+        {{- raise_exception('Unexpected content type.') }}
+    {%- endif %}
+{%- endmacro %}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- if messages[0].role == 'system' %}
+    {%- set content = render_content(messages[0].content, false, true)|trim %}
+    {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content, true)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role + '\n' + content }}
+        {{- '<|im_end|>\n' }}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

cpu_fp16/config.json

@@ -0,0 +1,101 @@
+{
+  "architectures": [
+    "Qwen3_5ForConditionalGeneration"
+  ],
+  "eos_token_id": 248044,
+  "image_token_id": 11,
+  "model_type": "qwen3_5",
+  "num_nextn_predict_layers": 1,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "attn_output_gate": true,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": 248044,
+    "full_attention_interval": 4,
+    "head_dim": 256,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3584,
+    "layer_types": [
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention"
+    ],
+    "linear_conv_kernel_dim": 4,
+    "linear_key_head_dim": 128,
+    "linear_num_key_heads": 16,
+    "linear_num_value_heads": 16,
+    "linear_value_head_dim": 128,
+    "mamba_ssm_dtype": "float32",
+    "max_position_embeddings": 262144,
+    "mlp_only_layers": [],
+    "model_type": "qwen3_5_text",
+    "mtp_num_hidden_layers": 1,
+    "mtp_use_dedicated_embeddings": false,
+    "num_attention_heads": 8,
+    "num_hidden_layers": 24,
+    "num_key_value_heads": 2,
+    "pad_token_id": null,
+    "partial_rotary_factor": 0.25,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "mrope_interleaved": true,
+      "mrope_section": [
+        11,
+        11,
+        10
+      ],
+      "partial_rotary_factor": 0.25,
+      "rope_theta": 10000000,
+      "rope_type": "default"
+    },
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "vocab_size": 65425
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.10.2",
+  "video_token_id": 12,
+  "vision_config": {
+    "deepstack_visual_indexes": [],
+    "depth": 12,
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 768,
+    "in_channels": 3,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "model_type": "qwen3_5_vision",
+    "num_heads": 12,
+    "num_position_embeddings": 2304,
+    "out_hidden_size": 1024,
+    "patch_size": 16,
+    "spatial_merge_size": 2,
+    "temporal_patch_size": 2
+  },
+  "vision_end_token_id": 10,
+  "vision_start_token_id": 9
+}

cpu_fp16/genai_config.json

@@ -0,0 +1,86 @@
+{
+    "model": {
+        "bos_token_id": 1,
+        "context_length": 262144,
+        "decoder": {
+            "session_options": {
+                "log_id": "onnxruntime-genai",
+                "provider_options": []
+            },
+            "filename": "text.onnx",
+            "head_size": 256,
+            "hidden_size": 1024,
+            "inputs": {
+                "inputs_embeds": "inputs_embeds",
+                "attention_mask": "attention_mask",
+                "position_ids": "position_ids",
+                "past_key_names": "past_key_values.%d.key",
+                "past_value_names": "past_key_values.%d.value"
+            },
+            "outputs": {
+                "logits": "logits",
+                "present_key_names": "present.%d.key",
+                "present_value_names": "present.%d.value"
+            },
+            "num_attention_heads": 8,
+            "num_hidden_layers": 24,
+            "num_key_value_heads": 2
+        },
+        "eos_token_id": 248044,
+        "pad_token_id": 248044,
+        "type": "qwen3_5",
+        "vocab_size": 65425,
+        "embedding": {
+            "filename": "embedding.onnx",
+            "inputs": {
+                "input_ids": "input_ids",
+                "image_features": "image_features"
+            },
+            "outputs": {
+                "inputs_embeds": "inputs_embeds"
+            },
+            "session_options": {
+                "log_id": "onnxruntime-genai",
+                "provider_options": []
+            }
+        },
+        "vision": {
+            "filename": "vision.onnx",
+            "config_filename": "processor_config.json",
+            "spatial_merge_size": 2,
+            "tokens_per_second": 2.0,
+            "patch_size": 16,
+            "window_size": 64,
+            "inputs": {
+                "pixel_values": "pixel_values",
+                "image_grid_thw": "image_grid_thw"
+            },
+            "outputs": {
+                "image_features": "image_features"
+            },
+            "session_options": {
+                "log_id": "onnxruntime-genai",
+                "provider_options": []
+            }
+        },
+        "image_token_id": 151655,
+        "video_token_id": 151656,
+        "vision_start_token_id": 151652
+    },
+    "search": {
+        "diversity_penalty": 0.0,
+        "do_sample": false,
+        "early_stopping": true,
+        "length_penalty": 1.0,
+        "max_length": 262144,
+        "min_length": 0,
+        "no_repeat_ngram_size": 0,
+        "num_beams": 1,
+        "num_return_sequences": 1,
+        "past_present_share_buffer": false,
+        "repetition_penalty": 1.0,
+        "temperature": 1.0,
+        "top_k": 50,
+        "top_p": 1.0
+    }
+}

cpu_fp16/processor_config.json

@@ -0,0 +1,76 @@
+{
+  "processor": {
+    "name": "qwen3_vl_image_processor",
+    "transforms": [
+      {
+        "operation": {
+          "name": "decode_image",
+          "type": "DecodeImage",
+          "attrs": {
+            "color_space": "RGB"
+          }
+        }
+      },
+      {
+        "operation": {
+          "name": "convert_to_rgb",
+          "type": "ConvertRGB"
+        }
+      },
+      {
+        "operation": {
+          "name": "resize",
+          "type": "Resize",
+          "attrs": {
+            "width": 540,
+            "height": 360,
+            "smart_resize": 1,
+            "min_pixels": 65536,
+            "max_pixels": 16777216,
+            "patch_size": 16,
+            "merge_size": 2
+          }
+        }
+      },
+      {
+        "operation": {
+          "name": "rescale",
+          "type": "Rescale",
+          "attrs": {
+            "rescale_factor": 0.00392156862745098
+          }
+        }
+      },
+      {
+        "operation": {
+          "name": "normalize",
+          "type": "Normalize",
+          "attrs": {
+            "mean": [
+              0.5,
+              0.5,
+              0.5
+            ],
+            "std": [
+              0.5,
+              0.5,
+              0.5
+            ],
+            "qwen3_vl": 1
+          }
+        }
+      },
+      {
+        "operation": {
+          "name": "patch_image",
+          "type": "PatchImage",
+          "attrs": {
+            "patch_size": 16,
+            "temporal_patch_size": 2,
+            "merge_size": 2
+          }
+        }
+      }
+    ]
+  }
+}

cpu_fp16/text.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0875b00ffcc69afac6e9d0b9d41b2af9991f24e478d99d32083553724f0c18f0
+size 434813

cpu_fp16/text.onnx.data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a16e3996a2af7e53b74bdb352c63c92cce584a45ec5a3eabaf211fd44c97938
+size 1197574144

cpu_fp16/tokenizer_config.json

@@ -0,0 +1,25 @@
+{
+  "backend": "tokenizers",
+  "bos_token": null,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 262144,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "Qwen3VLProcessor",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<unk>"
+}

cpu_fp32/chat_template.jinja

@@ -0,0 +1,66 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+    {%- if content is string %}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping %}
+        {%- for item in content %}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain images.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Picture ' ~ image_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+            {%- elif 'video' in item or item.type == 'video' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain videos.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Video ' ~ video_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+            {%- elif 'text' in item %}
+                {{- item.text }}
+            {%- else %}
+                {{- raise_exception('Unexpected item type in content.') }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif content is none or content is undefined %}
+        {{- '' }}
+    {%- else %}
+        {{- raise_exception('Unexpected content type.') }}
+    {%- endif %}
+{%- endmacro %}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- if messages[0].role == 'system' %}
+    {%- set content = render_content(messages[0].content, false, true)|trim %}
+    {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content, true)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role + '\n' + content }}
+        {{- '<|im_end|>\n' }}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

cpu_fp32/config.json

@@ -0,0 +1,101 @@
+{
+  "architectures": [
+    "Qwen3_5ForConditionalGeneration"
+  ],
+  "eos_token_id": 248044,
+  "image_token_id": 11,
+  "model_type": "qwen3_5",
+  "num_nextn_predict_layers": 1,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "attn_output_gate": true,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": 248044,
+    "full_attention_interval": 4,
+    "head_dim": 256,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3584,
+    "layer_types": [
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention"
+    ],
+    "linear_conv_kernel_dim": 4,
+    "linear_key_head_dim": 128,
+    "linear_num_key_heads": 16,
+    "linear_num_value_heads": 16,
+    "linear_value_head_dim": 128,
+    "mamba_ssm_dtype": "float32",
+    "max_position_embeddings": 262144,
+    "mlp_only_layers": [],
+    "model_type": "qwen3_5_text",
+    "mtp_num_hidden_layers": 1,
+    "mtp_use_dedicated_embeddings": false,
+    "num_attention_heads": 8,
+    "num_hidden_layers": 24,
+    "num_key_value_heads": 2,
+    "pad_token_id": null,
+    "partial_rotary_factor": 0.25,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "mrope_interleaved": true,
+      "mrope_section": [
+        11,
+        11,
+        10
+      ],
+      "partial_rotary_factor": 0.25,
+      "rope_theta": 10000000,
+      "rope_type": "default"
+    },
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "vocab_size": 65425
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.10.2",
+  "video_token_id": 12,
+  "vision_config": {
+    "deepstack_visual_indexes": [],
+    "depth": 12,
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 768,
+    "in_channels": 3,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "model_type": "qwen3_5_vision",
+    "num_heads": 12,
+    "num_position_embeddings": 2304,
+    "out_hidden_size": 1024,
+    "patch_size": 16,
+    "spatial_merge_size": 2,
+    "temporal_patch_size": 2
+  },
+  "vision_end_token_id": 10,
+  "vision_start_token_id": 9
+}

cpu_fp32/genai_config.json

@@ -0,0 +1,86 @@
+{
+    "model": {
+        "bos_token_id": 1,
+        "context_length": 262144,
+        "decoder": {
+            "session_options": {
+                "log_id": "onnxruntime-genai",
+                "provider_options": []
+            },
+            "filename": "text.onnx",
+            "head_size": 256,
+            "hidden_size": 1024,
+            "inputs": {
+                "inputs_embeds": "inputs_embeds",
+                "attention_mask": "attention_mask",
+                "position_ids": "position_ids",
+                "past_key_names": "past_key_values.%d.key",
+                "past_value_names": "past_key_values.%d.value"
+            },
+            "outputs": {
+                "logits": "logits",
+                "present_key_names": "present.%d.key",
+                "present_value_names": "present.%d.value"
+            },
+            "num_attention_heads": 8,
+            "num_hidden_layers": 24,
+            "num_key_value_heads": 2
+        },
+        "eos_token_id": 248044,
+        "pad_token_id": 248044,
+        "type": "qwen3_5",
+        "vocab_size": 65425,
+        "embedding": {
+            "filename": "embedding.onnx",
+            "inputs": {
+                "input_ids": "input_ids",
+                "image_features": "image_features"
+            },
+            "outputs": {
+                "inputs_embeds": "inputs_embeds"
+            },
+            "session_options": {
+                "log_id": "onnxruntime-genai",
+                "provider_options": []
+            }
+        },
+        "vision": {
+            "filename": "vision.onnx",
+            "config_filename": "processor_config.json",
+            "spatial_merge_size": 2,
+            "tokens_per_second": 2.0,
+            "patch_size": 16,
+            "window_size": 64,
+            "inputs": {
+                "pixel_values": "pixel_values",
+                "image_grid_thw": "image_grid_thw"
+            },
+            "outputs": {
+                "image_features": "image_features"
+            },
+            "session_options": {
+                "log_id": "onnxruntime-genai",
+                "provider_options": []
+            }
+        },
+        "image_token_id": 151655,
+        "video_token_id": 151656,
+        "vision_start_token_id": 151652
+    },
+    "search": {
+        "diversity_penalty": 0.0,
+        "do_sample": false,
+        "early_stopping": true,
+        "length_penalty": 1.0,
+        "max_length": 262144,
+        "min_length": 0,
+        "no_repeat_ngram_size": 0,
+        "num_beams": 1,
+        "num_return_sequences": 1,
+        "past_present_share_buffer": true,
+        "repetition_penalty": 1.0,
+        "temperature": 1.0,
+        "top_k": 50,
+        "top_p": 1.0
+    }
+}

cpu_fp32/processor_config.json

@@ -0,0 +1,76 @@
+{
+  "processor": {
+    "name": "qwen3_vl_image_processor",
+    "transforms": [
+      {
+        "operation": {
+          "name": "decode_image",
+          "type": "DecodeImage",
+          "attrs": {
+            "color_space": "RGB"
+          }
+        }
+      },
+      {
+        "operation": {
+          "name": "convert_to_rgb",
+          "type": "ConvertRGB"
+        }
+      },
+      {
+        "operation": {
+          "name": "resize",
+          "type": "Resize",
+          "attrs": {
+            "width": 540,
+            "height": 360,
+            "smart_resize": 1,
+            "min_pixels": 65536,
+            "max_pixels": 16777216,
+            "patch_size": 16,
+            "merge_size": 2
+          }
+        }
+      },
+      {
+        "operation": {
+          "name": "rescale",
+          "type": "Rescale",
+          "attrs": {
+            "rescale_factor": 0.00392156862745098
+          }
+        }
+      },
+      {
+        "operation": {
+          "name": "normalize",
+          "type": "Normalize",
+          "attrs": {
+            "mean": [
+              0.5,
+              0.5,
+              0.5
+            ],
+            "std": [
+              0.5,
+              0.5,
+              0.5
+            ],
+            "qwen3_vl": 1
+          }
+        }
+      },
+      {
+        "operation": {
+          "name": "patch_image",
+          "type": "PatchImage",
+          "attrs": {
+            "patch_size": 16,
+            "temporal_patch_size": 2,
+            "merge_size": 2
+          }
+        }
+      }
+    ]
+  }
+}

cpu_fp32/text.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbfd23ae84c7d152b392413222663addc0f5dd182acd7ef497ddb7d68cf91c83
+size 411759

cpu_fp32/text.onnx.data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b837aaf83685c5cfdd4efe579cadf3e3fea9149931f473f7945b830be97f73c5
+size 2328039424

cpu_fp32/tokenizer_config.json

@@ -0,0 +1,25 @@
+{
+  "backend": "tokenizers",
+  "bos_token": null,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 262144,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "Qwen3VLProcessor",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<unk>"
+}

cpu_int4/chat_template.jinja

@@ -0,0 +1,66 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+    {%- if content is string %}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping %}
+        {%- for item in content %}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain images.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Picture ' ~ image_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+            {%- elif 'video' in item or item.type == 'video' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain videos.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Video ' ~ video_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+            {%- elif 'text' in item %}
+                {{- item.text }}
+            {%- else %}
+                {{- raise_exception('Unexpected item type in content.') }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif content is none or content is undefined %}
+        {{- '' }}
+    {%- else %}
+        {{- raise_exception('Unexpected content type.') }}
+    {%- endif %}
+{%- endmacro %}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- if messages[0].role == 'system' %}
+    {%- set content = render_content(messages[0].content, false, true)|trim %}
+    {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content, true)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role + '\n' + content }}
+        {{- '<|im_end|>\n' }}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

cpu_int4/config.json

@@ -0,0 +1,101 @@
+{
+  "architectures": [
+    "Qwen3_5ForConditionalGeneration"
+  ],
+  "eos_token_id": 248044,
+  "image_token_id": 11,
+  "model_type": "qwen3_5",
+  "num_nextn_predict_layers": 1,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "attn_output_gate": true,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": 248044,
+    "full_attention_interval": 4,
+    "head_dim": 256,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3584,
+    "layer_types": [
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention"
+    ],
+    "linear_conv_kernel_dim": 4,
+    "linear_key_head_dim": 128,
+    "linear_num_key_heads": 16,
+    "linear_num_value_heads": 16,
+    "linear_value_head_dim": 128,
+    "mamba_ssm_dtype": "float32",
+    "max_position_embeddings": 262144,
+    "mlp_only_layers": [],
+    "model_type": "qwen3_5_text",
+    "mtp_num_hidden_layers": 1,
+    "mtp_use_dedicated_embeddings": false,
+    "num_attention_heads": 8,
+    "num_hidden_layers": 24,
+    "num_key_value_heads": 2,
+    "pad_token_id": null,
+    "partial_rotary_factor": 0.25,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "mrope_interleaved": true,
+      "mrope_section": [
+        11,
+        11,
+        10
+      ],
+      "partial_rotary_factor": 0.25,
+      "rope_theta": 10000000,
+      "rope_type": "default"
+    },
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "vocab_size": 65425
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.10.2",
+  "video_token_id": 12,
+  "vision_config": {
+    "deepstack_visual_indexes": [],
+    "depth": 12,
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 768,
+    "in_channels": 3,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "model_type": "qwen3_5_vision",
+    "num_heads": 12,
+    "num_position_embeddings": 2304,
+    "out_hidden_size": 1024,
+    "patch_size": 16,
+    "spatial_merge_size": 2,
+    "temporal_patch_size": 2
+  },
+  "vision_end_token_id": 10,
+  "vision_start_token_id": 9
+}