trfms-integration (#23)

- trmfs integration (1f032b3c3f2c4d4c216945f6279069b743cdb2d7)
- update config (4eeddac9908f942fa3ed3e30e37485e6e7802f1f)


Co-authored-by: Eustache Le Bihan <eustlb@users.noreply.huggingface.co>

README.md

@@ -196,9 +196,45 @@ We have added two simple example files that allow you to:
 
 ### Transformers
 
-*WIP ⏳*
+Starting with `transformers >= 5.2.0`, you can run Voxtral Realtime natively in Transformers!
 
-For progress, see: https://github.com/huggingface/transformers/pull/43769
+For more details, refer to the [Transformers documentation](https://huggingface.co/docs/transformers/main/en/model_doc/voxtral_realtime).
+
+#### Installation
+
+Install Transformers:
+
+```bash
+pip install --upgrade transformers
+```
+
+Make sure to have `mistral-common` installed with audio dependencies:
+
+```bash
+pip install --upgrade "mistral-common[audio]"
+```
+
+```python
+import torch
+from transformers import VoxtralRealtimeForConditionalGeneration, AutoProcessor
+from datasets import load_dataset
+
+repo_id = "mistralai/Voxtral-Mini-4B-Realtime-2602"
+
+processor = AutoProcessor.from_pretrained(repo_id)
+model = VoxtralRealtimeForConditionalGeneration.from_pretrained(repo_id, device_map="auto")
+
+ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
+audio = ds[0]["audio"]["array"]
+
+inputs = processor(audio, return_tensors="pt")
+inputs = inputs.to(model.device, dtype=model.dtype)
+
+outputs = model.generate(**inputs)
+decoded_outputs = processor.batch_decode(outputs, skip_special_tokens=True)
+
+print(decoded_outputs[0])
+```
 
 ### Community Contributions (Untested)
 

config.json

@@ -0,0 +1,60 @@
+{
+  "architectures": [
+    "VoxtralRealtimeForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_function": "gelu",
+    "attention_dropout": 0.0,
+    "head_dim": 64,
+    "hidden_act": "silu",
+    "hidden_size": 1280,
+    "initializer_range": 0.02,
+    "intermediate_size": 5120,
+    "max_position_embeddings": 1500,
+    "model_type": "voxtral_realtime_encoder",
+    "num_attention_heads": 32,
+    "num_hidden_layers": 32,
+    "num_key_value_heads": 32,
+    "num_mel_bins": 128,
+    "rms_norm_eps": 1e-05,
+    "rope_parameters": {
+      "rope_theta": 1000000.0,
+      "rope_type": "default"
+    },
+    "sliding_window": 750,
+    "vocab_size": 131072
+  },
+  "audio_length_per_tok": 8,
+  "default_num_delay_tokens": 6,
+  "downsample_factor": 4,
+  "dtype": "bfloat16",
+  "hidden_size": 3072,
+  "model_type": "voxtral_realtime",
+  "projector_hidden_act": "gelu",
+  "text_config": {
+    "attention_dropout": 0.0,
+    "bos_token_id": 1,
+    "eos_token_id": 2,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 3072,
+    "initializer_range": 0.02,
+    "intermediate_size": 9216,
+    "max_position_embeddings": 131072,
+    "model_type": "voxtral_realtime_text",
+    "num_attention_heads": 32,
+    "num_hidden_layers": 26,
+    "num_key_value_heads": 8,
+    "pad_token_id": null,
+    "rms_norm_eps": 1e-05,
+    "rope_parameters": {
+      "rope_theta": 1000000.0,
+      "rope_type": "default"
+    },
+    "sliding_window": 8192,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "vocab_size": 131072
+  },
+  "transformers_version": "5.2.0.dev0"
+}

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 11,
+  "transformers_version": "5.2.0.dev0",
+  "use_cache": true
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e745e4902df6a4c48f29f2f8dc1f6d0fb4cc73c7156bc45923451a5bcdfcd1d6
+size 8859446848

processor_config.json

@@ -0,0 +1,15 @@
+{
+  "feature_extractor": {
+    "feature_extractor_type": "VoxtralRealtimeFeatureExtractor",
+    "feature_size": 128,
+    "global_log_mel_max": 1.5,
+    "hop_length": 160,
+    "n_fft": 400,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": true,
+    "sampling_rate": 16000,
+    "win_length": 400
+  },
+  "processor_class": "VoxtralRealtimeProcessor"
+}