max_model_len 8192 -> 4096

acestep/llm_inference.py

@@ -375,9 +375,9 @@ class LLMHandler:
                 max_ratio=0.9
             )
             if low_gpu_memory_mode:
-                self.max_model_len = 4096
+                self.max_model_len = 2048
             else:
-                self.max_model_len = 8192
+                self.max_model_len = 4096
             
             logger.info(f"Initializing 5Hz LM with model: {model_path}, enforce_eager: False, tensor_parallel_size: 1, max_model_len: {self.max_model_len}, gpu_memory_utilization: {gpu_memory_utilization}")
             start_time = time.time()

acestep/third_parts/nano-vllm/nanovllm/config.py

@@ -8,7 +8,7 @@ class Config:
     model: str
     max_num_batched_tokens: int = 16384
     max_num_seqs: int = 512
-    max_model_len: int = 8192
+    max_model_len: int = 4096
     gpu_memory_utilization: float = 0.9
     tensor_parallel_size: int = 1
     enforce_eager: bool = False