Upload folder using huggingface_hub

README.md

@@ -0,0 +1,65 @@
+---
+license: apache-2.0
+language:
+  - en
+pipeline_tag: text-generation
+tags:
+  - pytorch
+  - causal-lm
+  - from-scratch
+  - custom_code
+  - gqa
+  - rope
+---
+
+# G0-nano-base
+
+A 62M-parameter GPT trained **completely from scratch on a single 8GB-RAM device** (an NVIDIA Jetson) — no cloud cluster, no multi-GPU node. This is the base (pretrained, non-instruct) checkpoint: it completes text, it does not follow instructions. See [`G0-nano-instruct`](https://huggingface.co/AZERDSQ/G0-nano-instruct) for the chat-tuned version.
+
+The point of this model isn't to compete with billion-parameter LLMs — a 62M model is physically capped at roughly 15.5MB of factual knowledge (~2 bits/parameter, [Allen-Zhu & Li, ICLR'25](https://arxiv.org/abs/2404.05405)), regardless of training method. The point is that it was trained end-to-end, pretraining included, within an 8GB memory budget.
+
+## Architecture
+
+Llama-style decoder-only transformer:
+
+| | |
+|---|---|
+| Parameters | 62.1M (embeddings shared with LM head) |
+| Layers | 12 |
+| Hidden size | 640 |
+| Attention | Grouped-Query Attention, 10 query heads / 2 KV heads, head_dim 64 |
+| Position encoding | RoPE (θ=10000) |
+| Feed-forward | SwiGLU, hidden 1728 |
+| Normalization | RMSNorm |
+| Context length | 1024 tokens |
+| Vocabulary | 16,384 (SentencePiece, trained from scratch) |
+| Precision | fp32 |
+
+Trained on ~1.5B tokens of English web/book text (FineWeb-Edu, BookCorpus, OpenWebText, PG-19, WikiHow).
+
+## Usage
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+tok = AutoTokenizer.from_pretrained("AZERDSQ/G0-nano-base", trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained("AZERDSQ/G0-nano-base", trust_remote_code=True)
+
+inputs = tok("The city of Paris is", return_tensors="pt")
+out = model.generate(**inputs, max_new_tokens=50, do_sample=True, top_k=50, temperature=0.8)
+print(tok.decode(out[0]))
+```
+
+`trust_remote_code=True` is required — this is a custom architecture (GQA + RoPE + SwiGLU), not one of the built-in `transformers` model types.
+
+## Limitations
+
+- 62M parameters caps factual knowledge hard — expect confident, fluent, frequently wrong completions on anything knowledge-dense.
+- 1024-token context.
+- English only.
+- The custom tokenizer/model code does not handle padded batched inference — single-sequence generation only.
+- Base model: raw next-token prediction, no chat formatting. It will continue text, not answer questions.
+
+## License
+
+Apache 2.0. Weights only — this release does not include training code or data pipelines.

config.json

@@ -0,0 +1,22 @@
+{
+  "model_type": "nanogpt",
+  "architectures": [
+    "NanoGPTForCausalLM"
+  ],
+  "vocab_size": 16384,
+  "hidden_size": 640,
+  "num_hidden_layers": 12,
+  "num_attention_heads": 10,
+  "num_key_value_heads": 2,
+  "head_dim": 64,
+  "intermediate_size": 1728,
+  "max_position_embeddings": 1024,
+  "rope_theta": 10000.0,
+  "rms_norm_eps": 1e-05,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float32",
+  "auto_map": {
+    "AutoConfig": "configuration_nanogpt.NanoGPTConfig",
+    "AutoModelForCausalLM": "modeling_nanogpt.NanoGPTForCausalLM"
+  }
+}

configuration_nanogpt.py

@@ -0,0 +1,32 @@
+from transformers import PretrainedConfig
+
+
+class NanoGPTConfig(PretrainedConfig):
+    model_type = "nanogpt"
+
+    def __init__(
+        self,
+        vocab_size=16384,
+        hidden_size=640,
+        num_hidden_layers=12,
+        num_attention_heads=10,
+        num_key_value_heads=2,
+        head_dim=64,
+        intermediate_size=1728,
+        max_position_embeddings=1024,
+        rope_theta=10000.0,
+        rms_norm_eps=1e-5,
+        tie_word_embeddings=True,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_key_value_heads = num_key_value_heads
+        self.head_dim = head_dim
+        self.intermediate_size = intermediate_size
+        self.max_position_embeddings = max_position_embeddings
+        self.rope_theta = rope_theta
+        self.rms_norm_eps = rms_norm_eps
+        super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "bos_token_id": 2,
+  "eos_token_id": [
+    3
+  ],
+  "pad_token_id": 0
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac6db6b91d6d0106a6e762fd1037f07c7db579502f7efdad604913fd7e139457
+size 248456088

modeling_nanogpt.py

@@ -0,0 +1,204 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import PreTrainedModel
+from transformers.cache_utils import Cache, DynamicCache
+from transformers.modeling_outputs import CausalLMOutputWithPast
+
+from .configuration_nanogpt import NanoGPTConfig
+
+
+class RMSNorm(nn.Module):
+    def __init__(self, dim: int, eps: float):
+        super().__init__()
+        self.eps = eps
+        self.weight = nn.Parameter(torch.ones(dim))
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        norm = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
+        return norm * self.weight
+
+
+def build_rope_cache(seq_len: int, head_dim: int, theta: float, device, dtype):
+    inv_freq = 1.0 / (theta ** (torch.arange(0, head_dim, 2, device=device).float() / head_dim))
+    t = torch.arange(seq_len, device=device).float()
+    freqs = torch.outer(t, inv_freq)
+    return freqs.cos().to(dtype), freqs.sin().to(dtype)
+
+
+def apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
+    x1, x2 = x[..., ::2], x[..., 1::2]
+    cos = cos[None, None, :x.size(2), :]
+    sin = sin[None, None, :x.size(2), :]
+    rotated1 = x1 * cos - x2 * sin
+    rotated2 = x1 * sin + x2 * cos
+    return torch.stack([rotated1, rotated2], dim=-1).flatten(-2).to(x.dtype)
+
+
+class GQAAttention(nn.Module):
+    def __init__(self, config: NanoGPTConfig):
+        super().__init__()
+        self.n_heads = config.num_attention_heads
+        self.n_kv_heads = config.num_key_value_heads
+        self.head_dim = config.head_dim
+        self.n_rep = self.n_heads // self.n_kv_heads
+
+        self.wq = nn.Linear(config.hidden_size, self.n_heads * self.head_dim, bias=False)
+        self.wk = nn.Linear(config.hidden_size, self.n_kv_heads * self.head_dim, bias=False)
+        self.wv = nn.Linear(config.hidden_size, self.n_kv_heads * self.head_dim, bias=False)
+        self.wo = nn.Linear(self.n_heads * self.head_dim, config.hidden_size, bias=False)
+
+    def forward(self, x, cos, sin, past_key_values=None, layer_idx=None, use_cache=False):
+        B, T, _ = x.shape
+        q = self.wq(x).view(B, T, self.n_heads, self.head_dim).transpose(1, 2)
+        k = self.wk(x).view(B, T, self.n_kv_heads, self.head_dim).transpose(1, 2)
+        v = self.wv(x).view(B, T, self.n_kv_heads, self.head_dim).transpose(1, 2)
+
+        past_len = 0 if past_key_values is None else past_key_values.get_seq_length(layer_idx)
+        q = apply_rope(q, cos[past_len:past_len + T], sin[past_len:past_len + T])
+        k = apply_rope(k, cos[past_len:past_len + T], sin[past_len:past_len + T])
+
+        if past_key_values is not None:
+            k, v = past_key_values.update(k, v, layer_idx)
+
+        k = k.repeat_interleave(self.n_rep, dim=1)
+        v = v.repeat_interleave(self.n_rep, dim=1)
+
+        if past_len == 0 and T > 1:
+            out = F.scaled_dot_product_attention(q, k, v, is_causal=True)
+        else:
+            key_len = k.size(2)
+            query_positions = torch.arange(T, device=x.device) + past_len
+            key_positions = torch.arange(key_len, device=x.device)
+            attn_mask = key_positions.unsqueeze(0) <= query_positions.unsqueeze(1)
+            out = F.scaled_dot_product_attention(q, k, v, attn_mask=attn_mask)
+
+        out = out.transpose(1, 2).contiguous().view(B, T, self.n_heads * self.head_dim)
+        return self.wo(out)
+
+
+class SwiGLU(nn.Module):
+    def __init__(self, config: NanoGPTConfig):
+        super().__init__()
+        self.w_gate = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
+        self.w_up = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
+        self.w_down = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
+
+    def forward(self, x):
+        return self.w_down(F.silu(self.w_gate(x)) * self.w_up(x))
+
+
+class Block(nn.Module):
+    def __init__(self, config: NanoGPTConfig):
+        super().__init__()
+        self.attn_norm = RMSNorm(config.hidden_size, config.rms_norm_eps)
+        self.attn = GQAAttention(config)
+        self.mlp_norm = RMSNorm(config.hidden_size, config.rms_norm_eps)
+        self.mlp = SwiGLU(config)
+
+    def forward(self, x, cos, sin, past_key_values=None, layer_idx=None, use_cache=False):
+        x = x + self.attn(self.attn_norm(x), cos, sin, past_key_values, layer_idx, use_cache)
+        x = x + self.mlp(self.mlp_norm(x))
+        return x
+
+
+class NanoGPTPreTrainedModel(PreTrainedModel):
+    config_class = NanoGPTConfig
+    base_model_prefix = "nanogpt"
+    supports_gradient_checkpointing = False
+
+    def _init_weights(self, module):
+        if isinstance(module, nn.Linear):
+            nn.init.normal_(module.weight, mean=0.0, std=0.02)
+        elif isinstance(module, nn.Embedding):
+            nn.init.normal_(module.weight, mean=0.0, std=0.02)
+
+
+class NanoGPTForCausalLM(NanoGPTPreTrainedModel):
+    _tied_weights_keys = {"lm_head.weight": "tok_emb.weight"}
+
+    def __init__(self, config: NanoGPTConfig):
+        super().__init__(config)
+        self.tok_emb = nn.Embedding(config.vocab_size, config.hidden_size)
+        self.blocks = nn.ModuleList([Block(config) for _ in range(config.num_hidden_layers)])
+        self.final_norm = RMSNorm(config.hidden_size, config.rms_norm_eps)
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        self._rope_cache = {}
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.tok_emb
+
+    def set_input_embeddings(self, value):
+        self.tok_emb = value
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, value):
+        self.lm_head = value
+
+    def _get_rope(self, seq_len, device, dtype):
+        key = (seq_len, device, dtype)
+        if key not in self._rope_cache:
+            self._rope_cache[key] = build_rope_cache(
+                seq_len, self.config.head_dim, self.config.rope_theta, device, dtype
+            )
+        return self._rope_cache[key]
+
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        past_key_values=None,
+        labels=None,
+        use_cache=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+        **kwargs,
+    ):
+        use_cache = bool(use_cache) if use_cache is not None else False
+        B, T = input_ids.shape
+
+        if use_cache and past_key_values is None:
+            past_key_values = DynamicCache(config=self.config)
+        past_len = past_key_values.get_seq_length() if past_key_values is not None else 0
+        assert past_len + T <= self.config.max_position_embeddings, (
+            f"sequence length {past_len + T} > max_position_embeddings "
+            f"{self.config.max_position_embeddings}"
+        )
+
+        x = self.tok_emb(input_ids)
+        cos, sin = self._get_rope(past_len + T, input_ids.device, x.dtype)
+
+        for i, block in enumerate(self.blocks):
+            x = block(x, cos, sin, past_key_values, i, use_cache)
+        x = self.final_norm(x)
+        logits = self.lm_head(x)
+
+        loss = None
+        if labels is not None:
+            loss = F.cross_entropy(
+                logits[:, :-1, :].reshape(-1, logits.size(-1)),
+                labels[:, 1:].reshape(-1),
+                ignore_index=-100,
+            )
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=past_key_values if use_cache else None,
+        )
+
+    def prepare_inputs_for_generation(
+        self, input_ids, past_key_values=None, attention_mask=None, use_cache=True, **kwargs
+    ):
+        if past_key_values is not None and past_key_values.get_seq_length() > 0:
+            input_ids = input_ids[:, -1:]
+        return {
+            "input_ids": input_ids,
+            "past_key_values": past_key_values,
+            "use_cache": use_cache,
+        }

spm_16384.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a76c144aef571d3db4f18b6c57fd91adee39f85fdad722fb3c893b3a827a81c5
+size 513334

tokenization_nanogpt.py

@@ -0,0 +1,78 @@
+import os
+import shutil
+
+import sentencepiece as spm
+from transformers import PreTrainedTokenizer
+
+VOCAB_FILES_NAMES = {"vocab_file": "spm_16384.model"}
+
+
+class NanoGPTTokenizer(PreTrainedTokenizer):
+    """Wrapper SentencePiece + tokens de chat additionnels (>= 16384).
+
+    Les tokens de chat (``<|user|>``, ``<|assistant|>``, ``<|end|>``,
+    ``<|system|>``) sont geres par le mecanisme "added tokens" standard de
+    ``transformers`` (passe via ``additional_special_tokens``) plutot que par
+    une logique maison : HF les decoupe avant tokenization et les reinsere
+    correctement au decode, et leur assigne des ids sequentiels a partir de
+    ``len(self)`` au moment du ``__init__`` -- ce qui reproduit exactement le
+    mapping fige dans ``training/chat_format.py`` (16384..16387), du moment
+    que la liste est fournie dans le meme ordre.
+    """
+
+    vocab_files_names = VOCAB_FILES_NAMES
+    model_input_names = ["input_ids", "attention_mask"]
+
+    def __init__(
+        self,
+        vocab_file,
+        bos_token="<s>",
+        eos_token="</s>",
+        unk_token="<unk>",
+        pad_token="<pad>",
+        additional_special_tokens=None,
+        **kwargs,
+    ):
+        self.vocab_file = vocab_file
+        self.sp_model = spm.SentencePieceProcessor()
+        self.sp_model.Load(vocab_file)
+
+        super().__init__(
+            bos_token=bos_token,
+            eos_token=eos_token,
+            unk_token=unk_token,
+            pad_token=pad_token,
+            additional_special_tokens=additional_special_tokens or [],
+            **kwargs,
+        )
+
+    @property
+    def vocab_size(self):
+        return self.sp_model.get_piece_size()
+
+    def get_vocab(self):
+        vocab = {self.sp_model.id_to_piece(i): i for i in range(self.vocab_size)}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+
+    def _tokenize(self, text, **kwargs):
+        return self.sp_model.encode(text, out_type=str)
+
+    def _convert_token_to_id(self, token):
+        return self.sp_model.piece_to_id(token)
+
+    def _convert_id_to_token(self, index):
+        return self.sp_model.id_to_piece(index)
+
+    def convert_tokens_to_string(self, tokens):
+        return self.sp_model.decode(tokens) if tokens else ""
+
+    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+        return [self.bos_token_id] + token_ids_0
+
+    def save_vocabulary(self, save_directory, filename_prefix=None):
+        out_name = (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
+        out_path = os.path.join(save_directory, out_name)
+        if os.path.abspath(self.vocab_file) != os.path.abspath(out_path):
+            shutil.copyfile(self.vocab_file, out_path)
+        return (out_path,)

tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "tokenizer_class": "NanoGPTTokenizer",
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_nanogpt.NanoGPTTokenizer",
+      null
+    ]
+  },
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "unk_token": "<unk>",
+  "pad_token": "<pad>",
+  "additional_special_tokens": [],
+  "clean_up_tokenization_spaces": false
+}