Upload folder using huggingface_hub

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "/workspace/HENLA/hf_exports/HENLA-CONFED-3B-FINEWEB-GATEFIX",
+  "architectures": [
+    "HenlaConfedForCausalLM"
+  ],
+  "area_hidden_mult": 4,
+  "attn_pdrop": 0.1,
+  "auto_map": {
+    "AutoConfig": "modeling_henla_confed.HenlaConfedConfig",
+    "AutoModelForCausalLM": "modeling_henla_confed.HenlaConfedForCausalLM"
+  },
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "model_type": "henla_confed",
+  "n_areas": 8,
+  "n_embd": 1280,
+  "n_head": 16,
+  "n_layer": 24,
+  "n_positions": 512,
+  "pad_token_id": 50256,
+  "resid_pdrop": 0.1,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.2",
+  "vocab_size": 50257
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "pad_token_id": 50256,
+  "transformers_version": "4.44.2"
+}

modeling_henla_confed.py

@@ -0,0 +1,245 @@
+
+import math
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from transformers import PretrainedConfig, PreTrainedModel
+from transformers.modeling_outputs import CausalLMOutputWithPast
+
+
+class HenlaConfedConfig(PretrainedConfig):
+    model_type = "henla_confed"
+
+    def __init__(
+        self,
+        vocab_size=50257,
+        n_positions=1024,
+        n_embd=1024,
+        n_layer=24,
+        n_head=16,
+        n_areas=8,
+        area_hidden_mult=4,
+        resid_pdrop=0.1,
+        embd_pdrop=0.1,
+        attn_pdrop=0.1,
+        bos_token_id=50256,
+        eos_token_id=50256,
+        pad_token_id=50256,
+        **kwargs,
+    ):
+        super().__init__(
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            pad_token_id=pad_token_id,
+            **kwargs,
+        )
+        self.vocab_size = vocab_size
+        self.n_positions = n_positions
+        self.n_embd = n_embd
+        self.n_layer = n_layer
+        self.n_head = n_head
+        self.n_areas = n_areas
+        self.area_hidden_mult = area_hidden_mult
+        self.resid_pdrop = resid_pdrop
+        self.embd_pdrop = embd_pdrop
+        self.attn_pdrop = attn_pdrop
+
+
+class CausalSelfAttention(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        assert config.n_embd % config.n_head == 0
+        self.n_head = config.n_head
+        self.head_dim = config.n_embd // config.n_head
+        self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd)
+        self.c_proj = nn.Linear(config.n_embd, config.n_embd)
+        self.attn_dropout = nn.Dropout(config.attn_pdrop)
+        self.resid_dropout = nn.Dropout(config.resid_pdrop)
+
+    def forward(self, x, attention_mask=None):
+        b, t, c = x.size()
+        qkv = self.c_attn(x)
+        q, k, v = qkv.split(c, dim=2)
+
+        q = q.view(b, t, self.n_head, self.head_dim).transpose(1, 2)
+        k = k.view(b, t, self.n_head, self.head_dim).transpose(1, 2)
+        v = v.view(b, t, self.n_head, self.head_dim).transpose(1, 2)
+
+        att = (q @ k.transpose(-2, -1)) / math.sqrt(self.head_dim)
+
+        causal = torch.tril(torch.ones(t, t, device=x.device, dtype=torch.bool))
+        att = att.masked_fill(~causal.view(1, 1, t, t), torch.finfo(att.dtype).min)
+
+        if attention_mask is not None:
+            # attention_mask: [B,T], 1 keep, 0 mask
+            m = attention_mask.view(b, 1, 1, t).to(torch.bool)
+            att = att.masked_fill(~m, torch.finfo(att.dtype).min)
+
+        att = F.softmax(att, dim=-1)
+        att = self.attn_dropout(att)
+
+        y = att @ v
+        y = y.transpose(1, 2).contiguous().view(b, t, c)
+        y = self.resid_dropout(self.c_proj(y))
+        return y
+
+
+class CognitiveAreaMLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        hidden = config.area_hidden_mult * config.n_embd
+        self.net = nn.Sequential(
+            nn.Linear(config.n_embd, hidden),
+            nn.GELU(),
+            nn.Linear(hidden, config.n_embd),
+            nn.Dropout(config.resid_pdrop),
+        )
+
+    def forward(self, x):
+        return self.net(x)
+
+
+class HenlaConfederatedBlock(nn.Module):
+    """
+    One transformer block with federated cognitive areas.
+
+    Flow:
+      token stream -> shared attention
+      -> parallel cognitive areas
+      -> metacognitive gate over areas
+      -> orchestrated residual
+    """
+    def __init__(self, config):
+        super().__init__()
+        self.ln_1 = nn.LayerNorm(config.n_embd)
+        self.attn = CausalSelfAttention(config)
+
+        self.ln_2 = nn.LayerNorm(config.n_embd)
+        self.areas = nn.ModuleList([CognitiveAreaMLP(config) for _ in range(config.n_areas)])
+
+        self.gate = nn.Sequential(
+            nn.Linear(config.n_embd, config.n_embd),
+            nn.GELU(),
+            nn.Linear(config.n_embd, config.n_areas),
+        )
+
+        self.area_norm = nn.LayerNorm(config.n_embd)
+        self.resid_dropout = nn.Dropout(config.resid_pdrop)
+
+    def forward(self, x, attention_mask=None):
+        x = x + self.attn(self.ln_1(x), attention_mask=attention_mask)
+
+        h = self.ln_2(x)
+        area_outs = torch.stack([area(h) for area in self.areas], dim=2)  # B,T,A,C
+        gate_logits = self.gate(h)                                        # B,T,A
+        gate = F.softmax(gate_logits, dim=-1).unsqueeze(-1)               # B,T,A,1
+
+        fused = (area_outs * gate).sum(dim=2)
+        fused = self.area_norm(fused)
+
+        x = x + self.resid_dropout(fused)
+        return x
+
+
+class HenlaConfedForCausalLM(PreTrainedModel):
+    config_class = HenlaConfedConfig
+    base_model_prefix = "henla_confed"
+    supports_gradient_checkpointing = True
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.config = config
+
+        self.wte = nn.Embedding(config.vocab_size, config.n_embd)
+        self.wpe = nn.Embedding(config.n_positions, config.n_embd)
+        self.drop = nn.Dropout(config.embd_pdrop)
+
+        self.blocks = nn.ModuleList([HenlaConfederatedBlock(config) for _ in range(config.n_layer)])
+        self.ln_f = nn.LayerNorm(config.n_embd)
+
+        self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
+        self.lm_head.weight = self.wte.weight
+
+        self.gradient_checkpointing = False
+        self.post_init()
+
+    def _init_weights(self, module):
+        if isinstance(module, nn.Linear):
+            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
+            if module.bias is not None:
+                torch.nn.init.zeros_(module.bias)
+        elif isinstance(module, nn.Embedding):
+            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
+
+    def gradient_checkpointing_enable(self, gradient_checkpointing_kwargs=None):
+        self.gradient_checkpointing = True
+
+    def gradient_checkpointing_disable(self):
+        self.gradient_checkpointing = False
+
+    def get_input_embeddings(self):
+        return self.wte
+
+    def set_input_embeddings(self, value):
+        self.wte = value
+        self.lm_head.weight = self.wte.weight
+
+    def prepare_inputs_for_generation(self, input_ids, attention_mask=None, **kwargs):
+        return {
+            "input_ids": input_ids,
+            "attention_mask": attention_mask,
+        }
+
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        labels=None,
+        **kwargs,
+    ):
+        b, t = input_ids.shape
+        if t > self.config.n_positions:
+            input_ids = input_ids[:, -self.config.n_positions:]
+            if attention_mask is not None:
+                attention_mask = attention_mask[:, -self.config.n_positions:]
+            if labels is not None:
+                labels = labels[:, -self.config.n_positions:]
+            b, t = input_ids.shape
+
+        pos = torch.arange(0, t, dtype=torch.long, device=input_ids.device).unsqueeze(0)
+
+        x = self.wte(input_ids) + self.wpe(pos)
+        x = self.drop(x)
+
+        for block in self.blocks:
+            if self.gradient_checkpointing and self.training:
+                x = torch.utils.checkpoint.checkpoint(
+                    block,
+                    x,
+                    attention_mask,
+                    use_reentrant=False,
+                )
+            else:
+                x = block(x, attention_mask=attention_mask)
+
+        x = self.ln_f(x)
+        logits = self.lm_head(x)
+
+        loss = None
+        if labels is not None:
+            shift_logits = logits[:, :-1, :].contiguous()
+            shift_labels = labels[:, 1:].contiguous()
+            loss = F.cross_entropy(
+                shift_logits.view(-1, shift_logits.size(-1)),
+                shift_labels.view(-1),
+                ignore_index=-100,
+            )
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=None,
+            hidden_states=None,
+            attentions=None,
+        )

pytorch_model-00001-of-00012.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43b60c4554b52fa91e0b86329df093fee77f682d4f2c232cc2d7da78e8dbdc47
+size 500508482

pytorch_model-00002-of-00012.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11096dd34250da87b1e6f26cd31f23493cfe0e4392d2054a0aa8c28dde289a29
+size 504988914

pytorch_model-00003-of-00012.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:994551afeeb9224f8fb2385b8a781ef8ab95ff862a4bc5c84565fd15929139c1
+size 508315734

pytorch_model-00004-of-00012.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e46c3eae9225cf375a47d9c1e6a567085793e95177aa070fb56b264cb6fef86a
+size 504988914

pytorch_model-00005-of-00012.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd2d1995bc22e74342e5af7c5e51dd1870f3d656e7f4f8d5f60c282f842b38ec
+size 504988914

pytorch_model-00006-of-00012.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bac995c6cd6e0cc043b65e4560e6638d815a15eb79fe736ab7fbc07841fe2de1
+size 504988978

pytorch_model-00007-of-00012.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbc1e6f0e9415bdaa0024f7df77b27b1b4b3b6ac4330e007c30528573e4c989a
+size 508308182

pytorch_model-00008-of-00012.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee82374510ed839479d0f8eedc23ee2303e34375cd33dc2a04ddfcbab01890ac
+size 504988978

pytorch_model-00009-of-00012.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9add2d0eedc2312d482fc58af5e872936279ef358181f95804bd46f044d5ab6d
+size 504988978

pytorch_model-00010-of-00012.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8ba589a0104c43d7b1a965bc09701c9372e603f6085d38e33bd8c0158df5f4d
+size 504988978

pytorch_model-00011-of-00012.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d12e1da8ed78a963e77a9650b933ae279dee7bac7744ad83ef8b99adfbe6012
+size 508301706

pytorch_model-00012-of-00012.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d67d91f1628fae60205428a17c0559fb80d0be81a59c0c235175df9c4fc5046a
+size 128659333

pytorch_model.bin.index.json

@@ -0,0 +1,1116 @@
+{
+  "metadata": {
+    "total_size": 5688637824
+  },
+  "weight_map": {
+    "blocks.0.area_norm.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.area_norm.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.0.net.0.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.0.net.0.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.0.net.2.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.0.net.2.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.1.net.0.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.1.net.0.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.1.net.2.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.1.net.2.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.2.net.0.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.2.net.0.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.2.net.2.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.2.net.2.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.3.net.0.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.3.net.0.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.3.net.2.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.3.net.2.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.4.net.0.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.4.net.0.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.4.net.2.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.4.net.2.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.5.net.0.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.5.net.0.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.5.net.2.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.5.net.2.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.6.net.0.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.6.net.0.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.6.net.2.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.6.net.2.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.7.net.0.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.7.net.0.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.7.net.2.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.areas.7.net.2.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.attn.c_attn.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.attn.c_attn.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.attn.c_proj.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.attn.c_proj.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.gate.0.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.gate.0.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.gate.2.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.gate.2.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.ln_1.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.ln_1.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.ln_2.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.0.ln_2.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.area_norm.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.1.area_norm.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.1.areas.0.net.0.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.areas.0.net.0.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.areas.0.net.2.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.areas.0.net.2.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.areas.1.net.0.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.areas.1.net.0.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.areas.1.net.2.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.areas.1.net.2.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.areas.2.net.0.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.areas.2.net.0.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.areas.2.net.2.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.areas.2.net.2.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.areas.3.net.0.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.areas.3.net.0.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.areas.3.net.2.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.areas.3.net.2.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.areas.4.net.0.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.areas.4.net.0.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.areas.4.net.2.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.areas.4.net.2.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.areas.5.net.0.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.1.areas.5.net.0.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.1.areas.5.net.2.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.1.areas.5.net.2.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.1.areas.6.net.0.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.1.areas.6.net.0.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.1.areas.6.net.2.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.1.areas.6.net.2.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.1.areas.7.net.0.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.1.areas.7.net.0.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.1.areas.7.net.2.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.1.areas.7.net.2.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.1.attn.c_attn.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.attn.c_attn.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.attn.c_proj.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.attn.c_proj.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.gate.0.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.1.gate.0.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.1.gate.2.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.1.gate.2.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.1.ln_1.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.ln_1.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.ln_2.bias": "pytorch_model-00001-of-00012.bin",
+    "blocks.1.ln_2.weight": "pytorch_model-00001-of-00012.bin",
+    "blocks.10.area_norm.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.10.area_norm.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.10.areas.0.net.0.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.areas.0.net.0.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.areas.0.net.2.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.areas.0.net.2.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.areas.1.net.0.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.areas.1.net.0.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.areas.1.net.2.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.areas.1.net.2.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.areas.2.net.0.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.areas.2.net.0.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.areas.2.net.2.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.areas.2.net.2.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.areas.3.net.0.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.areas.3.net.0.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.areas.3.net.2.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.areas.3.net.2.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.areas.4.net.0.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.areas.4.net.0.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.areas.4.net.2.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.10.areas.4.net.2.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.10.areas.5.net.0.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.10.areas.5.net.0.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.10.areas.5.net.2.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.10.areas.5.net.2.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.10.areas.6.net.0.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.10.areas.6.net.0.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.10.areas.6.net.2.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.10.areas.6.net.2.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.10.areas.7.net.0.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.10.areas.7.net.0.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.10.areas.7.net.2.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.10.areas.7.net.2.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.10.attn.c_attn.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.attn.c_attn.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.attn.c_proj.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.attn.c_proj.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.gate.0.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.10.gate.0.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.10.gate.2.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.10.gate.2.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.10.ln_1.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.ln_1.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.ln_2.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.10.ln_2.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.11.area_norm.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.area_norm.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.0.net.0.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.0.net.0.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.0.net.2.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.0.net.2.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.1.net.0.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.1.net.0.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.1.net.2.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.1.net.2.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.2.net.0.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.2.net.0.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.2.net.2.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.2.net.2.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.3.net.0.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.3.net.0.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.3.net.2.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.3.net.2.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.4.net.0.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.4.net.0.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.4.net.2.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.4.net.2.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.5.net.0.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.5.net.0.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.5.net.2.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.5.net.2.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.6.net.0.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.6.net.0.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.6.net.2.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.6.net.2.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.7.net.0.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.7.net.0.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.7.net.2.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.areas.7.net.2.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.attn.c_attn.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.attn.c_attn.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.attn.c_proj.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.attn.c_proj.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.gate.0.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.gate.0.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.gate.2.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.gate.2.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.ln_1.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.ln_1.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.ln_2.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.11.ln_2.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.area_norm.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.12.area_norm.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.12.areas.0.net.0.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.0.net.0.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.0.net.2.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.0.net.2.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.1.net.0.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.1.net.0.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.1.net.2.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.1.net.2.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.2.net.0.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.2.net.0.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.2.net.2.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.2.net.2.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.3.net.0.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.3.net.0.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.3.net.2.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.3.net.2.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.4.net.0.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.4.net.0.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.4.net.2.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.4.net.2.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.5.net.0.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.5.net.0.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.5.net.2.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.5.net.2.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.6.net.0.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.6.net.0.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.areas.6.net.2.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.12.areas.6.net.2.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.12.areas.7.net.0.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.12.areas.7.net.0.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.12.areas.7.net.2.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.12.areas.7.net.2.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.12.attn.c_attn.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.attn.c_attn.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.attn.c_proj.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.attn.c_proj.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.gate.0.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.12.gate.0.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.12.gate.2.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.12.gate.2.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.12.ln_1.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.ln_1.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.ln_2.bias": "pytorch_model-00006-of-00012.bin",
+    "blocks.12.ln_2.weight": "pytorch_model-00006-of-00012.bin",
+    "blocks.13.area_norm.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.area_norm.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.0.net.0.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.0.net.0.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.0.net.2.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.0.net.2.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.1.net.0.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.1.net.0.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.1.net.2.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.1.net.2.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.2.net.0.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.2.net.0.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.2.net.2.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.2.net.2.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.3.net.0.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.3.net.0.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.3.net.2.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.3.net.2.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.4.net.0.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.4.net.0.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.4.net.2.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.4.net.2.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.5.net.0.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.5.net.0.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.5.net.2.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.5.net.2.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.6.net.0.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.6.net.0.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.6.net.2.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.6.net.2.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.7.net.0.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.7.net.0.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.7.net.2.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.areas.7.net.2.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.attn.c_attn.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.attn.c_attn.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.attn.c_proj.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.attn.c_proj.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.gate.0.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.gate.0.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.gate.2.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.gate.2.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.ln_1.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.ln_1.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.ln_2.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.13.ln_2.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.area_norm.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.area_norm.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.0.net.0.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.0.net.0.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.0.net.2.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.0.net.2.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.1.net.0.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.1.net.0.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.1.net.2.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.1.net.2.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.2.net.0.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.2.net.0.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.2.net.2.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.2.net.2.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.3.net.0.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.3.net.0.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.3.net.2.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.3.net.2.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.4.net.0.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.4.net.0.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.4.net.2.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.4.net.2.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.5.net.0.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.5.net.0.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.5.net.2.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.5.net.2.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.6.net.0.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.6.net.0.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.6.net.2.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.6.net.2.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.7.net.0.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.7.net.0.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.7.net.2.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.areas.7.net.2.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.attn.c_attn.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.attn.c_attn.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.attn.c_proj.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.attn.c_proj.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.gate.0.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.gate.0.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.gate.2.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.gate.2.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.ln_1.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.ln_1.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.ln_2.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.14.ln_2.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.15.area_norm.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.area_norm.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.0.net.0.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.0.net.0.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.0.net.2.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.0.net.2.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.1.net.0.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.1.net.0.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.1.net.2.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.1.net.2.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.2.net.0.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.2.net.0.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.2.net.2.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.2.net.2.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.3.net.0.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.3.net.0.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.3.net.2.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.3.net.2.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.4.net.0.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.4.net.0.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.4.net.2.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.4.net.2.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.5.net.0.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.5.net.0.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.5.net.2.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.5.net.2.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.6.net.0.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.6.net.0.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.6.net.2.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.6.net.2.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.7.net.0.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.7.net.0.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.7.net.2.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.areas.7.net.2.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.attn.c_attn.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.15.attn.c_attn.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.15.attn.c_proj.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.15.attn.c_proj.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.15.gate.0.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.gate.0.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.gate.2.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.gate.2.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.15.ln_1.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.15.ln_1.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.15.ln_2.bias": "pytorch_model-00007-of-00012.bin",
+    "blocks.15.ln_2.weight": "pytorch_model-00007-of-00012.bin",
+    "blocks.16.area_norm.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.area_norm.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.0.net.0.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.0.net.0.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.0.net.2.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.0.net.2.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.1.net.0.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.1.net.0.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.1.net.2.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.1.net.2.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.2.net.0.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.2.net.0.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.2.net.2.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.2.net.2.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.3.net.0.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.3.net.0.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.3.net.2.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.3.net.2.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.4.net.0.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.4.net.0.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.4.net.2.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.4.net.2.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.5.net.0.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.5.net.0.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.5.net.2.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.5.net.2.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.6.net.0.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.6.net.0.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.6.net.2.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.6.net.2.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.7.net.0.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.7.net.0.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.7.net.2.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.areas.7.net.2.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.attn.c_attn.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.attn.c_attn.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.attn.c_proj.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.attn.c_proj.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.gate.0.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.gate.0.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.gate.2.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.gate.2.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.ln_1.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.ln_1.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.ln_2.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.16.ln_2.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.17.area_norm.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.area_norm.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.areas.0.net.0.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.17.areas.0.net.0.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.17.areas.0.net.2.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.17.areas.0.net.2.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.17.areas.1.net.0.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.17.areas.1.net.0.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.17.areas.1.net.2.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.17.areas.1.net.2.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.17.areas.2.net.0.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.areas.2.net.0.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.areas.2.net.2.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.areas.2.net.2.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.areas.3.net.0.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.areas.3.net.0.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.areas.3.net.2.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.areas.3.net.2.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.areas.4.net.0.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.areas.4.net.0.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.areas.4.net.2.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.areas.4.net.2.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.areas.5.net.0.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.areas.5.net.0.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.areas.5.net.2.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.areas.5.net.2.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.areas.6.net.0.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.areas.6.net.0.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.areas.6.net.2.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.areas.6.net.2.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.areas.7.net.0.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.areas.7.net.0.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.areas.7.net.2.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.areas.7.net.2.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.attn.c_attn.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.17.attn.c_attn.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.17.attn.c_proj.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.17.attn.c_proj.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.17.gate.0.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.gate.0.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.gate.2.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.gate.2.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.17.ln_1.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.17.ln_1.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.17.ln_2.bias": "pytorch_model-00008-of-00012.bin",
+    "blocks.17.ln_2.weight": "pytorch_model-00008-of-00012.bin",
+    "blocks.18.area_norm.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.area_norm.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.0.net.0.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.0.net.0.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.0.net.2.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.0.net.2.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.1.net.0.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.1.net.0.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.1.net.2.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.1.net.2.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.2.net.0.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.2.net.0.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.2.net.2.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.2.net.2.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.3.net.0.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.3.net.0.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.3.net.2.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.3.net.2.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.4.net.0.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.4.net.0.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.4.net.2.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.4.net.2.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.5.net.0.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.5.net.0.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.5.net.2.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.5.net.2.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.6.net.0.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.6.net.0.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.6.net.2.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.6.net.2.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.7.net.0.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.7.net.0.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.7.net.2.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.areas.7.net.2.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.attn.c_attn.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.attn.c_attn.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.attn.c_proj.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.attn.c_proj.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.gate.0.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.gate.0.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.gate.2.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.gate.2.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.ln_1.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.ln_1.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.ln_2.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.18.ln_2.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.19.area_norm.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.19.area_norm.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.19.areas.0.net.0.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.19.areas.0.net.0.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.19.areas.0.net.2.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.19.areas.0.net.2.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.19.areas.1.net.0.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.19.areas.1.net.0.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.19.areas.1.net.2.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.19.areas.1.net.2.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.19.areas.2.net.0.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.19.areas.2.net.0.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.19.areas.2.net.2.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.19.areas.2.net.2.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.19.areas.3.net.0.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.19.areas.3.net.0.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.19.areas.3.net.2.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.19.areas.3.net.2.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.19.areas.4.net.0.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.19.areas.4.net.0.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.19.areas.4.net.2.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.19.areas.4.net.2.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.19.areas.5.net.0.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.19.areas.5.net.0.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.19.areas.5.net.2.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.19.areas.5.net.2.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.19.areas.6.net.0.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.19.areas.6.net.0.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.19.areas.6.net.2.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.19.areas.6.net.2.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.19.areas.7.net.0.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.19.areas.7.net.0.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.19.areas.7.net.2.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.19.areas.7.net.2.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.19.attn.c_attn.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.19.attn.c_attn.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.19.attn.c_proj.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.19.attn.c_proj.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.19.gate.0.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.19.gate.0.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.19.gate.2.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.19.gate.2.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.19.ln_1.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.19.ln_1.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.19.ln_2.bias": "pytorch_model-00009-of-00012.bin",
+    "blocks.19.ln_2.weight": "pytorch_model-00009-of-00012.bin",
+    "blocks.2.area_norm.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.area_norm.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.0.net.0.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.0.net.0.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.0.net.2.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.0.net.2.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.1.net.0.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.1.net.0.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.1.net.2.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.1.net.2.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.2.net.0.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.2.net.0.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.2.net.2.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.2.net.2.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.3.net.0.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.3.net.0.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.3.net.2.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.3.net.2.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.4.net.0.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.4.net.0.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.4.net.2.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.4.net.2.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.5.net.0.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.5.net.0.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.5.net.2.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.5.net.2.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.6.net.0.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.6.net.0.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.6.net.2.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.6.net.2.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.7.net.0.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.7.net.0.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.7.net.2.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.areas.7.net.2.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.attn.c_attn.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.attn.c_attn.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.attn.c_proj.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.attn.c_proj.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.gate.0.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.gate.0.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.gate.2.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.gate.2.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.ln_1.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.ln_1.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.ln_2.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.2.ln_2.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.20.area_norm.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.area_norm.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.0.net.0.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.0.net.0.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.0.net.2.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.0.net.2.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.1.net.0.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.1.net.0.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.1.net.2.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.1.net.2.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.2.net.0.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.2.net.0.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.2.net.2.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.2.net.2.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.3.net.0.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.3.net.0.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.3.net.2.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.3.net.2.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.4.net.0.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.4.net.0.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.4.net.2.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.4.net.2.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.5.net.0.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.5.net.0.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.5.net.2.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.5.net.2.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.6.net.0.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.6.net.0.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.6.net.2.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.6.net.2.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.7.net.0.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.7.net.0.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.7.net.2.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.areas.7.net.2.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.attn.c_attn.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.attn.c_attn.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.attn.c_proj.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.attn.c_proj.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.gate.0.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.gate.0.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.gate.2.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.gate.2.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.ln_1.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.ln_1.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.ln_2.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.20.ln_2.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.area_norm.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.21.area_norm.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.21.areas.0.net.0.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.areas.0.net.0.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.areas.0.net.2.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.areas.0.net.2.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.areas.1.net.0.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.areas.1.net.0.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.areas.1.net.2.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.areas.1.net.2.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.areas.2.net.0.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.areas.2.net.0.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.areas.2.net.2.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.areas.2.net.2.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.areas.3.net.0.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.areas.3.net.0.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.areas.3.net.2.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.areas.3.net.2.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.areas.4.net.0.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.areas.4.net.0.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.areas.4.net.2.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.areas.4.net.2.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.areas.5.net.0.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.areas.5.net.0.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.areas.5.net.2.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.areas.5.net.2.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.areas.6.net.0.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.21.areas.6.net.0.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.21.areas.6.net.2.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.21.areas.6.net.2.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.21.areas.7.net.0.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.21.areas.7.net.0.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.21.areas.7.net.2.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.21.areas.7.net.2.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.21.attn.c_attn.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.attn.c_attn.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.attn.c_proj.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.attn.c_proj.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.gate.0.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.21.gate.0.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.21.gate.2.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.21.gate.2.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.21.ln_1.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.ln_1.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.ln_2.bias": "pytorch_model-00010-of-00012.bin",
+    "blocks.21.ln_2.weight": "pytorch_model-00010-of-00012.bin",
+    "blocks.22.area_norm.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.area_norm.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.0.net.0.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.0.net.0.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.0.net.2.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.0.net.2.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.1.net.0.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.1.net.0.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.1.net.2.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.1.net.2.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.2.net.0.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.2.net.0.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.2.net.2.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.2.net.2.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.3.net.0.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.3.net.0.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.3.net.2.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.3.net.2.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.4.net.0.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.4.net.0.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.4.net.2.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.4.net.2.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.5.net.0.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.5.net.0.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.5.net.2.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.5.net.2.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.6.net.0.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.6.net.0.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.6.net.2.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.6.net.2.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.7.net.0.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.7.net.0.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.7.net.2.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.areas.7.net.2.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.attn.c_attn.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.attn.c_attn.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.attn.c_proj.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.attn.c_proj.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.gate.0.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.gate.0.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.gate.2.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.gate.2.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.ln_1.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.ln_1.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.ln_2.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.22.ln_2.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.area_norm.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.area_norm.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.0.net.0.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.0.net.0.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.0.net.2.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.0.net.2.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.1.net.0.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.1.net.0.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.1.net.2.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.1.net.2.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.2.net.0.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.2.net.0.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.2.net.2.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.2.net.2.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.3.net.0.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.3.net.0.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.3.net.2.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.3.net.2.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.4.net.0.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.4.net.0.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.4.net.2.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.4.net.2.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.5.net.0.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.5.net.0.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.5.net.2.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.5.net.2.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.6.net.0.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.6.net.0.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.6.net.2.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.6.net.2.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.7.net.0.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.7.net.0.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.7.net.2.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.areas.7.net.2.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.attn.c_attn.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.attn.c_attn.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.attn.c_proj.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.attn.c_proj.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.gate.0.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.gate.0.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.gate.2.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.gate.2.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.ln_1.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.ln_1.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.ln_2.bias": "pytorch_model-00011-of-00012.bin",
+    "blocks.23.ln_2.weight": "pytorch_model-00011-of-00012.bin",
+    "blocks.3.area_norm.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.3.area_norm.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.3.areas.0.net.0.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.0.net.0.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.0.net.2.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.0.net.2.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.1.net.0.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.1.net.0.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.1.net.2.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.1.net.2.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.2.net.0.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.2.net.0.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.2.net.2.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.2.net.2.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.3.net.0.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.3.net.0.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.3.net.2.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.3.net.2.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.4.net.0.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.4.net.0.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.4.net.2.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.4.net.2.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.5.net.0.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.5.net.0.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.5.net.2.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.5.net.2.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.6.net.0.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.6.net.0.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.6.net.2.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.6.net.2.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.areas.7.net.0.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.3.areas.7.net.0.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.3.areas.7.net.2.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.3.areas.7.net.2.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.3.attn.c_attn.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.attn.c_attn.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.attn.c_proj.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.attn.c_proj.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.gate.0.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.3.gate.0.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.3.gate.2.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.3.gate.2.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.3.ln_1.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.ln_1.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.ln_2.bias": "pytorch_model-00002-of-00012.bin",
+    "blocks.3.ln_2.weight": "pytorch_model-00002-of-00012.bin",
+    "blocks.4.area_norm.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.area_norm.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.0.net.0.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.0.net.0.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.0.net.2.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.0.net.2.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.1.net.0.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.1.net.0.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.1.net.2.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.1.net.2.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.2.net.0.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.2.net.0.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.2.net.2.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.2.net.2.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.3.net.0.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.3.net.0.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.3.net.2.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.3.net.2.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.4.net.0.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.4.net.0.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.4.net.2.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.4.net.2.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.5.net.0.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.5.net.0.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.5.net.2.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.5.net.2.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.6.net.0.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.6.net.0.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.6.net.2.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.6.net.2.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.7.net.0.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.7.net.0.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.7.net.2.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.areas.7.net.2.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.attn.c_attn.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.attn.c_attn.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.attn.c_proj.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.attn.c_proj.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.gate.0.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.gate.0.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.gate.2.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.gate.2.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.ln_1.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.ln_1.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.ln_2.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.4.ln_2.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.area_norm.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.area_norm.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.0.net.0.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.0.net.0.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.0.net.2.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.0.net.2.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.1.net.0.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.1.net.0.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.1.net.2.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.1.net.2.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.2.net.0.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.2.net.0.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.2.net.2.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.2.net.2.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.3.net.0.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.3.net.0.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.3.net.2.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.3.net.2.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.4.net.0.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.4.net.0.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.4.net.2.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.4.net.2.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.5.net.0.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.5.net.0.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.5.net.2.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.5.net.2.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.6.net.0.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.6.net.0.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.6.net.2.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.6.net.2.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.7.net.0.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.7.net.0.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.7.net.2.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.areas.7.net.2.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.attn.c_attn.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.attn.c_attn.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.attn.c_proj.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.attn.c_proj.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.gate.0.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.gate.0.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.gate.2.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.gate.2.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.ln_1.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.ln_1.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.ln_2.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.5.ln_2.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.6.area_norm.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.area_norm.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.0.net.0.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.6.areas.0.net.0.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.6.areas.0.net.2.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.0.net.2.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.1.net.0.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.1.net.0.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.1.net.2.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.1.net.2.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.2.net.0.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.2.net.0.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.2.net.2.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.2.net.2.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.3.net.0.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.3.net.0.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.3.net.2.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.3.net.2.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.4.net.0.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.4.net.0.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.4.net.2.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.4.net.2.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.5.net.0.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.5.net.0.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.5.net.2.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.5.net.2.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.6.net.0.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.6.net.0.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.6.net.2.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.6.net.2.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.7.net.0.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.7.net.0.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.7.net.2.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.areas.7.net.2.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.attn.c_attn.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.6.attn.c_attn.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.6.attn.c_proj.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.6.attn.c_proj.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.6.gate.0.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.gate.0.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.gate.2.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.gate.2.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.6.ln_1.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.6.ln_1.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.6.ln_2.bias": "pytorch_model-00003-of-00012.bin",
+    "blocks.6.ln_2.weight": "pytorch_model-00003-of-00012.bin",
+    "blocks.7.area_norm.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.area_norm.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.0.net.0.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.0.net.0.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.0.net.2.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.0.net.2.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.1.net.0.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.1.net.0.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.1.net.2.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.1.net.2.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.2.net.0.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.2.net.0.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.2.net.2.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.2.net.2.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.3.net.0.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.3.net.0.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.3.net.2.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.3.net.2.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.4.net.0.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.4.net.0.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.4.net.2.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.4.net.2.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.5.net.0.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.5.net.0.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.5.net.2.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.5.net.2.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.6.net.0.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.6.net.0.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.6.net.2.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.6.net.2.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.7.net.0.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.7.net.0.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.7.net.2.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.areas.7.net.2.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.attn.c_attn.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.attn.c_attn.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.attn.c_proj.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.attn.c_proj.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.gate.0.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.gate.0.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.gate.2.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.gate.2.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.ln_1.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.ln_1.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.ln_2.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.7.ln_2.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.8.area_norm.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.area_norm.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.areas.0.net.0.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.8.areas.0.net.0.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.8.areas.0.net.2.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.8.areas.0.net.2.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.8.areas.1.net.0.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.8.areas.1.net.0.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.8.areas.1.net.2.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.8.areas.1.net.2.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.8.areas.2.net.0.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.8.areas.2.net.0.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.8.areas.2.net.2.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.areas.2.net.2.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.areas.3.net.0.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.areas.3.net.0.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.areas.3.net.2.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.areas.3.net.2.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.areas.4.net.0.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.areas.4.net.0.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.areas.4.net.2.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.areas.4.net.2.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.areas.5.net.0.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.areas.5.net.0.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.areas.5.net.2.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.areas.5.net.2.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.areas.6.net.0.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.areas.6.net.0.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.areas.6.net.2.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.areas.6.net.2.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.areas.7.net.0.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.areas.7.net.0.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.areas.7.net.2.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.areas.7.net.2.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.attn.c_attn.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.8.attn.c_attn.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.8.attn.c_proj.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.8.attn.c_proj.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.8.gate.0.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.gate.0.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.gate.2.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.gate.2.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.8.ln_1.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.8.ln_1.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.8.ln_2.bias": "pytorch_model-00004-of-00012.bin",
+    "blocks.8.ln_2.weight": "pytorch_model-00004-of-00012.bin",
+    "blocks.9.area_norm.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.area_norm.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.0.net.0.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.0.net.0.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.0.net.2.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.0.net.2.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.1.net.0.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.1.net.0.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.1.net.2.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.1.net.2.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.2.net.0.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.2.net.0.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.2.net.2.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.2.net.2.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.3.net.0.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.3.net.0.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.3.net.2.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.3.net.2.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.4.net.0.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.4.net.0.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.4.net.2.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.4.net.2.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.5.net.0.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.5.net.0.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.5.net.2.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.5.net.2.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.6.net.0.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.6.net.0.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.6.net.2.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.6.net.2.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.7.net.0.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.7.net.0.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.7.net.2.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.areas.7.net.2.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.attn.c_attn.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.attn.c_attn.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.attn.c_proj.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.attn.c_proj.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.gate.0.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.gate.0.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.gate.2.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.gate.2.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.ln_1.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.ln_1.weight": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.ln_2.bias": "pytorch_model-00005-of-00012.bin",
+    "blocks.9.ln_2.weight": "pytorch_model-00005-of-00012.bin",
+    "lm_head.weight": "pytorch_model-00012-of-00012.bin",
+    "ln_f.bias": "pytorch_model-00011-of-00012.bin",
+    "ln_f.weight": "pytorch_model-00011-of-00012.bin",
+    "wpe.weight": "pytorch_model-00001-of-00012.bin",
+    "wte.weight": "pytorch_model-00001-of-00012.bin"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}