Super-squash branch 'main' using huggingface_hub

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,119 @@
+---
+license: apache-2.0
+base_model: BEE-spoke-data/smol_llama-220M-GQA
+datasets:
+- BEE-spoke-data/pypi_clean-deduped
+- bigcode/the-stack-smol-xl
+- EleutherAI/proof-pile-2
+language:
+- en
+tags:
+- python
+- codegen
+- markdown
+- smol_llama
+metrics:
+- accuracy
+inference:
+  parameters:
+    max_new_tokens: 64
+    min_new_tokens: 8
+    do_sample: true
+    epsilon_cutoff: 0.0008
+    temperature: 0.3
+    top_p: 0.9
+    repetition_penalty: 1.02
+    no_repeat_ngram_size: 8
+    renormalize_logits: true
+widget:
+- text: |
+    def add_numbers(a, b):
+        return
+  example_title: Add Numbers Function
+- text: |
+    class Car:
+        def __init__(self, make, model):
+            self.make = make
+            self.model = model
+
+        def display_car(self):
+  example_title: Car Class
+- text: |
+    import pandas as pd
+    data = {'Name': ['Tom', 'Nick', 'John'], 'Age': [20, 21, 19]}
+    df = pd.DataFrame(data).convert_dtypes()
+    # eda
+  example_title: Pandas DataFrame
+- text: |
+    def factorial(n):
+        if n == 0:
+            return 1
+        else:
+  example_title: Factorial Function
+- text: |
+    def fibonacci(n):
+        if n <= 0:
+            raise ValueError("Incorrect input")
+        elif n == 1:
+            return 0
+        elif n == 2:
+            return 1
+        else:
+  example_title: Fibonacci Function
+- text: |
+    import matplotlib.pyplot as plt
+    import numpy as np
+    x = np.linspace(0, 10, 100)
+    # simple plot
+  example_title: Matplotlib Plot
+- text: |
+    def reverse_string(s:str) -> str:
+        return
+  example_title: Reverse String Function
+- text: |
+    def is_palindrome(word:str) -> bool:
+        return
+  example_title: Palindrome Function
+- text: |
+    def bubble_sort(lst: list):
+        n = len(lst)
+        for i in range(n):
+            for j in range(0, n-i-1):
+  example_title: Bubble Sort Function
+- text: |
+    def binary_search(arr, low, high, x):
+        if high >= low:
+            mid = (high + low) // 2
+            if arr[mid] == x:
+                return mid
+            elif arr[mid] > x:
+  example_title: Binary Search Function
+pipeline_tag: text-generation
+---
+
+# BEE-spoke-data/beecoder-220M-python
+
+
+
+
+This is `BEE-spoke-data/smol_llama-220M-GQA` fine-tuned for code generation on:
+
+- filtered version of stack-smol-XL
+- deduped version of 'algebraic stack' from proof-pile-2
+- cleaned and deduped pypi (last dataset)
+
+This model (and the base model) were both trained using ctx length 2048.  
+
+## examples
+
+> Example script for inference testing: [here](https://gist.github.com/pszemraj/c7738f664a64b935a558974d23a7aa8c)
+
+It has its limitations at 220M, but seems decent for single-line or docstring generation, and/or being used for speculative decoding for such purposes.
+
+
+
+![image/png](https://cdn-uploads.huggingface.co/production/uploads/60bccec062080d33f875cd0c/bLrtpr7Vi_MPvtF7mozDN.png)
+
+The screenshot is on CPU on a laptop.
+
+---

config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "BEE-spoke-data/smol_llama-220M-GQA-pypi-v2",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "max_position_embeddings": 2048,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 10,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.36.2",
+  "use_cache": false,
+  "vocab_size": 32128
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.36.2",
+  "use_cache": false
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e24cdfb3893f42e077d13d91c94eb5cfeb3e13d3d979b327dd15e92b1ae56a10
+size 435736840

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,45 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "trust_remote_code": false,
+  "unk_token": "<unk>",
+  "use_default_system_prompt": true,
+  "use_fast": true,
+  "use_fast_tokenizer": true
+}