tokenizer.json

{
  "version": "1.0",
  "truncation": null,
  "padding": null,
  "added_tokens": [],
  "normalizer": null,
  "pre_tokenizer": {
    "type": "ByteLevel",
    "add_prefix_space": false,
    "trim_offsets": true,
    "use_regex": true
  },
  "post_processor": null,
  "decoder": {
    "type": "ByteLevel",
    "add_prefix_space": true,
    "trim_offsets": true,
    "use_regex": true
  },
  "model": {
    "type": "WordLevel",
    "vocab": {
      "<unk>": 0,
      "<|endoftext|>": 1,
      "<|im_start|>": 2,
      "<|im_end|>": 3,
      "<|object_ref_start|>": 4,
      "<|object_ref_end|>": 5,
      "<|box_start|>": 6,
      "<|box_end|>": 7,
      "<|quad_start|>": 8,
      "<|quad_end|>": 9,
      "<|vision_start|>": 10,
      "<|vision_end|>": 11,
      "<|vision_pad|>": 12,
      "<|image_pad|>": 13,
      "<|video_pad|>": 14,
      "<tool_call>": 15,
      "</tool_call>": 16,
      "<|fim_prefix|>": 17,
      "<|fim_middle|>": 18,
      "<|fim_suffix|>": 19,
      "<|fim_pad|>": 20,
      "<|repo_name|>": 21,
      "<|file_sep|>": 22,
      "<image>": 23,
      "</image>": 24,
      "<ref>": 25,
      "</ref>": 26,
      "<box>": 27,
      "</box>": 28,
      "<quad>": 29,
      "</quad>": 30,
      "<point>": 31,
      "</point>": 32,
      "<slice>": 33,
      "</slice>": 34,
      "<image_id>": 35,
      "</image_id>": 36,
      "<unit>": 37,
      "</unit>": 38,
      "<asr>": 39,
      "</asr>": 40,
      "<query>": 41,
      "</query>": 42,
      "<|audio_start|>": 43,
      "<|audio|>": 44,
      "<|audio_end|>": 45,
      "<|spk_bos|>": 46,
      "<|spk|>": 47,
      "<|spk_eos|>": 48,
      "<|tts_bos|>": 49,
      "<|tts_eos|>": 50,
      "<|listen|>": 51,
      "<|speak|>": 52,
      "<|interrupt|>": 53,
      "<|vad_start|>": 54,
      "<|vad_end|>": 55,
      "<reserved_43>": 56,
      "<reserved_53>": 57,
      "!": 58,
      "\"": 59,
      "#": 60,
      "$": 61,
      "%": 62,
      "&": 63,
      "'": 64,
      "(": 65,
      ")": 66,
      "*": 67,
      "+": 68,
      ",": 69,
      "-": 70,
      ".": 71,
      "/": 72,
      "0": 73,
      "1": 74,
      "2": 75,
      "3": 76,
      "4": 77,
      "5": 78,
      "6": 79,
      "7": 80,
      "8": 81,
      "9": 82,
      ":": 83,
      ";": 84,
      "<": 85,
      "=": 86,
      ">": 87,
      "?": 88,
      "@": 89,
      "A": 90,
      "B": 91,
      "C": 92,
      "D": 93,
      "E": 94,
      "F": 95,
      "G": 96,
      "H": 97,
      "I": 98,
      "J": 99,
      "K": 100,
      "L": 101,
      "M": 102,
      "N": 103,
      "O": 104,
      "P": 105,
      "Q": 106,
      "R": 107,
      "S": 108,
      "T": 109,
      "U": 110,
      "V": 111,
      "W": 112,
      "X": 113,
      "Y": 114,
      "Z": 115,
      "[": 116,
      "\\": 117,
      "]": 118,
      "^": 119,
      "_": 120,
      "`": 121,
      "a": 122,
      "b": 123,
      "c": 124,
      "d": 125,
      "e": 126,
      "f": 127,
      "g": 128,
      "h": 129,
      "i": 130,
      "j": 131,
      "k": 132,
      "l": 133,
      "m": 134,
      "n": 135,
      "o": 136,
      "p": 137,
      "q": 138,
      "r": 139,
      "s": 140,
      "t": 141,
      "u": 142,
      "v": 143,
      "w": 144,
      "x": 145,
      "y": 146,
      "z": 147,
      "{": 148,
      "|": 149
    },
    "unk_token": "<unk>"
  }
}