Microbot-v1 / tokenizer.json

Upload 9 files

7e36384 verified 21 days ago

20.4 kB

	{
	"version": "1.0",
	"truncation": null,
	"padding": null,
	"added_tokens": [
	{
	"id": 0,
	"content": "[PAD]",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 1,
	"content": "[UNK]",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 2,
	"content": "[BOS]",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 3,
	"content": "[EOS]",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	}
	],
	"normalizer": null,
	"pre_tokenizer": {
	"type": "Whitespace"
	},
	"post_processor": null,
	"decoder": null,
	"model": {
	"type": "BPE",
	"dropout": null,
	"unk_token": "[UNK]",
	"continuing_subword_prefix": null,
	"end_of_word_suffix": null,
	"fuse_unk": false,
	"byte_fallback": false,
	"ignore_merges": false,
	"vocab": {
	"[PAD]": 0,
	"[UNK]": 1,
	"[BOS]": 2,
	"[EOS]": 3,
	"\"": 4,
	"$": 5,
	"&": 6,
	"(": 7,
	")": 8,
	",": 9,
	"-": 10,
	".": 11,
	"/": 12,
	"0": 13,
	"1": 14,
	"2": 15,
	"3": 16,
	"4": 17,
	"5": 18,
	"6": 19,
	"7": 20,
	"8": 21,
	"9": 22,
	":": 23,
	"=": 24,
	"@": 25,
	"A": 26,
	"C": 27,
	"D": 28,
	"E": 29,
	"I": 30,
	"J": 31,
	"L": 32,
	"M": 33,
	"O": 34,
	"P": 35,
	"R": 36,
	"S": 37,
	"T": 38,
	"V": 39,
	"W": 40,
	"[": 41,
	"\\": 42,
	"]": 43,
	"^": 44,
	"_": 45,
	"a": 46,
	"b": 47,
	"c": 48,
	"d": 49,
	"e": 50,
	"f": 51,
	"g": 52,
	"h": 53,
	"i": 54,
	"j": 55,
	"k": 56,
	"l": 57,
	"m": 58,
	"n": 59,
	"o": 60,
	"p": 61,
	"r": 62,
	"s": 63,
	"t": 64,
	"u": 65,
	"v": 66,
	"w": 67,
	"x": 68,
	"y": 69,
	"{": 70,
	"\|": 71,
	"}": 72,
	"\":": 73,
	"\",": 74,
	"es": 75,
	"on": 76,
	"ion": 77,
	"de": 78,
	"it": 79,
	"ve": 80,
	"re": 81,
	"in": 82,
	"co": 83,
	"ar": 84,
	"ara": 85,
	"pe": 86,
	"ru": 87,
	"ty": 88,
	"\"^": 89,
	"vel": 90,
	"aravel": 91,
	"ac": 92,
	"od": 93,
	"gn": 94,
	"is": 95,
	"pre": 96,
	"ition": 97,
	"cogn": 98,
	"cognition": 99,
	"le": 100,
	"--": 101,
	"laravel": 102,
	"ts": 103,
	"ist": 104,
	"precognition": 105,
	"dist": 106,
	"np": 107,
	"or": 108,
	"ri": 109,
	"ule": 110,
	"odule": 111,
	"at": 112,
	"pac": 113,
	"ut": 114,
	"},": 115,
	"npm": 116,
	"ch": 117,
	"atch": 118,
	"tru": 119,
	"true": 120,
	"20": 121,
	"cri": 122,
	"pes": 123,
	"types": 124,
	"crip": 125,
	"ag": 126,
	"ci": 127,
	"lin": 128,
	"nde": 129,
	"nci": 130,
	"watch": 131,
	"pende": 132,
	"type": 133,
	"ncies": 134,
	"pendencies": 135,
	"il": 136,
	"run": 137,
	"module": 138,
	"wor": 139,
	"lint": 140,
	"\"@": 141,
	"er": 142,
	"git": 143,
	"tion": 144,
	"ub": 145,
	"eslint": 146,
	"com": 147,
	"tsc": 148,
	"kag": 149,
	"packag": 150,
	"\\\"": 151,
	"bu": 152,
	"rs": 153,
	"vers": 154,
	"ild": 155,
	"build": 156,
	"version": 157,
	"],": 158,
	"ks": 159,
	"no": 160,
	"res": 161,
	"ue": 162,
	"dex": 163,
	"index": 164,
	"act": 165,
	"works": 166,
	"workspac": 167,
	"\".": 168,
	"&&": 169,
	"//": 170,
	"17": 171,
	"://": 172,
	"De": 173,
	"ES": 174,
	"Laravel": 175,
	"Module": 176,
	"al": 177,
	"as": 178,
	"cl": 179,
	"ht": 180,
	"hub": 181,
	"lod": 182,
	"me": 183,
	"ol": 184,
	"ps": 185,
	"pin": 186,
	"rf": 187,
	"rm": 188,
	"tps": 189,
	"vue": 190,
	"react": 191,
	"2020": 192,
	"github": 193,
	"packages": 194,
	"\"./": 195,
	"Dependencies": 196,
	"ES2020": 197,
	"ash": 198,
	"https": 199,
	"lodash": 200,
	"pine": 201,
	"Ch": 202,
	"ck": 203,
	"et": 204,
	"eck": 205,
	"js": 206,
	"dependencies": 207,
	"typescrip": 208,
	"alpine": 209,
	"Check": 210,
	"typescript": 211,
	"nt": 212,
	"typeCheck": 213,
	"12": 214,
	"ic": 215,
	"ly": 216,
	"na": 217,
	"os": 218,
	"pres": 219,
	"rc": 220,
	"scrip": 221,
	"ur": 222,
	"vDependencies": 223,
	"est": 224,
	"devDependencies": 225,
	"node": 226,
	"workspaces": 227,
	"scripts": 228,
	"\"/": 229,
	".\",": 230,
	"21": 231,
	"Di": 232,
	"Em": 233,
	"IT": 234,
	"Int": 235,
	"Js": 236,
	"MIT": 237,
	"On": 238,
	"Op": 239,
	"Out": 240,
	"Pre": 241,
	"Res": 242,
	"Watch": 243,
	"ain": 244,
	"aut": 245,
	"ct": 246,
	"ds": 247,
	"des": 248,
	"en": 249,
	"ey": 250,
	"fil": 251,
	"gin": 252,
	"get": 253,
	"ho": 254,
	"hor": 255,
	"hOn": 256,
	"ile": 257,
	"key": 258,
	"lu": 259,
	"lis": 260,
	"lic": 261,
	"main": 262,
	"op": 263,
	"out": 264,
	"put": 265,
	"pag": 266,
	"pub": 267,
	"pos": 268,
	"pile": 269,
	"plu": 270,
	"rOp": 271,
	"se": 272,
	"st": 273,
	"src": 274,
	"tar": 275,
	"ude": 276,
	"esModule": 277,
	"onModule": 278,
	"decl": 279,
	"itor": 280,
	"veJs": 281,
	"veWatch": 282,
	"repos": 283,
	"incl": 284,
	"aration": 285,
	"prepub": 286,
	"rict": 287,
	"ution": 288,
	"cription": 289,
	"moduleRes": 290,
	"words": 291,
	"erop": 292,
	"erveWatch": 293,
	"tions": 294,
	"compile": 295,
	"noEm": 296,
	"resol": 297,
	"workspace": 298,
	"mepag": 299,
	"olution": 300,
	"name": 301,
	"preserveWatch": 302,
	"url": 303,
	"Dir": 304,
	"Interop": 305,
	"Output": 306,
	"Precognition": 307,
	"author": 308,
	"description": 309,
	"ense": 310,
	"files": 311,
	"homepag": 312,
	"hOnly": 313,
	"keywords": 314,
	"lishOnly": 315,
	"license": 316,
	"outDir": 317,
	"plugin": 318,
	"rOptions": 319,
	"strict": 320,
	"target": 321,
	"esModuleInterop": 322,
	"declaration": 323,
	"itory": 324,
	"veJsonModule": 325,
	"repository": 326,
	"include": 327,
	"prepublishOnly": 328,
	"moduleResolution": 329,
	"compilerOptions": 330,
	"noEmit": 331,
	"resolveJsonModule": 332,
	"preserveWatchOutput": 333,
	"homepage": 334,
	").\",": 335,
	"=$": 336,
	"_packag": 337,
	"_version": 338,
	"e_version": 339,
	"kg": 340,
	"pkg": 341,
	"set": 342,
	"test": 343,
	"core": 344,
	"peer": 345,
	"npm_packag": 346,
	"peerDependencies": 347,
	"npm_package_version": 348,
	"18": 349,
	"19": 350,
	"vit": 351,
	"\|\|": 352,
	"link": 353,
	"alpinejs": 354,
	"vitest": 355
	},
	"merges": [
	[
	"\"",
	":"
	],
	[
	"\"",
	","
	],
	[
	"e",
	"s"
	],
	[
	"o",
	"n"
	],
	[
	"i",
	"on"
	],
	[
	"d",
	"e"
	],
	[
	"i",
	"t"
	],
	[
	"v",
	"e"
	],
	[
	"r",
	"e"
	],
	[
	"i",
	"n"
	],
	[
	"c",
	"o"
	],
	[
	"a",
	"r"
	],
	[
	"ar",
	"a"
	],
	[
	"p",
	"e"
	],
	[
	"r",
	"u"
	],
	[
	"t",
	"y"
	],
	[
	"\"",
	"^"
	],
	[
	"ve",
	"l"
	],
	[
	"ara",
	"vel"
	],
	[
	"a",
	"c"
	],
	[
	"o",
	"d"
	],
	[
	"g",
	"n"
	],
	[
	"i",
	"s"
	],
	[
	"p",
	"re"
	],
	[
	"it",
	"ion"
	],
	[
	"co",
	"gn"
	],
	[
	"cogn",
	"ition"
	],
	[
	"l",
	"e"
	],
	[
	"-",
	"-"
	],
	[
	"l",
	"aravel"
	],
	[
	"t",
	"s"
	],
	[
	"is",
	"t"
	],
	[
	"pre",
	"cognition"
	],
	[
	"d",
	"ist"
	],
	[
	"n",
	"p"
	],
	[
	"o",
	"r"
	],
	[
	"r",
	"i"
	],
	[
	"u",
	"le"
	],
	[
	"od",
	"ule"
	],
	[
	"a",
	"t"
	],
	[
	"p",
	"ac"
	],
	[
	"u",
	"t"
	],
	[
	"}",
	","
	],
	[
	"np",
	"m"
	],
	[
	"c",
	"h"
	],
	[
	"at",
	"ch"
	],
	[
	"t",
	"ru"
	],
	[
	"tru",
	"e"
	],
	[
	"2",
	"0"
	],
	[
	"c",
	"ri"
	],
	[
	"p",
	"es"
	],
	[
	"ty",
	"pes"
	],
	[
	"cri",
	"p"
	],
	[
	"a",
	"g"
	],
	[
	"c",
	"i"
	],
	[
	"l",
	"in"
	],
	[
	"n",
	"de"
	],
	[
	"n",
	"ci"
	],
	[
	"w",
	"atch"
	],
	[
	"pe",
	"nde"
	],
	[
	"ty",
	"pe"
	],
	[
	"nci",
	"es"
	],
	[
	"pende",
	"ncies"
	],
	[
	"i",
	"l"
	],
	[
	"ru",
	"n"
	],
	[
	"m",
	"odule"
	],
	[
	"w",
	"or"
	],
	[
	"lin",
	"t"
	],
	[
	"\"",
	"@"
	],
	[
	"e",
	"r"
	],
	[
	"g",
	"it"
	],
	[
	"t",
	"ion"
	],
	[
	"u",
	"b"
	],
	[
	"es",
	"lint"
	],
	[
	"co",
	"m"
	],
	[
	"ts",
	"c"
	],
	[
	"k",
	"ag"
	],
	[
	"pac",
	"kag"
	],
	[
	"\\",
	"\""
	],
	[
	"b",
	"u"
	],
	[
	"r",
	"s"
	],
	[
	"ve",
	"rs"
	],
	[
	"il",
	"d"
	],
	[
	"bu",
	"ild"
	],
	[
	"vers",
	"ion"
	],
	[
	"]",
	","
	],
	[
	"k",
	"s"
	],
	[
	"n",
	"o"
	],
	[
	"r",
	"es"
	],
	[
	"u",
	"e"
	],
	[
	"de",
	"x"
	],
	[
	"in",
	"dex"
	],
	[
	"ac",
	"t"
	],
	[
	"wor",
	"ks"
	],
	[
	"works",
	"pac"
	],
	[
	"\"",
	"."
	],
	[
	"&",
	"&"
	],
	[
	"/",
	"/"
	],
	[
	"1",
	"7"
	],
	[
	":",
	"//"
	],
	[
	"D",
	"e"
	],
	[
	"E",
	"S"
	],
	[
	"L",
	"aravel"
	],
	[
	"M",
	"odule"
	],
	[
	"a",
	"l"
	],
	[
	"a",
	"s"
	],
	[
	"c",
	"l"
	],
	[
	"h",
	"t"
	],
	[
	"h",
	"ub"
	],
	[
	"l",
	"od"
	],
	[
	"m",
	"e"
	],
	[
	"o",
	"l"
	],
	[
	"p",
	"s"
	],
	[
	"p",
	"in"
	],
	[
	"r",
	"f"
	],
	[
	"r",
	"m"
	],
	[
	"t",
	"ps"
	],
	[
	"v",
	"ue"
	],
	[
	"re",
	"act"
	],
	[
	"20",
	"20"
	],
	[
	"git",
	"hub"
	],
	[
	"packag",
	"es"
	],
	[
	"\".",
	"/"
	],
	[
	"De",
	"pendencies"
	],
	[
	"ES",
	"2020"
	],
	[
	"as",
	"h"
	],
	[
	"ht",
	"tps"
	],
	[
	"lod",
	"ash"
	],
	[
	"pin",
	"e"
	],
	[
	"C",
	"h"
	],
	[
	"c",
	"k"
	],
	[
	"e",
	"t"
	],
	[
	"e",
	"ck"
	],
	[
	"j",
	"s"
	],
	[
	"de",
	"pendencies"
	],
	[
	"types",
	"crip"
	],
	[
	"al",
	"pine"
	],
	[
	"Ch",
	"eck"
	],
	[
	"typescrip",
	"t"
	],
	[
	"n",
	"t"
	],
	[
	"type",
	"Check"
	],
	[
	"1",
	"2"
	],
	[
	"i",
	"c"
	],
	[
	"l",
	"y"
	],
	[
	"n",
	"a"
	],
	[
	"o",
	"s"
	],
	[
	"p",
	"res"
	],
	[
	"r",
	"c"
	],
	[
	"s",
	"crip"
	],
	[
	"u",
	"r"
	],
	[
	"v",
	"Dependencies"
	],
	[
	"es",
	"t"
	],
	[
	"de",
	"vDependencies"
	],
	[
	"no",
	"de"
	],
	[
	"workspac",
	"es"
	],
	[
	"scrip",
	"ts"
	],
	[
	"\"",
	"/"
	],
	[
	".",
	"\","
	],
	[
	"2",
	"1"
	],
	[
	"D",
	"i"
	],
	[
	"E",
	"m"
	],
	[
	"I",
	"T"
	],
	[
	"I",
	"nt"
	],
	[
	"J",
	"s"
	],
	[
	"M",
	"IT"
	],
	[
	"O",
	"n"
	],
	[
	"O",
	"p"
	],
	[
	"O",
	"ut"
	],
	[
	"P",
	"re"
	],
	[
	"R",
	"es"
	],
	[
	"W",
	"atch"
	],
	[
	"a",
	"in"
	],
	[
	"a",
	"ut"
	],
	[
	"c",
	"t"
	],
	[
	"d",
	"s"
	],
	[
	"d",
	"es"
	],
	[
	"e",
	"n"
	],
	[
	"e",
	"y"
	],
	[
	"f",
	"il"
	],
	[
	"g",
	"in"
	],
	[
	"g",
	"et"
	],
	[
	"h",
	"o"
	],
	[
	"h",
	"or"
	],
	[
	"h",
	"On"
	],
	[
	"i",
	"le"
	],
	[
	"k",
	"ey"
	],
	[
	"l",
	"u"
	],
	[
	"l",
	"is"
	],
	[
	"l",
	"ic"
	],
	[
	"m",
	"ain"
	],
	[
	"o",
	"p"
	],
	[
	"o",
	"ut"
	],
	[
	"p",
	"ut"
	],
	[
	"p",
	"ag"
	],
	[
	"p",
	"ub"
	],
	[
	"p",
	"os"
	],
	[
	"p",
	"ile"
	],
	[
	"p",
	"lu"
	],
	[
	"r",
	"Op"
	],
	[
	"s",
	"e"
	],
	[
	"s",
	"t"
	],
	[
	"s",
	"rc"
	],
	[
	"t",
	"ar"
	],
	[
	"u",
	"de"
	],
	[
	"es",
	"Module"
	],
	[
	"on",
	"Module"
	],
	[
	"de",
	"cl"
	],
	[
	"it",
	"or"
	],
	[
	"ve",
	"Js"
	],
	[
	"ve",
	"Watch"
	],
	[
	"re",
	"pos"
	],
	[
	"in",
	"cl"
	],
	[
	"ara",
	"tion"
	],
	[
	"pre",
	"pub"
	],
	[
	"ri",
	"ct"
	],
	[
	"ut",
	"ion"
	],
	[
	"crip",
	"tion"
	],
	[
	"module",
	"Res"
	],
	[
	"wor",
	"ds"
	],
	[
	"er",
	"op"
	],
	[
	"er",
	"veWatch"
	],
	[
	"tion",
	"s"
	],
	[
	"com",
	"pile"
	],
	[
	"no",
	"Em"
	],
	[
	"res",
	"ol"
	],
	[
	"workspac",
	"e"
	],
	[
	"me",
	"pag"
	],
	[
	"ol",
	"ution"
	],
	[
	"na",
	"me"
	],
	[
	"pres",
	"erveWatch"
	],
	[
	"ur",
	"l"
	],
	[
	"Di",
	"r"
	],
	[
	"Int",
	"erop"
	],
	[
	"Out",
	"put"
	],
	[
	"Pre",
	"cognition"
	],
	[
	"aut",
	"hor"
	],
	[
	"des",
	"cription"
	],
	[
	"en",
	"se"
	],
	[
	"fil",
	"es"
	],
	[
	"ho",
	"mepag"
	],
	[
	"hOn",
	"ly"
	],
	[
	"key",
	"words"
	],
	[
	"lis",
	"hOnly"
	],
	[
	"lic",
	"ense"
	],
	[
	"out",
	"Dir"
	],
	[
	"plu",
	"gin"
	],
	[
	"rOp",
	"tions"
	],
	[
	"st",
	"rict"
	],
	[
	"tar",
	"get"
	],
	[
	"esModule",
	"Interop"
	],
	[
	"decl",
	"aration"
	],
	[
	"itor",
	"y"
	],
	[
	"veJs",
	"onModule"
	],
	[
	"repos",
	"itory"
	],
	[
	"incl",
	"ude"
	],
	[
	"prepub",
	"lishOnly"
	],
	[
	"moduleRes",
	"olution"
	],
	[
	"compile",
	"rOptions"
	],
	[
	"noEm",
	"it"
	],
	[
	"resol",
	"veJsonModule"
	],
	[
	"preserveWatch",
	"Output"
	],
	[
	"homepag",
	"e"
	],
	[
	")",
	".\","
	],
	[
	"=",
	"$"
	],
	[
	"_",
	"packag"
	],
	[
	"_",
	"version"
	],
	[
	"e",
	"_version"
	],
	[
	"k",
	"g"
	],
	[
	"p",
	"kg"
	],
	[
	"s",
	"et"
	],
	[
	"t",
	"est"
	],
	[
	"co",
	"re"
	],
	[
	"pe",
	"er"
	],
	[
	"npm",
	"_packag"
	],
	[
	"peer",
	"Dependencies"
	],
	[
	"npm_packag",
	"e_version"
	],
	[
	"1",
	"8"
	],
	[
	"1",
	"9"
	],
	[
	"v",
	"it"
	],
	[
	"\|",
	"\|"
	],
	[
	"lin",
	"k"
	],
	[
	"alpine",
	"js"
	],
	[
	"vit",
	"est"
	]
	]
	}
	}