Upload processor

added_tokens.json

@@ -0,0 +1,770 @@
+{
+  "<garbage_0>": 32014,
+  "<garbage_100>": 32114,
+  "<garbage_101>": 32115,
+  "<garbage_102>": 32116,
+  "<garbage_103>": 32117,
+  "<garbage_104>": 32118,
+  "<garbage_105>": 32119,
+  "<garbage_106>": 32120,
+  "<garbage_107>": 32121,
+  "<garbage_108>": 32122,
+  "<garbage_109>": 32123,
+  "<garbage_10>": 32024,
+  "<garbage_110>": 32124,
+  "<garbage_111>": 32125,
+  "<garbage_112>": 32126,
+  "<garbage_113>": 32127,
+  "<garbage_114>": 32128,
+  "<garbage_115>": 32129,
+  "<garbage_116>": 32130,
+  "<garbage_117>": 32131,
+  "<garbage_118>": 32132,
+  "<garbage_119>": 32133,
+  "<garbage_11>": 32025,
+  "<garbage_120>": 32134,
+  "<garbage_121>": 32135,
+  "<garbage_122>": 32136,
+  "<garbage_123>": 32137,
+  "<garbage_124>": 32138,
+  "<garbage_125>": 32139,
+  "<garbage_126>": 32140,
+  "<garbage_127>": 32141,
+  "<garbage_128>": 32142,
+  "<garbage_129>": 32143,
+  "<garbage_12>": 32026,
+  "<garbage_130>": 32144,
+  "<garbage_131>": 32145,
+  "<garbage_132>": 32146,
+  "<garbage_133>": 32147,
+  "<garbage_134>": 32148,
+  "<garbage_135>": 32149,
+  "<garbage_136>": 32150,
+  "<garbage_137>": 32151,
+  "<garbage_138>": 32152,
+  "<garbage_139>": 32153,
+  "<garbage_13>": 32027,
+  "<garbage_140>": 32154,
+  "<garbage_141>": 32155,
+  "<garbage_142>": 32156,
+  "<garbage_143>": 32157,
+  "<garbage_144>": 32158,
+  "<garbage_145>": 32159,
+  "<garbage_146>": 32160,
+  "<garbage_147>": 32161,
+  "<garbage_148>": 32162,
+  "<garbage_149>": 32163,
+  "<garbage_14>": 32028,
+  "<garbage_150>": 32164,
+  "<garbage_151>": 32165,
+  "<garbage_152>": 32166,
+  "<garbage_153>": 32167,
+  "<garbage_154>": 32168,
+  "<garbage_155>": 32169,
+  "<garbage_156>": 32170,
+  "<garbage_157>": 32171,
+  "<garbage_158>": 32172,
+  "<garbage_159>": 32173,
+  "<garbage_15>": 32029,
+  "<garbage_160>": 32174,
+  "<garbage_161>": 32175,
+  "<garbage_162>": 32176,
+  "<garbage_163>": 32177,
+  "<garbage_164>": 32178,
+  "<garbage_165>": 32179,
+  "<garbage_166>": 32180,
+  "<garbage_167>": 32181,
+  "<garbage_168>": 32182,
+  "<garbage_169>": 32183,
+  "<garbage_16>": 32030,
+  "<garbage_170>": 32184,
+  "<garbage_171>": 32185,
+  "<garbage_172>": 32186,
+  "<garbage_173>": 32187,
+  "<garbage_174>": 32188,
+  "<garbage_175>": 32189,
+  "<garbage_176>": 32190,
+  "<garbage_177>": 32191,
+  "<garbage_178>": 32192,
+  "<garbage_179>": 32193,
+  "<garbage_17>": 32031,
+  "<garbage_180>": 32194,
+  "<garbage_181>": 32195,
+  "<garbage_182>": 32196,
+  "<garbage_183>": 32197,
+  "<garbage_184>": 32198,
+  "<garbage_185>": 32199,
+  "<garbage_186>": 32200,
+  "<garbage_187>": 32201,
+  "<garbage_188>": 32202,
+  "<garbage_189>": 32203,
+  "<garbage_18>": 32032,
+  "<garbage_190>": 32204,
+  "<garbage_191>": 32205,
+  "<garbage_192>": 32206,
+  "<garbage_193>": 32207,
+  "<garbage_194>": 32208,
+  "<garbage_195>": 32209,
+  "<garbage_196>": 32210,
+  "<garbage_197>": 32211,
+  "<garbage_198>": 32212,
+  "<garbage_199>": 32213,
+  "<garbage_19>": 32033,
+  "<garbage_1>": 32015,
+  "<garbage_200>": 32214,
+  "<garbage_201>": 32215,
+  "<garbage_202>": 32216,
+  "<garbage_203>": 32217,
+  "<garbage_204>": 32218,
+  "<garbage_205>": 32219,
+  "<garbage_206>": 32220,
+  "<garbage_207>": 32221,
+  "<garbage_208>": 32222,
+  "<garbage_209>": 32223,
+  "<garbage_20>": 32034,
+  "<garbage_210>": 32224,
+  "<garbage_211>": 32225,
+  "<garbage_212>": 32226,
+  "<garbage_213>": 32227,
+  "<garbage_214>": 32228,
+  "<garbage_215>": 32229,
+  "<garbage_216>": 32230,
+  "<garbage_217>": 32231,
+  "<garbage_218>": 32232,
+  "<garbage_219>": 32233,
+  "<garbage_21>": 32035,
+  "<garbage_220>": 32234,
+  "<garbage_221>": 32235,
+  "<garbage_222>": 32236,
+  "<garbage_223>": 32237,
+  "<garbage_224>": 32238,
+  "<garbage_225>": 32239,
+  "<garbage_226>": 32240,
+  "<garbage_227>": 32241,
+  "<garbage_228>": 32242,
+  "<garbage_229>": 32243,
+  "<garbage_22>": 32036,
+  "<garbage_230>": 32244,
+  "<garbage_231>": 32245,
+  "<garbage_232>": 32246,
+  "<garbage_233>": 32247,
+  "<garbage_234>": 32248,
+  "<garbage_235>": 32249,
+  "<garbage_236>": 32250,
+  "<garbage_237>": 32251,
+  "<garbage_238>": 32252,
+  "<garbage_239>": 32253,
+  "<garbage_23>": 32037,
+  "<garbage_240>": 32254,
+  "<garbage_241>": 32255,
+  "<garbage_242>": 32256,
+  "<garbage_243>": 32257,
+  "<garbage_244>": 32258,
+  "<garbage_245>": 32259,
+  "<garbage_246>": 32260,
+  "<garbage_247>": 32261,
+  "<garbage_248>": 32262,
+  "<garbage_249>": 32263,
+  "<garbage_24>": 32038,
+  "<garbage_250>": 32264,
+  "<garbage_251>": 32265,
+  "<garbage_252>": 32266,
+  "<garbage_253>": 32267,
+  "<garbage_254>": 32268,
+  "<garbage_255>": 32269,
+  "<garbage_256>": 32270,
+  "<garbage_257>": 32271,
+  "<garbage_258>": 32272,
+  "<garbage_259>": 32273,
+  "<garbage_25>": 32039,
+  "<garbage_260>": 32274,
+  "<garbage_261>": 32275,
+  "<garbage_262>": 32276,
+  "<garbage_263>": 32277,
+  "<garbage_264>": 32278,
+  "<garbage_265>": 32279,
+  "<garbage_266>": 32280,
+  "<garbage_267>": 32281,
+  "<garbage_268>": 32282,
+  "<garbage_269>": 32283,
+  "<garbage_26>": 32040,
+  "<garbage_270>": 32284,
+  "<garbage_271>": 32285,
+  "<garbage_272>": 32286,
+  "<garbage_273>": 32287,
+  "<garbage_274>": 32288,
+  "<garbage_275>": 32289,
+  "<garbage_276>": 32290,
+  "<garbage_277>": 32291,
+  "<garbage_278>": 32292,
+  "<garbage_279>": 32293,
+  "<garbage_27>": 32041,
+  "<garbage_280>": 32294,
+  "<garbage_281>": 32295,
+  "<garbage_282>": 32296,
+  "<garbage_283>": 32297,
+  "<garbage_284>": 32298,
+  "<garbage_285>": 32299,
+  "<garbage_286>": 32300,
+  "<garbage_287>": 32301,
+  "<garbage_288>": 32302,
+  "<garbage_289>": 32303,
+  "<garbage_28>": 32042,
+  "<garbage_290>": 32304,
+  "<garbage_291>": 32305,
+  "<garbage_292>": 32306,
+  "<garbage_293>": 32307,
+  "<garbage_294>": 32308,
+  "<garbage_295>": 32309,
+  "<garbage_296>": 32310,
+  "<garbage_297>": 32311,
+  "<garbage_298>": 32312,
+  "<garbage_299>": 32313,
+  "<garbage_29>": 32043,
+  "<garbage_2>": 32016,
+  "<garbage_300>": 32314,
+  "<garbage_301>": 32315,
+  "<garbage_302>": 32316,
+  "<garbage_303>": 32317,
+  "<garbage_304>": 32318,
+  "<garbage_305>": 32319,
+  "<garbage_306>": 32320,
+  "<garbage_307>": 32321,
+  "<garbage_308>": 32322,
+  "<garbage_309>": 32323,
+  "<garbage_30>": 32044,
+  "<garbage_310>": 32324,
+  "<garbage_311>": 32325,
+  "<garbage_312>": 32326,
+  "<garbage_313>": 32327,
+  "<garbage_314>": 32328,
+  "<garbage_315>": 32329,
+  "<garbage_316>": 32330,
+  "<garbage_317>": 32331,
+  "<garbage_318>": 32332,
+  "<garbage_319>": 32333,
+  "<garbage_31>": 32045,
+  "<garbage_320>": 32334,
+  "<garbage_321>": 32335,
+  "<garbage_322>": 32336,
+  "<garbage_323>": 32337,
+  "<garbage_324>": 32338,
+  "<garbage_325>": 32339,
+  "<garbage_326>": 32340,
+  "<garbage_327>": 32341,
+  "<garbage_328>": 32342,
+  "<garbage_329>": 32343,
+  "<garbage_32>": 32046,
+  "<garbage_330>": 32344,
+  "<garbage_331>": 32345,
+  "<garbage_332>": 32346,
+  "<garbage_333>": 32347,
+  "<garbage_334>": 32348,
+  "<garbage_335>": 32349,
+  "<garbage_336>": 32350,
+  "<garbage_337>": 32351,
+  "<garbage_338>": 32352,
+  "<garbage_339>": 32353,
+  "<garbage_33>": 32047,
+  "<garbage_340>": 32354,
+  "<garbage_341>": 32355,
+  "<garbage_342>": 32356,
+  "<garbage_343>": 32357,
+  "<garbage_344>": 32358,
+  "<garbage_345>": 32359,
+  "<garbage_346>": 32360,
+  "<garbage_347>": 32361,
+  "<garbage_348>": 32362,
+  "<garbage_349>": 32363,
+  "<garbage_34>": 32048,
+  "<garbage_350>": 32364,
+  "<garbage_351>": 32365,
+  "<garbage_352>": 32366,
+  "<garbage_353>": 32367,
+  "<garbage_354>": 32368,
+  "<garbage_355>": 32369,
+  "<garbage_356>": 32370,
+  "<garbage_357>": 32371,
+  "<garbage_358>": 32372,
+  "<garbage_359>": 32373,
+  "<garbage_35>": 32049,
+  "<garbage_360>": 32374,
+  "<garbage_361>": 32375,
+  "<garbage_362>": 32376,
+  "<garbage_363>": 32377,
+  "<garbage_364>": 32378,
+  "<garbage_365>": 32379,
+  "<garbage_366>": 32380,
+  "<garbage_367>": 32381,
+  "<garbage_368>": 32382,
+  "<garbage_369>": 32383,
+  "<garbage_36>": 32050,
+  "<garbage_370>": 32384,
+  "<garbage_371>": 32385,
+  "<garbage_372>": 32386,
+  "<garbage_373>": 32387,
+  "<garbage_374>": 32388,
+  "<garbage_375>": 32389,
+  "<garbage_376>": 32390,
+  "<garbage_377>": 32391,
+  "<garbage_378>": 32392,
+  "<garbage_379>": 32393,
+  "<garbage_37>": 32051,
+  "<garbage_380>": 32394,
+  "<garbage_381>": 32395,
+  "<garbage_382>": 32396,
+  "<garbage_383>": 32397,
+  "<garbage_384>": 32398,
+  "<garbage_385>": 32399,
+  "<garbage_386>": 32400,
+  "<garbage_387>": 32401,
+  "<garbage_388>": 32402,
+  "<garbage_389>": 32403,
+  "<garbage_38>": 32052,
+  "<garbage_390>": 32404,
+  "<garbage_391>": 32405,
+  "<garbage_392>": 32406,
+  "<garbage_393>": 32407,
+  "<garbage_394>": 32408,
+  "<garbage_395>": 32409,
+  "<garbage_396>": 32410,
+  "<garbage_397>": 32411,
+  "<garbage_398>": 32412,
+  "<garbage_399>": 32413,
+  "<garbage_39>": 32053,
+  "<garbage_3>": 32017,
+  "<garbage_400>": 32414,
+  "<garbage_401>": 32415,
+  "<garbage_402>": 32416,
+  "<garbage_403>": 32417,
+  "<garbage_404>": 32418,
+  "<garbage_405>": 32419,
+  "<garbage_406>": 32420,
+  "<garbage_407>": 32421,
+  "<garbage_408>": 32422,
+  "<garbage_409>": 32423,
+  "<garbage_40>": 32054,
+  "<garbage_410>": 32424,
+  "<garbage_411>": 32425,
+  "<garbage_412>": 32426,
+  "<garbage_413>": 32427,
+  "<garbage_414>": 32428,
+  "<garbage_415>": 32429,
+  "<garbage_416>": 32430,
+  "<garbage_417>": 32431,
+  "<garbage_418>": 32432,
+  "<garbage_419>": 32433,
+  "<garbage_41>": 32055,
+  "<garbage_420>": 32434,
+  "<garbage_421>": 32435,
+  "<garbage_422>": 32436,
+  "<garbage_423>": 32437,
+  "<garbage_424>": 32438,
+  "<garbage_425>": 32439,
+  "<garbage_426>": 32440,
+  "<garbage_427>": 32441,
+  "<garbage_428>": 32442,
+  "<garbage_429>": 32443,
+  "<garbage_42>": 32056,
+  "<garbage_430>": 32444,
+  "<garbage_431>": 32445,
+  "<garbage_432>": 32446,
+  "<garbage_433>": 32447,
+  "<garbage_434>": 32448,
+  "<garbage_435>": 32449,
+  "<garbage_436>": 32450,
+  "<garbage_437>": 32451,
+  "<garbage_438>": 32452,
+  "<garbage_439>": 32453,
+  "<garbage_43>": 32057,
+  "<garbage_440>": 32454,
+  "<garbage_441>": 32455,
+  "<garbage_442>": 32456,
+  "<garbage_443>": 32457,
+  "<garbage_444>": 32458,
+  "<garbage_445>": 32459,
+  "<garbage_446>": 32460,
+  "<garbage_447>": 32461,
+  "<garbage_448>": 32462,
+  "<garbage_449>": 32463,
+  "<garbage_44>": 32058,
+  "<garbage_450>": 32464,
+  "<garbage_451>": 32465,
+  "<garbage_452>": 32466,
+  "<garbage_453>": 32467,
+  "<garbage_454>": 32468,
+  "<garbage_455>": 32469,
+  "<garbage_456>": 32470,
+  "<garbage_457>": 32471,
+  "<garbage_458>": 32472,
+  "<garbage_459>": 32473,
+  "<garbage_45>": 32059,
+  "<garbage_460>": 32474,
+  "<garbage_461>": 32475,
+  "<garbage_462>": 32476,
+  "<garbage_463>": 32477,
+  "<garbage_464>": 32478,
+  "<garbage_465>": 32479,
+  "<garbage_466>": 32480,
+  "<garbage_467>": 32481,
+  "<garbage_468>": 32482,
+  "<garbage_469>": 32483,
+  "<garbage_46>": 32060,
+  "<garbage_470>": 32484,
+  "<garbage_471>": 32485,
+  "<garbage_472>": 32486,
+  "<garbage_473>": 32487,
+  "<garbage_474>": 32488,
+  "<garbage_475>": 32489,
+  "<garbage_476>": 32490,
+  "<garbage_477>": 32491,
+  "<garbage_478>": 32492,
+  "<garbage_479>": 32493,
+  "<garbage_47>": 32061,
+  "<garbage_480>": 32494,
+  "<garbage_481>": 32495,
+  "<garbage_482>": 32496,
+  "<garbage_483>": 32497,
+  "<garbage_484>": 32498,
+  "<garbage_485>": 32499,
+  "<garbage_486>": 32500,
+  "<garbage_487>": 32501,
+  "<garbage_488>": 32502,
+  "<garbage_489>": 32503,
+  "<garbage_48>": 32062,
+  "<garbage_490>": 32504,
+  "<garbage_491>": 32505,
+  "<garbage_492>": 32506,
+  "<garbage_493>": 32507,
+  "<garbage_494>": 32508,
+  "<garbage_495>": 32509,
+  "<garbage_496>": 32510,
+  "<garbage_497>": 32511,
+  "<garbage_498>": 32512,
+  "<garbage_499>": 32513,
+  "<garbage_49>": 32063,
+  "<garbage_4>": 32018,
+  "<garbage_500>": 32514,
+  "<garbage_501>": 32515,
+  "<garbage_502>": 32516,
+  "<garbage_503>": 32517,
+  "<garbage_504>": 32518,
+  "<garbage_505>": 32519,
+  "<garbage_506>": 32520,
+  "<garbage_507>": 32521,
+  "<garbage_508>": 32522,
+  "<garbage_509>": 32523,
+  "<garbage_50>": 32064,
+  "<garbage_510>": 32524,
+  "<garbage_511>": 32525,
+  "<garbage_512>": 32526,
+  "<garbage_513>": 32527,
+  "<garbage_514>": 32528,
+  "<garbage_515>": 32529,
+  "<garbage_516>": 32530,
+  "<garbage_517>": 32531,
+  "<garbage_518>": 32532,
+  "<garbage_519>": 32533,
+  "<garbage_51>": 32065,
+  "<garbage_520>": 32534,
+  "<garbage_521>": 32535,
+  "<garbage_522>": 32536,
+  "<garbage_523>": 32537,
+  "<garbage_524>": 32538,
+  "<garbage_525>": 32539,
+  "<garbage_526>": 32540,
+  "<garbage_527>": 32541,
+  "<garbage_528>": 32542,
+  "<garbage_529>": 32543,
+  "<garbage_52>": 32066,
+  "<garbage_530>": 32544,
+  "<garbage_531>": 32545,
+  "<garbage_532>": 32546,
+  "<garbage_533>": 32547,
+  "<garbage_534>": 32548,
+  "<garbage_535>": 32549,
+  "<garbage_536>": 32550,
+  "<garbage_537>": 32551,
+  "<garbage_538>": 32552,
+  "<garbage_539>": 32553,
+  "<garbage_53>": 32067,
+  "<garbage_540>": 32554,
+  "<garbage_541>": 32555,
+  "<garbage_542>": 32556,
+  "<garbage_543>": 32557,
+  "<garbage_544>": 32558,
+  "<garbage_545>": 32559,
+  "<garbage_546>": 32560,
+  "<garbage_547>": 32561,
+  "<garbage_548>": 32562,
+  "<garbage_549>": 32563,
+  "<garbage_54>": 32068,
+  "<garbage_550>": 32564,
+  "<garbage_551>": 32565,
+  "<garbage_552>": 32566,
+  "<garbage_553>": 32567,
+  "<garbage_554>": 32568,
+  "<garbage_555>": 32569,
+  "<garbage_556>": 32570,
+  "<garbage_557>": 32571,
+  "<garbage_558>": 32572,
+  "<garbage_559>": 32573,
+  "<garbage_55>": 32069,
+  "<garbage_560>": 32574,
+  "<garbage_561>": 32575,
+  "<garbage_562>": 32576,
+  "<garbage_563>": 32577,
+  "<garbage_564>": 32578,
+  "<garbage_565>": 32579,
+  "<garbage_566>": 32580,
+  "<garbage_567>": 32581,
+  "<garbage_568>": 32582,
+  "<garbage_569>": 32583,
+  "<garbage_56>": 32070,
+  "<garbage_570>": 32584,
+  "<garbage_571>": 32585,
+  "<garbage_572>": 32586,
+  "<garbage_573>": 32587,
+  "<garbage_574>": 32588,
+  "<garbage_575>": 32589,
+  "<garbage_576>": 32590,
+  "<garbage_577>": 32591,
+  "<garbage_578>": 32592,
+  "<garbage_579>": 32593,
+  "<garbage_57>": 32071,
+  "<garbage_580>": 32594,
+  "<garbage_581>": 32595,
+  "<garbage_582>": 32596,
+  "<garbage_583>": 32597,
+  "<garbage_584>": 32598,
+  "<garbage_585>": 32599,
+  "<garbage_586>": 32600,
+  "<garbage_587>": 32601,
+  "<garbage_588>": 32602,
+  "<garbage_589>": 32603,
+  "<garbage_58>": 32072,
+  "<garbage_590>": 32604,
+  "<garbage_591>": 32605,
+  "<garbage_592>": 32606,
+  "<garbage_593>": 32607,
+  "<garbage_594>": 32608,
+  "<garbage_595>": 32609,
+  "<garbage_596>": 32610,
+  "<garbage_597>": 32611,
+  "<garbage_598>": 32612,
+  "<garbage_599>": 32613,
+  "<garbage_59>": 32073,
+  "<garbage_5>": 32019,
+  "<garbage_600>": 32614,
+  "<garbage_601>": 32615,
+  "<garbage_602>": 32616,
+  "<garbage_603>": 32617,
+  "<garbage_604>": 32618,
+  "<garbage_605>": 32619,
+  "<garbage_606>": 32620,
+  "<garbage_607>": 32621,
+  "<garbage_608>": 32622,
+  "<garbage_609>": 32623,
+  "<garbage_60>": 32074,
+  "<garbage_610>": 32624,
+  "<garbage_611>": 32625,
+  "<garbage_612>": 32626,
+  "<garbage_613>": 32627,
+  "<garbage_614>": 32628,
+  "<garbage_615>": 32629,
+  "<garbage_616>": 32630,
+  "<garbage_617>": 32631,
+  "<garbage_618>": 32632,
+  "<garbage_619>": 32633,
+  "<garbage_61>": 32075,
+  "<garbage_620>": 32634,
+  "<garbage_621>": 32635,
+  "<garbage_622>": 32636,
+  "<garbage_623>": 32637,
+  "<garbage_624>": 32638,
+  "<garbage_625>": 32639,
+  "<garbage_626>": 32640,
+  "<garbage_627>": 32641,
+  "<garbage_628>": 32642,
+  "<garbage_629>": 32643,
+  "<garbage_62>": 32076,
+  "<garbage_630>": 32644,
+  "<garbage_631>": 32645,
+  "<garbage_632>": 32646,
+  "<garbage_633>": 32647,
+  "<garbage_634>": 32648,
+  "<garbage_635>": 32649,
+  "<garbage_636>": 32650,
+  "<garbage_637>": 32651,
+  "<garbage_638>": 32652,
+  "<garbage_639>": 32653,
+  "<garbage_63>": 32077,
+  "<garbage_640>": 32654,
+  "<garbage_641>": 32655,
+  "<garbage_642>": 32656,
+  "<garbage_643>": 32657,
+  "<garbage_644>": 32658,
+  "<garbage_645>": 32659,
+  "<garbage_646>": 32660,
+  "<garbage_647>": 32661,
+  "<garbage_648>": 32662,
+  "<garbage_649>": 32663,
+  "<garbage_64>": 32078,
+  "<garbage_650>": 32664,
+  "<garbage_651>": 32665,
+  "<garbage_652>": 32666,
+  "<garbage_653>": 32667,
+  "<garbage_654>": 32668,
+  "<garbage_655>": 32669,
+  "<garbage_656>": 32670,
+  "<garbage_657>": 32671,
+  "<garbage_658>": 32672,
+  "<garbage_659>": 32673,
+  "<garbage_65>": 32079,
+  "<garbage_660>": 32674,
+  "<garbage_661>": 32675,
+  "<garbage_662>": 32676,
+  "<garbage_663>": 32677,
+  "<garbage_664>": 32678,
+  "<garbage_665>": 32679,
+  "<garbage_666>": 32680,
+  "<garbage_667>": 32681,
+  "<garbage_668>": 32682,
+  "<garbage_669>": 32683,
+  "<garbage_66>": 32080,
+  "<garbage_670>": 32684,
+  "<garbage_671>": 32685,
+  "<garbage_672>": 32686,
+  "<garbage_673>": 32687,
+  "<garbage_674>": 32688,
+  "<garbage_675>": 32689,
+  "<garbage_676>": 32690,
+  "<garbage_677>": 32691,
+  "<garbage_678>": 32692,
+  "<garbage_679>": 32693,
+  "<garbage_67>": 32081,
+  "<garbage_680>": 32694,
+  "<garbage_681>": 32695,
+  "<garbage_682>": 32696,
+  "<garbage_683>": 32697,
+  "<garbage_684>": 32698,
+  "<garbage_685>": 32699,
+  "<garbage_686>": 32700,
+  "<garbage_687>": 32701,
+  "<garbage_688>": 32702,
+  "<garbage_689>": 32703,
+  "<garbage_68>": 32082,
+  "<garbage_690>": 32704,
+  "<garbage_691>": 32705,
+  "<garbage_692>": 32706,
+  "<garbage_693>": 32707,
+  "<garbage_694>": 32708,
+  "<garbage_695>": 32709,
+  "<garbage_696>": 32710,
+  "<garbage_697>": 32711,
+  "<garbage_698>": 32712,
+  "<garbage_699>": 32713,
+  "<garbage_69>": 32083,
+  "<garbage_6>": 32020,
+  "<garbage_700>": 32714,
+  "<garbage_701>": 32715,
+  "<garbage_702>": 32716,
+  "<garbage_703>": 32717,
+  "<garbage_704>": 32718,
+  "<garbage_705>": 32719,
+  "<garbage_706>": 32720,
+  "<garbage_707>": 32721,
+  "<garbage_708>": 32722,
+  "<garbage_709>": 32723,
+  "<garbage_70>": 32084,
+  "<garbage_710>": 32724,
+  "<garbage_711>": 32725,
+  "<garbage_712>": 32726,
+  "<garbage_713>": 32727,
+  "<garbage_714>": 32728,
+  "<garbage_715>": 32729,
+  "<garbage_716>": 32730,
+  "<garbage_717>": 32731,
+  "<garbage_718>": 32732,
+  "<garbage_719>": 32733,
+  "<garbage_71>": 32085,
+  "<garbage_720>": 32734,
+  "<garbage_721>": 32735,
+  "<garbage_722>": 32736,
+  "<garbage_723>": 32737,
+  "<garbage_724>": 32738,
+  "<garbage_725>": 32739,
+  "<garbage_726>": 32740,
+  "<garbage_727>": 32741,
+  "<garbage_728>": 32742,
+  "<garbage_729>": 32743,
+  "<garbage_72>": 32086,
+  "<garbage_730>": 32744,
+  "<garbage_731>": 32745,
+  "<garbage_732>": 32746,
+  "<garbage_733>": 32747,
+  "<garbage_734>": 32748,
+  "<garbage_735>": 32749,
+  "<garbage_736>": 32750,
+  "<garbage_737>": 32751,
+  "<garbage_738>": 32752,
+  "<garbage_739>": 32753,
+  "<garbage_73>": 32087,
+  "<garbage_740>": 32754,
+  "<garbage_741>": 32755,
+  "<garbage_742>": 32756,
+  "<garbage_743>": 32757,
+  "<garbage_744>": 32758,
+  "<garbage_745>": 32759,
+  "<garbage_746>": 32760,
+  "<garbage_747>": 32761,
+  "<garbage_748>": 32762,
+  "<garbage_749>": 32763,
+  "<garbage_74>": 32088,
+  "<garbage_750>": 32764,
+  "<garbage_751>": 32765,
+  "<garbage_752>": 32766,
+  "<garbage_753>": 32767,
+  "<garbage_75>": 32089,
+  "<garbage_76>": 32090,
+  "<garbage_77>": 32091,
+  "<garbage_78>": 32092,
+  "<garbage_79>": 32093,
+  "<garbage_7>": 32021,
+  "<garbage_80>": 32094,
+  "<garbage_81>": 32095,
+  "<garbage_82>": 32096,
+  "<garbage_83>": 32097,
+  "<garbage_84>": 32098,
+  "<garbage_85>": 32099,
+  "<garbage_86>": 32100,
+  "<garbage_87>": 32101,
+  "<garbage_88>": 32102,
+  "<garbage_89>": 32103,
+  "<garbage_8>": 32022,
+  "<garbage_90>": 32104,
+  "<garbage_91>": 32105,
+  "<garbage_92>": 32106,
+  "<garbage_93>": 32107,
+  "<garbage_94>": 32108,
+  "<garbage_95>": 32109,
+  "<garbage_96>": 32110,
+  "<garbage_97>": 32111,
+  "<garbage_98>": 32112,
+  "<garbage_99>": 32113,
+  "<garbage_9>": 32023,
+  "<|box_end|>": 32005,
+  "<|box_start|>": 32004,
+  "<|endoftext|>": 32013,
+  "<|im_end|>": 32001,
+  "<|im_start|>": 32000,
+  "<|image_pad|>": 32011,
+  "<|object_ref_end|>": 32003,
+  "<|object_ref_start|>": 32002,
+  "<|quad_end|>": 32007,
+  "<|quad_start|>": 32006,
+  "<|video_pad|>": 32012,
+  "<|vision_end|>": 32009,
+  "<|vision_pad|>": 32010,
+  "<|vision_start|>": 32008
+}

chat_template.jinja

@@ -0,0 +1,7 @@
+{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system
+You are a helpful assistant.<|im_end|>
+{% endif %}<|im_start|>{{ message['role'] }}
+{% if message['content'] is string %}{{ message['content'] }}<|im_end|>
+{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>
+{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
+{% endif %}

preprocessor_config.json

@@ -0,0 +1,39 @@
+{
+  "crop_size": null,
+  "data_format": "channels_first",
+  "default_to_square": true,
+  "device": null,
+  "disable_grouping": null,
+  "do_center_crop": null,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_pad": null,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "Qwen2VLImageProcessorFast",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "input_data_format": null,
+  "max_pixels": 12845056,
+  "merge_size": 2,
+  "min_pixels": 3136,
+  "pad_size": null,
+  "patch_size": 14,
+  "processor_class": "Qwen2_5_VLProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "return_tensors": null,
+  "size": {
+    "longest_edge": 12845056,
+    "shortest_edge": 3136
+  },
+  "temporal_patch_size": 2
+}

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

video_preprocessor_config.json

@@ -0,0 +1,43 @@
+{
+  "crop_size": null,
+  "data_format": "channels_first",
+  "default_to_square": true,
+  "device": null,
+  "do_center_crop": null,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "do_sample_frames": false,
+  "fps": null,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "input_data_format": null,
+  "max_frames": 768,
+  "max_pixels": 12845056,
+  "merge_size": 2,
+  "min_frames": 4,
+  "min_pixels": 3136,
+  "num_frames": null,
+  "pad_size": null,
+  "patch_size": 14,
+  "processor_class": "Qwen2_5_VLProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "return_metadata": false,
+  "size": {
+    "longest_edge": 12845056,
+    "shortest_edge": 3136
+  },
+  "temporal_patch_size": 2,
+  "video_metadata": null,
+  "video_processor_type": "Qwen2VLVideoProcessor"
+}