| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.0015996826229676031, |
| "eval_steps": 500, |
| "global_step": 500, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.25, |
| "completions/max_length": 213.8, |
| "completions/max_terminated_length": 147.2, |
| "completions/mean_length": 115.175, |
| "completions/mean_terminated_length": 72.4022647857666, |
| "completions/min_length": 46.8, |
| "completions/min_terminated_length": 21.2, |
| "entropy": 0.4786868065595627, |
| "epoch": 3.1993652459352066e-05, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 2.6119611263275146, |
| "learning_rate": 9.999712057127867e-06, |
| "loss": -0.1609, |
| "num_tokens": 61746.0, |
| "reward": 0.4312597796320915, |
| "reward_std": 0.10828691087663174, |
| "rewards/reward_func/mean": 0.4312597796320915, |
| "rewards/reward_func/std": 0.10828691087663174, |
| "sampling/importance_sampling_ratio/max": 1.3771747946739197, |
| "sampling/importance_sampling_ratio/mean": 0.9703039705753327, |
| "sampling/importance_sampling_ratio/min": 0.6949578016996384, |
| "sampling/sampling_logp_difference/max": 0.1786184310913086, |
| "sampling/sampling_logp_difference/mean": 0.009788535302504897, |
| "step": 10, |
| "step_time": 5.622241817000395 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.8375, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 126.1, |
| "completions/mean_length": 232.35, |
| "completions/mean_terminated_length": 114.25, |
| "completions/min_length": 158.1, |
| "completions/min_terminated_length": 106.9, |
| "entropy": 0.3626169502735138, |
| "epoch": 6.398730491870413e-05, |
| "frac_reward_zero_std": 0.3, |
| "grad_norm": 0.0, |
| "learning_rate": 9.999392120603273e-06, |
| "loss": -0.04, |
| "num_tokens": 128850.0, |
| "reward": 0.5504927486181259, |
| "reward_std": 0.19645964950323105, |
| "rewards/reward_func/mean": 0.5504927486181259, |
| "rewards/reward_func/std": 0.19645965695381165, |
| "sampling/importance_sampling_ratio/max": 1.4168030858039855, |
| "sampling/importance_sampling_ratio/mean": 0.9663502812385559, |
| "sampling/importance_sampling_ratio/min": 0.5392230927944184, |
| "sampling/sampling_logp_difference/max": 0.23053348064422607, |
| "sampling/sampling_logp_difference/mean": 0.007877917354926467, |
| "step": 20, |
| "step_time": 5.55996063089915 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.9875, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 19.0, |
| "completions/mean_length": 255.175, |
| "completions/mean_terminated_length": 19.0, |
| "completions/min_length": 249.4, |
| "completions/min_terminated_length": 19.0, |
| "entropy": 0.29454511404037476, |
| "epoch": 9.59809573780562e-05, |
| "frac_reward_zero_std": 0.35, |
| "grad_norm": 0.0, |
| "learning_rate": 9.99907218407868e-06, |
| "loss": -0.0185, |
| "num_tokens": 207336.0, |
| "reward": 0.5643827587366104, |
| "reward_std": 0.15548617392778397, |
| "rewards/reward_func/mean": 0.5643827587366104, |
| "rewards/reward_func/std": 0.15548617392778397, |
| "sampling/importance_sampling_ratio/max": 1.3922772645950316, |
| "sampling/importance_sampling_ratio/mean": 0.9528154253959655, |
| "sampling/importance_sampling_ratio/min": 0.5780708372592926, |
| "sampling/sampling_logp_difference/max": 0.24547218680381774, |
| "sampling/sampling_logp_difference/mean": 0.007050963398069143, |
| "step": 30, |
| "step_time": 5.868266978400788 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.9875, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 19.4, |
| "completions/mean_length": 255.225, |
| "completions/mean_terminated_length": 19.4, |
| "completions/min_length": 249.8, |
| "completions/min_terminated_length": 19.4, |
| "entropy": 0.2484108805656433, |
| "epoch": 0.00012797460983740827, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 1.5433145761489868, |
| "learning_rate": 9.998752247554087e-06, |
| "loss": -0.0153, |
| "num_tokens": 282170.0, |
| "reward": 0.5426100015640258, |
| "reward_std": 0.1483269989490509, |
| "rewards/reward_func/mean": 0.5426100015640258, |
| "rewards/reward_func/std": 0.14832699857652187, |
| "sampling/importance_sampling_ratio/max": 1.4858020782470702, |
| "sampling/importance_sampling_ratio/mean": 0.9998360693454742, |
| "sampling/importance_sampling_ratio/min": 0.6810799956321716, |
| "sampling/sampling_logp_difference/max": 0.2127821445465088, |
| "sampling/sampling_logp_difference/mean": 0.006208276166580618, |
| "step": 40, |
| "step_time": 5.823335035099808 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.9875, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 13.9, |
| "completions/mean_length": 254.5375, |
| "completions/mean_terminated_length": 13.9, |
| "completions/min_length": 244.3, |
| "completions/min_terminated_length": 13.9, |
| "entropy": 0.3215895563364029, |
| "epoch": 0.00015996826229676032, |
| "frac_reward_zero_std": 0.4, |
| "grad_norm": 0.0, |
| "learning_rate": 9.998432311029492e-06, |
| "loss": 0.0418, |
| "num_tokens": 351413.0, |
| "reward": 0.5794098913669586, |
| "reward_std": 0.16170265898108482, |
| "rewards/reward_func/mean": 0.5794098913669586, |
| "rewards/reward_func/std": 0.1617026634514332, |
| "sampling/importance_sampling_ratio/max": 1.47185720205307, |
| "sampling/importance_sampling_ratio/mean": 1.0133295714855195, |
| "sampling/importance_sampling_ratio/min": 0.6691074788570404, |
| "sampling/sampling_logp_difference/max": 0.20463817119598388, |
| "sampling/sampling_logp_difference/mean": 0.00659145483514294, |
| "step": 50, |
| "step_time": 5.593381853300161 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.9625, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 24.6, |
| "completions/mean_length": 253.55, |
| "completions/mean_terminated_length": 19.06666717529297, |
| "completions/min_length": 242.9, |
| "completions/min_terminated_length": 12.5, |
| "entropy": 0.29620115645229816, |
| "epoch": 0.0001919619147561124, |
| "frac_reward_zero_std": 0.45, |
| "grad_norm": 3.978233814239502, |
| "learning_rate": 9.998112374504899e-06, |
| "loss": -0.014, |
| "num_tokens": 414597.0, |
| "reward": 0.5179664015769958, |
| "reward_std": 0.19976407699286938, |
| "rewards/reward_func/mean": 0.5179664015769958, |
| "rewards/reward_func/std": 0.19976408630609513, |
| "sampling/importance_sampling_ratio/max": 1.484041118621826, |
| "sampling/importance_sampling_ratio/mean": 1.022961837053299, |
| "sampling/importance_sampling_ratio/min": 0.6446415841579437, |
| "sampling/sampling_logp_difference/max": 0.1990389347076416, |
| "sampling/sampling_logp_difference/mean": 0.005983676365576685, |
| "step": 60, |
| "step_time": 5.404291736499363 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 1.0, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 0.0, |
| "completions/mean_length": 256.0, |
| "completions/mean_terminated_length": 0.0, |
| "completions/min_length": 256.0, |
| "completions/min_terminated_length": 0.0, |
| "entropy": 0.17363325618207454, |
| "epoch": 0.00022395556721546445, |
| "frac_reward_zero_std": 0.55, |
| "grad_norm": 2.7598984241485596, |
| "learning_rate": 9.997792437980306e-06, |
| "loss": -0.0027, |
| "num_tokens": 480129.0, |
| "reward": 0.5814546048641205, |
| "reward_std": 0.2727541446685791, |
| "rewards/reward_func/mean": 0.5814546048641205, |
| "rewards/reward_func/std": 0.2727541491389275, |
| "sampling/importance_sampling_ratio/max": 1.3518780946731568, |
| "sampling/importance_sampling_ratio/mean": 1.0044036984443665, |
| "sampling/importance_sampling_ratio/min": 0.7056601405143738, |
| "sampling/sampling_logp_difference/max": 0.18388597369194032, |
| "sampling/sampling_logp_difference/mean": 0.004156309820245951, |
| "step": 70, |
| "step_time": 5.455206840401297 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.8625, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 90.2, |
| "completions/mean_length": 238.375, |
| "completions/mean_terminated_length": 61.20000076293945, |
| "completions/min_length": 161.3, |
| "completions/min_terminated_length": 33.3, |
| "entropy": 0.36540624052286147, |
| "epoch": 0.00025594921967481653, |
| "frac_reward_zero_std": 0.35, |
| "grad_norm": 2.457448959350586, |
| "learning_rate": 9.997472501455711e-06, |
| "loss": -0.0748, |
| "num_tokens": 548531.0, |
| "reward": 0.5621683537960053, |
| "reward_std": 0.21145149692893028, |
| "rewards/reward_func/mean": 0.5621683537960053, |
| "rewards/reward_func/std": 0.21145151033997536, |
| "sampling/importance_sampling_ratio/max": 1.4461460351943969, |
| "sampling/importance_sampling_ratio/mean": 1.0051416873931884, |
| "sampling/importance_sampling_ratio/min": 0.7030990332365036, |
| "sampling/sampling_logp_difference/max": 0.20937676429748536, |
| "sampling/sampling_logp_difference/mean": 0.006805244646966457, |
| "step": 80, |
| "step_time": 5.632404646299255 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.8125, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 61.2, |
| "completions/mean_length": 224.2375, |
| "completions/mean_terminated_length": 36.83452453613281, |
| "completions/min_length": 173.3, |
| "completions/min_terminated_length": 19.7, |
| "entropy": 0.5844738900661468, |
| "epoch": 0.00028794287213416856, |
| "frac_reward_zero_std": 0.2, |
| "grad_norm": 1.992931842803955, |
| "learning_rate": 9.997152564931118e-06, |
| "loss": 0.0096, |
| "num_tokens": 610270.0, |
| "reward": 0.5034450203180313, |
| "reward_std": 0.13549208920449018, |
| "rewards/reward_func/mean": 0.5034450203180313, |
| "rewards/reward_func/std": 0.1354920944198966, |
| "sampling/importance_sampling_ratio/max": 1.4657144665718078, |
| "sampling/importance_sampling_ratio/mean": 0.9980016946792603, |
| "sampling/importance_sampling_ratio/min": 0.5777884721755981, |
| "sampling/sampling_logp_difference/max": 0.16048599481582643, |
| "sampling/sampling_logp_difference/mean": 0.00926766509655863, |
| "step": 90, |
| "step_time": 5.383288126800471 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.525, |
| "completions/max_length": 252.5, |
| "completions/max_terminated_length": 116.3, |
| "completions/mean_length": 179.65, |
| "completions/mean_terminated_length": 77.10000095367431, |
| "completions/min_length": 87.0, |
| "completions/min_terminated_length": 35.8, |
| "entropy": 0.8687089055776596, |
| "epoch": 0.00031993652459352064, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 5.623146057128906, |
| "learning_rate": 9.996832628406525e-06, |
| "loss": -0.0832, |
| "num_tokens": 671718.0, |
| "reward": 0.4710801362991333, |
| "reward_std": 0.1657375056296587, |
| "rewards/reward_func/mean": 0.4710801362991333, |
| "rewards/reward_func/std": 0.1657375056296587, |
| "sampling/importance_sampling_ratio/max": 1.3921684265136718, |
| "sampling/importance_sampling_ratio/mean": 0.9789192140102386, |
| "sampling/importance_sampling_ratio/min": 0.579711401462555, |
| "sampling/sampling_logp_difference/max": 0.24069670140743255, |
| "sampling/sampling_logp_difference/mean": 0.012631397787481546, |
| "step": 100, |
| "step_time": 5.387485970201669 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.625, |
| "completions/max_length": 240.8, |
| "completions/max_terminated_length": 89.3, |
| "completions/mean_length": 192.85, |
| "completions/mean_terminated_length": 65.42381134033204, |
| "completions/min_length": 94.1, |
| "completions/min_terminated_length": 42.9, |
| "entropy": 0.43052335977554324, |
| "epoch": 0.0003519301770528727, |
| "frac_reward_zero_std": 0.45, |
| "grad_norm": 2.4985339641571045, |
| "learning_rate": 9.996512691881932e-06, |
| "loss": -0.0615, |
| "num_tokens": 740086.0, |
| "reward": 0.437653186917305, |
| "reward_std": 0.12863893285393715, |
| "rewards/reward_func/mean": 0.437653186917305, |
| "rewards/reward_func/std": 0.12863893881440164, |
| "sampling/importance_sampling_ratio/max": 1.4863561749458314, |
| "sampling/importance_sampling_ratio/mean": 1.0037799596786499, |
| "sampling/importance_sampling_ratio/min": 0.5982317864894867, |
| "sampling/sampling_logp_difference/max": 0.17598365545272826, |
| "sampling/sampling_logp_difference/mean": 0.007958207954652607, |
| "step": 110, |
| "step_time": 5.308089344799373 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.9875, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 7.5, |
| "completions/mean_length": 253.7375, |
| "completions/mean_terminated_length": 7.5, |
| "completions/min_length": 237.9, |
| "completions/min_terminated_length": 7.5, |
| "entropy": 0.2582511238753796, |
| "epoch": 0.0003839238295122248, |
| "frac_reward_zero_std": 0.35, |
| "grad_norm": 2.053544044494629, |
| "learning_rate": 9.996192755357337e-06, |
| "loss": 0.0053, |
| "num_tokens": 810633.0, |
| "reward": 0.25738574657589197, |
| "reward_std": 0.1840340968221426, |
| "rewards/reward_func/mean": 0.25738574657589197, |
| "rewards/reward_func/std": 0.18403410203754902, |
| "sampling/importance_sampling_ratio/max": 1.2579748988151551, |
| "sampling/importance_sampling_ratio/mean": 0.9866569399833679, |
| "sampling/importance_sampling_ratio/min": 0.766651064157486, |
| "sampling/sampling_logp_difference/max": 0.1475762128829956, |
| "sampling/sampling_logp_difference/mean": 0.0038984777755104007, |
| "step": 120, |
| "step_time": 5.620803888401861 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.9625, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 34.5, |
| "completions/mean_length": 250.7125, |
| "completions/mean_terminated_length": 34.5, |
| "completions/min_length": 213.7, |
| "completions/min_terminated_length": 34.5, |
| "entropy": 0.47940588295459746, |
| "epoch": 0.0004159174819715768, |
| "frac_reward_zero_std": 0.55, |
| "grad_norm": 3.1486337184906006, |
| "learning_rate": 9.995872818832744e-06, |
| "loss": 0.0107, |
| "num_tokens": 877890.0, |
| "reward": 0.526879546046257, |
| "reward_std": 0.16744473129510878, |
| "rewards/reward_func/mean": 0.526879546046257, |
| "rewards/reward_func/std": 0.16744473874568938, |
| "sampling/importance_sampling_ratio/max": 1.3929537773132323, |
| "sampling/importance_sampling_ratio/mean": 0.964176470041275, |
| "sampling/importance_sampling_ratio/min": 0.5460495926439762, |
| "sampling/sampling_logp_difference/max": 0.45359928011894224, |
| "sampling/sampling_logp_difference/mean": 0.007244944549165666, |
| "step": 130, |
| "step_time": 5.525064078100695 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.925, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 80.6, |
| "completions/mean_length": 248.8125, |
| "completions/mean_terminated_length": 77.1, |
| "completions/min_length": 201.6, |
| "completions/min_terminated_length": 73.6, |
| "entropy": 0.7605804890394211, |
| "epoch": 0.0004479111344309289, |
| "frac_reward_zero_std": 0.4, |
| "grad_norm": 4.323256969451904, |
| "learning_rate": 9.995552882308151e-06, |
| "loss": -0.0069, |
| "num_tokens": 947371.0, |
| "reward": 0.5220189839601517, |
| "reward_std": 0.1725775882601738, |
| "rewards/reward_func/mean": 0.5220189839601517, |
| "rewards/reward_func/std": 0.17257759273052214, |
| "sampling/importance_sampling_ratio/max": 1.3841626405715943, |
| "sampling/importance_sampling_ratio/mean": 0.9834085047245026, |
| "sampling/importance_sampling_ratio/min": 0.6015414744615555, |
| "sampling/sampling_logp_difference/max": 0.1806986689567566, |
| "sampling/sampling_logp_difference/mean": 0.010641565825790168, |
| "step": 140, |
| "step_time": 5.569494440500421 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.9875, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 10.9, |
| "completions/mean_length": 254.1625, |
| "completions/mean_terminated_length": 10.9, |
| "completions/min_length": 241.3, |
| "completions/min_terminated_length": 10.9, |
| "entropy": 1.031371247768402, |
| "epoch": 0.000479904786890281, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.0, |
| "learning_rate": 9.995232945783558e-06, |
| "loss": 0.0098, |
| "num_tokens": 1018908.0, |
| "reward": 0.49765680730342865, |
| "reward_std": 0.11617601253092288, |
| "rewards/reward_func/mean": 0.49765680730342865, |
| "rewards/reward_func/std": 0.11617602296173572, |
| "sampling/importance_sampling_ratio/max": 1.4478043794631958, |
| "sampling/importance_sampling_ratio/mean": 0.9885349094867706, |
| "sampling/importance_sampling_ratio/min": 0.502427950501442, |
| "sampling/sampling_logp_difference/max": 0.32693520486354827, |
| "sampling/sampling_logp_difference/mean": 0.012870562355965376, |
| "step": 150, |
| "step_time": 5.713950043598743 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 1.0, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 0.0, |
| "completions/mean_length": 256.0, |
| "completions/mean_terminated_length": 0.0, |
| "completions/min_length": 256.0, |
| "completions/min_terminated_length": 0.0, |
| "entropy": 1.0644956052303314, |
| "epoch": 0.0005118984393496331, |
| "frac_reward_zero_std": 0.6, |
| "grad_norm": 5.655723571777344, |
| "learning_rate": 9.994913009258965e-06, |
| "loss": 0.0009, |
| "num_tokens": 1091120.0, |
| "reward": 0.5022399604320527, |
| "reward_std": 0.1322677180171013, |
| "rewards/reward_func/mean": 0.5022399604320527, |
| "rewards/reward_func/std": 0.1322677217423916, |
| "sampling/importance_sampling_ratio/max": 1.565798020362854, |
| "sampling/importance_sampling_ratio/mean": 0.9627444028854371, |
| "sampling/importance_sampling_ratio/min": 0.5701836764812469, |
| "sampling/sampling_logp_difference/max": 0.2968409895896912, |
| "sampling/sampling_logp_difference/mean": 0.014765473827719689, |
| "step": 160, |
| "step_time": 5.714645506398665 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 1.0, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 0.0, |
| "completions/mean_length": 256.0, |
| "completions/mean_terminated_length": 0.0, |
| "completions/min_length": 256.0, |
| "completions/min_terminated_length": 0.0, |
| "entropy": 1.1330764949321748, |
| "epoch": 0.0005438920918089851, |
| "frac_reward_zero_std": 0.4, |
| "grad_norm": 3.06475830078125, |
| "learning_rate": 9.99459307273437e-06, |
| "loss": -0.0131, |
| "num_tokens": 1160588.0, |
| "reward": 0.4231398433446884, |
| "reward_std": 0.10350660830736161, |
| "rewards/reward_func/mean": 0.4231398433446884, |
| "rewards/reward_func/std": 0.10350661091506481, |
| "sampling/importance_sampling_ratio/max": 1.7593751907348634, |
| "sampling/importance_sampling_ratio/mean": 0.9571119487285614, |
| "sampling/importance_sampling_ratio/min": 0.3742682170093758, |
| "sampling/sampling_logp_difference/max": 1.9030644834041595, |
| "sampling/sampling_logp_difference/mean": 0.015633477736264466, |
| "step": 170, |
| "step_time": 5.528405581899278 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.9875, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 18.9, |
| "completions/mean_length": 255.1625, |
| "completions/mean_terminated_length": 18.9, |
| "completions/min_length": 249.3, |
| "completions/min_terminated_length": 18.9, |
| "entropy": 0.8980084896087647, |
| "epoch": 0.0005758857442683371, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 4.3491716384887695, |
| "learning_rate": 9.994273136209777e-06, |
| "loss": -0.0362, |
| "num_tokens": 1229921.0, |
| "reward": 0.513152438402176, |
| "reward_std": 0.11573685109615325, |
| "rewards/reward_func/mean": 0.513152438402176, |
| "rewards/reward_func/std": 0.1157368503510952, |
| "sampling/importance_sampling_ratio/max": 1.649842381477356, |
| "sampling/importance_sampling_ratio/mean": 1.0105858862400054, |
| "sampling/importance_sampling_ratio/min": 0.5301016598939896, |
| "sampling/sampling_logp_difference/max": 0.4382950246334076, |
| "sampling/sampling_logp_difference/mean": 0.013723865710198879, |
| "step": 180, |
| "step_time": 5.58106996090064 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.9875, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 23.5, |
| "completions/mean_length": 255.7375, |
| "completions/mean_terminated_length": 23.5, |
| "completions/min_length": 253.9, |
| "completions/min_terminated_length": 23.5, |
| "entropy": 0.6676588207483292, |
| "epoch": 0.0006078793967276892, |
| "frac_reward_zero_std": 0.65, |
| "grad_norm": 0.0, |
| "learning_rate": 9.993953199685184e-06, |
| "loss": -0.0132, |
| "num_tokens": 1301612.0, |
| "reward": 0.46829662322998045, |
| "reward_std": 0.09823110550642014, |
| "rewards/reward_func/mean": 0.46829662322998045, |
| "rewards/reward_func/std": 0.09823111034929752, |
| "sampling/importance_sampling_ratio/max": 1.6070812821388245, |
| "sampling/importance_sampling_ratio/mean": 0.9719215571880341, |
| "sampling/importance_sampling_ratio/min": 0.4822065860033035, |
| "sampling/sampling_logp_difference/max": 0.3739572048187256, |
| "sampling/sampling_logp_difference/mean": 0.011020558141171932, |
| "step": 190, |
| "step_time": 5.662645305100159 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.9375, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 55.1, |
| "completions/mean_length": 248.0875, |
| "completions/mean_terminated_length": 48.15, |
| "completions/min_length": 194.8, |
| "completions/min_terminated_length": 41.2, |
| "entropy": 0.5728133380413055, |
| "epoch": 0.0006398730491870413, |
| "frac_reward_zero_std": 0.6, |
| "grad_norm": 2.902703285217285, |
| "learning_rate": 9.99363326316059e-06, |
| "loss": 0.04, |
| "num_tokens": 1370639.0, |
| "reward": 0.5136901259422302, |
| "reward_std": 0.11567677184939384, |
| "rewards/reward_func/mean": 0.5136901259422302, |
| "rewards/reward_func/std": 0.11567677184939384, |
| "sampling/importance_sampling_ratio/max": 1.5911941289901734, |
| "sampling/importance_sampling_ratio/mean": 0.9673095345497131, |
| "sampling/importance_sampling_ratio/min": 0.5568610072135926, |
| "sampling/sampling_logp_difference/max": 0.20886776447296143, |
| "sampling/sampling_logp_difference/mean": 0.009916677000001073, |
| "step": 200, |
| "step_time": 5.727214452699991 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.9625, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 23.8, |
| "completions/mean_length": 249.375, |
| "completions/mean_terminated_length": 23.8, |
| "completions/min_length": 203.0, |
| "completions/min_terminated_length": 23.8, |
| "entropy": 0.7014953553676605, |
| "epoch": 0.0006718667016463934, |
| "frac_reward_zero_std": 0.6, |
| "grad_norm": 0.0, |
| "learning_rate": 9.993313326635996e-06, |
| "loss": -0.0173, |
| "num_tokens": 1440089.0, |
| "reward": 0.42509948313236234, |
| "reward_std": 0.09789986498653888, |
| "rewards/reward_func/mean": 0.42509948313236234, |
| "rewards/reward_func/std": 0.09789986498653888, |
| "sampling/importance_sampling_ratio/max": 1.6502047896385192, |
| "sampling/importance_sampling_ratio/mean": 0.9715989649295806, |
| "sampling/importance_sampling_ratio/min": 0.5182478070259094, |
| "sampling/sampling_logp_difference/max": 0.24867323338985442, |
| "sampling/sampling_logp_difference/mean": 0.01255566468462348, |
| "step": 210, |
| "step_time": 5.595404389499163 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.9625, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 15.5, |
| "completions/mean_length": 248.3375, |
| "completions/mean_terminated_length": 15.5, |
| "completions/min_length": 194.7, |
| "completions/min_terminated_length": 15.5, |
| "entropy": 0.7566814184188843, |
| "epoch": 0.0007038603541057454, |
| "frac_reward_zero_std": 0.35, |
| "grad_norm": 0.0, |
| "learning_rate": 9.992993390111403e-06, |
| "loss": -0.0316, |
| "num_tokens": 1510280.0, |
| "reward": 0.4379701763391495, |
| "reward_std": 0.15445991717278956, |
| "rewards/reward_func/mean": 0.4379701763391495, |
| "rewards/reward_func/std": 0.15445992574095727, |
| "sampling/importance_sampling_ratio/max": 1.6620585918426514, |
| "sampling/importance_sampling_ratio/mean": 1.0050045073032379, |
| "sampling/importance_sampling_ratio/min": 0.5981552571058273, |
| "sampling/sampling_logp_difference/max": 0.2270957887172699, |
| "sampling/sampling_logp_difference/mean": 0.013740905188024044, |
| "step": 220, |
| "step_time": 5.674174832900462 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.9875, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 15.8, |
| "completions/mean_length": 254.775, |
| "completions/mean_terminated_length": 15.8, |
| "completions/min_length": 246.2, |
| "completions/min_terminated_length": 15.8, |
| "entropy": 0.6086385309696197, |
| "epoch": 0.0007358540065650975, |
| "frac_reward_zero_std": 0.45, |
| "grad_norm": 2.3130009174346924, |
| "learning_rate": 9.99267345358681e-06, |
| "loss": -0.0006, |
| "num_tokens": 1578514.0, |
| "reward": 0.4830972790718079, |
| "reward_std": 0.1875777266919613, |
| "rewards/reward_func/mean": 0.4830972790718079, |
| "rewards/reward_func/std": 0.1875777341425419, |
| "sampling/importance_sampling_ratio/max": 1.6131272673606873, |
| "sampling/importance_sampling_ratio/mean": 1.0001753568649292, |
| "sampling/importance_sampling_ratio/min": 0.5552087921649218, |
| "sampling/sampling_logp_difference/max": 0.43015940189361573, |
| "sampling/sampling_logp_difference/mean": 0.011561960633844136, |
| "step": 230, |
| "step_time": 5.544450350801344 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 1.0, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 0.0, |
| "completions/mean_length": 256.0, |
| "completions/mean_terminated_length": 0.0, |
| "completions/min_length": 256.0, |
| "completions/min_terminated_length": 0.0, |
| "entropy": 0.3803480386734009, |
| "epoch": 0.0007678476590244496, |
| "frac_reward_zero_std": 0.55, |
| "grad_norm": 1.6584447622299194, |
| "learning_rate": 9.992353517062216e-06, |
| "loss": -0.0284, |
| "num_tokens": 1642058.0, |
| "reward": 0.37710185497999194, |
| "reward_std": 0.15426886081695557, |
| "rewards/reward_func/mean": 0.37710185497999194, |
| "rewards/reward_func/std": 0.1542688600718975, |
| "sampling/importance_sampling_ratio/max": 1.6463178157806397, |
| "sampling/importance_sampling_ratio/mean": 1.0489038527011871, |
| "sampling/importance_sampling_ratio/min": 0.5974440038204193, |
| "sampling/sampling_logp_difference/max": 0.25446571707725524, |
| "sampling/sampling_logp_difference/mean": 0.007619179924950004, |
| "step": 240, |
| "step_time": 5.355880505999084 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.975, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 40.4, |
| "completions/mean_length": 254.65, |
| "completions/mean_terminated_length": 40.4, |
| "completions/min_length": 245.2, |
| "completions/min_terminated_length": 40.4, |
| "entropy": 0.27831714749336245, |
| "epoch": 0.0007998413114838016, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.0, |
| "learning_rate": 9.992033580537622e-06, |
| "loss": -0.0032, |
| "num_tokens": 1711846.0, |
| "reward": 0.42411058843135835, |
| "reward_std": 0.16632129922509192, |
| "rewards/reward_func/mean": 0.42411058843135835, |
| "rewards/reward_func/std": 0.16632130593061448, |
| "sampling/importance_sampling_ratio/max": 1.5203413009643554, |
| "sampling/importance_sampling_ratio/mean": 1.0328932225704193, |
| "sampling/importance_sampling_ratio/min": 0.6588745385408401, |
| "sampling/sampling_logp_difference/max": 0.18178346157073974, |
| "sampling/sampling_logp_difference/mean": 0.0056556087452918295, |
| "step": 250, |
| "step_time": 5.5352098594987185 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 1.0, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 0.0, |
| "completions/mean_length": 256.0, |
| "completions/mean_terminated_length": 0.0, |
| "completions/min_length": 256.0, |
| "completions/min_terminated_length": 0.0, |
| "entropy": 0.1278106041252613, |
| "epoch": 0.0008318349639431536, |
| "frac_reward_zero_std": 0.8, |
| "grad_norm": 0.0, |
| "learning_rate": 9.99171364401303e-06, |
| "loss": -0.0046, |
| "num_tokens": 1786514.0, |
| "reward": 0.2930680900812149, |
| "reward_std": 0.17092638909816743, |
| "rewards/reward_func/mean": 0.2930680900812149, |
| "rewards/reward_func/std": 0.1709263913333416, |
| "sampling/importance_sampling_ratio/max": 1.3693155288696288, |
| "sampling/importance_sampling_ratio/mean": 1.0101397514343262, |
| "sampling/importance_sampling_ratio/min": 0.706161481142044, |
| "sampling/sampling_logp_difference/max": 0.2914245367050171, |
| "sampling/sampling_logp_difference/mean": 0.003187426913063973, |
| "step": 260, |
| "step_time": 6.023757166299765 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.7875, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 75.3, |
| "completions/mean_length": 220.3625, |
| "completions/mean_terminated_length": 43.833333587646486, |
| "completions/min_length": 142.5, |
| "completions/min_terminated_length": 14.5, |
| "entropy": 0.1212063867598772, |
| "epoch": 0.0008638286164025057, |
| "frac_reward_zero_std": 0.85, |
| "grad_norm": 0.0, |
| "learning_rate": 9.991393707488435e-06, |
| "loss": -0.0154, |
| "num_tokens": 1858843.0, |
| "reward": 0.33083333671092985, |
| "reward_std": 0.16369171999394894, |
| "rewards/reward_func/mean": 0.33083333671092985, |
| "rewards/reward_func/std": 0.16369172520935535, |
| "sampling/importance_sampling_ratio/max": 1.2816577553749084, |
| "sampling/importance_sampling_ratio/mean": 0.9920831918716431, |
| "sampling/importance_sampling_ratio/min": 0.6905932784080505, |
| "sampling/sampling_logp_difference/max": 0.28521891832351687, |
| "sampling/sampling_logp_difference/mean": 0.0035509371664375067, |
| "step": 270, |
| "step_time": 5.597285097798886 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.9375, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 5.0, |
| "completions/mean_length": 240.9375, |
| "completions/mean_terminated_length": 4.75, |
| "completions/min_length": 183.7, |
| "completions/min_terminated_length": 4.5, |
| "entropy": 0.07088708113878965, |
| "epoch": 0.0008958222688618578, |
| "frac_reward_zero_std": 0.8, |
| "grad_norm": 0.0, |
| "learning_rate": 9.991073770963842e-06, |
| "loss": 0.0048, |
| "num_tokens": 1932334.0, |
| "reward": 0.3946428582072258, |
| "reward_std": 0.22504092752933502, |
| "rewards/reward_func/mean": 0.3946428582072258, |
| "rewards/reward_func/std": 0.22504092901945114, |
| "sampling/importance_sampling_ratio/max": 1.2599162340164185, |
| "sampling/importance_sampling_ratio/mean": 1.0197808623313904, |
| "sampling/importance_sampling_ratio/min": 0.7929178476333618, |
| "sampling/sampling_logp_difference/max": 0.1750430405139923, |
| "sampling/sampling_logp_difference/mean": 0.0020096191816264763, |
| "step": 280, |
| "step_time": 5.630174891401111 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.8, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 88.3, |
| "completions/mean_length": 228.1625, |
| "completions/mean_terminated_length": 70.35500030517578, |
| "completions/min_length": 156.3, |
| "completions/min_terminated_length": 53.9, |
| "entropy": 0.08419435755349695, |
| "epoch": 0.0009278159213212099, |
| "frac_reward_zero_std": 0.8, |
| "grad_norm": 0.0, |
| "learning_rate": 9.990753834439249e-06, |
| "loss": 0.0111, |
| "num_tokens": 2008323.0, |
| "reward": 0.26570436991751195, |
| "reward_std": 0.18353180214762688, |
| "rewards/reward_func/mean": 0.26570436991751195, |
| "rewards/reward_func/std": 0.1835318051278591, |
| "sampling/importance_sampling_ratio/max": 1.4831114292144776, |
| "sampling/importance_sampling_ratio/mean": 1.0381149470806121, |
| "sampling/importance_sampling_ratio/min": 0.769954988360405, |
| "sampling/sampling_logp_difference/max": 0.2556483805179596, |
| "sampling/sampling_logp_difference/mean": 0.002364744877559133, |
| "step": 290, |
| "step_time": 5.769126031998894 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.8125, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 62.9, |
| "completions/mean_length": 225.9875, |
| "completions/mean_terminated_length": 53.37333374023437, |
| "completions/min_length": 172.1, |
| "completions/min_terminated_length": 44.1, |
| "entropy": 0.07666442807785642, |
| "epoch": 0.000959809573780562, |
| "frac_reward_zero_std": 0.8, |
| "grad_norm": 4.1501336097717285, |
| "learning_rate": 9.990433897914654e-06, |
| "loss": 0.0336, |
| "num_tokens": 2082826.0, |
| "reward": 0.23347222283482552, |
| "reward_std": 0.16722211516462265, |
| "rewards/reward_func/mean": 0.23347222283482552, |
| "rewards/reward_func/std": 0.1672221203800291, |
| "sampling/importance_sampling_ratio/max": 1.4259886741638184, |
| "sampling/importance_sampling_ratio/mean": 1.0574868023395538, |
| "sampling/importance_sampling_ratio/min": 0.789874866604805, |
| "sampling/sampling_logp_difference/max": 0.23401358423907367, |
| "sampling/sampling_logp_difference/mean": 0.0024510751294855027, |
| "step": 300, |
| "step_time": 5.732418804300687 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.75, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 75.1, |
| "completions/mean_length": 211.2875, |
| "completions/mean_terminated_length": 41.28166732788086, |
| "completions/min_length": 120.0, |
| "completions/min_terminated_length": 17.6, |
| "entropy": 0.0686798977241864, |
| "epoch": 0.000991803226239914, |
| "frac_reward_zero_std": 0.8, |
| "grad_norm": 1.8366484642028809, |
| "learning_rate": 9.99011396139006e-06, |
| "loss": 0.0239, |
| "num_tokens": 2152257.0, |
| "reward": 0.12645833492279052, |
| "reward_std": 0.14587621986865998, |
| "rewards/reward_func/mean": 0.12645833492279052, |
| "rewards/reward_func/std": 0.14587622582912446, |
| "sampling/importance_sampling_ratio/max": 1.1935789823532104, |
| "sampling/importance_sampling_ratio/mean": 0.974252438545227, |
| "sampling/importance_sampling_ratio/min": 0.7667523324489594, |
| "sampling/sampling_logp_difference/max": 0.1506180524825588, |
| "sampling/sampling_logp_difference/mean": 0.002328540227609641, |
| "step": 310, |
| "step_time": 5.518856391799636 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.8375, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 64.5, |
| "completions/mean_length": 226.7375, |
| "completions/mean_terminated_length": 52.0, |
| "completions/min_length": 142.4, |
| "completions/min_terminated_length": 40.0, |
| "entropy": 0.09266874022340943, |
| "epoch": 0.0010237968786992661, |
| "frac_reward_zero_std": 0.75, |
| "grad_norm": 0.0, |
| "learning_rate": 9.989794024865468e-06, |
| "loss": -0.0364, |
| "num_tokens": 2214244.0, |
| "reward": 0.15312500447034835, |
| "reward_std": 0.1446988269686699, |
| "rewards/reward_func/mean": 0.15312500447034835, |
| "rewards/reward_func/std": 0.1446988321840763, |
| "sampling/importance_sampling_ratio/max": 1.241053009033203, |
| "sampling/importance_sampling_ratio/mean": 1.0196372747421265, |
| "sampling/importance_sampling_ratio/min": 0.8321290433406829, |
| "sampling/sampling_logp_difference/max": 0.15206519961356973, |
| "sampling/sampling_logp_difference/mean": 0.0024405727162957165, |
| "step": 320, |
| "step_time": 5.378200366300734 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.9125, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 66.0, |
| "completions/mean_length": 246.2, |
| "completions/mean_terminated_length": 63.2, |
| "completions/min_length": 188.3, |
| "completions/min_terminated_length": 60.3, |
| "entropy": 0.09167351502528617, |
| "epoch": 0.0010557905311586182, |
| "frac_reward_zero_std": 0.6, |
| "grad_norm": 0.0, |
| "learning_rate": 9.989474088340873e-06, |
| "loss": 0.0217, |
| "num_tokens": 2285904.0, |
| "reward": 0.3576704621315002, |
| "reward_std": 0.3019014626741409, |
| "rewards/reward_func/mean": 0.3576704621315002, |
| "rewards/reward_func/std": 0.3019014626741409, |
| "sampling/importance_sampling_ratio/max": 1.4326660275459289, |
| "sampling/importance_sampling_ratio/mean": 1.037734228372574, |
| "sampling/importance_sampling_ratio/min": 0.7440287649631501, |
| "sampling/sampling_logp_difference/max": 0.23049730334314517, |
| "sampling/sampling_logp_difference/mean": 0.002569542225171517, |
| "step": 330, |
| "step_time": 5.626144137700612 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.9875, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 20.7, |
| "completions/mean_length": 255.3875, |
| "completions/mean_terminated_length": 20.7, |
| "completions/min_length": 251.1, |
| "completions/min_terminated_length": 20.7, |
| "entropy": 0.04270339908171365, |
| "epoch": 0.0010877841836179703, |
| "frac_reward_zero_std": 0.9, |
| "grad_norm": 2.1360747814178467, |
| "learning_rate": 9.98915415181628e-06, |
| "loss": -0.0077, |
| "num_tokens": 2360267.0, |
| "reward": 0.4468750059604645, |
| "reward_std": 0.24749347418546677, |
| "rewards/reward_func/mean": 0.4468750059604645, |
| "rewards/reward_func/std": 0.24749347791075707, |
| "sampling/importance_sampling_ratio/max": 1.2458698153495789, |
| "sampling/importance_sampling_ratio/mean": 0.9976316630840302, |
| "sampling/importance_sampling_ratio/min": 0.7613322377204895, |
| "sampling/sampling_logp_difference/max": 0.23170500993728638, |
| "sampling/sampling_logp_difference/mean": 0.00149433299084194, |
| "step": 340, |
| "step_time": 5.726679722499102 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.9875, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 0.8, |
| "completions/mean_length": 252.9, |
| "completions/mean_terminated_length": 0.8, |
| "completions/min_length": 231.2, |
| "completions/min_terminated_length": 0.8, |
| "entropy": 0.04858234422281384, |
| "epoch": 0.0011197778360773224, |
| "frac_reward_zero_std": 0.7, |
| "grad_norm": 0.0, |
| "learning_rate": 9.988834215291687e-06, |
| "loss": 0.0049, |
| "num_tokens": 2425455.0, |
| "reward": 0.4354166708886623, |
| "reward_std": 0.21333095170557498, |
| "rewards/reward_func/mean": 0.4354166708886623, |
| "rewards/reward_func/std": 0.21333095692098142, |
| "sampling/importance_sampling_ratio/max": 1.4613646507263183, |
| "sampling/importance_sampling_ratio/mean": 1.050947654247284, |
| "sampling/importance_sampling_ratio/min": 0.814390116930008, |
| "sampling/sampling_logp_difference/max": 0.21158765852451325, |
| "sampling/sampling_logp_difference/mean": 0.0014789702137932182, |
| "step": 350, |
| "step_time": 5.466575936901063 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.925, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 61.9, |
| "completions/mean_length": 245.925, |
| "completions/mean_terminated_length": 55.55, |
| "completions/min_length": 177.2, |
| "completions/min_terminated_length": 49.2, |
| "entropy": 0.053529321749115864, |
| "epoch": 0.0011517714885366742, |
| "frac_reward_zero_std": 0.75, |
| "grad_norm": 0.0, |
| "learning_rate": 9.988514278767092e-06, |
| "loss": 0.0049, |
| "num_tokens": 2504269.0, |
| "reward": 0.3968750059604645, |
| "reward_std": 0.1802075080573559, |
| "rewards/reward_func/mean": 0.3968750059604645, |
| "rewards/reward_func/std": 0.18020751103758811, |
| "sampling/importance_sampling_ratio/max": 1.177919292449951, |
| "sampling/importance_sampling_ratio/mean": 0.9963529229164123, |
| "sampling/importance_sampling_ratio/min": 0.790158474445343, |
| "sampling/sampling_logp_difference/max": 0.1852503061294435, |
| "sampling/sampling_logp_difference/mean": 0.0018855397938748685, |
| "step": 360, |
| "step_time": 5.941310021798563 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.9, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 84.4, |
| "completions/mean_length": 242.7125, |
| "completions/mean_terminated_length": 64.95, |
| "completions/min_length": 180.3, |
| "completions/min_terminated_length": 52.3, |
| "entropy": 0.06859993665476623, |
| "epoch": 0.0011837651409960263, |
| "frac_reward_zero_std": 0.85, |
| "grad_norm": 0.0, |
| "learning_rate": 9.988194342242499e-06, |
| "loss": 0.0173, |
| "num_tokens": 2569206.0, |
| "reward": 0.28229166865348815, |
| "reward_std": 0.2816430002450943, |
| "rewards/reward_func/mean": 0.28229166865348815, |
| "rewards/reward_func/std": 0.28164300322532654, |
| "sampling/importance_sampling_ratio/max": 1.124396276473999, |
| "sampling/importance_sampling_ratio/mean": 0.9633953630924225, |
| "sampling/importance_sampling_ratio/min": 0.7915119707584382, |
| "sampling/sampling_logp_difference/max": 0.16286268234239287, |
| "sampling/sampling_logp_difference/mean": 0.0019630593538717987, |
| "step": 370, |
| "step_time": 5.463180380301492 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.875, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 50.0, |
| "completions/mean_length": 231.6, |
| "completions/mean_terminated_length": 31.625, |
| "completions/min_length": 176.7, |
| "completions/min_terminated_length": 23.1, |
| "entropy": 0.06270983926292502, |
| "epoch": 0.0012157587934553784, |
| "frac_reward_zero_std": 0.8, |
| "grad_norm": 2.672822952270508, |
| "learning_rate": 9.987874405717906e-06, |
| "loss": -0.055, |
| "num_tokens": 2629322.0, |
| "reward": 0.3541666716337204, |
| "reward_std": 0.20473803579807281, |
| "rewards/reward_func/mean": 0.3541666716337204, |
| "rewards/reward_func/std": 0.20473804622888564, |
| "sampling/importance_sampling_ratio/max": 1.2506105780601502, |
| "sampling/importance_sampling_ratio/mean": 1.0168899595737457, |
| "sampling/importance_sampling_ratio/min": 0.824669885635376, |
| "sampling/sampling_logp_difference/max": 0.20925886631014237, |
| "sampling/sampling_logp_difference/mean": 0.0017543567693905971, |
| "step": 380, |
| "step_time": 5.311027956598991 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 1.0, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 0.0, |
| "completions/mean_length": 256.0, |
| "completions/mean_terminated_length": 0.0, |
| "completions/min_length": 256.0, |
| "completions/min_terminated_length": 0.0, |
| "entropy": 0.03655383495010937, |
| "epoch": 0.0012477524459147305, |
| "frac_reward_zero_std": 0.85, |
| "grad_norm": 0.0, |
| "learning_rate": 9.987554469193313e-06, |
| "loss": -0.0051, |
| "num_tokens": 2703414.0, |
| "reward": 0.377202383056283, |
| "reward_std": 0.1339038535952568, |
| "rewards/reward_func/mean": 0.377202383056283, |
| "rewards/reward_func/std": 0.13390385918319225, |
| "sampling/importance_sampling_ratio/max": 1.1611881256103516, |
| "sampling/importance_sampling_ratio/mean": 0.9962027847766877, |
| "sampling/importance_sampling_ratio/min": 0.854848849773407, |
| "sampling/sampling_logp_difference/max": 0.16494819521891485, |
| "sampling/sampling_logp_difference/mean": 0.0010984212101902812, |
| "step": 390, |
| "step_time": 5.715437156000553 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.95, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 50.9, |
| "completions/mean_length": 249.5625, |
| "completions/mean_terminated_length": 50.9, |
| "completions/min_length": 204.5, |
| "completions/min_terminated_length": 50.9, |
| "entropy": 0.0142990311000176, |
| "epoch": 0.0012797460983740825, |
| "frac_reward_zero_std": 0.85, |
| "grad_norm": 0.9834340214729309, |
| "learning_rate": 9.98723453266872e-06, |
| "loss": -0.0033, |
| "num_tokens": 2774675.0, |
| "reward": 0.2778645843267441, |
| "reward_std": 0.19682962205260993, |
| "rewards/reward_func/mean": 0.2778645843267441, |
| "rewards/reward_func/std": 0.19682962205260993, |
| "sampling/importance_sampling_ratio/max": 1.1094371914863586, |
| "sampling/importance_sampling_ratio/mean": 0.9909844160079956, |
| "sampling/importance_sampling_ratio/min": 0.8944578170776367, |
| "sampling/sampling_logp_difference/max": 0.1037120074032714, |
| "sampling/sampling_logp_difference/mean": 0.00037660791385860646, |
| "step": 400, |
| "step_time": 5.656911766299163 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.8375, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 94.3, |
| "completions/mean_length": 229.3875, |
| "completions/mean_terminated_length": 78.7, |
| "completions/min_length": 114.1, |
| "completions/min_terminated_length": 62.9, |
| "entropy": 0.07908494919538497, |
| "epoch": 0.0013117397508334346, |
| "frac_reward_zero_std": 0.75, |
| "grad_norm": 0.0, |
| "learning_rate": 9.986914596144125e-06, |
| "loss": -0.0365, |
| "num_tokens": 2845326.0, |
| "reward": 0.3717905465513468, |
| "reward_std": 0.1484815463423729, |
| "rewards/reward_func/mean": 0.3717905465513468, |
| "rewards/reward_func/std": 0.14848154485225679, |
| "sampling/importance_sampling_ratio/max": 1.2370566368103026, |
| "sampling/importance_sampling_ratio/mean": 0.9945116937160492, |
| "sampling/importance_sampling_ratio/min": 0.7393935590982437, |
| "sampling/sampling_logp_difference/max": 0.23011792302131653, |
| "sampling/sampling_logp_difference/mean": 0.0021225204749498517, |
| "step": 410, |
| "step_time": 5.705939891400339 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.9875, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 23.2, |
| "completions/mean_length": 255.7, |
| "completions/mean_terminated_length": 23.2, |
| "completions/min_length": 253.6, |
| "completions/min_terminated_length": 23.2, |
| "entropy": 0.11622791923582554, |
| "epoch": 0.0013437334032927867, |
| "frac_reward_zero_std": 0.55, |
| "grad_norm": 1.0996203422546387, |
| "learning_rate": 9.986594659619532e-06, |
| "loss": 0.0185, |
| "num_tokens": 2920366.0, |
| "reward": 0.4640106737613678, |
| "reward_std": 0.18370848000049592, |
| "rewards/reward_func/mean": 0.4640106737613678, |
| "rewards/reward_func/std": 0.183708493411541, |
| "sampling/importance_sampling_ratio/max": 1.4171173453330994, |
| "sampling/importance_sampling_ratio/mean": 1.003091186285019, |
| "sampling/importance_sampling_ratio/min": 0.667611488699913, |
| "sampling/sampling_logp_difference/max": 0.2792933344841003, |
| "sampling/sampling_logp_difference/mean": 0.003231652465183288, |
| "step": 420, |
| "step_time": 5.756985362099658 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.95, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 24.9, |
| "completions/mean_length": 247.1875, |
| "completions/mean_terminated_length": 19.0, |
| "completions/min_length": 192.3, |
| "completions/min_terminated_length": 13.1, |
| "entropy": 0.14475581012666225, |
| "epoch": 0.0013757270557521388, |
| "frac_reward_zero_std": 0.55, |
| "grad_norm": 11.813094139099121, |
| "learning_rate": 9.98627472309494e-06, |
| "loss": -0.016, |
| "num_tokens": 2990521.0, |
| "reward": 0.5379598774015903, |
| "reward_std": 0.2365218322724104, |
| "rewards/reward_func/mean": 0.5379598774015903, |
| "rewards/reward_func/std": 0.23652183823287487, |
| "sampling/importance_sampling_ratio/max": 1.3119380474090576, |
| "sampling/importance_sampling_ratio/mean": 0.9913179278373718, |
| "sampling/importance_sampling_ratio/min": 0.6958961844444275, |
| "sampling/sampling_logp_difference/max": 0.2457788407802582, |
| "sampling/sampling_logp_difference/mean": 0.00400251317769289, |
| "step": 430, |
| "step_time": 5.510892314800003 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.8625, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 92.2, |
| "completions/mean_length": 242.7125, |
| "completions/mean_terminated_length": 81.75, |
| "completions/min_length": 194.9, |
| "completions/min_terminated_length": 66.9, |
| "entropy": 0.1159008003771305, |
| "epoch": 0.0014077207082114909, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 1.4708253145217896, |
| "learning_rate": 9.985954786570346e-06, |
| "loss": -0.0072, |
| "num_tokens": 3058810.0, |
| "reward": 0.5677399009466171, |
| "reward_std": 0.2517916664481163, |
| "rewards/reward_func/mean": 0.5677399009466171, |
| "rewards/reward_func/std": 0.2517916738986969, |
| "sampling/importance_sampling_ratio/max": 1.3556796312332153, |
| "sampling/importance_sampling_ratio/mean": 0.9918678581714631, |
| "sampling/importance_sampling_ratio/min": 0.7032382875680924, |
| "sampling/sampling_logp_difference/max": 0.2598047614097595, |
| "sampling/sampling_logp_difference/mean": 0.0033211032161489127, |
| "step": 440, |
| "step_time": 5.6251649013007405 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.6625, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 166.1, |
| "completions/mean_length": 222.0, |
| "completions/mean_terminated_length": 129.63095703125, |
| "completions/min_length": 107.7, |
| "completions/min_terminated_length": 82.1, |
| "entropy": 0.1969615861773491, |
| "epoch": 0.001439714360670843, |
| "frac_reward_zero_std": 0.65, |
| "grad_norm": 0.0, |
| "learning_rate": 9.985634850045753e-06, |
| "loss": -0.0013, |
| "num_tokens": 3127434.0, |
| "reward": 0.5485919028520584, |
| "reward_std": 0.17314926013350487, |
| "rewards/reward_func/mean": 0.5485919028520584, |
| "rewards/reward_func/std": 0.17314926385879517, |
| "sampling/importance_sampling_ratio/max": 1.5697351694107056, |
| "sampling/importance_sampling_ratio/mean": 1.0374357163906098, |
| "sampling/importance_sampling_ratio/min": 0.6470421016216278, |
| "sampling/sampling_logp_difference/max": 0.2132512927055359, |
| "sampling/sampling_logp_difference/mean": 0.005053542635869235, |
| "step": 450, |
| "step_time": 5.532932810001512 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.525, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 176.8, |
| "completions/mean_length": 193.9625, |
| "completions/mean_terminated_length": 136.64166984558105, |
| "completions/min_length": 93.8, |
| "completions/min_terminated_length": 93.8, |
| "entropy": 0.3504923045635223, |
| "epoch": 0.001471708013130195, |
| "frac_reward_zero_std": 0.4, |
| "grad_norm": 0.0, |
| "learning_rate": 9.985314913521158e-06, |
| "loss": -0.1111, |
| "num_tokens": 3192059.0, |
| "reward": 0.4286737531423569, |
| "reward_std": 0.16954719498753548, |
| "rewards/reward_func/mean": 0.4286737531423569, |
| "rewards/reward_func/std": 0.1695472002029419, |
| "sampling/importance_sampling_ratio/max": 1.558855003118515, |
| "sampling/importance_sampling_ratio/mean": 0.959910637140274, |
| "sampling/importance_sampling_ratio/min": 0.4642279316438362, |
| "sampling/sampling_logp_difference/max": 1.0550345182418823, |
| "sampling/sampling_logp_difference/mean": 0.009053306374698877, |
| "step": 460, |
| "step_time": 5.39767370289992 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.8625, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 104.6, |
| "completions/mean_length": 239.975, |
| "completions/mean_terminated_length": 93.55, |
| "completions/min_length": 157.6, |
| "completions/min_terminated_length": 80.8, |
| "entropy": 0.5055756628513336, |
| "epoch": 0.001503701665589547, |
| "frac_reward_zero_std": 0.6, |
| "grad_norm": 3.5918967723846436, |
| "learning_rate": 9.984994976996565e-06, |
| "loss": -0.0296, |
| "num_tokens": 3260961.0, |
| "reward": 0.5198034137487412, |
| "reward_std": 0.20930832698941232, |
| "rewards/reward_func/mean": 0.5198034137487412, |
| "rewards/reward_func/std": 0.20930833518505096, |
| "sampling/importance_sampling_ratio/max": 1.7837756991386413, |
| "sampling/importance_sampling_ratio/mean": 1.0180927097797394, |
| "sampling/importance_sampling_ratio/min": 0.4891276627779007, |
| "sampling/sampling_logp_difference/max": 0.4487993657588959, |
| "sampling/sampling_logp_difference/mean": 0.010052280919626354, |
| "step": 470, |
| "step_time": 5.559607181699539 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.975, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 19.6, |
| "completions/mean_length": 252.05, |
| "completions/mean_terminated_length": 19.6, |
| "completions/min_length": 224.4, |
| "completions/min_terminated_length": 19.6, |
| "entropy": 0.20359312649816275, |
| "epoch": 0.0015356953180488992, |
| "frac_reward_zero_std": 0.55, |
| "grad_norm": 11.020283699035645, |
| "learning_rate": 9.984675040471972e-06, |
| "loss": -0.0173, |
| "num_tokens": 3328505.0, |
| "reward": 0.4618885099887848, |
| "reward_std": 0.2562588080763817, |
| "rewards/reward_func/mean": 0.4618885099887848, |
| "rewards/reward_func/std": 0.25625881403684614, |
| "sampling/importance_sampling_ratio/max": 1.4413376688957213, |
| "sampling/importance_sampling_ratio/mean": 1.0164444029331208, |
| "sampling/importance_sampling_ratio/min": 0.6880250811576843, |
| "sampling/sampling_logp_difference/max": 0.36722205877304076, |
| "sampling/sampling_logp_difference/mean": 0.003699347673682496, |
| "step": 480, |
| "step_time": 5.570423536199814 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.95, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 35.3, |
| "completions/mean_length": 247.6125, |
| "completions/mean_terminated_length": 35.3, |
| "completions/min_length": 188.9, |
| "completions/min_terminated_length": 35.3, |
| "entropy": 0.25075023621320724, |
| "epoch": 0.0015676889705082513, |
| "frac_reward_zero_std": 0.6, |
| "grad_norm": 3.4280946254730225, |
| "learning_rate": 9.984355103947377e-06, |
| "loss": 0.041, |
| "num_tokens": 3393326.0, |
| "reward": 0.4563781010918319, |
| "reward_std": 0.19282358512282372, |
| "rewards/reward_func/mean": 0.4563781010918319, |
| "rewards/reward_func/std": 0.1928235985338688, |
| "sampling/importance_sampling_ratio/max": 1.6902172982692718, |
| "sampling/importance_sampling_ratio/mean": 0.8757067143917083, |
| "sampling/importance_sampling_ratio/min": 0.30804501920938493, |
| "sampling/sampling_logp_difference/max": 1.0456432223320007, |
| "sampling/sampling_logp_difference/mean": 0.006558918510563672, |
| "step": 490, |
| "step_time": 5.501604846100963 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.9625, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 40.2, |
| "completions/mean_length": 251.5875, |
| "completions/mean_terminated_length": 28.7, |
| "completions/min_length": 222.0, |
| "completions/min_terminated_length": 17.2, |
| "entropy": 0.34359313249588014, |
| "epoch": 0.0015996826229676031, |
| "frac_reward_zero_std": 0.35, |
| "grad_norm": 6.27419376373291, |
| "learning_rate": 9.984035167422784e-06, |
| "loss": 0.0614, |
| "num_tokens": 3458237.0, |
| "reward": 0.435734324157238, |
| "reward_std": 0.1720918044447899, |
| "rewards/reward_func/mean": 0.435734324157238, |
| "rewards/reward_func/std": 0.17209180817008018, |
| "sampling/importance_sampling_ratio/max": 1.7418416380882262, |
| "sampling/importance_sampling_ratio/mean": 0.862521804869175, |
| "sampling/importance_sampling_ratio/min": 0.3627486750483513, |
| "sampling/sampling_logp_difference/max": 0.7209866166114807, |
| "sampling/sampling_logp_difference/mean": 0.009752574237063527, |
| "step": 500, |
| "step_time": 5.387810998200439 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 312562, |
| "num_input_tokens_seen": 3458237, |
| "num_train_epochs": 1, |
| "save_steps": 50, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 8, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
