File size: 10,293 Bytes
45e4806
 
 
 
 
 
 
 
d4922e5
 
 
ee0acaa
45e4806
 
 
 
d4922e5
45e4806
d4922e5
 
 
45e4806
ee0acaa
 
 
 
 
d4922e5
ee0acaa
d4922e5
 
 
ee0acaa
 
 
 
 
 
d4922e5
ee0acaa
d4922e5
 
 
ee0acaa
 
 
 
 
 
d4922e5
ee0acaa
d4922e5
 
 
ee0acaa
 
 
 
 
 
d4922e5
ee0acaa
d4922e5
 
 
ee0acaa
 
 
 
 
 
d4922e5
ee0acaa
d4922e5
 
 
ee0acaa
 
 
 
 
 
d4922e5
ee0acaa
d4922e5
 
 
 
 
ee0acaa
 
 
 
 
 
d4922e5
ee0acaa
d4922e5
 
 
 
 
ee0acaa
 
 
 
 
 
d4922e5
ee0acaa
d4922e5
 
 
 
ee0acaa
 
 
 
 
 
d4922e5
ee0acaa
d4922e5
 
 
 
ee0acaa
 
 
 
 
 
d4922e5
ee0acaa
d4922e5
 
 
 
ee0acaa
 
 
 
 
 
d4922e5
ee0acaa
d4922e5
 
 
 
ee0acaa
 
45e4806
 
 
 
d4922e5
45e4806
d4922e5
 
 
 
ee0acaa
 
 
 
 
 
d4922e5
ee0acaa
d4922e5
 
 
 
 
 
ee0acaa
 
 
 
 
 
d4922e5
ee0acaa
d4922e5
 
 
 
 
 
ee0acaa
 
 
 
 
 
d4922e5
ee0acaa
d4922e5
 
 
 
ee0acaa
 
 
 
 
 
d4922e5
ee0acaa
d4922e5
 
 
 
ee0acaa
 
 
 
 
 
d4922e5
ee0acaa
d4922e5
d038c6e
d4922e5
 
ee0acaa
 
 
 
 
 
d4922e5
ee0acaa
d4922e5
 
 
 
45e4806
ee0acaa
45e4806
 
 
 
d4922e5
45e4806
d038c6e
 
d4922e5
d038c6e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d4922e5
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
spec_version: 1
name: netweaver_sre
type: space
runtime: fastapi
app: server.app:app
port: 7860

tasks:

  # ── EASY ──────────────────────────────────────────────────────────────────

  - id: netweaver_sre_t01
    difficulty: easy
    grader:
      type: deterministic
      endpoint: /grader
      task_level: t01
    description: >
      Node Offline Triage. Read hardware_logs to find the offline node.
      Remediate with DRAIN_TRAFFIC on the identified node.
      Scored on: diagnosis (40%) + resolution (40%) + best practice (20%).

  - id: netweaver_sre_t02
    difficulty: easy
    grader:
      type: deterministic
      endpoint: /grader
      task_level: t02
    description: >
      Stale DNS Cache. Read hardware_logs to identify the node.
      Issue CLEAR_DNS_CACHE on the affected node.
      Scored on: diagnosis (40%) + resolution (40%) + best practice (20%).

  - id: netweaver_sre_t03
    difficulty: easy
    grader:
      type: deterministic
      endpoint: /grader
      task_level: t03
    description: >
      OOM Crash. Read hardware_logs for the crashed service name.
      Issue RESTART_SERVICE on that service.
      Scored on: diagnosis (40%) + resolution (40%) + best practice (20%).

  - id: netweaver_sre_t04
    difficulty: easy
    grader:
      type: deterministic
      endpoint: /grader
      task_level: t04
    description: >
      TLS Certificate Expiry. Read hardware_logs for the affected node.
      Issue RENEW_CERTIFICATE on that node.
      Scored on: diagnosis (40%) + resolution (40%) + best practice (20%).

  - id: netweaver_sre_t05
    difficulty: easy
    grader:
      type: deterministic
      endpoint: /grader
      task_level: t05
    description: >
      Disk Space Full. Read hardware_logs for the full node.
      Issue CLEAR_TEMP_FILES on that node.
      Scored on: diagnosis (40%) + resolution (40%) + best practice (20%).

  - id: netweaver_sre_t06
    difficulty: easy
    grader:
      type: deterministic
      endpoint: /grader
      task_level: t06
    description: >
      Unhealthy Pod in CrashLoopBackOff. Read hardware_logs for the pod name.
      Issue RESTART_POD on that pod.
      Scored on: diagnosis (40%) + resolution (40%) + best practice (20%).

  - id: netweaver_sre_t07
    difficulty: easy
    grader:
      type: deterministic
      endpoint: /grader
      task_level: t07
    description: >
      Zombie Process filling PID table. Read hardware_logs for the node.
      Issue KILL_ZOMBIE_PROCESS on that node.
      Scored on: diagnosis (40%) + resolution (40%) + best practice (20%).

  # ── MEDIUM ────────────────────────────────────────────────────────────────

  - id: netweaver_sre_t08
    difficulty: medium
    grader:
      type: deterministic
      endpoint: /grader
      task_level: t08
    description: >
      PFC Buffer Congestion. Read queue_depths for the switch near 99.9.
      Issue TUNE_PFC_THRESHOLD on that switch with a numeric value (1000-9000).
      Grader checks correct target AND value in valid range.
      Efficiency penalty applied for steps beyond ideal (3).
      Scored on: diagnosis (40%) + resolution+efficiency (40%) + best practice (20%).

  - id: netweaver_sre_t09
    difficulty: medium
    grader:
      type: deterministic
      endpoint: /grader
      task_level: t09
    description: >
      Power Throttling. Read hardware_logs for node and recommended wattage.
      Issue ADJUST_POWER_CAP on that node with the watt value.
      Efficiency penalty applied for steps beyond ideal (3).
      Scored on: diagnosis (40%) + resolution+efficiency (40%) + best practice (20%).

  - id: netweaver_sre_t10
    difficulty: medium
    grader:
      type: deterministic
      endpoint: /grader
      task_level: t10
    description: >
      BGP Route Flapping. Read hardware_logs for router and AS number.
      Issue MITIGATE_ROUTE_FLAP on that router with the AS number as value.
      Efficiency penalty applied for steps beyond ideal (3).
      Scored on: diagnosis (40%) + resolution+efficiency (40%) + best practice (20%).

  - id: netweaver_sre_t11
    difficulty: medium
    grader:
      type: deterministic
      endpoint: /grader
      task_level: t11
    description: >
      Jumbo Frame Packet Drop (MTU mismatch). Read queue_depths/logs for switch.
      Issue INCREASE_MTU on that switch with value exactly 9000.
      Efficiency penalty applied for steps beyond ideal (3).
      Scored on: diagnosis (40%) + resolution+efficiency (40%) + best practice (20%).

  - id: netweaver_sre_t12
    difficulty: medium
    grader:
      type: deterministic
      endpoint: /grader
      task_level: t12
    description: >
      DDoS API flood. Read hardware_logs for the gateway name.
      Issue SET_RATE_LIMIT on the gateway with a numeric req/s value.
      Efficiency penalty applied for steps beyond ideal (3).
      Scored on: diagnosis (40%) + resolution+efficiency (40%) + best practice (20%).

  - id: netweaver_sre_t13
    difficulty: medium
    grader:
      type: deterministic
      endpoint: /grader
      task_level: t13
    description: >
      DB Connection Pool Exhaustion. Read hardware_logs for DB target.
      Issue SCALE_CONN_POOL on that DB with the new pool size as value.
      Efficiency penalty applied for steps beyond ideal (3).
      Scored on: diagnosis (40%) + resolution+efficiency (40%) + best practice (20%).

  - id: netweaver_sre_t14
    difficulty: medium
    grader:
      type: deterministic
      endpoint: /grader
      task_level: t14
    description: >
      CPU Context Switch Storm. Read hardware_logs for node and thread count.
      Issue PIN_CPU_THREADS on that node with thread count as value.
      Efficiency penalty applied for steps beyond ideal (3).
      Scored on: diagnosis (40%) + resolution+efficiency (40%) + best practice (20%).

  # ── HARD ──────────────────────────────────────────────────────────────────

  - id: netweaver_sre_t15
    difficulty: hard
    grader:
      type: deterministic
      endpoint: /grader
      task_level: t15
    description: >
      Silent NaN Contagion. Read gradient_variances for ranks with value -1.0.
      Multi-step: (1) RUN_MINI_ITERATION on affected cluster to isolate,
      then (2) DRAIN_TRAFFIC on same cluster to remove from pool.
      Both commands required for full resolution score.
      Efficiency penalty applied for steps beyond ideal (4).
      Scored on: diagnosis (40%) + resolution+efficiency (40%) + best practice (20%).

  - id: netweaver_sre_t16
    difficulty: hard
    grader:
      type: deterministic
      endpoint: /grader
      task_level: t16
    description: >
      Broadcast Storm. Read queue_depths for the switch at 99.9.
      Issue ISOLATE_BROADCAST_STORM on that exact switch.
      Efficiency penalty applied for steps beyond ideal (3).
      Scored on: diagnosis (40%) + resolution+efficiency (40%) + best practice (20%).

  - id: netweaver_sre_t17
    difficulty: hard
    grader:
      type: deterministic
      endpoint: /grader
      task_level: t17
    description: >
      GPU Memory Leak. Read gpu_memory_usage for the cluster index near 1.0.
      Issue RESTART_GPU_DAEMON on cluster_<that index>.
      Efficiency penalty applied for steps beyond ideal (3).
      Scored on: diagnosis (40%) + resolution+efficiency (40%) + best practice (20%).

  - id: netweaver_sre_t18
    difficulty: hard
    grader:
      type: deterministic
      endpoint: /grader
      task_level: t18
    description: >
      Full Cluster Deadlock. All arrays frozen at 0.0 β€” distributed deadlock.
      Issue ISSUE_GLOBAL_ROLLBACK with the exact cluster target from logs.
      Efficiency penalty applied for steps beyond ideal (3).
      Scored on: diagnosis (40%) + resolution (40%) + best practice (20%).

  - id: netweaver_sre_t19
    difficulty: hard
    grader:
      type: deterministic
      endpoint: /grader
      task_level: t19
    description: >
      Network Partition. Read queue_depths for the split (0.01 vs 99.9).
      Issue REBOOT_LEAF_SWITCHES on the pod with the maxed entry.
      Efficiency penalty applied for steps beyond ideal (4).
      Scored on: diagnosis (40%) + resolution+efficiency (40%) + best practice (20%).

  - id: netweaver_sre_t20
    difficulty: hard
    grader:
      type: deterministic
      endpoint: /grader
      task_level: t20
    description: >
      Corrupt DB Block. Read logs for the randomized DB cluster target.
      Issue PURGE_CORRUPT_BLOCK on that exact DB target.
      Efficiency penalty applied for steps beyond ideal (4).
      Scored on: diagnosis (40%) + resolution+efficiency (40%) + best practice (20%).

  - id: netweaver_sre_t21
    difficulty: hard
    grader:
      type: deterministic
      endpoint: /grader
      task_level: t21
    description: >
      Cascading Failure Chain. A power fault triggers a CPU context switch storm
      which overflows the connection pool. Requires 3 sequential remediations:
      (1) ADJUST_POWER_CAP on the throttled node,
      (2) PIN_CPU_THREADS on the same node,
      (3) SCALE_CONN_POOL on the database.
      Order matters. Partial credit for each completed step.
      Scored on: diagnosis (40%) + resolution+efficiency (40%) + best practice (20%).

  - id: netweaver_sre_t22
    difficulty: hard
    grader:
      type: deterministic
      endpoint: /grader
      task_level: t22
    description: >
      Silent Gradient Poisoning with Network Amplification.
      NaN is corrupting gradient sync AND a broadcast storm is amplifying the corrupted
      gradients across the fabric. Requires:
      (1) RUN_MINI_ITERATION to isolate NaN source cluster,
      (2) ISOLATE_BROADCAST_STORM on the storm switch,
      (3) DRAIN_TRAFFIC on the poisoned cluster.
      All 3 commands required in this order for full resolution.
      Scored on: diagnosis (40%) + resolution+efficiency (40%) + best practice (20%).