grafted-fixed/wandb/run-20241209_200047-torch-grafted-the-charm/logs/debug-internal.log

{"time":"2024-12-09T20:00:47.449350846-08:00","level":"INFO","msg":"using version","core version":"0.18.7"}
{"time":"2024-12-09T20:00:47.449359816-08:00","level":"INFO","msg":"created symlink","path":"/home/jason-chou/Downloads/big_vision/wandb/run-20241209_200047-torch-grafted-the-charm/logs/debug-core.log"}
{"time":"2024-12-09T20:00:47.551810178-08:00","level":"INFO","msg":"created new stream","id":"torch-grafted-the-charm"}
{"time":"2024-12-09T20:00:47.551852977-08:00","level":"INFO","msg":"stream: started","id":"torch-grafted-the-charm"}
{"time":"2024-12-09T20:00:47.552155988-08:00","level":"INFO","msg":"handler: started","stream_id":"torch-grafted-the-charm"}
{"time":"2024-12-09T20:00:47.552238982-08:00","level":"INFO","msg":"sender: started","stream_id":"torch-grafted-the-charm"}
{"time":"2024-12-09T20:00:47.552242529-08:00","level":"INFO","msg":"writer: Do: started","stream_id":"torch-grafted-the-charm"}
{"time":"2024-12-09T20:00:48.132894291-08:00","level":"INFO","msg":"Starting system monitor"}
{"time":"2024-12-09T21:55:18.337868527-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-09T22:09:18.343882156-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-09T22:12:33.35181566-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-09T22:15:05.895562279-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-the-charm/file_stream"}
{"time":"2024-12-10T06:57:48.812039829-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-10T06:58:21.269593144-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-10T06:58:55.688038738-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-10T06:59:35.68254092-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-10T07:00:24.620322236-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-10T07:28:18.878094248-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-10T07:28:50.962490401-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-10T07:29:25.103368986-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-10T07:30:04.599024906-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-10T07:30:54.286803162-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-10T09:36:42.863726052-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-the-charm/file_stream\": unexpected EOF"}
{"time":"2024-12-10T09:42:19.012597191-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-10T09:42:51.070922518-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-10T09:43:25.378998359-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-10T09:44:04.575850506-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-10T09:44:52.88655775-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-10T17:43:34.443445373-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-10T17:44:06.93043529-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-10T17:44:41.352440591-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-10T17:45:20.743742501-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-10T17:46:08.141469304-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-10T18:02:42.333697207-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-the-charm/file_stream"}
{"time":"2024-12-10T18:03:49.485889478-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-10T18:04:21.528555101-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-10T18:04:55.804416059-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-10T18:05:35.660171287-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-10T18:06:23.525574621-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-10T19:28:19.608224899-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-10T19:28:51.806061596-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-10T19:29:26.800667429-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-10T19:30:04.816063078-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-10T19:30:52.812060771-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-10T19:53:34.670035711-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-10T19:54:06.977130349-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-10T19:54:41.576597171-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-10T19:55:21.554299063-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-10T19:56:07.94782966-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-10T22:02:00.554690138-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-the-charm/file_stream"}
{"time":"2024-12-11T09:44:05.446217936-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-11T09:44:37.859760806-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-11T09:45:12.288444067-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-11T09:45:50.921498923-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-11T09:46:38.469641311-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-11T11:19:05.57844594-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-11T11:19:37.697619303-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-11T11:20:12.080840442-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-11T11:20:50.792075563-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-11T11:21:38.283775375-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-11T11:22:41.620946001-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": read tcp 10.0.0.84:43994->35.186.228.49:443: read: connection reset by peer"}
{"time":"2024-12-11T11:39:05.649733723-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-11T11:39:37.77306646-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-11T11:40:12.111468894-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-11T11:40:51.962899173-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-11T11:41:38.390618819-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-11T19:54:35.555435248-08:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-the-charm/file_stream"}
{"time":"2024-12-12T02:18:21.443500235-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-12T02:18:53.923176433-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-12T02:19:28.026484112-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-12T02:20:06.482347454-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-12T02:20:53.489307691-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-12T02:28:51.489330731-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-12T02:29:23.628060435-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-12T02:29:57.771259295-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-12T02:30:36.731471107-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-12T02:31:26.644552765-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-12T12:16:07.051712483-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-12T12:16:39.190372519-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-12T12:17:13.985712831-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-12T12:17:53.43817487-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-12T12:18:39.597076671-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-12T12:26:22.108020559-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-12T12:26:54.57687148-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-12T12:27:28.647446374-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-12T12:28:07.688958342-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-12T12:28:57.547098341-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-12T14:16:07.23903407-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-12T14:16:39.390301955-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-12T14:17:13.981602637-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-12T14:17:52.668032171-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-12T14:18:40.683899565-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-12T14:48:22.310195481-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-12T14:48:54.463008334-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-12T14:49:29.197682043-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-12T14:50:09.016939995-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-12T14:50:55.980211119-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-13T00:01:21.104220708-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-the-charm/file_stream\": dial tcp 35.186.228.49:443: connect: no route to host"}
{"time":"2024-12-13T03:27:27.941753483-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-the-charm/file_stream\": read tcp 10.0.0.84:46014->35.186.228.49:443: read: connection reset by peer"}
{"time":"2024-12-13T08:03:23.255206173-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-13T08:03:55.486160975-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-13T08:04:29.755448667-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-13T08:05:08.547182012-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-13T08:05:54.703829757-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-13T08:13:38.445571587-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-13T08:14:10.851069374-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-13T08:14:45.806421997-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-13T08:15:24.023491607-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-13T08:16:11.885843772-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-13T10:11:38.583819827-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-13T10:29:11.203012924-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-the-charm/file_stream"}
{"time":"2024-12-13T11:33:03.268785476-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-the-charm/file_stream"}
{"time":"2024-12-13T20:18:39.103776723-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-13T20:19:11.454173389-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-13T20:19:46.408965335-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-13T20:20:26.355569053-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-13T20:21:12.931442216-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-13T22:15:39.229142978-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-13T22:16:11.581178825-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-13T22:16:46.156108543-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-13T22:17:24.642971988-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-13T22:18:13.488730153-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-13T22:24:24.277332999-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-13T22:24:56.711667386-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-13T22:25:30.863733417-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-13T22:26:09.829008172-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-13T22:26:57.250803957-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-13T22:27:34.604346158-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": read tcp 10.0.0.84:51008->35.186.228.49:443: read: connection reset by peer"}
{"time":"2024-12-14T01:19:24.493733654-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-14T01:19:56.826230141-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-14T01:20:31.461396731-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-14T01:21:09.493298907-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-14T01:21:58.390754123-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-14T02:24:54.58759746-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-14T02:25:27.021624552-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-14T02:26:02.019694238-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-14T02:26:40.554208551-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-14T02:27:28.096817098-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-14T02:28:04.813861439-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": read tcp 10.0.0.84:46106->35.186.228.49:443: read: connection reset by peer"}
{"time":"2024-12-14T10:25:30.311359617-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-the-charm/file_stream\": dial tcp 35.186.228.49:443: connect: no route to host"}
{"time":"2024-12-14T12:12:40.144841085-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-14T12:13:12.636177975-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-14T12:13:47.454722317-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-14T12:14:26.193246239-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-14T12:15:12.465569841-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-14T12:15:51.370573502-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": read tcp 10.0.0.84:60550->35.186.228.49:443: read: connection reset by peer"}
{"time":"2024-12-14T12:57:55.235438379-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-14T12:58:27.368362544-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-14T12:59:01.623117758-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-14T12:59:41.427311337-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-14T13:00:27.465187213-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-14T13:01:07.020724228-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": read tcp 10.0.0.84:37682->35.186.228.49:443: read: connection reset by peer"}
{"time":"2024-12-14T14:57:25.387292073-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-14T14:57:57.417457739-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-14T14:58:31.668620272-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-14T14:59:10.911399993-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-14T15:00:00.779320828-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-14T15:07:40.429900179-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-14T15:08:12.441292257-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-14T15:08:46.69689517-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-14T15:09:25.511004803-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-14T15:10:11.988778687-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-14T15:10:53.516040911-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": read tcp 10.0.0.84:33002->35.186.228.49:443: read: connection reset by peer"}
{"time":"2024-12-14T17:07:25.575615365-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-14T17:07:58.035825011-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-14T17:08:33.027568844-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-14T17:09:12.706516553-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-14T17:09:59.051100482-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-15T01:12:41.039661947-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-15T01:13:13.261161675-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-15T01:13:47.383598849-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-15T01:14:25.940842689-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-15T01:15:13.385892375-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-15T01:20:41.083848619-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-15T01:21:13.518808547-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-15T01:21:47.897249069-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-15T01:22:27.259488649-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-15T01:23:14.077974021-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-15T03:14:32.007332221-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-the-charm/file_stream\": dial tcp 35.186.228.49:443: connect: no route to host"}
{"time":"2024-12-15T11:16:29.703331511-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-the-charm/file_stream\": dial tcp 35.186.228.49:443: connect: no route to host"}
{"time":"2024-12-15T15:14:26.552012127-08:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-the-charm/file_stream"}
{"time":"2024-12-15T15:14:33.812701767-08:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-the-charm/file_stream"}
{"time":"2024-12-15T15:14:49.274411959-08:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-the-charm/file_stream"}
{"time":"2024-12-15T19:18:08.136295319-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-the-charm/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
{"time":"2024-12-16T05:21:48.232146816-08:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-the-charm/file_stream"}
{"time":"2024-12-16T05:27:10.599456797-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-the-charm/file_stream\": dial tcp 35.186.228.49:443: connect: no route to host"}
{"time":"2024-12-16T15:27:58.098597759-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-16T20:42:15.780784315-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-the-charm/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
{"time":"2024-12-16T20:42:18.215017044-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-the-charm/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
{"time":"2024-12-17T04:44:13.854729611-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-17T04:44:46.006964921-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-17T04:45:20.167712408-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-17T04:45:58.5311663-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-17T04:46:46.952116639-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-17T04:47:19.64269303-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": read tcp 10.0.0.84:53814->35.186.228.49:443: read: connection reset by peer"}
{"time":"2024-12-17T04:49:28.903025784-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2024-12-17T04:50:00.989363571-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-17T04:50:35.654573479-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-17T04:51:13.693432859-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-17T04:52:02.897165691-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2024-12-17T08:02:23.415945378-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-the-charm/file_stream"}
{"time":"2024-12-17T11:23:03.376517984-08:00","level":"INFO","msg":"stream: closing","id":"torch-grafted-the-charm"}
{"time":"2024-12-17T11:23:03.376743809-08:00","level":"INFO","msg":"Stopping system monitor"}
{"time":"2024-12-17T11:23:03.378778157-08:00","level":"INFO","msg":"Stopped system monitor"}
{"time":"2024-12-17T11:23:03.490358715-08:00","level":"WARN","msg":"No program path found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job"}
{"time":"2024-12-17T11:23:03.490374403-08:00","level":"INFO","msg":"sender: sendDefer: no job artifact to save"}
{"time":"2024-12-17T11:23:04.873709393-08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
{"time":"2024-12-17T11:23:04.990284517-08:00","level":"INFO","msg":"handler: closed","stream_id":"torch-grafted-the-charm"}
{"time":"2024-12-17T11:23:04.99036012-08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"torch-grafted-the-charm"}
{"time":"2024-12-17T11:23:04.990611581-08:00","level":"INFO","msg":"sender: closed","stream_id":"torch-grafted-the-charm"}
{"time":"2024-12-17T11:23:04.990650372-08:00","level":"INFO","msg":"stream: closed","id":"torch-grafted-the-charm"}