Upload debug-internal.log
Browse files
logs/20250418155709/wandb/run-20250418_160153-y58xpaxf/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2025-04-18T16:01:53.487244656Z","level":"INFO","msg":"using version","core version":"0.19.1"}
|
| 2 |
+
{"time":"2025-04-18T16:01:53.48726703Z","level":"INFO","msg":"created symlink","path":"/kaggle/working/model/logs/20250418155709/wandb/run-20250418_160153-y58xpaxf/logs/debug-core.log"}
|
| 3 |
+
{"time":"2025-04-18T16:01:53.593053691Z","level":"INFO","msg":"created new stream","id":"y58xpaxf"}
|
| 4 |
+
{"time":"2025-04-18T16:01:53.593098122Z","level":"INFO","msg":"stream: started","id":"y58xpaxf"}
|
| 5 |
+
{"time":"2025-04-18T16:01:53.593163737Z","level":"INFO","msg":"handler: started","stream_id":"y58xpaxf"}
|
| 6 |
+
{"time":"2025-04-18T16:01:53.594145548Z","level":"INFO","msg":"sender: started","stream_id":"y58xpaxf"}
|
| 7 |
+
{"time":"2025-04-18T16:01:53.594154082Z","level":"INFO","msg":"writer: Do: started","stream_id":"y58xpaxf"}
|
| 8 |
+
{"time":"2025-04-18T16:01:53.847207658Z","level":"INFO","msg":"Starting system monitor"}
|
| 9 |
+
{"time":"2025-04-18T16:07:40.598113153Z","level":"WARN","msg":"handler: ignoring partial history record","step":1,"current":17}
|
| 10 |
+
{"time":"2025-04-18T16:13:27.961251266Z","level":"WARN","msg":"handler: ignoring partial history record","step":2,"current":34}
|
| 11 |
+
{"time":"2025-04-18T16:19:24.134690807Z","level":"WARN","msg":"handler: ignoring partial history record","step":3,"current":51}
|
| 12 |
+
{"time":"2025-04-18T16:25:22.823564901Z","level":"WARN","msg":"handler: ignoring partial history record","step":4,"current":68}
|
| 13 |
+
{"time":"2025-04-18T16:30:52.951744415Z","level":"WARN","msg":"handler: ignoring partial history record","step":5,"current":85}
|
| 14 |
+
{"time":"2025-04-18T16:36:41.969244114Z","level":"WARN","msg":"handler: ignoring partial history record","step":6,"current":102}
|
| 15 |
+
{"time":"2025-04-18T16:42:25.627272865Z","level":"WARN","msg":"handler: ignoring partial history record","step":7,"current":119}
|
| 16 |
+
{"time":"2025-04-18T16:48:24.398433696Z","level":"WARN","msg":"handler: ignoring partial history record","step":8,"current":136}
|
| 17 |
+
{"time":"2025-04-18T16:54:33.184790273Z","level":"WARN","msg":"handler: ignoring partial history record","step":9,"current":153}
|
| 18 |
+
{"time":"2025-04-18T16:59:52.638324077Z","level":"WARN","msg":"handler: ignoring partial history record","step":10,"current":170}
|
| 19 |
+
{"time":"2025-04-18T17:06:01.203206271Z","level":"WARN","msg":"handler: ignoring partial history record","step":11,"current":187}
|
| 20 |
+
{"time":"2025-04-18T17:11:32.295994989Z","level":"WARN","msg":"handler: ignoring partial history record","step":12,"current":204}
|
| 21 |
+
{"time":"2025-04-18T17:16:44.057825199Z","level":"WARN","msg":"handler: ignoring partial history record","step":13,"current":221}
|
| 22 |
+
{"time":"2025-04-18T17:22:20.985419787Z","level":"WARN","msg":"handler: ignoring partial history record","step":14,"current":238}
|
| 23 |
+
{"time":"2025-04-18T17:28:10.320801961Z","level":"WARN","msg":"handler: ignoring partial history record","step":15,"current":255}
|
| 24 |
+
{"time":"2025-04-18T17:34:02.257505652Z","level":"WARN","msg":"handler: ignoring partial history record","step":16,"current":272}
|
| 25 |
+
{"time":"2025-04-18T17:39:39.974482121Z","level":"WARN","msg":"handler: ignoring partial history record","step":17,"current":289}
|
| 26 |
+
{"time":"2025-04-18T17:45:29.368226987Z","level":"WARN","msg":"handler: ignoring partial history record","step":18,"current":306}
|
| 27 |
+
{"time":"2025-04-18T17:51:30.53157647Z","level":"WARN","msg":"handler: ignoring partial history record","step":19,"current":323}
|
| 28 |
+
{"time":"2025-04-18T17:56:39.560168224Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
|
| 29 |
+
{"time":"2025-04-18T17:57:10.552991941Z","level":"WARN","msg":"handler: ignoring partial history record","step":20,"current":340}
|
| 30 |
+
{"time":"2025-04-18T17:57:42.157984447Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/kymi-kymi/Illustrious-XL-v0.1_LoRA/y58xpaxf/file_stream","body":"\n<html><head>\n<meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\">\n<title>502 Server Error</title>\n</head>\n<body text=#000000 bgcolor=#ffffff>\n<h1>Error: Server Error</h1>\n<h2>The server encountered a temporary error and could not complete your request.<p>Please try again in 30 seconds.</h2>\n<h2></h2>\n</body></html>\n"}
|
| 31 |
+
{"time":"2025-04-18T18:00:24.567132821Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": read tcp 172.19.2.2:37876->35.186.228.49:443: read: connection reset by peer"}
|
| 32 |
+
{"time":"2025-04-18T18:02:55.635252503Z","level":"WARN","msg":"handler: ignoring partial history record","step":21,"current":357}
|
| 33 |
+
{"time":"2025-04-18T18:08:47.998249122Z","level":"WARN","msg":"handler: ignoring partial history record","step":22,"current":374}
|
| 34 |
+
{"time":"2025-04-18T18:14:16.660264792Z","level":"WARN","msg":"handler: ignoring partial history record","step":23,"current":391}
|
| 35 |
+
{"time":"2025-04-18T18:19:43.957537783Z","level":"WARN","msg":"handler: ignoring partial history record","step":24,"current":408}
|
| 36 |
+
{"time":"2025-04-18T18:25:22.560265197Z","level":"WARN","msg":"handler: ignoring partial history record","step":25,"current":425}
|
| 37 |
+
{"time":"2025-04-18T18:31:05.976242802Z","level":"WARN","msg":"handler: ignoring partial history record","step":26,"current":442}
|
| 38 |
+
{"time":"2025-04-18T18:36:55.965262909Z","level":"WARN","msg":"handler: ignoring partial history record","step":27,"current":459}
|
| 39 |
+
{"time":"2025-04-18T18:42:47.804634266Z","level":"WARN","msg":"handler: ignoring partial history record","step":28,"current":476}
|
| 40 |
+
{"time":"2025-04-18T18:48:53.149787125Z","level":"WARN","msg":"handler: ignoring partial history record","step":29,"current":493}
|
| 41 |
+
{"time":"2025-04-18T18:54:49.862248851Z","level":"WARN","msg":"handler: ignoring partial history record","step":30,"current":510}
|
| 42 |
+
{"time":"2025-04-18T19:00:32.383571937Z","level":"WARN","msg":"handler: ignoring partial history record","step":31,"current":527}
|
| 43 |
+
{"time":"2025-04-18T19:05:26.574243133Z","level":"WARN","msg":"handler: ignoring partial history record","step":32,"current":544}
|
| 44 |
+
{"time":"2025-04-18T19:11:11.569219661Z","level":"WARN","msg":"handler: ignoring partial history record","step":33,"current":561}
|
| 45 |
+
{"time":"2025-04-18T19:17:23.444836071Z","level":"WARN","msg":"handler: ignoring partial history record","step":34,"current":578}
|
| 46 |
+
{"time":"2025-04-18T19:23:20.265263228Z","level":"WARN","msg":"handler: ignoring partial history record","step":35,"current":595}
|
| 47 |
+
{"time":"2025-04-18T19:28:57.021478169Z","level":"WARN","msg":"handler: ignoring partial history record","step":36,"current":612}
|
| 48 |
+
{"time":"2025-04-18T19:34:38.155235953Z","level":"WARN","msg":"handler: ignoring partial history record","step":37,"current":629}
|
| 49 |
+
{"time":"2025-04-18T19:40:39.421274665Z","level":"WARN","msg":"handler: ignoring partial history record","step":38,"current":646}
|
| 50 |
+
{"time":"2025-04-18T19:46:04.477292516Z","level":"WARN","msg":"handler: ignoring partial history record","step":39,"current":663}
|
| 51 |
+
{"time":"2025-04-18T19:51:22.396133473Z","level":"WARN","msg":"handler: ignoring partial history record","step":40,"current":680}
|
| 52 |
+
{"time":"2025-04-18T19:51:22.402364392Z","level":"INFO","msg":"Stopping system monitor"}
|
| 53 |
+
{"time":"2025-04-18T19:51:22.403171168Z","level":"INFO","msg":"Stopped system monitor"}
|
| 54 |
+
{"time":"2025-04-18T19:51:22.884979622Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
|
| 55 |
+
{"time":"2025-04-18T19:51:23.014790461Z","level":"INFO","msg":"handler: operation stats","stats":{}}
|
| 56 |
+
{"time":"2025-04-18T19:51:24.030169191Z","level":"INFO","msg":"stream: closing","id":"y58xpaxf"}
|
| 57 |
+
{"time":"2025-04-18T19:51:24.030200908Z","level":"INFO","msg":"handler: closed","stream_id":"y58xpaxf"}
|
| 58 |
+
{"time":"2025-04-18T19:51:24.030235237Z","level":"INFO","msg":"writer: Close: closed","stream_id":"y58xpaxf"}
|
| 59 |
+
{"time":"2025-04-18T19:51:24.030302398Z","level":"INFO","msg":"sender: closed","stream_id":"y58xpaxf"}
|
| 60 |
+
{"time":"2025-04-18T19:51:24.03033681Z","level":"INFO","msg":"stream: closed","id":"y58xpaxf"}
|