Grafted experiment: PyTorch model trained with tfds imagenet2012 data

.gitattributes

@@ -35,3 +35,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 05-15_2003/wandb/run-20240515_200355-torch-on-big-vision-bfloat16/logs/debug-internal.log filter=lfs diff=lfs merge=lfs -text
 05-15_2003/wandb/run-20240515_200355-torch-on-big-vision-bfloat16/run-torch-on-big-vision-bfloat16.wandb filter=lfs diff=lfs merge=lfs -text
+grafted/wandb/run-20241127_215015-torch-grafted-redux/run-torch-grafted-redux.wandb filter=lfs diff=lfs merge=lfs -text

grafted/checkpoint.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1a8535185a902993c4e798b63171f29063335d6073303d6f357609c25b6aea7
+size 264187339

grafted/config.json

@@ -0,0 +1 @@
+{"seed": 0, "total_epochs": 90, "num_classes": 1000, "loss": "softmax_xent", "input": {"data": {"name": "imagenet2012", "split": "train"}, "batch_size": 1024, "accum_freq": 8, "cache_raw": false, "shuffle_buffer_size": 150000, "pp": "decode_jpeg_and_inception_crop(224)|flip_lr|randaug(2,10)|value_range(-1, 1)|onehot(1000, key=\"label\", key_result=\"labels\")|keep(\"image\", \"labels\")"}, "pp_modules": ["ops_general", "ops_image", "ops_text", "archive.randaug"], "log_training_steps": 50, "ckpt_steps": 1000, "model_name": "vit", "model": {"variant": "S/16", "rep_size": false, "pool_type": "gap", "posemb": "sincos2d"}, "grad_clip_norm": 1.0, "optax_name": "scale_by_adam", "optax": {"mu_dtype": "bfloat16"}, "lr": 0.001, "wd": 0.0001, "schedule": {"warmup_steps": 10000, "decay_type": "cosine"}, "mixup": {"p": 0.2, "fold_in": null}, "evals": {"val": {"type": "classification", "data": {"name": "imagenet2012", "split": "validation"}, "pp_fn": "decode|resize_small(256)|central_crop(224)|value_range(-1, 1)|onehot(1000, key=\"label\", key_result=\"labels\")|keep(\"image\", \"labels\")", "loss_name": "softmax_xent", "log_steps": 2500}}}

grafted/wandb/run-20241127_213238-torch-grafted-redux/files/config.yaml

@@ -0,0 +1,95 @@
+_allow_dotted_keys:
+    value: false
+_convert_dict:
+    value: true
+_fields:
+    value:
+        ckpt_steps: 1000
+        evals: |
+            val:
+              data:
+                name: imagenet2012
+                split: validation
+              log_steps: 2500
+              loss_name: softmax_xent
+              pp_fn: decode|resize_small(256)|central_crop(224)|value_range(-1, 1)|onehot(1000,
+                key="label", key_result="labels")|keep("image", "labels")
+              type: classification
+        grad_clip_norm: 1
+        input: |
+            accum_freq: 8
+            batch_size: 1024
+            cache_raw: false
+            data:
+              name: imagenet2012
+              split: train
+            pp: decode_jpeg_and_inception_crop(224)|flip_lr|randaug(2,10)|value_range(-1, 1)|onehot(1000,
+              key="label", key_result="labels")|keep("image", "labels")
+            shuffle_buffer_size: 150000
+        log_training_steps: 50
+        loss: softmax_xent
+        lr: 0.001
+        mixup: |
+            fold_in: null
+            p: 0.2
+        model: |
+            pool_type: gap
+            posemb: sincos2d
+            rep_size: false
+            variant: S/16
+        model_name: vit
+        num_classes: 1000
+        optax: |
+            mu_dtype: bfloat16
+        optax_name: scale_by_adam
+        pp_modules:
+            - ops_general
+            - ops_image
+            - ops_text
+            - archive.randaug
+        schedule: |
+            decay_type: cosine
+            warmup_steps: 10000
+        seed: 0
+        total_epochs: 90
+        wd: 0.0001
+_locked:
+    value: true
+_sort_keys:
+    value: true
+_type_safe:
+    value: true
+_wandb:
+    value:
+        cli_version: 0.18.7
+        m: []
+        python_version: 3.11.10
+        t:
+            "1":
+                - 1
+                - 2
+                - 3
+                - 12
+                - 41
+                - 45
+                - 55
+            "2":
+                - 1
+                - 2
+                - 3
+                - 12
+                - 41
+                - 45
+                - 55
+            "3":
+                - 13
+                - 14
+                - 16
+                - 23
+                - 55
+            "4": 3.11.10
+            "5": 0.18.7
+            "8":
+                - 5
+            "12": 0.18.7
+            "13": linux-x86_64

grafted/wandb/run-20241127_213238-torch-grafted-redux/files/output.log

@@ -0,0 +1,31 @@
+I1127 21:32:38.985260 139930170870656 train.py:125] NOTE: Initializing train dataset...
+I1127 21:32:38.985419 139930170870656 train.py:125] NOTE: Global batch size 1024 on 1 hosts results in 1024 local batch size. With 1 dev per host (1 dev total), that's a 1024 per-device batch size.
+I1127 21:32:39.139223 139930170870656 logging_logger.py:49] Constructing tf.data.Dataset imagenet2012 for split _EvenSplit(split='train', index=0, count=1, drop_remainder=False), from /home/jason-chou/tensorflow_datasets/imagenet2012/5.1.0
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/home/jason-chou/Downloads/big_vision/big_vision/train.py", line 396, in <module>
+    app.run(main)
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/absl/app.py", line 308, in run
+    _run_main(main, args)
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/absl/app.py", line 254, in _run_main
+    sys.exit(main(argv))
+             ^^^^^^^^^^
+  File "/home/jason-chou/Downloads/big_vision/big_vision/train.py", line 189, in main
+    train_ds, ntrain_img = input_pipeline.training(config.input)
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/jason-chou/Downloads/big_vision/big_vision/input_pipeline.py", line 101, in training
+    data=train_data.get_tfdata(ordered=False),
+         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/jason-chou/Downloads/big_vision/big_vision/datasets/tfds.py", line 39, in get_tfdata
+    return _get_dataset_from_builder(
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/jason-chou/Downloads/big_vision/big_vision/datasets/tfds.py", line 70, in _get_dataset_from_builder
+    ds = builder.as_dataset(
+         ^^^^^^^^^^^^^^^^^^^
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/tensorflow_datasets/core/logging/__init__.py", line 176, in __call__
+    return function(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/tensorflow_datasets/core/dataset_builder.py", line 1008, in as_dataset
+    raise AssertionError(
+AssertionError: Dataset imagenet2012: could not find data in /home/jason-chou/tensorflow_datasets. Please make sure to call dataset_builder.download_and_prepare(), or pass download=True to tfds.load() before trying to access the tf.data.Dataset object.

grafted/wandb/run-20241127_213238-torch-grafted-redux/files/requirements.txt

@@ -0,0 +1,133 @@
+array_record==0.5.1
+wandb==0.18.7
+nvidia-curand-cu12==10.3.2.106
+requests-oauthlib==2.0.0
+zipp==3.21.0
+Werkzeug==3.1.3
+simple-parsing==0.1.6
+mdurl==0.1.2
+keras==2.15.0
+nvidia-cuda-nvcc-cu12==12.6.85
+google-auth-oauthlib==1.2.1
+jaxlib==0.4.34
+tf_keras==2.15.1
+oauthlib==3.2.2
+tensorflow-probability==0.25.0
+cachetools==5.5.0
+Jinja2==3.1.3
+rich==13.9.4
+filelock==3.13.1
+google-pasta==0.2.0
+optax==0.2.4
+toolz==1.0.0
+gast==0.6.0
+tensorboard==2.15.2
+pyasn1_modules==0.4.1
+nvidia-cudnn-cu12==9.1.0.70
+opt_einsum==3.4.0
+nvidia-nvjitlink-cu12==12.6.85
+chex==0.1.87
+namex==0.0.8
+termcolor==2.5.0
+flax==0.10.2
+cloudpickle==3.1.0
+numpy==1.26.4
+nvidia-nccl-cu12==2.21.5
+tensorflow-cpu==2.15.0
+nvidia-cusolver-cu12==11.4.5.107
+typing_extensions==4.12.2
+tensorflow-addons==0.23.0
+typeguard==2.13.3
+absl-py==2.1.0
+flatbuffers==24.3.25
+dlpack==0.1
+setuptools==65.5.0
+protobuf==4.25.5
+jax-cuda12-plugin==0.4.35
+tensorflow==2.15.0
+msgpack==1.1.0
+networkx==3.2.1
+docker-pycreds==0.4.0
+nvidia-cuda-cupti-cu12==12.1.105
+pillow==11.0.0
+libclang==18.1.1
+nvidia-cuda-nvrtc-cu12==12.1.105
+distrax==0.1.5
+orbax-checkpoint==0.10.1
+PyYAML==6.0.2
+urllib3==2.2.3
+aqtp==0.8.2
+tensorflow-metadata==1.16.1
+etils==1.11.0
+smmap==5.0.1
+pyasn1==0.6.1
+docstring_parser==0.16
+google-auth==2.36.0
+simplejson==3.19.3
+mpmath==1.3.0
+h5py==3.12.1
+jax-cuda12-pjrt==0.4.35
+tensorflow-io-gcs-filesystem==0.37.1
+tensorflow-estimator==2.15.0
+triton==3.1.0
+rsa==4.9
+panopticapi==0.1
+tensorflow-hub==0.16.1
+requests==2.32.3
+scipy==1.14.1
+ml-dtypes==0.2.0
+markdown-it-py==3.0.0
+tensorflow-text==2.15.0
+wrapt==1.14.1
+immutabledict==4.2.1
+MarkupSafe==3.0.2
+jax==0.4.35
+torch==2.5.1+cu121
+wheel==0.45.1
+einops==0.8.0
+sentry-sdk==2.19.0
+torchvision==0.20.1+cu121
+humanize==4.11.0
+toml==0.10.2
+tensorstore==0.1.69
+six==1.16.0
+promise==2.3
+certifi==2024.8.30
+nvidia-cuda-runtime-cu12==12.1.105
+flaxformer==0.8.8
+nvidia-cufft-cu12==11.0.2.54
+psutil==6.1.0
+GitPython==3.1.43
+platformdirs==4.3.6
+importlib_resources==6.4.5
+tfds-nightly==4.9.7.dev202411280044
+tensorflow-gan==2.1.0
+googleapis-common-protos==1.66.0
+overrides==7.7.0
+optree==0.13.1
+Pygments==2.18.0
+astunparse==1.6.3
+ml_collections==1.0.0
+setproctitle==1.3.4
+tensorboard-data-server==0.7.2
+sympy==1.13.1
+packaging==24.2
+nest-asyncio==1.6.0
+nvidia-cublas-cu12==12.1.3.1
+gitdb==4.0.11
+click==8.1.7
+idna==3.10
+tqdm==4.67.1
+grpcio==1.68.0
+decorator==5.1.1
+pyarrow==18.1.0
+clu==0.0.12
+charset-normalizer==3.4.0
+fsspec==2024.10.0
+dm-tree==0.1.8
+sentencepiece==0.2.0
+nvidia-cusparse-cu12==12.1.0.106
+torchaudio==2.5.1+cu121
+pip==24.3.1
+Markdown==3.7
+nvidia-nvtx-cu12==12.1.105

grafted/wandb/run-20241127_213238-torch-grafted-redux/files/wandb-metadata.json

@@ -0,0 +1,49 @@
+{
+  "os": "Linux-6.8.0-49-generic-x86_64-with-glibc2.39",
+  "python": "3.11.10",
+  "startedAt": "2024-11-28T05:32:38.332723Z",
+  "args": [
+    "--config",
+    "/home/jason-chou/Downloads/big_vision/big_vision/configs/vit_s16_i1k_single_gpu_test.py",
+    "--workdir",
+    "/data/imagenet/grafted",
+    "--name",
+    "torch-grafted-redux"
+  ],
+  "program": "-m big_vision.train",
+  "git": {
+    "remote": "https://github.com/EIFY/big_vision.git",
+    "commit": "44649a64ff67e709f55cdb5e3adcf52064b17de5"
+  },
+  "email": "[email protected]",
+  "root": "/home/jason-chou/Downloads/big_vision",
+  "host": "jasonchou-TensorBook-late-2021",
+  "username": "jason-chou",
+  "executable": "/home/jason-chou/.pyenv/versions/3.11.10/bin/python",
+  "cpu_count": 8,
+  "cpu_count_logical": 16,
+  "gpu": "NVIDIA GeForce RTX 3080 Laptop GPU",
+  "gpu_count": 1,
+  "disk": {
+    "/": {
+      "total": "1006450962432",
+      "used": "584312053760"
+    }
+  },
+  "memory": {
+    "total": "67162914816"
+  },
+  "cpu": {
+    "count": 8,
+    "countLogical": 16
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA GeForce RTX 3080 Laptop GPU",
+      "memoryTotal": "17179869184",
+      "cudaCores": 6144,
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.2"
+}

grafted/wandb/run-20241127_213238-torch-grafted-redux/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":0}}

grafted/wandb/run-20241127_213238-torch-grafted-redux/logs/debug-internal.log

@@ -0,0 +1,18 @@
+{"time":"2024-11-27T21:32:38.333839182-08:00","level":"INFO","msg":"using version","core version":"0.18.7"}
+{"time":"2024-11-27T21:32:38.333850342-08:00","level":"INFO","msg":"created symlink","path":"/home/jason-chou/Downloads/big_vision/wandb/run-20241127_213238-torch-grafted-redux/logs/debug-core.log"}
+{"time":"2024-11-27T21:32:38.437006136-08:00","level":"INFO","msg":"created new stream","id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:32:38.437103435-08:00","level":"INFO","msg":"stream: started","id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:32:38.437187603-08:00","level":"INFO","msg":"writer: Do: started","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:32:38.437273441-08:00","level":"INFO","msg":"handler: started","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:32:38.437384909-08:00","level":"INFO","msg":"sender: started","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:32:38.935119097-08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-27T21:32:39.180591766-08:00","level":"INFO","msg":"stream: closing","id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:32:39.180690557-08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-27T21:32:39.181672971-08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-27T21:32:39.361912852-08:00","level":"WARN","msg":"No program path found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job"}
+{"time":"2024-11-27T21:32:39.361944716-08:00","level":"INFO","msg":"sender: sendDefer: no job artifact to save"}
+{"time":"2024-11-27T21:32:39.739296662-08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-27T21:32:39.941553056-08:00","level":"INFO","msg":"handler: closed","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:32:39.941619024-08:00","level":"INFO","msg":"sender: closed","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:32:39.941589779-08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:32:39.941798781-08:00","level":"INFO","msg":"stream: closed","id":"torch-grafted-redux"}

grafted/wandb/run-20241127_213238-torch-grafted-redux/logs/debug.log

@@ -0,0 +1,54 @@
+2024-11-27 21:32:38,330 INFO    MainThread:68601 [wandb_setup.py:_flush():79] Current SDK version is 0.18.7
+2024-11-27 21:32:38,330 INFO    MainThread:68601 [wandb_setup.py:_flush():79] Configure stats pid to 68601
+2024-11-27 21:32:38,330 INFO    MainThread:68601 [wandb_setup.py:_flush():79] Loading settings from /home/jason-chou/.config/wandb/settings
+2024-11-27 21:32:38,330 INFO    MainThread:68601 [wandb_setup.py:_flush():79] Loading settings from /home/jason-chou/Downloads/big_vision/wandb/settings
+2024-11-27 21:32:38,330 INFO    MainThread:68601 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-27 21:32:38,330 INFO    MainThread:68601 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-27 21:32:38,330 WARNING MainThread:68601 [wandb_setup.py:_flush():79] Could not find program at -m big_vision.train
+2024-11-27 21:32:38,330 INFO    MainThread:68601 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m big_vision.train'}
+2024-11-27 21:32:38,330 INFO    MainThread:68601 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-27 21:32:38,330 INFO    MainThread:68601 [wandb_init.py:_log_setup():533] Logging user logs to /home/jason-chou/Downloads/big_vision/wandb/run-20241127_213238-torch-grafted-redux/logs/debug.log
+2024-11-27 21:32:38,330 INFO    MainThread:68601 [wandb_init.py:_log_setup():534] Logging internal logs to /home/jason-chou/Downloads/big_vision/wandb/run-20241127_213238-torch-grafted-redux/logs/debug-internal.log
+2024-11-27 21:32:38,330 INFO    MainThread:68601 [wandb_init.py:init():619] calling init triggers
+2024-11-27 21:32:38,331 INFO    MainThread:68601 [wandb_init.py:init():626] wandb.init called with sweep_config: {}
+config: {'_fields': {'seed': 0, 'total_epochs': 90, 'num_classes': 1000, 'loss': 'softmax_xent', 'input': accum_freq: 8
+batch_size: 1024
+cache_raw: false
+data:
+  name: imagenet2012
+  split: train
+pp: decode_jpeg_and_inception_crop(224)|flip_lr|randaug(2,10)|value_range(-1, 1)|onehot(1000,
+  key="label", key_result="labels")|keep("image", "labels")
+shuffle_buffer_size: 150000
+, 'pp_modules': ['ops_general', 'ops_image', 'ops_text', 'archive.randaug'], 'log_training_steps': 50, 'ckpt_steps': 1000, 'model_name': 'vit', 'model': pool_type: gap
+posemb: sincos2d
+rep_size: false
+variant: S/16
+, 'grad_clip_norm': 1.0, 'optax_name': 'scale_by_adam', 'optax': mu_dtype: bfloat16
+, 'lr': 0.001, 'wd': 0.0001, 'schedule': decay_type: cosine
+warmup_steps: 10000
+, 'mixup': fold_in: null
+p: 0.2
+, 'evals': val:
+  data:
+    name: imagenet2012
+    split: validation
+  log_steps: 2500
+  loss_name: softmax_xent
+  pp_fn: decode|resize_small(256)|central_crop(224)|value_range(-1, 1)|onehot(1000,
+    key="label", key_result="labels")|keep("image", "labels")
+  type: classification
+}, '_locked': True, '_type_safe': True, '_convert_dict': True, '_allow_dotted_keys': False, '_sort_keys': True}
+2024-11-27 21:32:38,331 INFO    MainThread:68601 [wandb_init.py:init():669] starting backend
+2024-11-27 21:32:38,331 INFO    MainThread:68601 [wandb_init.py:init():673] sending inform_init request
+2024-11-27 21:32:38,332 INFO    MainThread:68601 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-27 21:32:38,332 INFO    MainThread:68601 [wandb_init.py:init():686] backend started and connected
+2024-11-27 21:32:38,336 INFO    MainThread:68601 [wandb_init.py:init():781] updated telemetry
+2024-11-27 21:32:38,339 INFO    MainThread:68601 [wandb_init.py:init():814] communicating run to backend with 90.0 second timeout
+2024-11-27 21:32:38,933 INFO    MainThread:68601 [wandb_init.py:init():867] starting run threads in backend
+2024-11-27 21:32:38,983 INFO    MainThread:68601 [wandb_run.py:_console_start():2456] atexit reg
+2024-11-27 21:32:38,983 INFO    MainThread:68601 [wandb_run.py:_redirect():2305] redirect: wrap_raw
+2024-11-27 21:32:38,984 INFO    MainThread:68601 [wandb_run.py:_redirect():2370] Wrapping output streams.
+2024-11-27 21:32:38,984 INFO    MainThread:68601 [wandb_run.py:_redirect():2395] Redirects installed.
+2024-11-27 21:32:38,984 INFO    MainThread:68601 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-27 21:32:39,180 WARNING MsgRouterThr:68601 [router.py:message_loop():75] message_loop has been closed

grafted/wandb/run-20241127_213407-torch-grafted-redux/files/config.yaml

@@ -0,0 +1,97 @@
+_allow_dotted_keys:
+    value: false
+_convert_dict:
+    value: true
+_fields:
+    value:
+        ckpt_steps: 1000
+        evals: |
+            val:
+              data:
+                name: imagenet2012
+                split: validation
+              log_steps: 2500
+              loss_name: softmax_xent
+              pp_fn: decode|resize_small(256)|central_crop(224)|value_range(-1, 1)|onehot(1000,
+                key="label", key_result="labels")|keep("image", "labels")
+              type: classification
+        grad_clip_norm: 1
+        input: |
+            accum_freq: 8
+            batch_size: 1024
+            cache_raw: false
+            data:
+              name: imagenet2012
+              split: train
+            pp: decode_jpeg_and_inception_crop(224)|flip_lr|randaug(2,10)|value_range(-1, 1)|onehot(1000,
+              key="label", key_result="labels")|keep("image", "labels")
+            shuffle_buffer_size: 150000
+        log_training_steps: 50
+        loss: softmax_xent
+        lr: 0.001
+        mixup: |
+            fold_in: null
+            p: 0.2
+        model: |
+            pool_type: gap
+            posemb: sincos2d
+            rep_size: false
+            variant: S/16
+        model_name: vit
+        num_classes: 1000
+        optax: |
+            mu_dtype: bfloat16
+        optax_name: scale_by_adam
+        pp_modules:
+            - ops_general
+            - ops_image
+            - ops_text
+            - archive.randaug
+        schedule: |
+            decay_type: cosine
+            warmup_steps: 10000
+        seed: 0
+        total_epochs: 90
+        wd: 0.0001
+_locked:
+    value: true
+_sort_keys:
+    value: true
+_type_safe:
+    value: true
+_wandb:
+    value:
+        cli_version: 0.18.7
+        m: []
+        python_version: 3.11.10
+        t:
+            "1":
+                - 1
+                - 2
+                - 3
+                - 12
+                - 41
+                - 45
+                - 55
+            "2":
+                - 1
+                - 2
+                - 3
+                - 12
+                - 41
+                - 45
+                - 55
+            "3":
+                - 5
+                - 13
+                - 14
+                - 16
+                - 23
+                - 55
+                - 62
+            "4": 3.11.10
+            "5": 0.18.7
+            "8":
+                - 5
+            "12": 0.18.7
+            "13": linux-x86_64

grafted/wandb/run-20241127_213407-torch-grafted-redux/files/output.log

@@ -0,0 +1,31 @@
+I1127 21:34:08.317430 123604392237952 train.py:125] NOTE: Initializing train dataset...
+I1127 21:34:08.317584 123604392237952 train.py:125] NOTE: Global batch size 1024 on 1 hosts results in 1024 local batch size. With 1 dev per host (1 dev total), that's a 1024 per-device batch size.
+I1127 21:34:08.468933 123604392237952 logging_logger.py:49] Constructing tf.data.Dataset imagenet2012 for split _EvenSplit(split='train', index=0, count=1, drop_remainder=False), from /home/jason-chou/tensorflow_datasets/imagenet2012/5.1.0
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/home/jason-chou/Downloads/big_vision/big_vision/train.py", line 396, in <module>
+    app.run(main)
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/absl/app.py", line 308, in run
+    _run_main(main, args)
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/absl/app.py", line 254, in _run_main
+    sys.exit(main(argv))
+             ^^^^^^^^^^
+  File "/home/jason-chou/Downloads/big_vision/big_vision/train.py", line 189, in main
+    train_ds, ntrain_img = input_pipeline.training(config.input)
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/jason-chou/Downloads/big_vision/big_vision/input_pipeline.py", line 101, in training
+    data=train_data.get_tfdata(ordered=False),
+         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/jason-chou/Downloads/big_vision/big_vision/datasets/tfds.py", line 39, in get_tfdata
+    return _get_dataset_from_builder(
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/jason-chou/Downloads/big_vision/big_vision/datasets/tfds.py", line 70, in _get_dataset_from_builder
+    ds = builder.as_dataset(
+         ^^^^^^^^^^^^^^^^^^^
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/tensorflow_datasets/core/logging/__init__.py", line 176, in __call__
+    return function(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/tensorflow_datasets/core/dataset_builder.py", line 1008, in as_dataset
+    raise AssertionError(
+AssertionError: Dataset imagenet2012: could not find data in /home/jason-chou/tensorflow_datasets. Please make sure to call dataset_builder.download_and_prepare(), or pass download=True to tfds.load() before trying to access the tf.data.Dataset object.

grafted/wandb/run-20241127_213407-torch-grafted-redux/files/requirements.txt

@@ -0,0 +1,133 @@
+array_record==0.5.1
+wandb==0.18.7
+nvidia-curand-cu12==10.3.2.106
+requests-oauthlib==2.0.0
+zipp==3.21.0
+Werkzeug==3.1.3
+simple-parsing==0.1.6
+mdurl==0.1.2
+keras==2.15.0
+nvidia-cuda-nvcc-cu12==12.6.85
+google-auth-oauthlib==1.2.1
+jaxlib==0.4.34
+tf_keras==2.15.1
+oauthlib==3.2.2
+tensorflow-probability==0.25.0
+cachetools==5.5.0
+Jinja2==3.1.3
+rich==13.9.4
+filelock==3.13.1
+google-pasta==0.2.0
+optax==0.2.4
+toolz==1.0.0
+gast==0.6.0
+tensorboard==2.15.2
+pyasn1_modules==0.4.1
+nvidia-cudnn-cu12==9.1.0.70
+opt_einsum==3.4.0
+nvidia-nvjitlink-cu12==12.6.85
+chex==0.1.87
+namex==0.0.8
+termcolor==2.5.0
+flax==0.10.2
+cloudpickle==3.1.0
+numpy==1.26.4
+nvidia-nccl-cu12==2.21.5
+tensorflow-cpu==2.15.0
+nvidia-cusolver-cu12==11.4.5.107
+typing_extensions==4.12.2
+tensorflow-addons==0.23.0
+typeguard==2.13.3
+absl-py==2.1.0
+flatbuffers==24.3.25
+dlpack==0.1
+setuptools==65.5.0
+protobuf==4.25.5
+jax-cuda12-plugin==0.4.35
+tensorflow==2.15.0
+msgpack==1.1.0
+networkx==3.2.1
+docker-pycreds==0.4.0
+nvidia-cuda-cupti-cu12==12.1.105
+pillow==11.0.0
+libclang==18.1.1
+nvidia-cuda-nvrtc-cu12==12.1.105
+distrax==0.1.5
+orbax-checkpoint==0.10.1
+PyYAML==6.0.2
+urllib3==2.2.3
+aqtp==0.8.2
+tensorflow-metadata==1.16.1
+etils==1.11.0
+smmap==5.0.1
+pyasn1==0.6.1
+docstring_parser==0.16
+google-auth==2.36.0
+simplejson==3.19.3
+mpmath==1.3.0
+h5py==3.12.1
+jax-cuda12-pjrt==0.4.35
+tensorflow-io-gcs-filesystem==0.37.1
+tensorflow-estimator==2.15.0
+triton==3.1.0
+rsa==4.9
+panopticapi==0.1
+tensorflow-hub==0.16.1
+requests==2.32.3
+scipy==1.14.1
+ml-dtypes==0.2.0
+markdown-it-py==3.0.0
+tensorflow-text==2.15.0
+wrapt==1.14.1
+immutabledict==4.2.1
+MarkupSafe==3.0.2
+jax==0.4.35
+torch==2.5.1+cu121
+wheel==0.45.1
+einops==0.8.0
+sentry-sdk==2.19.0
+torchvision==0.20.1+cu121
+humanize==4.11.0
+toml==0.10.2
+tensorstore==0.1.69
+six==1.16.0
+promise==2.3
+certifi==2024.8.30
+nvidia-cuda-runtime-cu12==12.1.105
+flaxformer==0.8.8
+nvidia-cufft-cu12==11.0.2.54
+psutil==6.1.0
+GitPython==3.1.43
+platformdirs==4.3.6
+importlib_resources==6.4.5
+tfds-nightly==4.9.7.dev202411280044
+tensorflow-gan==2.1.0
+googleapis-common-protos==1.66.0
+overrides==7.7.0
+optree==0.13.1
+Pygments==2.18.0
+astunparse==1.6.3
+ml_collections==1.0.0
+setproctitle==1.3.4
+tensorboard-data-server==0.7.2
+sympy==1.13.1
+packaging==24.2
+nest-asyncio==1.6.0
+nvidia-cublas-cu12==12.1.3.1
+gitdb==4.0.11
+click==8.1.7
+idna==3.10
+tqdm==4.67.1
+grpcio==1.68.0
+decorator==5.1.1
+pyarrow==18.1.0
+clu==0.0.12
+charset-normalizer==3.4.0
+fsspec==2024.10.0
+dm-tree==0.1.8
+sentencepiece==0.2.0
+nvidia-cusparse-cu12==12.1.0.106
+torchaudio==2.5.1+cu121
+pip==24.3.1
+Markdown==3.7
+nvidia-nvtx-cu12==12.1.105

grafted/wandb/run-20241127_213407-torch-grafted-redux/files/wandb-metadata.json

@@ -0,0 +1,49 @@
+{
+  "os": "Linux-6.8.0-49-generic-x86_64-with-glibc2.39",
+  "python": "3.11.10",
+  "startedAt": "2024-11-28T05:34:07.870557Z",
+  "args": [
+    "--config",
+    "/home/jason-chou/Downloads/big_vision/big_vision/configs/vit_s16_i1k_single_gpu_test.py",
+    "--workdir",
+    "/data/imagenet/grafted",
+    "--name",
+    "torch-grafted-redux"
+  ],
+  "program": "-m big_vision.train",
+  "git": {
+    "remote": "https://github.com/EIFY/big_vision.git",
+    "commit": "44649a64ff67e709f55cdb5e3adcf52064b17de5"
+  },
+  "email": "[email protected]",
+  "root": "/home/jason-chou/Downloads/big_vision",
+  "host": "jasonchou-TensorBook-late-2021",
+  "username": "jason-chou",
+  "executable": "/home/jason-chou/.pyenv/versions/3.11.10/bin/python",
+  "cpu_count": 8,
+  "cpu_count_logical": 16,
+  "gpu": "NVIDIA GeForce RTX 3080 Laptop GPU",
+  "gpu_count": 1,
+  "disk": {
+    "/": {
+      "total": "1006450962432",
+      "used": "584312168448"
+    }
+  },
+  "memory": {
+    "total": "67162914816"
+  },
+  "cpu": {
+    "count": 8,
+    "countLogical": 16
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA GeForce RTX 3080 Laptop GPU",
+      "memoryTotal": "17179869184",
+      "cudaCores": 6144,
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.2"
+}

grafted/wandb/run-20241127_213407-torch-grafted-redux/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":0}}

grafted/wandb/run-20241127_213407-torch-grafted-redux/logs/debug-internal.log

@@ -0,0 +1,18 @@
+{"time":"2024-11-27T21:34:07.871305008-08:00","level":"INFO","msg":"using version","core version":"0.18.7"}
+{"time":"2024-11-27T21:34:07.871312795-08:00","level":"INFO","msg":"created symlink","path":"/home/jason-chou/Downloads/big_vision/wandb/run-20241127_213407-torch-grafted-redux/logs/debug-core.log"}
+{"time":"2024-11-27T21:34:07.974433557-08:00","level":"INFO","msg":"created new stream","id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:34:07.974483345-08:00","level":"INFO","msg":"stream: started","id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:34:07.974637769-08:00","level":"INFO","msg":"writer: Do: started","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:34:07.974694546-08:00","level":"INFO","msg":"handler: started","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:34:07.974824798-08:00","level":"INFO","msg":"sender: started","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:34:08.265740433-08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-27T21:34:08.510645063-08:00","level":"INFO","msg":"stream: closing","id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:34:08.510725098-08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-27T21:34:08.511601534-08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-27T21:34:08.671262357-08:00","level":"WARN","msg":"No program path found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job"}
+{"time":"2024-11-27T21:34:08.671295778-08:00","level":"INFO","msg":"sender: sendDefer: no job artifact to save"}
+{"time":"2024-11-27T21:34:09.08563661-08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-27T21:34:09.270472343-08:00","level":"INFO","msg":"handler: closed","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:34:09.270565975-08:00","level":"INFO","msg":"sender: closed","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:34:09.270568744-08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:34:09.270733853-08:00","level":"INFO","msg":"stream: closed","id":"torch-grafted-redux"}

grafted/wandb/run-20241127_213407-torch-grafted-redux/logs/debug.log

@@ -0,0 +1,55 @@
+2024-11-27 21:34:07,867 INFO    MainThread:68916 [wandb_setup.py:_flush():79] Current SDK version is 0.18.7
+2024-11-27 21:34:07,867 INFO    MainThread:68916 [wandb_setup.py:_flush():79] Configure stats pid to 68916
+2024-11-27 21:34:07,867 INFO    MainThread:68916 [wandb_setup.py:_flush():79] Loading settings from /home/jason-chou/.config/wandb/settings
+2024-11-27 21:34:07,867 INFO    MainThread:68916 [wandb_setup.py:_flush():79] Loading settings from /home/jason-chou/Downloads/big_vision/wandb/settings
+2024-11-27 21:34:07,867 INFO    MainThread:68916 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-27 21:34:07,867 INFO    MainThread:68916 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-27 21:34:07,867 WARNING MainThread:68916 [wandb_setup.py:_flush():79] Could not find program at -m big_vision.train
+2024-11-27 21:34:07,867 INFO    MainThread:68916 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m big_vision.train'}
+2024-11-27 21:34:07,867 INFO    MainThread:68916 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-27 21:34:07,867 INFO    MainThread:68916 [wandb_init.py:_log_setup():533] Logging user logs to /home/jason-chou/Downloads/big_vision/wandb/run-20241127_213407-torch-grafted-redux/logs/debug.log
+2024-11-27 21:34:07,867 INFO    MainThread:68916 [wandb_init.py:_log_setup():534] Logging internal logs to /home/jason-chou/Downloads/big_vision/wandb/run-20241127_213407-torch-grafted-redux/logs/debug-internal.log
+2024-11-27 21:34:07,867 INFO    MainThread:68916 [wandb_init.py:init():619] calling init triggers
+2024-11-27 21:34:07,869 INFO    MainThread:68916 [wandb_init.py:init():626] wandb.init called with sweep_config: {}
+config: {'_fields': {'seed': 0, 'total_epochs': 90, 'num_classes': 1000, 'loss': 'softmax_xent', 'input': accum_freq: 8
+batch_size: 1024
+cache_raw: false
+data:
+  name: imagenet2012
+  split: train
+pp: decode_jpeg_and_inception_crop(224)|flip_lr|randaug(2,10)|value_range(-1, 1)|onehot(1000,
+  key="label", key_result="labels")|keep("image", "labels")
+shuffle_buffer_size: 150000
+, 'pp_modules': ['ops_general', 'ops_image', 'ops_text', 'archive.randaug'], 'log_training_steps': 50, 'ckpt_steps': 1000, 'model_name': 'vit', 'model': pool_type: gap
+posemb: sincos2d
+rep_size: false
+variant: S/16
+, 'grad_clip_norm': 1.0, 'optax_name': 'scale_by_adam', 'optax': mu_dtype: bfloat16
+, 'lr': 0.001, 'wd': 0.0001, 'schedule': decay_type: cosine
+warmup_steps: 10000
+, 'mixup': fold_in: null
+p: 0.2
+, 'evals': val:
+  data:
+    name: imagenet2012
+    split: validation
+  log_steps: 2500
+  loss_name: softmax_xent
+  pp_fn: decode|resize_small(256)|central_crop(224)|value_range(-1, 1)|onehot(1000,
+    key="label", key_result="labels")|keep("image", "labels")
+  type: classification
+}, '_locked': True, '_type_safe': True, '_convert_dict': True, '_allow_dotted_keys': False, '_sort_keys': True}
+2024-11-27 21:34:07,869 INFO    MainThread:68916 [wandb_init.py:init():669] starting backend
+2024-11-27 21:34:07,869 INFO    MainThread:68916 [wandb_init.py:init():673] sending inform_init request
+2024-11-27 21:34:07,870 INFO    MainThread:68916 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-27 21:34:07,870 INFO    MainThread:68916 [wandb_init.py:init():686] backend started and connected
+2024-11-27 21:34:07,873 INFO    MainThread:68916 [wandb_init.py:init():781] updated telemetry
+2024-11-27 21:34:07,875 INFO    MainThread:68916 [wandb_init.py:init():814] communicating run to backend with 90.0 second timeout
+2024-11-27 21:34:08,241 INFO    MainThread:68916 [wandb_init.py:init():859] run resumed
+2024-11-27 21:34:08,261 INFO    MainThread:68916 [wandb_init.py:init():867] starting run threads in backend
+2024-11-27 21:34:08,315 INFO    MainThread:68916 [wandb_run.py:_console_start():2456] atexit reg
+2024-11-27 21:34:08,315 INFO    MainThread:68916 [wandb_run.py:_redirect():2305] redirect: wrap_raw
+2024-11-27 21:34:08,316 INFO    MainThread:68916 [wandb_run.py:_redirect():2370] Wrapping output streams.
+2024-11-27 21:34:08,316 INFO    MainThread:68916 [wandb_run.py:_redirect():2395] Redirects installed.
+2024-11-27 21:34:08,317 INFO    MainThread:68916 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-27 21:34:08,511 WARNING MsgRouterThr:68916 [router.py:message_loop():75] message_loop has been closed

grafted/wandb/run-20241127_213838-torch-grafted-redux/files/config.yaml

@@ -0,0 +1,97 @@
+_allow_dotted_keys:
+    value: false
+_convert_dict:
+    value: true
+_fields:
+    value:
+        ckpt_steps: 1000
+        evals: |
+            val:
+              data:
+                name: imagenet2012
+                split: validation
+              log_steps: 2500
+              loss_name: softmax_xent
+              pp_fn: decode|resize_small(256)|central_crop(224)|value_range(-1, 1)|onehot(1000,
+                key="label", key_result="labels")|keep("image", "labels")
+              type: classification
+        grad_clip_norm: 1
+        input: |
+            accum_freq: 8
+            batch_size: 1024
+            cache_raw: false
+            data:
+              name: imagenet2012
+              split: train
+            pp: decode_jpeg_and_inception_crop(224)|flip_lr|randaug(2,10)|value_range(-1, 1)|onehot(1000,
+              key="label", key_result="labels")|keep("image", "labels")
+            shuffle_buffer_size: 150000
+        log_training_steps: 50
+        loss: softmax_xent
+        lr: 0.001
+        mixup: |
+            fold_in: null
+            p: 0.2
+        model: |
+            pool_type: gap
+            posemb: sincos2d
+            rep_size: false
+            variant: S/16
+        model_name: vit
+        num_classes: 1000
+        optax: |
+            mu_dtype: bfloat16
+        optax_name: scale_by_adam
+        pp_modules:
+            - ops_general
+            - ops_image
+            - ops_text
+            - archive.randaug
+        schedule: |
+            decay_type: cosine
+            warmup_steps: 10000
+        seed: 0
+        total_epochs: 90
+        wd: 0.0001
+_locked:
+    value: true
+_sort_keys:
+    value: true
+_type_safe:
+    value: true
+_wandb:
+    value:
+        cli_version: 0.18.7
+        m: []
+        python_version: 3.11.10
+        t:
+            "1":
+                - 1
+                - 2
+                - 3
+                - 12
+                - 41
+                - 45
+                - 55
+            "2":
+                - 1
+                - 2
+                - 3
+                - 12
+                - 41
+                - 45
+                - 55
+            "3":
+                - 5
+                - 13
+                - 14
+                - 16
+                - 23
+                - 55
+                - 62
+            "4": 3.11.10
+            "5": 0.18.7
+            "8":
+                - 5
+            "12": 0.18.7
+            "13": linux-x86_64

grafted/wandb/run-20241127_213838-torch-grafted-redux/files/output.log

@@ -0,0 +1,152 @@
+I1127 21:38:39.183649 134109837835136 train.py:125] NOTE: Initializing train dataset...
+I1127 21:38:39.183780 134109837835136 train.py:125] NOTE: Global batch size 1024 on 1 hosts results in 1024 local batch size. With 1 dev per host (1 dev total), that's a 1024 per-device batch size.
+I1127 21:38:39.334601 134109837835136 dataset_info.py:707] Load dataset info from /data/tensorflow_datasets/imagenet2012/5.1.0
+I1127 21:38:39.359155 134109837835136 reader.py:261] Creating a tf.data.Dataset reading 1024 files located in folders: /data/tensorflow_datasets/imagenet2012/5.1.0.
+WARNING:tensorflow:From /home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/tensorflow_datasets/core/reader.py:101: CounterV2 (from tensorflow.python.data.experimental.ops.counter) is deprecated and will be removed in a future version.
+Instructions for updating:
+Use `tf.data.Dataset.counter(...)` instead.
+W1127 21:38:39.392815 134109837835136 deprecation.py:50] From /home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/tensorflow_datasets/core/reader.py:101: CounterV2 (from tensorflow.python.data.experimental.ops.counter) is deprecated and will be removed in a future version.
+Instructions for updating:
+Use `tf.data.Dataset.counter(...)` instead.
+I1127 21:38:39.415093 134109837835136 logging_logger.py:49] Constructing tf.data.Dataset imagenet2012 for split _EvenSplit(split='train', index=0, count=1, drop_remainder=False), from /data/tensorflow_datasets/imagenet2012/5.1.0
+I1127 21:38:39.481757 134109837835136 api.py:460] Data before pre-processing:
+{'file_name': <tf.Tensor 'args_1:0' shape=() dtype=string>, 'image': <tf.Tensor 'args_2:0' shape=() dtype=string>, 'label': <tf.Tensor 'args_3:0' shape=() dtype=int64>, 'tfds_id': <tf.Tensor 'args_4:0' shape=() dtype=string>, '_id': <tf.Tensor 'args_0:0' shape=() dtype=int32>}
+INFO:tensorflow:Using RandAug.
+I1127 21:38:39.744788 134109837835136 api.py:460] Using RandAug.
+WARNING:tensorflow:From /home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/tensorflow/python/util/dispatch.py:1260: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
+Instructions for updating:
+Use `tf.cast` instead.
+W1127 21:38:39.922924 134109837835136 deprecation.py:50] From /home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/tensorflow/python/util/dispatch.py:1260: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
+Instructions for updating:
+Use `tf.cast` instead.
+I1127 21:38:42.147573 134109837835136 api.py:460] Data after pre-processing:
+{'image': <tf.Tensor 'add:0' shape=(224, 224, 3) dtype=float32>, 'labels': <tf.Tensor 'one_hot:0' shape=(1000,) dtype=float32>}
+I1127 21:38:42.218924 134109837835136 train.py:125] NOTE: Running for 112603 steps, that means 90.000345 epochs
+I1127 21:38:42.850795 134109837835136 train.py:125] NOTE: Creating model...
+Weight decay for: conv_proj.weight
+Weight decay for: encoder.layers.encoder_layer_0.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_0.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_0.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_0.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_1.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_1.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_1.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_1.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_2.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_2.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_2.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_2.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_3.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_3.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_3.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_3.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_4.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_4.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_4.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_4.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_5.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_5.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_5.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_5.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_6.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_6.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_6.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_6.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_7.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_7.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_7.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_7.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_8.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_8.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_8.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_8.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_9.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_9.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_9.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_9.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_10.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_10.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_10.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_10.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_11.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_11.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_11.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_11.mlp.3.weight
+Weight decay for: heads.head.weight
+Weight decay for: conv_proj.weight
+Weight decay for: encoder.layers.encoder_layer_0.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_0.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_0.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_0.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_1.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_1.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_1.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_1.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_2.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_2.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_2.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_2.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_3.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_3.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_3.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_3.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_4.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_4.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_4.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_4.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_5.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_5.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_5.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_5.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_6.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_6.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_6.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_6.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_7.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_7.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_7.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_7.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_8.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_8.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_8.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_8.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_9.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_9.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_9.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_9.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_10.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_10.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_10.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_10.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_11.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_11.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_11.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_11.mlp.3.weight
+Weight decay for: heads.head.weight
+I1127 21:38:43.733023 134109837835136 train.py:125] NOTE: Running initial or final evals...
+I1127 21:38:43.733439 134109837835136 train.py:125] NOTE: Init evaluator: val…
+Steps:0/112603 [0.0%]
+I1127 21:38:43.735805 134109837835136 reader.py:261] Creating a tf.data.Dataset reading 64 files located in folders: /data/tensorflow_datasets/imagenet2012/5.1.0.
+I1127 21:38:43.766425 134109837835136 logging_logger.py:49] Constructing tf.data.Dataset imagenet2012 for split _EvenSplit(split='validation', index=0, count=1, drop_remainder=False), from /data/tensorflow_datasets/imagenet2012/5.1.0
+I1127 21:38:43.800241 134109837835136 api.py:460] Data before pre-processing:
+{'file_name': <tf.Tensor 'args_1:0' shape=() dtype=string>, 'image': <tf.Tensor 'args_2:0' shape=() dtype=string>, 'label': <tf.Tensor 'args_3:0' shape=() dtype=int64>, 'tfds_id': <tf.Tensor 'args_4:0' shape=() dtype=string>, '_id': <tf.Tensor 'args_0:0' shape=() dtype=int32>}
+I1127 21:38:43.969007 134109837835136 api.py:460] Data after pre-processing:
+{'image': <tf.Tensor 'add:0' shape=(224, 224, 3) dtype=float32>, 'labels': <tf.Tensor 'one_hot:0' shape=(1000,) dtype=float32>}
+I1127 21:38:44.062357 134109837835136 train.py:125] NOTE: val evaluation...
+Steps:0/112603 [0.0%]
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/home/jason-chou/Downloads/big_vision/big_vision/train.py", line 396, in <module>
+    app.run(main)
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/absl/app.py", line 308, in run
+    _run_main(main, args)
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/absl/app.py", line 254, in _run_main
+    sys.exit(main(argv))
+             ^^^^^^^^^^
+  File "/home/jason-chou/Downloads/big_vision/big_vision/train.py", line 296, in main
+    for key, value in evaluator.run(model, criterion, config.input.accum_freq):
+  File "/home/jason-chou/Downloads/big_vision/big_vision/evaluators/classification.py", line 74, in run
+    images, target = torch.from_dlpack(dlpack.asdlpack(images)), torch.from_dlpack(dlpack.asdlpack(target))
+                                       ^^^^^^^^^^^^^^^
+AttributeError: module 'dlpack' has no attribute 'asdlpack'

grafted/wandb/run-20241127_213838-torch-grafted-redux/files/requirements.txt

@@ -0,0 +1,133 @@
+array_record==0.5.1
+wandb==0.18.7
+nvidia-curand-cu12==10.3.2.106
+requests-oauthlib==2.0.0
+zipp==3.21.0
+Werkzeug==3.1.3
+simple-parsing==0.1.6
+mdurl==0.1.2
+keras==2.15.0
+nvidia-cuda-nvcc-cu12==12.6.85
+google-auth-oauthlib==1.2.1
+jaxlib==0.4.34
+tf_keras==2.15.1
+oauthlib==3.2.2
+tensorflow-probability==0.25.0
+cachetools==5.5.0
+Jinja2==3.1.3
+rich==13.9.4
+filelock==3.13.1
+google-pasta==0.2.0
+optax==0.2.4
+toolz==1.0.0
+gast==0.6.0
+tensorboard==2.15.2
+pyasn1_modules==0.4.1
+nvidia-cudnn-cu12==9.1.0.70
+opt_einsum==3.4.0
+nvidia-nvjitlink-cu12==12.6.85
+chex==0.1.87
+namex==0.0.8
+termcolor==2.5.0
+flax==0.10.2
+cloudpickle==3.1.0
+numpy==1.26.4
+nvidia-nccl-cu12==2.21.5
+tensorflow-cpu==2.15.0
+nvidia-cusolver-cu12==11.4.5.107
+typing_extensions==4.12.2
+tensorflow-addons==0.23.0
+typeguard==2.13.3
+absl-py==2.1.0
+flatbuffers==24.3.25
+dlpack==0.1
+setuptools==65.5.0
+protobuf==4.25.5
+jax-cuda12-plugin==0.4.35
+tensorflow==2.15.0
+msgpack==1.1.0
+networkx==3.2.1
+docker-pycreds==0.4.0
+nvidia-cuda-cupti-cu12==12.1.105
+pillow==11.0.0
+libclang==18.1.1
+nvidia-cuda-nvrtc-cu12==12.1.105
+distrax==0.1.5
+orbax-checkpoint==0.10.1
+PyYAML==6.0.2
+urllib3==2.2.3
+aqtp==0.8.2
+tensorflow-metadata==1.16.1
+etils==1.11.0
+smmap==5.0.1
+pyasn1==0.6.1
+docstring_parser==0.16
+google-auth==2.36.0
+simplejson==3.19.3
+mpmath==1.3.0
+h5py==3.12.1
+jax-cuda12-pjrt==0.4.35
+tensorflow-io-gcs-filesystem==0.37.1
+tensorflow-estimator==2.15.0
+triton==3.1.0
+rsa==4.9
+panopticapi==0.1
+tensorflow-hub==0.16.1
+requests==2.32.3
+scipy==1.14.1
+ml-dtypes==0.2.0
+markdown-it-py==3.0.0
+tensorflow-text==2.15.0
+wrapt==1.14.1
+immutabledict==4.2.1
+MarkupSafe==3.0.2
+jax==0.4.35
+torch==2.5.1+cu121
+wheel==0.45.1
+einops==0.8.0
+sentry-sdk==2.19.0
+torchvision==0.20.1+cu121
+humanize==4.11.0
+toml==0.10.2
+tensorstore==0.1.69
+six==1.16.0
+promise==2.3
+certifi==2024.8.30
+nvidia-cuda-runtime-cu12==12.1.105
+flaxformer==0.8.8
+nvidia-cufft-cu12==11.0.2.54
+psutil==6.1.0
+GitPython==3.1.43
+platformdirs==4.3.6
+importlib_resources==6.4.5
+tfds-nightly==4.9.7.dev202411280044
+tensorflow-gan==2.1.0
+googleapis-common-protos==1.66.0
+overrides==7.7.0
+optree==0.13.1
+Pygments==2.18.0
+astunparse==1.6.3
+ml_collections==1.0.0
+setproctitle==1.3.4
+tensorboard-data-server==0.7.2
+sympy==1.13.1
+packaging==24.2
+nest-asyncio==1.6.0
+nvidia-cublas-cu12==12.1.3.1
+gitdb==4.0.11
+click==8.1.7
+idna==3.10
+tqdm==4.67.1
+grpcio==1.68.0
+decorator==5.1.1
+pyarrow==18.1.0
+clu==0.0.12
+charset-normalizer==3.4.0
+fsspec==2024.10.0
+dm-tree==0.1.8
+sentencepiece==0.2.0
+nvidia-cusparse-cu12==12.1.0.106
+torchaudio==2.5.1+cu121
+pip==24.3.1
+Markdown==3.7
+nvidia-nvtx-cu12==12.1.105

grafted/wandb/run-20241127_213838-torch-grafted-redux/files/wandb-metadata.json

@@ -0,0 +1,49 @@
+{
+  "os": "Linux-6.8.0-49-generic-x86_64-with-glibc2.39",
+  "python": "3.11.10",
+  "startedAt": "2024-11-28T05:38:38.667951Z",
+  "args": [
+    "--config",
+    "/home/jason-chou/Downloads/big_vision/big_vision/configs/vit_s16_i1k_single_gpu_test.py",
+    "--workdir",
+    "/data/imagenet/grafted",
+    "--name",
+    "torch-grafted-redux"
+  ],
+  "program": "-m big_vision.train",
+  "git": {
+    "remote": "https://github.com/EIFY/big_vision.git",
+    "commit": "44649a64ff67e709f55cdb5e3adcf52064b17de5"
+  },
+  "email": "[email protected]",
+  "root": "/home/jason-chou/Downloads/big_vision",
+  "host": "jasonchou-TensorBook-late-2021",
+  "username": "jason-chou",
+  "executable": "/home/jason-chou/.pyenv/versions/3.11.10/bin/python",
+  "cpu_count": 8,
+  "cpu_count_logical": 16,
+  "gpu": "NVIDIA GeForce RTX 3080 Laptop GPU",
+  "gpu_count": 1,
+  "disk": {
+    "/": {
+      "total": "1006450962432",
+      "used": "584311005184"
+    }
+  },
+  "memory": {
+    "total": "67162914816"
+  },
+  "cpu": {
+    "count": 8,
+    "countLogical": 16
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA GeForce RTX 3080 Laptop GPU",
+      "memoryTotal": "17179869184",
+      "cudaCores": 6144,
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.2"
+}

grafted/wandb/run-20241127_213838-torch-grafted-redux/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":6}}

grafted/wandb/run-20241127_213838-torch-grafted-redux/logs/debug-internal.log

@@ -0,0 +1,18 @@
+{"time":"2024-11-27T21:38:38.66921709-08:00","level":"INFO","msg":"using version","core version":"0.18.7"}
+{"time":"2024-11-27T21:38:38.669224917-08:00","level":"INFO","msg":"created symlink","path":"/home/jason-chou/Downloads/big_vision/wandb/run-20241127_213838-torch-grafted-redux/logs/debug-core.log"}
+{"time":"2024-11-27T21:38:38.772438836-08:00","level":"INFO","msg":"created new stream","id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:38:38.772485797-08:00","level":"INFO","msg":"stream: started","id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:38:38.772617036-08:00","level":"INFO","msg":"writer: Do: started","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:38:38.77266667-08:00","level":"INFO","msg":"sender: started","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:38:38.772762681-08:00","level":"INFO","msg":"handler: started","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:38:39.13369457-08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-27T21:38:45.284216421-08:00","level":"INFO","msg":"stream: closing","id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:38:45.284267357-08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-27T21:38:45.285410301-08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-27T21:38:45.379673564-08:00","level":"WARN","msg":"No program path found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job"}
+{"time":"2024-11-27T21:38:45.37971701-08:00","level":"INFO","msg":"sender: sendDefer: no job artifact to save"}
+{"time":"2024-11-27T21:38:45.839243387-08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-27T21:38:45.960027486-08:00","level":"INFO","msg":"handler: closed","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:38:45.960124482-08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:38:45.96015466-08:00","level":"INFO","msg":"sender: closed","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:38:45.960287984-08:00","level":"INFO","msg":"stream: closed","id":"torch-grafted-redux"}

grafted/wandb/run-20241127_213838-torch-grafted-redux/logs/debug.log

@@ -0,0 +1,55 @@
+2024-11-27 21:38:38,665 INFO    MainThread:69602 [wandb_setup.py:_flush():79] Current SDK version is 0.18.7
+2024-11-27 21:38:38,665 INFO    MainThread:69602 [wandb_setup.py:_flush():79] Configure stats pid to 69602
+2024-11-27 21:38:38,665 INFO    MainThread:69602 [wandb_setup.py:_flush():79] Loading settings from /home/jason-chou/.config/wandb/settings
+2024-11-27 21:38:38,665 INFO    MainThread:69602 [wandb_setup.py:_flush():79] Loading settings from /home/jason-chou/Downloads/big_vision/wandb/settings
+2024-11-27 21:38:38,665 INFO    MainThread:69602 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-27 21:38:38,665 INFO    MainThread:69602 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-27 21:38:38,665 WARNING MainThread:69602 [wandb_setup.py:_flush():79] Could not find program at -m big_vision.train
+2024-11-27 21:38:38,665 INFO    MainThread:69602 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m big_vision.train'}
+2024-11-27 21:38:38,665 INFO    MainThread:69602 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-27 21:38:38,665 INFO    MainThread:69602 [wandb_init.py:_log_setup():533] Logging user logs to /home/jason-chou/Downloads/big_vision/wandb/run-20241127_213838-torch-grafted-redux/logs/debug.log
+2024-11-27 21:38:38,665 INFO    MainThread:69602 [wandb_init.py:_log_setup():534] Logging internal logs to /home/jason-chou/Downloads/big_vision/wandb/run-20241127_213838-torch-grafted-redux/logs/debug-internal.log
+2024-11-27 21:38:38,665 INFO    MainThread:69602 [wandb_init.py:init():619] calling init triggers
+2024-11-27 21:38:38,667 INFO    MainThread:69602 [wandb_init.py:init():626] wandb.init called with sweep_config: {}
+config: {'_fields': {'seed': 0, 'total_epochs': 90, 'num_classes': 1000, 'loss': 'softmax_xent', 'input': accum_freq: 8
+batch_size: 1024
+cache_raw: false
+data:
+  name: imagenet2012
+  split: train
+pp: decode_jpeg_and_inception_crop(224)|flip_lr|randaug(2,10)|value_range(-1, 1)|onehot(1000,
+  key="label", key_result="labels")|keep("image", "labels")
+shuffle_buffer_size: 150000
+, 'pp_modules': ['ops_general', 'ops_image', 'ops_text', 'archive.randaug'], 'log_training_steps': 50, 'ckpt_steps': 1000, 'model_name': 'vit', 'model': pool_type: gap
+posemb: sincos2d
+rep_size: false
+variant: S/16
+, 'grad_clip_norm': 1.0, 'optax_name': 'scale_by_adam', 'optax': mu_dtype: bfloat16
+, 'lr': 0.001, 'wd': 0.0001, 'schedule': decay_type: cosine
+warmup_steps: 10000
+, 'mixup': fold_in: null
+p: 0.2
+, 'evals': val:
+  data:
+    name: imagenet2012
+    split: validation
+  log_steps: 2500
+  loss_name: softmax_xent
+  pp_fn: decode|resize_small(256)|central_crop(224)|value_range(-1, 1)|onehot(1000,
+    key="label", key_result="labels")|keep("image", "labels")
+  type: classification
+}, '_locked': True, '_type_safe': True, '_convert_dict': True, '_allow_dotted_keys': False, '_sort_keys': True}
+2024-11-27 21:38:38,667 INFO    MainThread:69602 [wandb_init.py:init():669] starting backend
+2024-11-27 21:38:38,667 INFO    MainThread:69602 [wandb_init.py:init():673] sending inform_init request
+2024-11-27 21:38:38,667 INFO    MainThread:69602 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-27 21:38:38,667 INFO    MainThread:69602 [wandb_init.py:init():686] backend started and connected
+2024-11-27 21:38:38,670 INFO    MainThread:69602 [wandb_init.py:init():781] updated telemetry
+2024-11-27 21:38:38,672 INFO    MainThread:69602 [wandb_init.py:init():814] communicating run to backend with 90.0 second timeout
+2024-11-27 21:38:39,116 INFO    MainThread:69602 [wandb_init.py:init():859] run resumed
+2024-11-27 21:38:39,131 INFO    MainThread:69602 [wandb_init.py:init():867] starting run threads in backend
+2024-11-27 21:38:39,182 INFO    MainThread:69602 [wandb_run.py:_console_start():2456] atexit reg
+2024-11-27 21:38:39,182 INFO    MainThread:69602 [wandb_run.py:_redirect():2305] redirect: wrap_raw
+2024-11-27 21:38:39,182 INFO    MainThread:69602 [wandb_run.py:_redirect():2370] Wrapping output streams.
+2024-11-27 21:38:39,182 INFO    MainThread:69602 [wandb_run.py:_redirect():2395] Redirects installed.
+2024-11-27 21:38:39,183 INFO    MainThread:69602 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-27 21:38:45,287 WARNING MsgRouterThr:69602 [router.py:message_loop():75] message_loop has been closed

grafted/wandb/run-20241127_214327-torch-grafted-redux/files/config.yaml

@@ -0,0 +1,97 @@
+_allow_dotted_keys:
+    value: false
+_convert_dict:
+    value: true
+_fields:
+    value:
+        ckpt_steps: 1000
+        evals: |
+            val:
+              data:
+                name: imagenet2012
+                split: validation
+              log_steps: 2500
+              loss_name: softmax_xent
+              pp_fn: decode|resize_small(256)|central_crop(224)|value_range(-1, 1)|onehot(1000,
+                key="label", key_result="labels")|keep("image", "labels")
+              type: classification
+        grad_clip_norm: 1
+        input: |
+            accum_freq: 8
+            batch_size: 1024
+            cache_raw: false
+            data:
+              name: imagenet2012
+              split: train
+            pp: decode_jpeg_and_inception_crop(224)|flip_lr|randaug(2,10)|value_range(-1, 1)|onehot(1000,
+              key="label", key_result="labels")|keep("image", "labels")
+            shuffle_buffer_size: 150000
+        log_training_steps: 50
+        loss: softmax_xent
+        lr: 0.001
+        mixup: |
+            fold_in: null
+            p: 0.2
+        model: |
+            pool_type: gap
+            posemb: sincos2d
+            rep_size: false
+            variant: S/16
+        model_name: vit
+        num_classes: 1000
+        optax: |
+            mu_dtype: bfloat16
+        optax_name: scale_by_adam
+        pp_modules:
+            - ops_general
+            - ops_image
+            - ops_text
+            - archive.randaug
+        schedule: |
+            decay_type: cosine
+            warmup_steps: 10000
+        seed: 0
+        total_epochs: 90
+        wd: 0.0001
+_locked:
+    value: true
+_sort_keys:
+    value: true
+_type_safe:
+    value: true
+_wandb:
+    value:
+        cli_version: 0.18.7
+        m: []
+        python_version: 3.11.10
+        t:
+            "1":
+                - 1
+                - 2
+                - 3
+                - 12
+                - 41
+                - 45
+                - 55
+            "2":
+                - 1
+                - 2
+                - 3
+                - 12
+                - 41
+                - 45
+                - 55
+            "3":
+                - 5
+                - 13
+                - 14
+                - 16
+                - 23
+                - 55
+                - 62
+            "4": 3.11.10
+            "5": 0.18.7
+            "8":
+                - 5
+            "12": 0.18.7
+            "13": linux-x86_64

grafted/wandb/run-20241127_214327-torch-grafted-redux/files/output.log

@@ -0,0 +1,211 @@
+I1127 21:43:28.307721 123439371660160 train.py:125] NOTE: Initializing train dataset...
+I1127 21:43:28.307887 123439371660160 train.py:125] NOTE: Global batch size 1024 on 1 hosts results in 1024 local batch size. With 1 dev per host (1 dev total), that's a 1024 per-device batch size.
+I1127 21:43:28.459977 123439371660160 dataset_info.py:707] Load dataset info from /data/tensorflow_datasets/imagenet2012/5.1.0
+I1127 21:43:28.478083 123439371660160 reader.py:261] Creating a tf.data.Dataset reading 1024 files located in folders: /data/tensorflow_datasets/imagenet2012/5.1.0.
+WARNING:tensorflow:From /home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/tensorflow_datasets/core/reader.py:101: CounterV2 (from tensorflow.python.data.experimental.ops.counter) is deprecated and will be removed in a future version.
+Instructions for updating:
+Use `tf.data.Dataset.counter(...)` instead.
+W1127 21:43:28.511688 123439371660160 deprecation.py:50] From /home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/tensorflow_datasets/core/reader.py:101: CounterV2 (from tensorflow.python.data.experimental.ops.counter) is deprecated and will be removed in a future version.
+Instructions for updating:
+Use `tf.data.Dataset.counter(...)` instead.
+I1127 21:43:28.534458 123439371660160 logging_logger.py:49] Constructing tf.data.Dataset imagenet2012 for split _EvenSplit(split='train', index=0, count=1, drop_remainder=False), from /data/tensorflow_datasets/imagenet2012/5.1.0
+I1127 21:43:28.601723 123439371660160 api.py:460] Data before pre-processing:
+{'file_name': <tf.Tensor 'args_1:0' shape=() dtype=string>, 'image': <tf.Tensor 'args_2:0' shape=() dtype=string>, 'label': <tf.Tensor 'args_3:0' shape=() dtype=int64>, 'tfds_id': <tf.Tensor 'args_4:0' shape=() dtype=string>, '_id': <tf.Tensor 'args_0:0' shape=() dtype=int32>}
+INFO:tensorflow:Using RandAug.
+I1127 21:43:28.866792 123439371660160 api.py:460] Using RandAug.
+WARNING:tensorflow:From /home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/tensorflow/python/util/dispatch.py:1260: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
+Instructions for updating:
+Use `tf.cast` instead.
+W1127 21:43:29.046336 123439371660160 deprecation.py:50] From /home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/tensorflow/python/util/dispatch.py:1260: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
+Instructions for updating:
+Use `tf.cast` instead.
+I1127 21:43:31.278083 123439371660160 api.py:460] Data after pre-processing:
+{'image': <tf.Tensor 'add:0' shape=(224, 224, 3) dtype=float32>, 'labels': <tf.Tensor 'one_hot:0' shape=(1000,) dtype=float32>}
+I1127 21:43:31.349534 123439371660160 train.py:125] NOTE: Running for 112603 steps, that means 90.000345 epochs
+I1127 21:43:31.991644 123439371660160 train.py:125] NOTE: Creating model...
+Weight decay for: conv_proj.weight
+Weight decay for: encoder.layers.encoder_layer_0.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_0.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_0.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_0.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_1.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_1.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_1.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_1.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_2.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_2.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_2.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_2.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_3.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_3.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_3.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_3.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_4.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_4.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_4.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_4.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_5.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_5.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_5.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_5.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_6.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_6.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_6.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_6.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_7.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_7.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_7.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_7.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_8.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_8.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_8.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_8.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_9.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_9.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_9.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_9.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_10.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_10.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_10.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_10.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_11.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_11.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_11.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_11.mlp.3.weight
+Weight decay for: heads.head.weight
+Weight decay for: conv_proj.weight
+Weight decay for: encoder.layers.encoder_layer_0.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_0.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_0.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_0.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_1.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_1.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_1.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_1.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_2.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_2.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_2.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_2.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_3.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_3.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_3.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_3.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_4.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_4.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_4.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_4.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_5.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_5.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_5.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_5.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_6.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_6.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_6.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_6.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_7.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_7.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_7.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_7.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_8.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_8.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_8.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_8.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_9.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_9.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_9.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_9.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_10.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_10.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_10.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_10.mlp.3.weight
+Weight decay for: encoder.layers.encoder_layer_11.self_attention.in_proj_weight
+Weight decay for: encoder.layers.encoder_layer_11.self_attention.out_proj.weight
+Weight decay for: encoder.layers.encoder_layer_11.mlp.0.weight
+Weight decay for: encoder.layers.encoder_layer_11.mlp.3.weight
+Weight decay for: heads.head.weight
+I1127 21:43:32.855304 123439371660160 train.py:125] NOTE: Running initial or final evals...
+I1127 21:43:32.855672 123439371660160 train.py:125] NOTE: Init evaluator: val…
+Steps:0/112603 [0.0%]
+I1127 21:43:32.857478 123439371660160 reader.py:261] Creating a tf.data.Dataset reading 64 files located in folders: /data/tensorflow_datasets/imagenet2012/5.1.0.
+I1127 21:43:32.889285 123439371660160 logging_logger.py:49] Constructing tf.data.Dataset imagenet2012 for split _EvenSplit(split='validation', index=0, count=1, drop_remainder=False), from /data/tensorflow_datasets/imagenet2012/5.1.0
+I1127 21:43:32.923067 123439371660160 api.py:460] Data before pre-processing:
+{'file_name': <tf.Tensor 'args_1:0' shape=() dtype=string>, 'image': <tf.Tensor 'args_2:0' shape=() dtype=string>, 'label': <tf.Tensor 'args_3:0' shape=() dtype=int64>, 'tfds_id': <tf.Tensor 'args_4:0' shape=() dtype=string>, '_id': <tf.Tensor 'args_0:0' shape=() dtype=int32>}
+I1127 21:43:33.088486 123439371660160 api.py:460] Data after pre-processing:
+{'image': <tf.Tensor 'add:0' shape=(224, 224, 3) dtype=float32>, 'labels': <tf.Tensor 'one_hot:0' shape=(1000,) dtype=float32>}
+I1127 21:43:33.183008 123439371660160 train.py:125] NOTE: val evaluation...
+Steps:0/112603 [0.0%]
+I1127 21:44:58.298490 123439371660160 utils.py:1231] [0] val/acc@1 = 0.0045041454081632655
+I1127 21:44:58.298775 123439371660160 utils.py:1231] [0] val/loss = 6.883525214633163
+I1127 21:44:58.298943 123439371660160 utils.py:1231] [0] z/secs/eval/val = 85.11567897199711
+I1127 21:44:58.299036 123439371660160 utils.py:560] TIMING[z/secs/eval/val]: 85.11567897199711
+I1127 21:44:58.299128 123439371660160 train.py:125] NOTE: Starting training loop, compiling the first step...
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/home/jason-chou/Downloads/big_vision/big_vision/train.py", line 396, in <module>
+    app.run(main)
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/absl/app.py", line 308, in run
+    _run_main(main, args)
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/absl/app.py", line 254, in _run_main
+    sys.exit(main(argv))
+             ^^^^^^^^^^
+  File "/home/jason-chou/Downloads/big_vision/big_vision/train.py", line 328, in main
+    output = model(img)
+             ^^^^^^^^^^
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py", line 465, in _fn
+    return fn(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/jason-chou/Downloads/big_vision/big_vision/simple_vit.py", line 216, in forward
+    def forward(self, x: torch.Tensor):
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py", line 632, in _fn
+    return fn(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/torch/_functorch/aot_autograd.py", line 1100, in forward
+    return compiled_fn(full_args)
+           ^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 308, in runtime_wrapper
+    all_outs = call_func_at_runtime_with_args(
+               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/utils.py", line 124, in call_func_at_runtime_with_args
+    out = normalize_as_list(f(args))
+                            ^^^^^^^
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/utils.py", line 98, in g
+    return f(*args)
+           ^^^^^^^^
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/torch/autograd/function.py", line 575, in apply
+    return super().apply(*args, **kwargs)  # type: ignore[misc]
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 1525, in forward
+    fw_outs = call_func_at_runtime_with_args(
+              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/utils.py", line 124, in call_func_at_runtime_with_args
+    out = normalize_as_list(f(args))
+                            ^^^^^^^
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 488, in wrapper
+    return compiled_fn(runtime_args)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 667, in inner_fn
+    outs = compiled_fn(args)
+           ^^^^^^^^^^^^^^^^^
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/torch/_inductor/codecache.py", line 1478, in __call__
+    return self.current_callable(inputs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/jason-chou/.pyenv/versions/3.11.10/lib/python3.11/site-packages/torch/_inductor/utils.py", line 1977, in run
+    return model(new_inputs)
+           ^^^^^^^^^^^^^^^^^
+  File "/tmp/torchinductor_jason-chou/ka/ckacaj7mldyv4qgozpmsomr7zpv44qchrg7rtykk4mbmnm67wz36.py", line 1386, in call
+    buf141 = empty_strided_cuda((128, 6, 196, 64), (384, 64, 49152, 1), torch.float32)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 38.00 MiB. GPU 0 has a total capacity of 15.74 GiB of which 30.62 MiB is free. Including non-PyTorch memory, this process has 15.69 GiB memory in use. Of the allocated memory 3.51 GiB is allocated by PyTorch, and 176.27 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)

grafted/wandb/run-20241127_214327-torch-grafted-redux/files/requirements.txt

@@ -0,0 +1,134 @@
+array_record==0.5.1
+wandb==0.18.7
+nvidia-curand-cu12==10.3.2.106
+requests-oauthlib==2.0.0
+zipp==3.21.0
+Werkzeug==3.1.3
+simple-parsing==0.1.6
+mdurl==0.1.2
+keras==2.15.0
+nvidia-cuda-nvcc-cu12==12.6.85
+google-auth-oauthlib==1.2.1
+jaxlib==0.4.34
+tf_keras==2.15.1
+oauthlib==3.2.2
+tensorflow-probability==0.25.0
+cachetools==5.5.0
+Jinja2==3.1.3
+rich==13.9.4
+filelock==3.13.1
+google-pasta==0.2.0
+optax==0.2.4
+toolz==1.0.0
+gast==0.6.0
+tensorboard==2.15.2
+pyasn1_modules==0.4.1
+nvidia-cudnn-cu12==9.1.0.70
+opt_einsum==3.4.0
+nvidia-nvjitlink-cu12==12.6.85
+chex==0.1.87
+namex==0.0.8
+termcolor==2.5.0
+flax==0.10.2
+cloudpickle==3.1.0
+numpy==1.26.4
+nvidia-nccl-cu12==2.21.5
+tensorflow-cpu==2.15.0
+nvidia-cusolver-cu12==11.4.5.107
+typing_extensions==4.12.2
+tensorflow-addons==0.23.0
+typeguard==2.13.3
+absl-py==2.1.0
+flatbuffers==24.3.25
+dlpack==0.1
+setuptools==65.5.0
+protobuf==4.25.5
+jax-cuda12-plugin==0.4.35
+tensorflow==2.15.0
+msgpack==1.1.0
+networkx==3.2.1
+docker-pycreds==0.4.0
+nvidia-cuda-cupti-cu12==12.1.105
+pillow==11.0.0
+libclang==18.1.1
+nvidia-cuda-nvrtc-cu12==12.1.105
+distrax==0.1.5
+orbax-checkpoint==0.10.1
+PyYAML==6.0.2
+urllib3==2.2.3
+aqtp==0.8.2
+tensorflow-metadata==1.16.1
+etils==1.11.0
+smmap==5.0.1
+pyasn1==0.6.1
+docstring_parser==0.16
+google-auth==2.36.0
+simplejson==3.19.3
+mpmath==1.3.0
+h5py==3.12.1
+jax-cuda12-pjrt==0.4.35
+tensorflow-io-gcs-filesystem==0.37.1
+tensorflow-estimator==2.15.0
+pydlpack==0.2.1
+triton==3.1.0
+rsa==4.9
+panopticapi==0.1
+tensorflow-hub==0.16.1
+requests==2.32.3
+scipy==1.14.1
+ml-dtypes==0.2.0
+markdown-it-py==3.0.0
+tensorflow-text==2.15.0
+wrapt==1.14.1
+immutabledict==4.2.1
+MarkupSafe==3.0.2
+jax==0.4.35
+torch==2.5.1+cu121
+wheel==0.45.1
+einops==0.8.0
+sentry-sdk==2.19.0
+torchvision==0.20.1+cu121
+humanize==4.11.0
+toml==0.10.2
+tensorstore==0.1.69
+six==1.16.0
+promise==2.3
+certifi==2024.8.30
+nvidia-cuda-runtime-cu12==12.1.105
+flaxformer==0.8.8
+nvidia-cufft-cu12==11.0.2.54
+psutil==6.1.0
+GitPython==3.1.43
+platformdirs==4.3.6
+importlib_resources==6.4.5
+tfds-nightly==4.9.7.dev202411280044
+tensorflow-gan==2.1.0
+googleapis-common-protos==1.66.0
+overrides==7.7.0
+optree==0.13.1
+Pygments==2.18.0
+astunparse==1.6.3
+ml_collections==1.0.0
+setproctitle==1.3.4
+tensorboard-data-server==0.7.2
+sympy==1.13.1
+packaging==24.2
+nest-asyncio==1.6.0
+nvidia-cublas-cu12==12.1.3.1
+gitdb==4.0.11
+click==8.1.7
+idna==3.10
+tqdm==4.67.1
+grpcio==1.68.0
+decorator==5.1.1
+pyarrow==18.1.0
+clu==0.0.12
+charset-normalizer==3.4.0
+fsspec==2024.10.0
+dm-tree==0.1.8
+sentencepiece==0.2.0
+nvidia-cusparse-cu12==12.1.0.106
+torchaudio==2.5.1+cu121
+pip==24.3.1
+Markdown==3.7
+nvidia-nvtx-cu12==12.1.105

grafted/wandb/run-20241127_214327-torch-grafted-redux/files/wandb-metadata.json

@@ -0,0 +1,49 @@
+{
+  "os": "Linux-6.8.0-49-generic-x86_64-with-glibc2.39",
+  "python": "3.11.10",
+  "startedAt": "2024-11-28T05:43:27.740135Z",
+  "args": [
+    "--config",
+    "/home/jason-chou/Downloads/big_vision/big_vision/configs/vit_s16_i1k_single_gpu_test.py",
+    "--workdir",
+    "/data/imagenet/grafted",
+    "--name",
+    "torch-grafted-redux"
+  ],
+  "program": "-m big_vision.train",
+  "git": {
+    "remote": "https://github.com/EIFY/big_vision.git",
+    "commit": "44649a64ff67e709f55cdb5e3adcf52064b17de5"
+  },
+  "email": "[email protected]",
+  "root": "/home/jason-chou/Downloads/big_vision",
+  "host": "jasonchou-TensorBook-late-2021",
+  "username": "jason-chou",
+  "executable": "/home/jason-chou/.pyenv/versions/3.11.10/bin/python",
+  "cpu_count": 8,
+  "cpu_count_logical": 16,
+  "gpu": "NVIDIA GeForce RTX 3080 Laptop GPU",
+  "gpu_count": 1,
+  "disk": {
+    "/": {
+      "total": "1006450962432",
+      "used": "584312172544"
+    }
+  },
+  "memory": {
+    "total": "67162914816"
+  },
+  "cpu": {
+    "count": 8,
+    "countLogical": 16
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA GeForce RTX 3080 Laptop GPU",
+      "memoryTotal": "17179869184",
+      "cudaCores": 6144,
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.2"
+}

grafted/wandb/run-20241127_214327-torch-grafted-redux/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":118}}

grafted/wandb/run-20241127_214327-torch-grafted-redux/logs/debug-internal.log

@@ -0,0 +1,18 @@
+{"time":"2024-11-27T21:43:27.742580416-08:00","level":"INFO","msg":"using version","core version":"0.18.7"}
+{"time":"2024-11-27T21:43:27.742604921-08:00","level":"INFO","msg":"created symlink","path":"/home/jason-chou/Downloads/big_vision/wandb/run-20241127_214327-torch-grafted-redux/logs/debug-core.log"}
+{"time":"2024-11-27T21:43:27.845793864-08:00","level":"INFO","msg":"created new stream","id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:43:27.845820975-08:00","level":"INFO","msg":"stream: started","id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:43:27.845933768-08:00","level":"INFO","msg":"writer: Do: started","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:43:27.846046124-08:00","level":"INFO","msg":"handler: started","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:43:27.846062359-08:00","level":"INFO","msg":"sender: started","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:43:28.25683485-08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-27T21:45:20.049310897-08:00","level":"INFO","msg":"stream: closing","id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:45:20.0498735-08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-27T21:45:20.052125471-08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-27T21:45:20.144258999-08:00","level":"WARN","msg":"No program path found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job"}
+{"time":"2024-11-27T21:45:20.144292029-08:00","level":"INFO","msg":"sender: sendDefer: no job artifact to save"}
+{"time":"2024-11-27T21:45:20.611424696-08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-27T21:45:20.75808152-08:00","level":"INFO","msg":"handler: closed","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:45:20.758156552-08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:45:20.75839778-08:00","level":"INFO","msg":"sender: closed","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:45:20.758445214-08:00","level":"INFO","msg":"stream: closed","id":"torch-grafted-redux"}

grafted/wandb/run-20241127_214327-torch-grafted-redux/logs/debug.log

@@ -0,0 +1,55 @@
+2024-11-27 21:43:27,731 INFO    MainThread:70768 [wandb_setup.py:_flush():79] Current SDK version is 0.18.7
+2024-11-27 21:43:27,731 INFO    MainThread:70768 [wandb_setup.py:_flush():79] Configure stats pid to 70768
+2024-11-27 21:43:27,731 INFO    MainThread:70768 [wandb_setup.py:_flush():79] Loading settings from /home/jason-chou/.config/wandb/settings
+2024-11-27 21:43:27,731 INFO    MainThread:70768 [wandb_setup.py:_flush():79] Loading settings from /home/jason-chou/Downloads/big_vision/wandb/settings
+2024-11-27 21:43:27,731 INFO    MainThread:70768 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-27 21:43:27,731 INFO    MainThread:70768 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-27 21:43:27,731 WARNING MainThread:70768 [wandb_setup.py:_flush():79] Could not find program at -m big_vision.train
+2024-11-27 21:43:27,731 INFO    MainThread:70768 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m big_vision.train'}
+2024-11-27 21:43:27,732 INFO    MainThread:70768 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-27 21:43:27,732 INFO    MainThread:70768 [wandb_init.py:_log_setup():533] Logging user logs to /home/jason-chou/Downloads/big_vision/wandb/run-20241127_214327-torch-grafted-redux/logs/debug.log
+2024-11-27 21:43:27,732 INFO    MainThread:70768 [wandb_init.py:_log_setup():534] Logging internal logs to /home/jason-chou/Downloads/big_vision/wandb/run-20241127_214327-torch-grafted-redux/logs/debug-internal.log
+2024-11-27 21:43:27,732 INFO    MainThread:70768 [wandb_init.py:init():619] calling init triggers
+2024-11-27 21:43:27,737 INFO    MainThread:70768 [wandb_init.py:init():626] wandb.init called with sweep_config: {}
+config: {'_fields': {'seed': 0, 'total_epochs': 90, 'num_classes': 1000, 'loss': 'softmax_xent', 'input': accum_freq: 8
+batch_size: 1024
+cache_raw: false
+data:
+  name: imagenet2012
+  split: train
+pp: decode_jpeg_and_inception_crop(224)|flip_lr|randaug(2,10)|value_range(-1, 1)|onehot(1000,
+  key="label", key_result="labels")|keep("image", "labels")
+shuffle_buffer_size: 150000
+, 'pp_modules': ['ops_general', 'ops_image', 'ops_text', 'archive.randaug'], 'log_training_steps': 50, 'ckpt_steps': 1000, 'model_name': 'vit', 'model': pool_type: gap
+posemb: sincos2d
+rep_size: false
+variant: S/16
+, 'grad_clip_norm': 1.0, 'optax_name': 'scale_by_adam', 'optax': mu_dtype: bfloat16
+, 'lr': 0.001, 'wd': 0.0001, 'schedule': decay_type: cosine
+warmup_steps: 10000
+, 'mixup': fold_in: null
+p: 0.2
+, 'evals': val:
+  data:
+    name: imagenet2012
+    split: validation
+  log_steps: 2500
+  loss_name: softmax_xent
+  pp_fn: decode|resize_small(256)|central_crop(224)|value_range(-1, 1)|onehot(1000,
+    key="label", key_result="labels")|keep("image", "labels")
+  type: classification
+}, '_locked': True, '_type_safe': True, '_convert_dict': True, '_allow_dotted_keys': False, '_sort_keys': True}
+2024-11-27 21:43:27,737 INFO    MainThread:70768 [wandb_init.py:init():669] starting backend
+2024-11-27 21:43:27,737 INFO    MainThread:70768 [wandb_init.py:init():673] sending inform_init request
+2024-11-27 21:43:27,739 INFO    MainThread:70768 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-27 21:43:27,739 INFO    MainThread:70768 [wandb_init.py:init():686] backend started and connected
+2024-11-27 21:43:27,745 INFO    MainThread:70768 [wandb_init.py:init():781] updated telemetry
+2024-11-27 21:43:27,748 INFO    MainThread:70768 [wandb_init.py:init():814] communicating run to backend with 90.0 second timeout
+2024-11-27 21:43:28,243 INFO    MainThread:70768 [wandb_init.py:init():859] run resumed
+2024-11-27 21:43:28,254 INFO    MainThread:70768 [wandb_init.py:init():867] starting run threads in backend
+2024-11-27 21:43:28,306 INFO    MainThread:70768 [wandb_run.py:_console_start():2456] atexit reg
+2024-11-27 21:43:28,306 INFO    MainThread:70768 [wandb_run.py:_redirect():2305] redirect: wrap_raw
+2024-11-27 21:43:28,306 INFO    MainThread:70768 [wandb_run.py:_redirect():2370] Wrapping output streams.
+2024-11-27 21:43:28,306 INFO    MainThread:70768 [wandb_run.py:_redirect():2395] Redirects installed.
+2024-11-27 21:43:28,307 INFO    MainThread:70768 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-27 21:45:20,049 WARNING MsgRouterThr:70768 [router.py:message_loop():75] message_loop has been closed

grafted/wandb/run-20241127_215015-torch-grafted-redux/files/config.yaml

@@ -0,0 +1,98 @@
+_allow_dotted_keys:
+    value: false
+_convert_dict:
+    value: true
+_fields:
+    value:
+        ckpt_steps: 1000
+        evals: |
+            val:
+              data:
+                name: imagenet2012
+                split: validation
+              log_steps: 2500
+              loss_name: softmax_xent
+              pp_fn: decode|resize_small(256)|central_crop(224)|value_range(-1, 1)|onehot(1000,
+                key="label", key_result="labels")|keep("image", "labels")
+              type: classification
+        grad_clip_norm: 1
+        input: |
+            accum_freq: 8
+            batch_size: 1024
+            cache_raw: false
+            data:
+              name: imagenet2012
+              split: train
+            pp: decode_jpeg_and_inception_crop(224)|flip_lr|randaug(2,10)|value_range(-1, 1)|onehot(1000,
+              key="label", key_result="labels")|keep("image", "labels")
+            shuffle_buffer_size: 150000
+        log_training_steps: 50
+        loss: softmax_xent
+        lr: 0.001
+        mixup: |
+            fold_in: null
+            p: 0.2
+        model: |
+            pool_type: gap
+            posemb: sincos2d
+            rep_size: false
+            variant: S/16
+        model_name: vit
+        num_classes: 1000
+        optax: |
+            mu_dtype: bfloat16
+        optax_name: scale_by_adam
+        pp_modules:
+            - ops_general
+            - ops_image
+            - ops_text
+            - archive.randaug
+        schedule: |
+            decay_type: cosine
+            warmup_steps: 10000
+        seed: 0
+        total_epochs: 90
+        wd: 0.0001
+_locked:
+    value: true
+_sort_keys:
+    value: true
+_type_safe:
+    value: true
+_wandb:
+    value:
+        cli_version: 0.18.7
+        m: []
+        python_version: 3.11.10
+        t:
+            "1":
+                - 1
+                - 2
+                - 3
+                - 12
+                - 41
+                - 45
+                - 55
+            "2":
+                - 1
+                - 2
+                - 3
+                - 12
+                - 41
+                - 45
+                - 55
+            "3":
+                - 5
+                - 13
+                - 14
+                - 16
+                - 23
+                - 55
+                - 61
+                - 62
+            "4": 3.11.10
+            "5": 0.18.7
+            "8":
+                - 5
+            "12": 0.18.7
+            "13": linux-x86_64

grafted/wandb/run-20241127_215015-torch-grafted-redux/files/requirements.txt

@@ -0,0 +1,134 @@
+array_record==0.5.1
+wandb==0.18.7
+nvidia-curand-cu12==10.3.2.106
+requests-oauthlib==2.0.0
+zipp==3.21.0
+Werkzeug==3.1.3
+simple-parsing==0.1.6
+mdurl==0.1.2
+keras==2.15.0
+nvidia-cuda-nvcc-cu12==12.6.85
+google-auth-oauthlib==1.2.1
+jaxlib==0.4.34
+tf_keras==2.15.1
+oauthlib==3.2.2
+tensorflow-probability==0.25.0
+cachetools==5.5.0
+Jinja2==3.1.3
+rich==13.9.4
+filelock==3.13.1
+google-pasta==0.2.0
+optax==0.2.4
+toolz==1.0.0
+gast==0.6.0
+tensorboard==2.15.2
+pyasn1_modules==0.4.1
+nvidia-cudnn-cu12==9.1.0.70
+opt_einsum==3.4.0
+nvidia-nvjitlink-cu12==12.6.85
+chex==0.1.87
+namex==0.0.8
+termcolor==2.5.0
+flax==0.10.2
+cloudpickle==3.1.0
+numpy==1.26.4
+nvidia-nccl-cu12==2.21.5
+tensorflow-cpu==2.15.0
+nvidia-cusolver-cu12==11.4.5.107
+typing_extensions==4.12.2
+tensorflow-addons==0.23.0
+typeguard==2.13.3
+absl-py==2.1.0
+flatbuffers==24.3.25
+dlpack==0.1
+setuptools==65.5.0
+protobuf==4.25.5
+jax-cuda12-plugin==0.4.35
+tensorflow==2.15.0
+msgpack==1.1.0
+networkx==3.2.1
+docker-pycreds==0.4.0
+nvidia-cuda-cupti-cu12==12.1.105
+pillow==11.0.0
+libclang==18.1.1
+nvidia-cuda-nvrtc-cu12==12.1.105
+distrax==0.1.5
+orbax-checkpoint==0.10.1
+PyYAML==6.0.2
+urllib3==2.2.3
+aqtp==0.8.2
+tensorflow-metadata==1.16.1
+etils==1.11.0
+smmap==5.0.1
+pyasn1==0.6.1
+docstring_parser==0.16
+google-auth==2.36.0
+simplejson==3.19.3
+mpmath==1.3.0
+h5py==3.12.1
+jax-cuda12-pjrt==0.4.35
+tensorflow-io-gcs-filesystem==0.37.1
+tensorflow-estimator==2.15.0
+pydlpack==0.2.1
+triton==3.1.0
+rsa==4.9
+panopticapi==0.1
+tensorflow-hub==0.16.1
+requests==2.32.3
+scipy==1.14.1
+ml-dtypes==0.2.0
+markdown-it-py==3.0.0
+tensorflow-text==2.15.0
+wrapt==1.14.1
+immutabledict==4.2.1
+MarkupSafe==3.0.2
+jax==0.4.35
+torch==2.5.1+cu121
+wheel==0.45.1
+einops==0.8.0
+sentry-sdk==2.19.0
+torchvision==0.20.1+cu121
+humanize==4.11.0
+toml==0.10.2
+tensorstore==0.1.69
+six==1.16.0
+promise==2.3
+certifi==2024.8.30
+nvidia-cuda-runtime-cu12==12.1.105
+flaxformer==0.8.8
+nvidia-cufft-cu12==11.0.2.54
+psutil==6.1.0
+GitPython==3.1.43
+platformdirs==4.3.6
+importlib_resources==6.4.5
+tfds-nightly==4.9.7.dev202411280044
+tensorflow-gan==2.1.0
+googleapis-common-protos==1.66.0
+overrides==7.7.0
+optree==0.13.1
+Pygments==2.18.0
+astunparse==1.6.3
+ml_collections==1.0.0
+setproctitle==1.3.4
+tensorboard-data-server==0.7.2
+sympy==1.13.1
+packaging==24.2
+nest-asyncio==1.6.0
+nvidia-cublas-cu12==12.1.3.1
+gitdb==4.0.11
+click==8.1.7
+idna==3.10
+tqdm==4.67.1
+grpcio==1.68.0
+decorator==5.1.1
+pyarrow==18.1.0
+clu==0.0.12
+charset-normalizer==3.4.0
+fsspec==2024.10.0
+dm-tree==0.1.8
+sentencepiece==0.2.0
+nvidia-cusparse-cu12==12.1.0.106
+torchaudio==2.5.1+cu121
+pip==24.3.1
+Markdown==3.7
+nvidia-nvtx-cu12==12.1.105

grafted/wandb/run-20241127_215015-torch-grafted-redux/files/wandb-metadata.json

@@ -0,0 +1,49 @@
+{
+  "os": "Linux-6.8.0-49-generic-x86_64-with-glibc2.39",
+  "python": "3.11.10",
+  "startedAt": "2024-11-28T05:50:15.071664Z",
+  "args": [
+    "--config",
+    "/home/jason-chou/Downloads/big_vision/big_vision/configs/vit_s16_i1k_single_gpu_test.py",
+    "--workdir",
+    "/data/imagenet/grafted",
+    "--name",
+    "torch-grafted-redux"
+  ],
+  "program": "-m big_vision.train",
+  "git": {
+    "remote": "https://github.com/EIFY/big_vision.git",
+    "commit": "44649a64ff67e709f55cdb5e3adcf52064b17de5"
+  },
+  "email": "[email protected]",
+  "root": "/home/jason-chou/Downloads/big_vision",
+  "host": "jasonchou-TensorBook-late-2021",
+  "username": "jason-chou",
+  "executable": "/home/jason-chou/.pyenv/versions/3.11.10/bin/python",
+  "cpu_count": 8,
+  "cpu_count_logical": 16,
+  "gpu": "NVIDIA GeForce RTX 3080 Laptop GPU",
+  "gpu_count": 1,
+  "disk": {
+    "/": {
+      "total": "1006450962432",
+      "used": "584324419584"
+    }
+  },
+  "memory": {
+    "total": "67162914816"
+  },
+  "cpu": {
+    "count": 8,
+    "countLogical": 16
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA GeForce RTX 3080 Laptop GPU",
+      "memoryTotal": "17179869184",
+      "cudaCores": 6144,
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.2"
+}

grafted/wandb/run-20241127_215015-torch-grafted-redux/files/wandb-summary.json

@@ -0,0 +1 @@
+{"lr":2.3437951579552636e-13,"progress":1,"l2_grads":2.6440370082855225,"val/loss":0.919307008081553,"val/acc@1":0.7654257015306123,"l2_params":237.9905797356025,"uptime":705834.820052186,"_wandb":{"runtime":706047},"_step":112603,"core_hours_NVIDIA GeForce RTX 3080 Laptop GPU":196.03091018991725,"train/loss":1.7738897949457169,"img/sec/core":164.21697038941156,"examples_seen":1.15305472e+08,"_timestamp":1.7334789428918557e+09,"_runtime":706047.114314126,"epoch":90.00034499795889,"core_hours":196.03091018991725,"z/secs/eval/val":97.07507979194634}

grafted/wandb/run-20241127_215015-torch-grafted-redux/logs/debug-internal.log

@@ -0,0 +1,247 @@
+{"time":"2024-11-27T21:50:15.074175754-08:00","level":"INFO","msg":"using version","core version":"0.18.7"}
+{"time":"2024-11-27T21:50:15.074199926-08:00","level":"INFO","msg":"created symlink","path":"/home/jason-chou/Downloads/big_vision/wandb/run-20241127_215015-torch-grafted-redux/logs/debug-core.log"}
+{"time":"2024-11-27T21:50:15.178795589-08:00","level":"INFO","msg":"created new stream","id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:50:15.178841831-08:00","level":"INFO","msg":"stream: started","id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:50:15.179017283-08:00","level":"INFO","msg":"writer: Do: started","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:50:15.179013899-08:00","level":"INFO","msg":"sender: started","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:50:15.179140053-08:00","level":"INFO","msg":"handler: started","stream_id":"torch-grafted-redux"}
+{"time":"2024-11-27T21:50:15.742647044-08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-28T02:34:16.031157097-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-11-28T02:34:48.365741462-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-11-28T02:35:23.360466143-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-11-28T02:36:02.179163429-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-11-28T02:36:48.506458125-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-11-28T02:37:26.943002842-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": read tcp 10.0.0.84:35898->35.186.228.49:443: read: connection reset by peer"}
+{"time":"2024-11-28T03:42:18.129115838-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream\": dial tcp 35.186.228.49:443: connect: no route to host"}
+{"time":"2024-11-28T03:42:20.139827897-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
+{"time":"2024-11-28T03:42:24.372179347-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
+{"time":"2024-11-28T03:42:32.586084468-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
+{"time":"2024-11-28T03:42:46.132654012-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-11-28T03:42:52.120982292-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
+{"time":"2024-11-28T03:43:18.285293593-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-11-28T03:43:28.907782037-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
+{"time":"2024-11-28T03:43:53.098087627-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-11-28T03:44:28.909285538-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
+{"time":"2024-11-28T03:44:31.614526811-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-11-28T03:45:17.871096255-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-11-28T03:45:28.911117824-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
+{"time":"2024-11-28T04:27:01.204698067-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-11-28T04:27:33.238366122-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-11-28T04:28:07.420519765-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-11-28T04:28:47.085988867-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-11-28T04:29:35.767860694-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-11-28T08:14:46.411826832-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-11-29T07:48:56.401765469-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream\": dial tcp 35.186.228.49:443: connect: no route to host"}
+{"time":"2024-11-29T15:48:03.006388984-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-11-29T15:48:35.3494534-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-11-29T15:49:10.275739645-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-11-30T03:43:35.70265325-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream\": read tcp 10.0.0.84:53632->35.186.228.49:443: read: connection reset by peer"}
+{"time":"2024-11-30T04:33:41.905126209-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream\": dial tcp 35.186.228.49:443: connect: no route to host"}
+{"time":"2024-11-30T12:32:33.434061272-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream\": dial tcp 35.186.228.49:443: connect: no route to host"}
+{"time":"2024-11-30T16:47:09.087172552-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream\": read tcp 10.0.0.84:57124->35.186.228.49:443: read: connection reset by peer"}
+{"time":"2024-11-30T20:42:04.468697581-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-11-30T20:42:36.856408234-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-11-30T20:43:11.213955553-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-11-30T20:43:51.102603998-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-11-30T20:44:37.626630185-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-11-30T20:45:14.657693577-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": read tcp 10.0.0.84:52436->35.186.228.49:443: read: connection reset by peer"}
+{"time":"2024-12-01T00:45:34.74621148-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-01T00:46:06.894294578-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-01T00:46:41.891784748-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-01T00:47:20.240406935-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-01T00:48:08.857417923-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-01T03:18:49.908220918-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-01T03:19:22.177996379-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-01T03:19:57.167870716-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-01T03:20:35.786919209-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-01T03:21:23.817288934-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-01T04:35:50.003784946-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-01T04:36:22.26631399-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-01T04:36:56.640870294-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-01T04:37:35.443320555-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-01T04:38:23.987753899-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-01T04:46:05.057961007-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-01T04:46:37.546859547-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-01T04:47:11.581392685-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-01T04:47:50.252999457-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-01T04:48:37.650736566-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-01T04:49:11.191378623-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": read tcp 10.0.0.84:38402->35.186.228.49:443: read: connection reset by peer"}
+{"time":"2024-12-01T16:38:05.68648332-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-01T16:38:37.887436779-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-01T16:39:12.133333955-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-01T16:39:52.130430618-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-01T16:40:39.445088692-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-01T16:43:20.726602894-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-01T16:43:52.91904545-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-01T16:44:27.667498554-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-01T16:45:07.347776589-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-01T16:45:54.2028342-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-01T18:32:20.873980572-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-01T18:32:53.205902113-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-01T18:33:27.963313543-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-01T18:34:07.633098053-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-01T18:34:57.576633533-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-01T18:37:35.91680404-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-01T18:38:08.405404154-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-01T18:38:42.587426715-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-01T18:39:20.751273113-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-01T18:40:07.44210804-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-02T02:35:51.391509841-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-02T02:36:23.802341886-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-02T02:36:57.877540099-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-02T02:37:37.844207985-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-02T02:38:24.231618803-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-02T12:37:52.036374796-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-02T12:38:24.246737479-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-02T12:38:58.86214325-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-02T12:39:37.93956232-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-02T12:40:27.832359297-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-02T12:43:22.078082806-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-02T12:43:54.260401543-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-02T12:44:28.422029172-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-02T12:45:07.133435109-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-02T12:45:55.595030082-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-02T12:46:39.639355887-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": read tcp 10.0.0.84:53050->35.186.228.49:443: read: connection reset by peer"}
+{"time":"2024-12-02T14:32:22.213600631-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-02T14:32:54.409672429-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-02T14:33:29.313237828-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-02T14:34:08.561175398-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-02T14:34:58.280659826-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-02T14:35:40.950530237-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": read tcp 10.0.0.84:53982->35.186.228.49:443: read: connection reset by peer"}
+{"time":"2024-12-02T22:38:52.669417282-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-02T22:39:24.808021475-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-02T22:39:59.412976592-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-02T22:40:38.099901646-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-02T22:41:27.308506641-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-03T00:27:22.905950569-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-03T00:27:55.246674688-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-03T00:28:29.944219256-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-03T00:29:08.68500746-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-03T00:29:56.678315339-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-03T08:51:00.224082728-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream\": read tcp 10.0.0.84:54348->35.186.228.49:443: read: connection reset by peer"}
+{"time":"2024-12-03T08:55:53.399562922-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-03T08:56:25.491144652-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-03T08:57:00.241113785-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-03T08:57:40.21004883-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-03T08:58:26.526472631-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-03T10:36:19.661719074-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
+{"time":"2024-12-03T10:36:38.530396963-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-03T10:37:10.818041328-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-03T10:37:44.924643065-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-03T10:38:24.006775695-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-03T10:39:10.789085577-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-03T10:47:08.577691365-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-03T10:47:40.945024167-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-03T10:48:15.894144562-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-03T10:48:54.876775304-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-03T10:49:41.094974693-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-03T10:50:21.078869466-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": read tcp 10.0.0.84:48148->35.186.228.49:443: read: connection reset by peer"}
+{"time":"2024-12-03T12:23:51.403070328-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream"}
+{"time":"2024-12-03T12:40:53.731145089-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-03T23:32:54.290577171-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-03T23:33:26.617679262-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-03T23:34:00.757344429-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-03T23:34:39.966159301-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-03T23:35:26.529833061-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-04T14:19:55.049105743-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-04T15:28:55.116890309-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-04T15:29:27.135172446-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-04T15:30:02.078524164-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-04T15:30:40.31468816-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-04T15:31:29.901859494-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-04T17:18:55.237241846-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-04T17:19:27.334026079-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-04T17:20:02.022011033-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-04T17:20:41.434273822-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-04T17:21:29.254101761-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-04T18:04:10.322394576-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-04T18:04:42.484364529-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-04T18:05:16.646794563-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-04T18:05:55.608027075-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-04T18:06:44.553620127-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-04T22:27:40.607937922-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-04T22:28:12.836516946-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-04T22:28:47.459223154-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-04T22:29:26.147960147-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-04T22:30:15.062106005-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-05T01:47:39.133391386-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream"}
+{"time":"2024-12-05T01:47:55.82293439-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-05T01:47:58.69274009-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream"}
+{"time":"2024-12-05T01:48:14.974980479-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-12-05T01:48:20.402384234-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream"}
+{"time":"2024-12-05T01:48:35.966316067-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-12-05T01:48:45.592640286-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream"}
+{"time":"2024-12-05T01:49:02.593889106-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-12-05T01:49:19.131511604-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream"}
+{"time":"2024-12-05T01:49:36.282249089-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-12-05T01:50:15.299754947-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream"}
+{"time":"2024-12-05T01:50:27.504309484-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-12-05T01:51:32.820040328-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream"}
+{"time":"2024-12-05T01:51:45.124161387-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-12-05T01:52:32.91054814-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream"}
+{"time":"2024-12-05T01:53:01.802742601-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-12-05T01:53:49.829573071-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream"}
+{"time":"2024-12-05T01:54:01.810922094-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-12-05T01:55:01.864377192-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-12-05T01:55:07.143197064-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream"}
+{"time":"2024-12-05T01:56:07.173072531-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream"}
+{"time":"2024-12-05T01:56:18.825182849-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-12-05T01:57:24.055080888-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream"}
+{"time":"2024-12-05T01:57:25.832815739-08:00","level":"WARN","msg":"sender: taking a long time","seconds":600.000041136,"work":"WorkRecord(*service_go_proto.Request_StopStatus); Control(local:true mailbox_slot:\"a5uf5b0ketip\" connection_id:\"127.0.0.1:40934\")"}
+{"time":"2024-12-05T01:57:36.240628481-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-12-05T01:58:32.869123973-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream"}
+{"time":"2024-12-05T01:58:36.256158272-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-12-05T01:59:33.099733752-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream"}
+{"time":"2024-12-05T01:59:53.664227715-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-12-05T01:59:59.516848736-08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.00106625,"work":"WorkRecord(*service_go_proto.Request_Keepalive); Control(local:true connection_id:\"127.0.0.1:40934\")"}
+{"time":"2024-12-05T02:00:05.744419675-08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000352825,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2024-12-05T02:00:05.744480013-08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000396572,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2024-12-05T02:00:05.745541341-08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000591547,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2024-12-05T02:00:05.745550259-08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000894539,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2024-12-05T02:00:05.747662841-08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000444392,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2024-12-05T02:00:33.136975098-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream"}
+{"time":"2024-12-05T02:01:11.07999853-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-12-05T02:01:41.902148444-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream"}
+{"time":"2024-12-05T02:02:27.673478132-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-12-05T02:02:50.303630914-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream"}
+{"time":"2024-12-05T02:03:36.076486567-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-12-05T02:04:07.620541718-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream"}
+{"time":"2024-12-05T02:04:52.67722497-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-12-05T02:05:07.755640903-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream"}
+{"time":"2024-12-05T02:05:52.699769224-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-12-05T02:06:56.575240743-08:00","level":"INFO","msg":"sender: succeeded after taking longer than expected","seconds":1170.752439014,"work":"WorkRecord(*service_go_proto.Request_StopStatus); Control(local:true mailbox_slot:\"a5uf5b0ketip\" connection_id:\"127.0.0.1:40934\")"}
+{"time":"2024-12-05T02:06:56.575254227-08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1010.828027475,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2024-12-05T02:06:56.575270113-08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1010.830331617,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2024-12-05T02:06:56.57528404-08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1010.830637182,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2024-12-05T02:06:56.575306681-08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1010.831224773,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2024-12-05T02:06:56.575330718-08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1017.05961612,"work":"WorkRecord(*service_go_proto.Request_Keepalive); Control(local:true connection_id:\"127.0.0.1:40934\")"}
+{"time":"2024-12-05T02:06:56.5753365-08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1010.831326961,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2024-12-05T09:15:26.449381152-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-05T09:15:58.57629318-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-05T09:16:33.095093096-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-05T11:13:14.836120253-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
+{"time":"2024-12-05T11:13:16.949701235-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
+{"time":"2024-12-05T15:47:11.925458079-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-05T18:23:27.087099548-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": read tcp 10.0.0.84:41532->35.186.228.49:443: read: connection reset by peer"}
+{"time":"2024-12-05T19:12:12.239890815-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-05T19:12:44.526201007-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-05T19:13:18.940020619-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-05T19:13:56.999079567-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-05T19:14:43.370678467-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-05T19:27:27.291303123-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-05T19:27:59.337388458-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-12-05T19:28:33.843756768-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-05T19:29:13.488021118-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-05T19:29:59.640302841-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-12-05T21:23:37.131648811-08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream"}
+{"time":"2024-12-06T01:06:34.449145502-08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/eify/mup-vit/torch-grafted-redux/file_stream\": dial tcp 35.186.228.49:443: connect: no route to host"}
+{"time":"2024-12-06T01:55:44.185807115-08:00","level":"INFO","msg":"stream: closing","id":"torch-grafted-redux"}
+{"time":"2024-12-06T01:55:44.186163552-08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-12-06T01:55:44.188365369-08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-12-06T01:55:44.281103284-08:00","level":"WARN","msg":"No program path found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job"}
+{"time":"2024-12-06T01:55:44.281124225-08:00","level":"INFO","msg":"sender: sendDefer: no job artifact to save"}
+{"time":"2024-12-06T01:55:45.642796519-08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-12-06T01:55:45.770734316-08:00","level":"INFO","msg":"handler: closed","stream_id":"torch-grafted-redux"}
+{"time":"2024-12-06T01:55:45.77080105-08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"torch-grafted-redux"}
+{"time":"2024-12-06T01:55:45.770825146-08:00","level":"INFO","msg":"sender: closed","stream_id":"torch-grafted-redux"}
+{"time":"2024-12-06T01:55:45.77092765-08:00","level":"INFO","msg":"stream: closed","id":"torch-grafted-redux"}

grafted/wandb/run-20241127_215015-torch-grafted-redux/logs/debug.log

@@ -0,0 +1,55 @@
+2024-11-27 21:50:15,067 INFO    MainThread:72239 [wandb_setup.py:_flush():79] Current SDK version is 0.18.7
+2024-11-27 21:50:15,067 INFO    MainThread:72239 [wandb_setup.py:_flush():79] Configure stats pid to 72239
+2024-11-27 21:50:15,067 INFO    MainThread:72239 [wandb_setup.py:_flush():79] Loading settings from /home/jason-chou/.config/wandb/settings
+2024-11-27 21:50:15,067 INFO    MainThread:72239 [wandb_setup.py:_flush():79] Loading settings from /home/jason-chou/Downloads/big_vision/wandb/settings
+2024-11-27 21:50:15,067 INFO    MainThread:72239 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-27 21:50:15,067 INFO    MainThread:72239 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-27 21:50:15,067 WARNING MainThread:72239 [wandb_setup.py:_flush():79] Could not find program at -m big_vision.train
+2024-11-27 21:50:15,067 INFO    MainThread:72239 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m big_vision.train'}
+2024-11-27 21:50:15,067 INFO    MainThread:72239 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-27 21:50:15,067 INFO    MainThread:72239 [wandb_init.py:_log_setup():533] Logging user logs to /home/jason-chou/Downloads/big_vision/wandb/run-20241127_215015-torch-grafted-redux/logs/debug.log
+2024-11-27 21:50:15,067 INFO    MainThread:72239 [wandb_init.py:_log_setup():534] Logging internal logs to /home/jason-chou/Downloads/big_vision/wandb/run-20241127_215015-torch-grafted-redux/logs/debug-internal.log
+2024-11-27 21:50:15,067 INFO    MainThread:72239 [wandb_init.py:init():619] calling init triggers
+2024-11-27 21:50:15,068 INFO    MainThread:72239 [wandb_init.py:init():626] wandb.init called with sweep_config: {}
+config: {'_fields': {'seed': 0, 'total_epochs': 90, 'num_classes': 1000, 'loss': 'softmax_xent', 'input': accum_freq: 8
+batch_size: 1024
+cache_raw: false
+data:
+  name: imagenet2012
+  split: train
+pp: decode_jpeg_and_inception_crop(224)|flip_lr|randaug(2,10)|value_range(-1, 1)|onehot(1000,
+  key="label", key_result="labels")|keep("image", "labels")
+shuffle_buffer_size: 150000
+, 'pp_modules': ['ops_general', 'ops_image', 'ops_text', 'archive.randaug'], 'log_training_steps': 50, 'ckpt_steps': 1000, 'model_name': 'vit', 'model': pool_type: gap
+posemb: sincos2d
+rep_size: false
+variant: S/16
+, 'grad_clip_norm': 1.0, 'optax_name': 'scale_by_adam', 'optax': mu_dtype: bfloat16
+, 'lr': 0.001, 'wd': 0.0001, 'schedule': decay_type: cosine
+warmup_steps: 10000
+, 'mixup': fold_in: null
+p: 0.2
+, 'evals': val:
+  data:
+    name: imagenet2012
+    split: validation
+  log_steps: 2500
+  loss_name: softmax_xent
+  pp_fn: decode|resize_small(256)|central_crop(224)|value_range(-1, 1)|onehot(1000,
+    key="label", key_result="labels")|keep("image", "labels")
+  type: classification
+}, '_locked': True, '_type_safe': True, '_convert_dict': True, '_allow_dotted_keys': False, '_sort_keys': True}
+2024-11-27 21:50:15,068 INFO    MainThread:72239 [wandb_init.py:init():669] starting backend
+2024-11-27 21:50:15,068 INFO    MainThread:72239 [wandb_init.py:init():673] sending inform_init request
+2024-11-27 21:50:15,070 INFO    MainThread:72239 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-27 21:50:15,071 INFO    MainThread:72239 [wandb_init.py:init():686] backend started and connected
+2024-11-27 21:50:15,076 INFO    MainThread:72239 [wandb_init.py:init():781] updated telemetry
+2024-11-27 21:50:15,080 INFO    MainThread:72239 [wandb_init.py:init():814] communicating run to backend with 90.0 second timeout
+2024-11-27 21:50:15,728 INFO    MainThread:72239 [wandb_init.py:init():859] run resumed
+2024-11-27 21:50:15,740 INFO    MainThread:72239 [wandb_init.py:init():867] starting run threads in backend
+2024-11-27 21:50:15,808 INFO    MainThread:72239 [wandb_run.py:_console_start():2456] atexit reg
+2024-11-27 21:50:15,808 INFO    MainThread:72239 [wandb_run.py:_redirect():2305] redirect: wrap_raw
+2024-11-27 21:50:15,809 INFO    MainThread:72239 [wandb_run.py:_redirect():2370] Wrapping output streams.
+2024-11-27 21:50:15,809 INFO    MainThread:72239 [wandb_run.py:_redirect():2395] Redirects installed.
+2024-11-27 21:50:15,810 INFO    MainThread:72239 [wandb_init.py:init():911] run started, returning control to user process
+2024-12-06 01:55:44,185 WARNING MsgRouterThr:72239 [router.py:message_loop():75] message_loop has been closed

grafted/wandb/run-20241127_215015-torch-grafted-redux/run-torch-grafted-redux.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57dfc6760ef360b69a2d3a3f55620d815f8518b1f2ddc461bcb21aaf665973a9
+size 115832137

grafted/wandb/wandb-resume.json

@@ -0,0 +1 @@
+{"run_id": "torch-grafted-redux"}