Upload folder using huggingface_hub

README.md

@@ -0,0 +1,129 @@
+---
+library_name: transformers
+license: llama2
+base_model: meta-llama/CodeLlama-7b-hf
+tags:
+- generated_from_trainer
+datasets:
+- steve329/svgen-500k-2k
+model-index:
+- name: outputs/mymodel
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
+<details><summary>See axolotl config</summary>
+
+axolotl version: `0.8.0.dev0`
+```yaml
+base_model: meta-llama/CodeLlama-7b-hf
+bf16: auto
+dataset_processes: 32
+datasets:
+- message_property_mappings:
+    content: content
+    role: role
+  path: steve329/svgen-500k-2k
+  trust_remote_code: false
+  type: alpaca
+gradient_accumulation_steps: 1
+gradient_checkpointing: false
+learning_rate: 0.0002
+lisa_layers_attribute: model.layers
+load_best_model_at_end: false
+load_in_4bit: false
+load_in_8bit: false
+lora_alpha: 16
+lora_dropout: 0.05
+lora_r: 8
+lora_target_modules:
+- q_proj
+- v_proj
+- k_proj
+- o_proj
+- gate_proj
+- down_proj
+- up_proj
+loraplus_lr_embedding: 1.0e-06
+lr_scheduler: cosine
+max_prompt_len: 1400
+mean_resizing_embeddings: false
+micro_batch_size: 1
+num_epochs: 1.0
+optimizer: adamw_bnb_8bit
+output_dir: ./outputs/mymodel
+pretrain_multipack_attn: true
+pretrain_multipack_buffer_size: 10000
+qlora_sharded_model_loading: false
+ray_num_workers: 1
+resources_per_worker:
+  GPU: 1
+sample_packing_bin_size: 200
+sample_packing_group_size: 100000
+save_only_model: false
+save_safetensors: true
+sequence_len: 1400
+shuffle_merged_datasets: true
+skip_prepare_dataset: false
+strict: false
+train_on_inputs: false
+trl:
+  log_completions: false
+  ref_model_mixup_alpha: 0.9
+  ref_model_sync_steps: 64
+  sync_ref_model: false
+  use_vllm: false
+  vllm_device: auto
+  vllm_dtype: auto
+  vllm_gpu_memory_utilization: 0.9
+use_ray: false
+val_set_size: 0.0
+weight_decay: 0.0
+
+```
+
+</details><br>
+
+# outputs/mymodel
+
+This model is a fine-tuned version of [meta-llama/CodeLlama-7b-hf](https://huggingface.co/meta-llama/CodeLlama-7b-hf) on the steve329/svgen-500k-2k dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 1
+- eval_batch_size: 1
+- seed: 42
+- optimizer: Use OptimizerNames.ADAMW_BNB with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_steps: 49
+- num_epochs: 1.0
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.49.0
+- Pytorch 2.5.1+cu124
+- Datasets 3.2.0
+- Tokenizers 0.21.0

adapter_config.json

@@ -0,0 +1,37 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/CodeLlama-7b-hf",
+  "bias": "none",
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": false,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "gate_proj",
+    "up_proj",
+    "down_proj",
+    "o_proj",
+    "q_proj",
+    "k_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

checkpoint-1635/config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "meta-llama/CodeLlama-7b-hf",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 16384,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 1000000,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.49.0",
+  "use_cache": false,
+  "vocab_size": 32016
+}

checkpoint-1635/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "do_sample": true,
+  "eos_token_id": 2,
+  "transformers_version": "4.49.0"
+}

checkpoint-1635/model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11cc76d12eb9f7c80e7fc5baa91943bf4009a265075041faf5dcc0e38f6500e3
+size 4977175816

checkpoint-1635/model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f670def765d33cd57a88239eaf2139666563657e3b02f73db29fcf97836d9d5
+size 4991627832

checkpoint-1635/model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcf12a93d75ddab3df93f30127683d1bd2d8983ce1d97c387bc5c4f699435dfb
+size 4033405872

checkpoint-1635/model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14002176000
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

checkpoint-1635/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5bf7652eb0e38b8876c135497a5eccbdf2ab747e74defec201f8b4bef05abbd
+size 13688025904

checkpoint-1635/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:386fcc8cc1089aade9450d86fb239ea3483f455fd2d78d8378645feecfec9d69
+size 14244

checkpoint-1635/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8df906b15d6f3a5cc0a427d6d51369611352e747e0268c38e9bc42b9a1050ca
+size 1064

checkpoint-1635/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "additional_special_tokens": [
+    "▁<PRE>",
+    "▁<MID>",
+    "▁<SUF>",
+    "▁<EOT>"
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-1635/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45ccb9c8b6b561889acea59191d66986d314e7cbd6a78abc6e49b139ca91c1e6
+size 500058

checkpoint-1635/tokenizer_config.json

@@ -0,0 +1,84 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32007": {
+      "content": "▁<PRE>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32008": {
+      "content": "▁<SUF>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32009": {
+      "content": "▁<MID>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32010": {
+      "content": "▁<EOT>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "▁<PRE>",
+    "▁<MID>",
+    "▁<SUF>",
+    "▁<EOT>"
+  ],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "eot_token": "▁<EOT>",
+  "extra_special_tokens": {},
+  "fill_token": "<FILL_ME>",
+  "legacy": null,
+  "middle_token": "▁<MID>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "prefix_token": "▁<PRE>",
+  "sp_model_kwargs": {},
+  "suffix_token": "▁<SUF>",
+  "tokenizer_class": "CodeLlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoint-1635/trainer_state.json

@@ -0,0 +1,1461 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1635,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004892966360856269,
+      "grad_norm": 7.852441310882568,
+      "learning_rate": 3.265306122448979e-05,
+      "loss": 0.6394,
+      "step": 8
+    },
+    {
+      "epoch": 0.009785932721712538,
+      "grad_norm": 2.014887809753418,
+      "learning_rate": 6.530612244897959e-05,
+      "loss": 0.4965,
+      "step": 16
+    },
+    {
+      "epoch": 0.014678899082568808,
+      "grad_norm": 3.126901388168335,
+      "learning_rate": 9.79591836734694e-05,
+      "loss": 0.4726,
+      "step": 24
+    },
+    {
+      "epoch": 0.019571865443425075,
+      "grad_norm": 5.920114040374756,
+      "learning_rate": 0.00013061224489795917,
+      "loss": 0.5767,
+      "step": 32
+    },
+    {
+      "epoch": 0.024464831804281346,
+      "grad_norm": 5.9961957931518555,
+      "learning_rate": 0.00016326530612244898,
+      "loss": 0.6178,
+      "step": 40
+    },
+    {
+      "epoch": 0.029357798165137616,
+      "grad_norm": 3.531881332397461,
+      "learning_rate": 0.0001959183673469388,
+      "loss": 0.8085,
+      "step": 48
+    },
+    {
+      "epoch": 0.03425076452599388,
+      "grad_norm": 4.50161600112915,
+      "learning_rate": 0.00019999038714247288,
+      "loss": 0.9277,
+      "step": 56
+    },
+    {
+      "epoch": 0.03914373088685015,
+      "grad_norm": 4.269163131713867,
+      "learning_rate": 0.0001999558618678069,
+      "loss": 0.7473,
+      "step": 64
+    },
+    {
+      "epoch": 0.044036697247706424,
+      "grad_norm": 12.633176803588867,
+      "learning_rate": 0.000199896236661804,
+      "loss": 1.0754,
+      "step": 72
+    },
+    {
+      "epoch": 0.04892966360856269,
+      "grad_norm": 2.925950527191162,
+      "learning_rate": 0.00019981152649695857,
+      "loss": 0.6154,
+      "step": 80
+    },
+    {
+      "epoch": 0.05382262996941896,
+      "grad_norm": 3.7569785118103027,
+      "learning_rate": 0.00019970175264485266,
+      "loss": 0.763,
+      "step": 88
+    },
+    {
+      "epoch": 0.05871559633027523,
+      "grad_norm": 2.0997650623321533,
+      "learning_rate": 0.00019956694267081462,
+      "loss": 0.8468,
+      "step": 96
+    },
+    {
+      "epoch": 0.06360856269113149,
+      "grad_norm": 4.1135687828063965,
+      "learning_rate": 0.00019940713042699708,
+      "loss": 0.6468,
+      "step": 104
+    },
+    {
+      "epoch": 0.06850152905198777,
+      "grad_norm": 7.240751266479492,
+      "learning_rate": 0.0001992223560438763,
+      "loss": 0.6959,
+      "step": 112
+    },
+    {
+      "epoch": 0.07339449541284404,
+      "grad_norm": 0.8589597344398499,
+      "learning_rate": 0.0001990126659201752,
+      "loss": 0.65,
+      "step": 120
+    },
+    {
+      "epoch": 0.0782874617737003,
+      "grad_norm": 3.9679760932922363,
+      "learning_rate": 0.0001987781127112118,
+      "loss": 0.8298,
+      "step": 128
+    },
+    {
+      "epoch": 0.08318042813455657,
+      "grad_norm": 1.5270042419433594,
+      "learning_rate": 0.00019851875531567737,
+      "loss": 0.7995,
+      "step": 136
+    },
+    {
+      "epoch": 0.08807339449541285,
+      "grad_norm": 1.534912109375,
+      "learning_rate": 0.000198234658860846,
+      "loss": 0.7828,
+      "step": 144
+    },
+    {
+      "epoch": 0.09296636085626911,
+      "grad_norm": 2.2733259201049805,
+      "learning_rate": 0.0001979258946862208,
+      "loss": 0.5727,
+      "step": 152
+    },
+    {
+      "epoch": 0.09785932721712538,
+      "grad_norm": 1.2200807332992554,
+      "learning_rate": 0.0001975925403256195,
+      "loss": 0.7983,
+      "step": 160
+    },
+    {
+      "epoch": 0.10275229357798166,
+      "grad_norm": 2.681774139404297,
+      "learning_rate": 0.00019723467948770519,
+      "loss": 0.9528,
+      "step": 168
+    },
+    {
+      "epoch": 0.10764525993883792,
+      "grad_norm": 1.5814114809036255,
+      "learning_rate": 0.00019685240203496605,
+      "loss": 0.5555,
+      "step": 176
+    },
+    {
+      "epoch": 0.11253822629969419,
+      "grad_norm": 1.3297685384750366,
+      "learning_rate": 0.00019644580396115013,
+      "loss": 0.5567,
+      "step": 184
+    },
+    {
+      "epoch": 0.11743119266055047,
+      "grad_norm": 1.335259199142456,
+      "learning_rate": 0.00019601498736716017,
+      "loss": 0.6869,
+      "step": 192
+    },
+    {
+      "epoch": 0.12232415902140673,
+      "grad_norm": 1.0334993600845337,
+      "learning_rate": 0.00019556006043541522,
+      "loss": 0.6384,
+      "step": 200
+    },
+    {
+      "epoch": 0.12721712538226299,
+      "grad_norm": 2.511375904083252,
+      "learning_rate": 0.0001950811374026847,
+      "loss": 0.7303,
+      "step": 208
+    },
+    {
+      "epoch": 0.13211009174311927,
+      "grad_norm": 3.1611194610595703,
+      "learning_rate": 0.00019457833853140253,
+      "loss": 0.4528,
+      "step": 216
+    },
+    {
+      "epoch": 0.13700305810397553,
+      "grad_norm": 2.4775168895721436,
+      "learning_rate": 0.0001940517900794681,
+      "loss": 0.9094,
+      "step": 224
+    },
+    {
+      "epoch": 0.1418960244648318,
+      "grad_norm": 0.9594115018844604,
+      "learning_rate": 0.0001935016242685415,
+      "loss": 0.4906,
+      "step": 232
+    },
+    {
+      "epoch": 0.14678899082568808,
+      "grad_norm": 4.8106818199157715,
+      "learning_rate": 0.00019292797925084124,
+      "loss": 0.9268,
+      "step": 240
+    },
+    {
+      "epoch": 0.15168195718654434,
+      "grad_norm": 1.094592571258545,
+      "learning_rate": 0.0001923309990744531,
+      "loss": 0.4574,
+      "step": 248
+    },
+    {
+      "epoch": 0.1565749235474006,
+      "grad_norm": 1.3729877471923828,
+      "learning_rate": 0.0001917108336471579,
+      "loss": 0.6359,
+      "step": 256
+    },
+    {
+      "epoch": 0.1614678899082569,
+      "grad_norm": 1.7321268320083618,
+      "learning_rate": 0.0001910676386987883,
+      "loss": 0.6083,
+      "step": 264
+    },
+    {
+      "epoch": 0.16636085626911315,
+      "grad_norm": 1.0573010444641113,
+      "learning_rate": 0.00019040157574212345,
+      "loss": 0.5363,
+      "step": 272
+    },
+    {
+      "epoch": 0.1712538226299694,
+      "grad_norm": 0.817417323589325,
+      "learning_rate": 0.0001897128120323314,
+      "loss": 0.5954,
+      "step": 280
+    },
+    {
+      "epoch": 0.1761467889908257,
+      "grad_norm": 0.924107551574707,
+      "learning_rate": 0.00018900152052496978,
+      "loss": 0.4581,
+      "step": 288
+    },
+    {
+      "epoch": 0.18103975535168196,
+      "grad_norm": 1.5086708068847656,
+      "learning_rate": 0.00018826787983255473,
+      "loss": 0.7325,
+      "step": 296
+    },
+    {
+      "epoch": 0.18593272171253822,
+      "grad_norm": 1.3204253911972046,
+      "learning_rate": 0.00018751207417970966,
+      "loss": 0.6294,
+      "step": 304
+    },
+    {
+      "epoch": 0.1908256880733945,
+      "grad_norm": 0.9874266386032104,
+      "learning_rate": 0.00018673429335690442,
+      "loss": 0.5757,
+      "step": 312
+    },
+    {
+      "epoch": 0.19571865443425077,
+      "grad_norm": 0.6221067905426025,
+      "learning_rate": 0.00018593473267279707,
+      "loss": 0.735,
+      "step": 320
+    },
+    {
+      "epoch": 0.20061162079510703,
+      "grad_norm": 1.4100340604782104,
+      "learning_rate": 0.0001851135929051899,
+      "loss": 0.6051,
+      "step": 328
+    },
+    {
+      "epoch": 0.20550458715596331,
+      "grad_norm": 1.4263930320739746,
+      "learning_rate": 0.00018427108025061205,
+      "loss": 0.6215,
+      "step": 336
+    },
+    {
+      "epoch": 0.21039755351681957,
+      "grad_norm": 3.1889100074768066,
+      "learning_rate": 0.0001834074062725415,
+      "loss": 0.404,
+      "step": 344
+    },
+    {
+      "epoch": 0.21529051987767583,
+      "grad_norm": 1.0194282531738281,
+      "learning_rate": 0.0001825227878482793,
+      "loss": 0.5193,
+      "step": 352
+    },
+    {
+      "epoch": 0.22018348623853212,
+      "grad_norm": 1.3357785940170288,
+      "learning_rate": 0.0001816174471144895,
+      "loss": 0.6355,
+      "step": 360
+    },
+    {
+      "epoch": 0.22507645259938838,
+      "grad_norm": 1.151123046875,
+      "learning_rate": 0.00018069161141141842,
+      "loss": 0.6605,
+      "step": 368
+    },
+    {
+      "epoch": 0.22996941896024464,
+      "grad_norm": 0.5690786838531494,
+      "learning_rate": 0.0001797455132258071,
+      "loss": 0.9187,
+      "step": 376
+    },
+    {
+      "epoch": 0.23486238532110093,
+      "grad_norm": 3.2555487155914307,
+      "learning_rate": 0.00017877939013251161,
+      "loss": 0.7481,
+      "step": 384
+    },
+    {
+      "epoch": 0.2397553516819572,
+      "grad_norm": 1.0890693664550781,
+      "learning_rate": 0.00017779348473484556,
+      "loss": 0.6539,
+      "step": 392
+    },
+    {
+      "epoch": 0.24464831804281345,
+      "grad_norm": 1.0565882921218872,
+      "learning_rate": 0.00017678804460366,
+      "loss": 0.5726,
+      "step": 400
+    },
+    {
+      "epoch": 0.24954128440366974,
+      "grad_norm": 0.3272620141506195,
+      "learning_rate": 0.00017576332221517577,
+      "loss": 0.6694,
+      "step": 408
+    },
+    {
+      "epoch": 0.25443425076452597,
+      "grad_norm": 0.7338294386863708,
+      "learning_rate": 0.0001747195748875842,
+      "loss": 0.4905,
+      "step": 416
+    },
+    {
+      "epoch": 0.25932721712538226,
+      "grad_norm": 1.1237382888793945,
+      "learning_rate": 0.00017365706471643191,
+      "loss": 0.5868,
+      "step": 424
+    },
+    {
+      "epoch": 0.26422018348623855,
+      "grad_norm": 4.097552299499512,
+      "learning_rate": 0.00017257605850880598,
+      "loss": 0.4719,
+      "step": 432
+    },
+    {
+      "epoch": 0.2691131498470948,
+      "grad_norm": 0.6702149510383606,
+      "learning_rate": 0.00017147682771633598,
+      "loss": 0.3598,
+      "step": 440
+    },
+    {
+      "epoch": 0.27400611620795107,
+      "grad_norm": 0.7430676817893982,
+      "learning_rate": 0.00017035964836702963,
+      "loss": 0.579,
+      "step": 448
+    },
+    {
+      "epoch": 0.27889908256880735,
+      "grad_norm": 0.6901967525482178,
+      "learning_rate": 0.00016922480099595947,
+      "loss": 0.4065,
+      "step": 456
+    },
+    {
+      "epoch": 0.2837920489296636,
+      "grad_norm": 0.5310859084129333,
+      "learning_rate": 0.00016807257057481763,
+      "loss": 0.5841,
+      "step": 464
+    },
+    {
+      "epoch": 0.2886850152905199,
+      "grad_norm": 1.785436987876892,
+      "learning_rate": 0.00016690324644035641,
+      "loss": 0.4939,
+      "step": 472
+    },
+    {
+      "epoch": 0.29357798165137616,
+      "grad_norm": 0.4744262993335724,
+      "learning_rate": 0.0001657171222217332,
+      "loss": 0.2827,
+      "step": 480
+    },
+    {
+      "epoch": 0.2984709480122324,
+      "grad_norm": 0.628414511680603,
+      "learning_rate": 0.00016451449576677685,
+      "loss": 0.3409,
+      "step": 488
+    },
+    {
+      "epoch": 0.3033639143730887,
+      "grad_norm": 0.81764817237854,
+      "learning_rate": 0.00016329566906719527,
+      "loss": 0.627,
+      "step": 496
+    },
+    {
+      "epoch": 0.30825688073394497,
+      "grad_norm": 0.6422339677810669,
+      "learning_rate": 0.00016206094818274229,
+      "loss": 0.6052,
+      "step": 504
+    },
+    {
+      "epoch": 0.3131498470948012,
+      "grad_norm": 0.5135409832000732,
+      "learning_rate": 0.00016081064316436278,
+      "loss": 0.4868,
+      "step": 512
+    },
+    {
+      "epoch": 0.3180428134556575,
+      "grad_norm": 0.9049602150917053,
+      "learning_rate": 0.00015954506797633583,
+      "loss": 0.3631,
+      "step": 520
+    },
+    {
+      "epoch": 0.3229357798165138,
+      "grad_norm": 1.073359727859497,
+      "learning_rate": 0.00015826454041743503,
+      "loss": 0.5811,
+      "step": 528
+    },
+    {
+      "epoch": 0.32782874617737,
+      "grad_norm": 0.4618200659751892,
+      "learning_rate": 0.0001569693820411262,
+      "loss": 0.3276,
+      "step": 536
+    },
+    {
+      "epoch": 0.3327217125382263,
+      "grad_norm": 0.4611513912677765,
+      "learning_rate": 0.0001556599180748218,
+      "loss": 0.35,
+      "step": 544
+    },
+    {
+      "epoch": 0.3376146788990826,
+      "grad_norm": 1.2012866735458374,
+      "learning_rate": 0.00015433647733821304,
+      "loss": 0.4439,
+      "step": 552
+    },
+    {
+      "epoch": 0.3425076452599388,
+      "grad_norm": 0.4604026675224304,
+      "learning_rate": 0.00015299939216070017,
+      "loss": 0.4703,
+      "step": 560
+    },
+    {
+      "epoch": 0.3474006116207951,
+      "grad_norm": 0.6705082654953003,
+      "learning_rate": 0.0001516489982979409,
+      "loss": 0.4798,
+      "step": 568
+    },
+    {
+      "epoch": 0.3522935779816514,
+      "grad_norm": 1.997828722000122,
+      "learning_rate": 0.0001502856348475389,
+      "loss": 0.4541,
+      "step": 576
+    },
+    {
+      "epoch": 0.35718654434250763,
+      "grad_norm": 0.5123810172080994,
+      "learning_rate": 0.00014890964416389293,
+      "loss": 0.3792,
+      "step": 584
+    },
+    {
+      "epoch": 0.3620795107033639,
+      "grad_norm": 0.33416327834129333,
+      "learning_rate": 0.0001475213717722282,
+      "loss": 0.5155,
+      "step": 592
+    },
+    {
+      "epoch": 0.3669724770642202,
+      "grad_norm": 0.6584266424179077,
+      "learning_rate": 0.00014612116628183138,
+      "loss": 0.6743,
+      "step": 600
+    },
+    {
+      "epoch": 0.37186544342507644,
+      "grad_norm": 1.927019715309143,
+      "learning_rate": 0.0001447093792985114,
+      "loss": 0.6441,
+      "step": 608
+    },
+    {
+      "epoch": 0.3767584097859327,
+      "grad_norm": 0.3161625862121582,
+      "learning_rate": 0.00014328636533630758,
+      "loss": 0.334,
+      "step": 616
+    },
+    {
+      "epoch": 0.381651376146789,
+      "grad_norm": 0.8822286128997803,
+      "learning_rate": 0.00014185248172846748,
+      "loss": 0.4072,
+      "step": 624
+    },
+    {
+      "epoch": 0.38654434250764524,
+      "grad_norm": 0.4811040461063385,
+      "learning_rate": 0.0001404080885377172,
+      "loss": 0.4618,
+      "step": 632
+    },
+    {
+      "epoch": 0.39143730886850153,
+      "grad_norm": 0.806867241859436,
+      "learning_rate": 0.00013895354846584556,
+      "loss": 0.4391,
+      "step": 640
+    },
+    {
+      "epoch": 0.3963302752293578,
+      "grad_norm": 0.3224145770072937,
+      "learning_rate": 0.00013748922676262636,
+      "loss": 0.4006,
+      "step": 648
+    },
+    {
+      "epoch": 0.40122324159021405,
+      "grad_norm": 0.9635908007621765,
+      "learning_rate": 0.0001360154911341002,
+      "loss": 0.3608,
+      "step": 656
+    },
+    {
+      "epoch": 0.40611620795107034,
+      "grad_norm": 1.0050801038742065,
+      "learning_rate": 0.00013453271165023994,
+      "loss": 0.593,
+      "step": 664
+    },
+    {
+      "epoch": 0.41100917431192663,
+      "grad_norm": 0.7276580333709717,
+      "learning_rate": 0.0001330412606520222,
+      "loss": 0.3562,
+      "step": 672
+    },
+    {
+      "epoch": 0.41590214067278286,
+      "grad_norm": 0.8939726948738098,
+      "learning_rate": 0.00013154151265792893,
+      "loss": 0.3767,
+      "step": 680
+    },
+    {
+      "epoch": 0.42079510703363915,
+      "grad_norm": 0.2041948288679123,
+      "learning_rate": 0.000130033844269902,
+      "loss": 0.4444,
+      "step": 688
+    },
+    {
+      "epoch": 0.42568807339449544,
+      "grad_norm": 0.8207905888557434,
+      "learning_rate": 0.00012851863407877445,
+      "loss": 0.5678,
+      "step": 696
+    },
+    {
+      "epoch": 0.43058103975535167,
+      "grad_norm": 0.35527288913726807,
+      "learning_rate": 0.0001269962625692027,
+      "loss": 0.3501,
+      "step": 704
+    },
+    {
+      "epoch": 0.43547400611620796,
+      "grad_norm": 0.510108232498169,
+      "learning_rate": 0.00012546711202412287,
+      "loss": 0.3471,
+      "step": 712
+    },
+    {
+      "epoch": 0.44036697247706424,
+      "grad_norm": 0.39940977096557617,
+      "learning_rate": 0.0001239315664287558,
+      "loss": 0.6869,
+      "step": 720
+    },
+    {
+      "epoch": 0.4452599388379205,
+      "grad_norm": 0.7470912337303162,
+      "learning_rate": 0.00012239001137418417,
+      "loss": 0.5023,
+      "step": 728
+    },
+    {
+      "epoch": 0.45015290519877676,
+      "grad_norm": 0.20923694968223572,
+      "learning_rate": 0.00012084283396052703,
+      "loss": 0.3574,
+      "step": 736
+    },
+    {
+      "epoch": 0.45504587155963305,
+      "grad_norm": 0.9651541709899902,
+      "learning_rate": 0.00011929042269973478,
+      "loss": 0.3703,
+      "step": 744
+    },
+    {
+      "epoch": 0.4599388379204893,
+      "grad_norm": 0.9883789420127869,
+      "learning_rate": 0.00011773316741802994,
+      "loss": 0.463,
+      "step": 752
+    },
+    {
+      "epoch": 0.4648318042813456,
+      "grad_norm": 0.7703452110290527,
+      "learning_rate": 0.00011617145915801786,
+      "loss": 0.3881,
+      "step": 760
+    },
+    {
+      "epoch": 0.46972477064220186,
+      "grad_norm": 0.17868715524673462,
+      "learning_rate": 0.00011460569008049191,
+      "loss": 0.4687,
+      "step": 768
+    },
+    {
+      "epoch": 0.4746177370030581,
+      "grad_norm": 0.2721906304359436,
+      "learning_rate": 0.00011303625336595805,
+      "loss": 0.39,
+      "step": 776
+    },
+    {
+      "epoch": 0.4795107033639144,
+      "grad_norm": 0.7796443700790405,
+      "learning_rate": 0.00011146354311590297,
+      "loss": 0.3948,
+      "step": 784
+    },
+    {
+      "epoch": 0.48440366972477067,
+      "grad_norm": 0.6448463201522827,
+      "learning_rate": 0.00010988795425383144,
+      "loss": 0.4489,
+      "step": 792
+    },
+    {
+      "epoch": 0.4892966360856269,
+      "grad_norm": 0.6768741011619568,
+      "learning_rate": 0.00010830988242609696,
+      "loss": 0.5671,
+      "step": 800
+    },
+    {
+      "epoch": 0.4941896024464832,
+      "grad_norm": 0.8228215575218201,
+      "learning_rate": 0.00010672972390255074,
+      "loss": 0.4973,
+      "step": 808
+    },
+    {
+      "epoch": 0.4990825688073395,
+      "grad_norm": 0.44409453868865967,
+      "learning_rate": 0.00010514787547703466,
+      "loss": 0.5171,
+      "step": 816
+    },
+    {
+      "epoch": 0.5039755351681957,
+      "grad_norm": 1.1631615161895752,
+      "learning_rate": 0.00010356473436774206,
+      "loss": 0.3232,
+      "step": 824
+    },
+    {
+      "epoch": 0.5088685015290519,
+      "grad_norm": 0.7018297910690308,
+      "learning_rate": 0.00010198069811747233,
+      "loss": 0.4563,
+      "step": 832
+    },
+    {
+      "epoch": 0.5137614678899083,
+      "grad_norm": 0.8886349201202393,
+      "learning_rate": 0.00010039616449380378,
+      "loss": 0.3581,
+      "step": 840
+    },
+    {
+      "epoch": 0.5186544342507645,
+      "grad_norm": 0.6620282530784607,
+      "learning_rate": 9.881153138921029e-05,
+      "loss": 0.3712,
+      "step": 848
+    },
+    {
+      "epoch": 0.5235474006116208,
+      "grad_norm": 0.5291142463684082,
+      "learning_rate": 9.722719672114635e-05,
+      "loss": 0.3966,
+      "step": 856
+    },
+    {
+      "epoch": 0.5284403669724771,
+      "grad_norm": 0.37543076276779175,
+      "learning_rate": 9.564355833212613e-05,
+      "loss": 0.3773,
+      "step": 864
+    },
+    {
+      "epoch": 0.5333333333333333,
+      "grad_norm": 0.1506412774324417,
+      "learning_rate": 9.406101388982087e-05,
+      "loss": 0.4028,
+      "step": 872
+    },
+    {
+      "epoch": 0.5382262996941896,
+      "grad_norm": 0.8637451529502869,
+      "learning_rate": 9.247996078720086e-05,
+      "loss": 0.5459,
+      "step": 880
+    },
+    {
+      "epoch": 0.5431192660550459,
+      "grad_norm": 0.5877803564071655,
+      "learning_rate": 9.090079604274581e-05,
+      "loss": 0.3886,
+      "step": 888
+    },
+    {
+      "epoch": 0.5480122324159021,
+      "grad_norm": 0.7344499230384827,
+      "learning_rate": 8.932391620074952e-05,
+      "loss": 0.3507,
+      "step": 896
+    },
+    {
+      "epoch": 0.5529051987767584,
+      "grad_norm": 0.6422482132911682,
+      "learning_rate": 8.774971723174355e-05,
+      "loss": 0.5612,
+      "step": 904
+    },
+    {
+      "epoch": 0.5577981651376147,
+      "grad_norm": 0.4334872364997864,
+      "learning_rate": 8.617859443306512e-05,
+      "loss": 0.4623,
+      "step": 912
+    },
+    {
+      "epoch": 0.5626911314984709,
+      "grad_norm": 0.5234427452087402,
+      "learning_rate": 8.461094232959381e-05,
+      "loss": 0.5449,
+      "step": 920
+    },
+    {
+      "epoch": 0.5675840978593272,
+      "grad_norm": 0.6410064101219177,
+      "learning_rate": 8.304715457468249e-05,
+      "loss": 0.2146,
+      "step": 928
+    },
+    {
+      "epoch": 0.5724770642201835,
+      "grad_norm": 0.7764425277709961,
+      "learning_rate": 8.148762385130673e-05,
+      "loss": 0.5368,
+      "step": 936
+    },
+    {
+      "epoch": 0.5773700305810398,
+      "grad_norm": 0.2351657897233963,
+      "learning_rate": 7.993274177345846e-05,
+      "loss": 0.4047,
+      "step": 944
+    },
+    {
+      "epoch": 0.582262996941896,
+      "grad_norm": 1.0872235298156738,
+      "learning_rate": 7.838289878780744e-05,
+      "loss": 0.5856,
+      "step": 952
+    },
+    {
+      "epoch": 0.5871559633027523,
+      "grad_norm": 0.40746617317199707,
+      "learning_rate": 7.683848407565621e-05,
+      "loss": 0.4328,
+      "step": 960
+    },
+    {
+      "epoch": 0.5920489296636086,
+      "grad_norm": 0.38916969299316406,
+      "learning_rate": 7.529988545521281e-05,
+      "loss": 0.3651,
+      "step": 968
+    },
+    {
+      "epoch": 0.5969418960244648,
+      "grad_norm": 0.7692975997924805,
+      "learning_rate": 7.376748928420563e-05,
+      "loss": 0.2715,
+      "step": 976
+    },
+    {
+      "epoch": 0.6018348623853211,
+      "grad_norm": 0.529144287109375,
+      "learning_rate": 7.224168036286506e-05,
+      "loss": 0.422,
+      "step": 984
+    },
+    {
+      "epoch": 0.6067278287461774,
+      "grad_norm": 0.6713446974754333,
+      "learning_rate": 7.072284183729617e-05,
+      "loss": 0.7623,
+      "step": 992
+    },
+    {
+      "epoch": 0.6116207951070336,
+      "grad_norm": 0.8760319352149963,
+      "learning_rate": 6.921135510326697e-05,
+      "loss": 0.296,
+      "step": 1000
+    },
+    {
+      "epoch": 0.6165137614678899,
+      "grad_norm": 0.41893482208251953,
+      "learning_rate": 6.770759971043604e-05,
+      "loss": 0.2702,
+      "step": 1008
+    },
+    {
+      "epoch": 0.6214067278287462,
+      "grad_norm": 0.6046426892280579,
+      "learning_rate": 6.621195326704361e-05,
+      "loss": 0.4951,
+      "step": 1016
+    },
+    {
+      "epoch": 0.6262996941896024,
+      "grad_norm": 0.5448675751686096,
+      "learning_rate": 6.472479134509052e-05,
+      "loss": 0.3988,
+      "step": 1024
+    },
+    {
+      "epoch": 0.6311926605504588,
+      "grad_norm": 0.6648489236831665,
+      "learning_rate": 6.324648738602817e-05,
+      "loss": 0.5318,
+      "step": 1032
+    },
+    {
+      "epoch": 0.636085626911315,
+      "grad_norm": 0.4234471619129181,
+      "learning_rate": 6.177741260698348e-05,
+      "loss": 0.2669,
+      "step": 1040
+    },
+    {
+      "epoch": 0.6409785932721712,
+      "grad_norm": 0.6632813811302185,
+      "learning_rate": 6.031793590754269e-05,
+      "loss": 0.3598,
+      "step": 1048
+    },
+    {
+      "epoch": 0.6458715596330276,
+      "grad_norm": 0.23808319866657257,
+      "learning_rate": 5.88684237771168e-05,
+      "loss": 0.2686,
+      "step": 1056
+    },
+    {
+      "epoch": 0.6507645259938838,
+      "grad_norm": 0.9806786179542542,
+      "learning_rate": 5.742924020291246e-05,
+      "loss": 0.5166,
+      "step": 1064
+    },
+    {
+      "epoch": 0.65565749235474,
+      "grad_norm": 0.3415721654891968,
+      "learning_rate": 5.6000746578530984e-05,
+      "loss": 0.3834,
+      "step": 1072
+    },
+    {
+      "epoch": 0.6605504587155964,
+      "grad_norm": 0.2674581706523895,
+      "learning_rate": 5.4583301613218764e-05,
+      "loss": 0.3373,
+      "step": 1080
+    },
+    {
+      "epoch": 0.6654434250764526,
+      "grad_norm": 0.6766890287399292,
+      "learning_rate": 5.317726124179183e-05,
+      "loss": 0.4293,
+      "step": 1088
+    },
+    {
+      "epoch": 0.6703363914373088,
+      "grad_norm": 0.6667268872261047,
+      "learning_rate": 5.1782978535256974e-05,
+      "loss": 0.3177,
+      "step": 1096
+    },
+    {
+      "epoch": 0.6752293577981652,
+      "grad_norm": 0.37588515877723694,
+      "learning_rate": 5.040080361215188e-05,
+      "loss": 0.4755,
+      "step": 1104
+    },
+    {
+      "epoch": 0.6801223241590214,
+      "grad_norm": 0.2179548591375351,
+      "learning_rate": 4.903108355062707e-05,
+      "loss": 0.3994,
+      "step": 1112
+    },
+    {
+      "epoch": 0.6850152905198776,
+      "grad_norm": 0.3848086893558502,
+      "learning_rate": 4.7674162301290894e-05,
+      "loss": 0.3222,
+      "step": 1120
+    },
+    {
+      "epoch": 0.689908256880734,
+      "grad_norm": 0.8084197640419006,
+      "learning_rate": 4.633038060083996e-05,
+      "loss": 0.3848,
+      "step": 1128
+    },
+    {
+      "epoch": 0.6948012232415902,
+      "grad_norm": 0.5524225831031799,
+      "learning_rate": 4.500007588649694e-05,
+      "loss": 0.2426,
+      "step": 1136
+    },
+    {
+      "epoch": 0.6996941896024464,
+      "grad_norm": 0.47692129015922546,
+      "learning_rate": 4.368358221127647e-05,
+      "loss": 0.3014,
+      "step": 1144
+    },
+    {
+      "epoch": 0.7045871559633028,
+      "grad_norm": 0.2953272759914398,
+      "learning_rate": 4.2381230160101115e-05,
+      "loss": 0.4501,
+      "step": 1152
+    },
+    {
+      "epoch": 0.709480122324159,
+      "grad_norm": 0.5517217516899109,
+      "learning_rate": 4.109334676678804e-05,
+      "loss": 0.2776,
+      "step": 1160
+    },
+    {
+      "epoch": 0.7143730886850153,
+      "grad_norm": 0.3988036811351776,
+      "learning_rate": 3.982025543192772e-05,
+      "loss": 0.283,
+      "step": 1168
+    },
+    {
+      "epoch": 0.7192660550458716,
+      "grad_norm": 0.7574611902236938,
+      "learning_rate": 3.856227584167449e-05,
+      "loss": 0.4228,
+      "step": 1176
+    },
+    {
+      "epoch": 0.7241590214067278,
+      "grad_norm": 0.6677691340446472,
+      "learning_rate": 3.731972388747038e-05,
+      "loss": 0.4192,
+      "step": 1184
+    },
+    {
+      "epoch": 0.7290519877675841,
+      "grad_norm": 0.2024712860584259,
+      "learning_rate": 3.60929115867213e-05,
+      "loss": 0.3201,
+      "step": 1192
+    },
+    {
+      "epoch": 0.7339449541284404,
+      "grad_norm": 0.25624072551727295,
+      "learning_rate": 3.488214700444656e-05,
+      "loss": 0.5136,
+      "step": 1200
+    },
+    {
+      "epoch": 0.7388379204892966,
+      "grad_norm": 0.3890518248081207,
+      "learning_rate": 3.36877341759205e-05,
+      "loss": 0.3831,
+      "step": 1208
+    },
+    {
+      "epoch": 0.7437308868501529,
+      "grad_norm": 1.2776991128921509,
+      "learning_rate": 3.2509973030326014e-05,
+      "loss": 0.3482,
+      "step": 1216
+    },
+    {
+      "epoch": 0.7486238532110092,
+      "grad_norm": 0.22818905115127563,
+      "learning_rate": 3.134915931543954e-05,
+      "loss": 0.2121,
+      "step": 1224
+    },
+    {
+      "epoch": 0.7535168195718654,
+      "grad_norm": 0.5469585657119751,
+      "learning_rate": 3.0205584523365626e-05,
+      "loss": 0.4208,
+      "step": 1232
+    },
+    {
+      "epoch": 0.7584097859327217,
+      "grad_norm": 0.15633587539196014,
+      "learning_rate": 2.9079535817340408e-05,
+      "loss": 0.3314,
+      "step": 1240
+    },
+    {
+      "epoch": 0.763302752293578,
+      "grad_norm": 1.0666848421096802,
+      "learning_rate": 2.7971295959621824e-05,
+      "loss": 0.253,
+      "step": 1248
+    },
+    {
+      "epoch": 0.7681957186544343,
+      "grad_norm": 0.17933392524719238,
+      "learning_rate": 2.6881143240485407e-05,
+      "loss": 0.3803,
+      "step": 1256
+    },
+    {
+      "epoch": 0.7730886850152905,
+      "grad_norm": 0.46415796875953674,
+      "learning_rate": 2.5809351408342485e-05,
+      "loss": 0.4174,
+      "step": 1264
+    },
+    {
+      "epoch": 0.7779816513761468,
+      "grad_norm": 0.48421502113342285,
+      "learning_rate": 2.475618960099949e-05,
+      "loss": 0.3176,
+      "step": 1272
+    },
+    {
+      "epoch": 0.7828746177370031,
+      "grad_norm": 0.9857699275016785,
+      "learning_rate": 2.3721922278074382e-05,
+      "loss": 2.1419,
+      "step": 1280
+    },
+    {
+      "epoch": 0.7877675840978593,
+      "grad_norm": 0.5873011946678162,
+      "learning_rate": 2.270680915458846e-05,
+      "loss": 0.3703,
+      "step": 1288
+    },
+    {
+      "epoch": 0.7926605504587156,
+      "grad_norm": 1.011412501335144,
+      "learning_rate": 2.1711105135749122e-05,
+      "loss": 0.4611,
+      "step": 1296
+    },
+    {
+      "epoch": 0.7975535168195719,
+      "grad_norm": 0.8163596987724304,
+      "learning_rate": 2.0735060252940553e-05,
+      "loss": 0.2956,
+      "step": 1304
+    },
+    {
+      "epoch": 0.8024464831804281,
+      "grad_norm": 0.7531338334083557,
+      "learning_rate": 1.9778919600938438e-05,
+      "loss": 0.593,
+      "step": 1312
+    },
+    {
+      "epoch": 0.8073394495412844,
+      "grad_norm": 0.1579836755990982,
+      "learning_rate": 1.884292327636411e-05,
+      "loss": 0.2335,
+      "step": 1320
+    },
+    {
+      "epoch": 0.8122324159021407,
+      "grad_norm": 0.4287964105606079,
+      "learning_rate": 1.7927306317393755e-05,
+      "loss": 0.5929,
+      "step": 1328
+    },
+    {
+      "epoch": 0.8171253822629969,
+      "grad_norm": 0.646743655204773,
+      "learning_rate": 1.703229864473811e-05,
+      "loss": 0.4847,
+      "step": 1336
+    },
+    {
+      "epoch": 0.8220183486238533,
+      "grad_norm": 0.6975810527801514,
+      "learning_rate": 1.615812500390694e-05,
+      "loss": 0.4609,
+      "step": 1344
+    },
+    {
+      "epoch": 0.8269113149847095,
+      "grad_norm": 0.33001571893692017,
+      "learning_rate": 1.5305004908773234e-05,
+      "loss": 0.3397,
+      "step": 1352
+    },
+    {
+      "epoch": 0.8318042813455657,
+      "grad_norm": 0.21669529378414154,
+      "learning_rate": 1.4473152586450889e-05,
+      "loss": 0.2275,
+      "step": 1360
+    },
+    {
+      "epoch": 0.8366972477064221,
+      "grad_norm": 0.20811989903450012,
+      "learning_rate": 1.366277692350023e-05,
+      "loss": 0.3166,
+      "step": 1368
+    },
+    {
+      "epoch": 0.8415902140672783,
+      "grad_norm": 0.7644585967063904,
+      "learning_rate": 1.287408141347456e-05,
+      "loss": 0.3106,
+      "step": 1376
+    },
+    {
+      "epoch": 0.8464831804281345,
+      "grad_norm": 0.7066561579704285,
+      "learning_rate": 1.2107264105820593e-05,
+      "loss": 0.4042,
+      "step": 1384
+    },
+    {
+      "epoch": 0.8513761467889909,
+      "grad_norm": 0.47039857506752014,
+      "learning_rate": 1.1362517556146534e-05,
+      "loss": 0.3046,
+      "step": 1392
+    },
+    {
+      "epoch": 0.8562691131498471,
+      "grad_norm": 0.649107813835144,
+      "learning_rate": 1.0640028777869203e-05,
+      "loss": 0.3874,
+      "step": 1400
+    },
+    {
+      "epoch": 0.8611620795107033,
+      "grad_norm": 0.4716101288795471,
+      "learning_rate": 9.939979195253157e-06,
+      "loss": 0.4792,
+      "step": 1408
+    },
+    {
+      "epoch": 0.8660550458715597,
+      "grad_norm": 0.5243660807609558,
+      "learning_rate": 9.262544597853163e-06,
+      "loss": 0.3829,
+      "step": 1416
+    },
+    {
+      "epoch": 0.8709480122324159,
+      "grad_norm": 0.9584677815437317,
+      "learning_rate": 8.60789509637171e-06,
+      "loss": 0.5921,
+      "step": 1424
+    },
+    {
+      "epoch": 0.8758409785932721,
+      "grad_norm": 0.9656602144241333,
+      "learning_rate": 7.976195079942461e-06,
+      "loss": 0.6334,
+      "step": 1432
+    },
+    {
+      "epoch": 0.8807339449541285,
+      "grad_norm": 0.47991645336151123,
+      "learning_rate": 7.367603174850502e-06,
+      "loss": 0.4313,
+      "step": 1440
+    },
+    {
+      "epoch": 0.8856269113149847,
+      "grad_norm": 0.25357896089553833,
+      "learning_rate": 6.78227220469958e-06,
+      "loss": 0.2116,
+      "step": 1448
+    },
+    {
+      "epoch": 0.890519877675841,
+      "grad_norm": 0.12222320586442947,
+      "learning_rate": 6.220349152036676e-06,
+      "loss": 0.4204,
+      "step": 1456
+    },
+    {
+      "epoch": 0.8954128440366973,
+      "grad_norm": 0.4386347234249115,
+      "learning_rate": 5.681975121443084e-06,
+      "loss": 0.3288,
+      "step": 1464
+    },
+    {
+      "epoch": 0.9003058103975535,
+      "grad_norm": 0.3939024806022644,
+      "learning_rate": 5.167285304101677e-06,
+      "loss": 0.1985,
+      "step": 1472
+    },
+    {
+      "epoch": 0.9051987767584098,
+      "grad_norm": 0.14102672040462494,
+      "learning_rate": 4.67640894384902e-06,
+      "loss": 0.2843,
+      "step": 1480
+    },
+    {
+      "epoch": 0.9100917431192661,
+      "grad_norm": 0.6808680295944214,
+      "learning_rate": 4.2094693047209476e-06,
+      "loss": 0.4995,
+      "step": 1488
+    },
+    {
+      "epoch": 0.9149847094801223,
+      "grad_norm": 0.9039413332939148,
+      "learning_rate": 3.766583639999666e-06,
+      "loss": 0.3095,
+      "step": 1496
+    },
+    {
+      "epoch": 0.9198776758409786,
+      "grad_norm": 0.6358740329742432,
+      "learning_rate": 3.3478631627702928e-06,
+      "loss": 0.3858,
+      "step": 1504
+    },
+    {
+      "epoch": 0.9247706422018349,
+      "grad_norm": 0.554490327835083,
+      "learning_rate": 2.9534130179941334e-06,
+      "loss": 0.2273,
+      "step": 1512
+    },
+    {
+      "epoch": 0.9296636085626911,
+      "grad_norm": 0.5606730580329895,
+      "learning_rate": 2.5833322561057085e-06,
+      "loss": 0.2062,
+      "step": 1520
+    },
+    {
+      "epoch": 0.9345565749235474,
+      "grad_norm": 0.5869613289833069,
+      "learning_rate": 2.2377138081401406e-06,
+      "loss": 0.4307,
+      "step": 1528
+    },
+    {
+      "epoch": 0.9394495412844037,
+      "grad_norm": 0.8386426568031311,
+      "learning_rate": 1.9166444623973103e-06,
+      "loss": 0.2351,
+      "step": 1536
+    },
+    {
+      "epoch": 0.94434250764526,
+      "grad_norm": 0.8785386085510254,
+      "learning_rate": 1.6202048426483651e-06,
+      "loss": 0.29,
+      "step": 1544
+    },
+    {
+      "epoch": 0.9492354740061162,
+      "grad_norm": 0.4885581433773041,
+      "learning_rate": 1.3484693878902942e-06,
+      "loss": 0.2552,
+      "step": 1552
+    },
+    {
+      "epoch": 0.9541284403669725,
+      "grad_norm": 0.5360342264175415,
+      "learning_rate": 1.101506333653568e-06,
+      "loss": 0.2805,
+      "step": 1560
+    },
+    {
+      "epoch": 0.9590214067278288,
+      "grad_norm": 0.9303220510482788,
+      "learning_rate": 8.793776948674337e-07,
+      "loss": 0.5197,
+      "step": 1568
+    },
+    {
+      "epoch": 0.963914373088685,
+      "grad_norm": 0.8274058103561401,
+      "learning_rate": 6.82139250287428e-07,
+      "loss": 0.4381,
+      "step": 1576
+    },
+    {
+      "epoch": 0.9688073394495413,
+      "grad_norm": 0.7039580345153809,
+      "learning_rate": 5.098405284886809e-07,
+      "loss": 0.2944,
+      "step": 1584
+    },
+    {
+      "epoch": 0.9737003058103976,
+      "grad_norm": 0.15521883964538574,
+      "learning_rate": 3.6252479542884243e-07,
+      "loss": 0.1466,
+      "step": 1592
+    },
+    {
+      "epoch": 0.9785932721712538,
+      "grad_norm": 0.18543364107608795,
+      "learning_rate": 2.4022904358357303e-07,
+      "loss": 0.3412,
+      "step": 1600
+    },
+    {
+      "epoch": 0.9834862385321101,
+      "grad_norm": 0.5296732187271118,
+      "learning_rate": 1.4298398265727432e-07,
+      "loss": 0.2791,
+      "step": 1608
+    },
+    {
+      "epoch": 0.9883792048929664,
+      "grad_norm": 0.19710539281368256,
+      "learning_rate": 7.08140318716799e-08,
+      "loss": 0.3469,
+      "step": 1616
+    },
+    {
+      "epoch": 0.9932721712538226,
+      "grad_norm": 0.20537033677101135,
+      "learning_rate": 2.373731383383815e-08,
+      "loss": 0.136,
+      "step": 1624
+    },
+    {
+      "epoch": 0.998165137614679,
+      "grad_norm": 0.3667503297328949,
+      "learning_rate": 1.7656499854412645e-09,
+      "loss": 0.3888,
+      "step": 1632
+    }
+  ],
+  "logging_steps": 8,
+  "max_steps": 1635,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.714708871518618e+16,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1635/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bb11e38f79afb93f6464f1016d1ef2f8dde68a54fd4597053d826d246a7e758
+size 6456

config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "meta-llama/CodeLlama-7b-hf",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 16384,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 1000000,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.49.0",
+  "use_cache": false,
+  "vocab_size": 32016
+}

config.yaml

@@ -0,0 +1,63 @@
+base_model: meta-llama/CodeLlama-7b-hf
+bf16: auto
+dataset_processes: 32
+datasets:
+- message_property_mappings:
+    content: content
+    role: role
+  path: steve329/svgen-500k-2k
+  trust_remote_code: false
+  type: alpaca
+gradient_accumulation_steps: 1
+gradient_checkpointing: false
+learning_rate: 0.0002
+lisa_layers_attribute: model.layers
+load_best_model_at_end: false
+load_in_4bit: false
+load_in_8bit: false
+lora_alpha: 16
+lora_dropout: 0.05
+lora_r: 8
+lora_target_modules:
+- q_proj
+- v_proj
+- k_proj
+- o_proj
+- gate_proj
+- down_proj
+- up_proj
+loraplus_lr_embedding: 1.0e-06
+lr_scheduler: cosine
+max_prompt_len: 1400
+mean_resizing_embeddings: false
+micro_batch_size: 1
+num_epochs: 1.0
+optimizer: adamw_bnb_8bit
+output_dir: ./outputs/mymodel
+pretrain_multipack_attn: true
+pretrain_multipack_buffer_size: 10000
+qlora_sharded_model_loading: false
+ray_num_workers: 1
+resources_per_worker:
+  GPU: 1
+sample_packing_bin_size: 200
+sample_packing_group_size: 100000
+save_only_model: false
+save_safetensors: true
+sequence_len: 1400
+shuffle_merged_datasets: true
+skip_prepare_dataset: false
+strict: false
+train_on_inputs: false
+trl:
+  log_completions: false
+  ref_model_mixup_alpha: 0.9
+  ref_model_sync_steps: 64
+  sync_ref_model: false
+  use_vllm: false
+  vllm_device: auto
+  vllm_dtype: auto
+  vllm_gpu_memory_utilization: 0.9
+use_ray: false
+val_set_size: 0.0
+weight_decay: 0.0

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "do_sample": true,
+  "eos_token_id": 2,
+  "transformers_version": "4.49.0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11cc76d12eb9f7c80e7fc5baa91943bf4009a265075041faf5dcc0e38f6500e3
+size 4977175816

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f670def765d33cd57a88239eaf2139666563657e3b02f73db29fcf97836d9d5
+size 4991627832

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcf12a93d75ddab3df93f30127683d1bd2d8983ce1d97c387bc5c4f699435dfb
+size 4033405872

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14002176000
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "additional_special_tokens": [
+    "▁<PRE>",
+    "▁<MID>",
+    "▁<SUF>",
+    "▁<EOT>"
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45ccb9c8b6b561889acea59191d66986d314e7cbd6a78abc6e49b139ca91c1e6
+size 500058

tokenizer_config.json

@@ -0,0 +1,84 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32007": {
+      "content": "▁<PRE>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32008": {
+      "content": "▁<SUF>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32009": {
+      "content": "▁<MID>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32010": {
+      "content": "▁<EOT>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "▁<PRE>",
+    "▁<MID>",
+    "▁<SUF>",
+    "▁<EOT>"
+  ],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "eot_token": "▁<EOT>",
+  "extra_special_tokens": {},
+  "fill_token": "<FILL_ME>",
+  "legacy": null,
+  "middle_token": "▁<MID>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "prefix_token": "▁<PRE>",
+  "sp_model_kwargs": {},
+  "suffix_token": "▁<SUF>",
+  "tokenizer_class": "CodeLlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}