Training in progress, epoch 0

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

KETI_b1_s4_e1_training_log.log

@@ -0,0 +1,39 @@
+03/31/2025 01:23:04 - INFO  - Train data file: finetuning_data.json
+03/31/2025 01:23:04 - INFO  - Output Directory: output/gemma-3-27b-pt/20250331_012249_gemma-3-27b-pt_LoRA
+03/31/2025 01:23:04 - INFO  - Experiment name: KETI_b1_s4_e1
+03/31/2025 01:23:04 - INFO  - Using 6 GPU(s): NVIDIA A100-SXM4-80GB
+03/31/2025 01:23:05 - INFO  - 🔢 Training samples: 101601
+03/31/2025 01:23:05 - INFO  - 🔍 Evaluation samples: 11289
+03/31/2025 01:23:05 - INFO  - 📊 Steps per epoch: 4233
+03/31/2025 01:23:05 - INFO  - 🪜 Total training steps: 4233
+03/31/2025 01:23:05 - INFO  - torch_dtype: torch.bfloat16
+03/31/2025 01:23:05 - INFO  - ✅ FFT or LoRA 모드로 학습합니다.
+03/31/2025 01:23:25 - INFO  - Initializing LORA model...
+03/31/2025 01:23:32 - INFO  - gcc -pthread -B /root/pai/envs/llm-finetuning/compiler_compat -DNDEBUG -fwrapv -O2 -Wall -fPIC -O2 -isystem /root/pai/envs/llm-finetuning/include -fPIC -O2 -isystem /root/pai/envs/llm-finetuning/include -fPIC -c /tmp/tmpb98cgay3/test.c -o /tmp/tmpb98cgay3/test.o
+03/31/2025 01:23:32 - INFO  - gcc -pthread -B /root/pai/envs/llm-finetuning/compiler_compat -DNDEBUG -fwrapv -O2 -Wall -fPIC -O2 -isystem /root/pai/envs/llm-finetuning/include -fPIC -O2 -isystem /root/pai/envs/llm-finetuning/include -fPIC -c /tmp/tmp8kmegipo/test.c -o /tmp/tmp8kmegipo/test.o
+03/31/2025 01:23:33 - INFO  - Start Training !
+03/31/2025 01:24:01 - INFO  - [Epoch 0.07] [Step 10] loss: 3.8061
+03/31/2025 01:24:24 - INFO  - [Epoch 0.13] [Step 20] loss: 3.0707
+03/31/2025 01:24:47 - INFO  - [Epoch 0.20] [Step 30] loss: 2.9570
+03/31/2025 01:25:10 - INFO  - [Epoch 0.26] [Step 40] loss: 2.9072
+03/31/2025 01:25:33 - INFO  - [Epoch 0.33] [Step 50] loss: 2.8972
+03/31/2025 01:25:55 - INFO  - [Epoch 0.40] [Step 60] loss: 2.8640
+03/31/2025 01:26:18 - INFO  - [Epoch 0.46] [Step 70] loss: 2.8533
+03/31/2025 01:26:41 - INFO  - [Epoch 0.53] [Step 80] loss: 2.8426
+03/31/2025 01:27:03 - INFO  - [Epoch 0.59] [Step 90] loss: 2.8257
+03/31/2025 01:27:26 - INFO  - [Epoch 0.66] [Step 100] loss: 2.8181
+03/31/2025 01:27:49 - INFO  - [Epoch 0.72] [Step 110] loss: 2.8075
+03/31/2025 01:28:11 - INFO  - [Epoch 0.79] [Step 120] loss: 2.8010
+03/31/2025 01:28:33 - INFO  - [Epoch 0.86] [Step 130] loss: 2.8073
+03/31/2025 01:28:56 - INFO  - [Epoch 0.92] [Step 140] loss: 2.7980
+03/31/2025 01:29:18 - INFO  - [Epoch 0.99] [Step 150] loss: 2.8007
+03/31/2025 01:29:39 - INFO  - ✅ Training complete. Logging system usage...
+03/31/2025 01:29:39 - INFO  - >> System Usage - CPU: 3.2%, RAM: 2.6%, SSD: 75.74GB / 1888.43GB
+03/31/2025 01:29:39 - INFO  - >> GPU 0: 71.28 GB used
+03/31/2025 01:29:39 - INFO  - >> GPU 1: 79.90 GB used
+03/31/2025 01:29:39 - INFO  - >> GPU 2: 70.95 GB used
+03/31/2025 01:29:39 - INFO  - >> GPU 3: 78.04 GB used
+03/31/2025 01:29:39 - INFO  - >> GPU 4: 79.88 GB used
+03/31/2025 01:29:39 - INFO  - >> GPU 5: 77.00 GB used
+03/31/2025 01:29:39 - INFO  - >> Total GPU Memory Used: 457.05 GB
+03/31/2025 01:29:39 - INFO  - >> Total GPU Power Consumption: 540.07 W

adapter_config.json

@@ -0,0 +1,39 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "google/gemma-3-27b-pt",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "up_proj",
+    "k_proj",
+    "o_proj",
+    "gate_proj",
+    "q_proj",
+    "down_proj",
+    "v_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1c47b1d5277ad90994756c1e11e4b56c89bc894631daa3d04be3e9b033cdc70
+size 454183400

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<image_soft_token>": 262144
+}

runs/Mar31_01-23-27_llm-server-779876f58-9zzqd/events.out.tfevents.1743384214.llm-server-779876f58-9zzqd.4164641.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0accf6fbbb6658f59d3a13ca1a0078a0a53289023916060eac9875772ee298b2
+size 11214

special_tokens_map.json

@@ -0,0 +1,33 @@
+{
+  "boi_token": "<start_of_image>",
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eoi_token": "<end_of_image>",
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "image_token": "<image_soft_token>",
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4667f2089529e8e7657cfb6d1c19910ae71ff5f28aa7ab2ff2763330affad795
+size 33384568

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1299c11d7cf632ef3b4e11937501358ada021bbdf7c47638d13c0ee982f2e79c
+size 4689074

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0524d70b5438b81dfd429c5e371f15f469593975d6921ace87f503ec38471854
+size 5816