Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,46 @@
+---
+base_model: []
+library_name: transformers
+tags:
+- mergekit
+- merge
+
+---
+# pixtral_merge_task_3
+
+This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
+
+## Merge Details
+### Merge Method
+
+This model was merged using the [task arithmetic](https://arxiv.org/abs/2212.04089) merge method using /opt/dlami/nvme/models/unsloth_Pixtral-12B-2409 as a base.
+
+### Models Merged
+
+The following models were included in the merge:
+* /opt/dlami/nvme/models/TheDrummer_Rocinante-12B-v1.1_llava
+* /opt/dlami/nvme/models/TheDrummer_UnslopNemo-12B-v3_llava
+
+### Configuration
+
+The following YAML configuration was used to produce this model:
+
+```yaml
+merge_method: task_arithmetic
+dtype: bfloat16
+base_model: /opt/dlami/nvme/models/unsloth_Pixtral-12B-2409
+models:
+  - model: /opt/dlami/nvme/models/unsloth_Pixtral-12B-2409
+    parameters:
+      weight: 1.0  
+  - model: /opt/dlami/nvme/models/TheDrummer_UnslopNemo-12B-v3_llava
+    parameters:
+      weight: 0.3
+  - model: /opt/dlami/nvme/models/TheDrummer_Rocinante-12B-v1.1_llava
+    parameters:
+      weight: 0.2
+tokenizer:
+  source: base
+chat_template: "pixtral12b"
+
+```

config.json

@@ -0,0 +1,43 @@
+{
+  "_name_or_path": "/opt/dlami/nvme/models/unsloth_Pixtral-12B-2409",
+  "architectures": [
+    "LlavaForConditionalGeneration"
+  ],
+  "ignore_index": -100,
+  "image_seq_length": 1,
+  "image_token_index": 10,
+  "model_type": "llava",
+  "num_hidden_layers": 40,
+  "projector_hidden_act": "gelu",
+  "initializer_range": 0.02,
+  "text_config": {
+    "hidden_size": 5120,
+    "head_dim": 128,
+    "intermediate_size": 14336,
+    "is_composition": true,
+    "max_position_embeddings": 1024000,
+    "model_type": "mistral",
+    "num_hidden_layers": 40,
+    "num_attention_heads": 32,
+    "num_key_value_heads": 8,
+    "rms_norm_eps": 1e-05,
+    "rope_theta": 1000000000.0,
+    "sliding_window": null,
+    "vocab_size": 131072
+  },
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.47.1",
+  "vision_config": {
+    "head_dim": 64,
+    "num_attention_heads": 16,
+    "hidden_act": "silu",
+    "image_size": 1024,
+    "is_composition": true,
+    "model_type": "pixtral",
+    "patch_size": 16,
+    "rope_theta": 10000.0,
+    "tie_word_embeddings": false
+  },
+  "vision_feature_layer": -1,
+  "vision_feature_select_strategy": "full"
+}

mergekit_config.yml

@@ -0,0 +1,16 @@
+merge_method: task_arithmetic
+dtype: bfloat16
+base_model: /opt/dlami/nvme/models/unsloth_Pixtral-12B-2409
+models:
+  - model: /opt/dlami/nvme/models/unsloth_Pixtral-12B-2409
+    parameters:
+      weight: 1.0  
+  - model: /opt/dlami/nvme/models/TheDrummer_UnslopNemo-12B-v3_llava
+    parameters:
+      weight: 0.3
+  - model: /opt/dlami/nvme/models/TheDrummer_Rocinante-12B-v1.1_llava
+    parameters:
+      weight: 0.2
+tokenizer:
+  source: base
+chat_template: "pixtral12b"

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66d88ce1ad546d6afaa00ae738c5614e185926a1ac1dd170698ba5d18d56dc8b
+size 25365558112

preprocessor_config.json

@@ -0,0 +1,27 @@
+{
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "PixtralImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "patch_size": {
+    "height": 16,
+    "width": 16
+  },
+  "processor_class": "PixtralProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "longest_edge": 1024
+  }
+}

processor_config.json

@@ -0,0 +1,7 @@
+{
+  "image_break_token": "[IMG_BREAK]",
+  "image_end_token": "[IMG_END]",
+  "image_token": "[IMG]",
+  "patch_size": 16,
+  "processor_class": "PixtralProcessor"
+}

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84f33e6f52b2833e8cc17229af8eea363f640a898f19a48184a2c7f6f5a88337
+size 17077329