Hugging Face's logo

MerantixMomentum
/
acip_llama1_7b

Text Generation
Transformers
Safetensors
PyTorch
English
acip_model
feature-extraction
acip
custom_code
Model card Files Community
acip_llama1_7b
File size: 2,454 Bytes 
7836cdd
768f224
7836cdd
 
 
 
 
 
 
 
 
d5e0e16
9de62de
a29081d
7836cdd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a29081d
9de62de
a29081d
 
9de62de
5eb20d4
9de62de
 
 
7836cdd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
{
  "_name_or_path": "MerantixMomentum/acip_llama1_7b",
  "adapter_config": {
    "peft_config": {
      "default": {
        "alpha_pattern": {},
        "auto_mapping": null,
        "base_model_name_or_path": "jeffwan/llama-7b-hf",
        "bias": "none",
        "eva_config": null,
        "exclude_modules": [
          "parametrization",
          "base",
          "ortho"
        ],
        "fan_in_fan_out": false,
        "inference_mode": false,
        "init_lora_weights": true,
        "layer_replication": null,
        "layers_pattern": null,
        "layers_to_transform": null,
        "loftq_config": {},
        "lora_alpha": 16,
        "lora_bias": false,
        "lora_dropout": 0.05,
        "megatron_config": null,
        "megatron_core": "megatron.core",
        "modules_to_save": null,
        "peft_type": "LORA",
        "r": 32,
        "rank_pattern": {},
        "revision": null,
        "target_modules": [
          "up_proj",
          "q_proj",
          "k_proj",
          "v_proj",
          "ortho",
          "base",
          "gate_proj",
          "o_proj",
          "down_proj"
        ],
        "task_type": "CAUSAL_LM",
        "use_dora": false,
        "use_rslora": false
      }
    }
  },
  "architectures": [
    "ACIPModel"
  ],
  "auto_map": {
    "AutoConfig": "acip_model.ACIPModelConfig",
    "AutoModel": "acip_model.ACIPModel"
  },
  "base_model_config": {
    "pretrained_config": null,
    "pretrained_model_cls": "transformers.models.auto.modeling_auto.AutoModelForCausalLM",
    "pretrained_model_kwargs": {
      "pretrained_model_name_or_path": "jeffwan/llama-7b-hf",
      "torch_dtype": "bfloat16"
    }
  },
  "model_mode": "train",
  "model_type": "acip_model",
  "parametrization_config": {
    "exclude_modules": null,
    "module_factory_cls": "svd",
    "module_factory_kwargs": {
      "mask_func": "ste",
      "mask_scaling_factor": 0.02
    },
    "target_modules": [
      "k_proj",
      "down_proj",
      "o_proj",
      "v_proj",
      "gate_proj",
      "q_proj",
      "up_proj"
    ]
  },
  "torch_dtype": "bfloat16",
  "transformers_version": "4.46.3",
  "weight_quantization_config": {
    "exclude_modules": null,
    "module_factory_cls": "bitsandbytes.nn.Linear4bit",
    "module_factory_kwargs": {
      "compute_dtype": "torch.bfloat16",
      "quant_type": "fp4"
    },
    "target_modules": [
      "ortho",
      "base",
      "base_layer"
    ]
  }
}