Upload 24 files

.gitattributes

@@ -1,35 +1,6 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
 *.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -textLLM/tokenizer.json filter=lfs diff=lfs merge=lfs -text

BiCodec/config.yaml

@@ -0,0 +1,60 @@
+audio_tokenizer:
+  mel_params:
+    sample_rate: 16000
+    n_fft: 1024
+    win_length: 640
+    hop_length: 320
+    mel_fmin: 10 
+    mel_fmax: null
+    num_mels: 128
+
+  encoder:
+    input_channels: 1024
+    vocos_dim: 384
+    vocos_intermediate_dim: 2048
+    vocos_num_layers: 12
+    out_channels: 1024
+    sample_ratios: [1,1]
+
+  decoder:
+    input_channel: 1024
+    channels: 1536
+    rates: [8, 5, 4, 2]
+    kernel_sizes: [16,11,8,4]
+
+  quantizer:
+    input_dim: 1024
+    codebook_size: 8192
+    codebook_dim: 8
+    commitment: 0.25
+    codebook_loss_weight: 2.0
+    use_l2_normlize: True
+    threshold_ema_dead_code: 0.2
+  
+  speaker_encoder:
+    input_dim: 128
+    out_dim: 1024
+    latent_dim: 128
+    token_num: 32
+    fsq_levels: [4, 4, 4, 4, 4, 4]
+    fsq_num_quantizers: 1
+
+  prenet:
+    input_channels: 1024
+    vocos_dim: 384
+    vocos_intermediate_dim: 2048
+    vocos_num_layers: 12
+    out_channels: 1024
+    condition_dim: 1024
+    sample_ratios: [1,1]
+    use_tanh_at_final: False
+
+  postnet: 
+    input_channels: 1024
+    vocos_dim: 384
+    vocos_intermediate_dim: 2048
+    vocos_num_layers: 6
+    out_channels: 1024
+    use_tanh_at_final: False
+
+

BiCodec/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9940cd48d4446e4340ced82d234bf5618350dd9f5db900ebe47a4fdb03867ec
+size 625518756

LLM/config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 21,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.1",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 166000
+}

LLM/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54825baf0a2f6076eb3c78fa1d22a95aee225f59070a8b295f8169db860eb109
+size 2026568968

LLM/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

LLM/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c8b057d6ca205a429cc3428b9fc815f0d6ee1d53106dd5e5b129ef9db2ff057
+size 14129172

__init__.py

@@ -0,0 +1,32 @@
+# # SparkAudio/Spark-TTS-0.5B_v2/__init__.py
+
+# import os
+# from pathlib import Path
+# # Import registry và AutoModel để đăng ký
+# from transformers.pipelines import PIPELINE_REGISTRY
+# from transformers import AutoModel
+
+# # Print để xác nhận file này được thực thi
+# print(f"Executing __init__.py for SparkTTS custom module in: {__file__}")
+
+# try:
+#     # Import lớp Pipeline tùy chỉnh để đăng ký
+#     from .pipeline_spark_tts import SparkTTSPipeline
+
+#     # Đăng ký pipeline tùy chỉnh với registry
+#     # Đảm bảo tên task "text-to-speech" khớp với tên bạn dùng khi gọi pipeline()
+#     print(f"Attempting to register SparkTTSPipeline for task 'text-to-speech'...")
+#     PIPELINE_REGISTRY.register_pipeline(
+#         "text-to-speech",                 # Tên task
+#         pipeline_class=SparkTTSPipeline,  # Lớp pipeline tùy chỉnh của bạn
+#         pt_model=AutoModel,               # Lớp AutoModel tương thích cho PyTorch
+#         # tf_model=None,                  # Thêm lớp TF nếu có
+#         type="text",                      # Kiểu input chính (ví dụ: text, audio, image)
+#     )
+#     print("Pipeline registration call completed.")
+
+# # Bắt lỗi nếu import thất bại (ví dụ: thiếu dependency)
+# except ImportError as e:
+#     print(f"WARNING: Could not import SparkTTSPipeline in __init__.py: {e}. Pipeline registration failed.")
+# except Exception as e:
+#      print(f"ERROR: An unexpected error occurred during SparkTTS __init__ registration: {e}")

_utils.py

@@ -0,0 +1,151 @@
+# Copyright (c) 2025 SparkAudio & The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Utility functions for SparkTTS """
+
+import random
+import soxr
+import soundfile
+import torch
+import torchaudio
+import numpy as np
+
+from pathlib import Path
+from typing import Tuple, Dict, Any
+from numpy.lib.stride_tricks import sliding_window_view
+from omegaconf import OmegaConf # Keep if BiCodec config loading needs it
+
+
+# --- Token Maps (from sparktts/utils/token_parser.py) ---
+TASK_TOKEN_MAP = {
+    "vc": "<|task_vc|>",
+    "tts": "<|task_tts|>",
+    "asr": "<|task_asr|>",
+    "s2s": "<|task_s2s|>",
+    "t2s": "<|task_t2s|>",
+    "understand": "<|task_understand|>",
+    "caption": "<|task_cap|>",
+    "controllable_tts": "<|task_controllable_tts|>",
+    "prompt_tts": "<|task_prompt_tts|>",
+    "speech_edit": "<|task_edit|>",
+}
+
+LEVELS_MAP = {
+    "very_low": 0,
+    "low": 1,
+    "moderate": 2,
+    "high": 3,
+    "very_high": 4,
+}
+
+LEVELS_MAP_UI = {
+    1: 'very_low',
+    2: 'low',
+    3: 'moderate',
+    4: 'high',
+    5: 'very_high'
+}
+
+GENDER_MAP = {
+    "female": 0,
+    "male": 1,
+}
+
+# --- Audio Utils (from sparktts/utils/audio.py) ---
+def audio_volume_normalize(audio: np.ndarray, coeff: float = 0.2) -> np.ndarray:
+    temp = np.sort(np.abs(audio))
+    if len(temp) == 0: # Handle empty audio case
+        return audio
+    if temp[-1] < 0.1:
+        scaling_factor = max(temp[-1], 1e-3)
+        audio = audio / scaling_factor * 0.1
+    temp = temp[temp > 0.01]
+    L = temp.shape[0]
+    if L <= 10:
+        return audio
+    volume = np.mean(temp[int(0.9 * L) : int(0.99 * L)])
+    if volume == 0: # Avoid division by zero if volume is effectively zero
+        return audio
+    audio = audio * np.clip(coeff / volume, a_min=0.1, a_max=10)
+    max_value = np.max(np.abs(audio)) if len(audio) > 0 else 0
+    if max_value > 1:
+        audio = audio / max_value
+    return audio
+
+def load_audio(
+    adfile: Path,
+    sampling_rate: int = None,
+    length: int = None,
+    volume_normalize: bool = False,
+    segment_duration: int = None,
+) -> np.ndarray:
+    try:
+        audio, sr = soundfile.read(adfile, dtype='float32') # Ensure float32
+    except Exception as e:
+        raise IOError(f"Could not read audio file {adfile}: {e}")
+
+    if audio is None or len(audio) == 0:
+        raise ValueError(f"Audio file {adfile} is empty or invalid.")
+
+    if len(audio.shape) > 1:
+        audio = audio[:, 0]
+
+    if sampling_rate is not None and sr != sampling_rate:
+        try:
+            # Ensure input is float64 for soxr
+            audio = audio.astype(np.float64)
+            audio = soxr.resample(audio, sr, sampling_rate, quality="VHQ")
+            # Convert back to float32
+            audio = audio.astype(np.float32)
+            sr = sampling_rate
+        except Exception as e:
+             raise RuntimeError(f"Failed to resample audio from {sr}Hz to {sampling_rate}Hz: {e}")
+
+    if segment_duration is not None:
+        seg_length = int(sr * segment_duration)
+        audio = random_select_audio_segment(audio, seg_length)
+
+    if volume_normalize:
+        audio = audio_volume_normalize(audio)
+
+    if length is not None:
+        if audio.shape[0] > length:
+            audio = audio[:length]
+        else:
+            audio = np.pad(audio, (0, int(length - audio.shape[0])), mode='constant')
+    return audio
+
+def random_select_audio_segment(audio: np.ndarray, length: int) -> np.ndarray:
+    if audio.shape[0] < length:
+        audio = np.pad(audio, (0, int(length - audio.shape[0])), mode='constant')
+        start_index = 0 # If padded, start from beginning
+    elif audio.shape[0] == length:
+         start_index = 0 # If exact length, start from beginning
+    else:
+        start_index = random.randint(0, audio.shape[0] - length)
+
+    end_index = int(start_index + length)
+    return audio[start_index:end_index]
+
+# --- File Utils (Minimal required) ---
+def load_config_yaml(config_path: Path) -> Dict:
+    """Loads a YAML configuration file using OmegaConf."""
+    # Check if path exists
+    if not Path(config_path).is_file():
+        raise FileNotFoundError(f"YAML Config file not found: {config_path}")
+    try:
+        config = OmegaConf.load(config_path)
+        # Convert OmegaConf DictConfig to standard Python dict
+        return OmegaConf.to_container(config, resolve=True)
+    except Exception as e:
+        raise IOError(f"Error loading YAML config file {config_path}: {e}")

config.json

@@ -0,0 +1,90 @@
+{
+    "model_type": "spark-tts",
+    "architectures": [
+      "SparkTTSModel"
+    ],
+    "auto_map": {
+      "AutoConfig": "configuration_spark_tts.SparkTTSConfig",
+      "AutoModel": "modeling_spark_tts.SparkTTSModel",
+      "AutoProcessor": "processing_spark_tts.SparkTTSProcessor"
+    },
+    "processor_class": "processing_spark_tts.SparkTTSProcessor",
+    "custom_pipelines": {
+        "text-to-speech": {
+            "impl": "pipeline_spark_tts.SparkTTSPipeline",
+            "pt": ["AutoModel"]
+        }
+    },
+    "llm_model_name_or_path": "./LLM",
+    "bicodec_model_name_or_path": "./BiCodec",
+    "wav2vec2_model_name_or_path": "./wav2vec2-large-xlsr-53",
+    "sample_rate": 16000,
+    "highpass_cutoff_freq": 40,
+    "latent_hop_length": 320,
+    "ref_segment_duration": 6.0,
+    "volume_normalize": true,
+    "bicodec_config": {
+      "audio_tokenizer": {
+        "mel_params": {
+        "sample_rate": 16000,
+        "n_fft": 1024,
+        "win_length": 640,
+        "hop_length": 320,
+        "mel_fmin": 10,
+        "mel_fmax": null,
+        "num_mels": 128
+        },
+        "encoder": {
+        "input_channels": 1024,
+        "vocos_dim": 384,
+        "vocos_intermediate_dim": 2048,
+        "vocos_num_layers": 12,
+        "out_channels": 1024,
+        "sample_ratios": [1, 1]
+        },
+        "decoder": {
+        "input_channel": 1024,
+        "channels": 1536,
+        "rates": [8, 5, 4, 2],
+        "kernel_sizes": [16, 11, 8, 4]
+        },
+        "quantizer": {
+        "input_dim": 1024,
+        "codebook_size": 8192,
+        "codebook_dim": 8,
+        "commitment": 0.25,
+        "codebook_loss_weight": 2.0,
+        "use_l2_normlize": true,
+        "threshold_ema_dead_code": 0.2
+        },
+        "speaker_encoder": {
+        "input_dim": 128,
+        "out_dim": 1024,
+        "latent_dim": 128,
+        "token_num": 32,
+        "fsq_levels": [4, 4, 4, 4, 4, 4],
+        "fsq_num_quantizers": 1
+        },
+        "prenet": {
+        "input_channels": 1024,
+        "vocos_dim": 384,
+        "vocos_intermediate_dim": 2048,
+        "vocos_num_layers": 12,
+        "out_channels": 1024,
+        "condition_dim": 1024,
+        "sample_ratios": [1, 1],
+        "use_tanh_at_final": false
+        },
+        "postnet": {
+        "input_channels": 1024,
+        "vocos_dim": 384,
+        "vocos_intermediate_dim": 2048,
+        "vocos_num_layers": 6,
+        "out_channels": 1024,
+        "use_tanh_at_final": false
+        }
+    }
+    },
+    "torch_dtype": "bfloat16",
+    "transformers_version": "4.43.1"
+  }

configuration_spark_tts.py

@@ -0,0 +1,86 @@
+# Copyright (c) 2025 SparkAudio & The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" SparkTTS model configuration"""
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+class SparkTTSConfig(PretrainedConfig):
+    """
+    This is the configuration class to store the configuration of a [`SparkTTSModel`].
+    It is used to instantiate a SparkTTS model according to the specified arguments, defining the model
+    architecture and sub-component paths.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs.
+    Read the documentation from [`PretrainedConfig`] for more information.
+
+    Args:
+        llm_model_name_or_path (`str`, *optional*, defaults to `"./LLM"`):
+            Path to the pretrained LLM model or model identifier from huggingface.co/models.
+        bicodec_model_name_or_path (`str`, *optional*, defaults to `"./BiCodec"`):
+            Path to the pretrained BiCodec model directory.
+        wav2vec2_model_name_or_path (`str`, *optional*, defaults to `"./wav2vec2-large-xlsr-53"`):
+            Path to the pretrained Wav2Vec2 model directory.
+        sample_rate (`int`, *optional*, defaults to 16000):
+            The sampling rate of the audio files.
+        highpass_cutoff_freq (`int`, *optional*, defaults to 40):
+             Highpass filter cutoff frequency for audio processing.
+        latent_hop_length (`int`, *optional*, defaults to 320):
+             Hop length used in BiCodec processing.
+        ref_segment_duration (`float`, *optional*, defaults to 6.0):
+            Duration (in seconds) of the reference audio clip used for speaker embedding.
+        volume_normalize (`bool`, *optional*, defaults to `True`):
+            Whether to normalize the volume of audio inputs.
+        bicodec_config (`dict`, *optional*):
+             A dictionary containing the configuration for the BiCodec model components (encoder, decoder, etc.).
+             This is typically loaded from the `BiCodec/config.yaml` originally.
+        **kwargs
+            Additional keyword arguments passed along to [`PretrainedConfig`].
+    """
+
+    model_type = "spark-tts"
+    processor_class = "SparkTTSProcessor"
+    attribute_map = {} # Add mappings if needed for renaming attributes
+
+    def __init__(
+        self,
+        llm_model_name_or_path="./LLM",
+        bicodec_model_name_or_path="./BiCodec",
+        wav2vec2_model_name_or_path="./wav2vec2-large-xlsr-53",
+        sample_rate=16000,
+        highpass_cutoff_freq=40,
+        latent_hop_length=320,
+        ref_segment_duration=6.0,
+        volume_normalize=True,
+        bicodec_config=None,
+        **kwargs,
+    ):
+        self.llm_model_name_or_path = llm_model_name_or_path
+        self.bicodec_model_name_or_path = bicodec_model_name_or_path
+        self.wav2vec2_model_name_or_path = wav2vec2_model_name_or_path
+        self.sample_rate = sample_rate
+        self.highpass_cutoff_freq = highpass_cutoff_freq
+        self.latent_hop_length = latent_hop_length
+        self.ref_segment_duration = ref_segment_duration
+        self.volume_normalize = volume_normalize
+        self.bicodec_config = bicodec_config if bicodec_config is not None else {}
+
+        # REMOVE THIS WARNING - the check in SparkTTSModel is better
+        # if not self.bicodec_config:
+        #     logger.warning("BiCodec config is empty. BiCodec model might not load correctly.")
+
+        super().__init__(**kwargs)

model_index.json

@@ -0,0 +1,39 @@
+{
+    "model_index": [
+      {
+        "name": "SparkTTS",
+        "results": [],
+        "config": {
+          "architectures": [
+            "SparkTTSForConditionalGeneration"
+          ],
+          "model_type": "sparktts"
+        },
+        "filenames": [
+          "config.json",
+          "model_index.json",
+          "modeling_sparktts.py",
+          "pipeline_sparktts.py",
+          "__init__.py",
+          "LLM/config.json",
+          "LLM/pytorch_model.bin",
+          "LLM/tokenizer.json", 
+          "BiCodec/config.yaml",
+          "BiCodec/model.safetensors",
+          "wav2vec2-large-xlsr-53/config.json",
+          "wav2vec2-large-xlsr-53/pytorch_model.bin",
+          "wav2vec2-large-xlsr-53/preprocessor_config.json",
+          "sparktts_code/models/audio_tokenizer.py",
+          "sparktts_code/models/bicodec.py",
+          "sparktts_code/modules/speaker/speaker_encoder.py",
+          "sparktts_code/modules/encoder_decoder/feat_encoder.py",
+          "sparktts_code/modules/encoder_decoder/feat_decoder.py",
+          "sparktts_code/modules/encoder_decoder/wave_generator.py",
+          "sparktts_code/modules/vq/factorized_vector_quantize.py",
+          "sparktts_code/utils/audio.py",
+          "sparktts_code/utils/file.py",
+          "sparktts_code/utils/token_parser.py"
+        ]
+      }
+    ]
+  }

pipeline_spark_tts.py

@@ -0,0 +1,303 @@
+# Copyright (c) 2025 SparkAudio & The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+import re
+import numpy as np
+from typing import Optional, Dict, Any, Union, List
+from pathlib import Path
+from transformers import Pipeline, PreTrainedModel # Inherit directly from the base class
+from transformers.utils import logging
+
+# Import necessary items from this module (assuming they are defined in modeling_spark_tts)
+from .modeling_spark_tts import SparkTTSModel
+from .configuration_spark_tts import SparkTTSConfig
+from .modeling_spark_tts import (
+    load_audio,
+    TASK_TOKEN_MAP,
+    LEVELS_MAP,
+    GENDER_MAP,
+    LEVELS_MAP_UI,
+)
+# No need to import SparkTTSModel/Config here, pipeline gets them during init
+
+logger = logging.get_logger(__name__)
+
+# Define constants if needed (e.g., for default generation args)
+DEFAULT_MAX_NEW_TOKENS = 3000
+DEFAULT_TEMPERATURE = 0.8
+DEFAULT_TOP_K = 50
+DEFAULT_TOP_P = 0.95
+
+class SparkTTSPipeline(Pipeline):
+    """
+    Custom Pipeline for SparkTTS text-to-speech generation, following HF documentation structure.
+    Handles voice cloning and voice creation modes.
+    """
+    def __init__(self, model, tokenizer=None, framework="pt", device=None, **kwargs):
+        # --- KHÔNG NÊN load model ở đây nữa ---
+        # __init__ của pipeline tùy chỉnh nên nhận model và tokenizer đã được load
+        # Việc load nên xảy ra BÊN NGOÀI trước khi gọi pipeline() hoặc do pipeline factory xử lý
+
+        # Kiểm tra model và tokenizer được truyền vào
+        if model is None:
+             raise ValueError("SparkTTSPipeline requires a 'model' argument.")
+        if not isinstance(model, SparkTTSModel):
+             # Có thể model được load bằng AutoModel nên là PreTrainedModel, kiểm tra config
+             if isinstance(model, PreTrainedModel) and isinstance(model.config, SparkTTSConfig):
+                  pass # OK, model tương thích dựa trên config
+             else:
+                  raise TypeError(f"Expected model compatible with SparkTTSConfig, but got {type(model)}")
+
+        if tokenizer is None:
+             raise ValueError("SparkTTSPipeline requires a 'tokenizer' argument.")
+        if not hasattr(tokenizer, 'encode') or not hasattr(tokenizer, 'batch_decode'):
+              raise TypeError("Tokenizer does not seem to be a valid Transformers tokenizer.")
+
+        # Gọi super().__init__ với model/tokenizer đã nhận
+        super().__init__(model=model, tokenizer=tokenizer, framework=framework, device=device, **kwargs)
+        if hasattr(self.model, 'config') and hasattr(self.model.config, 'sample_rate'):
+             self.sampling_rate = self.model.config.sample_rate
+        else:
+             # Nên đặt giá trị mặc định hoặc lấy từ nơi khác nếu config không có
+             logger.warning("Could not determine sampling rate from model config. Defaulting to 16000.")
+             self.sampling_rate = 16000
+
+    def _sanitize_parameters(self, **kwargs) -> tuple[dict, dict, dict]:
+        """
+        Sanitizes pipeline parameters and separates them for preprocess, forward, and postprocess.
+
+        Returns:
+            Tuple[dict, dict, dict]: preprocess_kwargs, forward_kwargs, postprocess_kwargs
+        """
+        preprocess_kwargs = {}
+        # --- Preprocessing specific args ---
+        if "prompt_speech_path" in kwargs:
+            preprocess_kwargs["prompt_speech_path"] = kwargs["prompt_speech_path"]
+        if "prompt_text" in kwargs:
+            preprocess_kwargs["prompt_text"] = kwargs["prompt_text"]
+        if "gender" in kwargs:
+            preprocess_kwargs["gender"] = kwargs["gender"]
+        if "pitch" in kwargs:
+            preprocess_kwargs["pitch"] = kwargs["pitch"]
+        if "speed" in kwargs:
+            preprocess_kwargs["speed"] = kwargs["speed"]
+
+        forward_kwargs = {}
+        # --- Forward specific args (LLM generation) ---
+        # Use kwargs.get to allow users to override defaults
+        forward_kwargs["max_new_tokens"] = kwargs.get("max_new_tokens", DEFAULT_MAX_NEW_TOKENS)
+        forward_kwargs["do_sample"] = kwargs.get("do_sample", True)
+        forward_kwargs["temperature"] = kwargs.get("temperature", DEFAULT_TEMPERATURE)
+        forward_kwargs["top_k"] = kwargs.get("top_k", DEFAULT_TOP_K)
+        forward_kwargs["top_p"] = kwargs.get("top_p", DEFAULT_TOP_P)
+        # Ensure essential generation tokens are present if needed
+        if self.tokenizer.eos_token_id is not None:
+            forward_kwargs["eos_token_id"] = self.tokenizer.eos_token_id
+        if self.tokenizer.pad_token_id is not None:
+            forward_kwargs["pad_token_id"] = self.tokenizer.pad_token_id
+        elif self.tokenizer.eos_token_id is not None:
+            logger.warning("Setting pad_token_id to eos_token_id for open-end generation.")
+            forward_kwargs["pad_token_id"] = self.tokenizer.eos_token_id
+        # Filter out None values that might cause issues with generate
+        forward_kwargs = {k: v for k, v in forward_kwargs.items() if v is not None}
+
+        postprocess_kwargs = {}
+        # --- Postprocessing specific args (if any in the future) ---
+        # Example: if you added an option to return tokens instead of audio
+        # if "return_tokens" in kwargs:
+        #    postprocess_kwargs["return_tokens"] = kwargs["return_tokens"]
+
+        return preprocess_kwargs, forward_kwargs, postprocess_kwargs
+
+    def preprocess(self, inputs, **preprocess_kwargs) -> dict:
+        """
+        Transforms text input and preprocess_kwargs into model input format.
+
+        Args:
+            inputs (str): The text to synthesize.
+            preprocess_kwargs (dict): Arguments relevant to preprocessing (e.g., prompt paths, controls).
+
+        Returns:
+            dict: Containing `model_inputs` (tokenized dict) and `global_token_ids_prompt` (optional Tensor).
+        """
+        text = inputs
+        prompt_speech_path = preprocess_kwargs.get("prompt_speech_path")
+        prompt_text = preprocess_kwargs.get("prompt_text")
+        gender = preprocess_kwargs.get("gender")
+        pitch = preprocess_kwargs.get("pitch")
+        speed = preprocess_kwargs.get("speed")
+
+        global_token_ids = None
+        llm_prompt_string = ""
+
+        # --- Logic to build llm_prompt_string and get global_token_ids ---
+        if prompt_speech_path is not None:
+            # Voice Cloning Mode
+            logger.info(f"Preprocessing for Voice Cloning (prompt: {prompt_speech_path})")
+            if not Path(prompt_speech_path).exists():
+                 raise FileNotFoundError(f"Prompt speech file not found: {prompt_speech_path}")
+            # Use the MODEL's method for tokenization (self.model is set by base Pipeline class)
+            global_tokens, semantic_tokens = self.model._tokenize_audio(prompt_speech_path)
+            global_token_ids = global_tokens # Keep Tensor for detokenization
+
+            global_tokens_str = "".join([f"<|bicodec_global_{i}|>" for i in global_tokens.squeeze().tolist()])
+            if prompt_text and len(prompt_text) > 1:
+                semantic_tokens_str = "".join([f"<|bicodec_semantic_{i}|>" for i in semantic_tokens.squeeze().tolist()])
+                llm_prompt_parts = [
+                    TASK_TOKEN_MAP["tts"], "<|start_content|>", prompt_text, text, "<|end_content|>",
+                    "<|start_global_token|>", global_tokens_str, "<|end_global_token|>",
+                    "<|start_semantic_token|>", semantic_tokens_str,
+                ]
+            else:
+                 llm_prompt_parts = [
+                    TASK_TOKEN_MAP["tts"], "<|start_content|>", text, "<|end_content|>",
+                    "<|start_global_token|>", global_tokens_str, "<|end_global_token|>",
+                 ]
+            llm_prompt_string = "".join(llm_prompt_parts)
+        elif gender is not None and pitch is not None and speed is not None:
+            # Voice Creation Mode
+            logger.info(f"Preprocessing for Voice Creation (gender: {gender}, pitch: {pitch}, speed: {speed})")
+            if gender not in GENDER_MAP: raise ValueError(f"Invalid gender: {gender}")
+            if pitch not in LEVELS_MAP: raise ValueError(f"Invalid pitch: {pitch}")
+            if speed not in LEVELS_MAP: raise ValueError(f"Invalid speed: {speed}")
+
+            gender_id = GENDER_MAP[gender]
+            pitch_level_id = LEVELS_MAP[pitch]
+            speed_level_id = LEVELS_MAP[speed]
+            attribute_tokens = f"<|gender_{gender_id}|><|pitch_label_{pitch_level_id}|><|speed_label_{speed_level_id}|>"
+            llm_prompt_parts = [
+                TASK_TOKEN_MAP["controllable_tts"], "<|start_content|>", text, "<|end_content|>",
+                "<|start_style_label|>", attribute_tokens, "<|end_style_label|>",
+            ]
+            llm_prompt_string = "".join(llm_prompt_parts)
+        else:
+             raise ValueError("Pipeline requires 'prompt_speech_path' (for cloning) or 'gender', 'pitch', 'speed' (for creation).")
+        # --- End prompt building logic ---
+
+        # Tokenize the final prompt for the LLM
+        # Use self.tokenizer (set by base Pipeline class)
+        model_inputs = self.tokenizer(llm_prompt_string, return_tensors=self.framework, padding=False)
+
+        return {"model_inputs": model_inputs, "global_token_ids_prompt": global_token_ids}
+
+
+    def _forward(self, model_inputs, **forward_kwargs) -> dict:
+        """
+        Passes model_inputs to the model's LLM generate method with forward_kwargs.
+
+        Args:
+            model_inputs (dict): Output from `preprocess`.
+            forward_kwargs (dict): Generation arguments from `_sanitize_parameters`.
+
+        Returns:
+            dict: Containing `generated_ids`, `input_ids_len`, and context (`global_token_ids_prompt`).
+        """
+        llm_inputs = model_inputs["model_inputs"]
+        global_token_ids_prompt = model_inputs.get("global_token_ids_prompt")
+
+        # Move inputs to the correct device (self.device is set by base Pipeline class)
+        llm_inputs = {k: v.to(self.device) for k, v in llm_inputs.items()}
+        input_ids = llm_inputs["input_ids"]
+        input_ids_len = input_ids.shape[-1]
+
+        # Combine LLM inputs and generation arguments
+        generate_kwargs = {**llm_inputs, **forward_kwargs}
+
+        logger.info(f"Generating tokens with args: {forward_kwargs}")
+        # Use the model's LLM component (self.model is set by base Pipeline class)
+        with torch.no_grad():
+            generated_ids = self.model.llm.generate(**generate_kwargs)
+
+        # Prepare output dict to pass to postprocess
+        output_dict = {
+            "generated_ids": generated_ids,
+            "input_ids_len": input_ids_len,
+        }
+        if global_token_ids_prompt is not None:
+            output_dict["global_token_ids_prompt"] = global_token_ids_prompt
+
+        return output_dict
+
+    def postprocess(self, model_outputs, **postprocess_kwargs) -> dict:
+        """
+        Transforms model outputs (from _forward) into the final audio dictionary.
+
+        Args:
+            model_outputs (dict): Dictionary from `_forward`.
+            postprocess_kwargs (dict): Arguments relevant to postprocessing (currently none).
+
+        Returns:
+            dict: Containing `audio` (np.ndarray) and `sampling_rate` (int).
+        """
+        generated_ids = model_outputs["generated_ids"]
+        input_ids_len = model_outputs["input_ids_len"]
+        global_token_ids_prompt = model_outputs.get("global_token_ids_prompt")
+
+        # --- Logic to extract tokens and detokenize ---
+        output_ids = generated_ids[0, input_ids_len:] # Assumes batch size 1
+        # Use self.tokenizer (set by base Pipeline class)
+        predicts_text = self.tokenizer.decode(output_ids, skip_special_tokens=True)
+
+        semantic_matches = re.findall(r"bicodec_semantic_(\d+)", predicts_text)
+        if not semantic_matches:
+             logger.warning("No semantic tokens found. Returning empty audio.")
+             # Use self.model.config for sampling rate
+             return {"audio": np.array([], dtype=np.float32), "sampling_rate": self.model.config.sample_rate}
+
+        pred_semantic_ids = torch.tensor([int(token) for token in semantic_matches]).long().unsqueeze(0).to(self.device)
+
+        if global_token_ids_prompt is not None:
+             # Voice Cloning: Use prompt tokens
+             global_token_ids = global_token_ids_prompt.to(self.device)
+             logger.info("Using global tokens from prompt.")
+        else:
+             # Voice Creation: Extract generated tokens
+             global_matches = re.findall(r"bicodec_global_(\d+)", predicts_text)
+             if not global_matches:
+                  raise ValueError("Voice creation failed: No bicodec_global tokens found.")
+             global_token_ids = torch.tensor([int(token) for token in global_matches]).long().unsqueeze(0).to(self.device)
+             if global_token_ids.ndim == 2: global_token_ids = global_token_ids.unsqueeze(1)
+             logger.info("Using global tokens from generated text.")
+
+        # Use the MODEL's method for detokenization (self.model is set by base Pipeline class)
+        wav_np = self.model._detokenize_audio(global_token_ids, pred_semantic_ids)
+        logger.info(f"Generated audio shape: {wav_np.shape}")
+        # --- End detokenization logic ---
+
+        # Return final output dictionary
+        return {"audio": wav_np, "sampling_rate": self.model.config.sample_rate}
+
+
+
+# --- Add Registration Code Here ---
+# This code will execute when this file is loaded via trust_remote_code
+try:
+    from transformers.pipelines import PIPELINE_REGISTRY
+    from transformers import AutoModel # Use AutoModel for registration
+
+    print(f"Registering SparkTTSPipeline for task 'text-to-speech' from pipeline_spark_tts.py...")
+    PIPELINE_REGISTRY.register_pipeline(
+        "text-to-speech",                 # Task name
+        pipeline_class=SparkTTSPipeline,  # The class defined above
+        pt_model=AutoModel,               # Compatible PT AutoModel class
+        # tf_model=None,                  # Add TF class if needed
+    )
+    print("Pipeline registration call completed successfully.")
+except ImportError:
+     # Handle potential import error if transformers structure changes
+     print("WARNING: Could not import PIPELINE_REGISTRY or AutoModel. Pipeline registration failed.")
+except Exception as e:
+     print(f"ERROR: An unexpected error occurred during pipeline registration: {e}")
+# --- End Registration Code ---

processing_spark_tts.py

@@ -0,0 +1,345 @@
+# coding=utf-8
+# Copyright 2024 The SparkAudio Authors and The HuggingFace Inc. team. All rights reserved.
+# ... (license) ...
+"""Processor class for SparkTTS."""
+
+import torch
+import re
+import numpy as np
+import warnings
+from typing import Optional, Dict, Any, Union, List, Tuple
+from pathlib import Path
+
+from transformers.processing_utils import ProcessorMixin
+from transformers.feature_extraction_utils import FeatureExtractionMixin
+from transformers.tokenization_utils_base import BatchEncoding, PreTrainedTokenizerBase
+from transformers import AutoTokenizer, Wav2Vec2FeatureExtractor
+from transformers.utils import logging
+
+# Import necessary items directly or ensure they are available via model reference
+# Note: Avoid direct model imports here if possible, rely on the model reference.
+# from .modeling_spark_tts import SparkTTSModel # Avoid direct model import if possible
+from .configuration_spark_tts import SparkTTSConfig # Config is okay
+
+# Import utils needed for prompt formatting (assuming they are merged into modeling)
+# We'll access them via the model reference if needed, or duplicate simple ones like token maps.
+
+logger = logging.get_logger(__name__)
+
+# --- Token Maps (Duplicate here for direct use in processor) ---
+TASK_TOKEN_MAP = {
+    "tts": "<|task_tts|>",
+    "controllable_tts": "<|task_controllable_tts|>",
+    # Add other tasks if needed by processor logic
+}
+LEVELS_MAP = {"very_low": 0, "low": 1, "moderate": 2, "high": 3, "very_high": 4}
+GENDER_MAP = {"female": 0, "male": 1}
+# --- End Token Maps ---
+
+
+class SparkTTSProcessor(ProcessorMixin):
+    r"""
+    Constructs a SparkTTS processor which wraps a text tokenizer and an audio feature extractor
+    into a single processor.
+
+    [`SparkTTSProcessor`] offers all the functionalities of [`AutoTokenizer`] and [`Wav2Vec2FeatureExtractor`].
+    It processes text input for the LLM and prepares audio inputs if needed (delegating actual audio tokenization
+    to the model). It also handles decoding the final output.
+
+    Args:
+        tokenizer (`PreTrainedTokenizerBase`):
+            An instance of [`AutoTokenizer`]. The tokenizer is used to encode the prompt text.
+        feature_extractor (`Wav2Vec2FeatureExtractor`):
+            An instance of [`Wav2Vec2FeatureExtractor`]. The feature extractor is used to processor reference audio
+            (though the main processing happens inside the model).
+        model (`PreTrainedModel`, *optional*):
+            A reference to the loaded `SparkTTSModel`. This is REQUIRED for voice cloning (prompt audio processing)
+            and final audio decoding, as these steps rely on the model's internal BiCodec and Wav2Vec2 components.
+            Set this using `processor.model = model` after loading both.
+        config (`SparkTTSConfig`, *optional*):
+             The configuration object, needed for parameters like sample_rate. Can often be inferred from the model.
+    """
+    attributes = ["tokenizer", "feature_extractor"]
+    tokenizer_class = ("Qwen2TokenizerFast", "Qwen2Tokenizer") # Specify the underlying tokenizer type
+    feature_extractor_class = ("Wav2Vec2FeatureExtractor",) # Specify the underlying feature extractor type
+
+    def __init__(self, tokenizer=None, feature_extractor=None, model=None, config=None, **kwargs):
+        if tokenizer is None:
+            raise ValueError("SparkTTSProcessor requires a `tokenizer`.")
+        if feature_extractor is None:
+            # Attempt to load default if path is known or provide clearer error
+             raise ValueError("SparkTTSProcessor requires a `feature_extractor` (Wav2Vec2FeatureExtractor).")
+
+        super().__init__(tokenizer, feature_extractor)
+        self.model = model # Store model reference (can be None initially)
+        self.config = config # Store config reference
+
+        # Get sampling rate from config if available
+        self.sampling_rate = None
+        if self.config and hasattr(self.config, 'sample_rate'):
+            self.sampling_rate = self.config.sample_rate
+        elif self.model and hasattr(self.model, 'config') and hasattr(self.model.config, 'sample_rate'):
+             self.sampling_rate = self.model.config.sample_rate
+        else:
+             # Try feature extractor default, or raise warning
+             if hasattr(self.feature_extractor, 'sampling_rate'):
+                 self.sampling_rate = self.feature_extractor.sampling_rate
+             else:
+                 logger.warning("Could not determine sampling rate. Defaulting to 16000. Set `processor.sampling_rate` manually if needed.")
+                 self.sampling_rate = 16000
+
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
+        """
+        Instantiate a [`SparkTTSProcessor`] from a pretrained processor configuration.
+
+        Args:
+            pretrained_model_name_or_path (`str` or `os.PathLike`):
+                This can be either:
+                - a string, the *model id* of a pretrained processor hosted inside a model repo on huggingface.co.
+                - a path to a *directory* containing processor files saved using the `save_pretrained()` method,
+                  e.g., `./my_model_directory/`.
+            **kwargs:
+                Additional keyword arguments passed along to both `AutoTokenizer.from_pretrained()` and
+                `AutoFeatureExtractor.from_pretrained()`.
+        """
+        config = kwargs.pop("config", None)
+        if config is None:
+            # Try loading the specific config first
+            try:
+                 config = SparkTTSConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
+            except Exception:
+                 logger.warning(f"Could not load SparkTTSConfig from {pretrained_model_name_or_path}. Processor might lack some config values.")
+                 config = None
+
+
+        # Resolve component paths relative to the main path
+        def _resolve_path(sub_path):
+            p = Path(sub_path)
+            if p.is_absolute():
+                return str(p)
+            # Try resolving relative to the main path if it's a directory
+            main_path = Path(pretrained_model_name_or_path)
+            if main_path.is_dir():
+                 resolved = main_path / p
+                 if resolved.exists():
+                      return str(resolved)
+            # Fallback to assuming sub_path is relative within a repo structure (might fail for local non-dirs)
+            return sub_path
+
+        # Determine paths from config or assume defaults
+        llm_tokenizer_path = "./LLM"
+        w2v_processor_path = "./wav2vec2-large-xlsr-53"
+        if config:
+            llm_tokenizer_path = getattr(config, 'llm_model_name_or_path', llm_tokenizer_path)
+            w2v_processor_path = getattr(config, 'wav2vec2_model_name_or_path', w2v_processor_path)
+
+        resolved_tokenizer_path = _resolve_path(llm_tokenizer_path)
+        resolved_w2v_path = _resolve_path(w2v_processor_path)
+
+        try:
+            tokenizer = AutoTokenizer.from_pretrained(resolved_tokenizer_path, **kwargs)
+        except Exception as e:
+            raise OSError(f"Could not load tokenizer from {resolved_tokenizer_path}. Ensure path is correct and files exist. Original error: {e}")
+
+        try:
+            feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(resolved_w2v_path, **kwargs)
+        except Exception as e:
+            raise OSError(f"Could not load feature extractor from {resolved_w2v_path}. Ensure path is correct and files exist. Original error: {e}")
+
+        # The 'model' attribute will be set later externally
+        return cls(tokenizer=tokenizer, feature_extractor=feature_extractor, config=config)
+
+
+    def __call__(self, text: str = None,
+                 prompt_speech_path: Optional[str] = None,
+                 prompt_text: Optional[str] = None,
+                 gender: Optional[str] = None,
+                 pitch: Optional[str] = None,
+                 speed: Optional[str] = None,
+                 return_tensors: Optional[str] = "pt",
+                 **kwargs) -> BatchEncoding:
+        """
+        Main method to process inputs for the SparkTTS model.
+
+        Args:
+            text (`str`): The text to be synthesized.
+            prompt_speech_path (`str`, *optional*): Path to prompt audio for voice cloning.
+            prompt_text (`str`, *optional*): Transcript of prompt audio.
+            gender (`str`, *optional*): Target gender ('male' or 'female') for voice creation.
+            pitch (`str`, *optional*): Target pitch level ('very_low'...'very_high') for voice creation.
+            speed (`str`, *optional*): Target speed level ('very_low'...'very_high') for voice creation.
+            return_tensors (`str`, *optional*, defaults to `"pt"`):
+                Framework of the returned tensors (`"pt"` for PyTorch, `"np"` for NumPy).
+            **kwargs: Additional arguments (currently ignored).
+
+        Returns:
+            `BatchEncoding`: A dictionary containing the `input_ids`, `attention_mask`, and optionally
+                             `global_token_ids_prompt` ready for the model's `.generate()` method.
+        """
+        if text is None:
+            raise ValueError("`text` input must be provided.")
+
+        global_token_ids_prompt = None
+        llm_prompt_string = ""
+
+        if prompt_speech_path is not None:
+            # --- Voice Cloning Mode ---
+            if self.model is None:
+                 raise ValueError("Processor requires a loaded `model` reference (`processor.model = model`) for voice cloning.")
+            if not hasattr(self.model, '_tokenize_audio'):
+                raise AttributeError("The provided model object does not have the required '_tokenize_audio' method.")
+
+            logger.info(f"Processing prompt audio: {prompt_speech_path}")
+            # Delegate audio tokenization to the model
+            try:
+                # _tokenize_audio returns (global_tokens, semantic_tokens)
+                global_tokens, semantic_tokens = self.model._tokenize_audio(prompt_speech_path)
+                global_token_ids_prompt = global_tokens # Keep for decoding stage
+            except Exception as e:
+                 logger.error(f"Error tokenizing prompt audio: {e}", exc_info=True)
+                 raise RuntimeError(f"Failed to process prompt audio file: {prompt_speech_path}. Check file integrity and model compatibility.") from e
+
+            # Format prompt string using token maps
+            global_tokens_str = "".join([f"<|bicodec_global_{i}|>" for i in global_tokens.squeeze().tolist()])
+
+            if prompt_text and len(prompt_text) > 1:
+                semantic_tokens_str = "".join([f"<|bicodec_semantic_{i}|>" for i in semantic_tokens.squeeze().tolist()])
+                llm_prompt_parts = [
+                    TASK_TOKEN_MAP["tts"], "<|start_content|>", prompt_text, text, "<|end_content|>",
+                    "<|start_global_token|>", global_tokens_str, "<|end_global_token|>",
+                    "<|start_semantic_token|>", semantic_tokens_str,
+                ]
+            else:
+                llm_prompt_parts = [
+                    TASK_TOKEN_MAP["tts"], "<|start_content|>", text, "<|end_content|>",
+                    "<|start_global_token|>", global_tokens_str, "<|end_global_token|>",
+                ]
+            llm_prompt_string = "".join(llm_prompt_parts)
+
+        elif gender is not None and pitch is not None and speed is not None:
+            # --- Voice Creation Mode ---
+            if gender not in GENDER_MAP: raise ValueError(f"Invalid gender '{gender}'.")
+            if pitch not in LEVELS_MAP: raise ValueError(f"Invalid pitch '{pitch}'.")
+            if speed not in LEVELS_MAP: raise ValueError(f"Invalid speed '{speed}'.")
+
+            gender_id = GENDER_MAP[gender]
+            pitch_level_id = LEVELS_MAP[pitch]
+            speed_level_id = LEVELS_MAP[speed]
+
+            attribute_tokens = f"<|gender_{gender_id}|><|pitch_label_{pitch_level_id}|><|speed_label_{speed_level_id}|>"
+
+            llm_prompt_parts = [
+                TASK_TOKEN_MAP["controllable_tts"], "<|start_content|>", text, "<|end_content|>",
+                "<|start_style_label|>", attribute_tokens, "<|end_style_label|>",
+            ]
+            llm_prompt_string = "".join(llm_prompt_parts)
+            # No global_token_ids_prompt needed
+
+        else:
+            raise ValueError("Processor requires either 'prompt_speech_path' (for cloning) or 'gender', 'pitch', and 'speed' (for creation).")
+
+        # Tokenize the final LLM prompt string
+        inputs = self.tokenizer(llm_prompt_string, return_tensors=return_tensors, padding=False, truncation=False)
+
+        # Add prompt global tokens to the output if they exist (for passing to decode)
+        if global_token_ids_prompt is not None:
+             inputs["global_token_ids_prompt"] = global_token_ids_prompt
+
+        return inputs
+
+    def decode(self,
+               generated_ids: Union[List[int], np.ndarray, torch.Tensor],
+               global_token_ids_prompt: Optional[torch.Tensor] = None,
+               input_ids_len: Optional[int] = None,
+               skip_special_tokens: bool = True) -> Dict[str, Any]:
+        """
+        Decodes the raw token IDs generated by the model into an audio waveform.
+
+        Args:
+            generated_ids (`Union[List[int], np.ndarray, torch.Tensor]`):
+                The token IDs generated by the `model.generate()` method. Assumed to be a single sequence (batch size 1).
+            global_token_ids_prompt (`torch.Tensor`, *optional*):
+                The global tokens obtained from the prompt audio during preprocessing (needed for voice cloning).
+                Should be passed from the `__call__` output.
+            input_ids_len (`int`, *optional*):
+                The length of the original prompt `input_ids`. If provided, the prompt part will be stripped from
+                `generated_ids` before decoding the text representation. If None, assumes `generated_ids` contains
+                *only* the generated part.
+            skip_special_tokens (`bool`, *optional*, defaults to `True`):
+                Whether to skip special tokens when decoding the text representation for parsing.
+
+        Returns:
+            `Dict[str, Any]`: A dictionary containing:
+                - `audio` (`np.ndarray`): The generated audio waveform.
+                - `sampling_rate` (`int`): The sampling rate of the audio.
+        """
+        if self.model is None:
+            raise ValueError("Processor requires a loaded `model` reference (`processor.model = model`) for decoding.")
+        if not hasattr(self.model, '_detokenize_audio'):
+            raise AttributeError("The provided model object does not have the required '_detokenize_audio' method.")
+        if self.sampling_rate is None:
+             raise ValueError("Processor could not determine sampling_rate. Set `processor.sampling_rate`.")
+
+        # Ensure generated_ids is a tensor on the correct device
+        if isinstance(generated_ids, (list, np.ndarray)):
+             output_ids_tensor = torch.tensor(generated_ids)
+        else:
+             output_ids_tensor = generated_ids
+
+        # Remove prompt if input_ids_len is provided
+        if input_ids_len is not None:
+             # Handle potential batch dimension if present (though usually not for decode)
+             if output_ids_tensor.ndim > 1:
+                  output_ids = output_ids_tensor[0, input_ids_len:]
+             else:
+                  output_ids = output_ids_tensor[input_ids_len:]
+        else:
+             if output_ids_tensor.ndim > 1:
+                  output_ids = output_ids_tensor[0]
+             else:
+                  output_ids = output_ids_tensor
+
+        if output_ids.numel() == 0:
+            logger.warning("Received empty generated IDs after removing prompt. Returning empty audio.")
+            return {"audio": np.array([], dtype=np.float32), "sampling_rate": self.sampling_rate}
+
+        # Decode the text representation to parse tokens
+        predicts_text = self.tokenizer.decode(output_ids, skip_special_tokens=skip_special_tokens)
+
+        # Extract semantic tokens
+        semantic_matches = re.findall(r"bicodec_semantic_(\d+)", predicts_text)
+        if not semantic_matches:
+             logger.warning("No semantic tokens found in the generated output text. Cannot synthesize audio.")
+             return {"audio": np.array([], dtype=np.float32), "sampling_rate": self.sampling_rate}
+        # Use model's device for tensors
+        device = self.model.device
+        pred_semantic_ids = torch.tensor([int(token) for token in semantic_matches], dtype=torch.long, device=device).unsqueeze(0) # Add batch dim
+
+        # Determine global tokens
+        if global_token_ids_prompt is not None:
+            # Voice Cloning: Use prompt global tokens
+            global_token_ids = global_token_ids_prompt.to(device)
+             # Ensure correct shape (B, T_token, Q) or (B, D) - BiCodec detokenize needs to handle this
+            if global_token_ids.ndim == 2: # If (B, D), maybe unsqueeze? Check BiCodec.detokenize expectation
+                 global_token_ids = global_token_ids.unsqueeze(1) # Assume (B, 1, D) might be needed
+        else:
+            # Voice Creation: Parse global tokens from generated text
+            global_matches = re.findall(r"bicodec_global_(\d+)", predicts_text)
+            if not global_matches:
+                 logger.error("Voice creation failed: No global tokens found in generated text.")
+                 raise ValueError("Voice creation failed: Could not find bicodec_global tokens in the LLM output.")
+            global_token_ids = torch.tensor([int(token) for token in global_matches], dtype=torch.long, device=device).unsqueeze(0) # Add batch dim
+            # Add sequence dimension if needed (check BiCodec.detokenize)
+            if global_token_ids.ndim == 2:
+                 global_token_ids = global_token_ids.unsqueeze(1) # Assume (B, 1, D)
+
+        # Detokenize audio using the model's method
+        try:
+            wav_np = self.model._detokenize_audio(global_token_ids, pred_semantic_ids)
+        except Exception as e:
+             logger.error(f"Error during audio detokenization: {e}", exc_info=True)
+             raise RuntimeError("Failed to synthesize audio waveform from generated tokens.") from e
+
+        return {"audio": wav_np, "sampling_rate": self.sampling_rate}

wav2vec2-large-xlsr-53/README.md

@@ -0,0 +1,29 @@
+---
+language: multilingual
+datasets:
+- common_voice
+tags:
+- speech
+license: apache-2.0
+---
+
+# Wav2Vec2-XLSR-53 
+
+[Facebook's XLSR-Wav2Vec2](https://ai.facebook.com/blog/wav2vec-20-learning-the-structure-of-speech-from-raw-audio/)
+
+The base model pretrained on 16kHz sampled speech audio. When using the model make sure that your speech input is also sampled at 16Khz. Note that this model should be fine-tuned on a downstream task, like Automatic Speech Recognition. Check out [this blog](https://huggingface.co/blog/fine-tune-wav2vec2-english) for more information.
+
+[Paper](https://arxiv.org/abs/2006.13979)
+
+Authors: Alexis Conneau, Alexei Baevski, Ronan Collobert, Abdelrahman Mohamed, Michael Auli
+
+**Abstract**
+This paper presents XLSR which learns cross-lingual speech representations by pretraining a single model from the raw waveform of speech in multiple languages. We build on wav2vec 2.0 which is trained by solving a contrastive task over masked latent speech representations and jointly learns a quantization of the latents shared across languages. The resulting model is fine-tuned on labeled data and experiments show that cross-lingual pretraining significantly outperforms monolingual pretraining. On the CommonVoice benchmark, XLSR shows a relative phoneme error rate reduction of 72% compared to the best known results. On BABEL, our approach improves word error rate by 16% relative compared to a comparable system. Our approach enables a single multilingual speech recognition model which is competitive to strong individual models. Analysis shows that the latent discrete speech representations are shared across languages with increased sharing for related languages. We hope to catalyze research in low-resource speech understanding by releasing XLSR-53, a large model pretrained in 53 languages.
+
+The original model can be found under https://github.com/pytorch/fairseq/tree/master/examples/wav2vec#wav2vec-20.
+
+# Usage
+
+See [this notebook](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/Fine_Tune_XLSR_Wav2Vec2_on_Turkish_ASR_with_%F0%9F%A4%97_Transformers.ipynb) for more information on how to fine-tune the model.
+
+![model image](https://raw.githubusercontent.com/patrickvonplaten/scientific_images/master/xlsr_wav2vec2.png)

wav2vec2-large-xlsr-53/config.json

@@ -0,0 +1,83 @@
+{
+  "activation_dropout": 0.0,
+  "apply_spec_augment": true,
+  "architectures": [
+    "Wav2Vec2ForPreTraining"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": 1,
+  "codevector_dim": 768,
+  "contrastive_logits_temperature": 0.1,
+  "conv_bias": true,
+  "conv_dim": [
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512
+  ],
+  "conv_kernel": [
+    10,
+    3,
+    3,
+    3,
+    3,
+    2,
+    2
+  ],
+  "conv_stride": [
+    5,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2
+  ],
+  "ctc_loss_reduction": "sum",
+  "ctc_zero_infinity": false,
+  "diversity_loss_weight": 0.1,
+  "do_stable_layer_norm": true,
+  "eos_token_id": 2,
+  "feat_extract_activation": "gelu",
+  "feat_extract_dropout": 0.0,
+  "feat_extract_norm": "layer",
+  "feat_proj_dropout": 0.1,
+  "feat_quantizer_dropout": 0.0,
+  "final_dropout": 0.0,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout": 0.1,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "layerdrop": 0.1,
+  "mask_channel_length": 10,
+  "mask_channel_min_space": 1,
+  "mask_channel_other": 0.0,
+  "mask_channel_prob": 0.0,
+  "mask_channel_selection": "static",
+  "mask_feature_length": 10,
+  "mask_feature_prob": 0.0,
+  "mask_time_length": 10,
+  "mask_time_min_space": 1,
+  "mask_time_other": 0.0,
+  "mask_time_prob": 0.075,
+  "mask_time_selection": "static",
+  "model_type": "wav2vec2",
+  "num_attention_heads": 16,
+  "num_codevector_groups": 2,
+  "num_codevectors_per_group": 320,
+  "num_conv_pos_embedding_groups": 16,
+  "num_conv_pos_embeddings": 128,
+  "num_feat_extract_layers": 7,
+  "num_hidden_layers": 24,
+  "num_negatives": 100,
+  "pad_token_id": 0,
+  "proj_codevector_dim": 768,
+  "transformers_version": "4.7.0.dev0",
+  "vocab_size": 32
+}

wav2vec2-large-xlsr-53/preprocessor_config.json

@@ -0,0 +1,9 @@
+{
+  "do_normalize": true,
+  "feature_extractor_type": "Wav2Vec2FeatureExtractor",
+  "feature_size": 1,
+  "padding_side": "right",
+  "padding_value": 0,
+  "return_attention_mask": true,
+  "sampling_rate": 16000
+}

wav2vec2-large-xlsr-53/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:314340227371a608f71adcd5f0de5933824fe77e55822aa4b24dba9c1c364dcb
+size 1269737156