Upload folder using huggingface_hub

README.md

@@ -20,21 +20,27 @@ This Hugging Face Space automatically installs dependencies from requirements.tx
 
 ### Installation Process
 
-For clearer dependency management, the installation is split into multiple files:
+The project uses a single consolidated requirements file that maintains the proper installation order of dependencies:
 
-1. **Base Dependencies (requirements-base.txt)**:
-   - Core packages like torch, transformers, accelerate, etc.
-   - Install with: `pip install -r requirements-base.txt`
-
-2. **Standard Dependencies (requirements.txt)**:
-   - References base requirements and adds additional packages
+1. **All Dependencies (requirements.txt)**:
+   - Contains all required packages in the correct installation order
    - Install with: `pip install -r requirements.txt`
+   - The file is organized with clear sections:
+     - Base dependencies (installed first)
+     - Main dependencies
+     - Optional dependencies (commented out by default)
 
-3. **Flash Attention (requirements-flash.txt)** (Optional):
+2. **Flash Attention** (Optional):
    - For faster attention computation
-   - Install with: `pip install -r requirements-flash.txt --no-build-isolation`
+   - Install with: `pip install flash-attn==2.5.2 --no-build-isolation`
+   - Or uncomment the flash-attn line in requirements.txt
+
+3. **Automated Installation**:
+   - For convenience, you can use the included script:
+   - Basic install: `python install_requirements.py`
+   - With flash-attn: `python install_requirements.py --flash`
 
-Using this staged approach helps prevent dependency conflicts and installation issues.
+This approach simplifies dependency management while still maintaining proper installation order.
 
 ### Essential Dependencies
 

install_requirements.py

@@ -22,43 +22,29 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 def install_requirements(include_flash=False):
-    """Install requirements in the correct order."""
+    """Install requirements from the consolidated requirements file."""
     current_dir = Path(__file__).parent
-    base_req_path = current_dir / "requirements-base.txt"
-    main_req_path = current_dir / "requirements.txt"
-    flash_req_path = current_dir / "requirements-flash.txt"
+    req_path = current_dir / "requirements.txt"
     
-    if not base_req_path.exists():
-        logger.error(f"Base requirements file not found: {base_req_path}")
+    if not req_path.exists():
+        logger.error(f"Requirements file not found: {req_path}")
         return False
     
-    if not main_req_path.exists():
-        logger.error(f"Main requirements file not found: {main_req_path}")
-        return False
-    
-    logger.info("Installing dependencies in sequential order...")
+    logger.info("Installing dependencies from consolidated requirements file...")
     
     try:
-        # Step 1: Install base requirements
-        logger.info(f"Step 1: Installing base requirements from {base_req_path}")
-        subprocess.run([sys.executable, "-m", "pip", "install", "-r", str(base_req_path)], 
-                      check=True)
-        logger.info("Base requirements installed successfully")
-        
-        # Step 2: Install main requirements
-        logger.info(f"Step 2: Installing additional requirements from {main_req_path}")
-        subprocess.run([sys.executable, "-m", "pip", "install", "-r", str(main_req_path)], 
+        # Install all requirements
+        logger.info(f"Installing requirements from {req_path}")
+        subprocess.run([sys.executable, "-m", "pip", "install", "-r", str(req_path)], 
                       check=True)
-        logger.info("Additional requirements installed successfully")
+        logger.info("Main requirements installed successfully")
         
-        # Step 3: Optionally install flash-attention
-        if include_flash and flash_req_path.exists():
-            logger.info(f"Step 3: Installing flash-attention from {flash_req_path}")
-            subprocess.run([sys.executable, "-m", "pip", "install", "-r", str(flash_req_path), "--no-build-isolation"], 
+        # Optionally install flash-attention
+        if include_flash:
+            logger.info("Installing flash-attention...")
+            subprocess.run([sys.executable, "-m", "pip", "install", "flash-attn==2.5.2", "--no-build-isolation"], 
                           check=True)
             logger.info("Flash-attention installed successfully")
-        elif include_flash:
-            logger.warning(f"Flash requirements file not found: {flash_req_path}")
         
         logger.info("All required packages installed successfully!")
         return True

requirements.txt

@@ -1,6 +1,18 @@
--r requirements-base.txt
+# BASE REQUIREMENTS - Install these critical dependencies first
+# ---------------------------------------------------------------------
+torch>=2.0.0
+accelerate>=0.27.0
+bitsandbytes>=0.41.0
+transformers>=4.36.0
+datasets>=2.15.0
+huggingface-hub>=0.19.0
+tensorboard>=2.15.0
+
+# MAIN REQUIREMENTS - Install these after base dependencies
+# ---------------------------------------------------------------------
 einops>=0.7.0
 filelock>=3.13.1
+gradio>=5.17.0
 matplotlib>=3.7.0
 numpy>=1.24.0
 packaging>=23.0
@@ -15,3 +27,7 @@ sentencepiece>=0.1.99
 tqdm>=4.65.0
 typing-extensions>=4.8.0
 unsloth>=2024.3
+
+# OPTIONAL DEPENDENCIES - Install these last (if needed)
+# ---------------------------------------------------------------------
+# flash-attn==2.5.2

update_space.py

@@ -99,29 +99,26 @@ def verify_configs():
             raise ValueError(f"Invalid JSON in {json_file}: {e}")
 
 def update_requirements():
-    """Update requirements.txt with necessary packages using a two-stage installation process."""
-    logger.info("Setting up requirements files for sequential installation...")
+    """Update consolidated requirements.txt with all necessary packages in the correct order."""
+    logger.info("Setting up consolidated requirements file...")
     current_dir = Path(__file__).parent
-    base_req_path = current_dir / "requirements-base.txt"
-    main_req_path = current_dir / "requirements.txt"
-    flash_req_path = current_dir / "requirements-flash.txt"
+    req_path = current_dir / "requirements.txt"
     
-    # First ensure base requirements exist
-    required_base_packages = {
+    # All required packages in the correct installation order
+    required_packages = [
+        # Base requirements (install first)
         "torch>=2.0.0",
-        "transformers>=4.36.0",
         "accelerate>=0.27.0",
-        "bitsandbytes>=0.41.0",
-        "tensorboard>=2.15.0",
-        "gradio>=5.17.0",
+        "bitsandbytes>=0.41.0", 
+        "transformers>=4.36.0",
+        "datasets>=2.15.0",
         "huggingface-hub>=0.19.0",
-        "datasets>=2.15.0"
-    }
-    
-    # Additional packages for main requirements
-    required_additional_packages = {
+        "tensorboard>=2.15.0",
+        
+        # Main requirements (install second)
         "einops>=0.7.0",
         "filelock>=3.13.1",
+        "gradio>=5.17.0",
         "matplotlib>=3.7.0",
         "numpy>=1.24.0",
         "packaging>=23.0",
@@ -136,43 +133,43 @@ def update_requirements():
         "tqdm>=4.65.0",
         "typing-extensions>=4.8.0",
         "unsloth>=2024.3"
-    }
-    
-    # Read existing base requirements
-    existing_requirements = set()
-    if base_req_path.exists():
-        with open(base_req_path) as f:
-            existing_requirements = {line.strip() for line in f if line.strip() and not line.startswith('-r')}
+    ]
     
-    # Add new requirements
-    updated_requirements = existing_requirements.union(required_base_packages)
+    # Optional packages (commented out by default)
+    optional_packages = [
+        "flash-attn==2.5.2"
+    ]
     
-    # 1. Write updated base requirements
-    with open(base_req_path, 'w') as f:
-        # Ensure torch is first
-        torch_req = next((req for req in updated_requirements if req.startswith("torch")), "torch>=2.0.0")
-        f.write(f"{torch_req}\n")
+    # Create consolidated requirements file
+    with open(req_path, 'w') as f:
+        f.write("# BASE REQUIREMENTS - Install these critical dependencies first\n")
+        f.write("# ---------------------------------------------------------------------\n")
         
-        # Write all other requirements (excluding torch)
-        for req in sorted(r for r in updated_requirements if not r.startswith("torch")):
+        # Write base dependencies first
+        for i, req in enumerate(required_packages):
+            if i == 7:  # After base requirements
+                f.write("\n# MAIN REQUIREMENTS - Install these after base dependencies\n")
+                f.write("# ---------------------------------------------------------------------\n")
             f.write(f"{req}\n")
+        
+        # Add optional dependencies section
+        f.write("\n# OPTIONAL DEPENDENCIES - Install these last (if needed)\n")
+        f.write("# ---------------------------------------------------------------------\n")
+        for opt_pkg in optional_packages:
+            f.write(f"# {opt_pkg}\n")
     
-    # 2. Create main requirements file (references base)
-    with open(main_req_path, 'w') as f:
-        f.write("-r requirements-base.txt\n")
-        for req in sorted(required_additional_packages):
-            f.write(f"{req}\n")
+    logger.info(f"Updated consolidated requirements file at {req_path}")
+    logger.info("Requirements are ordered for proper dependency installation")
     
-    # 3. Create or update flash-attn requirements
-    with open(flash_req_path, 'w') as f:
-        f.write("-r requirements-base.txt\n")
-        f.write("flash-attn==2.5.2\n")
+    # Remove old requirements files if they exist
+    old_files = ["requirements-base.txt", "requirements-flash.txt"]
+    for old_file in old_files:
+        old_path = current_dir / old_file
+        if old_path.exists():
+            old_path.unlink()
+            logger.info(f"Removed old requirements file: {old_file}")
     
-    logger.info("Updated requirements files for sequential installation:")
-    logger.info(f"1. Base requirements in {base_req_path}")
-    logger.info(f"2. Main requirements in {main_req_path}")
-    logger.info(f"3. Flash-attention requirements in {flash_req_path}")
-    logger.info("This ensures packages are installed in the correct order")
+    return True
 
 def create_space(username, space_name):
     """Create or get a Hugging Face Space."""