Fahimeh Orvati Nia
		
	commited on
		
		
					Commit 
							
							·
						
						7ac2007
	
1
								Parent(s):
							
							60e6efb
								
Fix storage limit by caching to /tmp
Browse files- .dockerignore +25 -0
- .gitignore +41 -0
- DEPLOYMENT.md +88 -0
- README.md +22 -3
- cleanup_cache.py +49 -0
- sorghum_pipeline/segmentation/manager.py +7 -2
    	
        .dockerignore
    ADDED
    
    | @@ -0,0 +1,25 @@ | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            # Ignore cache and unnecessary files
         | 
| 2 | 
            +
            __pycache__/
         | 
| 3 | 
            +
            *.pyc
         | 
| 4 | 
            +
            *.pyo
         | 
| 5 | 
            +
            *.pyd
         | 
| 6 | 
            +
            .Python
         | 
| 7 | 
            +
            *.so
         | 
| 8 | 
            +
            *.egg
         | 
| 9 | 
            +
            *.egg-info/
         | 
| 10 | 
            +
            dist/
         | 
| 11 | 
            +
            build/
         | 
| 12 | 
            +
            *.log
         | 
| 13 | 
            +
            .git/
         | 
| 14 | 
            +
            .gitignore
         | 
| 15 | 
            +
            .DS_Store
         | 
| 16 | 
            +
            *.swp
         | 
| 17 | 
            +
            *.swo
         | 
| 18 | 
            +
            *~
         | 
| 19 | 
            +
            .vscode/
         | 
| 20 | 
            +
            .idea/
         | 
| 21 | 
            +
            *.pt
         | 
| 22 | 
            +
            *.pth
         | 
| 23 | 
            +
            *.ckpt
         | 
| 24 | 
            +
            *.weights
         | 
| 25 | 
            +
             | 
    	
        .gitignore
    ADDED
    
    | @@ -0,0 +1,41 @@ | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            # Python
         | 
| 2 | 
            +
            __pycache__/
         | 
| 3 | 
            +
            *.py[cod]
         | 
| 4 | 
            +
            *$py.class
         | 
| 5 | 
            +
            *.so
         | 
| 6 | 
            +
            .Python
         | 
| 7 | 
            +
            env/
         | 
| 8 | 
            +
            venv/
         | 
| 9 | 
            +
            ENV/
         | 
| 10 | 
            +
            .venv
         | 
| 11 | 
            +
            *.egg-info/
         | 
| 12 | 
            +
            dist/
         | 
| 13 | 
            +
            build/
         | 
| 14 | 
            +
             | 
| 15 | 
            +
            # Hugging Face cache
         | 
| 16 | 
            +
            .cache/
         | 
| 17 | 
            +
            transformers_cache/
         | 
| 18 | 
            +
             | 
| 19 | 
            +
            # Model weights
         | 
| 20 | 
            +
            *.pt
         | 
| 21 | 
            +
            *.pth
         | 
| 22 | 
            +
            *.ckpt
         | 
| 23 | 
            +
            *.weights
         | 
| 24 | 
            +
             | 
| 25 | 
            +
            # IDE
         | 
| 26 | 
            +
            .vscode/
         | 
| 27 | 
            +
            .idea/
         | 
| 28 | 
            +
            *.swp
         | 
| 29 | 
            +
            *.swo
         | 
| 30 | 
            +
             | 
| 31 | 
            +
            # OS
         | 
| 32 | 
            +
            .DS_Store
         | 
| 33 | 
            +
            Thumbs.db
         | 
| 34 | 
            +
             | 
| 35 | 
            +
            # Gradio
         | 
| 36 | 
            +
            flagged/
         | 
| 37 | 
            +
            gradio_cached_examples/
         | 
| 38 | 
            +
             | 
| 39 | 
            +
            # Logs
         | 
| 40 | 
            +
            *.log
         | 
| 41 | 
            +
             | 
    	
        DEPLOYMENT.md
    ADDED
    
    | @@ -0,0 +1,88 @@ | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            # Deployment Guide for Hugging Face Spaces
         | 
| 2 | 
            +
             | 
| 3 | 
            +
            ## Storage Limit Error Fix
         | 
| 4 | 
            +
             | 
| 5 | 
            +
            If you see "Workload evicted, storage limit exceeded (50G)", here's how to fix it:
         | 
| 6 | 
            +
             | 
| 7 | 
            +
            ### Quick Fix (Recommended)
         | 
| 8 | 
            +
            The pipeline now uses `/tmp` for caching (ephemeral storage), which resets on each container restart. This should prevent storage buildup.
         | 
| 9 | 
            +
             | 
| 10 | 
            +
            **To apply the fix:**
         | 
| 11 | 
            +
            1. Push the updated code to your Space
         | 
| 12 | 
            +
            2. The Space will rebuild automatically
         | 
| 13 | 
            +
            3. The model will cache to `/tmp` instead of persistent storage
         | 
| 14 | 
            +
             | 
| 15 | 
            +
            ### Manual Cleanup
         | 
| 16 | 
            +
            If your Space is still stuck, you need to clean up old cached files:
         | 
| 17 | 
            +
             | 
| 18 | 
            +
            1. **Go to your Space settings** on Hugging Face
         | 
| 19 | 
            +
            2. **Factory Reboot** your Space:
         | 
| 20 | 
            +
               - Settings → Factory reboot
         | 
| 21 | 
            +
               - This will clear all persistent storage and restart fresh
         | 
| 22 | 
            +
             | 
| 23 | 
            +
            ### Alternative: Upgrade Space Storage
         | 
| 24 | 
            +
            If you need more persistent storage:
         | 
| 25 | 
            +
            1. Go to Settings → Hardware
         | 
| 26 | 
            +
            2. Upgrade to a tier with more storage (costs $$$)
         | 
| 27 | 
            +
             | 
| 28 | 
            +
            ## Storage Optimization Applied
         | 
| 29 | 
            +
             | 
| 30 | 
            +
            The following changes reduce storage usage:
         | 
| 31 | 
            +
             | 
| 32 | 
            +
            ### 1. Cache to /tmp (ephemeral)
         | 
| 33 | 
            +
            ```python
         | 
| 34 | 
            +
            # In sorghum_pipeline/segmentation/manager.py
         | 
| 35 | 
            +
            cache_dir = "/tmp/huggingface_cache"  # Cleared on restart
         | 
| 36 | 
            +
            ```
         | 
| 37 | 
            +
             | 
| 38 | 
            +
            ### 2. Low memory mode
         | 
| 39 | 
            +
            ```python
         | 
| 40 | 
            +
            low_cpu_mem_usage=True  # Reduces peak memory during model load
         | 
| 41 | 
            +
            ```
         | 
| 42 | 
            +
             | 
| 43 | 
            +
            ### 3. Ignore files
         | 
| 44 | 
            +
            - `.dockerignore`: Prevents copying cache/models during build
         | 
| 45 | 
            +
            - `.gitignore`: Prevents committing large files
         | 
| 46 | 
            +
             | 
| 47 | 
            +
            ### 4. Smaller model resolution
         | 
| 48 | 
            +
            - Using 512x512 instead of 1024x1024 for 4x speedup and less memory
         | 
| 49 | 
            +
             | 
| 50 | 
            +
            ## Monitoring Storage
         | 
| 51 | 
            +
             | 
| 52 | 
            +
            To check storage usage in your Space:
         | 
| 53 | 
            +
            1. Open the Space logs
         | 
| 54 | 
            +
            2. Look for disk usage warnings
         | 
| 55 | 
            +
            3. If approaching 50GB, do a factory reboot
         | 
| 56 | 
            +
             | 
| 57 | 
            +
            ## Expected Storage Usage
         | 
| 58 | 
            +
             | 
| 59 | 
            +
            - **BRIA RMBG-2.0 model**: ~350MB (cached to /tmp)
         | 
| 60 | 
            +
            - **PyTorch/Transformers libs**: ~2-3GB
         | 
| 61 | 
            +
            - **Application code**: <50MB
         | 
| 62 | 
            +
            - **Temporary files**: <1GB (cleared after each run)
         | 
| 63 | 
            +
             | 
| 64 | 
            +
            **Total**: ~3-4GB (well under 50GB limit)
         | 
| 65 | 
            +
             | 
| 66 | 
            +
            ## Troubleshooting
         | 
| 67 | 
            +
             | 
| 68 | 
            +
            ### "No space left on device"
         | 
| 69 | 
            +
            - Factory reboot the Space
         | 
| 70 | 
            +
            - Check if any large files were committed to git
         | 
| 71 | 
            +
             | 
| 72 | 
            +
            ### "Model download failed"
         | 
| 73 | 
            +
            - Check HF_TOKEN is set in Space secrets
         | 
| 74 | 
            +
            - Verify internet connectivity in Space
         | 
| 75 | 
            +
             | 
| 76 | 
            +
            ### Slow startup
         | 
| 77 | 
            +
            - First startup downloads model (~350MB)
         | 
| 78 | 
            +
            - Subsequent startups load from /tmp (fast)
         | 
| 79 | 
            +
            - After container restart, re-downloads to /tmp
         | 
| 80 | 
            +
             | 
| 81 | 
            +
            ## Best Practices
         | 
| 82 | 
            +
             | 
| 83 | 
            +
            1. ✅ Use `/tmp` for all caches
         | 
| 84 | 
            +
            2. ✅ Enable `low_cpu_mem_usage=True`
         | 
| 85 | 
            +
            3. ✅ Keep `.dockerignore` and `.gitignore` updated
         | 
| 86 | 
            +
            4. ❌ Don't commit model weights to git
         | 
| 87 | 
            +
            5. ❌ Don't use persistent cache directories
         | 
| 88 | 
            +
             | 
    	
        README.md
    CHANGED
    
    | @@ -1,12 +1,31 @@ | |
| 1 | 
             
            ---
         | 
| 2 | 
             
            title: Plant Analysis Demo
         | 
| 3 | 
            -
            emoji:  | 
| 4 | 
            -
            colorFrom:  | 
| 5 | 
             
            colorTo: blue
         | 
| 6 | 
             
            sdk: gradio
         | 
| 7 | 
             
            sdk_version: 5.48.0
         | 
| 8 | 
             
            app_file: app.py
         | 
| 9 | 
             
            pinned: false
         | 
|  | |
| 10 | 
             
            ---
         | 
| 11 |  | 
| 12 | 
            -
             | 
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
             
            ---
         | 
| 2 | 
             
            title: Plant Analysis Demo
         | 
| 3 | 
            +
            emoji: 🌿
         | 
| 4 | 
            +
            colorFrom: green
         | 
| 5 | 
             
            colorTo: blue
         | 
| 6 | 
             
            sdk: gradio
         | 
| 7 | 
             
            sdk_version: 5.48.0
         | 
| 8 | 
             
            app_file: app.py
         | 
| 9 | 
             
            pinned: false
         | 
| 10 | 
            +
            python_version: 3.10
         | 
| 11 | 
             
            ---
         | 
| 12 |  | 
| 13 | 
            +
            # 🌿 Automated Plant Analysis Pipeline
         | 
| 14 | 
            +
             | 
| 15 | 
            +
            Analyze plant images with automated segmentation, feature extraction, and morphological analysis.
         | 
| 16 | 
            +
             | 
| 17 | 
            +
            ## Features
         | 
| 18 | 
            +
            - **Multi-plant detection**: Automatically detects and analyzes multiple plants
         | 
| 19 | 
            +
            - **Segmentation**: BRIA RMBG-2.0 model for accurate plant segmentation
         | 
| 20 | 
            +
            - **Vegetation indices**: NDVI, GNDVI, SAVI with statistics
         | 
| 21 | 
            +
            - **Texture analysis**: LBP, HOG, Lacunarity on green band
         | 
| 22 | 
            +
            - **Morphology**: Height calculation for each plant, size visualization
         | 
| 23 | 
            +
             | 
| 24 | 
            +
            ## Storage Optimization
         | 
| 25 | 
            +
            This app uses `/tmp` for model caching to avoid persistent storage limits on Hugging Face Spaces.
         | 
| 26 | 
            +
             | 
| 27 | 
            +
            ## Usage
         | 
| 28 | 
            +
            1. Upload a plant image (TIFF, PNG, or JPG)
         | 
| 29 | 
            +
            2. Or select a preset image (Sorghum, Corn, Cotton)
         | 
| 30 | 
            +
            3. Click "Run Pipeline"
         | 
| 31 | 
            +
            4. Watch results appear progressively!
         | 
    	
        cleanup_cache.py
    ADDED
    
    | @@ -0,0 +1,49 @@ | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            #!/usr/bin/env python3
         | 
| 2 | 
            +
            """
         | 
| 3 | 
            +
            Cleanup script for Hugging Face Spaces to remove cached models and free up storage.
         | 
| 4 | 
            +
            Run this if you encounter storage limit errors.
         | 
| 5 | 
            +
            """
         | 
| 6 | 
            +
             | 
| 7 | 
            +
            import os
         | 
| 8 | 
            +
            import shutil
         | 
| 9 | 
            +
            from pathlib import Path
         | 
| 10 | 
            +
             | 
| 11 | 
            +
            def get_size(path):
         | 
| 12 | 
            +
                """Get size of directory in GB."""
         | 
| 13 | 
            +
                total = 0
         | 
| 14 | 
            +
                try:
         | 
| 15 | 
            +
                    for entry in os.scandir(path):
         | 
| 16 | 
            +
                        if entry.is_file(follow_symlinks=False):
         | 
| 17 | 
            +
                            total += entry.stat().st_size
         | 
| 18 | 
            +
                        elif entry.is_dir(follow_symlinks=False):
         | 
| 19 | 
            +
                            total += get_size(entry.path)
         | 
| 20 | 
            +
                except PermissionError:
         | 
| 21 | 
            +
                    pass
         | 
| 22 | 
            +
                return total / (1024**3)  # Convert to GB
         | 
| 23 | 
            +
             | 
| 24 | 
            +
            def cleanup():
         | 
| 25 | 
            +
                """Remove cache directories to free up space."""
         | 
| 26 | 
            +
                cache_dirs = [
         | 
| 27 | 
            +
                    Path.home() / ".cache" / "huggingface",
         | 
| 28 | 
            +
                    Path.home() / ".cache" / "torch",
         | 
| 29 | 
            +
                    Path("/tmp/huggingface_cache"),
         | 
| 30 | 
            +
                    Path("/tmp/torch_cache"),
         | 
| 31 | 
            +
                ]
         | 
| 32 | 
            +
                
         | 
| 33 | 
            +
                total_freed = 0
         | 
| 34 | 
            +
                for cache_dir in cache_dirs:
         | 
| 35 | 
            +
                    if cache_dir.exists():
         | 
| 36 | 
            +
                        size = get_size(str(cache_dir))
         | 
| 37 | 
            +
                        print(f"Removing {cache_dir} ({size:.2f} GB)...")
         | 
| 38 | 
            +
                        try:
         | 
| 39 | 
            +
                            shutil.rmtree(cache_dir)
         | 
| 40 | 
            +
                            total_freed += size
         | 
| 41 | 
            +
                            print(f"  ✓ Removed")
         | 
| 42 | 
            +
                        except Exception as e:
         | 
| 43 | 
            +
                            print(f"  ✗ Failed: {e}")
         | 
| 44 | 
            +
                
         | 
| 45 | 
            +
                print(f"\nTotal space freed: {total_freed:.2f} GB")
         | 
| 46 | 
            +
             | 
| 47 | 
            +
            if __name__ == "__main__":
         | 
| 48 | 
            +
                cleanup()
         | 
| 49 | 
            +
             | 
    	
        sorghum_pipeline/segmentation/manager.py
    CHANGED
    
    | @@ -29,13 +29,18 @@ class SegmentationManager: | |
| 29 | 
             
                    import os
         | 
| 30 | 
             
                    hf_token = os.environ.get("HF_TOKEN")
         | 
| 31 |  | 
| 32 | 
            -
                     | 
|  | |
|  | |
|  | |
|  | |
| 33 | 
             
                    self.model = AutoModelForImageSegmentation.from_pretrained(
         | 
| 34 | 
             
                        model_name,
         | 
| 35 | 
             
                        trust_remote_code=trust_remote_code,
         | 
| 36 | 
            -
                        cache_dir=cache_dir | 
| 37 | 
             
                        local_files_only=local_files_only,
         | 
| 38 | 
             
                        token=hf_token,
         | 
|  | |
| 39 | 
             
                    ).eval().to(self.device)
         | 
| 40 |  | 
| 41 | 
             
                    # Use 512x512 for 4x speed improvement
         | 
|  | |
| 29 | 
             
                    import os
         | 
| 30 | 
             
                    hf_token = os.environ.get("HF_TOKEN")
         | 
| 31 |  | 
| 32 | 
            +
                    # Set cache directory to /tmp to avoid persistent storage issues
         | 
| 33 | 
            +
                    if cache_dir is None:
         | 
| 34 | 
            +
                        cache_dir = "/tmp/huggingface_cache"
         | 
| 35 | 
            +
                    
         | 
| 36 | 
            +
                    logger.info(f"Loading BRIA model: {model_name} (cache: {cache_dir})")
         | 
| 37 | 
             
                    self.model = AutoModelForImageSegmentation.from_pretrained(
         | 
| 38 | 
             
                        model_name,
         | 
| 39 | 
             
                        trust_remote_code=trust_remote_code,
         | 
| 40 | 
            +
                        cache_dir=cache_dir,
         | 
| 41 | 
             
                        local_files_only=local_files_only,
         | 
| 42 | 
             
                        token=hf_token,
         | 
| 43 | 
            +
                        low_cpu_mem_usage=True,  # Reduce memory usage during loading
         | 
| 44 | 
             
                    ).eval().to(self.device)
         | 
| 45 |  | 
| 46 | 
             
                    # Use 512x512 for 4x speed improvement
         | 
