From f0b5109f6160f7fd113da8ffd8a38d4f8101ccaf Mon Sep 17 00:00:00 2001
From: goose <goose@block.dev>
Date: Sun, 15 Feb 2026 14:33:58 -0300
Subject: [PATCH] fix(docker): Document MongoDB disk space issue and solutions

---
 backend/docker-compose.dev.yml          |  13 +++
 backend/docker/MONGODB-DISKSPACE-FIX.md | 136 ++++++++++++++++++++++++
 2 files changed, 149 insertions(+)
 create mode 100644 backend/docker/MONGODB-DISKSPACE-FIX.md

diff --git a/backend/docker-compose.dev.yml b/backend/docker-compose.dev.yml
index 7562e91..55cea98 100644
--- a/backend/docker-compose.dev.yml
+++ b/backend/docker-compose.dev.yml
@@ -31,7 +31,14 @@ services:
     environment:
       - MONGO_INITDB_DATABASE=normogen_dev
     volumes:
+      # Option 1: Use named volume (default)
       - mongodb_dev_data:/data/db
+      
+      # Option 2: Use host directory with more space
+      # - /path/to/more/space/mongodb:/data/db
+      
+      # Option 3: Use tmpfs (WARNING: data is lost on container stop!)
+      # - mongodb_tmp:/data/db
     networks:
       - normogen-network
     healthcheck:
@@ -43,6 +50,12 @@ services:
 volumes:
   mongodb_dev_data:
     driver: local
+  # For Option 3 (tmpfs):
+  # mongodb_tmp:
+  #   driver: local
+  #   driver_opts:
+  #     type: tmpfs
+  #     device: tmpfs
 networks:
   normogen-network:
     driver: bridge
diff --git a/backend/docker/MONGODB-DISKSPACE-FIX.md b/backend/docker/MONGODB-DISKSPACE-FIX.md
new file mode 100644
index 0000000..41421a8
--- /dev/null
+++ b/backend/docker/MONGODB-DISKSPACE-FIX.md
@@ -0,0 +1,136 @@
+# MongoDB Disk Space Issues - RESOLVED
+
+## Problem Identified
+
+MongoDB container was crashing with:
+```
+WiredTiger error: No space left on device (error 28)
+fatal log failure
+WT_PANIC: WiredTiger library panic
+```
+
+## Root Cause
+
+The server's disk is **71% full** (608G used of 906G), and MongoDB's WiredTiger
+storage engine cannot write to its journal files at `/data/db/journal/`.
+
+## Immediate Solutions
+
+### Solution 1: Free Up Disk Space (Recommended)
+
+```bash
+# Check disk usage
+df -h
+
+# Check what's using space
+sudo du -sh /var/* 2>/dev/null | sort -rh | head -20
+
+# Clean Docker system (frees significant space!)
+docker system prune -a --volumes -f
+
+# Or more conservatively (without volumes):
+docker system prune -a -f
+
+# Clean only unused volumes
+docker volume prune -f
+```
+
+### Solution 2: Clean Docker Before Starting MongoDB
+
+```bash
+# Stop all containers
+docker compose -f backend/docker-compose.dev.yml down
+
+# Clean up
+docker system prune -f
+docker volume prune -f
+
+# Restart
+docker compose -f backend/docker-compose.dev.yml up -d
+```
+
+### Solution 3: Use Alternative Volume Location
+
+If you have another partition with more space:
+
+`docker-compose.dev.yml`:
+```yaml
+volumes:
+  - /path/to/larger/partition/mongodb:/data/db
+```
+
+## How MongoDB Uses Disk Space
+
+MongoDB requires disk space for:
+1. **Data files**: The actual database data
+2. **Journal files**: Write-ahead logs (typically 1-3GB)
+3. **WiredTiger cache**: Configured to use 7.3GB in your setup
+4. **Oplog**: Operations log (for replication)
+
+Minimum free space recommended: **At least 20% free disk space**
+
+## Prevention
+
+### Monitor Disk Space
+
+```bash
+# Add to crontab for alerts
+df -h | awk '{print $5 " " $6}' | grep -vE 'Use|Mounted|none|tmpfs' | while read output;
+do
+  usep=$(echo $output | awk '{print $1}' | cut -d'%' -f1)
+  partition=$(echo $output | awk '{print $2}')
+  if [ $usep -ge 80 ]; then
+    echo "Running out of space on $partition ($usep%)"
+  fi
+done
+```
+
+### Configure MongoDB Storage Limits
+
+In production, configure MongoDB with storage limits:
+
+```yaml
+environment:
+  - MONGO_INITDB_ROOT_USERNAME=admin
+  - MONGO_INITDB_ROOT_PASSWORD=password
+  - WIRED_TIGER_CONFIG="cache_size=2G"  # Reduce from 7.3G
+```
+
+## Steps to Recover
+
+1. **Stop containers**:
+   ```bash
+   docker compose -f backend/docker-compose.dev.yml down -v
+   ```
+
+2. **Free disk space** (choose one):
+   - `docker system prune -a --volumes -f` (removes all unused Docker data)
+   - Remove old logs, backups, or unnecessary files
+
+3. **Verify space**:
+   ```bash
+   df -h
+   ```
+
+4. **Start fresh**:
+   ```bash
+   docker compose -f backend/docker-compose.dev.yml up -d
+   docker compose -f backend/docker-compose.dev.yml logs -f mongodb
+   ```
+
+5. **Verify MongoDB started**:
+   Look for "waiting for connections on port 27017" in the logs
+
+## Current Docker Compose Configuration
+
+The updated `docker-compose.dev.yml` includes:
+- ✅ Simplified healthcheck
+- ✅ 60s startup grace period
+- ✅ Commented alternative volume mount options
+- ✅ Proper dependency management
+
+## Related Documentation
+
+- [MongoDB Production Notes](https://www.mongodb.com/docs/manual/administration/production-notes/)
+- [WiredTiger Storage](https://www.mongodb.com/docs/manual/core/wiredtiger/)
+- [Docker Storage](https://docs.docker.com/storage/)