Added files

.gitignore

@@ -0,0 +1,2 @@
+.venv
+dev/

README.md

@@ -1,2 +1,5 @@
-# minio_quota_checker
+## minio_quota_checker
 
+`minio_quota_checker` is a Python-based utility crafted to automate the enforcement of storage quotas within MinIO object storage systems. It operates by periodically scanning MinIO buckets, intelligently grouping them by user based on a `username-bucketname` naming convention, and calculating the total storage consumption for each user. Leveraging Redis for efficient data storage and retrieval, this script flags users who surpass their pre-defined storage limits. This is achieved by setting a specific key in Redis, which can then be used by other services, such as Nginx or API gateways, to enforce access restrictions or provide notifications. This tool is indispensable for administrators managing multi-tenant MinIO environments where granular, user-level storage quotas are essential.
+
+For detailed usage instructions, configuration options, and advanced deployment strategies, please refer to the project's wiki: [minio_quota_checker Wiki](https://git.bugzbunny.net/bugzbunnynet/minio_quota_checker/wiki/?action=_pages)

minio_quota_checker.py

@@ -0,0 +1,150 @@
+import os
+import time
+import logging
+import signal
+import sys
+import redis
+from datetime import datetime, timedelta
+from minio import Minio
+from minio.error import S3Error
+
+# Default Configuration
+CONFIG = {
+    "MINIO_ENDPOINT": "play.min.io",  # Change to your MinIO instance
+    "MINIO_ACCESS_KEY": "your-access-key",
+    "MINIO_SECRET_KEY": "your-secret-key",
+    "TOTAL_SIZE_LIMIT": "1G",  # Configurable, supports 1G, 500M, etc.
+    "WHITELIST": "admin,superuser",  # Users that are not restricted, comma-separated string
+    "AGGREGATE_INTERVAL": "600",  # 10 minutes in seconds as string
+    "LOG_FILE": "minio_quota.log",
+    "REDIS_HOST": "localhost",
+    "REDIS_PORT": "6379",
+    "REDIS_DB": "0"
+}
+
+# Load Configuration from /etc/minio_quota.conf if it exists
+CONFIG_FILE = "/etc/minio_quota.conf"
+if os.path.exists(CONFIG_FILE):
+    with open(CONFIG_FILE, "r") as f:
+        file_config = f.readlines()
+        for line in file_config:
+            if "=" in line:
+                key, value = line.strip().split("=", 1)
+                CONFIG[key.strip()] = value.strip()
+
+# Override with Environment Variables if set
+CONFIG.update({key: os.getenv(key, value) for key, value in CONFIG.items()})
+
+# Parse WHITELIST from string
+CONFIG["WHITELIST"] = CONFIG["WHITELIST"].split(",")
+
+# Parse AGGREGATE_INTERVAL as integer
+CONFIG["AGGREGATE_INTERVAL"] = int(CONFIG["AGGREGATE_INTERVAL"])
+
+# Determine if the MinIO server is using HTTPS
+USE_HTTPS = CONFIG["MINIO_ENDPOINT"].startswith("https://")
+
+# Initialize MinIO client
+minio_client = Minio(
+    CONFIG["MINIO_ENDPOINT"].replace("https://", "").replace("http://", ""),
+    access_key=CONFIG["MINIO_ACCESS_KEY"],
+    secret_key=CONFIG["MINIO_SECRET_KEY"],
+    secure=USE_HTTPS,
+)
+
+redis_client = redis.Redis(
+    host=CONFIG["REDIS_HOST"],
+    port=int(CONFIG["REDIS_PORT"]),
+    db=int(CONFIG["REDIS_DB"]),
+    decode_responses=True
+)
+
+
+# Setup logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s",
+    handlers=[
+        logging.FileHandler(CONFIG["LOG_FILE"]),
+        logging.StreamHandler()
+    ]
+)
+
+def parse_size(size_str):
+    """Convert human-readable size string (e.g., 1G, 500M) to bytes."""
+    size_str = size_str.upper().strip()
+    if size_str.endswith("G"):
+        return int(float(size_str[:-1]) * 1024**3)
+    elif size_str.endswith("M"):
+        return int(float(size_str[:-1]) * 1024**2)
+    elif size_str.endswith("K"):
+        return int(float(size_str[:-1]) * 1024)
+    else:
+        return int(size_str)  # Assume raw bytes if no suffix
+
+TOTAL_SIZE_LIMIT_BYTES = parse_size(CONFIG["TOTAL_SIZE_LIMIT"])
+
+def get_buckets_by_user():
+    """Fetch all buckets and group them by username."""
+    logging.info("Fetching user buckets...")
+    buckets = minio_client.list_buckets()
+    user_buckets = {}
+    
+    for bucket in buckets:
+        name = bucket.name
+        username = name.split("-")[0]  # Extract username from bucket name
+        if username not in CONFIG["WHITELIST"]:
+            user_buckets.setdefault(username, []).append(name)
+    
+    return user_buckets
+
+def get_bucket_size(bucket_name):
+    """Calculate the total size of objects in a bucket."""
+    total_size = 0
+    try:
+        objects = minio_client.list_objects(bucket_name, recursive=True)
+        for obj in objects:
+            total_size += obj.size
+    except S3Error as e:
+        logging.error(f"Error fetching objects for {bucket_name}: {e}")
+    return total_size
+
+def aggregate_disk_usage():
+    """Aggregate disk usage per user and log warnings if they exceed the limit."""
+    user_buckets = get_buckets_by_user()
+    user_usage = {}
+    
+    for user, buckets in user_buckets.items():
+        total_size = sum(get_bucket_size(bucket) for bucket in buckets)
+        user_usage[user] = total_size
+        if total_size > TOTAL_SIZE_LIMIT_BYTES:
+            excess_size = total_size - TOTAL_SIZE_LIMIT_BYTES
+            logging.warning(f"User {user} exceeded quota by {excess_size} bytes (Total size: {total_size} bytes)")
+            redis_client.set(f"quota_exceeded:{user}", "1")
+        else:
+            # Remove quota exceeded flag if user is under quota
+            if redis_client.exists(f"quota_exceeded:{user}"):
+                redis_client.delete(f"quota_exceeded:{user}")
+                logging.info(f"User {user} is now under quota. Removed quota exceeded flag.")
+
+    return user_usage
+
+def signal_handler(sig, frame):
+    """Handle SIGINT and SIGTERM to gracefully exit."""
+    logging.info(f"Received signal {sig}. Exiting gracefully...")
+    sys.exit(0)
+
+# Register the signal handler for SIGINT and SIGTERM
+signal.signal(signal.SIGINT, signal_handler)
+signal.signal(signal.SIGTERM, signal_handler)
+
+def main():
+    """Main loop running every 10 minutes."""
+    while True:
+        logging.info("Starting disk usage aggregation...")
+        aggregate_disk_usage()
+        logging.info("Sleeping for 10 minutes...")
+        time.sleep(CONFIG["AGGREGATE_INTERVAL"])
+
+if __name__ == "__main__":
+    main()

nginx.conf.example

@@ -0,0 +1,29 @@
+http {
+    # ... other http configurations ...
+
+    map $http_authorization $is_s3_auth {
+        default 0;
+        ~AWS4-HMAC-SHA256 1; # Check for AWS S3 v4 authorization
+    }
+
+    server {
+        listen 80; # Or your desired port
+        server_name yourdomain.com; # Or your domain
+
+        location / {
+            if ($request_method = PUT) {
+                if ($is_s3_auth = 1) {
+                    # PUT request with S3 authorization: proxy to s3_backend
+                    proxy_pass http://s3_backend;
+                    break; # Stop further processing in this location
+                }
+            }
+            # All other requests (or PUT without S3 auth): proxy to default_backend
+            proxy_pass http://default_backend;
+        }
+
+        # ... other locations ...
+    }
+
+    # ... other http configurations ...
+}

nginx_request_checker.py

@@ -0,0 +1,83 @@
+from fastapi import FastAPI, Request, Response
+import redis
+import httpx
+import os
+import logging
+
+# Default configuration
+CONFIG = {
+    "REDIS_HOST": "127.0.0.1",
+    "REDIS_PORT": "6379",
+    "REDIS_DB": "2",
+    "MINIO_ENDPOINT": "http://localhost:9000",
+    "LOG_FILE": "/var/log/minio_quota.log"
+}
+
+# Load config from /etc/minio_quota.conf if available
+CONFIG_FILE = "/etc/minio_quota.conf"
+if os.path.exists(CONFIG_FILE):
+    with open(CONFIG_FILE, "r") as f:
+        for line in f:
+            if "=" in line:
+                key, value = line.strip().split("=", 1)
+                CONFIG[key.strip()] = value.strip()
+
+# Read configuration with priority:
+REDIS_HOST = os.getenv("REDIS_HOST", CONFIG.get("REDIS_HOST"))
+REDIS_PORT = os.getenv("REDIS_PORT", CONFIG.get("REDIS_PORT"))
+REDIS_DB = int(os.getenv("REDIS_DB", CONFIG.get("REDIS_DB")))
+MINIO_ENDPOINT = os.getenv("MINIO_ENDPOINT", CONFIG.get("MINIO_ENDPOINT"))
+LOG_FILE = os.getenv("LOG_FILE", CONFIG.get("LOG_FILE"))
+
+# Set up logging
+handlers = [logging.StreamHandler()]  # Always log to stdout
+if LOG_FILE and LOG_FILE.strip():  # Log to file only if LOG_FILE is set and not empty
+    handlers.append(logging.FileHandler(LOG_FILE))
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s",
+    handlers=handlers
+)
+
+# Initialize FastAPI and Redis client
+app = FastAPI()
+redis_client = redis.Redis(host=REDIS_HOST, port=int(REDIS_PORT), db=REDIS_DB)
+
+@app.api_route("/{path:path}", methods=["GET", "PUT", "POST", "DELETE"])
+async def proxy_request(path: str, request: Request):
+    username = path.split("/")[0].split("-")[0]
+    logging.info(f"Received {request.method} request for: {path}")
+
+    if username:
+        quota_exceeded = redis_client.get(f"quota_exceeded:{username}")
+        if quota_exceeded:
+            logging.warning(f"Quota exceeded for user: {username}, blocking request.")
+            error_xml = f"""<?xml version="1.0" encoding="UTF-8"?>
+<Error>
+  <Code>QuotaExceeded</Code>
+  <Message>User has exceeded storage quota.</Message>
+  <Resource>/{path}</Resource>
+  <RequestId>request-id-12345</RequestId>
+</Error>"""
+            return Response(content=error_xml, status_code=403, media_type="application/xml")
+
+    # Proxy request to MinIO
+    async with httpx.AsyncClient() as client:
+        minio_url = f"{MINIO_ENDPOINT}/{path}"
+        headers = dict(request.headers)
+        body = await request.body()
+
+        logging.info(f"Proxying request to MinIO: {minio_url}")
+
+        minio_response = await client.request(
+            method=request.method, url=minio_url, headers=headers, content=body
+        )
+
+        logging.info(f"MinIO response: {minio_response.status_code}")
+
+        return Response(
+            content=minio_response.content,
+            status_code=minio_response.status_code,
+            headers=minio_response.headers
+        )