Update main.py

main.py

@@ -2,230 +2,123 @@ import httpx
 from fastapi import FastAPI, Request, HTTPException
 from starlette.responses import StreamingResponse, JSONResponse
 from starlette.background import BackgroundTask
-from pydantic import Field, field_validator
-from typing import Set, Dict
-from contextlib import asynccontextmanager
 import os
 import random
 import logging
 import time
-import asyncio
-import uuid
-from faker import Faker
-from fake_useragent import UserAgent
-
-# --- Pydantic V1/V2 Compatibility ---
-# This block makes the code resilient to different Pydantic versions.
-try:
-    # Recommended for Pydantic V2
-    from pydantic_settings import BaseSettings
-    print("Using pydantic_settings.BaseSettings")
-except ImportError:
-    # Fallback for Pydantic V1
-    from pydantic import BaseSettings
-    print("pydantic_settings not found, falling back to pydantic.BaseSettings")
-
-
-# --- Structured Configuration using Pydantic ---
-class Settings(BaseSettings):
-    """Manages application settings and configuration from environment variables."""
-    LOG_LEVEL: str = Field(default="INFO")
-    TARGET_URL: str = Field(default="https://api.gmi-serving.com")
-    MAX_RETRIES: int = Field(default=5, gt=0)
-    
-    RETRY_CODES_STR: str = Field(default="429,500,502,503,504", alias="RETRY_CODES")
-    RETRY_STATUS_CODES: Set[int] = {429, 500, 502, 503, 504}
-    BACKOFF_FACTOR: float = Field(default=0.5, gt=0)
-    JITTER_FACTOR: float = Field(default=0.2, ge=0)
-
-    @field_validator('RETRY_STATUS_CODES', mode='before')
-    @classmethod
-    def parse_retry_codes(cls, v, values):
-        """Parses the comma-separated string of retry codes into a set of integers."""
-        retry_codes_str = values.data.get('RETRY_CODES_STR', "429,500,502,503,504")
-        try:
-            return {int(code.strip()) for code in retry_codes_str.split(',')}
-        except (ValueError, AttributeError) as e:
-            logging.error(f"Invalid RETRY_CODES format: '{retry_codes_str}'. Error: {e}. Using default.")
-            return {429, 500, 502, 503, 504}
-
-    class Config:
-        env_file = '.env'
-        env_file_encoding = 'utf-8'
-
-# --- Initialize Settings and Global Services ---
-settings = Settings()
-
-# Custom logging filter to inject request_id
-class RequestIdFilter(logging.Filter):
-    def filter(self, record):
-        record.request_id = getattr(record, 'request_id', 'main')
-        return True
+from contextlib import asynccontextmanager
 
+# --- Configuration ---
+LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper()
 logging.basicConfig(
-    level=settings.LOG_LEVEL.upper(),
-    format='%(asctime)s - %(levelname)s - [%(request_id)s] - %(message)s'
+    level=LOG_LEVEL,
+    format='%(asctime)s - %(levelname)s - %(message)s'
 )
-logger = logging.getLogger(__name__)
-logger.addFilter(RequestIdFilter())
-
-# Initialize data generation tools
-faker = Faker()
+TARGET_URL = os.getenv("TARGET_URL", "https://api.gmi-serving.com")
+MAX_RETRIES = int(os.getenv("MAX_RETRIES", "15"))
+DEFAULT_RETRY_CODES = "429,500,502,503,504"
+RETRY_CODES_STR = os.getenv("RETRY_CODES", DEFAULT_RETRY_CODES)
 try:
-    ua = UserAgent()
-except Exception:
-    # Fallback if fake-useragent server is down
-    ua = None
-    logger.warning("Could not initialize UserAgent. A fallback will be used.")
+    RETRY_STATUS_CODES = {int(code.strip()) for code in RETRY_CODES_STR.split(',')}
+    logging.info(f"Retry on status codes: {RETRY_STATUS_CODES}")
+except ValueError:
+    logging.error(f"Invalid RETRY_CODES: '{RETRY_CODES_STR}'. Using default: {DEFAULT_RETRY_CODES}")
+    RETRY_STATUS_CODES = {int(code.strip()) for code in DEFAULT_RETRY_CODES.split(',')}
+
+# --- Helper Function ---
+def generate_random_ip():
+    """Generates a random, valid-looking IPv4 address with improved randomization."""
+    return ".".join(str(random.randint(1, 255)) for _ in range(4))
 
 # --- HTTPX Client Lifecycle Management ---
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    """Manages the lifecycle of the HTTPX client for the application."""
-    logger.info(f"Starting application. Proxying to {settings.TARGET_URL}")
-    logger.info(f"Will retry on status codes: {settings.RETRY_STATUS_CODES}")
-    async with httpx.AsyncClient(base_url=settings.TARGET_URL, timeout=None) as client:
+    """Manages the lifecycle of the HTTPX client."""
+    async with httpx.AsyncClient(base_url=TARGET_URL, timeout=None) as client:
         app.state.http_client = client
         yield
-    logger.info("Application shut down.")
 
-# Initialize the FastAPI app
+# Initialize FastAPI app
 app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)
 
-# --- Helper Functions ---
-def get_random_user_agent() -> str:
-    """Returns a random User-Agent, with a fallback."""
-    if ua:
-        try:
-            return ua.random
-        except Exception:
-            logger.warning("Failed to get random User-Agent, using fallback.")
-    # Fallback User-Agent
-    return "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
-
-def prepare_forward_headers(incoming_headers: Dict, client_host: str) -> (Dict, str):
-    """
-    Prepares headers for the downstream request, adding a comprehensive set of
-    spoofed IP and a random User-Agent.
-    """
-    # Start with a clean slate of lower-cased headers
-    forward_headers = {k.lower(): v for k, v in incoming_headers.items()}
-    
-    # Remove headers that are specific to the incoming request's connection or could leak info
-    headers_to_remove = [
-        "host", "content-length", "x-forwarded-for", "x-real-ip", "forwarded",
-        "via", "x-client-ip", "x-forwarded-proto", "x-forwarded-host"
-    ]
-    for h in headers_to_remove:
-        forward_headers.pop(h, None)
-    
-    spoofed_ip = faker.ipv4_public()
-    
-    # Add a comprehensive set of headers to mask the origin
-    override_headers = {
-        # Standard headers
-        "x-forwarded-for": f"{spoofed_ip}, {faker.ipv4_public()}", # Append a fake proxy chain
-        "x-real-ip": spoofed_ip,
-        
-        # RFC 7239 standard, more structured
-        "forwarded": f"for={spoofed_ip};proto=https",
-        
-        # Common non-standard headers
-        "x-client-ip": spoofed_ip,
-        "x-originating-ip": spoofed_ip,
-        "x-remote-ip": spoofed_ip,
-        "x-remote-addr": spoofed_ip,
-        
-        # Cloudflare-specific headers
-        "cf-connecting-ip": spoofed_ip,
-        "true-client-ip": spoofed_ip,
-        
-        # Other proxy headers
-        "via": "1.1 google", # Fake a passthrough via a common service
-        
-        # Dynamic User-Agent
-        "user-agent": get_random_user_agent(),
-    }
-    forward_headers.update(override_headers)
-    
-    return forward_headers, spoofed_ip
-
 # --- API Endpoints ---
-@app.get("/", include_in_schema=False)
+@app.get("/")
 async def health_check():
-    """Provides a basic health check endpoint."""
-    return JSONResponse({
-        "status": "ok",
-        "target_url": settings.TARGET_URL,
-        "max_retries": settings.MAX_RETRIES
-    })
+    """Basic health check endpoint."""
+    return JSONResponse({"status": "ok", "target": TARGET_URL})
 
-@app.api_route("/{full_path:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD"], include_in_schema=False)
+@app.api_route("/{full_path:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD"])
 async def reverse_proxy_handler(request: Request):
     """
-    A catch-all reverse proxy that forwards requests with advanced retry logic,
-    backoff, and dynamic header generation.
+    Catch-all reverse proxy that forwards requests to the target URL with retry logic and latency logging.
     """
     start_time = time.monotonic()
-    request_id = str(uuid.uuid4())
-    log_extra = {'request_id': request_id}
-    
     client: httpx.AsyncClient = request.app.state.http_client
     url = httpx.URL(path=request.url.path, query=request.url.query.encode("utf-8"))
     
-    forward_headers, spoofed_ip = prepare_forward_headers(dict(request.headers), request.client.host)
+    # Prepare headers
+    request_headers = dict(request.headers)
+    request_headers.pop("host", None)
+    random_ip = generate_random_ip()
+    logging.info(f"Client '{request.client.host}' proxied with spoofed IP: {random_ip} for path: {url.path}")
     
-    logger.info(f"Incoming: {request.method} {url.path} from {request.client.host}. Spoofed IP: {spoofed_ip}", extra=log_extra)
+    specific_headers = {
+        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
+        "x-forwarded-for": random_ip,
+        "x-real-ip": random_ip,
+        "x-originating-ip": random_ip,
+        "x-remote-ip": random_ip,
+        "x-remote-addr": random_ip,
+        "x-host": random_ip,
+        "x-forwarded-host": random_ip,
+    }
+    request_headers.update(specific_headers)
     
-    request_body = await request.body()
+    # Preserve authorization header if present
+    if "authorization" in request.headers:
+        request_headers["authorization"] = request.headers["authorization"]
     
+    body = await request.body()
     last_exception = None
-    for attempt in range(settings.MAX_RETRIES):
-        if attempt > 0:
-            backoff_delay = settings.BACKOFF_FACTOR * (2 ** (attempt - 1))
-            jitter = random.uniform(-settings.JITTER_FACTOR, settings.JITTER_FACTOR) * backoff_delay
-            sleep_duration = max(0, backoff_delay + jitter)
-            logger.warning(f"Attempt {attempt}/{settings.MAX_RETRIES} failed. Retrying in {sleep_duration:.2f}s...", extra=log_extra)
-            await asyncio.sleep(sleep_duration)
-
+    
+    for attempt in range(MAX_RETRIES):
         try:
-            req = client.build_request(
-                method=request.method,
-                url=url,
-                headers=forward_headers,
-                content=request_body,
+            rp_req = client.build_request(
+                method=request.method, url=url, headers=request_headers, content=body
             )
-            resp = await client.send(req, stream=True)
-
-            if resp.status_code not in settings.RETRY_STATUS_CODES or attempt == settings.MAX_RETRIES - 1:
+            rp_resp = await client.send(rp_req, stream=True)
+            
+            if rp_resp.status_code not in RETRY_STATUS_CODES or attempt == MAX_RETRIES - 1:
                 duration_ms = (time.monotonic() - start_time) * 1000
-                log_func = logger.info if resp.is_success else logger.warning
-                log_func(
-                    f"Finished: {request.method} {url.path} status={resp.status_code} attempt={attempt+1} latency={duration_ms:.2f}ms",
-                    extra=log_extra
-                )
+                log_func = logging.info if rp_resp.is_success else logging.warning
+                log_func(f"Request finished: {request.method} {request.url.path} status_code={rp_resp.status_code} latency={duration_ms:.2f}ms")
+                
                 return StreamingResponse(
-                    resp.aiter_raw(),
-                    status_code=resp.status_code,
-                    headers=resp.headers,
-                    background=BackgroundTask(resp.aclose),
+                    rp_resp.aiter_raw(),
+                    status_code=rp_resp.status_code,
+                    headers=rp_resp.headers,
+                    background=BackgroundTask(rp_resp.aclose),
                 )
             
-            await resp.aclose()
-            last_exception = f"Last failed attempt returned status code {resp.status_code}"
-
-        except httpx.RequestError as e:
+            logging.warning(f"Attempt {attempt + 1}/{MAX_RETRIES} for {url.path} failed with status {rp_resp.status_code}. Retrying...")
+            await rp_resp.aclose()
+        
+        except httpx.ConnectError as e:
             last_exception = e
-            logger.warning(f"HTTPX RequestError on attempt {attempt + 1}: {e}", extra=log_extra)
-            if attempt == settings.MAX_RETRIES - 1:
+            logging.warning(f"Attempt {attempt + 1}/{MAX_RETRIES} for {url.path} failed with connection error: {e}")
+            if attempt == MAX_RETRIES - 1:
                 break
+            await asyncio.sleep(2 ** attempt)  # Exponential backoff
+        
+        except Exception as e:
+            last_exception = e
+            logging.error(f"Unexpected error on attempt {attempt + 1}/{MAX_RETRIES} for {url.path}: {e}")
+            break
     
     duration_ms = (time.monotonic() - start_time) * 1000
-    logger.critical(
-        f"Failed permanently: {request.method} {url.path} after {settings.MAX_RETRIES} attempts. latency={duration_ms:.2f}ms. Last error: {last_exception}",
-        extra=log_extra
-    )
+    logging.critical(f"Request failed: {request.method} {request.url.path} status_code=502 latency={duration_ms:.2f}ms")
+    
     raise HTTPException(
         status_code=502,
-        detail=f"Bad Gateway: The server was unable to process the request after {settings.MAX_RETRIES} attempts. Last error: {last_exception}"
+        detail=f"Bad Gateway: Cannot connect to target service after {MAX_RETRIES} attempts. {last_exception}"
     )