AO3 Mirror v4 — initial commit

Cookie-aware proxy pool with CF challenge solving - Tiered proxy pool (fast 50 + main 676) - Per-proxy cf_clearance cookie persistence - CF challenge detection + user-browser solving - Safari + Chrome TLS fingerprint rotation - Async FastAPI backend with LRU cache - Passive daemon with systemd supervision - Stats dashboard + Prometheus metrics
2026-06-23 08:04:04 +00:00
commit 085cc7a140
16 changed files with 2842 additions and 0 deletions
--- a/scripts/scan_proxies_cffi.py
+++ b/scripts/scan_proxies_cffi.py
@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+"""
+Optimized proxy scanner — uses port-filtered Webshare proxies for 72.6% success rate.
+
+Key findings from v2 diagnostic scan (5000 proxies tested):
+- Optimal port range: 13500-14499 → 72.6% success rate
+- Best fingerprints: chrome123, chrome124, safari17_0 (equivalent)
+- 99.7% of failures = CF_BLOCK_403 (IP reputation, not TLS fingerprint)
+- Fingerprint rotation does NOT improve rate
+- chrome120 = 7.5%, edge120 = 0% (DO NOT USE)
+"""
+import concurrent.futures
+import os
+import sys
+import time
+
+from curl_cffi import requests as curl_requests
+
+TEST_URL = "https://archiveofourown.org/robots.txt"
+TIMEOUT = 12
+MAX_WORKERS = 50
+PROXY_FILE = "/home/ubuntu/proxy.txt"
+OUTPUT_FILE = "/dev/shm/working_proxies.txt"
+
+# Optimal port range for Webshare proxies (verified by full scan)
+#   Ports 13500-13999: 74.4% | Ports 14000-14499: 70.8%
+#   Ports 10000-13499: ~32%  | Ports 14500-14999: ~52%
+OPTIMAL_MIN_PORT = 13500
+OPTIMAL_MAX_PORT = 14499
+
+# Use only proven fingerprints
+BROWSER_FINGERPRINTS = ["chrome123", "chrome124"]
+
+
+def test_proxy(host_port: str) -> tuple:
+    """Test proxy with fallback through fingerprints."""
+    start = time.time()
+    for imp in BROWSER_FINGERPRINTS:
+        try:
+            session = curl_requests.Session()
+            session.impersonate = imp
+            session.timeout = TIMEOUT
+            session.proxies = {
+                "http": f"http://{host_port}",
+                "https": f"http://{host_port}",
+            }
+            session.headers.update({
+                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+                "Accept-Language": "en-US,en;q=0.9,zh-CN;q=0.8",
+            })
+            resp = session.get(TEST_URL)
+            elapsed = time.time() - start
+            session.close()
+            if resp.status_code == 200:
+                return (host_port, True, round(elapsed, 2), f"OK({imp})")
+            elif resp.status_code in (403, 503) and imp != BROWSER_FINGERPRINTS[-1]:
+                continue  # Try next fingerprint
+            else:
+                return (host_port, False, round(elapsed, 2), f"CF_BLOCK({resp.status_code})")
+        except Exception as e:
+            if imp != BROWSER_FINGERPRINTS[-1]:
+                continue
+            return (host_port, False, round(time.time() - start, 2), str(e)[:60])
+    return (host_port, False, round(time.time() - start, 2), "ALL_FAILED")
+
+
+def main():
+    print(f"[Port-Optimized Scanner] Loading proxies from {PROXY_FILE}")
+
+    proxies = []
+    with open(PROXY_FILE) as f:
+        for line in f:
+            line = line.strip()
+            if not line or ":" not in line:
+                continue
+            if "|" in line:
+                line = line.split("|")[-1].strip()
+            proxies.append(line)
+
+    # Filter to optimal port range
+    before = len(proxies)
+    proxies = [p for p in proxies if ":" in p and OPTIMAL_MIN_PORT <= int(p.split(":")[-1]) <= OPTIMAL_MAX_PORT]
+    print(f"Loaded {before} proxies, filtered to ports {OPTIMAL_MIN_PORT}-{OPTIMAL_MAX_PORT}: {len(proxies)}")
+    print(f"Starting scan with {MAX_WORKERS} workers (timeout={TIMEOUT}s)...")
+
+    working = []
+    checked = 0
+    start_time = time.time()
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+        futures = {executor.submit(test_proxy, p): p for p in proxies}
+
+        for future in concurrent.futures.as_completed(futures):
+            host_port, is_ok, elapsed, note = future.result()
+            checked += 1
+
+            if is_ok:
+                working.append((host_port, elapsed))
+                working.sort(key=lambda x: x[1])
+
+            if checked % 100 == 0 or is_ok:
+                elapsed_total = time.time() - start_time
+                rate = checked / elapsed_total if elapsed_total > 0 else 0
+                eta = (len(proxies) - checked) / rate if rate > 0 else 0
+                status = "✅" if is_ok else "❌"
+                print(f"  [{checked}/{len(proxies)}] {status} {host_port} ({elapsed:.1f}s) {note} | "
+                      f"Working: {len(working)} ({len(working)/checked*100:.1f}%) | "
+                      f"{elapsed_total:.0f}s | ETA: {eta:.0f}s")
+
+    with open(OUTPUT_FILE, "w") as f:
+        for host_port, elapsed in working:
+            f.write(f"{host_port}\n")
+
+    elapsed_total = time.time() - start_time
+    rate = len(working) / checked * 100 if checked else 0
+    print(f"\n[Scan complete] {len(working)}/{checked} = {rate:.1f}% working (target: 70%+)")
+    print(f"  Baseline (unfiltered): 44.8%")
+    print(f"  Improvement: +{rate - 44.8:.1f} percentage points")
+    print(f"  Target met: {'YES ✅' if rate >= 70 else 'NO ❌'}")
+    print(f"  Time: {elapsed_total:.0f}s")
+    print(f"  Saved to: {OUTPUT_FILE}")
+
+    if working:
+        print(f"\n  Top 10 fastest:")
+        for hp, el in working[:10]:
+            print(f"    {hp} ({el:.2f}s)")
+
+
+if __name__ == "__main__":
+    main()