Skip to content

S
spider
You need to sign in or sign up before continuing.
Commit 39a2e885 by Peng
Options

no message

parent fb663d1b
Showing with 3010 additions and 828 deletions
py_spider/amzon_parse_db_html/test_requests.py 0 → 100644
import requests
import json
import time
LENS_API = "http://192.168.10.8:8000"
def test_health():
r = requests.get(f"{LENS_API}/health", timeout=5)
print("Health:", r.json())
return r.status_code == 200
def test_upload_image(image_path):
print(f"\n上传图片: {image_path}")
t0 = time.time()
with open(image_path, "rb") as f:
r = requests.post(
f"{LENS_API}/api/lens",
files={"image": (image_path.split("\\")[-1], f, "image/png")},
data={"hl": "zh-CN"},
timeout=300,
)
elapsed = time.time() - t0
data = r.json()
print(f"状态: {r.status_code} | 耗时: {elapsed:.1f}s")
print(f"匹配数: {data.get('visual_matches_count', 0)}")
# AI 概览
ai = data.get("ai_overview", {})
if ai.get("available"):
print(f"\n--- AI 概览 ---")
print(f"摘要: {ai.get('summary', '')[:300]}")
for b in ai.get("bullets", [])[:10]:
print(f" - {b[:150]}")
else:
print(f"\nAI 概览: 无")
# 商品匹配
print(f"\n--- 商品匹配 (前10) ---")
for i, m in enumerate(data.get("visual_matches", [])[:10], 1):
price = f" | {m['price']}" if m.get("price") else ""
print(f" {i}. {m.get('title', '')[:60]}{price}")
print(f" {m.get('url', '')[:90]}")
return data
def test_url_image(image_url):
print(f"\nURL 搜图: {image_url[:80]}...")
t0 = time.time()
r = requests.post(
f"{LENS_API}/api/lens",
data={"image_url": image_url, "hl": "zh-CN"},
timeout=300,
)
elapsed = time.time() - t0
data = r.json()
print(f"状态: {r.status_code} | 耗时: {elapsed:.1f}s")
print(f"匹配数: {data.get('visual_matches_count', 0)}")
# AI 概览
ai = data.get("ai_overview", {})
if ai.get("available"):
print(f"\n--- AI 概览 ---")
print(f"摘要: {ai.get('summary', '')[:300]}")
for b in ai.get("bullets", [])[:10]:
print(f" - {b[:150]}")
else:
print(f"\nAI 概览: 无")
# 商品匹配
print(f"\n--- 商品匹配 (前10) ---")
for i, m in enumerate(data.get("visual_matches", [])[:10], 1):
price = f" | {m['price']}" if m.get("price") else ""
print(f" {i}. {m.get('title', '')[:60]}{price}")
print(f" {m.get('url', '')[:90]}")
return data
if __name__ == "__main__":
if not test_health():
print("服务不可用!")
exit(1)
# 上传图片搜图（走 Lens upload 路径，返回商品匹配）
test_upload_image(r"C:\Users\ASUS\Downloads\61fEImDN-kL.jpg")
# URL 搜图
# test_url_image("https://m.media-amazon.com/images/I/61fEImDN-kL.jpg")
py_spider/comment_spider/account_manager.py 0 → 100644
"""账号池管理 — 通过 RemoteEngine HTTP 代理，多机器竞争领取 + 状态回写 + 心跳"""
import os
import sys
import socket
import time
import uuid
import logging
import threading
import pandas as pd
from configparser import ConfigParser
from datetime import datetime
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
try:
from client import get_remote_engine
except ImportError:
get_remote_engine = None
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
class AccountManager:
def __init__(self, config_path="config.ini"):
cfg = ConfigParser()
cfg.read(config_path, encoding="utf-8")
mac_suffix = uuid.getnode().__format__('012x')[-4:]
self.machine_id = cfg.get("spider", "machine_id", fallback="") or f"{socket.gethostname()}-{mac_suffix}"
self.accounts_per_machine = cfg.getint("spider", "accounts_per_machine", fallback=3)
self.heartbeat_interval = cfg.getint("spider", "heartbeat_interval", fallback=600)
self._heartbeat_thread = None
self._heartbeat_stop = threading.Event()
if get_remote_engine:
self._engine = get_remote_engine("us", "mysql", database_name="selection")
else:
self._engine = None
logging.info(f"AccountManager init: machine={self.machine_id}")
def _query(self, sql, action="DB"):
for attempt in range(3):
try:
df = self._engine.read_sql(sql)
return df.to_dict("records")
except Exception as e:
logging.warning(f"{action}失败({attempt+1}/3): {e}")
time.sleep(3)
logging.error(f"{action}重试3次仍失败")
return []
def _exec(self, sql, action="DB"):
for attempt in range(3):
try:
with self._engine.begin() as tx:
tx.execute(sql)
return True
except Exception as e:
logging.warning(f"{action}失败({attempt+1}/3): {e}")
time.sleep(3)
logging.error(f"{action}重试3次仍失败")
return False
def reclaim_dead(self, timeout_minutes=60):
"""回收超时未心跳的僵尸账号"""
self._exec(
f"UPDATE share_account SET machine_id = NULL, locked_at = NULL "
f"WHERE locked_at < DATE_SUB(NOW(), INTERVAL {timeout_minutes} MINUTE) "
f"AND machine_id IS NOT NULL",
action="回收僵尸"
)
def acquire_accounts(self, count=None):
"""领取空闲账号（UPDATE 原子操作，不用 FOR UPDATE）"""
count = count or self.accounts_per_machine
self.reclaim_dead()
already = self._query(
f"SELECT * FROM share_account WHERE machine_id = '{self.machine_id}' AND status = 1",
)
if len(already) >= count:
logging.info(f"已持有 {len(already)} 个账号，无需再领")
return already
need = count - len(already)
self._exec(
f"UPDATE share_account SET machine_id = '{self.machine_id}', locked_at = NOW() "
f"WHERE status = 1 AND (machine_id IS NULL OR machine_id = '') "
f"AND (cooldown_until IS NULL OR cooldown_until < NOW()) "
f"ORDER BY fail_count ASC, account_id ASC LIMIT {need}",
action="领取账号"
)
result = self._query(
f"SELECT * FROM share_account WHERE machine_id = '{self.machine_id}' AND status = 1",
)
logging.info(f"当前持有 {len(result)} 个账号: {[r['account_id'] for r in result]}")
return result
def release_account(self, account_id):
self._exec(
f"UPDATE share_account SET machine_id = NULL, locked_at = NULL WHERE account_id = {account_id}",
action="释放账号"
)
def acquire_one_more(self):
self._exec(
f"UPDATE share_account SET machine_id = '{self.machine_id}', locked_at = NOW() "
f"WHERE status = 1 AND (machine_id IS NULL OR machine_id = '') "
f"AND (cooldown_until IS NULL OR cooldown_until < NOW()) "
f"ORDER BY fail_count ASC, account_id ASC LIMIT 1",
action="补领账号"
)
rows = self._query(
f"SELECT * FROM share_account WHERE machine_id = '{self.machine_id}' AND status = 1 "
f"ORDER BY locked_at DESC LIMIT 1",
)
if rows:
logging.info(f"补领账号 {rows[0]['account_id']}")
return rows[0]
logging.warning("无可用账号可补领")
return None
def mark_locked(self, account_id):
self._exec(
f"UPDATE share_account SET status = 2, fail_count = fail_count + 1, "
f"last_fail_at = NOW(), last_fail_reason = 'account_locked', "
f"machine_id = NULL, locked_at = NULL WHERE account_id = {account_id}",
action="标记锁定"
)
logging.warning(f"账号 {account_id} 被锁(account_locked) → status=2")
def mark_suspended(self, account_id):
self._exec(
f"UPDATE share_account SET status = 0, fail_count = fail_count + 1, "
f"last_fail_at = NOW(), last_fail_reason = 'suspended', "
f"machine_id = NULL, locked_at = NULL WHERE account_id = {account_id}",
action="标记封禁"
)
logging.warning(f"账号 {account_id} 被封禁(suspended) → status=0")
def mark_risk(self, account_id, reason="risk_detected"):
self._exec(
f"UPDATE share_account SET status = 2, fail_count = fail_count + 1, "
f"last_fail_at = NOW(), last_fail_reason = '{reason}', "
f"machine_id = NULL, locked_at = NULL, "
f"cooldown_until = DATE_ADD(NOW(), INTERVAL 30 MINUTE) "
f"WHERE account_id = {account_id}",
action="标记风控"
)
logging.warning(f"账号 {account_id} 风控({reason}) → status=2, 冷却30分钟")
def mark_login_fail(self, account_id, reason="wrong_password"):
self._exec(
f"UPDATE share_account SET fail_count = fail_count + 1, "
f"last_fail_at = NOW(), last_fail_reason = '{reason}' "
f"WHERE account_id = {account_id}",
action="标记登录失败"
)
def mark_login_ok(self, account_id):
self._exec(
f"UPDATE share_account SET fail_count = 0, last_fail_reason = NULL, "
f"locked_at = NOW() WHERE account_id = {account_id}",
action="标记登录成功"
)
def heartbeat(self):
self._exec(
f"UPDATE share_account SET locked_at = NOW() WHERE machine_id = '{self.machine_id}'",
action="心跳"
)
def start_heartbeat(self):
def _loop():
while not self._heartbeat_stop.is_set():
try:
self.heartbeat()
except Exception as e:
logging.warning(f"心跳失败: {e}")
self._heartbeat_stop.wait(self.heartbeat_interval)
self._heartbeat_thread = threading.Thread(target=_loop, daemon=True)
self._heartbeat_thread.start()
logging.info(f"心跳线程启动(每{self.heartbeat_interval}秒)")
def stop_heartbeat(self):
self._heartbeat_stop.set()
def release_all(self):
self._exec(
f"UPDATE share_account SET machine_id = NULL, locked_at = NULL "
f"WHERE machine_id = '{self.machine_id}'",
action="释放全部"
)
logging.info(f"释放本机全部账号")
def check_gateway(self):
"""轻量检查网关是否可达"""
try:
self._engine.read_sql("SELECT 1")
return True
except Exception:
return False
def get_my_accounts(self):
return self._query(
f"SELECT * FROM share_account WHERE machine_id = '{self.machine_id}' AND status = 1",
)
py_spider/comment_spider/human_behavior.py 0 → 100644
"""
人类行为模拟层 — 包装 Playwright Page 对象，让自动化操作像真人
核心能力:
1. 鼠标轨迹: Bezier曲线 + 加速/减速 + 微抖动 + 过冲修正
2. 键盘输入: 逐字符 + 变速延迟 + 偶尔停顿 + 打错修正
3. 行为注入: 随机滚动、悬停、阅读模拟
4. 时间随机: 高斯分布替代固定sleep
用法:
from human_behavior import HumanPage
hp = HumanPage(page)
hp.human_click('#submit')
hp.human_type('#name', 'John Smith')
hp.human_wait(2, 4)
"""
import math
import random
import time
def _bezier_curve(p0, p1, p2, p3, steps=30):
"""三次Bezier曲线生成鼠标路径点"""
points = []
for i in range(steps + 1):
t = i / steps
u = 1 - t
x = u**3 * p0[0] + 3 * u**2 * t * p1[0] + 3 * u * t**2 * p2[0] + t**3 * p3[0]
y = u**3 * p0[1] + 3 * u**2 * t * p1[1] + 3 * u * t**2 * p2[1] + t**3 * p3[1]
points.append((x, y))
return points
def _add_jitter(points, intensity=1.5):
"""给路径点加微抖动（模拟手部微颤）"""
result = [points[0]]
for i in range(1, len(points) - 1):
x, y = points[i]
jx = x + random.gauss(0, intensity)
jy = y + random.gauss(0, intensity)
result.append((jx, jy))
result.append(points[-1])
return result
def _ease_timing(n, total_time_ms=300):
"""生成缓入缓出的时间间隔（中间快两头慢）"""
if n <= 1:
return [total_time_ms]
raw = []
for i in range(n):
t = i / (n - 1)
speed = 0.3 + 0.7 * math.sin(t * math.pi)
raw.append(1.0 / max(speed, 0.1))
s = sum(raw)
return [r / s * total_time_ms for r in raw]
def _gauss_wait(center, spread=0.3):
"""高斯分布等待，center为中心秒数，spread为标准差比例"""
t = max(0.1, random.gauss(center, center * spread))
time.sleep(t)
return t
class HumanPage:
"""包装 Playwright Page，所有交互走人类行为模拟"""
def __init__(self, page):
self.page = page
self._mouse_x = random.randint(300, 800)
self._mouse_y = random.randint(200, 500)
self._last_action = time.time()
@property
def url(self):
return self.page.url
def _move_mouse_to(self, target_x, target_y, duration_ms=None):
"""Bezier曲线移动鼠标到目标位置"""
sx, sy = self._mouse_x, self._mouse_y
tx, ty = target_x, target_y
dist = math.hypot(tx - sx, ty - sy)
if dist < 5:
return
if duration_ms is None:
duration_ms = max(150, min(800, dist * 1.5 + random.gauss(100, 30)))
mid_x = (sx + tx) / 2
mid_y = (sy + ty) / 2
spread = max(30, dist * 0.3)
cp1 = (sx + (mid_x - sx) * 0.3 + random.gauss(0, spread * 0.5),
sy + (mid_y - sy) * 0.3 + random.gauss(0, spread * 0.5))
cp2 = (tx - (tx - mid_x) * 0.3 + random.gauss(0, spread * 0.3),
ty - (ty - mid_y) * 0.3 + random.gauss(0, spread * 0.3))
steps = max(15, min(60, int(dist / 8)))
if random.random() < 0.15 and dist > 100:
overshoot = random.uniform(3, 12)
angle = math.atan2(ty - sy, tx - sx)
otx = tx + math.cos(angle) * overshoot
oty = ty + math.sin(angle) * overshoot
pts1 = _bezier_curve((sx, sy), cp1, cp2, (otx, oty), steps=steps - 5)
pts2 = _bezier_curve((otx, oty),
(otx + random.gauss(0, 2), oty + random.gauss(0, 2)),
(tx + random.gauss(0, 1), ty + random.gauss(0, 1)),
(tx, ty), steps=5)
path = pts1 + pts2[1:]
else:
path = _bezier_curve((sx, sy), cp1, cp2, (tx, ty), steps=steps)
path = _add_jitter(path, intensity=max(0.5, min(2.0, dist * 0.01)))
delays = _ease_timing(len(path), duration_ms)
for i, (px, py) in enumerate(path):
try:
self.page.mouse.move(px, py)
except Exception:
break
if i < len(delays):
time.sleep(delays[i] / 1000.0)
self._mouse_x = tx
self._mouse_y = ty
def _get_element_center(self, selector, timeout=5000):
"""获取元素中心坐标（加随机偏移，不总是正中心）"""
try:
loc = self.page.locator(selector)
if loc.count() == 0:
return None
box = loc.first.bounding_box(timeout=timeout)
if not box:
return None
offset_x = random.gauss(0, box['width'] * 0.15)
offset_y = random.gauss(0, box['height'] * 0.15)
cx = box['x'] + box['width'] / 2 + offset_x
cy = box['y'] + box['height'] / 2 + offset_y
cx = max(box['x'] + 2, min(box['x'] + box['width'] - 2, cx))
cy = max(box['y'] + 2, min(box['y'] + box['height'] - 2, cy))
return cx, cy
except Exception:
return None
def human_click(self, selector, timeout=5000, pre_hover_ms=None):
"""人类式点击: 移动鼠标→悬停→按下→松开"""
pos = self._get_element_center(selector, timeout)
if not pos:
try:
self.page.locator(selector).first.click(timeout=timeout)
except Exception:
return False
return True
cx, cy = pos
self._move_mouse_to(cx, cy)
if pre_hover_ms is None:
pre_hover_ms = random.gauss(120, 40)
time.sleep(max(30, pre_hover_ms) / 1000.0)
try:
down_delay = random.gauss(80, 20)
self.page.mouse.down()
time.sleep(max(30, down_delay) / 1000.0)
self.page.mouse.up()
except Exception:
try:
self.page.locator(selector).first.click(timeout=timeout)
except Exception:
return False
self._last_action = time.time()
return True
def human_type(self, selector, text, click_first=True, wpm=None):
"""人类式输入: 先点击输入框→逐字符输入(变速)→偶尔停顿"""
if click_first:
self.human_click(selector)
_gauss_wait(0.3, 0.3)
if wpm is None:
wpm = random.gauss(55, 15)
base_delay = 60.0 / max(20, wpm) / 5.0
for i, ch in enumerate(text):
try:
self.page.keyboard.type(ch, delay=0)
except Exception:
break
delay = base_delay * random.gauss(1.0, 0.25)
if ch == ' ':
delay *= random.uniform(1.2, 2.0)
elif ch in '.,!?@#':
delay *= random.uniform(1.5, 2.5)
elif i > 0 and text[i-1].isupper() and ch.islower():
delay *= random.uniform(1.1, 1.5)
if random.random() < 0.03 and i > 2 and i < len(text) - 2:
delay += random.uniform(0.3, 0.8)
time.sleep(max(0.02, delay))
self._last_action = time.time()
def human_fill(self, selector, text, clear_first=True):
"""人类式填写: 点击→全选→删除→逐字输入"""
self.human_click(selector)
_gauss_wait(0.2, 0.3)
if clear_first:
try:
self.page.keyboard.press("Control+a")
time.sleep(random.uniform(0.05, 0.15))
self.page.keyboard.press("Backspace")
time.sleep(random.uniform(0.1, 0.3))
except Exception:
pass
self.human_type(selector, text, click_first=False)
def human_wait(self, min_s=1.0, max_s=3.0):
"""人类式等待: 高斯分布 + 偶尔微小鼠标移动"""
center = (min_s + max_s) / 2
spread = (max_s - min_s) / 4
total = max(min_s, random.gauss(center, spread))
if total > 2.0 and random.random() < 0.4:
part1 = total * random.uniform(0.3, 0.6)
time.sleep(part1)
dx = random.gauss(0, 30)
dy = random.gauss(0, 20)
try:
self.page.mouse.move(self._mouse_x + dx, self._mouse_y + dy)
self._mouse_x += dx
self._mouse_y += dy
except Exception:
pass
time.sleep(total - part1)
else:
time.sleep(total)
self._last_action = time.time()
def random_scroll(self, direction=None, amount=None):
"""随机滚动页面"""
if direction is None:
direction = random.choice(['down', 'down', 'down', 'up'])
if amount is None:
amount = random.randint(100, 400)
if direction == 'up':
amount = -amount
try:
self.page.mouse.wheel(0, amount)
except Exception:
try:
self.page.evaluate(f"window.scrollBy(0, {amount})")
except Exception:
pass
time.sleep(random.uniform(0.3, 0.8))
def simulate_reading(self, duration=3.0):
"""模拟阅读: 小幅鼠标移动 + 偶尔滚动"""
end_time = time.time() + duration
while time.time() < end_time:
action = random.random()
if action < 0.4:
dx = random.gauss(0, 50)
dy = random.gauss(0, 30)
try:
nx = max(50, min(1400, self._mouse_x + dx))
ny = max(50, min(900, self._mouse_y + dy))
self._move_mouse_to(nx, ny, duration_ms=random.randint(200, 500))
except Exception:
pass
elif action < 0.55:
self.random_scroll(amount=random.randint(50, 200))
else:
time.sleep(random.uniform(0.5, 1.5))
def before_action_pause(self):
"""在关键动作前的自然停顿（模拟视觉搜索+决策时间）"""
elapsed = time.time() - self._last_action
if elapsed < 0.5:
_gauss_wait(random.uniform(0.3, 0.8))
if random.random() < 0.3:
dx = random.gauss(0, 20)
dy = random.gauss(0, 15)
try:
self.page.mouse.move(self._mouse_x + dx, self._mouse_y + dy)
self._mouse_x += dx
self._mouse_y += dy
except Exception:
pass
time.sleep(random.uniform(0.1, 0.3))
def move_to_random(self):
"""鼠标移到随机位置"""
tx = random.randint(100, 1200)
ty = random.randint(100, 700)
self._move_mouse_to(tx, ty)
py_spider/comment_spider/profile_manager.py 0 → 100644
"""
指纹浏览器环境管理器 v3 — fingerprint-chromium + 反检测增强
架构:
fingerprint-chromium (C++级): Canvas/WebGL/Audio/Fonts/UA/WebRTC/TLS/CDP
_build_minimal_fingerprint_script (JS补充): 以下fingerprint-chromium未覆盖的项
- Function.prototype.toString伪装 (防止hook被fn.toString()检测)
- DOMRect微扰动 (getBoundingClientRect加1e-6噪声，Arkose检测此项)
- Audio双拦截 (DynamicsCompressor.reduction + getChannelData)
- Geolocation/Battery/Speech/MediaDevices/端口扫描保护
_build_fingerprint_script (JS全量): fallback用，fingerprint-chromium不可用时的完整JS注入
启动参数:
--fingerprint={seed} 确定性指纹种子
--fingerprint-storage-quota=2147483648 绕过隐私模式检测
--ignore_default_args 去掉Patchright的--no-sandbox和--disable-blink-features警告条
"""
import json
import random
import hashlib
import shutil
from pathlib import Path
from datetime import datetime
from dataclasses import asdict
from browserforge.fingerprints import FingerprintGenerator
import threading
_thread_local = threading.local()
def _get_thread_playwright():
"""每个线程独立的 Playwright 实例（线程本地存储，创建后不销毁）"""
if not hasattr(_thread_local, 'pw') or _thread_local.pw is None:
from patchright.sync_api import sync_playwright
_thread_local.pw = sync_playwright().start()
return _thread_local.pw
BASE_DIR = Path(__file__).parent / "browser_profiles"
FINGERPRINT_CHROME_DIR = Path(__file__).parent / "fingerprint-chromium" / "ungoogled-chromium_148.0.7778.215-1.1_windows_x64"
FINGERPRINT_CHROME_EXE = FINGERPRINT_CHROME_DIR / "chrome.exe"
US_TIMEZONES = [
("America/New_York", 40.7128, -74.0060),
("America/Chicago", 41.8781, -87.6298),
("America/Denver", 39.7392, -104.9903),
("America/Los_Angeles", 34.0522, -118.2437),
("America/Phoenix", 33.4484, -112.0740),
("America/Anchorage", 61.2181, -149.9003),
("Pacific/Honolulu", 21.3069, -157.8583),
]
_ip_tz_cache = None
def _detect_ip_timezone():
"""检测出口IP的实际时区和地理位置 — 随机选时区会被Pixelscan检出timezone spoofed"""
global _ip_tz_cache
if _ip_tz_cache:
return _ip_tz_cache
try:
import urllib.request, json as _json
data = _json.loads(urllib.request.urlopen(
"http://ip-api.com/json/?fields=timezone,lat,lon,city,regionName", timeout=10
).read())
tz = data.get("timezone", "")
lat = data.get("lat", 0)
lon = data.get("lon", 0)
if tz:
_ip_tz_cache = (tz, lat, lon)
print(f"[IP-TZ] Detected: {tz} ({data.get('city')}, {data.get('regionName')}) [{lat:.2f},{lon:.2f}]")
return _ip_tz_cache
except Exception as e:
print(f"[IP-TZ] Detection failed: {e}, using fallback")
for tz_name, lat, lng in US_TIMEZONES:
if "Los_Angeles" in tz_name:
_ip_tz_cache = (tz_name, lat, lng)
return _ip_tz_cache
_ip_tz_cache = US_TIMEZONES[3]
return _ip_tz_cache
GPU_POOL = [
("ANGLE (NVIDIA, NVIDIA GeForce GTX 1060 6GB Direct3D11 vs_5_0 ps_5_0, D3D11)", "Google Inc. (NVIDIA)"),
("ANGLE (NVIDIA, NVIDIA GeForce GTX 1650 Direct3D11 vs_5_0 ps_5_0, D3D11)", "Google Inc. (NVIDIA)"),
("ANGLE (NVIDIA, NVIDIA GeForce GTX 1660 Ti Direct3D11 vs_5_0 ps_5_0, D3D11)", "Google Inc. (NVIDIA)"),
("ANGLE (NVIDIA, NVIDIA GeForce RTX 2060 Direct3D11 vs_5_0 ps_5_0, D3D11)", "Google Inc. (NVIDIA)"),
("ANGLE (NVIDIA, NVIDIA GeForce RTX 2070 SUPER Direct3D11 vs_5_0 ps_5_0, D3D11)", "Google Inc. (NVIDIA)"),
("ANGLE (NVIDIA, NVIDIA GeForce RTX 3060 Direct3D11 vs_5_0 ps_5_0, D3D11)", "Google Inc. (NVIDIA)"),
("ANGLE (NVIDIA, NVIDIA GeForce RTX 3070 Direct3D11 vs_5_0 ps_5_0, D3D11)", "Google Inc. (NVIDIA)"),
("ANGLE (NVIDIA, NVIDIA GeForce RTX 3080 Direct3D11 vs_5_0 ps_5_0, D3D11)", "Google Inc. (NVIDIA)"),
("ANGLE (NVIDIA, NVIDIA GeForce RTX 4060 Direct3D11 vs_5_0 ps_5_0, D3D11)", "Google Inc. (NVIDIA)"),
("ANGLE (NVIDIA, NVIDIA GeForce RTX 4070 Direct3D11 vs_5_0 ps_5_0, D3D11)", "Google Inc. (NVIDIA)"),
("ANGLE (AMD, AMD Radeon RX 580 Direct3D11 vs_5_0 ps_5_0, D3D11)", "Google Inc. (AMD)"),
("ANGLE (AMD, AMD Radeon RX 5700 XT Direct3D11 vs_5_0 ps_5_0, D3D11)", "Google Inc. (AMD)"),
("ANGLE (AMD, AMD Radeon RX 6600 XT Direct3D11 vs_5_0 ps_5_0, D3D11)", "Google Inc. (AMD)"),
("ANGLE (AMD, AMD Radeon RX 6700 XT Direct3D11 vs_5_0 ps_5_0, D3D11)", "Google Inc. (AMD)"),
("ANGLE (AMD, AMD Radeon RX 7600 Direct3D11 vs_5_0 ps_5_0, D3D11)", "Google Inc. (AMD)"),
("ANGLE (Intel, Intel(R) UHD Graphics 620 Direct3D11 vs_5_0 ps_5_0, D3D11)", "Google Inc. (Intel)"),
("ANGLE (Intel, Intel(R) UHD Graphics 630 Direct3D11 vs_5_0 ps_5_0, D3D11)", "Google Inc. (Intel)"),
("ANGLE (Intel, Intel(R) UHD Graphics 770 Direct3D11 vs_5_0 ps_5_0, D3D11)", "Google Inc. (Intel)"),
("ANGLE (Intel, Intel(R) Iris(R) Xe Graphics Direct3D11 vs_5_0 ps_5_0, D3D11)", "Google Inc. (Intel)"),
]
FONT_POOL = [
"Arial","Arial Black","Calibri","Cambria","Cambria Math","Comic Sans MS",
"Consolas","Constantia","Courier New","Georgia","Helvetica","Impact",
"Lucida Console","Lucida Sans Unicode","Microsoft Sans Serif",
"Palatino Linotype","Segoe UI","Tahoma","Times New Roman","Trebuchet MS",
"Verdana","Garamond","Book Antiqua","Century Gothic","Franklin Gothic Medium",
"Candara","Corbel","Ebrima","Gabriola","Leelawadee UI","Malgun Gothic",
"Microsoft YaHei","Nirmala UI","Sylfaen","Yu Gothic",
]
# 非标准分辨率(如1707x1067)乘DPR会出现小数像素(2560.5x1600.5)，直接暴露伪造
STANDARD_SCREENS = [
(1920, 1080, 1),
(1536, 864, 1.25),
(1440, 900, 2),
(1366, 768, 1),
(1280, 720, 1),
(1600, 900, 1),
(1680, 1050, 1),
(2560, 1440, 1),
(1920, 1200, 1),
(1280, 800, 1),
]
SPEECH_VOICES = [
"Microsoft David - English (United States)",
"Microsoft Mark - English (United States)",
"Microsoft Zira - English (United States)",
"Google US English",
"Google UK English Female",
"Google UK English Male",
]
def _build_fingerprint_script(extra):
"""v2 指纹注入 — 所有覆盖走Prototype级别(躲避own-property检测)，Canvas用getContext拦截(不碰toDataURL原型避免toString泄露)"""
seed = extra["canvas_seed"]
renderer = extra["gpu_renderer"]
vendor = extra["gpu_vendor"]
hw = extra["hw_concurrency"]
mem = extra["device_memory"]
cd = extra["color_depth"]
fonts = json.dumps(extra["fonts"])
voices = json.dumps(extra["speech_voices"])
lat = extra["geo_lat"]
lng = extra["geo_lng"]
battery_level = extra["battery_level"]
battery_charging = "true" if extra["battery_charging"] else "false"
media_devices = json.dumps(extra["media_devices"])
dnt = extra["dnt"]
rtt = random.choice([50, 100, 150, 200, 250, 300])
downlink = random.choice([1.5, 2.5, 5, 10, 15, 20])
# 用f-string拼接JS，避免复杂的字符串拼接
js = f"""(function() {{
const SEED = {seed};
function rng32(s) {{
return function() {{
s |= 0; s = s + 0x6D2B79F5 | 0;
var t = Math.imul(s ^ s >>> 15, 1 | s);
t = t + Math.imul(t ^ t >>> 7, 61 | t) ^ t;
return ((t ^ t >>> 14) >>> 0) / 4294967296;
}}
}}
const rng = rng32(SEED);
// ======== Canvas噪声 (通过getContext拦截，不改toDataURL原型) ========
const _getCtx = HTMLCanvasElement.prototype.getContext;
HTMLCanvasElement.prototype.getContext = function(type, attrs) {{
const ctx = _getCtx.apply(this, arguments);
if (ctx && type === '2d' && !ctx._ns) {{
ctx._ns = true;
const _gid = ctx.getImageData.bind(ctx);
ctx.getImageData = function(sx, sy, sw, sh) {{
const d = _gid(sx, sy, sw, sh);
if (sw * sh < 500) {{
for (let i = 0; i < d.data.length; i += 4)
d.data[i] = (d.data[i] + Math.floor(rng() * 3 - 1) + 256) % 256;
}}
return d;
}};
}}
return ctx;
}};
// ======== WebGL (Prototype级 + 容错null) ========
const RENDERER = '{renderer}';
const VENDOR = '{vendor}';
const _glGP = WebGLRenderingContext.prototype.getParameter;
Object.defineProperty(WebGLRenderingContext.prototype, 'getParameter', {{
value: function(p) {{
if (p === 0x9245) return VENDOR;
if (p === 0x9246) return RENDERER;
try {{ const r = _glGP.apply(this, arguments); return r !== null ? r : RENDERER; }}
catch(e) {{ return null; }}
}}, writable: true, configurable: true
}});
try {{
const _gl2GP = WebGL2RenderingContext.prototype.getParameter;
Object.defineProperty(WebGL2RenderingContext.prototype, 'getParameter', {{
value: function(p) {{
if (p === 0x9245) return VENDOR;
if (p === 0x9246) return RENDERER;
try {{ const r = _gl2GP.apply(this, arguments); return r !== null ? r : RENDERER; }}
catch(e) {{ return null; }}
}}, writable: true, configurable: true
}});
}} catch(e) {{}}
// ======== Audio噪声 (轻量) ========
const _gffd = AnalyserNode.prototype.getFloatFrequencyData;
AnalyserNode.prototype.getFloatFrequencyData = function(a) {{
_gffd.apply(this, arguments);
for (let i = 0; i < a.length; i++) a[i] += (rng() - 0.5) * 0.05;
}};
// ======== 硬件 (Prototype级 + 双层保障) ========
try {{ Object.defineProperty(Navigator.prototype, 'hardwareConcurrency', {{get: () => {hw}, configurable: true}}); }} catch(e) {{}}
try {{ Object.defineProperty(Navigator.prototype, 'deviceMemory', {{get: () => {mem}, configurable: true}}); }} catch(e) {{}}
// 备用: 如果 Prototype 级别不生效, 在实例上也加
if (navigator.deviceMemory !== {mem}) {{
try {{ Object.defineProperty(navigator, 'deviceMemory', {{get: () => {mem}, configurable: true}}); }} catch(e) {{}}
}}
if (navigator.hardwareConcurrency !== {hw}) {{
try {{ Object.defineProperty(navigator, 'hardwareConcurrency', {{get: () => {hw}, configurable: true}}); }} catch(e) {{}}
}}
try {{ Object.defineProperty(Screen.prototype, 'colorDepth', {{get: () => {cd}, configurable: true}}); }} catch(e) {{}}
try {{ Object.defineProperty(Screen.prototype, 'pixelDepth', {{get: () => {cd}, configurable: true}}); }} catch(e) {{}}
// ======== WebRTC (relay策略，不重写构造函数) ========
if (window.RTCPeerConnection) {{
const _RTC = window.RTCPeerConnection;
window.RTCPeerConnection = function(cfg) {{
if (cfg) {{ cfg.iceServers = cfg.iceServers || []; cfg.iceTransportPolicy = 'relay'; }}
return new _RTC(cfg);
}};
window.RTCPeerConnection.prototype = _RTC.prototype;
window.RTCPeerConnection.generateCertificate = _RTC.generateCertificate;
}}
// ======== Geolocation ========
if (navigator.geolocation) {{
navigator.geolocation.getCurrentPosition = function(success) {{
success({{
coords: {{
latitude: {lat} + (rng() - 0.5) * 0.01,
longitude: {lng} + (rng() - 0.5) * 0.01,
accuracy: 20 + Math.floor(rng() * 50),
altitude: null, altitudeAccuracy: null, heading: null, speed: null
}},
timestamp: Date.now()
}});
}};
navigator.geolocation.watchPosition = function(s) {{
navigator.geolocation.getCurrentPosition(s);
return Math.floor(rng() * 1000);
}};
}}
// ======== Speech ========
const VOICES = {voices};
if (window.speechSynthesis) {{
const fv = VOICES.map((n, i) => ({{name: n, lang: 'en-US', localService: i < 3, default: i === 0, voiceURI: n}}));
speechSynthesis.getVoices = () => fv;
}}
// ======== 媒体设备 ========
const DEVICES = {media_devices};
if (navigator.mediaDevices && navigator.mediaDevices.enumerateDevices)
navigator.mediaDevices.enumerateDevices = () => Promise.resolve(DEVICES);
// ======== 充电 ========
if (navigator.getBattery) {{
navigator.getBattery = () => Promise.resolve({{
charging: {battery_charging},
chargingTime: {battery_charging} ? Math.floor(rng() * 3600) : Infinity,
dischargingTime: {battery_charging} ? Infinity : 3600 + Math.floor(rng() * 14400),
level: {battery_level},
addEventListener: function() {{}},
}});
}}
// ======== 端口扫描保护 ========
const _fetch = window.fetch;
window.fetch = function(url, opts) {{
if (typeof url === 'string') {{
try {{
const u = new URL(url, location.href);
if (['localhost','127.0.0.1','0.0.0.0'].includes(u.hostname) || u.hostname.endsWith('.local'))
return Promise.reject(new TypeError('NetworkError'));
}} catch(e) {{}}
}}
return _fetch.apply(this, arguments);
}};
// ======== 字体 ========
const FONTS = new Set({fonts});
const _cf = document.fonts.check.bind(document.fonts);
document.fonts.check = function(font) {{
const f = font.replace(/^[\\d.]+[a-z]+\\s+/i, '').replace(/["']/g, '').split(',')[0].trim();
if (!FONTS.has(f) && !['serif','sans-serif','monospace','cursive','fantasy'].includes(f)) return false;
return _cf.apply(this, arguments);
}};
// ======== DNT (Prototype级) ========
Object.defineProperty(Navigator.prototype, 'doNotTrack', {{get: () => '{dnt}', configurable: true}});
// ======== SEC-CH-UA ========
if (navigator.userAgentData) {{
Object.defineProperty(navigator.userAgentData, 'platform', {{get: () => 'Windows'}});
Object.defineProperty(navigator.userAgentData, 'mobile', {{get: () => false}});
}}
// ======== Chrome对象完整模拟 ========
if (!window.chrome) window.chrome = {{}};
if (!window.chrome.runtime) {{
const noop = function(){{}};
const evt = {{addListener: noop, removeListener: noop, hasListener: () => false, hasListeners: () => false}};
window.chrome.runtime = {{
id: undefined,
connect: function(info) {{
return {{name: '', onDisconnect: evt, onMessage: evt, postMessage: noop, disconnect: noop}};
}},
sendMessage: function() {{ if (arguments.length && typeof arguments[arguments.length-1] === 'function') arguments[arguments.length-1](); }},
onMessage: evt, onConnect: evt, onInstalled: evt,
getManifest: () => ({{}}), getURL: (p) => p,
PlatformOs: {{MAC:'mac',WIN:'win',ANDROID:'android',CROS:'cros',LINUX:'linux',OPENBSD:'openbsd'}},
RequestUpdateCheckStatus: {{THROTTLED:'throttled',NO_UPDATE:'no_update',UPDATE_AVAILABLE:'update_available'}},
}};
}}
if (!window.chrome.app) {{
window.chrome.app = {{isInstalled: false, InstallState: {{DISABLED:'disabled',INSTALLED:'installed',NOT_INSTALLED:'not_installed'}}, RunningState: {{CANNOT_RUN:'cannot_run',READY_TO_RUN:'ready_to_run',RUNNING:'running'}}}};
}}
if (!window.chrome.csi) window.chrome.csi = function() {{ return {{startE: Date.now(), onloadT: Date.now(), pageT: Date.now()}} }};
if (!window.chrome.loadTimes) window.chrome.loadTimes = function() {{ return {{requestTime: Date.now()/1000, startLoadTime: Date.now()/1000, commitLoadTime: Date.now()/1000, finishDocumentLoadTime: Date.now()/1000, firstPaintTime: Date.now()/1000, firstPaintAfterLoadTime: 0, navigationType: 'Other', wasFetchedViaSpdy: true, wasNpnNegotiated: true, npnNegotiatedProtocol: 'h2', wasAlternateProtocolAvailable: false, connectionInfo: 'h2'}} }};
// ======== Notification.permission 强制 default (反复覆盖) ========
try {{
Object.defineProperty(Notification, 'permission', {{
get: () => 'default', set: () => {{}}, configurable: true, enumerable: true
}});
}} catch(e) {{}}
// 备用: 如果 defineProperty 被 Chrome 覆盖回去，用 Proxy 拦截
try {{
const _Notif = window.Notification;
window.Notification = new Proxy(_Notif, {{
get(target, prop) {{
if (prop === 'permission') return 'default';
return Reflect.get(target, prop);
}}
}});
window.Notification.prototype = _Notif.prototype;
}} catch(e) {{}}
// ======== outerWidth/Height ========
if (window.outerWidth === 0) {{
Object.defineProperty(window, 'outerWidth', {{get: () => screen.width + 16}});
Object.defineProperty(window, 'outerHeight', {{get: () => screen.height + 134}});
}}
// ======== 网络连接 ========
try {{
if (navigator.connection) {{
Object.defineProperty(navigator.connection, 'rtt', {{get: () => {rtt}, configurable: true}});
Object.defineProperty(navigator.connection, 'downlink', {{get: () => {downlink}, configurable: true}});
}}
}} catch(e) {{}}
// ======== Error堆栈清洗 (移除 UtilityScript/evaluate 等自动化痕迹) ========
const _origStack = Object.getOwnPropertyDescriptor(Error.prototype, 'stack');
if (_origStack && _origStack.get) {{
Object.defineProperty(Error.prototype, 'stack', {{
get() {{
const s = _origStack.get.call(this);
if (typeof s !== 'string') return s;
return s.replace(/UtilityScript\\.evaluate/g, 'Object.evaluate')
.replace(/<anonymous>:\\d+:\\d+/g, (m) => m)
.replace(/\\n\\s+at UtilityScript\\.[^\\n]*/g, '');
}},
set(v) {{ if (_origStack.set) _origStack.set.call(this, v); }},
configurable: true
}});
}}
}})();"""
return js
def _build_minimal_fingerprint_script(extra):
"""fingerprint-chromium已处理Canvas/WebGL/Audio/Fonts/UA/WebRTC/CDP/TLS/ClientRects等，只补它没覆盖的
v5: toString伪装 / Geo / Battery / Speech / MediaDevices / 端口保护
(Audio/DOMRect已由C++ patch 003/014处理，JS层不再重复hook)"""
seed = extra.get("canvas_seed", 12345)
lat = extra.get("geo_lat", 34.05)
lng = extra.get("geo_lng", -118.24)
battery_level = extra.get("battery_level", 0.85)
battery_charging = "true" if extra.get("battery_charging", True) else "false"
media_devices = json.dumps(extra.get("media_devices", []))
voices = json.dumps(extra.get("speech_voices", []))
return f"""(function() {{
const SEED = {seed};
function rng32(s) {{
return function() {{
s |= 0; s = s + 0x6D2B79F5 | 0;
var t = Math.imul(s ^ s >>> 15, 1 | s);
t = t + Math.imul(t ^ t >>> 7, 61 | t) ^ t;
return ((t ^ t >>> 14) >>> 0) / 4294967296;
}}
}}
const rng = rng32(SEED);
// ======== P0: Function.prototype.toString伪装 ========
// 被hook的函数调用toString()会暴露proxy/bound代码，用原始toString返回native code
const _nativeToString = Function.prototype.toString;
const _hookedFns = new WeakMap();
function disguise(hooked, original) {{
_hookedFns.set(hooked, original || hooked);
}}
Function.prototype.toString = function() {{
const orig = _hookedFns.get(this);
if (orig) return _nativeToString.call(orig);
return _nativeToString.call(this);
}};
disguise(Function.prototype.toString, _nativeToString);
// ======== Geolocation坐标 ========
if (navigator.geolocation) {{
const _geoOrig = navigator.geolocation.getCurrentPosition;
const _lat = {lat}, _lng = {lng};
navigator.geolocation.getCurrentPosition = function(success) {{
success({{
coords: {{
latitude: _lat + (rng() - 0.5) * 0.01,
longitude: _lng + (rng() - 0.5) * 0.01,
accuracy: 20 + Math.floor(rng() * 50),
altitude: null, altitudeAccuracy: null, heading: null, speed: null
}},
timestamp: Date.now()
}});
}};
disguise(navigator.geolocation.getCurrentPosition, _geoOrig);
const _watchOrig = navigator.geolocation.watchPosition;
navigator.geolocation.watchPosition = function(s) {{
navigator.geolocation.getCurrentPosition(s);
return Math.floor(rng() * 1000);
}};
disguise(navigator.geolocation.watchPosition, _watchOrig);
}}
// ======== Battery ========
if (navigator.getBattery) {{
const _batOrig = navigator.getBattery;
navigator.getBattery = () => Promise.resolve({{
charging: {battery_charging},
chargingTime: {battery_charging} ? Math.floor(rng() * 3600) : Infinity,
dischargingTime: {battery_charging} ? Infinity : 3600 + Math.floor(rng() * 14400),
level: {battery_level},
addEventListener: function() {{}},
}});
disguise(navigator.getBattery, _batOrig);
}}
// ======== Speech ========
const VOICES = {voices};
if (window.speechSynthesis && VOICES.length) {{
const fv = VOICES.map((n, i) => ({{name: n, lang: 'en-US', localService: i < 3, default: i === 0, voiceURI: n}}));
const _svOrig = speechSynthesis.getVoices;
speechSynthesis.getVoices = () => fv;
disguise(speechSynthesis.getVoices, _svOrig);
}}
// ======== Media devices ========
const DEVICES = {media_devices};
if (navigator.mediaDevices && navigator.mediaDevices.enumerateDevices && DEVICES.length) {{
const _edOrig = navigator.mediaDevices.enumerateDevices;
navigator.mediaDevices.enumerateDevices = () => Promise.resolve(DEVICES);
disguise(navigator.mediaDevices.enumerateDevices, _edOrig);
}}
// ======== 端口扫描保护 ========
const _fetch = window.fetch;
window.fetch = function(url, opts) {{
if (typeof url === 'string') {{
try {{
const u = new URL(url, location.href);
if (['localhost','127.0.0.1','0.0.0.0'].includes(u.hostname) || u.hostname.endsWith('.local'))
return Promise.reject(new TypeError('NetworkError'));
}} catch(e) {{}}
}}
return _fetch.apply(this, arguments);
}};
disguise(window.fetch, _fetch);
}})();"""
class ProfileManager:
def __init__(self, base_dir=None):
self.base = Path(base_dir) if base_dir else BASE_DIR
self.data_dir = self.base / "data"
self.index_path = self.base / "profiles.json"
self.base.mkdir(parents=True, exist_ok=True)
self.data_dir.mkdir(exist_ok=True)
self._load_index()
def _load_index(self):
if self.index_path.exists():
self.index = json.loads(self.index_path.read_text(encoding="utf-8-sig"))
else:
self.index = {}
def _save_index(self):
self.index_path.write_text(
json.dumps(self.index, indent=2, ensure_ascii=False), encoding="utf-8"
)
def create(self, name, proxy=None, account=None):
if name in self.index:
print(f"[ProfileManager] '{name}' already exists")
return self.get_config(name)
profile_dir = self.data_dir / name
profile_dir.mkdir(parents=True, exist_ok=True)
(profile_dir / "chrome_data").mkdir(exist_ok=True)
tz, geo_lat, geo_lng = _detect_ip_timezone()
fg = FingerprintGenerator(browser="chrome")
fp = fg.generate(os=("windows",), locale=("en-US",))
fp_dict = asdict(fp)
seed = int(hashlib.sha256(name.encode()).hexdigest()[:8], 16)
gpu = random.choice(GPU_POOL)
hw = random.choice([4, 6, 8, 8, 10, 12, 16])
mem = random.choice([4, 8, 8, 16, 16, 32])
cd = random.choice([24, 24, 32, 32, 32])
fonts = sorted(random.sample(FONT_POOL, random.randint(16, 24)))
voices = random.sample(SPEECH_VOICES, random.randint(3, 5))
device_id_base = hashlib.md5(f"{name}_device".encode()).hexdigest()
media_devices = [
{"deviceId": device_id_base[:16], "groupId": device_id_base[16:32],
"kind": "audioinput", "label": ""},
{"deviceId": hashlib.md5(f"{name}_speaker".encode()).hexdigest()[:16],
"groupId": device_id_base[16:32],
"kind": "audiooutput", "label": ""},
]
if random.random() > 0.3:
media_devices.append({
"deviceId": hashlib.md5(f"{name}_cam".encode()).hexdigest()[:16],
"groupId": hashlib.md5(f"{name}_camg".encode()).hexdigest()[:16],
"kind": "videoinput", "label": ""
})
fp_dict["_extra"] = {
"canvas_seed": seed,
"gpu_renderer": gpu[0],
"gpu_vendor": gpu[1],
"hw_concurrency": hw,
"device_memory": mem,
"color_depth": cd,
"fonts": fonts,
"speech_voices": voices,
"geo_lat": geo_lat + (random.random() - 0.5) * 0.1,
"geo_lng": geo_lng + (random.random() - 0.5) * 0.1,
"battery_level": round(random.uniform(0.2, 1.0), 2),
"battery_charging": random.random() > 0.4,
"media_devices": media_devices,
"dnt": random.choice(["1", "null", "unspecified"]),
}
(profile_dir / "fingerprint.json").write_text(
json.dumps(fp_dict, indent=2, ensure_ascii=False), encoding="utf-8"
)
config = {
"name": name,
"created": datetime.now().isoformat(),
"os": ["windows"],
"timezone": tz,
"proxy": proxy,
"account": account or {},
"status": "active",
"last_used": None,
"use_count": 0,
"ua": fp_dict.get("navigator", {}).get("userAgent", ""),
}
(profile_dir / "config.json").write_text(
json.dumps(config, indent=2, ensure_ascii=False), encoding="utf-8"
)
self.index[name] = {
"created": config["created"],
"os": config["os"],
"ua": config["ua"][:60],
"proxy": proxy or "direct",
"status": "active",
}
self._save_index()
screen = fp_dict.get("screen", {})
print(f"[+] Created: {name}")
print(f" Screen={screen.get('width')}x{screen.get('height')} GPU={gpu[0][:45]}")
print(f" UA={config['ua'][:70]}")
print(f" TZ={tz} HW={hw}core/{mem}GB CD={cd} Fonts={len(fonts)}")
print(f" Geo=({geo_lat:.2f},{geo_lng:.2f}) Proxy={proxy or 'direct'}")
return config
def launch(self, name, headless=False):
profile_dir = self.data_dir / name
config = json.loads((profile_dir / "config.json").read_text(encoding="utf-8"))
fp_dict = json.loads((profile_dir / "fingerprint.json").read_text(encoding="utf-8"))
nav = fp_dict.get("navigator", {})
screen = fp_dict.get("screen", {})
extra = fp_dict.get("_extra", {})
user_data_dir = str(profile_dir / "chrome_data")
tz = config.get("timezone", "America/Los_Angeles")
config["last_used"] = datetime.now().isoformat()
config["use_count"] = config.get("use_count", 0) + 1
(profile_dir / "config.json").write_text(
json.dumps(config, indent=2, ensure_ascii=False), encoding="utf-8"
)
seed = extra.get("canvas_seed", int(hashlib.sha256(name.encode()).hexdigest()[:8], 16))
hw = extra.get("hw_concurrency", 8)
mem = extra.get("device_memory", 8)
pw = _get_thread_playwright()
scr_w = screen.get("width", 1920)
scr_h = screen.get("height", 1080)
dpr = screen.get("devicePixelRatio", 1) or 1
if (scr_w * dpr) % 1 != 0 or (scr_h * dpr) % 1 != 0 or scr_w not in [s[0] for s in STANDARD_SCREENS]:
std = random.choice(STANDARD_SCREENS)
scr_w, scr_h, dpr = std
use_fp_chrome = FINGERPRINT_CHROME_EXE.exists()
if use_fp_chrome:
launch_args = [
f"--fingerprint={seed}",
"--fingerprint-platform=windows",
"--fingerprint-brand=Chrome",
f"--fingerprint-hardware-concurrency={hw}",
f"--fingerprint-device-memory={mem}",
f"--timezone={tz}",
"--lang=en-US",
"--accept-lang=en-US,en",
"--fingerprint-storage-quota=2147483648",
]
ctx_args = {
"executable_path": str(FINGERPRINT_CHROME_EXE),
"headless": headless,
"viewport": {"width": scr_w, "height": scr_h},
"locale": "en-US",
"timezone_id": tz,
"color_scheme": "light",
"device_scale_factor": dpr,
"args": launch_args,
"ignore_default_args": ["--no-sandbox", "--disable-blink-features=AutomationControlled"],
"permissions": ["geolocation", "notifications"],
"geolocation": {"latitude": extra.get("geo_lat", 34.05), "longitude": extra.get("geo_lng", -118.24)},
}
else:
launch_args = [
"--disable-blink-features=AutomationControlled",
"--force-webrtc-ip-handling-policy=disable_non_proxied_udp",
"--enforce-webrtc-ip-permission-check",
]
ctx_args = {
"channel": "chrome",
"headless": headless,
"user_agent": nav.get("userAgent", ""),
"viewport": {"width": scr_w, "height": scr_h},
"locale": "en-US",
"timezone_id": tz,
"color_scheme": "light",
"device_scale_factor": dpr,
"args": launch_args,
"permissions": ["geolocation", "notifications"],
"geolocation": {"latitude": extra.get("geo_lat", 34.05), "longitude": extra.get("geo_lng", -118.24)},
}
if config.get("proxy"):
ctx_args["proxy"] = {"server": config["proxy"]}
context = pw.chromium.launch_persistent_context(user_data_dir, **ctx_args)
if use_fp_chrome:
context.add_init_script(_build_minimal_fingerprint_script(extra))
print(f" [FP-Chrome] seed={seed} OS=windows TZ={tz} HW={hw}core use_count={config['use_count']}")
else:
context.add_init_script(_build_fingerprint_script(extra))
print(f" OS={config['os']} TZ={tz} use_count={config['use_count']}")
return _PatchrightBrowser(pw, None, context), config
def save_cookies(self, name, browser_or_ctx):
print(f"[ProfileManager] Cookies auto-saved (persistent): {name}")
def load_cookies(self, name, browser_or_ctx):
print(f"[ProfileManager] Cookies auto-loaded (persistent): {name}")
return True
def get_config(self, name):
return json.loads(
(self.data_dir / name / "config.json").read_text(encoding="utf-8")
)
def list_profiles(self):
if not self.index:
print("No profiles yet.")
return []
names = []
for name, info in self.index.items():
print(f" {name}: OS={info['os']} proxy={info.get('proxy','direct')} status={info['status']}")
if info.get("status") == "active":
names.append(name)
return names
def get_active_profiles(self):
return [n for n, info in self.index.items() if info.get("status") == "active"]
def delete(self, name):
if name in self.index:
d = self.data_dir / name
if d.exists():
shutil.rmtree(d)
del self.index[name]
self._save_index()
print(f"[-] Deleted: {name}")
def batch_create(self, prefix, count, proxies=None):
results = []
for i in range(count):
name = f"{prefix}_{i+1:03d}"
proxy = proxies[i] if proxies and i < len(proxies) else None
cfg = self.create(name, proxy=proxy)
results.append(cfg)
return results
class _PatchrightBrowser:
def __init__(self, pw, browser, context):
self._pw = pw
self._browser = browser
self.context = context
def new_page(self):
pages = self.context.pages
if pages:
return pages[0]
return self.context.new_page()
def close(self):
try:
self.context.close()
except Exception:
pass
if self._browser:
try:
self._browser.close()
except Exception:
pass
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
py_spider/comment_spider/review_parser.py 0 → 100644
"""评论解析器 — 从 comment_dp_spider.py 抽取的纯解析函数
无浏览器依赖(无DrissionPage/Patchright)，只依赖 parsel + re + json
包含：
- xpath_res: 评论 HTML 解析（25个字段）
- _build_api_data: 下游标准数据格式
- if_response: 页面异常检测（有评论数但解析不到→异常）
- time_ch / comment_time: 多语言日期解析（12种语言）
- _extract_numbers / _dedup_comments / get_page_num: 工具函数
"""
import re
import json
import logging
from datetime import datetime
from parsel import Selector
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
def _extract_numbers(text: str) -> list:
"""清洗 unicode 转义、标点后提取所有数字串"""
cleaned = (
re.sub(r"\\u.{4}", '', text.__repr__())
.replace(",", "").replace(".", "").replace("'", "")
)
return re.findall(r'(\d+)', cleaned)
def _dedup_comments(star_comment: list) -> list:
"""去重 + 填充空值"""
seen = set()
result = []
for item in star_comment:
key = (item.get('asin'), item.get('comment_id'))
if key in seen:
continue
seen.add(key)
if item.get('agree_num') is None:
item['agree_num'] = "0"
if item.get('img_num') is None:
item['img_num'] = "0"
result.append(item)
return result
def get_page_num(num: int) -> int:
"""评论数 → 最大页数（上限10页，即100条）"""
if num <= 10:
page = 2
elif num % 10:
page = num // 10 + 2
else:
page = num // 10 + 1
return min(page, 11)
# 各语言月份名称→数字（覆盖 tr/nl/us/de/es/fr/it/uk/ca/mx/au/be/ae）
_TIME_DICT = {
# tr
"Ağustos": "08", "Kasım": "11", "Aralık": "12", "Şubat": "02",
"Mart": "03", "Nisan": "04", "Temmuz": "07", "Ocak": "01",
"Ekim": "10", "Mayıs": "05", "Eylül": "09", "Haziran": "06",
# nl
"januari": "01", "december": "12", "april": "04", "oktober": '10',
"augustus": "08", "februari": '02', "maart": "03", "juli": "07",
"november": "11", "juni": '06', "mei": '05',
# us
"June": "06", "April": "04", "January": "01", "October": "10",
"November": "11", "August": "08", "March": "03", "December": "12",
"July": "07", "September": "09", "Feb": '02', "May": "05",
# de
"Januar": "01", "Februar": '02', "März": '03', "Mai": '05',
"Juni": '06', "Juli": '07', "september": "09", "Oktober": '10',
"Dezember": "12",
# es
"enero": '01', "febrero": '02', "marzo": '03', "abril": "04",
"mayo": "05", "junio": '06', "julio": "07", "agosto": "08",
"septiembre": '09', "octubre": '10', "noviembre": "11", "diciembre": "12",
# fr
"janvier": "01", "février": '02', "mars": "03", "avril": "04",
"mai": "05", "juin": "06", "juillet": "07", "août": "08",
"septembre": "09", "octobre": "10", "novembre": "11", "décembre": "12",
# it
"gennaio": "01", "febbraio": "02", "aprile": "04", "maggio": '05',
"giugno": "06", "luglio": "07", "settembre": '09', "ottobre": '10',
# uk abbr
"February": '02', "Aug": '08', "Jun": "06", "Mar": "03",
"Nov": "11", "Sept": "09", "Oct": "10", "Dec": "12", "Jan": "01",
}
def time_ch(site, time_msg) -> str:
"""日期格式转换 → YYYY-MM-DD（原样复制自同事代码，覆盖12种语言）"""
td = _TIME_DICT
if t := re.match(r"([0-9]{2})\.([0-9]{2})\.([0-9]{4})", time_msg):
return f"{t.group(3)}-{t.group(2)}-{t.group(1)}"
if site == "de":
if "/" in time_msg:
return time_msg.strip().replace("/", "-")
parts = time_msg.split(" ")
if not td.get(parts[1].strip()):
return ""
day = time_msg.split(".")[0].replace("\xa0", "").strip()
month = td[parts[1].strip()]
year = parts[-1]
elif site == "uk":
key = time_msg.split(" ")[1].strip().replace(".", "")
if not td.get(key):
return ""
day = time_msg.split(" ")[0].strip()
month = td[key]
year = time_msg.split(" ")[2]
elif site == "fr":
key = time_msg.split(" ")[1].strip()
if not td.get(key):
return ""
day = time_msg.split(" ")[0].strip()
month = td[key]
year = time_msg.split(" ")[2]
elif site == "es":
if len(time_msg.split(" ")) == 1:
return ""
key = time_msg.split(" ")[1].strip()
if not td.get(key):
return ""
day = time_msg.split(" ")[0].strip()
month = td[key]
year = time_msg.split(" ")[-1]
elif site == "it":
if len(time_msg) == 4:
return time_msg
key = time_msg.split(" ")[1].strip()
if not td.get(key):
return ""
day = time_msg.split(" ")[0].strip()
month = td[key]
year = time_msg.split(" ")[-1]
elif site == "ca":
key = time_msg.replace(".", "").split(" ")[0].strip()
if not td.get(key):
return ""
day = time_msg.split(" ")[1].strip().split(" ")[0].strip()
month = td[key]
year = time_msg.split(" ")[-1]
elif site == "mx":
key = time_msg.replace(".", "").split(" ")[1].strip()
if not td.get(key):
return ""
day = time_msg.split(" ")[0].strip()
month = td[key]
year = time_msg.split(" ")[-1]
elif site in ["nl", "au", "be", "tr", "ae"]:
key = time_msg.split(" ")[1]
if not td.get(key):
return ""
day = time_msg.split(",")[0].split(" ")[0].replace("\xa0", "")
month = td[key]
year = time_msg.split(" ")[-1].strip()
else:
key = time_msg.split(" ")[0]
if not td.get(key):
return ""
day = time_msg.split(",")[0].split(" ")[-1].replace("\xa0", "")
month = td[key]
year = time_msg.split(",")[-1].strip()
day = day if len(day) == 2 else "0" + day
return f"{year}-{month}-{day}"
def comment_time(time_msg, site) -> str:
"""评论日期原始文本预处理（去掉前缀词）"""
if not time_msg:
return ""
try:
if site == "de":
time_msg = (time_msg.split(".")[0][-2:] + "." + time_msg.split(".")[1]).strip()
elif site == "es":
time_msg = time_msg.split("el ")[-1].strip()
elif site == "fr":
time_msg = time_msg.split("le ")[-1].strip()
elif site == "it":
time_msg = time_msg.split("il ")[-1].strip()
else:
if "年" in time_msg:
date_object = datetime.strptime(time_msg.split(" ")[0], "%Y年%m月%d日")
time_msg = date_object.strftime("%Y-%m-%d")
else:
parts = time_msg.split(" on ")
if len(parts) == 1:
parts = time_msg.split("on")
time_msg = parts[-1].strip() if len(parts) > 1 else time_msg.strip()
except Exception as e:
logging.info(f"comment_time 解析失败 site={site} time_msg={time_msg!r}: {e}")
return ""
return time_msg
def get_review_count(html) -> list:
"""从 HTML 中提取评论总数"""
sel = Selector(text=html)
raw = sel.xpath('.//div[@data-hook="cr-filter-info-review-rating-count"]//text()').get()
if raw:
return _extract_numbers(raw) or ['0']
return ['0']
def if_response(html, spider_page, max_page, site="us") -> bool:
"""判断页面是否异常（返回 True=异常需重试，False=正常）
检测: 评论数显示不为0，但实际解析不到评论 → 页面异常
spider_page: (当前页码, 星级)
max_page: 该星级最大页数
"""
sel = Selector(text=html)
raw = sel.xpath('.//div[@class="a-row a-spacing-base a-size-base"]//text()').get()
comment_num_data = _extract_numbers(raw) if raw else []
if not comment_num_data:
return False
comment_num = comment_num_data[0]
comment_data_len = len(sel.xpath("//li[@data-hook='review']"))
if comment_num != "0" and comment_data_len:
return False
elif comment_num == "0":
return False
elif int(spider_page[0]) == 1 and len(comment_num_data) > 1 and int(comment_num_data[1]) == 1:
return False
elif int(spider_page[0]) == (int(max_page) - 1) and int(comment_num) % 10 == 1 and not comment_data_len:
return False
else:
logging.info(f"页面异常: 评论数={comment_num} 但解析到{comment_data_len}条")
return True
def xpath_res(html, seed, page, asin, site="us") -> list:
"""评论 HTML 解析 → 结构化数据列表（原样复制自同事的 xpath_res）
参数:
html: 评论页 HTML（page.content() 或 AJAX 响应清洗后的）
seed: 任务字典 {"taskId": ..., "siteName": ...}
page: 当前页码（int）
asin: 商品 ASIN
site: 站点代码（us/uk/de/fr/es/it 等）
"""
sel = Selector(text=html)
datas = sel.xpath("//li[@data-hook='review']")
items = []
for i in datas:
try:
title = (
i.xpath(".//h5/a/span//text()").get("").strip()
or i.xpath(".//h5//span[@data-hook='review-title']/span/text()").get("").strip()
)
user_name = i.xpath(".//span[@class='a-profile-name']//text()").get('').strip()
user_img = i.xpath(".//div[@class='a-profile-avatar']/img/@data-src").get('')
content = i.xpath(".//span[@data-hook='review-body']/span/text()").getall()
helpful = i.xpath(".//span[@data-hook='helpful-vote-statement']//text()").get()
if helpful:
token = helpful.split(" ")[1].strip() if site == "es" else helpful.split(" ")[0].strip()
review = 1 if token == "One" else token
else:
review = 0
try:
review = int(review)
except (ValueError, TypeError):
review = 0
size_mini = i.xpath(".//span[@class='a-size-mini a-color-state a-text-bold']//text()").get()
time_msg = i.xpath(".//span[@data-hook='review-date']//text()").get('')
time_msg = comment_time(time_msg, site)
comment_img = i.xpath(
".//div[@class='a-section a-spacing-top-mini cr-lightbox-image-thumbnails']/img/@src"
).getall()
video_url_list = i.xpath('.//input[contains(@class,"video-url")]/@value').getall()
video_url = ';'.join(video_url_list) if video_url_list else ""
user_page = i.xpath(".//div[@data-hook='genome-widget']/a/@href").get('')
rat = i.xpath(".//span[@class='a-icon-alt']//text()").get('')
rating = (
rat.split(".")[0].replace("stars", "").strip()
if len(rat.split(".")[0].replace("stars", "").strip()) == 1
else rat.split(",")[0].replace("stars", "").strip()
) if rat else ""
comment_url = i.xpath(".//*[@data-hook='review-title']/@href").get("")
variat_asin_href = i.xpath(".//a[@data-hook='format-strip']/@href").get()
variat_asin = ""
if variat_asin_href:
_m = re.findall(r"(?:product-reviews|customer-reviews)/(.*?)(?:/|\?)", variat_asin_href)
variat_asin = _m[0] if _m else ""
format_strip_texts = i.xpath(".//a[@data-hook='format-strip']//text()").getall()
vine = i.xpath(".//span[@class=\"a-color-success a-text-bold\"]//text()").get('')
item = {
"asin": asin,
"title": title,
"content": "".join(content).strip() if content else "",
"is_vp": str(1 if size_mini else 2),
"model": "|-|".join(format_strip_texts).strip().replace("What's this?", "") if format_strip_texts else "",
"rating": rating,
"agree_num": review,
"img_num": len(comment_img),
"img_url": ",".join(comment_img),
"is_video": str(1 if video_url else 2),
"video_url": video_url,
"comment_url": comment_url,
"user_name": user_name,
"user_img": user_img,
"country": site,
"user_page": user_page,
"is_earns_commissions": "",
"comment_time": time_msg,
"comment_time_format": time_ch(site, time_msg),
"page": int(page),
"vine_review_flag": '1' if vine else '2',
"asin_variation": variat_asin or "",
}
_keep_empty = {"asin_variation", "comment_id"}
item = {k: ("" if k in _keep_empty and (v == "" or v is None) else (None if (v == "" or v is None) else v)) for k, v in item.items()}
item["comment_id"] = i.xpath("./@id").get() or ""
item["page_state"] = 1 if item["comment_url"] else 2
items.append(item)
except Exception as e:
logging.info(f"单条评论解析失败，跳过: {e}")
continue
return items
def extract_detail_review_html(html):
"""从ASIN详情页HTML提取 localTopReviews + internationalTopReviews 的HTML片段
返回: (combined_html, review_count)
combined_html: 两块div的HTML拼接, 都没有则返回空字符串
review_count: 总评论条数
"""
from lxml import etree
tree = etree.HTML(html)
parts = []
total_count = 0
for div_id in ('localTopReviews', 'internationalTopReviews'):
nodes = tree.xpath(f'//div[@id="{div_id}"]')
if nodes:
div_html = etree.tostring(nodes[0], encoding='unicode', method='html')
parts.append(div_html)
total_count += len(nodes[0].xpath('.//*[@data-hook="review"]'))
return '\n'.join(parts), total_count
def build_api_data(seed, asin, total_count, v_count=None, values=None,
search=None, done_flag=None) -> dict:
"""构建上报数据（与同事的 _build_api_data 完全一致）"""
header = {
"totalReviews": total_count[0] if total_count else "0",
"totalRatings": '',
}
if v_count is not None:
header["reviews"] = v_count[0]
header["ratings"] = ''
data = {
"taskId": seed.get('taskId'),
"asin": asin,
"site": seed.get('siteName'),
"search": json.dumps(search) if search else "",
"header": json.dumps(header),
"values": json.dumps(values if values else []),
}
if done_flag is not None:
data['doneFlag'] = str(done_flag)
if values:
data['values_len'] = str(len(values))
return data
py_spider/comment_spider/review_spider.py 0 → 100644
"""评论爬虫 — 基于养号浏览器环境，page.goto 翻页 + review_parser 解析
每个账号一个独立指纹浏览器，爬评论就是养号行为的一部分
修复清单（对照10个运行时风险）：
1. import review_parser 替换自写解析 ✓
2. page.content() 前等评论元素渲染 ✓
3. URL 重定向检查 ✓
4. 每 ASIN 检查登录状态 ✓
5. 每页检查风控 ✓
6. get_page_num 避免浪费请求 ✓
7. 逐星级上报 ✓
8. goto 超时处理 ✓
9. MAX_STAR_RETRY=3 ✓
10. Redis 重试 ✓
"""
import os
import re
import sys
import json
import time
import random
import logging
import base64
import hashlib
import hmac
import urllib.request
import urllib.error
import redis as rd
from configparser import ConfigParser
from datetime import datetime
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import pyotp
import pandas as pd
from profile_manager import ProfileManager
from human_behavior import HumanPage
try:
from client import get_remote_engine
except ImportError:
get_remote_engine = None
from review_parser import (
xpath_res, build_api_data, if_response, get_review_count,
get_page_num, _dedup_comments, extract_detail_review_html,
)
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
NOTIFY_CONFIG = {
"test": {
"url": "http://119.23.61.82/soundasia_selection/asinComments/analysis/v3/notifyCrawlCompleted",
"secret": "Gq5kYj9uXr3Tn4sV2bWc6F1pLz8MdHaR",
},
"prod": {
"url": "https://selection.yswg.com.cn/soundasia_selection/asinComments/analysis/v3/notifyCrawlCompleted",
"secret": "YSKxIUfcaDJj961dbqfQo5XU9zje95g9",
},
}
TRANSLATE_SERVER = "http://61.145.136.61:30101"
def _base64url_encode(data: bytes) -> str:
return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii")
def _create_jwt(secret: str) -> str:
now = int(time.time())
header = json.dumps({"alg": "HS256", "typ": "JWT"}, separators=(",", ":")).encode()
payload = json.dumps({"iat": now, "nbf": now, "exp": now + 3600, "iss": "inv", "sub": "crawler generate token"}, separators=(",", ":")).encode()
h, p = _base64url_encode(header), _base64url_encode(payload)
sig = hmac.new(secret.encode(), f"{h}.{p}".encode(), hashlib.sha256).digest()
return f"{h}.{p}.{_base64url_encode(sig)}"
def notify_crawl_completed(env_type, site_name, asin, asin_type="1"):
cfg = NOTIFY_CONFIG[env_type]
body = json.dumps({"siteName": site_name, "asin": asin, "asinType": str(asin_type)}).encode()
req = urllib.request.Request(cfg["url"], data=body, headers={
"Content-Type": "application/json",
"jwt-token": _create_jwt(cfg["secret"]),
}, method="POST")
with urllib.request.urlopen(req, timeout=30) as resp:
return json.loads(resp.read().decode())
SEARCHES = [
# 电子数码
"wireless earbuds", "bluetooth speaker", "portable charger", "usb c cable",
"hdmi cable", "webcam", "keyboard", "mouse pad", "laptop stand", "phone case",
"tablet stand", "power strip", "smart watch", "screen protector", "ring light",
"microsd card", "wireless mouse", "usb hub", "monitor arm", "cable organizer",
# 家居厨房
"air fryer", "coffee maker", "water bottle", "desk lamp", "desk organizer",
"yoga mat", "resistance bands", "kitchen scale", "cutting board", "spice rack",
"storage bins", "shower curtain", "led strip lights", "candles", "throw blanket",
"wall hooks", "laundry basket", "dish soap", "paper towels", "trash bags",
# 户外运动
"running shoes", "backpack", "sunglasses", "hiking boots", "camping tent",
"sleeping bag", "fishing rod", "bike lock", "jump rope", "gym bag",
"baseball cap", "rain jacket", "swim goggles", "insulated tumbler", "cooler bag",
# 日用护理
"toothbrush", "shampoo", "face moisturizer", "sunscreen", "lip balm",
"hand cream", "body wash", "deodorant", "nail clipper", "cotton swabs",
"first aid kit", "vitamins", "protein powder", "melatonin", "eye drops",
# 宠物母婴
"dog treats", "cat litter", "pet bed", "baby wipes", "diaper bag",
"baby monitor", "sippy cup", "stroller", "teething toys", "nursing pillow",
# 办公文具
"sticky notes", "ballpoint pens", "notebook", "file folders", "tape dispenser",
"stapler", "highlighters", "whiteboard markers", "binder clips", "label maker",
# 服饰配件
"t shirt men", "socks women", "belt leather", "winter gloves", "beanie hat",
"tote bag", "wallet men", "scarf", "pajamas", "slippers",
]
class ReviewSpider:
def __init__(self, config_path="config.ini"):
cfg = ConfigParser()
cfg.read(config_path, encoding="utf-8")
self.pm = ProfileManager()
self.site = "us"
self._redis_conf = {
"host": cfg.get("redis", "host"),
"port": cfg.getint("redis", "port"),
"password": cfg.get("redis", "password"),
}
self.db_task = cfg.getint("redis", "db_task", fallback=10)
self.db_result = cfg.getint("redis", "db_result", fallback=0)
self.seed_key = "AsinCommentSpiderList"
self.save_key = "AsinCommentResultStream"
self.page_interval = (
cfg.getfloat("spider", "page_interval_min", fallback=3),
cfg.getfloat("spider", "page_interval_max", fallback=8),
)
self.asin_interval = (
cfg.getfloat("spider", "asin_interval_min", fallback=5),
cfg.getfloat("spider", "asin_interval_max", fallback=12),
)
self.asin_batch_limit = cfg.getint("spider", "asin_batch_limit", fallback=50)
self.asin_batch_sleep = (
cfg.getint("spider", "asin_batch_sleep_min", fallback=1800),
cfg.getint("spider", "asin_batch_sleep_max", fallback=3600),
)
self.nurture_every = cfg.getint("spider", "nurture_every", fallback=10)
self.max_pages_per_star = cfg.getint("spider", "max_pages_per_star", fallback=10)
self.account_rotate_hours = cfg.getint("spider", "account_rotate_hours", fallback=4)
# RemoteEngine 代理（MySQL + Doris）
if get_remote_engine:
self._engine_mysql = get_remote_engine("us", "mysql", database_name="selection")
self._engine_doris = get_remote_engine("us", "doris", database_name="selection")
else:
self._engine_mysql = None
self._engine_doris = None
self.notify_env = "prod"
self.asin_count = 0
self.page_count = 0
self._last_render_ok = True
# ==================== 数据库操作（全部走 RemoteEngine 代理） ====================
def _proxy_query(self, engine, sql, action="DB"):
"""通过 RemoteEngine 执行 SELECT，返回 list[dict]"""
for attempt in range(3):
try:
df = engine.read_sql(sql)
return df.to_dict("records")
except Exception as e:
logging.warning(f"{action}失败({attempt+1}/3): {e}")
time.sleep(3)
logging.error(f"{action}重试3次仍失败")
return []
def _proxy_execute(self, engine, sql, action="DB"):
"""通过 RemoteEngine 执行 UPDATE/DELETE"""
for attempt in range(3):
try:
with engine.begin() as conn:
conn.execute(sql)
return True
except Exception as e:
logging.warning(f"{action}失败({attempt+1}/3): {e}")
time.sleep(3)
logging.error(f"{action}重试3次仍失败")
return False
# ==================== 翻译（标题+内容 英译中） ====================
def _translate_texts(self, texts, batch_size=50, timeout=120):
"""批量调用翻译服务，返回与 texts 等长的译文列表，失败项为空串"""
import requests as req
results = [""] * len(texts)
tasks = [(i, t) for i, t in enumerate(texts) if t and t.strip()]
if not tasks:
return results
for start in range(0, len(tasks), batch_size):
batch = tasks[start:start + batch_size]
batch_indices = [i for i, _ in batch]
batch_texts = [t for _, t in batch]
translated = None
for attempt in range(3):
try:
resp = req.post(
f"{TRANSLATE_SERVER}/translate",
timeout=timeout,
json={"words": batch_texts},
)
resp.raise_for_status()
data = resp.json().get("result", [])
if len(data) == len(batch_texts):
translated = data
break
logging.warning(f"翻译返回数量不一致: 期望{len(batch_texts)} 实际{len(data)}")
except Exception as e:
logging.warning(f"翻译失败({attempt+1}/3): {e}")
time.sleep(3)
if translated:
for idx, tr in zip(batch_indices, translated):
results[idx] = tr or ""
return results
def _translate_items(self, items):
"""翻译评论标题和内容，添加 title_cn / content_cn 字段
翻译服务不可用或失败 → 字段置空串，不影响入库"""
import requests as req
if not items:
return items
try:
req.get(f"{TRANSLATE_SERVER}/health", timeout=30).raise_for_status()
except Exception as e:
logging.warning(f"翻译服务不可用，跳过本批翻译: {e}")
for it in items:
it["title_cn"] = ""
it["content_cn"] = ""
return items
titles = [it.get("title") or "" for it in items]
contents = [it.get("content") or "" for it in items]
title_trans = self._translate_texts(titles, batch_size=100, timeout=60)
content_trans = self._translate_texts(contents, batch_size=20, timeout=120)
for i, it in enumerate(items):
it["title_cn"] = title_trans[i]
it["content_cn"] = content_trans[i]
ok_count = sum(1 for t in title_trans if t) + sum(1 for t in content_trans if t)
total = sum(1 for t in titles if t.strip()) + sum(1 for t in contents if t.strip())
logging.info(f"翻译完成: {ok_count}/{total} 成功")
return items
# ==================== 数据存储 ====================
def save_reviews(self, items, task_id=""):
"""写入 Doris asin_comment_detail（重试3次）"""
if not items:
return 0
self._translate_items(items)
for attempt in range(3):
try:
self._save_to_doris_via_proxy(items)
return len(items)
except Exception as e:
logging.warning(f"Doris写入失败({attempt+1}/3): {e}")
time.sleep(3)
logging.error(f"Doris写入重试3次仍失败，丢失{len(items)}条")
return 0
def _save_to_doris_via_proxy(self, items):
"""通过 RemoteEngine HTTP 代理写入 Doris asin_comment_detail"""
if not get_remote_engine or not items:
return
rows = []
for item in items:
time_fmt = item.get("comment_time_format")
if time_fmt and len(time_fmt) < 8:
time_fmt = None
rating_val = None
if item.get("rating") and str(item["rating"]).isdigit():
rating_val = float(item["rating"])
rows.append({
"site_name": item.get("country") or self.site,
"asin": item.get("asin"),
"comment_id": item.get("comment_id") or "",
"title": (item.get("title") or "")[:255],
"content": item.get("content"),
"rating": rating_val,
"user_name": (item.get("user_name") or "")[:200],
"user_img": (item.get("user_img") or "")[:255],
"user_page": (item.get("user_page") or "")[:255],
"comment_time": (item.get("comment_time") or "")[:100],
"comment_time_format": time_fmt,
"is_vp": int(item["is_vp"]) if item.get("is_vp") else 0,
"agree_num": int(item["agree_num"]) if item.get("agree_num") is not None else 0,
"img_num": int(item["img_num"]) if item.get("img_num") is not None else 0,
"img_url": item.get("img_url") or "",
"is_video": int(item["is_video"]) if item.get("is_video") else 0,
"video_url": item.get("video_url") or "",
"model": (item.get("model") or "")[:255],
"comment_url": (item.get("comment_url") or "")[:255],
"country": (item.get("country") or self.site)[:100],
"vine_review_flag": int(item["vine_review_flag"]) if item.get("vine_review_flag") else 0,
"asin_variation": item.get("asin_variation") or "",
"title_cn": (item.get("title_cn") or "")[:500],
"content_cn": item.get("content_cn") or "",
})
df = pd.DataFrame(rows)
self._engine_doris.to_sql(df, "asin_comment_detail", if_exists="append")
logging.info(f"Doris代理写入: {len(rows)}条")
# ==================== Redis（带重试） ====================
def _redis(self, db=None):
kwargs = dict(
host=self._redis_conf["host"],
port=self._redis_conf["port"],
password=self._redis_conf["password"],
db=db if db is not None else self.db_task,
socket_timeout=10,
socket_connect_timeout=5,
retry_on_timeout=True,
)
try:
return rd.Redis(protocol=2, **kwargs)
except TypeError:
return rd.Redis(**kwargs)
def _redis_retry(self, fn, action_name="Redis"):
for attempt in range(3):
try:
return fn()
except (rd.exceptions.ConnectionError, rd.exceptions.TimeoutError) as e:
logging.warning(f"{action_name}失败({attempt+1}/3): {e}")
time.sleep(3)
logging.error(f"{action_name}重试3次仍失败")
return None
def get_task(self):
"""从 MySQL asin_comment_syn 领取一批 ASIN（read_then_update 原子操作，防多机重复）"""
for attempt in range(3):
try:
result = self._engine_mysql.read_then_update(
select_sql="SELECT id, site_name, asin, asin_type FROM asin_comment_syn WHERE state = 1 ORDER BY id ASC LIMIT 1 FOR UPDATE",
update_table="asin_comment_syn",
set_values={"state": 2},
where_keys=["id"],
as_df=False,
)
break
except Exception as e:
logging.warning(f"领取任务失败({attempt+1}/3): {e}")
time.sleep(3)
result = None
if not result:
return None
rows = result if isinstance(result, list) else result.to_dict("records") if hasattr(result, 'to_dict') else []
if not rows:
return None
seed = {
"taskId": f"comment_{datetime.now().strftime('%Y%m%d%H%M%S')}",
"siteName": rows[0]["site_name"],
"asin": [{"asin": r["asin"], "_syn_id": r["id"], "asin_type": r.get("asin_type", "1")} for r in rows],
}
logging.info(f"领取任务: {len(rows)}个ASIN ({[r['asin'] for r in rows]})")
return seed
def mark_task_done(self, asin, syn_id=None, asin_type="1"):
"""标记 ASIN 抓取完成（state=3），优先按 id 精确更新"""
if syn_id:
sql = f"UPDATE asin_comment_syn SET state = 3, updated_at = NOW() WHERE id = {syn_id}"
else:
sql = f"UPDATE asin_comment_syn SET state = 3, updated_at = NOW() WHERE asin = '{asin}' AND asin_type = '{asin_type}'"
self._proxy_execute(self._engine_mysql, sql, action=f"标记完成 {asin}")
def requeue_task(self, seed, asin, syn_id=None, asin_type="1"):
"""ASIN 抓取失败，退回 state=1，优先按 id 精确更新"""
if syn_id:
sql = f"UPDATE asin_comment_syn SET state = 1, updated_at = NOW() WHERE id = {syn_id} AND state = 2"
else:
sql = f"UPDATE asin_comment_syn SET state = 1, updated_at = NOW() WHERE asin = '{asin}' AND asin_type = '{asin_type}' AND state = 2"
self._proxy_execute(self._engine_mysql, sql, action=f"退回任务 {asin}")
def save_result(self, data):
"""Redis Stream 上报（非致命，失败只告警不中断，数据已存Doris）"""
try:
def _do():
r = self._redis(self.db_result)
r.xadd(self.save_key, data)
r.close()
self._redis_retry(_do, "Redis上报")
except Exception as e:
logging.warning(f"Redis上报失败(已忽略，数据在Doris): {e}")
def _notify_completed(self, asin, asin_type="1"):
"""通知后端 ASIN 抓取完成（重试15次，间隔5-8秒随机，应对后端负载均衡间歇性401）"""
for attempt in range(15):
try:
result = notify_crawl_completed(self.notify_env, self.site, asin, asin_type)
if isinstance(result, dict) and result.get("code") == 200:
logging.info(f"[{asin}] 通知成功(type={asin_type})")
return
logging.warning(f"[{asin}] 通知返回非200({attempt+1}/15): {result}")
except Exception as e:
logging.warning(f"[{asin}] 通知异常({attempt+1}/15): {e}")
if attempt < 14:
time.sleep(random.uniform(5, 8))
logging.error(f"[{asin}] 通知重试15次仍失败(已忽略，数据在Doris)")
# ==================== 浏览器操作 ====================
def ensure_profile(self, account):
"""确保账号有对应的浏览器 profile，没有就创建"""
name = f"share_{account['account_id']}"
try:
self.pm.get_config(name)
return name
except Exception:
pass
if name in self.pm.index:
del self.pm.index[name]
self.pm.index_path.write_text(
json.dumps(self.pm.index, indent=2, ensure_ascii=False), encoding="utf-8"
)
logging.info(f"[{name}] 索引残留已清理，重新创建profile")
self.pm.create(name, account={
"phone": account["username"],
"name": f"User_{account['account_id']}",
"password": account["password"],
"totp_secret": account.get("totp_secret", ""),
})
return name
def launch_browser(self, profile_name):
browser, config = self.pm.launch(profile_name, headless=False)
page = browser.new_page()
hp = HumanPage(page)
return browser, page, hp, config
def _safe_goto(self, page, url, timeout=30000):
"""安全导航：处理超时和导航错误"""
try:
page.goto(url, wait_until="domcontentloaded", timeout=timeout)
return True
except Exception as e:
logging.warning(f"导航失败 {url[:80]}: {e}")
return False
def _wait_reviews_render(self, page, timeout=10):
"""等评论元素渲染完再读 DOM，渲染成功设 _last_render_ok=True"""
try:
page.locator('li[data-hook="review"]').first.wait_for(timeout=timeout * 1000)
time.sleep(1)
self._last_render_ok = True
except Exception:
time.sleep(1)
self._last_render_ok = False
def _get_page_html(self, page):
"""读取页面 HTML（等评论渲染 + 读 DOM）"""
self._wait_reviews_render(page)
try:
return page.content()
except Exception as e:
logging.warning(f"读取页面内容失败: {e}")
return ""
def check_page_status(self, page):
"""检测页面状态（含反爬/验证码/变狗/风控等）"""
try:
text = page.inner_text("body", timeout=5000).lower()
except Exception:
return "unknown"
url = (page.url or "").lower()
if "account locked temporarily" in text or "account has been locked" in text:
return "account_locked"
if "account has been suspended" in text or "account is suspended" in text or "revoke-account-suspension" in url:
return "suspended"
if "password reset required" in text or "forgotpassword" in url:
return "password_reset"
if ("sorry! we couldn't find that page" in text or
"we just need to make sure you're not a robot" in text or
"sorry! something went wrong" in text):
return "anti_bot"
if "click the button below to continue" in text:
return "continue_shopping"
if "send a request" in text:
return "risk_detected"
if ("enter the characters you see below" in text
or "type the characters" in text
or "geben sie die zeichen unten ein" in text
or "introduce los caracteres que se muestran" in text
or "saisissez les caractères que vous voyez" in text
or "inserisci i caratteri visualizzati" in text):
return "captcha"
if "automated access" in text or "apis services" in text or "api services" in text:
return "anti_bot"
if "unusual activity" in text or "limit review access" in text:
return "risk_detected"
return "ok"
def check_login(self, page):
"""检查是否仍然登录（每 ASIN 前调用）"""
try:
nav = page.inner_text('#nav-link-accountList', timeout=3000).strip()
if 'sign in' in nav.lower():
return False
return True
except Exception:
return True
def auto_login(self, page, hp, account):
"""自动登录：密码 + TOTP"""
if not self._safe_goto(page, "https://www.amazon.com/"):
return "network_error"
time.sleep(3)
if self.check_page_status(page) == "continue_shopping":
self._handle_continue_shopping(page, hp)
time.sleep(3)
try:
nav = page.inner_text('#nav-link-accountList', timeout=5000).strip()
if 'sign in' not in nav.lower():
logging.info(f"已登录: {nav[:30]}")
return "ok"
except Exception:
pass
password = account["password"]
username = account["username"]
totp_secret = account.get("totp_secret", "")
if not self._safe_goto(page,
"https://www.amazon.com/ap/signin?openid.return_to=https%3A%2F%2Fwww.amazon.com"
"&openid.assoc_handle=usflex&openid.mode=checkid_setup"
"&openid.ns=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0"):
return "network_error"
time.sleep(3)
for step in range(15):
url = page.url
status = self.check_page_status(page)
if status != "ok":
return status
try:
text = page.inner_text("body", timeout=3000).lower()
except Exception:
text = ""
if "amazon.com" in url and "/ap/" not in url and "/cvf/" not in url and "/mfa" not in url:
try:
nav = page.inner_text('#nav-link-accountList', timeout=3000)
if 'sign in' not in nav.lower():
logging.info(f"登录成功: {nav.strip()[:30]}")
return "ok"
except Exception:
pass
if "/ap/signin" in url:
try:
if page.locator('#ap_password').is_visible(timeout=2000):
hp.human_fill('#ap_password', password)
time.sleep(0.5)
hp.human_click('#signInSubmit')
time.sleep(4)
continue
except Exception:
pass
try:
if page.locator('#ap_email_login').is_visible(timeout=2000):
hp.human_fill('#ap_email_login', username)
time.sleep(0.5)
hp.human_click('#continue')
time.sleep(4)
continue
except Exception:
pass
if "/ap/mfa" in url or "auth-mfa-otpcode" in text:
if not totp_secret:
return "no_totp"
code = pyotp.TOTP(totp_secret).now()
logging.info(f"TOTP: {code}")
for sel in ['#auth-mfa-otpcode', 'input[name="otpCode"]', 'input[name="code"]']:
try:
el = page.locator(sel)
if el.count() > 0 and el.first.is_visible(timeout=2000):
hp.human_fill(sel, code)
break
except Exception:
continue
time.sleep(0.5)
try:
hp.human_click('#auth-signin-button')
except Exception:
try:
page.locator('input[type="submit"]').first.click(timeout=3000)
except Exception:
pass
time.sleep(5)
continue
if "switch account" in text:
try:
page.locator('a:has-text("Sign out")').first.click()
time.sleep(3)
except Exception:
pass
continue
time.sleep(3)
return "timeout"
# ==================== 评论爬取 ====================
def _get_latest_comment_ids(self, asin, limit=50):
"""查该 ASIN 最近的 comment_id（通过代理查 Doris）"""
rows = self._proxy_query(
self._engine_doris,
f"SELECT comment_id FROM asin_comment_detail WHERE asin = '{asin}' AND site_name = '{self.site}' LIMIT {limit}",
action="查最新评论ID"
)
return set(r["comment_id"] for r in rows if r.get("comment_id"))
def _handle_continue_shopping(self, page, hp):
"""处理 "Click the button below to continue shopping" 拦截页"""
try:
btn = page.locator('a:has-text("Continue shopping"), button:has-text("Continue shopping")')
if btn.count() > 0:
logging.info("[拦截] 点击 Continue shopping")
btn.first.click(timeout=5000)
time.sleep(random.uniform(3, 5))
return True
except Exception:
pass
return False
def _click_show_more(self, page, hp):
"""滚动找 Show 10 more reviews 按钮并点击，返回是否成功加载了新评论"""
old_count = page.locator('li[data-hook="review"]').count()
try:
old_last_id = page.locator('li[data-hook="review"]').last.get_attribute('id')
except Exception:
old_last_id = None
btn = page.locator('a[data-hook="show-more-button"]')
if btn.count() == 0:
return False
for _ in range(15):
page.evaluate("window.scrollBy(0, Math.floor(window.innerHeight * 0.7))")
time.sleep(random.uniform(0.3, 0.6))
try:
box = btn.first.bounding_box()
if box and box['y'] < page.viewport_size['height']:
break
except Exception:
pass
old_url = page.url
try:
btn.first.scroll_into_view_if_needed()
time.sleep(random.uniform(1.5, 2.5))
btn.first.click()
except Exception:
return False
for _ in range(40):
time.sleep(0.5)
try:
new_count = page.locator('li[data-hook="review"]').count()
if new_count > old_count:
return True
if new_count > 0 and old_last_id:
new_last_id = page.locator('li[data-hook="review"]').last.get_attribute('id')
if new_last_id != old_last_id:
return True
except Exception:
pass
if page.url != old_url:
try:
page.locator('li[data-hook="review"]').first.wait_for(timeout=10000)
except Exception:
pass
return page.locator('li[data-hook="review"]').count() > 0
return False
def _enter_review_page(self, page, hp, asin):
"""进入评论页的公共流程：检查登录→商品页→评论页→风控检查→读评论总数"""
if not self.check_login(page):
return "session_expired", None, ['0'], ""
if not self._safe_goto(page, f"https://www.amazon.com/dp/{asin}/"):
return "network_error", None, ['0'], ""
hp.simulate_reading(random.uniform(2, 4))
status = self.check_page_status(page)
if status == "continue_shopping":
self._handle_continue_shopping(page, hp)
if not self._safe_goto(page, f"https://www.amazon.com/dp/{asin}/"):
return "network_error", None, ['0'], ""
hp.simulate_reading(random.uniform(2, 4))
status = self.check_page_status(page)
if status != "ok":
return status, None, ['0'], ""
base_url = f"https://www.amazon.com/product-reviews/{asin}"
if not self._safe_goto(page, f"{base_url}?reviewerType=all_reviews&sortBy=recent"):
return "network_error", None, ['0'], ""
time.sleep(random.uniform(3, 5))
status = self.check_page_status(page)
if status == "continue_shopping":
self._handle_continue_shopping(page, hp)
if not self._safe_goto(page, f"{base_url}?reviewerType=all_reviews&sortBy=recent"):
return "network_error", None, ['0'], ""
time.sleep(random.uniform(3, 5))
status = self.check_page_status(page)
if status != "ok":
return status, None, ['0'], ""
current_url = page.url or ""
if "product-reviews" not in current_url and "customer-reviews" not in current_url:
status = self.check_page_status(page)
return (status if status != "ok" else "redirect"), None, ['0'], ""
html = self._get_page_html(page)
total_count = get_review_count(html)
if not self._last_render_ok and int(total_count[-1]) == 0:
logging.warning(f"[{asin}] 评论未渲染且count=0，判定渲染超时，回滚重试")
return "render_timeout", None, ['0'], ""
return "ok", base_url, total_count, html
def crawl_reviews(self, page, hp, seed, asin):
"""智能爬取：首次全量（按星级），增量靠 INSERT IGNORE 影响行数判断
返回: (status, total_items_count, total_count)
"""
logging.info(f"===== 开始 {asin} =====")
# 判断首次还是增量（查最近50个 comment_id）
known_ids = self._get_latest_comment_ids(asin)
is_incremental = len(known_ids) > 0
logging.info(f"[{asin}] 模式: {'增量' if is_incremental else '首次全量'} (已有{len(known_ids)}条)")
# 进入评论页
status, base_url, total_count, html = self._enter_review_page(page, hp, asin)
if status != "ok":
return status, 0, ['0']
# 直接解析第1页，避免后续重复导航浪费数据
first_page_items = xpath_res(html, seed, 1, asin, site=self.site) if html else []
if int(total_count[-1]) == 0:
if not first_page_items:
logging.info(f"[{asin}] 无评论")
api_data = build_api_data(seed, asin, total_count, done_flag=1)
self.save_result(api_data)
return "ok", 0, total_count
logging.info(f"[{asin}] review count元素缺失但解析到{len(first_page_items)}条评论，继续抓取")
total_count = [str(len(first_page_items))]
logging.info(f"[{asin}] 总评论数: {total_count[0]}")
if is_incremental:
return self._crawl_incremental(page, hp, seed, asin, base_url, total_count, known_ids, first_page_items)
else:
return self._crawl_full(page, hp, seed, asin, base_url, total_count, first_page_items)
def _crawl_incremental(self, page, hp, seed, asin, base_url, total_count, known_ids, first_page_items=None):
"""增量抓取：逐页点击 Show more → 逐条比对已有 comment_id → 碰到已有就停"""
logging.info(f"[{asin}] 增量抓取开始 (已知{len(known_ids)}条)")
total_new = 0
seen_ids = set()
for p in range(1, self.max_pages_per_star + 1):
if p == 1 and first_page_items:
items = first_page_items
else:
if p > 1:
hp.human_wait(1, 2)
if not self._click_show_more(page, hp):
logging.info(f"[{asin}] 增量 第{p}页 无Show more按钮，停止")
break
time.sleep(random.uniform(*self.page_interval))
self.page_count += 1
status = self.check_page_status(page)
if status != "ok":
return status, total_new, total_count
html = self._get_page_html(page)
all_parsed = xpath_res(html, seed, p, asin, site=self.site)
items = [it for it in all_parsed if it.get("comment_id") not in seen_ids]
if not items:
logging.info(f"[{asin}] 增量 第{p}页 无数据，停止")
break
seen_ids.update(it.get("comment_id") for it in items)
new_items = []
hit_known = False
for item in items:
cid = item.get("comment_id")
if cid and cid in known_ids:
hit_known = True
break
new_items.append(item)
if new_items:
saved = self.save_reviews(new_items, task_id=seed.get("taskId", ""))
total_new += saved
logging.info(f"[{asin}] 增量 第{p}页 新增{saved}条")
if hit_known:
logging.info(f"[{asin}] 增量 第{p}页 碰到已有评论，停止翻页")
break
if len(new_items) == len(items):
logging.info(f"[{asin}] 增量 第{p}页 全新，继续")
else:
break
if total_new > 0:
api_data = build_api_data(
seed, asin, total_count,
search={"sortBy": "recent", "formatType": "current_format"},
done_flag=1,
)
self.save_result(api_data)
logging.info(f"[{asin}] 增量完成: 新增{total_new}条")
return "ok", total_new, total_count
def _crawl_full(self, page, hp, seed, asin, base_url, total_count, first_page_items=None):
"""首次全量抓取：第1页用缓存，后续点击 Show 10 more reviews 加载"""
logging.info(f"[{asin}] 全量抓取开始 (最多{self.max_pages_per_star}页)")
all_items = []
seen_ids = set()
if first_page_items:
items = first_page_items
else:
html = self._get_page_html(page)
items = xpath_res(html, seed, 1, asin, site=self.site)
if items:
all_items.extend(items)
seen_ids.update(it.get("comment_id") for it in items)
logging.info(f"[{asin}] 第1页 {len(items)}条 (累计{len(all_items)}条)")
else:
logging.info(f"[{asin}] 第1页 无数据，停止")
for p in range(2, self.max_pages_per_star + 1):
hp.human_wait(1, 2)
if not self._click_show_more(page, hp):
logging.info(f"[{asin}] 第{p}页 无Show more按钮，停止")
break
time.sleep(random.uniform(*self.page_interval))
self.page_count += 1
status = self.check_page_status(page)
if status != "ok":
return status, 0, total_count
html = self._get_page_html(page)
items = xpath_res(html, seed, p, asin, site=self.site)
new_items = [it for it in items if it.get("comment_id") not in seen_ids]
if not new_items:
logging.info(f"[{asin}] 第{p}页 全部重复，停止")
break
all_items.extend(new_items)
seen_ids.update(it.get("comment_id") for it in new_items)
logging.info(f"[{asin}] 第{p}页 {len(new_items)}条 (累计{len(all_items)}条)")
all_items = _dedup_comments(all_items)
saved = 0
if all_items:
saved = self.save_reviews(all_items, task_id=seed.get("taskId", ""))
api_data = build_api_data(
seed, asin, total_count, values=all_items,
search={"sortBy": "recent", "formatType": "current_format"},
done_flag=1,
)
self.save_result(api_data)
logging.info(f"[{asin}] 全量完成: 共{saved}条评论")
return "ok", saved, total_count
# ==================== 详情页评论HTML提取 (asin_type=2) ====================
def crawl_detail_reviews(self, page, hp, seed, asin):
"""asin_type=2: 访问详情页，提取评论区HTML存Doris
返回: (status, review_count, ['0'])
"""
logging.info(f"===== 开始 {asin} (详情页HTML模式) =====")
if not self.check_login(page):
return "session_expired", 0, ['0']
if not self._safe_goto(page, f"https://www.amazon.com/dp/{asin}/"):
return "network_error", 0, ['0']
hp.simulate_reading(random.uniform(2, 4))
status = self.check_page_status(page)
if status == "continue_shopping":
self._handle_continue_shopping(page, hp)
if not self._safe_goto(page, f"https://www.amazon.com/dp/{asin}/"):
return "network_error", 0, ['0']
hp.simulate_reading(random.uniform(2, 4))
status = self.check_page_status(page)
if status != "ok":
return status, 0, ['0']
current_url = (page.url or "").lower()
if "/dp/" not in current_url and asin.lower() not in current_url:
logging.warning(f"[{asin}] 详情页被重定向: {current_url[:80]}")
if "/ap/signin" in current_url or "/ap/cvf" in current_url:
return "session_expired", 0, ['0']
return "redirect", 0, ['0']
try:
page.locator('#customerReviews').wait_for(timeout=10000)
except Exception:
pass
time.sleep(1)
try:
html = page.content()
except Exception as e:
logging.warning(f"[{asin}] 读取详情页内容失败: {e}")
return "network_error", 0, ['0']
combined_html, review_count = extract_detail_review_html(html)
if not combined_html:
logging.info(f"[{asin}] 详情页未找到评论区HTML")
return "ok", 0, ['0']
self.save_detail_html(asin, combined_html)
logging.info(f"[{asin}] 详情页评论HTML已保存, {review_count}条评论, {len(combined_html)}字符")
return "ok", review_count, [str(review_count)]
def save_detail_html(self, asin, comment_html):
"""写入 Doris asin_comment_first_detail（重试3次）"""
for attempt in range(3):
try:
df = pd.DataFrame([{
"site_name": self.site,
"asin": asin,
"comment_html": comment_html,
"updated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
}])
self._engine_doris.to_sql(df, "asin_comment_first_detail", if_exists="append")
logging.info(f"[{asin}] Doris asin_comment_first_detail 写入成功")
return
except Exception as e:
logging.warning(f"[{asin}] 详情HTML写入失败({attempt+1}/3): {e}")
time.sleep(3)
logging.error(f"[{asin}] 详情HTML写入重试3次仍失败")
# ==================== 养号行为 ====================
def do_nurture(self, page, hp):
"""完整养号流程：搜索+浏览商品+看评论+逛Deals+看账户
模拟真人购物行为，总时长控制在3-7分钟"""
start = time.time()
max_duration = random.uniform(180, 420)
logging.info(f"[养号] 开始 (目标{max_duration/60:.1f}分钟)")
def _overtime():
return (time.time() - start) >= max_duration
try:
# 1. 随机搜索1-2个关键词，搜索+浏览商品+看评论
keywords = random.sample(SEARCHES, random.randint(1, 2))
for kw in keywords:
if _overtime():
break
logging.info(f"[养号] 搜索: {kw}")
# 搜索
try:
hp.human_fill('#twotabsearchtextbox', kw)
hp.human_wait(0.3, 0.8)
page.press('#twotabsearchtextbox', 'Enter')
page.wait_for_load_state("domcontentloaded", timeout=15000)
except Exception:
continue
# 搜索结果页模拟阅读 + 滚动
hp.simulate_reading(random.uniform(2, 4))
for _ in range(random.randint(2, 3)):
if _overtime():
break
hp.random_scroll()
hp.human_wait(1, 2)
if _overtime():
break
# 随机点1个商品进详情页
try:
product_url = page.evaluate("""() => {
const items = document.querySelectorAll('[data-component-type="s-search-result"] h2 a, .s-result-item h2 a');
const hrefs = [];
for (const a of items) { if (a.href && a.href.includes('/dp/')) hrefs.push(a.href); }
return hrefs.length ? hrefs[Math.floor(Math.random() * Math.min(hrefs.length, 6))] : null;
}""")
if product_url:
page.goto(product_url, wait_until="domcontentloaded", timeout=20000)
hp.simulate_reading(random.uniform(3, 5))
logging.info(f"[养号] 浏览商品: {page.title()[:40] if page.title() else '?'}")
# 滚动看商品
for _ in range(random.randint(2, 4)):
if _overtime():
break
hp.random_scroll()
hp.human_wait(1, 2)
# 50%概率进评论页看看
if not _overtime() and random.random() < 0.5:
asin_m = re.search(r'/dp/([A-Z0-9]{10})', product_url)
if asin_m:
review_url = f"https://www.amazon.com/product-reviews/{asin_m.group(1)}?sortBy=recent"
self._safe_goto(page, review_url, timeout=20000)
hp.simulate_reading(random.uniform(2, 4))
for _ in range(random.randint(2, 4)):
if _overtime():
break
hp.random_scroll()
hp.human_wait(1, 2)
logging.info(f"[养号] 浏览评论")
# 返回搜索结果
if not _overtime():
page.go_back()
hp.human_wait(1, 2)
except Exception as e:
logging.info(f"[养号] 浏览商品异常(忽略): {e}")
# 2. 逛 Today's Deals
if not _overtime():
logging.info("[养号] 逛 Deals")
self._safe_goto(page, "https://www.amazon.com/gp/goldbox", timeout=20000)
hp.simulate_reading(random.uniform(3, 5))
for _ in range(random.randint(1, 3)):
if _overtime():
break
hp.random_scroll()
hp.human_wait(1, 2)
# 3. 看 Your Account
if not _overtime():
logging.info("[养号] 看账户")
self._safe_goto(page, "https://www.amazon.com/gp/css/homepage.html", timeout=20000)
hp.simulate_reading(random.uniform(2, 4))
elapsed = (time.time() - start) / 60
logging.info(f"[养号] 完成 (耗时{elapsed:.1f}分钟)")
except Exception as e:
logging.warning(f"[养号] 异常(忽略): {e}")
def _cleanup_chrome_locks(self, profile_name):
"""清理 Chromium 残留锁文件，解决 'Opening in existing browser session' 死循环"""
chrome_data = self.pm.data_dir / profile_name / "chrome_data"
for lock_name in ["SingletonLock", "SingletonSocket", "SingletonCookie"]:
lock_file = chrome_data / lock_name
if lock_file.exists():
try:
lock_file.unlink()
logging.info(f"[{profile_name}] 清理锁文件: {lock_name}")
except Exception as ex:
logging.warning(f"[{profile_name}] 清理锁文件失败: {lock_name} {ex}")
# ==================== 单账号工作循环 ====================
def run_account(self, account, account_mgr):
"""单个账号的完整工作循环"""
account_id = account["account_id"]
profile_name = self.ensure_profile(account)
logging.info(f"启动账号 {account_id} profile={profile_name}")
browser = None
page = None
try:
browser, page, hp, config = self.launch_browser(profile_name)
except Exception as e:
logging.error(f"浏览器启动失败: {e}")
if "existing browser session" in str(e):
self._cleanup_chrome_locks(profile_name)
account_mgr.mark_login_fail(account_id, "browser_launch_fail")
return "browser_fail"
try:
login_status = self.auto_login(page, hp, account)
if login_status == "account_locked":
account_mgr.mark_locked(account_id)
return "locked"
elif login_status == "suspended":
account_mgr.mark_suspended(account_id)
return "suspended"
elif login_status == "password_reset":
account_mgr.mark_risk(account_id, "password_reset")
return "risk"
elif login_status != "ok":
account_mgr.mark_login_fail(account_id, login_status)
return "login_fail"
account_mgr.mark_login_ok(account_id)
start_time = time.time()
self.asin_count = 0
asin_retry_counts = {}
while True:
elapsed_hours = (time.time() - start_time) / 3600
if elapsed_hours >= 72:
logging.info(f"账号 {account_id} 运行{elapsed_hours:.1f}小时，重启浏览器释放内存")
break
seed = self.get_task()
if not seed:
logging.info("无任务，等待3-5分钟后重试")
time.sleep(random.uniform(180, 300))
continue
asins = seed.get("asin") or []
for asin_item in asins:
asin = asin_item.get("asin") if isinstance(asin_item, dict) else asin_item
syn_id = asin_item.get("_syn_id") if isinstance(asin_item, dict) else None
asin_type = asin_item.get("asin_type", "1") if isinstance(asin_item, dict) else "1"
if asin_retry_counts.get(asin, 0) >= 3:
logging.info(f"[{asin}] 本轮已失败3次，跳过不重试")
self.requeue_task(seed, asin, syn_id, asin_type)
time.sleep(random.uniform(*self.asin_interval))
continue
if str(asin_type) == "2":
status, items_count, total_count = self.crawl_detail_reviews(page, hp, seed, asin)
else:
status, items_count, total_count = self.crawl_reviews(page, hp, seed, asin)
if status == "account_locked":
account_mgr.mark_locked(account_id)
for a in asins[asins.index(asin_item):]:
_a = a.get("asin") if isinstance(a, dict) else a
_sid = a.get("_syn_id") if isinstance(a, dict) else None
_at = a.get("asin_type", "1") if isinstance(a, dict) else "1"
self.requeue_task(seed, _a, _sid, _at)
return "locked"
elif status == "suspended":
account_mgr.mark_suspended(account_id)
for a in asins[asins.index(asin_item):]:
_a = a.get("asin") if isinstance(a, dict) else a
_sid = a.get("_syn_id") if isinstance(a, dict) else None
_at = a.get("asin_type", "1") if isinstance(a, dict) else "1"
self.requeue_task(seed, _a, _sid, _at)
return "suspended"
elif status in ("risk_detected", "anti_bot", "password_reset", "captcha"):
account_mgr.mark_risk(account_id, status)
for a in asins[asins.index(asin_item):]:
_a = a.get("asin") if isinstance(a, dict) else a
_sid = a.get("_syn_id") if isinstance(a, dict) else None
_at = a.get("asin_type", "1") if isinstance(a, dict) else "1"
self.requeue_task(seed, _a, _sid, _at)
return "risk"
elif status == "continue_shopping":
logging.warning("频率限制，等待30秒后重试")
self._handle_continue_shopping(page, hp)
time.sleep(random.uniform(20, 40))
self.requeue_task(seed, asin, syn_id, asin_type)
continue
elif status == "session_expired":
logging.warning("session 过期，重新登录")
login_status = self.auto_login(page, hp, account)
if login_status != "ok":
for a in asins[asins.index(asin_item):]:
_a = a.get("asin") if isinstance(a, dict) else a
_sid = a.get("_syn_id") if isinstance(a, dict) else None
_at = a.get("asin_type", "1") if isinstance(a, dict) else "1"
self.requeue_task(seed, _a, _sid, _at)
return "login_fail"
self.requeue_task(seed, asin, syn_id, asin_type)
continue
if status != "ok":
asin_retry_counts[asin] = asin_retry_counts.get(asin, 0) + 1
retries = asin_retry_counts[asin]
if retries >= 3:
logging.warning(f"[{asin}] 连续失败{retries}次({status})，本轮放弃，退回队列等其他Worker处理")
self.requeue_task(seed, asin, syn_id, asin_type)
continue
delay = min(30 * retries, 120)
logging.warning(f"[{asin}] 异常状态: {status}，{delay}秒后重试({retries}/3)")
self.requeue_task(seed, asin, syn_id, asin_type)
time.sleep(delay)
continue
asin_retry_counts.pop(asin, None)
self.mark_task_done(asin, syn_id, asin_type)
if items_count > 0:
self._notify_completed(asin, asin_type)
logging.info(f"[{asin}] 保存完成 共{items_count}条 → state=3")
self.asin_count += 1
# 每10个 ASIN 养号一次（3-7分钟）
if self.asin_count % self.nurture_every == 0:
logging.info(f"已爬{self.asin_count}个ASIN，开始养号")
self.do_nurture(page, hp)
# 50个 ASIN 长休眠
if self.asin_count >= self.asin_batch_limit:
nap = random.uniform(*self.asin_batch_sleep)
logging.info(f"已爬{self.asin_count}个ASIN，长休眠{nap/60:.0f}分钟")
time.sleep(nap)
self.asin_count = 0
time.sleep(random.uniform(*self.asin_interval))
return "rotate"
except Exception as e:
logging.error(f"账号 {account_id} 异常: {e}")
return "error"
finally:
try:
if page:
self.pm.save_cookies(profile_name, page.context)
except Exception:
pass
try:
if browser:
browser.close()
except Exception:
pass
py_spider/send_email_account/logs/agent.log deleted 100644 → 0
[2026-06-23 10:56:54] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] ========== MailBot Agent v1.0.0 启动 ==========
[2026-06-23 10:56:54] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] agent_id=DESKTOP-SVPNHIJ_TT, account=TT, MAC=04:42:1A:22:A0:76, DiskSN=WD-WCC6Y3VFVXCE
[2026-06-23 10:56:54] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] 注册表自启动注册成功: MailBotAgent_TT
[2026-06-23 10:56:54] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] 注册设备: agent_id=DESKTOP-SVPNHIJ_TT, account=TT, mac=04:42:1A:22:A0:76, disk_sn=WD-WCC6Y3VFVXCE
[2026-06-23 10:56:54] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] 注册成功, Token有效期: 2026-07-23 10:56:53
[2026-06-23 10:56:55] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] 远程监控线程启动, 间隔 30 秒
[2026-06-23 10:56:55] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] 开始轮询任务, 间隔 30 秒
[2026-06-23 10:56:55] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] 日志目录: sunlogin=D:\开发安装\向日葵\SunloginClient\log, rustdesk=C:\Users\ASUS\AppData\Roaming\RustDesk\log, todesk=D:\开发安装\远程\ToDesk\Logs
[2026-06-23 10:56:55] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] Token有效, 到期: 2026-07-23 10:56:53
[2026-06-23 10:56:56] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 无待处理任务
[2026-06-23 10:56:58] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 远程监控: 无远程连接
[2026-06-23 10:56:58] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 上报在线状态
[2026-06-23 10:57:26] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] Token有效, 到期: 2026-07-23 10:56:53
[2026-06-23 10:57:26] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 无待处理任务
[2026-06-23 10:57:31] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 远程监控: 无远程连接
[2026-06-23 10:57:31] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 上报在线状态
[2026-06-23 10:57:56] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] Token有效, 到期: 2026-07-23 10:56:53
[2026-06-23 10:57:56] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 无待处理任务
[2026-06-23 10:58:03] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 远程监控: 无远程连接
[2026-06-23 10:58:03] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 上报在线状态
[2026-06-23 10:58:26] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] Token有效, 到期: 2026-07-23 10:56:53
[2026-06-23 10:58:27] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 无待处理任务
[2026-06-23 10:58:38] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 远程监控: 无远程连接
[2026-06-23 10:58:38] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 上报在线状态
[2026-06-23 10:58:57] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] Token有效, 到期: 2026-07-23 10:56:53
[2026-06-23 10:58:57] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 无待处理任务
[2026-06-23 10:59:12] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 远程监控: 无远程连接
[2026-06-23 10:59:12] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 上报在线状态
[2026-06-23 10:59:27] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] Token有效, 到期: 2026-07-23 10:56:53
[2026-06-23 10:59:27] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 无待处理任务
[2026-06-23 10:59:45] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 远程监控: 无远程连接
[2026-06-23 10:59:45] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 上报在线状态
[2026-06-23 10:59:57] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] Token有效, 到期: 2026-07-23 10:56:53
[2026-06-23 10:59:58] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 无待处理任务
[2026-06-23 11:00:19] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 远程监控: 无远程连接
[2026-06-23 11:00:19] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 上报在线状态
[2026-06-23 11:00:28] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] Token有效, 到期: 2026-07-23 10:56:53
[2026-06-23 11:00:28] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 无待处理任务
[2026-06-23 11:00:52] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 远程监控: 无远程连接
[2026-06-23 11:00:52] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 上报在线状态
[2026-06-23 11:00:58] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] Token有效, 到期: 2026-07-23 10:56:53
[2026-06-23 11:00:58] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 无待处理任务
[2026-06-23 11:01:25] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 远程监控: 无远程连接
[2026-06-23 11:01:25] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 上报在线状态
[2026-06-23 11:01:28] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] Token有效, 到期: 2026-07-23 10:56:53
[2026-06-23 11:01:29] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 无待处理任务
[2026-06-23 11:01:58] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 远程监控: 无远程连接
[2026-06-23 11:01:58] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 上报在线状态
[2026-06-23 11:01:59] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] Token有效, 到期: 2026-07-23 10:56:53
[2026-06-23 11:01:59] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 无待处理任务
[2026-06-23 11:02:29] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] Token有效, 到期: 2026-07-23 10:56:53
[2026-06-23 11:02:30] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 无待处理任务
[2026-06-23 11:02:31] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 远程监控: 无远程连接
[2026-06-23 11:02:31] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 上报在线状态
[2026-06-23 11:07:43] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] ========== MailBot Agent v1.0.0 启动 ==========
[2026-06-23 11:07:43] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] agent_id=DESKTOP-SVPNHIJ_TT, account=TT, MAC=04:42:1A:22:A0:76, DiskSN=WD-WCC6Y3VFVXCE
[2026-06-23 11:07:43] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] 注册表自启动注册成功: MailBotAgent_TT
[2026-06-23 11:07:43] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] 注册设备: agent_id=DESKTOP-SVPNHIJ_TT, account=TT, mac=04:42:1A:22:A0:76, disk_sn=WD-WCC6Y3VFVXCE
[2026-06-23 11:07:43] [ERROR ] [agent=DESKTOP-SVPNHIJ_TT] [404] Agent不存在或已禁用, 停止请求
[2026-06-23 11:07:43] [ERROR ] [agent=DESKTOP-SVPNHIJ_TT] 注册失败: None
[2026-06-23 11:07:43] [ERROR ] [agent=DESKTOP-SVPNHIJ_TT] Agent 注册阶段遇到致命错误(403/404), 无法启动
[2026-06-23 11:07:48] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] ========== MailBot Agent v1.0.0 启动 ==========
[2026-06-23 11:07:48] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] agent_id=DESKTOP-SVPNHIJ_TT, account=TT, MAC=04:42:1A:22:A0:76, DiskSN=WD-WCC6Y3VFVXCE
[2026-06-23 11:07:48] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] 注册表自启动注册成功: MailBotAgent_TT
[2026-06-23 11:07:48] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] 注册设备: agent_id=DESKTOP-SVPNHIJ_TT, account=TT, mac=04:42:1A:22:A0:76, disk_sn=WD-WCC6Y3VFVXCE
[2026-06-23 11:07:48] [ERROR ] [agent=DESKTOP-SVPNHIJ_TT] [404] Agent不存在或已禁用, 停止请求
[2026-06-23 11:07:48] [ERROR ] [agent=DESKTOP-SVPNHIJ_TT] 注册失败: None
[2026-06-23 11:07:48] [ERROR ] [agent=DESKTOP-SVPNHIJ_TT] Agent 注册阶段遇到致命错误(403/404), 无法启动
[2026-06-23 11:08:56] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] ========== MailBot Agent v1.0.0 启动 ==========
[2026-06-23 11:08:56] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] agent_id=DESKTOP-SVPNHIJ_TT, account=TT, MAC=04:42:1A:22:A0:76, DiskSN=WD-WCC6Y3VFVXCE
[2026-06-23 11:08:56] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] 注册表自启动注册成功: MailBotAgent_TT
[2026-06-23 11:08:56] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] 注册设备: agent_id=DESKTOP-SVPNHIJ_TT, account=TT, mac=04:42:1A:22:A0:76, disk_sn=WD-WCC6Y3VFVXCE
[2026-06-23 11:08:56] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] 注册成功, Token有效期: 2026-07-23 11:08:56
[2026-06-23 11:08:58] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] 远程监控线程启动, 间隔 30 秒
[2026-06-23 11:08:58] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] 开始轮询任务, 间隔 30 秒
[2026-06-23 11:08:58] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] 日志目录: sunlogin=D:\开发安装\向日葵\SunloginClient\log, rustdesk=C:\Users\ASUS\AppData\Roaming\RustDesk\log, todesk=D:\开发安装\远程\ToDesk\Logs
[2026-06-23 11:08:58] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] Token有效, 到期: 2026-07-23 11:08:56
[2026-06-23 11:08:58] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 无待处理任务
[2026-06-23 11:09:00] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 远程监控: 无远程连接
[2026-06-23 11:09:00] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 上报在线状态
[2026-06-23 11:09:28] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] Token有效, 到期: 2026-07-23 11:08:56
[2026-06-23 11:09:29] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 无待处理任务
[2026-06-23 11:09:33] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 远程监控: 无远程连接
[2026-06-23 11:09:33] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 上报在线状态
[2026-06-23 11:09:59] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] Token有效, 到期: 2026-07-23 11:08:56
[2026-06-23 11:09:59] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 无待处理任务
[2026-06-23 11:10:06] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 远程监控: 无远程连接
[2026-06-23 11:10:06] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 上报在线状态
[2026-06-23 11:10:29] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] Token有效, 到期: 2026-07-23 11:08:56
[2026-06-23 11:10:29] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 无待处理任务
[2026-06-23 11:10:38] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 远程监控: 无远程连接
[2026-06-23 11:10:38] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 上报在线状态
[2026-06-23 11:10:59] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] Token有效, 到期: 2026-07-23 11:08:56
[2026-06-23 11:11:00] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 无待处理任务
[2026-06-23 11:11:11] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 远程监控: 无远程连接
[2026-06-23 11:11:11] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 上报在线状态
[2026-06-23 11:11:30] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] Token有效, 到期: 2026-07-23 11:08:56
[2026-06-23 11:11:30] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 无待处理任务
[2026-06-23 11:11:44] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 远程监控: 无远程连接
[2026-06-23 11:11:44] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 上报在线状态
[2026-06-23 11:12:00] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] Token有效, 到期: 2026-07-23 11:08:56
[2026-06-23 11:12:01] [DEBUG ] [agent=DESKTOP-SVPNHIJ_TT] 无待处理任务
[2026-06-23 11:12:04] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] 收到退出信号
[2026-06-23 11:12:04] [INFO ] [agent=DESKTOP-SVPNHIJ_TT] ========== MailBot Agent 停止 ==========
py_spider/threading_spider/Poll_site_search_term_pg.py deleted 100644 → 0
import sys
import os
from datetime import datetime, timedelta
sys.path.append(os.path.dirname(sys.path[0])) # 上级目录
from multiprocessing import Pool
from amazon_spider.search_term_pg_week import search_temp_pg
from amazon_spider.search_term_mysql import search_temp_mysql
from threading_spider.db_connectivity import connect_db
import time
import pandas as pd
import random
import requests
import socket
from amazon_spider.VPS_IP import pppoe_ip
from threading_spider.post_to_dolphin import DolphinschedulerHelper
def select_sate_mysql(site, num=None, week=None):
db_class = connect_db(site)
engine_pg = db_class.pg_db() # pg
cursor_us, connect_us = db_class.us_mysql_db() # us站点
print('week::', week)
if int(week) < 10:
week = f'0{int(week)}'
else:
week = week
if num is None:
select_state1_sql = f"select status_val from workflow_progress where site_name='{site}' and date_info='2024-{week}' and date_type='week' and page='ABA搜索词';"
print(select_state1_sql)
cursor_us.execute(select_state1_sql)
site_sate1_list = cursor_us.fetchone()
try:
if site_sate1_list[0] == 2:
print(f"{site} 站点 {week} 周 搜索词 已完成导入到 search_term 执行下一步")
state1_sql = f"select status_val from workflow_progress where site_name='{site}' and date_info='2024-{week}' and date_type='week' and page='反查搜索词';"
print(state1_sql)
cursor_us.execute(state1_sql)
sate1_list = cursor_us.fetchone()
print('查看状态：', sate1_list)
if sate1_list[0] in (1, 2):
update_workflow_progress_2 = f"update workflow_progress set status_val=2,status='抓取中' where page='反查搜索词' and date_info='2024-{week}' and site_name='{site}' and date_type='week' and status_val=1"
print('修改状态2：：', update_workflow_progress_2)
cursor_us.execute(update_workflow_progress_2)
connect_us.commit()
return True
else:
print(33333333333336666666666666666666666666)
return False
except:
return False
if num == 1:
# 搜索词多进程已经抓完。最后执行单进程抓取。
sql_read = f'SELECT id FROM {site}_search_term_syn where state in (1,2) and week={week} LIMIT 1'
print(sql_read)
df = pd.read_sql(sql_read, con=engine_pg)
id_tuple = list(df.id)
if len(id_tuple) == 0:
id_tuple = None
print('id_tuple：', id_tuple)
if id_tuple is None:
status_sql = f"select status_val from workflow_progress where site_name='{site}' and date_info='2024-{week}' and date_type='week' and page='反查搜索词';"
print('status_sql:', status_sql)
cursor_us.execute(status_sql)
status_list = cursor_us.fetchone()
print(status_list)
if status_list:
if status_list[0] == 2:
update_workflow_progress = f"update workflow_progress set status_val=3,status='抓取结束' where page='反查搜索词' and date_info='2024-{week}' and site_name='{site}' and date_type='week'"
print('update_workflow_progress: 修改状态3 ', update_workflow_progress)
cursor_us.execute(update_workflow_progress)
connect_us.commit()
account = 'pengyanbing,chenyuanjie,hezhe,wangrui4,fangxingjun'
title = site + '站点 搜索词'
content = str(week) + ' 周 搜索词 已结束,请确认下一步流程!时间:'
db_class.send_mg(account, title, content)
DolphinschedulerHelper.start_process_instance_common(
project_name="big_data_selection",
process_df_name='ALL站点-asin同步-pg-api',
startParams={
"site_name": f"{site}",
"date_type": "week",
"date_info": f'2024-{week}'
}
)
if site == 'us':
pass
return True
else:
print('5555555555555555555555555555555555')
return False
if num == 3:
# 搜索词多进程已经抓完。最后执行单进程抓取。
sql_read = f'SELECT id FROM {site}_search_term_syn where state in (1,2) and week={week} LIMIT 1'
print(sql_read)
df = pd.read_sql(sql_read, con=engine_pg)
id_tuple = list(df.id)
if id_tuple:
id_tuple = [1]
else:
id_tuple = None
if id_tuple is None:
select_sate_mysql(site, num=1, week=week)
return False
else:
return True
def get_ip_address():
# 返回内网ip
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('baidu.com', 0))
ip = s.getsockname()[0]
return ip
def long_time_task(site, proxy_name, week):
print("当前 抓取 站点 ", site)
if site == 'us':
spider_us = search_temp_pg(site_name=site, read_size=180, proxy_name=proxy_name, week=week)
spider_us.run_pol()
else:
spider_mysql = search_temp_mysql(site_name=site, read_size=180, proxy_name=proxy_name,
week=week)
spider_mysql.run_pol()
print('结束')
def start_run(week):
pppoe_ip()
proxy_name = None
site_list = ['us','de', 'uk']
for site in site_list:
if select_sate_mysql(site, week=week):
while True:
current_time = datetime.now()
# 计算后五分钟的时间
five_minutes_later = current_time + timedelta(minutes=1)
print('后五分钟时间', five_minutes_later)
p = Pool(5)
for i in range(3):
p.apply_async(long_time_task, args=(site, proxy_name, week))
print('等待所有子进程运行完成')
# 执行该方法后不能继续添加新的Process
p.close()
# 等待所有子进程执行完毕
p.join()
print('所有进程运行完毕！')
print('所有进程运行完毕！开始切换ip')
print('所有进程运行完毕！开始切换ip')
print('所有进程运行完毕！开始切换ip')
print('所有进程运行完毕！开始切换ip')
current_time = datetime.now()
if current_time > five_minutes_later:
pppoe_ip()
else:
time.sleep(random.uniform(200, 300))
pppoe_ip()
if select_sate_mysql(site, num=3, week=week) == False:
break
if __name__ == '__main__':
try:
week = int(sys.argv[1])
except:
week = None
start_run(week)
py_spider/utils/secure_db_client.py deleted 100644 → 0
#
#
#
# '''旧版'''
#
# import json
#
# import pandas as pd
# import numpy as np
# import orjson, requests, time
# from typing import List
#
# # -------- 映射字典 --------
# site_name_db_dict = {
# "us": "selection",
# "uk": "selection_uk",
# "de": "selection_de",
# "es": "selection_es",
# "fr": "selection_fr",
# "it": "selection_it",
# }
#
# db_type_alias_map = {
# "mysql": "mysql", # 阿里云mysql
# "postgresql_14": "postgresql_14", # pg14爬虫库-内网
# "postgresql_14_outer": "postgresql_14", # pg14爬虫库-外网
# "postgresql_15": "postgresql_15", # pg15正式库-内网
# "postgresql_15_outer": "postgresql_15_outer", # pg15正式库-外网
# "postgresql_cluster": "postgresql_cluster", # pg集群-内网
# "postgresql_cluster_outer": "postgresql_cluster_outer", # pg集群-外网
# "doris": "doris", # doris集群-内网
# }
#
# DEFAULT_SERVERS = [
# # "http://192.168.200.210:7777", # 内网
# # "http://192.168.10.217:7777", # 内网-h7
# # "http://113.100.143.162:7777", # 外网
# # "http://113.100.143.162:7779", # 外网
# # "http://61.145.136.61:7777", # 外网
# # "http://61.145.136.61:7779", # 外网
# "http://192.168.200.210:7780"
#
#
# ]
# # ---------------------------
#
# def df_to_json_records(df: pd.DataFrame) -> list:
# """保证 DataFrame 可安全序列化为 JSON records（处理 NaN / ±Inf）"""
# df_clean = df.copy()
#
# # 1️⃣ 替换 ±Inf -> NaN
# num_cols = df_clean.select_dtypes(include=[np.number]).columns
# if len(num_cols):
# df_clean[num_cols] = df_clean[num_cols].replace([np.inf, -np.inf], np.nan)
#
# # 2️⃣ 替换 NaN -> None（注意：有时 astype(object) 不彻底，需用 applymap）
# df_clean = df_clean.applymap(lambda x: None if pd.isna(x) else x)
#
# # 3️⃣ 转为 dict records
# return df_clean.to_dict("records")
#
#
# def clean_json_field_for_orjson(v):
# """清洗单个 JSON 字段的值，使其符合 orjson 要求并避免空字典入库"""
# if v is None or pd.isna(v):
# return None
#
# # 1️⃣ 如果是空字典对象，返回 None
# if isinstance(v, dict) and not v:
# return None
#
# # 2️⃣ 如果是空字符串或仅为 "{}"，返回 None
# if isinstance(v, str):
# stripped = v.strip()
# if not stripped or stripped == "{}":
# return None
# try:
# parsed = json.loads(stripped)
# if isinstance(parsed, dict) and not parsed:
# return None
# return json.dumps(parsed, ensure_ascii=False)
# except Exception:
# return v # 非 JSON 字符串则原样保留
#
# return v
#
#
# def fully_clean_for_orjson(df: pd.DataFrame) -> pd.DataFrame:
# # """全面清洗 DataFrame 以符合 orjson 要求"""
# # df = df.replace([np.inf, -np.inf], np.nan)
# # df = df.applymap(lambda x: None if pd.isna(x) else x)
# #
# # # 找出所有可能为 JSON 字符串的字段
# # json_like_cols = [col for col in df.columns if col.endswith('_json')]
# #
# # # 针对每个 JSON-like 字段，应用清洗函数
# # for col in json_like_cols:
# # df[col] = df[col].apply(clean_json_field_for_orjson)
# #
# # return df
# """全面清洗 DataFrame 以符合 orjson 要求"""
# df = df.replace([np.inf, -np.inf], np.nan)
#
# # NaN → None （比 applymap 高效且不出错）
# df = df.where(pd.notna(df), None)
#
# # 找出所有可能为 JSON 字符串的字段
# json_like_cols = [col for col in df.columns if col.endswith('_json')]
#
# # 针对每个 JSON-like 字段，应用清洗函数
# for col in json_like_cols:
# df[col] = df[col].apply(clean_json_field_for_orjson)
#
# return df
#
#
# class RemoteTransaction:
#
# def __init__(self, db: str, database: str,
# session: requests.Session, urls: List[str]):
# self.db = db
# self.database = database
# self.session = session
# self.urls = urls
# self.sql_queue = []
#
# # def execute(self, sql: str):
# # self.sql_queue.append(sql)
# def execute(self, sql: str, params=None):
# """
# params 可取：
# • None → 纯文本 SQL
# • dict → 单条参数化 e.g. {"id":1,"name":"a"}
# • list/tuple → 批量 executemany
# - list[dict] ↔ INSERT .. VALUES (:id,:name)
# - list[tuple] ↔ INSERT .. VALUES (%s,%s)
# """
# self.sql_queue.append({"sql": sql, "params": params})
#
# def __enter__(self): return self
#
# def __exit__(self, exc_type, exc, tb):
# for url in self.urls:
# try:
# self.session.post(
# url + "/transaction",
# json={"db": self.db,
# "sql_list": self.sql_queue,
# "site_name": self.database}, # site_name not needed on server, kept for clarity
# timeout=3000,
# ).raise_for_status()
# return
# except Exception as e:
# print(f"[WARN] 事务失败 {url}: {e}")
# raise RuntimeError("All servers failed for transaction")
#
#
# class RemoteEngine:
# def __init__(self, db: str, database: str,
# server_urls: List[str], retries: int = 2):
# self.db = db
# self.database = database
# self.urls = [u.rstrip("/") for u in server_urls]
# self.session = requests.Session()
# self.retries = retries
#
# def _request(self, endpoint: str, payload):
# for url in self.urls:
# for _ in range(self.retries):
# try:
# json_bytes = orjson.dumps(payload)
# r = self.session.post(f"{url}/{endpoint}",
# data=json_bytes,
# headers={"Content-Type": "application/json"},
# timeout=3000)
#
# # r = self.session.post(f"{url}/{endpoint}",
# # json=payload, timeout=10)
# r.raise_for_status()
# return r.json()
# except Exception as e:
# print(f"[WARN] {endpoint} fail @ {url}: {e}")
# time.sleep(1)
# raise RuntimeError(f"All servers failed for {endpoint}")
# # def _request(self, endpoint: str, payload):
# # # 用 orjson，“allow_nan” 会把 NaN/Inf 写成 null
# # # json_bytes = orjson.dumps(payload,
# # # option=orjson.OPT_NON_STR_KEYS | orjson.OPT_NAIVE_UTC | orjson.OPT_OMIT_MICROSECOND | orjson.OPT_ALLOW_INF_AND_NAN)
# # json_bytes = orjson.dumps(
# # payload,
# # option=orjson.OPT_NON_STR_KEYS | orjson.OPT_NAIVE_UTC | orjson.OPT_ALLOW_INF_AND_NAN
# # )
# #
# # headers = {"Content-Type": "application/json"}
# #
# # for url in self.urls:
# # for _ in range(self.retries):
# # try:
# # r = self.session.post(f"{url}/{endpoint}",
# # data=json_bytes, headers=headers,
# # timeout=15)
# # r.raise_for_status()
# # return r.json()
# # except Exception as e:
# # print(f"[WARN] {endpoint} fail @ {url}: {e}")
# # time.sleep(1)
# # raise RuntimeError(f"All servers failed for {endpoint}")
#
# # ---------- 公共 API ----------
# def read_sql(self, sql: str) -> pd.DataFrame:
# data = self._request("query",
# {"db": self.db,
# "sql": sql,
# "site_name": self.database})
# return pd.DataFrame(data["result"])
#
# def to_sql(self, df: pd.DataFrame, table: str, if_exists="append"):
#
# return self._request("insert",
# {"db": self.db,
# "table": table,
# "if_exists": if_exists,
# "data": fully_clean_for_orjson(df=df).to_dict("records"),
# # "data": df_to_json_records(df), # ← 清洗后的 records
# "site_name": self.database})
#
# def read_then_update(
# self,
# select_sql: str,
# update_table: str,
# set_values: dict,
# where_keys: List[str],
# error_if_empty: bool = False,
# ):
# """
# 动态生成 UPDATE：把 select_sql 读到的行，按 where_keys 精准更新 set_values
# 返回 (DataFrame, rows_updated)
# """
# payload = {
# "db": self.db,
# "site_name": self.database,
# "select_sql": select_sql,
# "update_table": update_table,
# "set_values": set_values,
# "where_keys": where_keys,
# "error_if_empty": error_if_empty,
# }
# resp = self._request("read_then_update", payload)
# df = pd.DataFrame(resp["read_result"])
# rows_updated = resp.get("rows_updated", 0)
# return df
#
# def begin(self):
# return RemoteTransaction(self.db, self.database,
# self.session, self.urls)
# # ---------------------------------
#
#
# def get_remote_engine(site_name: str, db_type: str,
# servers: List[str] = None) -> RemoteEngine:
# if site_name not in site_name_db_dict:
# raise ValueError(f"Unknown site_name: {site_name}")
# if db_type not in db_type_alias_map:
# raise ValueError(f"Unknown db_type: {db_type}")
# return RemoteEngine(
# db=db_type_alias_map[db_type],
# database=site_name,
# server_urls=servers or DEFAULT_SERVERS,
# )
import json
import sys,os
sys.path.append(os.path.dirname(sys.path[0])) # 上级目录
import pandas as pd
import numpy as np
import orjson, requests, time
from typing import List
# -------- 映射字典 --------
site_name_db_dict = {
"us": "selection",
"uk": "selection_uk",
"de": "selection_de",
"es": "selection_es",
"fr": "selection_fr",
"it": "selection_it",
}
db_type_alias_map = {
"mysql": "mysql", # 阿里云mysql
"postgresql_14": "postgresql_14", # pg14爬虫库-内网
"postgresql_14_outer": "postgresql_14_outer", # pg14爬虫库-外网
"postgresql_15": "postgresql_15", # pg15正式库-内网
"postgresql_15_outer": "postgresql_15_outer", # pg15正式库-外网
"postgresql_cluster": "postgresql_cluster", # pg集群-内网
"postgresql_cluster_outer": "postgresql_cluster_outer", # pg集群-外网
"doris": "doris", # doris集群-内网
"doris_adv": "doris_adv",
}
DEFAULT_SERVERS = [
# "http://192.168.200.210:7777", # 内网
# "http://192.168.10.217:7777", # 内网-h7
# "http://61.145.136.61:7777", # 外网
# "http://61.145.136.61:7779", # 外网
"http://61.145.136.61:7780"
]
# ---------------------------
def df_to_json_records(df: pd.DataFrame) -> list:
"""保证 DataFrame 可安全序列化为 JSON records（处理 NaN / ±Inf）"""
df_clean = df.copy()
# 1️⃣ 替换 ±Inf -> NaN
num_cols = df_clean.select_dtypes(include=[np.number]).columns
if len(num_cols):
df_clean[num_cols] = df_clean[num_cols].replace([np.inf, -np.inf], np.nan)
# 2️⃣ 替换 NaN -> None（兼容 pandas 1.x 和 2.x）
df_clean = df_clean.where(pd.notna(df_clean), None)
# 3️⃣ 转为 dict records
return df_clean.to_dict("records")
def clean_json_field_for_orjson(v):
"""清洗单个 JSON 字段的值，使其符合 orjson 要求并避免空字典入库"""
if v is None or pd.isna(v):
return None
# 1️⃣ 如果是空字典对象，返回 None
if isinstance(v, dict) and not v:
return None
# 2️⃣ 如果是空字符串或仅为 "{}"，返回 None
if isinstance(v, str):
stripped = v.strip()
if not stripped or stripped == "{}":
return None
try:
parsed = json.loads(stripped)
if isinstance(parsed, dict) and not parsed:
return None
return json.dumps(parsed, ensure_ascii=False)
except Exception:
return v # 非 JSON 字符串则原样保留
return v
def fully_clean_for_orjson(df: pd.DataFrame) -> pd.DataFrame:
"""全面清洗 DataFrame 以符合 orjson 要求"""
df = df.replace([np.inf, -np.inf], np.nan)
df = df.where(pd.notna(df), None)
# 找出所有可能为 JSON 字符串的字段
json_like_cols = [col for col in df.columns if col.endswith('_json')]
# 针对每个 JSON-like 字段，应用清洗函数
for col in json_like_cols:
df[col] = df[col].apply(clean_json_field_for_orjson)
return df
class RemoteTransaction:
def __init__(self, db: str, database: str,
session: requests.Session, urls: List[str]):
self.db = db
self.database = database
self.session = session
self.urls = urls
self.sql_queue = []
# def execute(self, sql: str):
# self.sql_queue.append(sql)
def execute(self, sql: str, params=None):
"""
params 可取：
• None → 纯文本 SQL
• dict → 单条参数化 e.g. {"id":1,"name":"a"}
• list/tuple → 批量 executemany
- list[dict] ↔ INSERT .. VALUES (:id,:name)
- list[tuple] ↔ INSERT .. VALUES (%s,%s)
"""
self.sql_queue.append({"sql": sql, "params": params})
def __enter__(self): return self
def __exit__(self, exc_type, exc, tb):
for url in self.urls:
try:
self.session.post(
url + "/transaction",
json={"db_type": self.db,
"sql_list": self.sql_queue,
"database": self.database}, # site_name not needed on server, kept for clarity
timeout=3000,
).raise_for_status()
return
except Exception as e:
print(f"[WARN] 事务失败 {url}: {e}")
raise RuntimeError("All servers failed for transaction")
class RemoteEngine:
def __init__(self, db: str, database: str,
server_urls: List[str], retries: int = 20):
"""
:param db: db_type--数据库类型
:param database: 数据库名称
:param server_urls:
:param retries:
"""
self.db = db
self.database = database
self.urls = [u.rstrip("/") for u in server_urls]
self.session = requests.Session()
self.retries = retries
def _request(self, endpoint: str, payload):
for url in self.urls:
for _ in range(self.retries):
try:
json_bytes = orjson.dumps(payload)
r = self.session.post(f"{url}/{endpoint}",
data=json_bytes,
headers={"Content-Type": "application/json"},
timeout=3600)
# r = self.session.post(f"{url}/{endpoint}",
# json=payload, timeout=10)
r.raise_for_status()
return r.json()
except Exception as e:
print(f"[WARN] {endpoint} fail @ {url}: {e}")
time.sleep(1)
raise RuntimeError(f"All servers failed for {endpoint}")
# ---------- 公共 API ----------
def read_sql(self, sql: str) -> pd.DataFrame:
data = self._request("query",
{"db_type": self.db,
"sql": sql,
"database": self.database})
return pd.DataFrame(data["result"])
def to_sql(self, df: pd.DataFrame, table: str, if_exists="append"):
return self._request("insert",
{"db_type": self.db,
"table": table,
"if_exists": if_exists,
"data": fully_clean_for_orjson(df=df).to_dict("records"),
# "data": df_to_json_records(df), # ← 清洗后的 records
"database": self.database})
def read_then_update(
self,
select_sql: str,
update_table: str,
set_values: dict,
where_keys: List[str],
error_if_empty: bool = False,
):
"""
动态生成 UPDATE：把 select_sql 读到的行，按 where_keys 精准更新 set_values
返回 (DataFrame, rows_updated)
"""
payload = {
"db_type": self.db,
"database": self.database,
"select_sql": select_sql,
"update_table": update_table,
"set_values": set_values,
"where_keys": where_keys,
"error_if_empty": error_if_empty,
}
resp = self._request("read_then_update", payload)
df = pd.DataFrame(resp["read_result"])
rows_updated = resp.get("rows_updated", 0)
return df
def begin(self):
return RemoteTransaction(self.db, self.database,
self.session, self.urls)
# ---------------------------------
# db -- 数据库类型
# database -- 站点
def get_remote_engine(site_name: str, db_type: str, database: str = None,
servers: List[str] = None) -> RemoteEngine:
"""
:param site_name: 站点
:param db_type: 数据库类型
:param database: 数据库名称-默认为None， 否则通过站点来匹配
:param servers: 服务器url地址
:return: 数据库连接对象
"""
if not database:
database = site_name_db_dict[site_name]
if site_name not in site_name_db_dict:
raise ValueError(f"Unknown site_name: {site_name}")
if db_type not in db_type_alias_map:
raise ValueError(f"Unknown db_type: {db_type}")
print(f"db_type: {db_type_alias_map[db_type]}, database: {database}")
return RemoteEngine(
db=db_type_alias_map[db_type],
database=database,
server_urls=servers or DEFAULT_SERVERS,
)
def stream_load_df(df: pd.DataFrame, database: str='advertising_manager', table: str='', server_url: str = None):
server = (server_url or DEFAULT_SERVERS[0]).rstrip("/")
payload = {
"database": database,
"table": table,
"data": fully_clean_for_orjson(df).to_dict("records"),
}
json_bytes = orjson.dumps(payload)
r = requests.post(
f"{server}/stream_load_df",
data=json_bytes,
headers={"Content-Type": "application/json"},
timeout=3000,
)
r.raise_for_status()
return r.json()
if __name__ == '__main__':
engine_mysql = get_remote_engine(
site_name='us', # -> database "selection"
db_type="doris_adv", # -> 服务端 alias "mysql"
database="advertising_manager", # -> 服务端 alias "mysql"
)
# e = engine_mysql.read_sql('SELECT max(created_time),MIN(created_time) FROM us_sp_search_term_rank WHERE time_batch="2026-03-09-15"')
# e = engine_mysql.read_sql('SELECT * FROM us_sp_search_term_rank WHERE time_batch="2026-03-09-12" and page_row=68')
# e = engine_mysql.read_sql('SELECT * FROM us_sp_search_term_rank WHERE id=50564433')
e = engine_mysql.read_sql('SELECT count(id),time_batch FROM us_sp_search_term_rank GROUP BY time_batch ORDER BY time_batch ASC')
# e = engine_mysql.read_sql('SELECT count(id),time_batch FROM us_sp_search_term_rank GROUP BY time_batch ORDER BY time_batch ASC')
# e = engine_mysql.read_sql('SELECT count(id) FROM sp_keyword_position_keyword')
# e = engine_mysql.read_sql('select * from us_sp_search_term_syn limit 100')
# dog bandanas spring
# 2026-04-03-11
# e = engine_mysql.read_sql('select * from us_sp_search_term_syn where search_term = "dog bandanas spring"')
# e = engine_mysql.read_sql('select * from us_sp_search_term_rank where search_term="dog bandanas spring" and time_batch="2026-04-03-11" and cate_type="sp" and asin="B0GCN7CTCM"')
# e = engine_mysql.read_sql('select * from us_sp_search_term_rank where time_batch="2026-03-17-16" and cate_type="sp" limit 100')
# print(e)
print(e.columns)
print(e.values.tolist())
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment