import sys
import os
import traceback

sys.path.append(os.path.dirname(sys.path[0]))  # 上级目录
from amazon_params import py_ja3
from utils.requests_param import Requests_param_val
from utils.db_connect import BaseUtils
from amazon_params.params import DB_CONN_DICT
import pymysql
import random
import time
from lxml import etree
import re
import urllib3
from queue import Queue
import threading
import pandas as pd
import datetime
import requests
from amazon_spider.VPS_IP import pppoe_ip

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
sess = requests.Session()
import json
import uuid

"""抓取亚马逊 bsr分类，和 bsr asin top100"""


class bsr_catgory(BaseUtils):
    def __init__(self, site_name, table_name, save_table_name):
        super().__init__()
        self.site_name = site_name
        self.week = int(time.strftime("%W"))
        today = datetime.datetime.today()
        self.time_strftime_ = time.strftime("%Y-%m-%d", time.localtime())
        self.month = today.month
        self.name_path_queue = Queue()
        self.insert_list = []
        self.insert_id_pid_list = []
        self.id_and_en_name_list = []
        self.init_db(site_name)
        self.cookies_queue = Queue()
        self.item_list = []
        self.catgory_next_path_list = []
        self.asin_dict_ = {}
        self.category_first_id_list = []
        self.path_en_name_list = []
        self.headers_num_int = 0
        self.columns = ['cate_current_id', 'asin', 'bsr_rank', 'price', 'rating', 'total_comments', "week",
                        "year_month", 'category_id']
        self.table_name = table_name
        self.save_table_name = save_table_name

    def init_db(self, site_name):
        self.engine = self.mysql_connect()
        if site_name == 'us':
            self.connect = pymysql.connect(host=DB_CONN_DICT['mysql_host'], port=DB_CONN_DICT['mysql_port'],
                                           user=DB_CONN_DICT['mysql_user'],
                                           password=DB_CONN_DICT['mysql_pwd'], database="selection", charset="utf8mb4")
        else:
            self.connect = pymysql.connect(host=DB_CONN_DICT['mysql_host'], port=DB_CONN_DICT['mysql_port'],
                                           user=DB_CONN_DICT['mysql_user'],
                                           password=DB_CONN_DICT['mysql_pwd'], database="selection_" + site_name,
                                           charset="utf8mb4")
        connect_us = pymysql.connect(host=DB_CONN_DICT['mysql_host'], port=DB_CONN_DICT['mysql_port'],
                                     user=DB_CONN_DICT['mysql_user'],
                                     password=DB_CONN_DICT['mysql_pwd'], database="selection", charset="utf8mb4")

        cursor_us = connect_us.cursor()
        self.cursor = self.connect.cursor()
        if site_name == "us":
            self.site_url = 'https://www.amazon.com'
            self.host = 'www.amazon.com'
        elif site_name == 'uk':
            self.site_url = 'https://www.amazon.co.uk'  # 站点url
            self.host = 'www.amazon.co.uk'
        elif site_name == 'de':
            self.site_url = 'https://www.amazon.de'
            self.host = 'www.amazon.de'
        elif site_name == 'fr':
            self.site_url = 'https://www.amazon.fr'
            self.host = 'www.amazon.fr'
        elif site_name == 'es':
            self.site_url = 'https://www.amazon.es'
            self.host = 'www.amazon.es'
        elif site_name == 'it':
            self.site_url = 'https://www.amazon.it'
            self.host = 'www.amazon.it'
        self.reuests_para_val = Requests_param_val(site_name=self.site_name)
        self.year_month = f'2024_{self.month}'
        sele_sql = f"SELECT week FROM week_20_to_30 WHERE `year_month`='2024_{self.month}'"
        print(sele_sql)
        cursor_us.execute(sele_sql)
        year_week = cursor_us.fetchall()
        self.year_week = year_week[-1]
        print(self.year_week[0])

    def safeIndex(self, list: list, index: int, default: object = None):
        """
        安全获取list的索引对应的值
        :param list: 列表
        :param index: 索引
        :param default: 默认值
        :return:
        """
        if (index <= len(list) - 1):
            return list[index]

    def parse_url(self, nodes_num: int, url: str):
        print('解析当前nodes_num：：', nodes_num)
        arr = url.split("/")
        ref_suffix = self.safeIndex(arr, len(arr) - 1, None)
        category_first_id = self.safeIndex(arr, len(arr) - 3, None)
        category_id = self.safeIndex(arr, len(arr) - 2, None)
        if nodes_num == 1:
            level = 1
        elif url.endswith("_0"):
            level = 2
        elif url.endswith(f"{category_first_id}_1"):
            level = 3
        else:
            level = 4
        # 获取 parent id
        if level == 1:
            # 根节点
            category_id = "0"
            category_first_id = None
            category_parent_id = None
        elif level == 2:
            # 一级节点
            category_id = category_id
            category_first_id = category_id
            category_parent_id = "0"
        elif level == 3:
            # 一级节点下的次级节点
            category_id = category_id
            category_first_id = category_first_id
            category_parent_id = category_first_id
        elif level == 4:
            category_id = category_id
            category_first_id = category_first_id
            category_parent_id = ref_suffix[ref_suffix.rfind("_") + 1:]
        else:
            category_id = None
            category_first_id = None
            category_parent_id = None

        return {
            "category_id": category_id,
            "category_first_id": category_first_id,
            "category_parent_id": category_parent_id
        }

    def _requests(self, bsr_url):
        if self.cookies_queue.empty():
            cookies_dict = self.reuests_para_val.get_cookie()
            self.cookie_dict_delete_id = cookies_dict
            for ck in cookies_dict.values():
                self.cookies_queue.put(ck)
        while 1:
            cookie_str = self.cookies_queue.get()
            if len(cookie_str) > 50:
                try:
                    cookie_lsit = json.loads(cookie_str)
                except:
                    cookie_lsit = eval(cookie_str)
                cookie_dic = {}
                try:
                    for i in cookie_lsit:
                        if i:
                            cookie_dic[i["name"]] = i["value"]
                        else:
                            continue
                    cookie_str = ''
                    for k, v in cookie_dic.items():
                        cookie_str = cookie_str + str(k) + '=' + str(v) + ';'
                    break
                except:
                    cookie_str = ''
                    for k, v in cookie_lsit.items():
                        cookie_str = cookie_str + str(k) + '=' + str(v) + ';'
                    break
            else:
                break
        n = random.randint(70, 114)
        ua = f'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{n}.0.{random.randint(1000, 5000)}.{random.randint(1, 181)} Safari/537.36'
        headers = {
            'connection': 'close',
            'authority': self.host,
            'accept': 'text/html,*/*',
            'accept-language': 'zh-CN,zh;q=0.9',
            'cache-control': 'no-cache',
            'content-type': 'application/x-www-form-urlencoded;charset=UTF-8',
            'origin': f'{bsr_url}',
            'referer': f'{bsr_url}',
            'sec-ch-ua-mobile': '?0',
            'user-agent': ua
        }
        alphabet = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n']
        k = ""
        for i in (0, random.randint(0, 5)):
            k += random.choice(alphabet)
        headers[k] = str(uuid.uuid4())
        headers["cookie"] = cookie_str
        while True:
            try:
                sess.mount(self.site_url, py_ja3.DESAdapter())
                resp = sess.get(bsr_url, headers=headers,
                                timeout=20, verify=False)
                if ("Enter the characters you see below" in resp.text) or (
                        "Geben Sie die Zeichen unten ein" in resp.text) or (
                        "Introduce los caracteres que se muestran" in resp.text) or (
                        "Saisissez les caractères que vous voyez" in resp.text) or (
                        "Inserisci i caratteri visualizzati nello spazio" in resp.text):
                    print("出现验证码")
                    self.headers_num_int += 1
                    if self.headers_num_int > 120:
                        self.headers_num_int = 0
                        pppoe_ip()
                        time.sleep(5)
                    time.sleep(random.uniform(1.5, 5.5))
                    continue
                if resp.status_code == 200 or resp.status_code == 201:
                    response = etree.HTML(resp.text)
                    response.xpath("//span[@id='glow-ingress-line2']/text()")
                    return response
            except Exception as e:
                print("报错 继续 请求", e)
                time.sleep(random.uniform(1.5, 5.5))
                continue

    def html_4(self, bum):
        while True:
            if self.name_path_queue.empty() == False:
                querys = self.name_path_queue.get()
                print('请求=====：', querys)
                bsr_url = querys[2]
                bsr_class_name = querys[1]
                bsr_id = querys[0]
                category_id = querys[3]
                response = self._requests(bsr_url)
                a_2_list = response.xpath(
                    f'//div[@role="treeitem"]/span[contains(text(),"{bsr_class_name}")]/parent::div/following-sibling::div[@role="group"]/div[@role="treeitem"]/a')
                if self.site_name == 'us' or self.site_name == 'uk':
                    h1_bsr_name_lsit = response.xpath("//h1[contains(text(),'Best Sellers in')]/text()")
                    if h1_bsr_name_lsit:
                        h1_bsr_name = h1_bsr_name_lsit[0].replace("Best Sellers in", '').strip()
                    else:
                        h1_bsr_name = None
                elif self.site_name == 'de':
                    h1_bsr_name_lsit = response.xpath("//h1[contains(text(),'Bestseller in')]/text()")
                    if h1_bsr_name_lsit:
                        h1_bsr_name = h1_bsr_name_lsit[0].replace("Bestseller in", '').strip()
                    else:
                        h1_bsr_name = None
                elif self.site_name == 'fr':
                    h1_bsr_name_lsit = response.xpath("//h1[contains(text(),'Les meilleures ventes en')]/text()")
                    if h1_bsr_name_lsit:
                        h1_bsr_name = h1_bsr_name_lsit[0].replace("Les meilleures ventes en", '').strip()
                    else:
                        h1_bsr_name = None
                elif self.site_name == 'es':
                    h1_bsr_name_lsit = response.xpath("//h1[contains(text(),'Los más vendidos en')]/text()")
                    if h1_bsr_name_lsit:
                        h1_bsr_name = h1_bsr_name_lsit[0].replace("Los más vendidos en", '').strip()
                    else:
                        h1_bsr_name = None
                elif self.site_name == 'it':
                    h1_bsr_name_lsit = response.xpath("//h1[contains(text(),'Bestseller in')]/text()")
                    if h1_bsr_name_lsit:
                        h1_bsr_name = h1_bsr_name_lsit[0].replace("Bestseller in", '').strip()
                    else:
                        h1_bsr_name = None
                # self.id_and_en_name_list 1 代表还有 子分类 2代表没有子分类，为最终节点
                bsr_url_category_id_dict = self.parse_url(bum - 1, bsr_url)
                print('bsr_url_category_id_dict', bsr_url_category_id_dict)
                category_first_id_list = []
                if a_2_list:
                    self.id_and_en_name_list.append((bsr_id, h1_bsr_name, 1, bsr_url_category_id_dict['category_id'],
                                                     bsr_url_category_id_dict['category_parent_id']))
                    # 获取同级节点url
                    if (bum - 1) > 2:
                        # 获取同级节点url
                        if (bum - 1) > 2:
                            xpath_herf_list = [
                                f'//div[@role="treeitem"]/span[contains(text(),"{bsr_class_name}")]/parent::div/parent::div//a/@href',
                                f'//div[@role="treeitem"]/span[contains(text(),"{bsr_class_name}")]/parent::div/parent::div/parent::div//a/@href',
                                f'//div[@role="treeitem"]/span[contains(text(),"{bsr_class_name[0:5]}")]/parent::div/parent::div/parent::div//a/@href']
                            for xpath_herf in xpath_herf_list:
                                a_herf_list = response.xpath(xpath_herf)
                                if a_herf_list:
                                    a_url_dict = self.parse_url(bum, self.site_url + a_herf_list[0])
                                    print('a_url_dict22222:::', a_url_dict)
                                    category_first_id_list.append(
                                        [bsr_url_category_id_dict['category_first_id'],
                                         a_url_dict['category_first_id']])
                                    break
                    name_2_href_list = []
                    for a_2 in a_2_list:
                        name_2 = a_2.xpath('./text()')[0].strip()
                        name_2_href = a_2.xpath('./@href')[0].strip()
                        href = self.site_url + name_2_href
                        id_pid_data_dict = self.parse_url(bum, href)
                        print('id_pid_data_dict: ', id_pid_data_dict)
                        self.insert_id_pid_list.append(
                            [name_2, href, id_pid_data_dict['category_id'], id_pid_data_dict['category_parent_id']])
                        # 父级类目自增id  类目名称 二级类目， 链接
                        self.insert_list.append((bsr_id, name_2, bum, href, id_pid_data_dict['category_id'],
                                                 id_pid_data_dict['category_parent_id']))
                        # 存储当前分类id。，和页面获取的下一级url
                        name_2_href_list.append(href)
                        # 存储path 和 名称
                        self.path_en_name_list.append([href, name_2])
                    self.catgory_next_path_list.append(
                        [bsr_url_category_id_dict['category_id'], '|-|'.join(name_2_href_list)])
                else:
                    # 获取同级节点url
                    if (bum - 1) > 2:
                        xpath_herf_list = [
                            f'//div[@role="treeitem"]/span[contains(text(),"{bsr_class_name}")]/parent::div/parent::div//a/@href',
                            f'//div[@role="treeitem"]/span[contains(text(),"{bsr_class_name}")]/parent::div/parent::div/parent::div//a/@href',
                            f'//div[@role="treeitem"]/span[contains(text(),"{bsr_class_name[0:5]}")]/parent::div/parent::div/parent::div//a/@href']
                        for xpath_herf in xpath_herf_list:
                            a_herf_list = response.xpath(xpath_herf)
                            if a_herf_list:
                                print(self.site_url + a_herf_list[0])
                                a_url_dict = self.parse_url(bum, self.site_url + a_herf_list[0])
                                print('a_url_dict1111:::', a_url_dict)
                                category_first_id_list.append(
                                    [bsr_url_category_id_dict['category_first_id'],
                                     a_url_dict['category_first_id']])
                                break
                    self.id_and_en_name_list.append((bsr_id, h1_bsr_name, 2, bsr_url_category_id_dict['category_id'],
                                                     bsr_url_category_id_dict['category_parent_id']))
                if category_first_id_list:
                    # 获取当前请求url 分类id。对 first_id 拼接
                    self.category_first_id_list.append([bsr_url, '|-|'.join(category_first_id_list[0])])
                print(self.year_week[0], '====当前周===', self.week)
                ele_next = response.xpath(
                    '//a[contains(text(), "Weiter")]/@href|//a[contains(text(), "Next")]/@href|//a[contains(text(), "Page suivante")]/@href|//a[contains(text(), "Suivant")]/@href|//a[contains(text(), "Avanti")]/@href|//a[contains(text(), "Siguiente")]/@href|//a[contains(text(), "Seite")]/@href|//a[contains(text(), "siguiente")]/@href|//a[contains(text(), "Pagina")]/@href')
                data_client_recs_list = response.xpath(
                    "//div[@class='p13n-desktop-grid']/@data-client-recs-list")
                div_list = response.xpath("//div[@id='gridItemRoot']")
                for div in div_list:
                    bsr_rank_list = div.xpath(".//span[@class='zg-bdg-text']/text()")
                    if bsr_rank_list:
                        bsr_rank = re.findall(r'#(\d+)', bsr_rank_list[0])[0]
                    else:
                        bsr_rank = None
                    href_list = div.xpath(".//a[@role='link']/@href")
                    if href_list:
                        try:
                            asin = re.findall('dp/(.*)/ref', href_list[0])[0]
                        except:
                            asin = None
                    else:
                        asin = None
                    price = None
                    rating = None
                    reviews = None
                    if asin:
                        self.asin_dict_[asin] = [bsr_id, asin, bsr_rank, price, rating, reviews, self.week,
                                                 self.year_month, bsr_url_category_id_dict['category_id']]
                if data_client_recs_list:
                    data_list = eval(data_client_recs_list[0])
                    for data in data_list:
                        asin = data['id']
                        rank = data['metadataMap']['render.zg.rank']
                        if self.asin_dict_.get(asin):
                            tp_item = tuple(self.asin_dict_.get(asin))
                            self.item_list.append(tp_item)
                        else:
                            self.item_list.append(
                                (bsr_id, asin, int(rank), None, None, None, self.week, self.year_month,
                                 bsr_url_category_id_dict['category_id']))

                print(len(data_client_recs_list))
                if ele_next and data_client_recs_list:
                    self.get_bsr_asin_data(ele_next, bsr_id, bsr_url_category_id_dict['category_id'], asin_dict_={})
            else:
                break

    def get_bsr_asin_data(self, ele_next, c_id, bsr_url_category_id, asin_dict_):
        next_url = self.site_url + ele_next[0]
        response = self._requests(next_url)
        data_client_recs_list = response.xpath("//div[@class='p13n-desktop-grid']/@data-client-recs-list")
        div_list = response.xpath("//div[@id='gridItemRoot']")
        for div in div_list:
            bsr_rank_list = div.xpath(".//span[@class='zg-bdg-text']/text()")
            if bsr_rank_list:
                bsr_rank = re.findall(r'#(\d+)', bsr_rank_list[0])[0]
            else:
                bsr_rank = None
            href_list = div.xpath(".//a[@role='link']/@href")
            if href_list:
                try:
                    asin = re.findall('dp/(.*)/ref', href_list[0])[0]
                except:
                    asin = None
            else:
                asin = None
            price = None
            rating = None
            reviews = None
            if asin:
                asin_dict_[asin] = [c_id, asin, bsr_rank, price, rating, reviews, self.week, self.year_month,
                                    bsr_url_category_id]
        if data_client_recs_list:
            data_list = eval(data_client_recs_list[0])
            for data in data_list:
                asin = data['id']
                rank = data['metadataMap']['render.zg.rank']
                if asin_dict_.get(asin):
                    tp_item = tuple(asin_dict_.get(asin))
                    self.item_list.append(tp_item)
                else:
                    self.item_list.append(
                        (c_id, asin, int(rank), None, None, None, self.week, self.year_month, bsr_url_category_id))

    def save_asin_data(self):
        while True:
            try:
                print('存储：====', len(self.item_list))
                df = pd.DataFrame(data=self.item_list, columns=self.columns)
                df['date_info'] = self.time_strftime_
                df.drop_duplicates(['asin', 'bsr_rank', 'cate_current_id'], inplace=True)  # 去重
                print(df.shape)
                df.to_sql(f'{self.site_name}{self.save_table_name}', con=self.engine, if_exists="append", index=False)
                self.item_list = []
                break
            except Exception as e:
                print(e, '报错存储')
                self.mysql_reconnect(table_name=f'{self.site_name}{self.save_table_name}', e=e)
                time.sleep(10)
                continue

    def init_list(self):
        self.insert_list = []
        self.insert_id_pid_list = []
        self.name_path_queue = Queue()
        self.id_and_en_name_list = []
        self.catgory_next_path_list = []
        self.cookies_queue = Queue()
        self.asin_dict_ = {}
        self.item_list = []
        self.headers_num_int = 0
        self.path_en_name_list = []
        self.category_first_id_list = []

    def update_delete_time(self):
        try:
            if self.catgory_next_path_list:
                for category_path in self.catgory_next_path_list:
                    print('category_path::', category_path)
                    category_url_list = category_path[1].split('|-|')
                    category_id = category_path[0]
                    path_sql = f"""
                           select id, category_id, category_parent_id, en_name, `path`, delete_time
                       from {self.site_name}{self.table_name}
                       where (category_parent_id = "{category_id}")
                       order by {self.site_name}{self.table_name}.category_id, category_parent_id;
                       """
                    print('path_sql:', path_sql)
                    self.cursor.execute(path_sql)
                    exist_rows = self.cursor.fetchall()
                    group1_id = []
                    group2_id = []
                    for row in exist_rows:
                        print('rowrow::', row)
                        if row[4] in category_url_list:
                            group1_id.append(row[0])
                        else:
                            group2_id.append(row[0])
                    _strftime_ = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
                    print(_strftime_)
                    if group1_id:
                        if len(group1_id) == 1:
                            sql1 = f'update {self.site_name}{self.table_name} set delete_time = null where id in ({group1_id[0]})'
                        else:
                            sql1 = f"update {self.site_name}{self.table_name} set delete_time = null where id in {tuple(group1_id)}"
                            print('sql1::', sql1)
                        self.cursor.execute(sql1)
                        self.connect.commit()
                    if group2_id:
                        if len(group2_id) == 1:
                            sql2 = f"update {self.site_name}{self.table_name} set delete_time = '{_strftime_}' where id in ({group2_id[0]})"
                        else:
                            sql2 = f"update {self.site_name}{self.table_name} set delete_time = '{_strftime_}' where id in {tuple(group2_id)}"
                        print('sql2::', sql2)
                        self.cursor.execute(sql2)
                        self.connect.commit()

        except Exception as e:
            print('更新 delete_time 报错', traceback.format_exc(e))
        try:
            if self.category_first_id_list:
                for category_first_id in self.category_first_id_list:
                    print('category_first_id:', category_first_id)
                    # ['16380', 'industrial|-|hpc']
                    category_bsr_url = category_first_id[0]
                    category_first_name_id_list = category_first_id[1].split('|-|')
                    len_first_name = list(set(category_first_name_id_list))
                    if len(len_first_name) == 2:
                        print('重定向url：', category_bsr_url)
                        update_first_id_sql = f'''update {self.site_name}{self.table_name} set redirect_first_id="{category_first_name_id_list[1]}" where `path`="{category_bsr_url}"'''
                        print('update_first_id_sql::', update_first_id_sql)
                        self.cursor.execute(update_first_id_sql)
                        self.connect.commit()

        except Exception as e:
            print('更新 category_first_id_list 报错', traceback.format_exc(e))

    def save_date(self):
        for name_num_path in self.insert_list:
            self.connect.ping(reconnect=True)
            save_name_num_list = []
            while True:
                #  不存在就插入
                try:
                    self.connect.ping(reconnect=True)
                    select_sql_id = f'''SELECT id FROM {self.site_name}{self.table_name}  WHERE `path`="{name_num_path[3]}"'''
                    print('select_sql_id:', select_sql_id)
                    self.cursor.execute(select_sql_id)
                    name_id = self.cursor.fetchone()
                    if name_id is None:
                        save_name_num_list.append(name_num_path)
                    else:
                        select_sql_name = f'''SELECT en_name FROM {self.site_name}{self.table_name}  WHERE `path`="{name_num_path[3]}"'''
                        print('select_sql_name:', select_sql_name)
                        self.cursor.execute(select_sql_name)
                        en_name = self.cursor.fetchone()
                        print(en_name[0], '33333333333333333333333', name_num_path[1])
                        if en_name[0] == name_num_path[1]:
                            pass
                        else:
                            _strftime_ = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
                            update_name_sql = f'''update {self.site_name}{self.table_name} set delete_time = '2023-06-19 00:00:00' WHERE `path`="{name_num_path[3]}" and delete_time is null'''
                            print('更新 en_name：', update_name_sql)
                            self.cursor.execute(update_name_sql)
                            self.connect.commit()
                            save_name_num_list.append(name_num_path)
                    break
                except Exception as e:
                    print(e)
                    time.sleep(10)
                    self.init_db(self.site_name)
                    continue
            # # 插入新的数据
            self.cursor.executemany(
                f"insert into {self.site_name}{self.table_name} (p_id, en_name, nodes_num,path, category_id, category_parent_id) values (%s, %s,%s, %s,%s, %s)",
                save_name_num_list)
            self.connect.commit()

        for id_en_mane in self.id_and_en_name_list:
            self.connect.ping(reconnect=True)
            while True:
                try:
                    # self.id_and_en_name_list.append((bsr_id, h1_bsr_name, 2,bsr_url_category_id_dict['category_id'],bsr_url_category_id_dict['category_parent_id']))
                    # 更新 and_en_name， 根据 id 更新 分类名称
                    update_sql = f'update {self.site_name}{self.table_name} set and_en_name="{id_en_mane[1]}",leaf_node={id_en_mane[2]} WHERE category_id="{id_en_mane[3]}" and category_parent_id="{id_en_mane[4]}"'
                    print("更新 and_en_name：", update_sql)
                    self.cursor.execute(update_sql)
                    self.connect.commit()
                    break
                except Exception as e:
                    print('更新失败：', e)
                    time.sleep(10)
                    self.init_db(self.site_name)
                    continue

    def run(self):
        print(" run 函数 是抓取 分类节点，只新增，不删除")
        num = 1
        while True:
            print(num)
            b_num = num
            name_1_list = self.db_read_data(num, state_sql=True)
            if num < 15:
                for name_1 in name_1_list:
                    self.name_path_queue.put(name_1)
                if self.cookies_queue.empty():
                    cookies_dict = self.reuests_para_val.get_cookie()
                    self.cookie_dict_delete_id = cookies_dict
                    for ck in cookies_dict.values():
                        self.cookies_queue.put(ck)
                html_thread = []
                for i in range(50):
                    thread2 = threading.Thread(target=self.html_4, args=(b_num + 1,))
                    html_thread.append(thread2)
                for ti in html_thread:
                    ti.start()
                    time.sleep(1)
                for t2 in html_thread:
                    t2.join()
                print(f"当前 {num} 级分类 全部请求完毕", "获取二级类目：", len(self.insert_list))
                num += 1
            else:
                break
            self.save_date()
            self.update_delete_time()
            self.save_asin_data()
            self.init_list()
            pppoe_ip()
            time.sleep(5)
            self.init_db(self.site_name)
        self.select_id_1()

    def run_update_redirect_flag(self):
        print("run_update_redirect_flag 单独抓取 更新是否重定向的分类")
        num = 1
        while True:
            print(num)
            b_num = num
            name_1_list = self.db_read_data(num, state_sql=False)
            if num < 15:
                for name_1 in name_1_list:
                    self.name_path_queue.put(name_1)
                if self.cookies_queue.empty():
                    cookies_dict = self.reuests_para_val.get_cookie()
                    self.cookie_dict_delete_id = cookies_dict
                    for ck in cookies_dict.values():
                        self.cookies_queue.put(ck)
                html_thread = []
                for i in range(50):
                    thread2 = threading.Thread(target=self.html_4, args=(b_num + 1,))
                    html_thread.append(thread2)
                for ti in html_thread:
                    ti.start()
                    time.sleep(1)
                for t2 in html_thread:
                    t2.join()
                print(f"当前 {num} 级分类 全部请求完毕", "获取二级类目：", len(self.insert_list))
                num += 1
            else:
                break
            self.update_delete_time()
            self.init_list()

    def select_id_1(self):
        # 查询 子节点的顶级父类id
        select_sql_1 = f'select id from {self.site_name}{self.table_name} where nodes_num=2'
        self.cursor.execute(select_sql_1)
        id_1_list = self.cursor.fetchall()
        print(id_1_list)
        for id in id_1_list:
            en_name_id_list = []
            select_p_id = f"select t3.id,t4.en_name from (select t1.id,t1.parent_id,if(find_in_set(parent_id, @pids) > 0, @pids := concat(@pids, ',',id), 0) as ischild from (select id,p_id as parent_id from {self.site_name}{self.table_name} t  order by p_id,id) t1,(select @pids := {id[0]}) t2) t3 LEFT JOIN {self.site_name}{self.table_name} t4 on t3.id = t4.id where ischild != 0;"
            print("select_p_id:", select_p_id)
            self.cursor.execute(select_p_id)
            all_id_lsit = self.cursor.fetchall()
            if all_id_lsit:
                for en_name_id in all_id_lsit:
                    en_name_id_list.append(en_name_id[0])
                id_tuple = tuple(en_name_id_list)
                print(len(id_tuple))
                update_sql = f'update {self.site_name}{self.table_name} set one_category_id={id[0]} where id in {id_tuple}'
                self.cursor.execute(update_sql)
                self.connect.commit()

    def db_read_data(self, i, state_sql):
        while True:
            try:
                with self.engine.begin() as conn:
                    sql_read = f'select id,en_name,`path`,category_id from {self.site_name}{self.table_name} where nodes_num={i} and category_state=1 and delete_time is NULL'
                    print(sql_read)
                    a = conn.execute(sql_read)
                    df_read = pd.DataFrame(a, columns=['id', 'en_name', 'path', 'category_id'])
                    if df_read.shape[0] == 0:
                        return []
                    index_tuple = tuple(df_read['category_id'])
                    id_tuple = tuple(df_read['id'])
                    if state_sql == True:
                        if len(index_tuple) == 1:
                            sql_update = f"""UPDATE {self.site_name}{self.table_name} a set category_state=2 where a.category_id in ('{index_tuple[0]}')"""
                        else:
                            sql_update = f"""UPDATE {self.site_name}{self.table_name} a set category_state=2 where a.category_id in {index_tuple}"""
                    else:
                        if len(id_tuple) == 1:
                            sql_update = f"""UPDATE {self.site_name}{self.table_name} a set category_state=2 where a.id in ('{id_tuple[0]}')"""
                        else:
                            sql_update = f"""UPDATE {self.site_name}{self.table_name} a set category_state=2 where a.id in {id_tuple}"""
                    print('sql_update:', sql_update)
                    conn.execute(sql_update)
                categorys_list = list(df_read.id.astype(
                    "U") + '|-|' + df_read.en_name + '|-|' + df_read.path + '|-|' + df_read.category_id)
                category_url_list = []
                for i in categorys_list:
                    category_url_list.append(i.split('|-|'))
                return category_url_list
            except Exception as e:
                print(e)
                time.sleep(10)
                self.init_db(self.site_name)
                continue

    def update_category_state(self):
        id_sql = f"""
        select id
        from {self.site_name}{self.table_name}
        where category_id in (
                select category_id
                from (
                                 select category_id,
                                                count(distinct coalesce(redirect_first_id, category_first_id)) count
                                 from {self.site_name}{self.table_name}
                                 where delete_time is null
                                 group by category_id
                                 having count > 1) tmp
        )
          and delete_time is null
        order by category_id,category_first_id
        """
        self.cursor.execute(id_sql)
        id_tuple = self.cursor.fetchall()
        id_list = []
        for id in id_tuple:
            id_list.append(id[0])
        if id_list:
            update_sql = f"""
                update {self.site_name}{self.table_name} set category_state=1 where id in {tuple(id_list)}"""
            print('update_sql::', update_sql)
            self.cursor.execute(update_sql)
            self.connect.commit()

    def send_ms(self):
        for i in range(3):
            try:
                print('发送消息通知：')
                update_sql = f'UPDATE {self.site_name}{self.table_name} set category_state=1;'
                self.cursor.execute(update_sql)
                self.connect.commit()
                url = 'http://selection.yswg.com.cn:8080/soundasia_selection/workflow/emit'
                data = {"dateType": "day", "reportDate": self.time_strftime_, "statusVal": 3,
                        "siteName": self.site_name,
                        "remark": "bsr榜单爬取完毕 ", "isEnd": "是", "tableName": f"{self.site_name}{self.save_table_name}",
                        "status": "bsr榜单爬取完毕"}
                headers = {
                    'Content-Type': 'application/json',
                    'Accept': 'application/json'
                }
                t = requests.post(url, headers=headers, json=data)
                print(t.text)
                break
            except:
                self.init_db(self.site_name)
                time.sleep(10)
                continue

        for i in range(3):
            try:
                self.connect.ping(reconnect=True)
                days = ((datetime.datetime.now()) + datetime.timedelta(days=-8)).strftime("%Y-%m-%d")
                delect_sql = f"DELETE FROM {self.site_name}{self.save_table_name} WHERE date_info < '{days}';"
                print(delect_sql)
                self.cursor.execute(delect_sql)
                self.connect.commit()
                break
            except Exception as e:
                print(e, '222222222')
                time.sleep(20)
                continue


if __name__ == '__main__':
    spider_type = sys.argv[1]
    if spider_type == "new":
        table_name = '_new_releases'
        save_table_name = '_new_releases_top100_asin'
    elif spider_type == 'bsr':
        table_name = '_bs_category'
        save_table_name = '_bs_category_top100_asin'
    else:
        table_name = None
        save_table_name = None
    if table_name and save_table_name:
        for site in ['us', 'de', 'uk']:
            spider_us = bsr_catgory(site, table_name, save_table_name)
            spider_us.run()
            if spider_type == 'bsr':
                spider_us.update_category_state()
                spider_us.run_update_redirect_flag()
            spider_us.send_ms()
    else:
        print('启动参数错误:', spider_type)
