import sys
import os

sys.path.append(os.path.dirname(sys.path[0]))  # 上级目录
from amazon_params import py_ja3
from utils.requests_param import Requests_param_val
from utils.db_connect import BaseUtils
import requests as requests2

sess = requests2.Session()
import threading
import random
import uuid
import json
from queue import Queue
import time
import pandas as pd
import traceback
from lxml import etree
import re
import urllib3
from threading_spider.db_connectivity import connect_db
import ast

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
from datetime import datetime, timedelta


class Spiders_Demo(BaseUtils):
    def __init__(self, site_name):
        super().__init__()
        if site_name == "us":
            self.site_url = 'https://www.amazon.com/'
            self.host = 'www.amazon.com'
        elif site_name == 'uk':
            self.site_url = 'https://www.amazon.co.uk/'  # 站点url
            self.host = 'www.amazon.co.uk'
        elif site_name == 'de':
            self.site_url = 'https://www.amazon.de/'
            self.host = 'www.amazon.de'
        elif site_name == 'fr':
            self.site_url = 'https://www.amazon.fr/'
            self.host = 'www.amazon.fr'
        elif site_name == 'es':
            self.site_url = 'https://www.amazon.es/'
            self.host = 'www.amazon.es'
        elif site_name == 'it':
            self.site_url = 'https://www.amazon.it/'
            self.host = 'www.amazon.it'
        self.check_spider_syn = 'check_spider_html'
        self.site_name = site_name
        self.cookies_queue = Queue()
        self.queries_queue = Queue()
        self.delete_cookies_list = []
        self.update_state_list = []
        self.asin_not_found_list = []
        self.engine_pg6 = self.pg_connect_6()
        self.engine_pg14 = self.pg_connect()
        self.redis14 = self.redis_db()
        self.reuests_para_val = Requests_param_val(site_name=site_name, spider="seller_account_product",
                                                   proxy_name=None)
        self.items_list = []

    def get_url(self, url_list=None):
        '发送请求'
        for url_spider in url_list:
            url_spider_list = url_spider.split('|-|')
            scraper_url = url_spider_list[0]
            spider_type = url_spider_list[1]
            xpath_list = ast.literal_eval(url_spider_list[2])
            time.sleep(0.3)
            n = random.randint(70, 114)
            ua = f'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{n}.0.{random.randint(1000, 5000)}.{random.randint(1, 181)} Safari/537.36'
            headers = {
                'connection': 'close',
                'authority': self.host,
                'accept': 'text/html,*/*',
                'accept-language': 'zh-CN,zh;q=0.9',
                'cache-control': 'no-cache',
                'content-type': 'application/x-www-form-urlencoded;charset=UTF-8',
                'origin': scraper_url,
                'referer': self.site_url,
                'sec-ch-ua-mobile': '?0',
                'user-agent': ua
            }
            if self.cookies_queue.empty():
                cookies_dict = self.reuests_para_val.get_cookie()
                self.cookie_dict_delete_id = cookies_dict
                for ck in cookies_dict.values():
                    self.cookies_queue.put(ck)
            while 1:
                cookie_str = self.cookies_queue.get()
                if len(cookie_str) > 50:
                    try:
                        cookie_lsit = json.loads(cookie_str)
                    except:
                        cookie_lsit = eval(cookie_str)
                    cookie_dic = {}
                    try:
                        for i in cookie_lsit:
                            if i:
                                cookie_dic[i["name"]] = i["value"]
                            else:
                                continue
                        cookie_str = ''
                        for k, v in cookie_dic.items():
                            cookie_str = cookie_str + str(k) + '=' + str(v) + ';'
                        break
                    except:
                        cookie_str = ''
                        for k, v in cookie_lsit.items():
                            cookie_str = cookie_str + str(k) + '=' + str(v) + ';'
                        break
                else:
                    break
            alphabet = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r',
                        's', 't', 'u', 'v', 'w', 'x', 'y', 'z']
            k = ""
            for i in (0, random.randint(0, 26)):
                k += random.choice(alphabet)
            headers[k] = str(uuid.uuid4())
            headers["cookie"] = cookie_str
            print('scraper_url:', scraper_url)
            for i in range(5):
                time.sleep(60)
                try:
                    sess.mount(self.site_url, py_ja3.DESAdapter())
                    if spider_type != 'review':
                        resp = sess.get(scraper_url, headers=headers, timeout=10, verify=False)
                    else:
                        data = {
                            'sortBy': 'recent',
                            'reviewerType': 'all_reviews',
                            'pageNumber': '1',
                            'shouldAppend': 'undefined',
                            'deviceType': 'desktop',
                            'canShowIntHeader': 'undefined',
                            'reftag': 'cm_cr_arp_d_paging_btm_next_1',
                            'pageSize': '20',
                            'asin': 'B07TVWTG82',
                            'scope': 'reviewsAjax0'
                        }
                        resp = sess.post(scraper_url, headers=headers, timeout=10, verify=False, data=data)
                    if ("Enter the characters you see below" in resp.text) or (
                            "Geben Sie die Zeichen unten ein" in resp.text) or (
                            "Introduce los caracteres que se muestran" in resp.text) or (
                            "Saisissez les caractères que vous voyez" in resp.text) or (
                            "Inserisci i caratteri visualizzati nello spazio" in resp.text):
                        print(f"{self.site_name}  站点  +   使用代理ip出现验证码：{scraper_url}")
                        time.sleep(random.uniform(3.2, 4.5))
                        continue
                    break
                except Exception as e:
                    print('请求报错：', e)
                    time.sleep(random.uniform(3.2, 4.5))
                    self.update_state_list.append(scraper_url)
                    continue
            response = resp.text
            etree_html = etree.HTML(response)
            # 'data-hook=\\"review\\"'
            if spider_type != 'review':
                try:
                    ingress = etree_html.xpath("//span[@id='glow-ingress-line2']/text()")
                    print("**************  邮编  ***************", ingress)
                    ingress = ingress[0].strip()
                except:
                    ingress = None
                    print("获取邮编錯誤:")
                print('***** 邮编: ', ingress, ' 请求url: ', scraper_url)
                if ingress:
                    if ("中国大陆" in ingress) or ("China" in ingress) or ("Hong" in ingress) or (
                            "Chine" in ingress) or (
                            "Cina" in ingress):
                        if self.site_name != 'es' or self.site_name != 'it':
                            try:
                                cookie_ubid_main_id = re.findall(r'ubid-main=(.*?);', cookie_str)[0]
                            except:
                                cookie_ubid_main_id = re.findall(r'session-id=(.*?);', cookie_str)[0]
                            for cookie_key_value in self.cookie_dict_delete_id.items():
                                if cookie_ubid_main_id in cookie_key_value[1]:
                                    self.delete_cookies_list.append(cookie_key_value[0])
                            time.sleep(random.uniform(1.2, 3.5))
                        self.update_state_list.append(scraper_url)
                        continue
                else:
                    print("没有获取到邮编，", scraper_url)
                    time.sleep(random.uniform(1.5, 3.5))
                    self.update_state_list.append(scraper_url)
                    continue

                if ("Page Not Found" in response and ingress is None) or (
                        "We are sorry! This Gift Card is not available" in response and ingress is None) or (
                        "500 - An error occurred" in response and ingress is None) or (
                        "Sorry! Something went wrong!" in response and ingress is None) or (
                        "Tut uns Leid!" in response and ingress is None):
                    print("***** Page Not Found  关键词搜索出现问题：", scraper_url)
                    time.sleep(random.uniform(2.2, 4.5))
                    self.asin_not_found_list.append(scraper_url)
                    continue
            else:
                if 'data-hook=\\"review\\"' not in response:
                    time.sleep(random.uniform(2.2, 4.5))
                    self.update_state_list.append(scraper_url)
                    continue
            with open(f'{spider_type}.html', 'w', encoding='utf-8')as f:
                f.write(response)
            if spider_type == 'search_term':
                self.spider_seacr_term(response, etree_html, xpath_list)
            elif spider_type == 'asin':
                self.spider_asin(response, etree_html, xpath_list)
            elif spider_type == 'feedback':
                self.spider_feedback(response, etree_html, xpath_list)
            elif spider_type == 'product':
                self.spider_product(response, etree_html, xpath_list)
            elif spider_type == 'review':
                self.spider_review(response, etree_html, xpath_list)
            # elif spider_type == 'bsr':
            #     self.spider_bsr(response, etree_html, xpath_list)
            # elif spider_type == 'new_bsr':
            #     self.spider_new_bsr(response, etree_html, xpath_list)

    def spider_seacr_term(self, response, etree_html, xpath_list):
        sold_id_class_list = xpath_list[0]['sold']
        itmes = {}
        itmes['spider_type'] = 'seacr_term'
        for i in sold_id_class_list:
            data_list = etree_html.xpath(i)
            if data_list:
                itmes['sold'] = True
                break
            else:
                itmes['sold'] = False

        zr_list = xpath_list[1]['zr']
        for i in zr_list:
            data_list = etree_html.xpath(i)
            if data_list:
                itmes['zr'] = True
                break
            else:
                itmes['zr'] = False

        sp_list = xpath_list[2]['sp']
        for i in sp_list:
            data_list = etree_html.xpath(i)
            if data_list:
                itmes['sp'] = True
                break
            else:
                itmes['sp'] = False

        sb_list = xpath_list[3]['sb']
        for i in sb_list:
            print(i)
            data_list = etree_html.xpath(i)
            if data_list:
                itmes['sb'] = True
                break
            else:
                itmes['sb'] = False
        sb2_list = xpath_list[4]['sb2']
        for i in sb2_list:
            data_list = etree_html.xpath(i)
            if data_list:
                itmes['sb2'] = True
                break
            else:
                itmes['sb2'] = False
        sb3_list = xpath_list[5]['sb3']
        for i in sb3_list:
            data_list = etree_html.xpath(i)
            if data_list:
                itmes['sb3'] = True
                break
            else:
                itmes['sb3'] = False

        bs_list = xpath_list[6]['bs']
        for i in bs_list:
            data_list = etree_html.xpath(i)
            if data_list:
                itmes['bs'] = True
                break
            else:
                itmes['bs'] = False
        buy_list = xpath_list[7]['buy']
        for i in buy_list:
            data_list = etree_html.xpath(i)
            if data_list:
                itmes['buy'] = True
                break
            else:
                itmes['buy'] = False
        # 标签广告
        sb4_list = xpath_list[8]['sb4']
        for i in sb4_list:
            data_list = etree_html.xpath(i)
            if data_list:
                itmes['sb4'] = True
                break
            else:
                itmes['sb4'] = False

        self.items_list.append(itmes)

    def spider_asin(self, response, etree_html, xpath_list):
        items = {}
        items['spider_type'] = 'asin'
        div_head_list = xpath_list[0]['div_head']
        for i in div_head_list:
            div_data = etree_html.xpath(i)
            if div_data:
                items['div_head'] = True
                break
            else:
                items['div_head'] = False
        div_bottom_list = xpath_list[1]['div_bottom']
        for i in div_bottom_list:
            div_data = etree_html.xpath(i)
            if div_data:
                items['div_bottom'] = True
                break
            else:
                items['div_bottom'] = False
        # category
        category_list = xpath_list[2]['category']
        for i in category_list:
            div_data = etree_html.xpath(i)
            if div_data:
                items['category'] = True
                break
            else:
                items['category'] = False

        self.items_list.append(items)

    def spider_feedback(self, response, etree_html, xpath_list):
        items = {}
        items['spider_type'] = 'feedback'

        # div_feedback_list = ['//div[@id="page-section-feedback"]/@class']
        div_feedback_list = xpath_list[0]['div_feedback']
        for i in div_feedback_list:
            div_feedback = etree_html.xpath(i)
            if div_feedback:
                items['div_feedback'] = True
                break
            else:
                items['div_feedback'] = False
        # div_site_list = ['//div[@id="page-section-detail-seller-info"]/@class']
        div_site_list = xpath_list[1]['div_site']
        for i in div_site_list:
            div_site = etree_html.xpath(i)
            if div_site:
                items['div_site'] = True
                break
            else:
                items['div_site'] = False
        self.items_list.append(items)

    def spider_product(self, response, etree_html, xpath_list):
        items = {}
        items['spider_type'] = 'product'
        if "Need help?" in response and "Try checking your spelling or use more general terms" in response:
            items['product'] = None
        else:
            results_list = xpath_list[0]['results']
            for i in results_list:
                results = etree_html.xpath(i)
                if results:
                    items['results'] = True
                    break
                else:
                    items['results'] = False

            data_asin_list = xpath_list[1]['product_asin']
            for i in data_asin_list:
                product_asin = etree_html.xpath(i)
                if product_asin:
                    items['product_asin'] = True
                    break
                else:
                    items['product_asin'] = False

        self.items_list.append(items)

    def spider_review(self, response, etree_html, xpath_list):
        items = {}
        response = response.replace('\\n', '').replace('data-hook=\\"review\\"', 'data-hook="review"').replace(
            '\\"', '"')
        resp = etree.HTML(response)
        div_list = xpath_list[0]['review']
        for i in div_list:
            results = resp.xpath(i)
            if results:
                items['review'] = True
                break
            else:
                items['review'] = False
        self.items_list.append(items)

    def spider_bsr(self, response, etree_html, xpath_list):
        items = {}
        items['spider_type'] = 'bsr'
        div_bsr_name_list = xpath_list[0]['bsr_name']
        for i in div_bsr_name_list:
            bsr_name = etree_html.xpath(i)
            if bsr_name:
                items['bsr_name'] = True
                break
            else:
                items['bsr_name'] = False

        bsr_asin_list = xpath_list[1]['bsr_asin']
        for i in bsr_asin_list:
            bsr_asin = etree_html.xpath(i)
            if bsr_asin:
                items['bsr_asin'] = True
                break
            else:
                items['bsr_asin'] = False
        self.items_list.append(items)

    def spider_new_bsr(self, response, etree_html, xpath_list):
        items = {}
        items['spider_type'] = 'new_bsr'
        div_bsr_name_list = xpath_list[0]['new_name']
        for i in div_bsr_name_list:
            bsr_name = etree_html.xpath(i)
            if bsr_name:
                items['new_name'] = True
                break
            else:
                items['new_name'] = False

        bsr_asin_list = xpath_list[1]['new_asin']
        for i in bsr_asin_list:
            bsr_asin = etree_html.xpath(i)
            if bsr_asin:
                items['new_asin'] = True
                break
            else:
                items['new_asin'] = True
        self.items_list.append(items)

    def read_db_data(self):
        while True:
            try:
                self.engine_pg6 = self.pg_connect_6()
                with self.engine_pg6.begin() as conn:
                    sql_read = f"SELECT url, id, spider_type,xpath_list FROM {self.check_spider_syn} WHERE state=1 and site_name='{self.site_name}';"
                    print(sql_read)
                    a = conn.execute(sql_read)
                    self.df_read = pd.DataFrame(a, columns=['url', 'id', 'spider_type', 'xpath_list'])

                    if self.df_read.shape[0] == 0:
                        return []
                    self.index_tuple = tuple(self.df_read['id'])
                    # self.wx_user = list(self.df_read['wx_user'])[0]
                    # print('消息通知人：', self.wx_user)
                    if len(self.index_tuple) == 1:
                        sql_update = f"""UPDATE {self.check_spider_syn} a set state=2 where site_name='{self.site_name}' and a.id in ({self.index_tuple[0]})"""
                    else:
                        sql_update = f"""UPDATE {self.check_spider_syn} a set state=2 where site_name='{self.site_name}' and a.id in {self.index_tuple}"""
                    conn.execute(sql_update)
                data_list = list(self.df_read.url + '|-|' + self.df_read.spider_type + '|-|' + self.df_read.xpath_list)
                print(data_list)
                url_type_list = []
                for data in data_list:
                    url_list = data.split('|-|')[0].split(',')
                    for url in url_list:
                        url_type_list.append(url + '|-|' + data.split('|-|')[1] + '|-|' + data.split('|-|')[2])
                return url_type_list
            except Exception as e:
                print("读取数据出bug并等待5s继续", e, f"\n{traceback.format_exc()}")
                time.sleep(60)
                continue

    def update_syn_state(self, spider_type_list, state):
        print(f'更新 状态 ', spider_type_list)
        while True:
            try:
                self.engine_pg6 = self.pg_connect_6()
                spider_type_list = list(set(spider_type_list))
                if spider_type_list:
                    # 创建包含 URL 的数据框
                    df_url = pd.DataFrame(data={'url': spider_type_list})
                    search_tuple = tuple(
                        [search_term.replace("%", "%%").replace('"', '\\"').replace("'", "\\'") for search_term in
                         set(df_url.url)])
                    with self.engine_pg6.begin() as conn:
                        if len(search_tuple) == 1:
                            # 如果只有一个 URL，用单引号括起来
                            sql_update = f"UPDATE {self.check_spider_syn} SET state = {state} WHERE site_name='{self.site_name}' and url = ('{search_tuple[0]}');"
                        else:
                            # 如果有多个 URL，使用 IN 子句
                            sql_update = f"UPDATE {self.check_spider_syn} SET state = {state} WHERE site_name='{self.site_name}' and url IN {search_tuple};"
                        print(sql_update)
                        conn.execute(sql_update)

                break
            except Exception as e:
                print(f"{self.check_spider_syn}'等待5s继续", e, f"\n{traceback.format_exc()}")
                time.sleep(30)
                continue

    def run(self):
        if self.cookies_queue.empty():
            cookies_dict = self.reuests_para_val.get_cookie()
            self.cookie_dict_delete_id = cookies_dict
            for ck in cookies_dict.values():
                self.cookies_queue.put(ck)
        while True:
            url_list = self.read_db_data()
            if url_list:
                html_thread = []
                for i in range(1):
                    thread2 = threading.Thread(target=self.get_url, args=(url_list,))
                    html_thread.append(thread2)
                for ti in html_thread:
                    ti.start()
                for t2 in html_thread:
                    t2.join()
                if self.update_state_list:
                    self.update_syn_state(self.update_state_list, 1)
                    self.update_state_list = []
                if self.asin_not_found_list:
                    self.update_syn_state(self.asin_not_found_list, 4)
                    self.asin_not_found_list = []
                time.sleep(15)
            else:
                break
        print('items_list::::',self.items_list)
        self.send_ms()

    def send_ms(self):
        content_list = []
        content_data_list = []
        for items in self.items_list:
            if items.get('results') == False:
                print('店铺 总数 html 可能发生变化')
                content_list.append('店铺 总数 html 可能发生变化')
                content_data_list.append(['product', '店铺 总数 html 发生变化'])
            if items.get('product_asin') == False:
                print('店铺 asin html 可能发生变化')
                content_list.append('店铺 asin html 可能发生变化')
                content_data_list.append(['product', '店铺 asin html 发生变化'])
            if items.get('new_name') == False:
                print('新品榜单 name html 可能发生变化')
                content_list.append('新品榜单 name html 可能发生变化')
                content_data_list.append(['new_bsr', '新品榜单 name html 发生变化'])
            if items.get('new_asin') == False:
                print('新品榜单 asin html 可能发生变化')
                content_list.append('新品榜单 asin html 可能发生变化')
                content_data_list.append(['new_bsr', '新品榜单 asin html 发生变化'])
            if items.get('sold') == False:
                print('搜索词 搜索量 html 可能发生变化')
                content_list.append('搜索词 搜索量 html 可能发生变化')
                content_data_list.append(['search_term', '新品榜单 asin html 发生变化'])
            if items.get('zr') == False:
                print('搜索词 解析asin html 可能发生变化')
                content_list.append('搜索词 解析asin html 可能发生变化')
                content_data_list.append(['search_term', '搜索词 解析asin html 发生变化'])
            if items.get('div_head') == False:
                print('asin 头部 html 可能发生变化')
                content_list.append('asin 头部 html 可能发生变化')
                content_data_list.append(['asin', 'asin 头部 html 发生变化'])
            if items.get('div_bottom') == False:
                print('asin 底部 html 可能发生变化')
                content_list.append('asin 底部 html 可能发生变化')
                content_data_list.append(['asin', 'asin 底部 html 发生变化'])
            if items.get('bsr_name') == False:
                print('bsr榜单 name html 可能发生变化')
                content_list.append('bsr榜单 name html 可能发生变化')
                content_data_list.append(['bsr', 'bsr榜单 name html 发生变化'])
            if items.get('bsr_asin') == False:
                print('bsr榜单 asin html 可能发生变化')
                content_list.append('bsr榜单 asin html 可能发生变化')
                content_data_list.append(['bsr', 'bsr榜单 asin html 发生变化'])
            if items.get('review') == False:
                print('评论 post请求 html 可能发生变化')
                content_list.append('评论 post请求 html 可能发生变化')
                content_data_list.append(['review', '评论 post请求 html 发生变化'])
            if items.get('review') == False:
                print('评论 post请求 html 可能发生变化')
                content_list.append('评论 post请求 html 可能发生变化')
                content_data_list.append(['review', '评论 post请求 html 发生变化'])
            if items.get('category')== False:
                content_list.append('asin 头部分类 html 可能发生变化')
                content_data_list.append(['asin', 'category 头部分类 html 发生变化'])
            if items.get('sb4')== False:
                content_list.append('搜索词 标签下广告位 html 可能发生变化')
                content_data_list.append(['搜索词', '搜索词 标签下广告位 html 发生变化'])
            if items.get('buy')== False:
                content_list.append('搜索词 下的月销量 html 可能发生变化')
                content_data_list.append(['搜索词', '搜索词 下的月销量 html 发生变化'])
            if items.get('bs')== False:
                content_list.append('搜索词 bs类型 html 可能发生变化')
                content_data_list.append(['搜索词', '搜索词 bs类型 html 发生变化'])
            if items.get('sb3')== False:
                content_list.append('搜索词 sbv视频 html 可能发生变化')
                content_data_list.append(['搜索词', '搜索词 sbv视频 html 发生变化'])
            if items.get('sb')== False:
                content_list.append('搜索词 头部品牌 html 可能发生变化')
                content_data_list.append(['搜索词', '搜索词 头部品牌 html 发生变化'])
            if items.get('sb2')== False:
                content_list.append('搜索词 尾部品牌 html 可能发生变化')
                content_data_list.append(['搜索词', '搜索词 尾部品牌 html 发生变化'])
            if items.get('sp')== False:
                content_list.append('搜索词 sp类型 html 可能发生变化')
                content_data_list.append(['搜索词', '搜索词 sp类型 html 发生变化'])
        db_class = connect_db('us')
        # account = self.wx_user
        account = 'pengyanbing'
        title = '校验 html 是否有变化'
        time_strftime = time.strftime("%Y-%m-%d %X", time.localtime())
        if content_list:
            content_str = '\n'.join(content_list)
            content = content_str + time_strftime
            db_class.send_mg(account, title, content)
        for i in range(5):
            try:
                self.engine_pg6 = self.pg_connect_6()
                with self.engine_pg6.begin() as conn:
                    update_sql = f"update {self.check_spider_syn}  set state = 1,content='html无发现变动' where site_name='{self.site_name}'"
                    conn.execute(update_sql)
                    if content_data_list:
                        for content_data in content_data_list:
                            update_ = f"update {self.check_spider_syn}  set content = f'{content_data[1]}' where spider_type='{content_data[0]}' and where site_name='{self.site_name}'"
                            conn.execute(update_)
                break
            except:
                time.sleep(30)

    def get_redis_data(self):
        # 当前时间
        new_date = datetime.now().strftime("%Y-%m-%d")
        json_data = self.redis14.hget('2024-02', f'{new_date}')
        # 将 JSON 字符串解析为 Python 字典
        if json_data:
            nested_dict = json.loads(json_data)
            date_info = nested_dict['date_info']
            asin_total = nested_dict['asin_total']
            asin_request_err_total = len(nested_dict['asin_request_err_total'])
            yzm_err_total = len(nested_dict['yzm_err_total'])
            success_asin_total = len(nested_dict['success_asin_total'])
            request_total_count = len(nested_dict['request_total_count'])
            print(f'抓取日期：{date_info}，抓取总数：{asin_total}，异常总数：{asin_request_err_total}，'
                  f'验证码次数：{yzm_err_total}, 成功抓取总数：{success_asin_total}，一共发送请求总数：{request_total_count}')
            # with self.engine_pg6.begin() as conn:
            #     # 如果只有一个 URL，用单引号括起来
            #     sql_update = f"UPDATE {self.site_name}_count_request_data SET asin_total = {asin_total},asin_request_err_total={asin_request_err_total},yzm_err_total={yzm_err_total},success_asin_total={success_asin_total},request_total_count={request_total_count} where date_info='{new_date}'"
            #     print(sql_update)
            #     conn.execute(sql_update)
        else:
            print("Key or field not found.")

    def created_redis_hash_key(self):
        # 获取当前日期
        current_date = datetime.now()
        # 计算后一天的日期
        next_day_date = current_date + timedelta(days=1)
        # 格式化日期字符串
        new_date = next_day_date.strftime("%Y-%m-%d")
        print(new_date)
        new_date_month = datetime.now().strftime("%Y-%m")
        sql_count = f'select count(id) from {self.site_name}_all_syn_st_month_{new_date_month.replace("-", "_")}'
        print(sql_count)
        df_count_id = pd.read_sql(sql_count, con=self.engine_pg14)
        asin_total = df_count_id.values.tolist()[0][0]
        print(asin_total)
        # 定义要添加的不同字典
        user_info_dict = {
            'date_info': new_date,
            'asin_total': asin_total,
            'asin_request_err_total': [],
            'yzm_err_total': [],
            'success_asin_total': [],
            'request_total_count': []
        }
        # 将当前时间的小时、分钟和秒设置为0
        next_day_date = next_day_date.replace(hour=0, minute=0, second=0, microsecond=0)
        # 生成当天的24小时时间
        hourly_times = [next_day_date + timedelta(hours=i) for i in range(24)]
        # 打印每个小时的时间
        for hour_time in hourly_times:
            print(hour_time.strftime("%Y-%m-%d:%H"))
            user_info_dict[hour_time.strftime("%Y-%m-%d:%H")] = []
        # 将字典添加到相同的 user_data 键中
        self.redis14.hset(f'{new_date_month}', new_date, json.dumps(user_info_dict))
        with self.engine_pg6.begin() as conn:
            # 如果只有一个 URL，用单引号括起来
            sql_update = f"INSERT INTO {self.site_name}_count_request_data (date_info,asin_total) VALUES ('{new_date}',{asin_total});"
            print(sql_update)
            conn.execute(sql_update)

if __name__ == '__main__':
    Spiders_Demo('us').run()
    Spiders_Demo('us').created_redis_hash_key()