import time

import pandas as pd
from sqlalchemy import create_engine
from sqlalchemy.pool import NullPool
from sqlalchemy import text
from sqlalchemy.orm import sessionmaker
import platform
import traceback
import json
import uuid
from sqlalchemy.exc import SQLAlchemyError


class ConnectSpider:
    def __init__(self):
        self.pg_port = 54328
        self.pg_db = "selection"
        self.pg_user = "postgres"
        self.pg_pwd = "fazAqRRVV9vDmwDNRNb593ht5TxYVrfTyHJSJ3BS"
        self.pg_host = "61.145.136.61"
        self.db_engine = create_engine(f"postgresql://{self.pg_user}:{self.pg_pwd}@{self.pg_host}:{self.pg_port}/{self.pg_db}")

        pg_host = "192.168.10.223"
        self.db_engine192 = create_engine(
            f"postgresql://{self.pg_user}:{self.pg_pwd}@{self.pg_host}:{self.pg_port}/{self.pg_db}")

        # mysql
        self.sql_port = 3306
        self.sql_db = "selection"
        self.sql_user = "adv_yswg"
        self.sql_pwd = "Gd1pGJog1ysLMLBdML8w81"
        self.sql_host = "rm-wz9yg9bsb2zf01ea4yo.mysql.rds.aliyuncs.com"
        self.charset = 'utf8mb4'

        # 创建数据库连接字符串
        connection_string_mysql = f"mysql+pymysql://{self.sql_user}:{self.sql_pwd}@{self.sql_host}:{self.sql_port}/{self.sql_db}?charset={self.charset}"
        self.mysql_engine = create_engine(connection_string_mysql)

        # mysql
        sql_port = 19030
        sql_db = "test"
        sql_user = "fangxingjun"
        sql_pwd = "fangxingjun12345"
        sql_host = "192.168.10.151"
        wai_host = "113.100.143.162"

        # 创建数据库连接字符串
        connection_string_mysql = f"mysql+pymysql://{sql_user}:{sql_pwd}@{wai_host}:{sql_port}/{sql_db}"
        self.mysql_test = create_engine(connection_string_mysql)

        # mysql
        sql_port = 19030
        sql_db = "selection"
        sql_user = "fangxingjun"
        sql_pwd = "fangxingjun12345"
        sql_host = "192.168.10.151"
        wai_host = "113.100.143.162"

        # 创建数据库连接字符串
        connection_string_mysql = f"mysql+pymysql://{sql_user}:{sql_pwd}@{sql_host}:{sql_port}/{sql_db}"
        self.mysql_selection = create_engine(connection_string_mysql)

    def con (self):
        # pg_port = 5433
        # pg_db = "selection"
        # pg_user = "postgres"
        # pg_pwd = "fazAqRRVV9vDmwDNRNb593ht5TxYVrfTyHJSJ3BS"
        # pg_host = "61.145.136.61"
        # db_engine = create_engine(f"postgresql://{pg_user}:{pg_pwd}@{pg_host}:{pg_port}/{pg_db}")
        # return db_engine
        if platform.system().lower() == 'windows':
            PG_CONN_DICT = {
                "pg_port": 5433,
                "pg_db": "selection",
                "pg_user": "postgres",
                "pg_pwd": "fazAqRRVV9vDmwDNRNb593ht5TxYVrfTyHJSJ3BS",
                "pg_host": "192.168.10.223",
            }
        else:
            PG_CONN_DICT = {
                "pg_port": 5433,
                "pg_db": "selection",
                "pg_user": "postgres",
                "pg_pwd": "fazAqRRVV9vDmwDNRNb593ht5TxYVrfTyHJSJ3BS",
                "pg_host": "61.145.136.61",
            }
        nums = 0
        while True:
            try:
                db = 'selection'
                engine_pg = create_engine(
                    f"postgresql+psycopg2://{PG_CONN_DICT['pg_user']}:{PG_CONN_DICT['pg_pwd']}@{PG_CONN_DICT['pg_host']}:{PG_CONN_DICT['pg_port']}/{db}",
                    encoding='utf-8', connect_args={"connect_timeout": 10}, poolclass=NullPool)
                return engine_pg
            except Exception as e:
                print("pg_connect 14 t11111111111111111111111:", e, f"\n{traceback.format_exc()}")
                continue

    def mysql(self):
        sql_port = 3306
        sql_db = "us_spider"
        sql_user = "adv_yswg"
        sql_pwd = "Gd1pGJog1ysLMLBdML8w81"
        sql_host = "rm-wz9yg9bsb2zf01ea4yo.mysql.rds.aliyuncs.com"
        charset = 'utf8mb4'

        # 创建数据库连接字符串
        connection_string_mysql = f"mysql+pymysql://{sql_user}:{sql_pwd}@{sql_host}:{sql_port}/{sql_db}?charset={charset}"
        mysql_engine = create_engine(connection_string_mysql)
        return mysql_engine

    def mysql_us_spider(self):
        sql_port = 19030
        # sql_db = "us_spider"
        sql_db = "test"
        sql_user = "fangxingjun"
        sql_pwd = "fangxingjun12345"
        sql_host = "192.168.10.151"

        # 创建数据库连接字符串
        connection_string_mysql = f"mysql+pymysql://{sql_user}:{sql_pwd}@{sql_host}:{sql_port}/{sql_db}"
        mysql_us_spider_engine = create_engine(connection_string_mysql)
        return mysql_us_spider_engine

    def hd_search_keyword_test(self):
        with self.mysql_engine.connect() as connection:
            table_name = "us_brand_analytics_month_2025"
            query = text(f"select search_term from {table_name} where month= 2 and category1 in ('Home', 'Home Entertainment', 'Home Improvement') limit 10000")
            print(query)
            try:
                result = connection.execute(query)
                df_status = pd.DataFrame(result.fetchall())

                if df_status.empty:
                    print(f"没有找到记录")
                    return None

                ranges = df_status[['search_term']].values.tolist()
                connection.commit()  # 提交事务
                return ranges  # 返回第一行的范围

            except Exception as e:
                print(f"查询时发生错误: {e}")
                connection.rollback()  # 回滚事务以防异常
                return None


    def save_hd_search__test(self, items):
        """批量保存数据到数据库。"""
        table_name = "hd_search_terms"

        columns = ['id','search_term','state']
        df = pd.DataFrame(items, columns=columns)

        # 使用with语句管理数据库连接
        with self.db_engine.connect() as connection:
            df.to_sql(
                name=table_name,
                con=connection,
                if_exists='append',  # 如果表存在，则追加数据
                index=False,  # 不将DataFrame的index写入数据库表
                method='multi',  # 可选：使用'multi'方法提高批量插入效率
                chunksize=1000  # 可选：每次插入1000行，可以根据实际情况调整
            )


    def save_hd_count(self, item_list):
        """批量保存数据到数据库。"""
        table_name = "hd_count"
        columns = ['search_term', 'methods', 'page', 'project_count', 'created_at']

        # 创建DataFrame
        df = pd.DataFrame(item_list, columns=columns)

        # 使用with语句管理数据库连接
        with self.db_engine.connect() as connection:
            df.to_sql(
                name=table_name,
                con=connection,
                if_exists='append',  # 如果表存在，则追加数据
                index=False,  # 不将DataFrame的index写入数据库表
                method='multi',  # 可选：使用'multi'方法提高批量插入效率
                chunksize=1000  # 可选：每次插入1000行，可以根据实际情况调整
            )


    def save_stock_img_id(self, items):
        """批量保存数据到数据库。"""
        table_name = "stock_image_id_wj"

        # 提取 image_title 和 image_size_info
        processed_items = []
        for item in items:
            image_title = item.get('title', '')
            image_size_info = json.dumps(item.get('sizes', {}))
            processed_item = {
                'account_id': item['account_id'],
                'image_id': item['image_id'],
                'state': item['state'],
                'created_at': item['created_at'],
                'image_title': image_title,
                'image_size_info': image_size_info
            }
            processed_items.append(processed_item)

        # 定义DataFrame的列
        columns = ['account_id', 'image_id', 'state', 'created_at', 'image_title', 'image_size_info']
        df = pd.DataFrame(processed_items, columns=columns)

        # 使用with语句管理数据库连接
        with self.db_engine192.connect() as connection:
            df.to_sql(
                name=table_name,
                con=connection,
                if_exists='append',
                index=False
            )

    def update_search_id(self,search_id):
        with self.db_engine192.connect() as connection:
            table_name = "homedepot_search_terms"
            update_query = text(f"UPDATE {table_name} SET state = :new_state WHERE search_id = :search_id")

            try:
                # 执行更新操作
                result = connection.execute(update_query, {"new_state": 3, "search_id": search_id})

                # 检查受影响的行数
                rows_updated = result.rowcount

                if rows_updated > 0:
                    print(f"Successfully updated {rows_updated} record(s) with search_id: {search_id}")
                else:
                    print(f"No records found for search_id: {search_id}")

                # 提交事务
                connection.commit()
            except Exception as e:
                print(f"An error occurred while updating the record: {e}")
                # 回滚事务以防异常
                connection.rollback()




    # def homedepot_ranges(self,i):
    #     with self.db_engine192.connect() as connection:
    #         table_name = "homedepot_ranges"
    #         query = f"SELECT start_range, end_range, id FROM {table_name} WHERE( STATE = 1)"
    #         df_status = pd.read_sql(query, con=connection)
    #         try:
    #             start_range = df_status['start_range'].iloc[i]
    #             end_range = df_status['end_range'].iloc[i]
    #             id_ = df_status['id'].iloc[i]
    #             sql_update = text(f"UPDATE {table_name} SET state = 2 WHERE id = {id_}")
    #             result = connection.execute(sql_update, {"id_": id_})
    #
    #         except IndexError:
    #             print("没有找到记录")
    #             return None
    #         return [start_range, end_range]

    # def get_homedepot_search_terms(self):
    #     with self.db_engine192.connect() as connection:
    #         table_name = "homedepot_search_terms"
    #         query = text(f"SELECT search_term, id FROM {table_name} where state = 1")
    #
    #         # 执行查询
    #         result = connection.execute(query)
    #
    #         # 获取所有记录
    #         records = result.fetchall()
    #
    #         # 转换为列表并返回
    #         # terms_and_states = [{"search_term": row[0], "search_id": row[1]} for row in records]
    #         terms_and_states = [row[0]for row in records]
    #
    #         return terms_and_states

    def get_hd_search_terms(self,start_id,limit):
        with self.db_engine192.connect() as connection:
            table_name = "hd_search_terms"
            # query = f"-- SELECT search_term,id FROM {table_name} WHERE id BETWEEN {start_id} AND {start_id + limit - 1} and state = 1;"
            query = f"SELECT search_term,id FROM {table_name} WHERE id BETWEEN {start_id} AND {start_id + limit - 1} and state = 5;"
            df_status = pd.read_sql(query, con=connection)
            search_term_id_pairs = [f"{search_term}|-|-|-|-|-|{id}" for search_term, id in zip(df_status['search_term'], df_status['id'])]
            sql_update = text(f"UPDATE {table_name} SET state = 2 WHERE id IN :ids")
            connection.execute(sql_update, {"ids": tuple(df_status['id'])})
            connection.commit()  # 添加事务提交
            return search_term_id_pairs


    def get_hd_search_test(self,start_id,limit):
        with self.db_engine192.connect() as connection:
            table_name = "hd_search_test"
            # query = f"-- SELECT search_term,id FROM {table_name} WHERE id BETWEEN {start_id} AND {start_id + limit - 1} and state = 1;"
            query = f"SELECT search_term,id FROM {table_name} WHERE id BETWEEN {start_id} AND {start_id + limit - 1} and state = 1;"
            df_status = pd.read_sql(query, con=connection)
            search_term_id_pairs = [f"{search_term}|-|-|-|-|-|{id}" for search_term, id in zip(df_status['search_term'], df_status['id'])]
            sql_update = text(f"UPDATE {table_name} SET state = 2 WHERE id IN :ids")
            connection.execute(sql_update, {"ids": tuple(df_status['id'])})
            connection.commit()  # 添加事务提交
            return search_term_id_pairs




    def count_search_terms(self,start_id,limit):
        with self.db_engine192.connect() as connection:
            table_name = "hd_search_terms"
            # query = f"-- SELECT search_term,id FROM {table_name} WHERE id BETWEEN {start_id} AND {start_id + limit - 1} and state = 1;"
            query = f"SELECT search_term,id FROM {table_name} WHERE id BETWEEN {start_id} AND {start_id + limit - 1}  and state = 3;"
            df_status = pd.read_sql(query, con=connection)
            search_term_id_pairs = [f"{search_term}|-|-|-|-|-|{id}" for search_term, id in zip(df_status['search_term'], df_status['id'])]

            connection.commit()  # 添加事务提交
            return search_term_id_pairs




    def get_homedepot_search_terms(self,start_id,limit):
        with self.db_engine192.connect() as connection:
            table_name = "homedepot_search_terms"
            query = f"SELECT search_term,id FROM {table_name} WHERE id BETWEEN {start_id} AND {start_id + limit - 1} and state = 2;"
            df_status = pd.read_sql(query, con=connection)
            search_term_id_pairs = [f"{search_term}|-|-|-|-|-|{id}" for search_term, id in zip(df_status['search_term'], df_status['id'])]
            sql_update = text(f"UPDATE {table_name} SET state = 2 WHERE id IN :ids")
            connection.execute(sql_update, {"ids": tuple(df_status['id'])})
            connection.commit()  # 添加事务提交
            return search_term_id_pairs

    def get_homedepot_url(self,start_id,limit):
        with self.db_engine192.connect() as connection:
            table_name = "homedepot_projects_items"
            query = f"SELECT search_term,id,title,price,url,image,sku FROM {table_name} WHERE id BETWEEN '{str(start_id)}' AND '{str(start_id + limit - 1)}' AND state = '1';"
            df_status = pd.read_sql(query, con=connection)
            search_term_id_pairs = [f"{search_term}|-|-|-|-|-|{id}|-|-|-|-|-|{title}|-|-|-|-|-|{price}|-|-|-|-|-|{url}|-|-|-|-|-|{image}|-|-|-|-|-|{sku}" for search_term, id,title,price,url,image,sku in zip(df_status['search_term'], df_status['id'],df_status['title'],df_status['price'],df_status['url'],df_status['image'],df_status['sku'])]
            sql_update = text(f"UPDATE {table_name} SET state = 2 WHERE id IN :ids")
            connection.execute(sql_update, {"ids": tuple(df_status['id'])})
            connection.commit()  # 添加事务提交
            return search_term_id_pairs

    def hd_details_upstate_to_3(self, success_id):
        print('开始更新为3..............')
        with self.db_engine192.connect() as connection:
            table_name = "homedepot_projects_items"
            if len(success_id[0]) == 1:
                sql_update = text(f"UPDATE {table_name} SET state = '3' WHERE id = :id")
                connection.execute(sql_update, {"id": success_id[0][0]})
            else:
                placeholders = ",".join([f":id{i}" for i in range(len(success_id[0]))])
                sql_update = text(f"UPDATE {table_name} SET state = '3' WHERE id IN ({placeholders})")
                params = {f"id{i}": id for i, id in enumerate(success_id[0])}
                connection.execute(sql_update, params)
            connection.commit()
            print(str(sql_update), "执行中...")
        print('成功更新为3')

    def hd_details_upstate_to_4(self, failed_id):
        print('开始更新为4..............')
        with self.db_engine192.connect() as connection:
            table_name = "homedepot_projects_items"
            if len(failed_id) == 1:  # 注意这里检查的是failed_id的长度，不是failed_id[0]的长度
                sql_update = f"UPDATE {table_name} SET state = '4' WHERE id = :id"
                # 确保传递的id是整数
                connection.execute(text(sql_update), {"id": int(failed_id[0][0])})
            else:
                # 对于多个ID的情况，确保它们都是整数
                ids = tuple(int(id[0]) for id in failed_id)
                sql_update = f"UPDATE {table_name} SET state = '4' WHERE id IN :ids"
                connection.execute(text(sql_update), {"ids": ids})

            connection.commit()
            print(sql_update, "执行中...")

        print('成功更新为4')

    def hd_test_to_3(self, success_id):
        print('开始更新为3..............')
        with self.db_engine192.connect() as connection:
            table_name = "hd_search_test"
            if len(success_id[0]) == 1:
                sql_update = text(f"UPDATE {table_name} SET state = '3' WHERE id = :id")
                connection.execute(sql_update, {"id": success_id[0][0]})
            else:
                placeholders = ",".join([f":id{i}" for i in range(len(success_id[0]))])
                sql_update = text(f"UPDATE {table_name} SET state = '3' WHERE id IN ({placeholders})")
                params = {f"id{i}": id for i, id in enumerate(success_id[0])}
                connection.execute(sql_update, params)
            connection.commit()
            print(str(sql_update), "执行中...")
        print('成功更新为3')

    def hd_test_to_4(self, failed_id):
        print('开始更新为4..............')
        failed_id = [item[0] for item in failed_id]
        with self.db_engine192.connect() as connection:
            table_name = "hd_search_test"

            # 动态生成占位符和参数字典
            placeholders = ','.join([f':id{i}' for i in range(len(failed_id))])
            print(placeholders)
            sql_update = f"UPDATE {table_name} SET state = '4' WHERE id IN ({placeholders})"

            # 创建参数字典
            params = {f'id{i}': id for i, id in enumerate(failed_id)}

            # 使用text()方法并传递params参数
            result = connection.execute(text(sql_update), params)
            connection.commit()

        print('成功更新为4')

    def hd_test_to_5(self, invalid_id):
        print('开始更新为5..............')
        with self.db_engine192.connect() as connection:
            table_name = "hd_search_test"

            # 动态生成占位符和参数字典
            placeholders = ','.join([f':id{i}' for i in range(len(invalid_id))])
            sql_update = f"UPDATE {table_name} SET state = '5' WHERE id IN ({placeholders})"

            # 创建参数字典
            params = {f'id{i}': id for i, id in enumerate(invalid_id)}

            # 使用text()方法并传递params参数
            result = connection.execute(text(sql_update), params)
            connection.commit()

        print('成功更新为5')



    def hd_upstate_to_3(self, success_id):
        print('开始更新为3..............')
        with self.db_engine192.connect() as connection:
            table_name = "hd_search_terms"
            if len(success_id[0]) == 1:
                sql_update = text(f"UPDATE {table_name} SET state = '3' WHERE id = :id")
                connection.execute(sql_update, {"id": success_id[0][0]})
            else:
                placeholders = ",".join([f":id{i}" for i in range(len(success_id[0]))])
                sql_update = text(f"UPDATE {table_name} SET state = '3' WHERE id IN ({placeholders})")
                params = {f"id{i}": id for i, id in enumerate(success_id[0])}
                connection.execute(sql_update, params)
            connection.commit()
            print(str(sql_update), "执行中...")
        print('成功更新为3')



    def hd_upstate_to_4(self, failed_id):
        print('开始更新为4..............')
        failed_id = [item[0] for item in failed_id]
        with self.db_engine192.connect() as connection:
            table_name = "hd_search_terms"

            # 动态生成占位符和参数字典
            placeholders = ','.join([f':id{i}' for i in range(len(failed_id))])
            print(placeholders)
            sql_update = f"UPDATE {table_name} SET state = '4' WHERE id IN ({placeholders})"

            # 创建参数字典
            params = {f'id{i}': id for i, id in enumerate(failed_id)}

            # 使用text()方法并传递params参数
            result = connection.execute(text(sql_update), params)
            connection.commit()

        print('成功更新为4')



    def hd_upstate_to_5(self, invalid_id):
        print('开始更新为5..............')
        with self.db_engine192.connect() as connection:
            table_name = "hd_search_terms"

            # 动态生成占位符和参数字典
            placeholders = ','.join([f':id{i}' for i in range(len(invalid_id))])
            sql_update = f"UPDATE {table_name} SET state = '5' WHERE id IN ({placeholders})"

            # 创建参数字典
            params = {f'id{i}': id for i, id in enumerate(invalid_id)}

            # 使用text()方法并传递params参数
            result = connection.execute(text(sql_update), params)
            connection.commit()

        print('成功更新为5')

    def homedepot_ranges(self, i):
        with self.db_engine192.connect() as connection:
            table_name = "homedepot_ranges"
            query = text(f"SELECT start_range, end_range, id FROM {table_name} WHERE state = 1 AND id = :i")
            print(query)
            try:
                result = connection.execute(query, {"i": i})
                df_status = pd.DataFrame(result.fetchall())

                if df_status.empty:
                    print(f"没有找到记录，id: {i}")
                    return None

                ranges = df_status[['start_range', 'end_range']].values.tolist()
                ids_to_update = df_status['id'].tolist()

                # 批量更新状态
                sql_update = text(f"UPDATE {table_name} SET state = 2 WHERE id = ANY(:ids)")
                connection.execute(sql_update, {"ids": ids_to_update})
                connection.commit()  # 提交事务

                return ranges[0]  # 返回第一行的范围

            except Exception as e:
                print(f"查询或更新状态时发生错误: {e}")
                connection.rollback()  # 回滚事务以防异常
                return None

    def homedepot_details_ranges(self, i):
        with self.db_engine192.connect() as connection:
            table_name = "homedepot_details_ranges"
            query = text(f"SELECT start_range, end_range, id FROM {table_name} WHERE state = 1 AND id = :i")
            print(query)
            try:
                result = connection.execute(query, {"i": i})
                df_status = pd.DataFrame(result.fetchall())

                if df_status.empty:
                    print(f"没有找到记录，id: {i}")
                    return None

                ranges = df_status[['start_range', 'end_range']].values.tolist()
                ids_to_update = df_status['id'].tolist()

                # 批量更新状态
                sql_update = text(f"UPDATE {table_name} SET state = 2 WHERE id = ANY(:ids)")
                connection.execute(sql_update, {"ids": ids_to_update})
                connection.commit()  # 提交事务
                return ranges[0]  # 返回第一行的范围

            except Exception as e:
                print(f"查询或更新状态时发生错误: {e}")
                connection.rollback()  # 回滚事务以防异常
                return None

    def save_project_items(self, items):
        """批量保存数据到数据库。"""
        table_name = "homedepot_projects_items"
        # 提取并处理需要的数据
        processed_items = []
        for item in items:
            processed_item = {
                'title': item.get('title', ''),
                'price': item.get('price', None),
                'url': item.get('url', ''),
                'image': item.get('image', ''),
                'sku': item.get('sku', ''),
                'search_term': item.get('search_term', ''),
                'state':item.get('state', ''),
                'created_at': item.get('created_at', None)  # 如果'created_at'不在item中，可以考虑设置为当前时间
            }
            processed_items.append(processed_item)

        # 定义DataFrame的列
        columns = ['title', 'price', 'url', 'image', 'sku', 'search_term','state','created_at']
        df = pd.DataFrame(processed_items, columns=columns)

        # 使用with语句管理数据库连接
        with self.db_engine.connect() as connection:
            df.to_sql(
                name=table_name,
                con=connection,
                if_exists='append',  # 如果表存在，则追加数据
                index=False,  # 不将DataFrame的index写入数据库表
                method='multi',  # 可选：使用'multi'方法提高批量插入效率
                chunksize=1000  # 可选：每次插入1000行，可以根据实际情况调整
            )


    def save_hd_hd_test2(self, items):
        """批量保存数据到数据库。"""
        table_name = "hd_test2"
        # 提取并处理需要的数据
        processed_items = []
        for item in items:
            processed_item = {
                'title': item.get('title', ''),
                'price': item.get('price', None),
                'url': item.get('url', ''),
                'image': item.get('image', ''),
                'sku': item.get('sku', ''),
                'search_term': item.get('search_term', ''),
                'page': item.get('page', ''),
                'state': item.get('state', ''),
                'project_data': json.dumps(item.get('project_data', [])),
                'created_at': item.get('created_at', None)  # 如果'created_at'不在item中，可以考虑设置为当前时间
            }
            processed_items.append(processed_item)

        # 定义DataFrame的列
        columns = ['title', 'price', 'url', 'image', 'sku', 'search_term', 'page', 'state', 'project_data',
                   'created_at']
        df = pd.DataFrame(processed_items, columns=columns)

        # 使用with语句管理数据库连接
        with self.db_engine.connect() as connection:
            df.to_sql(
                name=table_name,
                con=connection,
                if_exists='append',  # 如果表存在，则追加数据
                index=False,  # 不将DataFrame的index写入数据库表
                method='multi',  # 可选：使用'multi'方法提高批量插入效率
                chunksize=1000  # 可选：每次插入1000行，可以根据实际情况调整
            )

    def save_hd_hd_test(self, items):
        """批量保存数据到数据库。"""
        table_name = "hd_test"
        # 提取并处理需要的数据
        processed_items = []
        for item in items:
            processed_item = {
                'title': item.get('title', ''),
                'price': item.get('price', None),
                'url': item.get('url', ''),
                'image': item.get('image', ''),
                'sku': item.get('sku', ''),
                'search_term': item.get('search_term', ''),
                'page': item.get('page', ''),
                'state': item.get('state', ''),
                'project_data': json.dumps(item.get('project_data', [])),
                'created_at': item.get('created_at', None)  # 如果'created_at'不在item中，可以考虑设置为当前时间
            }
            processed_items.append(processed_item)

        # 定义DataFrame的列
        columns = ['title', 'price', 'url', 'image', 'sku', 'search_term', 'page', 'state', 'project_data',
                   'created_at']
        df = pd.DataFrame(processed_items, columns=columns)

        # 使用with语句管理数据库连接
        with self.db_engine.connect() as connection:
            df.to_sql(
                name=table_name,
                con=connection,
                if_exists='append',  # 如果表存在，则追加数据
                index=False,  # 不将DataFrame的index写入数据库表
                method='multi',  # 可选：使用'multi'方法提高批量插入效率
                chunksize=1000  # 可选：每次插入1000行，可以根据实际情况调整
            )

    def save_hd_project_items(self, items):
        """批量保存数据到数据库。"""
        table_name = "hd_projects_items"
        # 提取并处理需要的数据
        processed_items = []
        for item in items:
            processed_item = {
                'title': item.get('title', ''),
                'price': item.get('price', None),
                'url': item.get('url', ''),
                'image': item.get('image', ''),
                'sku': item.get('sku', ''),
                'search_term': item.get('search_term', ''),
                'page': item.get('page', ''),
                'state':item.get('state', ''),
                'project_data': json.dumps(item.get('project_data', [])),
                'created_at': item.get('created_at', None)  # 如果'created_at'不在item中，可以考虑设置为当前时间
            }
            processed_items.append(processed_item)

        # 定义DataFrame的列
        columns = ['title', 'price', 'url', 'image', 'sku', 'search_term','page','state','project_data','created_at']
        df = pd.DataFrame(processed_items, columns=columns)

        # 使用with语句管理数据库连接
        with self.db_engine.connect() as connection:
            df.to_sql(
                name=table_name,
                con=connection,
                if_exists='append',  # 如果表存在，则追加数据
                index=False,  # 不将DataFrame的index写入数据库表
                method='multi',  # 可选：使用'multi'方法提高批量插入效率
                chunksize=1000  # 可选：每次插入1000行，可以根据实际情况调整
            )


    def save_hd_html(self, items):
        """批量保存数据到数据库。"""
        table_name = "hd_htmls"
        # 提取并处理需要的数据
        processed_items = []
        for item in items:
            processed_item = {
                'search_term': item['search_term'],
                'page': item['page'],
                'html_data':item['html_data'],
                'created_at': item['created_at']  # 如果'created_at'不在item中，可以考虑设置为当前时间
            }
            processed_items.append(processed_item)

        # 定义DataFrame的列
        columns = ['search_term', 'page', 'html_data', 'created_at']
        df = pd.DataFrame(processed_items, columns=columns)

        # 使用with语句管理数据库连接
        with self.db_engine.connect() as connection:
            df.to_sql(
                name=table_name,
                con=connection,
                if_exists='append',  # 如果表存在，则追加数据
                index=False,  # 不将DataFrame的index写入数据库表
                method='multi',  # 可选：使用'multi'方法提高批量插入效率
                chunksize=1000  # 可选：每次插入1000行，可以根据实际情况调整
            )


    def save_hd_details(self, items):
        """批量保存数据到数据库。"""
        table_name = "homedepot_details"

        # 提取并处理需要的数据
        processed_items = []
        for item in items:
            processed_item = {
                'search_term': item.get('search_term', ''),
                'title': item.get('title', ''),
                'price': item.get('price', None),
                'url': item.get('url', ''),
                'image': item.get('image', ''),
                'sku': item.get('sku', ''),
                'salient_points': item.get('salient_points', ''),
                'description':item.get('description', ''),
                'model': item.get('model', ''),
                'created_at': item.get('created_at', None)  # 如果'created_at'不在item中，可以考虑设置为当前时间
            }
            processed_items.append(processed_item)

        # 定义DataFrame的列
        columns = ['search_term','title', 'price', 'url', 'image', 'sku', 'salient_points','description','model','created_at']
        df = pd.DataFrame(processed_items, columns=columns)

        # 使用with语句管理数据库连接
        with self.db_engine.connect() as connection:
            df.to_sql(
                name=table_name,
                con=connection,
                if_exists='append',  # 如果表存在，则追加数据
                index=False,  # 不将DataFrame的index写入数据库表
                method='multi',  # 可选：使用'multi'方法提高批量插入效率
                chunksize=1000  # 可选：每次插入1000行，可以根据实际情况调整
            )



    def save_homedepot_range(self, items):
        """批量保存数据到数据库。"""
        table_name = "homedepot_details_ranges"

        # 提取并处理需要的数据
        processed_items = []
        for item in items:
            processed_item = {
                'start_range': item.get('start_range', ''),
                'end_range': item.get('end_range', ''),
                'state':item.get('state', ''),
            }
            processed_items.append(processed_item)

        # 定义DataFrame的列
        columns = ['start_range', 'end_range', 'state',]
        df = pd.DataFrame(processed_items, columns=columns)

        # 使用with语句管理数据库连接
        with self.db_engine.connect() as connection:
            df.to_sql(
                name=table_name,
                con=connection,
                if_exists='append',  # 如果表存在，则追加数据
                index=False,  # 不将DataFrame的index写入数据库表
                method='multi',  # 可选：使用'multi'方法提高批量插入效率
                chunksize=1000  # 可选：每次插入1000行，可以根据实际情况调整
            )



    def get_account_id(self, item_id):
        with self.db_engine192.connect() as connection:
            table_name = "stock_image_summary_wj"
            query = text(f"SELECT account_id, id FROM {table_name} WHERE id = :item_id")
            result = connection.execute(query, {"item_id": item_id})
            df_status = pd.DataFrame(result.fetchall())
            df_status.columns = result.keys()
            try:
                accounts = df_status.account_id.iloc[0]
            except IndexError:
                accounts = None  # 或者处理不存在的情况
            return accounts

    def update_id_to_3(self,account_id):
        with self.db_engine192.connect() as connection:
            table_name = "stock_image_summary_wj"
            success_id = tuple(account_id)
            sql_update = text(f"UPDATE {table_name} SET state = 3 WHERE account_id IN :success_id")
            result = connection.execute(sql_update, {"success_id": success_id})
            print('成功更新为3')
            connection.close()

    def save_account(self,items):
        """批量保存数据到数据库。"""
        table_name = "stock_image_summary_wj"

        # 定义DataFrame的列
        columns = ['account_id', 'account_secret', 'year_month', 'spider_date','state','created_time']
        df = pd.DataFrame(items, columns=columns)

        # 使用with语句管理数据库连接
        with self.db_engine192.connect() as connection:
            df.to_sql(
                name=table_name,
                con=connection,
                if_exists='append',
                index=False
            )
        print("保存成功！")


    def delet_datails(self, image_id_list):
        with self.db_engine192.connect() as connection:
            table_name = "stock_image_detail_wj"

            # 使用 SQLAlchemy 的 text 函数来创建 SQL 语句
            query = text(f"SELECT image_id FROM {table_name} WHERE account_id = 'zhouweiqing@yswg.com.cn';")

            # 使用 connection.execute() 来执行查询
            result = connection.execute(query).fetchall()

            # 获取表中的 image_id 列表
            db_image_ids = [row[0] for row in result]

            # 找出不在 image_id_list 中的 image_id
            non_existent_image_ids = set(db_image_ids) - set(image_id_list)

            # 删除不在 image_id_list 中的记录
            for image_id in non_existent_image_ids:
                delete_query = text(
                    f"DELETE FROM {table_name} WHERE account_id = 'zhouweiqing@yswg.com.cn' AND image_id = '{image_id}';")
                connection.execute(delete_query)

            # 提交更改
            connection.commit()


    def get_datails_image_id(self, account_id):
        with self.db_engine192.connect() as connection:
            table_name = "stock_image_detail_wj"
            sql_query = text(f"SELECT image_id FROM {table_name} WHERE account_id = :account_id and created_time  < '2024-09-02 00:00:00'")
            result = connection.execute(sql_query, {"account_id": account_id})
            image_id_list = [int(row[0]) for row in result.fetchall()]
            # 提交更改
            # connection.commit()
            return image_id_list



    # 1111111111111
    def save_stock_detail(self, item):
        """批量保存数据到数据库。"""
        table_name = "stock_image_detail_wj"

        # 将item包装成列表
        items_list = [item]

        # 定义DataFrame的列
        columns = ['account_id', 'image_id', 'image_size_info', 'image_title', 'image_type', 'image_url','state', 'created_time']
        df = pd.DataFrame(items_list, columns=columns)

        with self.db_engine192.connect() as connection:
            df.to_sql(
                name=table_name,
                con=connection,
                if_exists='append',
                index=False
            )
        # print("保存成功！")


    def get_pic_urls_limit1(self, account_id):
        pic_data_list = []  # 创建一个空列表来存储结果
        with self.db_engine192.connect() as connection:
            table_name = "stock_image_detail_wj"
            query = text(
                f"""select image_url, image_id, image_title from {table_name} where account_id = :account_id and state = 1 limit 1""")

            try:
                result = connection.execute(query, {'account_id': account_id})
                for row in result:  # 遍历所有的结果行
                    if row is not None:
                        # 将 RowProxy 转换为标准的字典
                        row_dict = dict(zip(result.keys(), row))  # 使用 keys 和 fetchone 的结果创建字典

                        # 直接构建所需格式的字符串
                        pic_datas = f"{row_dict['image_url']}||{row_dict['image_id']}||{row_dict['image_title']}"
                        pic_data_list.append(pic_datas)  # 添加到列表中

                if not pic_data_list:
                    # print("No data found for the given account_id")
                    return False
                else:
                    return pic_data_list  # 返回列表
            except Exception as e:
                print(f"An error occurred: {e}")
                return False


    def get_pic_urls(self, account_id):
        pic_data_list = []  # 创建一个空列表来存储结果
        with self.db_engine192.connect() as connection:
            table_name = "stock_image_detail_wj"
            query = text(
                f"""select image_url, image_id, image_title from {table_name} where account_id = :account_id and state = 1""")

            try:
                result = connection.execute(query, {'account_id': account_id})
                for row in result:  # 遍历所有的结果行
                    if row is not None:
                        # 将 RowProxy 转换为标准的字典
                        row_dict = dict(zip(result.keys(), row))  # 使用 keys 和 fetchone 的结果创建字典

                        # 直接构建所需格式的字符串
                        pic_datas = f"{row_dict['image_url']}||{row_dict['image_id']}||{row_dict['image_title']}"
                        pic_data_list.append(pic_datas)  # 添加到列表中

                if not pic_data_list:
                    # print("No data found for the given account_id")
                    return False
                else:
                    return pic_data_list  # 返回列表
            except Exception as e:
                print(f"An error occurred: {e}")
                return False


    def get_stock_images_id2(self, account_id):
        with self.db_engine192.connect() as connection:
            table_name = "stock_image_id_wj"
            # 特定的 image_id 列表
            specific_image_ids = [
'1025406430',
'782084149',
'2340663257',
'2444918601',
'2481076155',
'2534369399',
'2522128969',
'2522144147',
'2482077119',
'2475085855',
'2560247125',
'1115348984',
'2555951185',
'1644852415',
'1644852424',
'258700904',
'2540342353',
'2555951245',
'2529955899',
'1309059847',
'1899316957',
'2416180707',
'1978653428',
'2520112131',
'1447499252',
'2335787565',
'1780440524',
'2316295613',
'2463106909',
'2527382733',
'2548693637',
'2460743889',
'2489123001',
'2527399543',
'2456315025',
'2469939069',
'2305915213',
'1660111006',
'2218802639',
'453729808',
'2295540279',
'2323950095',
'2323950087',
'2057817146',
'2541104423',
'231076948',
'2196541827',
'2407612765',
'2521017693',
'2554778219',
'2523427909',
'2520799267',
'2533854931',
'2498052331',
'2521798533',
'2471652945',
'2445858817',
'2449783031',
'1735869230',
'1106587370',
'2393397957',
'2527382699',
'2348771553',
'1822384931',
'2564084221'
]
            # 修改查询语句以匹配你的数据表名称、列名称，并加入 image_id 条件
            query = text(f"""SELECT image_id,id,image_title,image_size_info FROM {table_name} 
                             WHERE account_id = :account_id 
                             AND image_id IN :image_ids""")
            print(query)
            result = connection.execute(query, {'account_id': account_id, 'image_ids': tuple(specific_image_ids)})
            try:
                df_status = pd.DataFrame(result.fetchall())
                df_status.columns = result.keys()

                df_status['id'] = df_status['id'].astype(str)
                image_id_id_pairs = list(
                    df_status['image_id'].astype(str) + '||-||' + df_status['id'] + '||-||' + df_status[
                        'image_title'] + '||-||' + df_status['image_size_info'])
                print(f'账号：{account_id}需爬取{len(image_id_id_pairs)}张')
                return image_id_id_pairs
            except Exception as e:
                print(e)
                return False

    def img_size_is_1(self, account_id, image_ids):
        # 确保image_ids列表非空
        if not image_ids:
            print("No image IDs provided.")
            return False

        with self.db_engine192.connect() as connection:
            table_name = "stock_image_id_wj"
            query = text(f"""SELECT image_id, id, image_title, image_size_info  FROM {table_name} WHERE account_id = :account_id AND image_id IN :image_ids""")
            try:
                result = connection.execute(query, {'account_id': account_id, 'image_ids': tuple(image_ids)})
                df_status = pd.DataFrame(result.fetchall())
                df_status.columns = result.keys()
                df_status['id'] = df_status['id'].astype(str)
                image_id_id_pairs = list(
                    df_status['image_id'] + '||-||' + df_status['id'] + '||-||' + df_status['image_title'] + '||-||' +
                    df_status['image_size_info'])
                print(f'账号：{account_id}需爬取{len(image_id_id_pairs)}张')
                return image_id_id_pairs
            except Exception as e:
                print(e)
                return False

    def get_stock_images_id(self,account_id):
        with self.db_engine192.connect() as connection:
            table_name = "stock_image_id_wj"
            # 修改查询语句以匹配你的数据表名称和列名称
            query = text(f""" SELECT image_id,id,image_title,image_size_info FROM {table_name} where account_id = :account_id and state = 1""")

            print(query)
            result = connection.execute(query, {'account_id': account_id})
            try:
                df_status = pd.DataFrame(result.fetchall())
                df_status.columns = result.keys()


                df_status['id'] = df_status['id'].astype(str)
                image_id_id_pairs = list(df_status['image_id'] + '||-||' + df_status['id'] + '||-||' + df_status['image_title'] + '||-||' + df_status['image_size_info'])
                print(f'账号：{account_id}需爬取{len(image_id_id_pairs)}张')
                connection.close()
                return image_id_id_pairs
            except Exception as e:
                print(e)
                return False

    def get_kong_images_id(self, account_id):
        with self.db_engine192.connect() as connection:
            table_name = "stock_image_detail_wj"
            # 使用子查询来过滤掉已经有 image_size_info 的 image_id
            query = text(
                f"""SELECT image_id, image_type, image_url 
                    FROM {table_name} 
                    WHERE account_id = :account_id 
                    AND image_id NOT IN (
                        SELECT image_id 
                        FROM {table_name} 
                        WHERE account_id = :account_id 
                        AND image_size_info != '{{}}'
                    )
                    AND image_size_info = '{{}}'"""
            )
            print(query)
            result = connection.execute(query, {'account_id': account_id})
            df_status = pd.DataFrame(result.fetchall(), columns=result.keys())
            if df_status.empty:
                return []
            data_list = list(df_status['image_id'] + '||' + df_status['image_type'] + '||' + df_status['image_url'])
            connection.close()
            return data_list

    def get_stock_image_detail(self, account_id):
        with self.mysql_selection.connect() as connection:
            table_name = "stock_image_detail"
            query = text(
                f"""SELECT account_id, image_id, image_size_info, image_title, image_type, image_url, created_time FROM {table_name} WHERE account_id = :account_id""")
            print(query)
            result = connection.execute(query, {'account_id': account_id})
            df_status = pd.DataFrame(result.fetchall())
            df_status.columns = result.keys()

            # 将 Timestamp 转换为字符串格式
            df_status['created_time'] = df_status['created_time'].dt.strftime('%Y-%m-%d %H:%M:%S')

            # 拼接字符串
            detail_datas = list(
                df_status['account_id'] + '||-||' +
                df_status['image_id'] + '||-||' +
                df_status['image_size_info'] + '||-||' +
                df_status['image_title'] + '||-||' +
                df_status['image_type'] + '||-||' +
                df_status['image_url'] + '||-||' +
                df_status['created_time']
            )
            print(f'账号：{account_id} 一共 {len(detail_datas)} 条数据')
            return detail_datas

    def save_stock_detail_move(self, data_list):
        table_name = "stock_image_detail_wj"

        # 定义DataFrame的列
        columns = ['account_id', 'image_id', 'image_size_info', 'image_title', 'image_type', 'image_url',
                   'created_time']
        df = pd.DataFrame(data_list, columns=columns)

        with self.db_engine192.connect() as connection:
            df.to_sql(
                name=table_name,
                con=connection,
                if_exists='append',
                index=False
            )
        print("保存成功！")

    # 11111111111
    def update_image_id_to_3(self, item_id):
        with self.db_engine192.connect() as connection:
            table_name = "stock_image_id_wj"
            trans = connection.begin()
            sql_update = text(f"UPDATE {table_name} SET state = 3 WHERE id = :item_id")
            result = connection.execute(sql_update, {"item_id": item_id})
            trans.commit()

    def update_url_state_to_3(self,image_id):
        with self.db_engine192.connect() as connection:
            table_name = "stock_image_detail_wj"
            trans = connection.begin()
            sql_update = text(f"UPDATE {table_name} SET state = 3 WHERE image_id = :image_id and state = 1")
            result = connection.execute(sql_update, {"image_id": image_id})
            trans.commit()

    # 11111111111
    def update_image_id_to_4(self, item_id):
        with self.db_engine192.connect() as connection:
            table_name = "stock_image_id_wj"
            trans = connection.begin()
            sql_update = text(f"UPDATE {table_name} SET state = 4 WHERE id = :item_id")
            result = connection.execute(sql_update, {"item_id": item_id})
            connection.close()
            trans.commit()

    def save_stock_cookie(self,item):
        table_name = "stock_cookie_wj"
        # 将item包装成列表
        items_list = [item]
        # 定义DataFrame的列
        columns = ['account_id', 'cookie', 'state', 'created_at']
        df = pd.DataFrame(items_list, columns=columns)

        with self.db_engine192.connect() as connection:
            df.to_sql(
                name=table_name,
                con=connection,
                if_exists='append',
                index=False
            )
        print("保存成功！")

    def updata_ck_state(self,ck_id):
        with self.db_engine192.connect() as connection:
            table_name = "stock_cookie_wj"
            # 使用参数化查询防止SQL注入
            query = text(
                f"""UPDATE {table_name} SET state = :new_state WHERE id = :ck_id;""")

            try:
                # 执行更新语句
                result = connection.execute(query, {'new_state': 3, 'ck_id': ck_id})
                # 提交事务以确保更改被保存到数据库中
                connection.commit()

                # 检查是否有行受到影响
                if result.rowcount > 0:
                    print('修改cookie状态为3')
                    return True  # 更新成功

                else:
                    return False  # 没有找到匹配项或没有更新任何行
            except Exception as e:
                # print(f"An error occurred: {e}")
                return False


    def get_stock_cookie(self, account):
        with self.db_engine192.connect() as connection:
            table_name = "stock_cookie_wj"
            # 使用参数化查询
            query = text(
                f"""SELECT id, cookie, state FROM {table_name} WHERE account_id = :account_id AND state = :state LIMIT 1;""")
            result = connection.execute(query, {'account_id': account, 'state': 1}).mappings().first()

            if result is not None:
                # 通过键名访问字典中的元素
                cookie_id_state = f"{result['id']}||-||{result['cookie']}||-||{result['state']}"
                return cookie_id_state
            else:
                return None  # 没有找到匹配项时返回None

    def get_stock_cookie_list(self, account):
        with self.db_engine192.connect() as connection:
            table_name = "stock_cookie_wj"
            # 使用参数化查询
            query = text(f"""SELECT cookie FROM {table_name} WHERE account_id = :account_id AND state = :state ; """)
            result = connection.execute(query, {'account_id': account, 'state': 1})
            df_status = pd.DataFrame(result.fetchall())
            df_status.columns = result.keys()
            cookie_list = df_status['cookie'].tolist() if 'cookie' in df_status.columns else []
            return cookie_list

    def get_cookie_account(self,item_id):
        # try:
            with self.db_engine192.connect() as connection:
                table_name = "stock_image_summary_wj"
                # 修改查询语句以匹配你的数据表名称和列名称
                query = text(f"""SELECT account_id,account_secret FROM {table_name} where id = :item_id ;""")
                result = connection.execute(query, {'item_id': item_id})
                # print(query)
                df_status = pd.DataFrame(result.fetchall())
                df_status.columns = result.keys()
                account_id = df_status.account_id.iloc[0]
                account_secret = df_status.account_secret.iloc[0]
                account_list = [account_id, account_secret]
                # print(account_list)
                # print(111111111111)
                connection.close()
                return account_list
        # except Exception as e:
        #     print(111111111)

    def get_stock_test_id(self,username):
        with self.db_engine192.connect() as connection:
            table_name = "stock_image_id_wj"
            # 修改查询语句以匹配你的数据表名称和列名称
            query = text(f"""SELECT image_id FROM {table_name} WHERE account_id = :username LIMIT 1;""")
            result = connection.execute(query, {'username': username})
            df_status = pd.DataFrame(result.fetchall())
            df_status.columns = result.keys()
            image_id = df_status.image_id.iloc[0]
            connection.close()
            return image_id

    def upload_data(self, account_id, image_id, upload_time, err_msg):
        with self.db_engine192.connect() as connection:
            table_name = "stock_image_detail_wj"
            # 强制将 image_id 转换为字符串类型
            image_id = str(image_id)
            err_msg_json = json.dumps(err_msg)
            sql_update = text(
                f"UPDATE {table_name} SET upload_time = :upload_time, err_msg = :err_msg WHERE account_id = :account_id AND image_id = :image_id AND created_time  < '2024-09-02 00:00:00'")
            result = connection.execute(sql_update, {
                "upload_time": upload_time,
                "err_msg": err_msg_json,
                "account_id": account_id,
                "image_id": image_id
            })

    def upload_success_data(self, account_id, image_id, upload_time):
        with self.db_engine192.connect() as connection:
            table_name = "stock_image_detail_wj"
            # 强制将 image_id 转换为字符串类型
            image_id = str(image_id)
            sql_update = text(
                f"UPDATE {table_name} SET upload_time = :upload_time WHERE account_id = :account_id AND image_id = :image_id AND created_time  < '2024-09-02 00:00:00'")
            result = connection.execute(sql_update, {
                "upload_time": upload_time,
                "account_id": account_id,
                "image_id": image_id
            })



    def get_all_image_id(self):
        with self.db_engine192.connect() as connection:
            table_name = "stock_image_detail_wj"
            sql_query = f"SELECT image_id FROM {table_name} "
            df_status = pd.read_sql(sql_query, con=connection)
            image_id = list(df_status['image_id'].astype(str))

            connection.close()
            return image_id




