# coding:utf-8
import os
import sys
import json
import time
import redis
import random
import scrapy
import logging
from urllib.parse import urlparse
sys.path.append(os.path.dirname(os.path.dirname(sys.path[0])))  # 上级目录
from scrapy import cmdline, signals
from amazon_spider.db.redis_db import spop, sadd
from func_timeout.exceptions import FunctionTimedOut
# mx站点  详情页产品描述获取
from amazon_spider.spiders.yswg_spider import SourceSpider
from amazon_spider.extractor.amazon_detail_extractor import AmazonDetailExtractor


class AmazonUpcAsinSpider(SourceSpider):
    name = 'upc_spider'
    custom_settings = {
        'CONCURRENT_REQUESTS': 30,
        'DOWNLOAD_TIMEOUT': 10,
        'allowed_domains': ['amazon.com'],
        # # 设置重启爬虫时是否清空爬取队列
        # 'SCHEDULER_FLUSH_ON_START': False,
        # # 启用Redis调度存储请求队列
        # 'SCHEDULER': "amazon_spider.scrapy_redis.scheduler.Scheduler",
        # # 确保所有的爬虫通过Redis去重
        # 'DUPEFILTER_CLASS': "amazon_spider.scrapy_redis.dupefilter.RFPDupeFilter",
        # 'SCHEDULER_QUEUE_CLASS': 'amazon_spider.scrapy_redis.queue.SpiderPriorityQueue',
        # # 种子队列的信息
        # 'REDIS_URL': None,
        # 'REDIS_HOST': '192.168.10.224',
        # 'REDIS_PORT': 6379,
        # 'REDIS_PARAMS': {
        #     'password': 'HCL1zcUgQesaaXNLbL37O5KhpSAy0c',
        #     'db': 0
        # },
        # # # 6379
        # # 'FILTER_URL': None,
        # # 'FILTER_HOST': '127.0.0.1',
        # # 'FILTER_PORT': 6379,
        # # # 6379
        # # 'FILTER_DB': 0,
        # 'SCHEDULER_QUEUE_KEY': "detail_seed",
        # Retry settings
        'RETRY_ENABLED': True,
        'RETRY_TIMES': 3,  # 想重试几次就写几
        # 'COOKIES_ENABLED': True,
        # 'COOKIES_DEBUG': False,
        'RETRY_HTTP_CODES': [203, 301, 403, 408, 429, 500, 502, 503, 504, 522, 524, 404],
        # downloader middlewares for spider.
        'DOWNLOADER_MIDDLEWARES': {
            'scrapy.downloadermiddlewares.retry.RetryMiddleware': None,
            # 递增调用
            'amazon_spider.middlewares.RandomUserAgentMiddleware': 460,
            # 'amazon_spider.middlewares.ProxyMiddleware': 450,
            'amazon_spider.middlewares.CookiesZip': 480,
            # 'amazon_spider.middlewares.GetCookieMiddleware': 460,
            'amazon_spider.middleware.http2.HttpxMiddleware': 490,
            'amazon_spider.middleware.aiohttp.AiohttpMiddleware': 490,
            'amazon_spider.middleware.temu.CurlCffiRequests': 490,
            # 递减调用
            # 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500
        },
        'ITEM_PIPELINES': {
            'amazon_spider.pipeline.amazon_upc_pip.AmazonUpcSpiderPipeline': 230,
        },
        'DOWNLOAD_HANDLERS': {
            'http': 'amazon_spider.downloadhandlers.ja3.MyHTTPDownloadHandler',
            'https': 'amazon_spider.downloadhandlers.ja3.MyHTTPDownloadHandler',
        }
    }

    def __init__(self, site='us'):
        super(AmazonUpcAsinSpider, self).__init__()
        self.site = site
        self.update_cookies()
        self.sleep_count = 0
        logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',
                            level=logging.INFO)

        self.col = ['asin', 'page_state', 'upc']

    @classmethod
    def from_crawler(cls, crawler, *args, **kwargs):
        spider = super(AmazonUpcAsinSpider, cls).from_crawler(crawler, *args, **kwargs)
        crawler.signals.connect(spider.close, signals.spider_closed)
        crawler.signals.connect(spider.spider_idle, signals.spider_idle)
        return spider

    def get_seeds(self):
        while True:
            try:
                seeds = spop(f'{self.site}_upc_video', 25)
                logging.info("get seeds ok o-_-o")
                break
            except redis.exceptions.ConnectionError as e:
                logging.info(f"get seeds error T_T --> {e}")
                time.sleep(5)
                continue
            except FunctionTimedOut as e:
                logging.info(f"get seeds timeout T_T -->{e}")
                continue
        return seeds

    def spider_idle(self, spider):
        logging.debug(f'IDLE------------------{self.site}')
        str_time = time.strftime("%H:%M:%S", time.localtime())
        # 更新cookies
        if str_time >= "09:09:00" and str_time <= "09:10:00":
            self.update_cookies()
        seeds = self.get_seeds()
        if seeds:
            for i in seeds:
                job = json.loads(i)
        # try:
        #     df = ReadDb("us").read_db_video_asin(self.site)
        # except OperationalError as e:
        #     logging.info(f'get seeds failure in link sleep 30s{e}')
        #     time.sleep(30)
        #     raise DontCloseSpider
        # except FunctionTimedOut as e:
        #     logging.info(f'get seeds time out sleep 30s{e}')
        #     time.sleep(30)
        #     raise DontCloseSpider

        # if df.shape[0] > 0:
        #     for i in df.values:
                url = f"{self.site_url.get(job['site'])}/dp/{job['asin']}"
                headers = {
                    'Connection': 'close',
                    'authority': urlparse(url).hostname,
                    'accept': 'text/html,*/*',
                    'accept-language':  'zh-CN,zh;q=0.9',
                    'cache-control': 'no-cache',
                    'content-type': 'application/x-www-form-urlencoded;charset=UTF-8',
                    'origin': url,
                    'referer': f'{url}/Bosch-ROS20VSK-Palm-Sander-Collector/product-reviews/B0018Z8D64/ref=cm_cr_arp_d_paging_btm_next_3?ie=UTF8&reviewerType=all_reviews&pageNumber=3',
                }

                meta = {
                    # "use_aiohttp": True,
                    "asin": job['asin'],
                    "source_asin": job['source_asin'],
                    "site": job['site'],
                    "sku": "",
                    "bsr_url": job.get("bsr_url", ""),

                    "choice_header": True,
                    "date_info": "",
                    # "cookie_id": cookies[0],
                    # "amazon_proxy": True,
                    "cookiejar": int(time.time()),
                }
                meta = self.random_r(meta)
                h_key = random.choice(list(self.h_dict))
                headers[h_key] = self.h_dict.get(h_key)
                logging.info(f'随机添加headers：{h_key}')
                headers['X-Forwarded-For'] = '1.1.1.1,2.2.2.2'
                cookies = self.json_cookies(meta.get('site'))
                print(cookies)
                cookies.update({
                    'csm-sid': '916-1904410-6680838',
                    'x-amz-captcha-1': '1706092664884599',
                    'x-amz-captcha-2': '5OU/Q7DyDmtg9QGCbivNXg=='
                })
                self.crawler.engine.crawl(scrapy.Request(url=url, cookies=cookies, headers=headers,
                                     callback=self.parse, errback=self.err_parse,
                                     dont_filter=True, meta=meta), self)
        else:
            # meta = {
            #     'handle_httpstatus_all': True
            # }
            # request = scrapy.Request(url="https://www.baidu.com/", callback=self.parse_news_page, errback=self.err_, dont_filter=True, meta=meta)
            # self.crawler.engine.crawl(request, spider=self)
            # logging.info('no task sleep 30s')
            # time.sleep(30)
            # raise DontCloseSpider()
            raise

    def page_state(self, response):
        if ("Die eingegebene Webadresse ist keine" in response.text) \
                    or ("The Web address you entered is not a functioning" in response.text) \
                    or ("saisie n'est pas une page fonctionnelle" in response.text) \
                    or ("web inserito non è una pagina funzionante" in response.text) \
                    or ("web que has especificado no es" in response.text) \
                    or ("Page Not Found" in response.text) \
                    or ("We are sorry! This Gift Card is not available" in response.text) \
                    or ("500 - An error occurred" in response.text):
            item = {
                "error_asin": True,
                "asin": (4, response.request.meta.get("asin"), response.request.meta.get("site")),
            }
            logging.info(f"页面为空:{item.get('asin')}")
            return {
                "asin": response.meta.get("asin"),
                "source_asin": response.meta.get('source_asin'),
                "bsr_url": response.meta.get('bsr_url'),

                "site": response.request.meta.get("site"),
                    "sku": response.meta.get("sku"), "page_state": "狗头页面", "video": "否", "add_url": "否"}
        elif ("keywords" in response.url) or ("dp/" not in response.url) or (
                "ref=" in response.url and "encoding=" in response.url) or (
                response.meta.get("asin") not in response.url):
            item = {
                "error_asin": True,
                "asin": (12, response.request.meta.get("asin"), response.request.meta.get("site")),
            }
            logging.info(f"跳转页面:{item.get('asin')}")
            return {
                "asin": response.meta.get("asin"),
                "source_asin": response.meta.get('source_asin'),
                "bsr_url": response.meta.get('bsr_url'),

                "site": response.request.meta.get("site"),
                   "sku": response.meta.get("sku"), "page_state": "跳转页面", "video": "否", "add_url": "否"}
        elif not response.xpath("//div[@id='dp']").getall():
            item = {
                "error_asin": True,
                "asin": (13, response.request.meta.get("asin"), response.request.meta.get("site")),
            }
            logging.info(f"跳转或者视频页面:{item.get('asin')}")
            return {
                "asin": response.meta.get("asin"),
                "source_asin": response.meta.get('source_asin'),
                "bsr_url": response.meta.get('bsr_url'),

                "site": response.request.meta.get("site"),
                   "sku": response.meta.get("sku"), "page_state": "跳转页面", "video": "否", "add_url": "否"}
        else:
            return None

    def parse(self, response, **kwargs):
        if item := self.page_state(response):
            print(item)
            yield item
        else:
            amazon_detail_extractor = AmazonDetailExtractor(self.site)
            items = amazon_detail_extractor.run(response)


            item = {
                "asin": response.meta.get("asin"),
                "source_asin": response.meta.get('source_asin'),
                "bsr_url": response.meta.get('bsr_url'),

                "site": response.request.meta.get("site"), "sku": response.meta.get("sku"), "page_state": "页面有数据", "video": "是" if items.get("video_url") else "否", "add_url": "是" if items.get("add_url") else "否"}
            print(item)
            yield item
            # upc = response.xpath('//div[@id="detailBulletsWrapper_feature_div"]//*[contains(text(), "UPC")]/parent::*/span[not(@class)]/text()').getall()
            # if upc:
            #     upc = upc[0]
            # else:
            #     upc = None
            # print(upc)
            # yield {"asin": response.meta.get("asin"), "page_state": "页面有数据", "upc": upc}


    def err_parse(self, failure, **kwargs):
        # item = {
        #     "error_asin": True,
        #     "asin": (1, failure.request.meta.get("asin"), failure.request.meta.get("site"))
        # }
        # logging.info(f"爬取失败asin:{item.get('asin')}")
        # yield item
        logging.info(f"error______ {failure.getErrorMessage()}, {failure.request.meta.get('asin')}")

        # 处理失败请求并将其重新发送到 Redis
        # 获取原始请求对象
        request = failure.request
        meta = {
            "asin": request.meta.get("asin"),
            "site": request.meta.get("site"),
            "bsr_url": request.meta.get("bsr_url"),
            "source_asin": request.meta.get("source_asin"),
        }
        k = json.dumps(meta)
        while True:
            try:
                sadd(f"{self.site}_upc_video", k, use_md5=False)
                logging.info(f"push {self.site}_upc_video succeed")
                break
            except redis.exceptions.ConnectionError as e:
                logging.info(f"push {self.site}_upc_video ConnectionError，afresh push --> T_T {e}")
                time.sleep(5)
                continue
            except FunctionTimedOut as e:
                logging.info(f"push {self.site}_upc_video time out --> T_T {e}")
                continue


if __name__ == '__main__':
    args = 'scrapy crawl upc_spider -a site=us'.split()
    cmdline.execute(args)


# nohup cd /mnt/hezhe/amazon_spider/amazon_spider scrapy crawl mx_self_asin -a site=mx  > mx_amazon1.log 2>&1 &
# nohup scrapy crawl upc_spider -a site=us  > upc_amazon.log 2>&1 &
# source activate pyspark
# for i in `ps -ef|grep "scrapy crawl upc_spider" |awk '{print $2}' `; do kill -9 $i ; done;

# C:\Users\Administrator\AppData\Local\Programs\Python\Python38\scrapy crawl mx_self_asin

# 0 0 * * * cd /mnt/hezhe/amazon_spider/amazon_spider && /opt/module/anaconda3/envs/pyspark/bin/scrapy crawl mx_self_asin -a site=mx > mx_asin1.log 2>&1 &

# cd /mnt/hezhe/amazon_spider/amazon_spider && /opt/module/anaconda3/envs/pyspark/bin/scrapy crawl mx_self_asin -a site=mx  > mx_asin1.log 2>&1 &
