from json import loads
from os import path, remove
from tempfile import gettempdir

from requests import get
from scrapy import Spider, Request
from scrapy.crawler import CrawlerProcess
from scrapy.pipelines.files import FilesPipeline
from twisted.internet import reactor
from twisted.internet.task import deferLater

from ElasticSearch import ElasticSearch
from MinIO import MinIO
from secrets import MINIO_ACCESS_KEY, MINIO_SECRET_KEY
from variables import ES_HOST
from variables import MINIO_SERVER

MAX_TASKS = 1
SLEEP_TIME = 0.5 * 60


class LocalFilesPipeline(FilesPipeline):
    pass


class CdnSpider(Spider):
    name = "cdn_spider"

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.base = "урл"
        self.es = ElasticSearch(hosts=[ES_HOST])
        self.temp_dir = gettempdir()
        self.aws = MinIO(
            endpoint=MINIO_SERVER,
            access_key=MINIO_ACCESS_KEY,
            secret_key=MINIO_SECRET_KEY,
        )
        self.urls = [
            obj["url"]
            for obj in self.es.search(
                index="videos",
                body={
                    "query": {"match": {"video_url": "NULL"}},
                    "size": f"{MAX_TASKS}",
                },
            )
        ]

    def start_requests(self):
        for url in self.urls:
            yield Request(
                url=url,
                callback=self.parse,
                method="GET",
                headers={
                    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                    "AppleWebKit/537.36 (KHTML, like Gecko) "
                    "Chrome/87.0.4280.88 Safari/537.36",
                    "content-type": "application/x-www-form-urlencoded",
                },
            )

    def parse(self, response, **kwargs):

        # страница не найдена
        not_found = response.css(
            "div[id=wrapper_div] > "
            "div[id=content_div] > "
            "div.main_content > "
            "span.page_message::text "
        ).get()

        if not_found == "Post Not Found":
            self.es.delete_by_query(
                index="videos",
                body={"query": {"match": {"url": f"{response.url}"}}},
            )
            return

        # видео удалено
        content_removed = response.css(
            "div[id=wrapper_div] > "
            "div[id=content_div] > "
            "div.main_content > "
            "div.post_el_small.post_el_post > "
            "div.post_el_wrap > "
            "div:nth-child(4) > span::text"
        ).get()

        if content_removed == "Sorry! Content removed.":
            self.es.delete_by_query(
                index="videos",
                body={"query": {"match": {"url": f"{response.url}"}}},
            )
            return

        data_vnfo = (
            response.css("div[id=wrapper_div]")
            .css("div[id=content_div]")
            .css("div.main_content > span.vidsnfo::attr(data-vnfo)")
        )
        video_url = list(loads(data_vnfo.get()).values())[0]
        video_url = f"{self.base}{decrypting(video_url)}"
        url = f"{response.url}"

        temp = download_file(url=video_url, directory=self.temp_dir)

        if self.aws.upload_file(
            filename=temp,
            bucket_name="raw_videos",
            filename_in_bucket=path.basename(temp),
        ):
            raw_filepath = path.join("raw_videos", path.basename(temp)).replace(
                "\\", "/"
            )
            if self.es.update_by_query(
                index="videos",
                body={
                    "query": {"term": {"url": url}},
                    "script": {
                        "source": "ctx._source.video_url = params.video_url ; ctx._source.raw_filepath = params.raw_filepath",
                        "lang": "painless",
                        "params": {
                            "video_url": f"{video_url}",
                            "raw_filepath": f"{raw_filepath}",
                        },
                    },
                },
            ):
                remove(temp)


def download_file(url: str, directory: str):
    local_filename = url.split("/")[-1]
    filename = path.join(directory, local_filename)
    # NOTE the stream=True parameter below
    with get(url, stream=True) as r:
        r.raise_for_status()
        with open(file=filename, mode="wb") as f:
            for chunk in r.iter_content(chunk_size=1024 * 1024):
                f.write(chunk)
    return filename


def decrypting(link: str):
    """
    :param link: урл вида '/cdn/c10/jy3p1xzk1u3e0xzs9l5yz71e5a3/06DNXKxRpMEDmZ1lqK1Iew/1610390527
                           /u5tb3dh8k0pb8ek68d21h4g0pdt/a5ffzfeb84yfle48iambcaaf81g.vid'
    :return: /cdn8/c10/lt3j1uz21t3q09zr9f5ez41a5g3/hIMNrXNQPRe5S1L1bW7ciw/1610390562/
             js5ebad38l09bree66dx1r4f0fd/p65ifkf5bx4gfbeh87a0biaofx1.vid
    """
    buffer = link.replace("/cdn/", "/cdn8/").split("/")
    bbuffer = buffer
    c = int(buffer[5])
    a = stripdown(buffer[6])
    b = stripdown(buffer[7])
    a = quersumm(a)
    b = quersumm(b)
    c = c - (a + b)
    bbuffer[5] = str(c)
    return "/".join([str(i) for i in bbuffer])


def stripdown(string_: str) -> str:
    """
    :param string_: входная строка
    :return: только цифры
    """
    return "".join(c for c in string_ if c.isdigit())


def quersumm(string_: str) -> int:
    """
    :param string_: входная строка
    :return:
    """
    return sum([int(i) for i in string_])


def _sleep(self, *args, seconds):
    """Non blocking sleep callback"""
    return deferLater(reactor, seconds, lambda: None)


process = CrawlerProcess(
    settings={
        "FEEDS": {
            "items.json": {"format": "json"},
        },
        "DOWNLOAD_DELAY": 5,
        "CONCURRENT_REQUESTS": 16,
        "AUTOTHROTTLE_ENABLED": True,
        "AUTOTHROTTLE_START_DELAY": 5,
        "ITEM_PIPELINES": {LocalFilesPipeline: 200},
    }
)
process.crawl(CdnSpider, name="cdn_parser")
process.start()