NI-HCM/src/scrapers/skrytapravda.py at master · Hackrrr/NI-HCM · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
from . import *


class SkrytaPravda(Scraper):
    def get_article(self, url: str) -> Article:
        r = self.s.get(url)
        soup = BeautifulSoup(r.text, "html.parser")
        title = soup.find("h1", itemprop="headline name").string

        time = datetime.fromisoformat(
            soup.find("time", itemprop="datePublished")["datetime"]
        )

        # https://www.skrytapravda.cz/z-domova/1120-neni-jine-cesty-tato-vlada-ztratila-mandat-jako-bolsevicka-v-roce-1989
        categories = [url.split("skrytapravda.cz/", 1)[1].split("/", 1)[0]]
        tags = [x.string for x in soup.find(class_="panel-title").find_all("a")]

        intro_text = soup.find("div", class_="introtext").get_text()
        main_content = soup.find("div", class_="fulltext")
        text = intro_text + "\n" + generic_text_extract([main_content])

        return Article(
            self.url_to_id(url),
            title,
            text,
            time,
            categories,
            tags,
            url,
        )

    def get_urls_from_time_range(self, time_range: TimeRange) -> list[str]:
        sitemap = self._scrape_sitemap(
            "https://www.skrytapravda.cz/index.php?option=com_jmap&view=sitemap&format=xml",
            time_range,
        )

        urls = []
        for x in sitemap.get_urls_in_timerange(time_range):
            # https://www.skrytapravda.cz/doporucene-clanky/955-dekujeme-udatny-americane-ze-jsi-osvobodil-evropu-a-ceskoslovensko-od-nacismu
            if (
                not x.split("skrytapravda.cz/", 1)[1]
                .split("/", 1)[1]
                .split("-", 1)[0]
                .isnumeric()
            ):
                continue
            urls.append(x)

        return urls


# print(SkrytaPravda().get_article("https://www.skrytapravda.cz/z-domova/1120-neni-jine-cesty-tato-vlada-ztratila-mandat-jako-bolsevicka-v-roce-1989").text)
# print(SkrytaPravda().get_article("https://www.skrytapravda.cz/ze-zahranici/2942-mapa-ukrajinskych-utoku-dronu-na-putinovu-prezidentskou-rezidenci-ktere-udajne-neexistuji").text)