Tibia-Fandom-Web-Scrapper/CreatureScrapper.py at main · alehee/Tibia-Fandom-Web-Scrapper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
from bs4 import BeautifulSoup
from urllib.request import urlopen
import re
from Models.Creature import Creature

class CreatureScrapper:
    def __init__(self):
        self.creature = Creature()

    def scrap(self, url):
        page = urlopen(url)
        html = page.read().decode("utf-8")
        soup = BeautifulSoup(html, "html.parser")
        self.scrapName(soup)
        self.scrapExperience(soup)
        self.scrapHealth(soup)
        self.scrapCreatureClass(soup)
        self.scrapCreatureClassification(soup)
        self.scrapCharmPoints(soup)
        self.scrapResistances(soup)
        self.scrapVersion(soup)
        return self.creature

    def scrapCreaturePropertyTable(self, soup, propertyName):
        parent = soup.find(string=propertyName).parent.parent
        value = parent.find('div').get_text().replace(',','')
        return value

    def scrapName(self, soup):
        parent = soup.find('span', 'mw-page-title-main')
        value = parent.get_text()
        print('Name: ' + value)
        self.creature.addName(value)

    def scrapExperience(self, soup):
        value = int(self.scrapCreaturePropertyTable(soup, 'Experience'))
        print('Experience: ' + str(value))
        self.creature.addExperience(value)

    def scrapHealth(self, soup):
        value = int(self.scrapCreaturePropertyTable(soup, 'Health'))
        print('Health: ' + str(value))
        self.creature.addHealth(value)

    def scrapCreatureClass(self, soup):
        value = self.scrapCreaturePropertyTable(soup, 'Class')
        print('Class: ' + value)
        self.creature.addCreatureClass(value)

    def scrapCreatureClassification(self, soup):
        value = self.scrapCreaturePropertyTable(soup, 'Classification')
        print('Classification: ' + value)
        self.creature.addCreatureClassification(value)

    def scrapCharmPoints(self, soup):
        value = int(self.scrapCreaturePropertyTable(soup, 'Charm Points'))
        print('Charm Points: ' + str(value))
        self.creature.addCharmPoints(value)

    def scrapResistances(self, soup):
        parent = soup.find('span', id='Damage_Taken_From_Elements').parent.parent

        resistances = {}
        rows = parent.get_text().split('\n')
        pattern = r'^(\w+)\s+(\d+)%\??$'
        for row in rows:
            match = re.match(pattern, row)
            if match:
                element, percent = match.groups()
                percent = -int(percent) + 100
                if percent != 0:
                    resistances[element] = percent

        if resistances == {}:
            resistances = None

        print('Resistences: ' + str(resistances))
        self.creature.addResistances(resistances)

    def scrapVersion(self, soup):
        value = self.scrapCreaturePropertyTable(soup, 'Version')

        strFlag = 0
        for i in range(0, len(value)):
            c = value[i]
            if c != '.' and not c.isdigit():
                strFlag = i
                break
        if strFlag > 0:
            value = value[0:strFlag]

        print('Version: ' + value)
        self.creature.addVersion(value)