PyWebscrapers/scraper2.py at main · M-Atkins/PyWebscrapers · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
import time
import random

def readBackwards(link, direction):
    #reverse and split
    exp = [*link][::-1]
    for i, s in enumerate(exp):
        if s == "/":
            # print("index ", i)
            if direction == "reverse":
                exp = link[:-i]
            if direction == "forward":
                exp = link[-i:]
            break
    return exp

def splitBetween(text, tag1, tag2):
    indexes = []

    # print(text)
    li = [*text]
    for i, s in enumerate(li):
        if(li[i] == tag1 or li[i] == tag2):
            indexes.append(i)
            # print(i)

    inb = ''
    for i in range(indexes[0]+1, indexes[1]):
        inb += li[i]
    # print(inb)

    return inb

def main():
    driver = webdriver.Firefox()
    driver.get("")
    try:
        elem = driver.find_elements(By.CLASS_NAME, "video-card")

        for i, f in enumerate(elem):

            id = f.find_element(By.TAG_NAME,"a").get_attribute("href")
            id = readBackwards(id, "forward")
            print(id)
            f = f.get_attribute("onclick")
            f = splitBetween(f,"(",")")
            f = readBackwards(f, "reverse")

            if "previews/" in f:
                f = f.replace("previews/", "videos/")

            link = str(f) + str(id) + ".mp4"
            with open("links.txt", "a") as a_file:
                a_file.write(link)
                a_file.write("\n")

        print(e)


    driver.close()
    return

main()