StudyBuddies/updateCourses.py at master · JVenberg/StudyBuddies · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
import requests
from bs4 import BeautifulSoup
import firebase_admin
from firebase_admin import credentials
from firebase_admin import db

# Use the application default credentials

cred = credentials.Certificate("/Users/jackv/serviceAccountKey.json")
firebase_admin.initialize_app(cred, {
    'databaseURL': 'https://studybuddy-19f01.firebaseio.com/'
})
# db = firestore.client()
ref = db.reference('curriculum_search')

courses = {}


payload = {
   'year': '2017',
   'quarter': 'autumn',
   'page_size': '1000',
   'exclude_courses_without_sections': 'on'
}
headers = {"Authorization":"Bearer CA102B39-4731-4782-BF86-2A6C0EA80FD4"}
r = requests.get('https://ws.admin.washington.edu/student/v5/course', params=payload, headers=headers)
soup = BeautifulSoup(r.text, 'html.parser')

page_num = 1
courses_soup = soup.findAll("li")
while len(courses_soup) > 0:
   payload = {
      'year': '2017',
      'quarter': 'autumn',
      'page_size': '1000',
      'exclude_courses_without_sections': 'on',
      'page_start': page_num
   }
   headers = {"Authorization":"Bearer CA102B39-4731-4782-BF86-2A6C0EA80FD4"}
   r = requests.get('https://ws.admin.washington.edu/student/v5/course', params=payload, headers=headers)
   soup = BeautifulSoup(r.text, 'html.parser')
   courses_soup = soup.findAll("li")
   print page_num
   page_num += 1000
   for course_soup in courses_soup:
      # print courses_soup
      name = course_soup.find("span", class_="CourseTitleWithCourseTitleLong").string
      curriculum = course_soup.find("span", class_="courseuri__curriculum_abbreviation").string
      quarter = course_soup.find("span", class_="course_uri_qtr").string
      year = course_soup.find("span", class_="course_uri_year").string
      number = course_soup.find("span", class_="courseuri_course_number").string

      # if (courses[curriculum] == undefined)
      if curriculum in courses:
         courses[curriculum] = True
      else:
         courses.update({curriculum: True})
# print courses
ref.set(courses)
# for key in courses:
#    doc_ref = db.collection(u'courses').document(key)
#    doc_ref.set(courses[key])