intro-data-sci/happiest_state.py at master · a1flecke/intro-data-sci · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
import sys
import re
import json

def main():
  sent_file = open(sys.argv[1])
  tweet_file = open(sys.argv[2])

  sentiments = loadSentiment(sent_file)
  tweets = loadTweets(tweet_file)
  state_sents = {}

  states = ["AL", "AK", "AZ", "AR", "CA", "CO", "CT", "DC", "DE", "FL", "GA",
          "HI", "ID", "IL", "IN", "IA", "KS", "KY", "LA", "ME", "MD",
          "MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", "NJ",
          "NM", "NY", "NC", "ND", "OH", "OK", "OR", "PA", "RI", "SC",
          "SD", "TN", "TX", "UT", "VT", "VA", "WA", "WV", "WI", "WY"]

  for tweet in tweets:
     addToStateSent(state_sents, tweet, calculateSentiment(tweet, sentiments))


  state_objs = buildStateObjs(state_sents)
  happiest = sorted(state_objs, key=StateObj.getSent).pop()
  print happiest.getName()

def loadSentiment(fp):
  sent = {}

  for line in fp:
    line_split = re.split(r'(\t+)',line)
    sent[line_split[0]] = int(line_split[2])

  return sent

def loadTweets(fp):
  tweets = []
  for line in fp:
    tweet = json.loads(line)
    if u'text' in tweet:
      state = getStateForTweet(tweet)
      if state != "":
        tweets.append(Tweet(tweet[u'text'], state))

  return tweets

def calculateSentiment(tweet, sentiments):
  tweet_split = re.split(r'(\s+)',tweet.getText())
  sent_sum = 0
  for tweet_word in tweet_split:
    if tweet_word in sentiments:
      sent_sum += sent_sum + int(sentiments[tweet_word])
  return sent_sum

def getStateForTweet(tweet):
  if u'user' in tweet:
    user = tweet[u'user']
    if u'lang' in user and user[u'lang'] == 'en':
      if u'location' in user:
        location = user[u'location']
        search = re.search(',\s+(AL|AK|AZ|AR|CA|CO|CT|DC|DE|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY)$', location)
        if search:
          return search.group(1)
  return ''

def addToStateSent(state_sents, tweet, sentiment):
  state_sents[tweet.getState()] = state_sents.get(tweet.getState(), 0) + 1

def buildStateObjs(state_sents):
  state_objs = []
  for k in state_sents:
    state_objs.append(StateObj(k, state_sents[k]))

  return state_objs

class Tweet:
  def __init__(self, text, state):
    self.text = text
    self.state = state

  def getText(self):
    return self.text

  def getState(self):
    return self.state

class StateObj:

  def __init__(self, name, sent):
    self.name = name
    self.sent = sent

  def getName(self):
    return self.name

  def getSent(self):
    return self.sent

if __name__ == '__main__':
    main()