This repository was archived by the owner on Mar 20, 2021. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathSnoopy.py
More file actions
830 lines (704 loc) · 30.5 KB
/
Snoopy.py
File metadata and controls
830 lines (704 loc) · 30.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
# -*- coding: utf-8 -*-
__author__ = 'Doscript'
__doc__ = """
Snoopy - 一个强大的PHP采集库
Snoopy for python
By Doscript
原PHP库,详见
http://sourceforge.net/projects/snoopy/
PHP版本的解释:
Snoopy是一个php类,用来模拟浏览器的功能,可以获取网页内容,发送表单。
Snoopy的特点:
1、抓取网页的内容 fetch
2、抓取网页的文本内容 (去除HTML标签) fetchtext
3、抓取网页的链接,表单 fetchlinks fetchform
4、支持代理主机
5、支持基本的用户名/密码验证
6、支持设置 user_agent, referer(来路), cookies 和 header content(头文件)
7、支持浏览器重定向,并能控制重定向深度
8、能把网页中的链接扩展成高质量的url(默认)
9、提交数据并且获取返回值
10、支持跟踪HTML框架
11、支持重定向的时候传递cookies
要求php4以上就可以了,由于本身是php一个类,无需扩支持,服务器不支持curl时候的最好选择。
"""
import urllib
import urlparse
import re
import os
import socket
import zlib
import base64
import uuid
import hashlib
def gzinflate(base64_string):
compressed_data = base64.b64decode(base64_string)
return zlib.decompress(compressed_data, -15)
def empty(variable):
if not variable:
return True
return False
def is_array(var):
return isinstance(var, (list, tuple))
def isset(variable):
return variable in locals() or variable in globals()
class Snoopy:
""" Public variables """
# usernamedefinable vars
def __init__(self):
self.scheme = 'http' # http or https
self.host = "www.python.org" # host name we are connecting to
self.port = 80 # port we are connecting to
self.proxy_host = "" # proxy host to use
self.proxy_port = "" # proxy port to use
self.proxy_username= "" # proxy usernameto use
self.proxy_password= "" # proxy password to use
self.agent = "Snoopy for python v1.0.0" # agent we masquerade as
self.referer = "" # referer info to pass
self.cookies = {} # array of cookies to pass
# cookies["username"]="joe"
self.rawheaders = {} # array of raw headers to send
# rawheaders["Content-type"]="text/html"
self.maxredirs = 5 # http redirection depth maximum. 0 = disallow
self.lastredirectaddr = "" # contains address of last redirected address
self.offsiteok = True # allows redirection off-site
self.maxframes = 0 # frame content depth maximum. 0 = disallow
self.expandlinks = True # expand links to fully qualified URLs.
# this only applies to fetchlinks()
# submitlinks(), and submittext()
self.passcookies = True # passwordset cookies back through redirects
# NOTE: this currently does not respect
# dates, domains or paths.
self.username = "" # username for http authentication
self.password = "" # password for http authentication
# http accept types
self.accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg,*"
self.results = "" # where the content is put
self.error = "" # error messages sent here
self.response_code = "" # response code returned from server
self.headers = {} # headers returned from server sent here
self.maxlength = 500000 # max return data length (body)
self.read_timeout = 0 # timeout on read operations, in seconds
# supported only since PHP 4 Beta 4
# set to 0 to disallow timeouts
self.timed_out = False # if a read operation timed out
self.status = 0 # http request status
self.temp_dir = "/tmp" # temporary directory that the webserver
# has permission to write to.
# under Windows, this should be C:\temp
self.curl_path = False
# deprecated, snoopy no longer uses curl for https requests,
# but instead requires the openssl extension.
# send Accept-encoding: gzip?
self.use_gzip = True
# file or directory with CA certificates to verify remote host with
self.cafile = ""
self.capath = ""
""" Private variables """
self._maxlinelen = 4096 # max line length (headers)
self._httpmethod = "GET" # default http request method
self._httpversion = "HTTP/1.1" # default http request version
self._submit_method = "POST" # default submit method
self._submit_type = "application/x-www-form-urlencoded" # default submit type
self._mime_boundary = "" # MIME boundary for multipart/form-data submit type
self._redirectaddr = "" # will be set if page fetched is a redirect
self._redirectdepth = 0 # increments on an http redirect
self._frameurls = {} # frame src urls
self._framedepth = 0 # increments on frame depth
self._isproxy = False # set if using a proxy server
self._fp_timeout = 30 # timeout for socket connection
self.u8_bom = chr(239)+chr(187)+chr(191)
"""
Function: fetch
Purpose: fetch the contents of a web page
(and possibly other protocols in the
future like ftp, nntp, gopher, etc.)
Input: URI the location of the page to fetch
Output: self.results the output text from the fetch
"""
def fetch(self, url):
uri_rarts = urlparse.urlparse(url)
if not empty(uri_rarts.username):
self.username = uri_rarts.username
if not empty(uri_rarts.password):
self.password = uri_rarts.password
fp = None
switch = uri_rarts.scheme.lower()
if switch == "https":
self.port = 443
elif switch == "http":
self.scheme = uri_rarts.scheme.lower()
self.host = uri_rarts.hostname
if not empty(uri_rarts.port):
self.port = uri_rarts.port
fp = self._connect(fp)
if fp:
if self._isproxy:
# using proxy, send entire URI
self._httprequest(url, fp, url, self._httpmethod)
else:
path = uri_rarts.path + ("?" + uri_rarts.query if uri_rarts.query else "")
# no proxy, send only the path
self._httprequest(path, fp, url, self._httpmethod)
self._disconnect(fp)
if self._redirectaddr:
#url was redirected, check if we've hit the max depth
if self.maxredirs > self._redirectdepth:
# only follow redirect if it's on this site, or offsiteok is True
if re.search("^https?://" + re.escape(self.host), self._redirectaddr, re.I) or self.offsiteok:
#follow the redirect
self._redirectdepth += 1
self.lastredirectaddr = self._redirectaddr
self.fetch(self._redirectaddr)
if self._framedepth < self.maxframes and len(self._frameurls) > 0:
frameurls = self._frameurls
self._frameurls = {}
for frameurl in frameurls.values():
if self._framedepth < self.maxframes:
self.fetch(frameurl)
self._framedepth += 1
else:
break
else:
return False
return self
else:
# not a valid protocol
self.error = 'Invalid protocol "' + uri_rarts.scheme + '"\n'
return False
return self
"""
Function: submit
Purpose: submit an http(s) form
Input: URI the location to post the data
formvars the formvars to use.
format: formvars["var"] = "val"
formfiles an array of files to submit
format: formfiles["var"] = "/dir/filename.ext"
Output: self.results the text output from the post
"""
def submit(self, uri, formvars = None, formfiles = None):
postdata = self._prepare_post_body(formvars, formfiles)
uri_rarts = urlparse.urlparse(uri)
if not empty(uri_rarts.username):
self.username= uri_rarts.username
if not empty(uri_rarts.password):
self.password= uri_rarts.password
fp = None
switch = uri_rarts.scheme.lower()
if switch == "https":
self.port = 443
elif switch == "http":
self.scheme = uri_rarts.scheme.lower()
self.host = uri_rarts.hostname
if not empty(uri_rarts.port):
self.port = uri_rarts.port
fp = self._connect(fp)
if fp:
if self._isproxy:
# using proxy, send entire URI
self._httprequest(uri, fp, uri, self._submit_method, self._submit_type, postdata)
else:
path = uri_rarts.path + ("?" + uri_rarts.query if uri_rarts.query else "")
# no proxy, send only the path
self._httprequest(path, fp, uri, self._submit_method, self._submit_type, postdata)
self._disconnect(fp)
if self._redirectaddr:
#url was redirected, check if we've hit the max depth
if self.maxredirs > self._redirectdepth:
if not re.search("^" + uri_rarts.scheme + "://", self._redirectaddr):
self._redirectaddr = self._expandlinks(self._redirectaddr, uri_rarts.scheme + "://" + uri_rarts.host)
# only follow redirect if it's on this site, or offsiteok is True
if re.search("^https?://" + re.escape(self.host), self._redirectaddr, re.I) or self.offsiteok:
#follow the redirect
self._redirectdepth += 1
self.lastredirectaddr = self._redirectaddr
if self._redirectaddr.find("?") > 0:
self.fetch(self._redirectaddr) # the redirect has changed the request method from post to get
else:
self.submit(self._redirectaddr, formvars, formfiles)
if self._framedepth < self.maxframes and len(self._frameurls) > 0:
frameurls = self._frameurls
self._frameurls = {}
for frameurl in frameurls.values():
if self._framedepth < self.maxframes:
self.fetch(frameurl)
self._framedepth += 1
else:
break
else:
return False
return self
else:
# not a valid protocol
self.error = 'Invalid protocol "' + uri_rarts.scheme + '"\n'
return False
return self
"""
Function: fetchlinks
Purpose: fetch the links from a web page
Input: URI where you are fetching from
Output: self.results an array of the URLs
"""
def fetchlinks(self, uri):
if self.fetch(uri):
if self.lastredirectaddr:
uri = self.lastredirectaddr
if is_array(self.results):
for i in xrange(len(self.results)):
self.results[i] = self._striplinks(self.results[i])
else:
self.results = self._striplinks(self.results)
if self.expandlinks:
self.results = self._expandlinks(self.results, uri)
return self
else:
return False
"""
Function: fetchform
Purpose: fetch the form elements from a web page
Input: URI where you are fetching from
Output: self.results the resulting html form
"""
def fetchform(self, uri):
if self.fetch(uri):
if is_array(self.results):
for i in xrange(len(self.results)):
self.results[i] = self._stripform(self.results[i])
else:
self.results = self._stripform(self.results)
return self
else:
return False
"""
Function: fetchtext
Purpose: fetch the text from a web page, stripping the links
Input: URI where you are fetching from
Output: self.results the text from the web page
"""
def fetchtext(self, uri):
if self.fetch(uri):
if is_array(self.results):
for i in xrange(len(self.results)):
self.results[i] = self._striptext(self.results[i])
else:
self.results = self._striptext(self.results)
return self
else:
return False
"""
Function: submitlinks
Purpose: grab links from a form submission
Input: URI where you are submitting from
Output: self.results an array of the links from the post
"""
def submitlinks(self, uri, formvars = "", formfiles = ""):
if self.submit(uri, formvars, formfiles):
if self.lastredirectaddr:
uri = self.lastredirectaddr
if is_array(self.results):
for i in xrange(len(self.results)):
self.results[i] = self._striplinks(self.results[i])
if self.expandlinks:
self.results[i] = self._expandlinks(self.results[i], uri)
else:
self.results = self._striplinks(self.results)
if self.expandlinks:
self.results = self._expandlinks(self.results, uri)
return self
else:
return False
"""
Function: submittext
Purpose: grab text from a form submission
Input: URI where you are submitting from
Output: self.results the text from the web page
"""
def submittext(self, uri, formvars = "", formfiles = ""):
if self.submit(uri, formvars, formfiles):
if self.lastredirectaddr:
uri = self.lastredirectaddr
if is_array(self.results):
for i in xrange(len(self.results)):
self.results[i] = self._striptext(self.results[i])
if self.expandlinks:
self.results[i] = self._expandlinks(self.results[i], uri)
else:
self.results = self._striptext(self.results)
if self.expandlinks:
self.results = self._expandlinks(self.results, uri)
return self
else:
return False
"""
Function: set_submit_multipart
Purpose: Set the form submission content type to
multipart/form-data
"""
def set_submit_multipart(self):
self._submit_type = "multipart/form-data"
return self
"""
Function: set_submit_normal
Purpose: Set the form submission content type to
application/x-www-form-urlencoded
"""
def set_submit_normal(self):
self._submit_type = "application/x-www-form-urlencoded"
return self
"""
Private functions
"""
"""
Function: _striplinks
Purpose: strip the hyperlinks from an html document
Input: document document to strip.
Output: match an array of the links
"""
def _striplinks(self, document):
links = re.findall("""<\s*a\s.*?href\s*=\s* # find <a href=
[\"\']? # find single or double quote
(?(1) (.*?)\\1 | ([^\s\>]+)) # if quote found, match up to next matching
# quote, otherwise match up to next space
""", document, re.X | re.I | re.S)
# catenate the non-empty matches from the conditional subpattern
match = []
for i in links:
if not empty(i[1]):
match.append(i[1])
if not empty(i[0]):
match.append(i[0])
# return the links
return match
"""
Function: _stripform
Purpose: strip the form elements from an html document
Input: document document to strip.
Output: match an array of the links
"""
def _stripform(self, document):
elements = re.findall("(<\/?(FORM|INPUT|SELECT|TEXTAREA|(OPTION))[^<>]*>(?(2)(.*(?=<\/?(option|select)[^<>]*>[\r\n]*)|(?=[\r\n]*))|(?=[\r\n]*)))", document, re.U | re.S | re.I)
# catenate the matches
match = "\r\n".join([i[0] for i in elements])
# return the links
return match
"""
Function: _striptext
Purpose: strip the text from an html document
Input: document document to strip.
Output: text the resulting text
"""
def _striptext(self, document):
# I didn't use preg eval (//e) since that is only available in PHP 4.0.
# so, list your entities one by one here. I included some of the
# more common ones.
search = [('<script[^>]*?>.*?</script>', '', re.I | re.S),
('<[\\/\\!]*?[^<>]*?>', '', re.I | re.S),
('(?:[\r\n])[\\s]+', '\\1', 0),
('&(?:quot|#34|#034|#x22);', '"', re.I),
('&(?:amp|#38|#038|#x26);', '&', re.I),
('&(?:lt|#60|#060|#x3c);', '<', re.I),
('&(?:gt|#62|#062|#x3e);', '>', re.I),
('&(?:nbsp|#160|#xa0);', ' ', re.I),
('&(?:iexcl|#161);', chr(161), re.I),
('&(?:cent|#162);', chr(162), re.I),
('&(?:pound|#163);', chr(163), re.I),
('&(?:copy|#169);', chr(169), re.I),
('&(?:reg|#174);', chr(174), re.I),
('&(?:deg|#176);', chr(176), re.I),
('&(?:#39|#039|#x27);', chr(39), 0),
('&(?:euro|#8364);', chr(128), re.I),
('&a(?:uml|UML);', "ä", 0),
('&o(?:uml|UML);', "ö", 0),
('&u(?:uml|UML);', "ü", 0),
('&A(?:uml|UML);', "Ä", 0),
('&O(?:uml|UML);', "Ö", 0),
('&U(?:uml|UML);', "Ü", 0),
('ß', "ß", re.I)]
for i in search:
document = re.sub(i[0], i[1], document, i[2])
return document
"""
Function: _expandlinks
Purpose: expand each link into a fully qualified URL
Input: links the links to qualify
URI the full URI to get the base from
Output: expandedLinks the expanded links
"""
def _expandlinks(self, links, uri):
match = re.search("^[^\?]+", uri)
match = re.sub(r"/[^\/\.]+\.[^\/\.]+", "", match.group(1))
match = re.sub("/", "", match)
match_part = urlparse.urlparse(match)
match_root = match_part.scheme + "://" + match_part.hostname
#print match
search = [("^http://" + re.escape(self.host), "", re.I),
("^(\/)", match_root + "/", re.I),
("^(?!http://)(?!mailto:)", match + "/", re.I),
("/\./", "/", 0),
("/[^\/]+/\.\./", "/", 0)]
for i in search:
links = re.sub(i[0], i[1], links, i[2])
return links
"""
Function: _httprequest
Purpose: go get the http(s) data from the server
Input: url the url to fetch
fp the current open file pointer
URI the full URI
body body contents to send if any (POST)
Output:
"""
def _httprequest(self, url, fp, URI, http_method, content_type = "", body = ""):
cookie_headers = ''
if self.passcookies and self._redirectaddr:
self.setcookies()
URI_PARTS = urlparse.urlparse(URI)
if empty(url):
url = "/"
headers = http_method + " " + url + " " + self._httpversion + "\r\n"
if not empty(self.host) and 'Host' not in self.rawheaders:
headers += "Host: " + self.host
if (not empty(self.port) and self.port != '80'):
headers += ":" + str(self.port)
headers += "\r\n"
if not empty(self.agent):
headers += "User-Agent: " + self.agent + "\r\n"
if not empty(self.accept):
headers += "Accept: " + self.accept + "\r\n"
if self.use_gzip:
pass
if not empty(self.referer):
headers += "Referer: " + self.referer + "\r\n"
if not empty(self.cookies):
if len(self.cookies) > 0:
cookie_headers += 'Cookie: '
for cookieKey in self.cookies.keys():
cookie_headers += cookieKey + "=" + self.cookies[cookieKey] + "; "#urllib.quote_plus(self.cookies[cookieKey]) + "; "
headers += cookie_headers[0:-1] + "\r\n"
if not empty(self.rawheaders):
for headerKey in self.rawheaders.keys():
headers += headerKey + ": " + self.rawheaders[headerKey] + "\r\n"
if not empty(content_type):
headers += "Content-type: content_type"
if content_type == "multipart/form-data":
headers += " boundary=" + self._mime_boundary
headers += "\r\n"
if not empty(body):
headers += "Content-length: " + str(len(body)) + "\r\n"
if not empty(self.username) or not empty(self.password):
headers += "Authorization: Basic " + (self.username + ":" + self.password).encode('base64') + "\r\n"
#add proxy auth headers
#if not empty(self.proxy_user):
# headers += 'Proxy-Authorization: ' + 'Basic ' + (self.proxy_username + ':' + self.proxy_pass).encode('base64') + "\r\n"
headers += "\r\n"
# set the read timeout if needed
if self.read_timeout > 0:
socket.setdefaulttimeout(self.read_timeout)
self.timed_out = False
fp.write(headers + body)
fp.flush()
self._redirectaddr = False
self.headers = []
# content was returned gzip encoded?
is_gzipped = False
while True:
try:
currentHeader = fp.readline(self._maxlinelen)
except socket.timeout:
self.status = -100
return False
if not currentHeader:
break
if currentHeader == "\r\n":
break
# if a header begins with Location: or URI:, set the redirect
if re.search("^(Location:|URI:)", currentHeader, re.I):
# get URL portion of the redirect
matches = re.search("^(Location:|URI:)[ ]+(.*)", currentHeader.rstrip(), re.I)
# look for :# in the Location header to see if hostname is included
if not re.search("\:\/\/", matches.group(2)):
# no host in the path, so prepend
self._redirectaddr = URI_PARTS.scheme + "://" + self.host + ":" + self.port
# eliminate double slash
if not re.search("^/", matches.group(2)):
self._redirectaddr += "/" + matches.group(2)
else:
self._redirectaddr += matches.group(2)
else:
self._redirectaddr = matches.group(2)
if re.search("^HTTP/", currentHeader):
status = re.search("^HTTP/[^\s]*\s(.*?)\s", currentHeader)
if status:
self.status = status.group(1)
self.response_code = currentHeader
if re.search("Content-Encoding: gzip", currentHeader):
is_gzipped = True
self.headers.append(currentHeader)
results = ''
while True:
try:
_data = fp.read(self.maxlength)
except socket.timeout:
self.status = -100
return False
if len(_data) == 0:
break
results += _data
if self.u8_bom in results:
results = results[results.find(self.u8_bom) + 3:]
# gunzip
if is_gzipped:
# per http://www.php.net/manual/en/function.gzencode.php
results = results[10:]
results = gzinflate(results)
# check if there is a a redirect meta tag
match = re.search("<meta[\s]*http-equiv[^>]*?content[\s]*=[\s]*[\"\']?\d+[\s]*URL[\s]*=[\s]*([^\"\']*?)[\"\']?>", results, re.I)
if match:
self._redirectaddr = self._expandlinks(match.group(1), URI)
# have we hit our frame depth and is there frame src to fetch?
match = re.findall("<frame\s+.*src[\s]*=[\'\"]?([^\'\"\>]+)", results, re.I)
if (self._framedepth < self.maxframes) and match:
self.results.append(results)
for i in match:
self._frameurls.append(self._expandlinks(i, URI_PARTS.scheme + "://" + self.host))
# have we already fetched framed content?
elif is_array(self.results):
self.results.append(results)
# no framed content
else:
self.results = results
return self
"""
Function: setcookies()
Purpose: set cookies for a redirection
"""
def setcookies(self):
for i in self.headers:
match = re.search("^set-cookie:[\s]+([^=]+)=([^;]+)", i, re.I)
if match:
self.cookies[match.group(1)] = urllib.unquote_plus(match.group(2))
return self
"""
Function: _connect
Purpose: make a socket connection
Input: fp file pointer
"""
def _connect(self, fp):
if not empty(self.proxy_host) and not empty(self.proxy_port):
self._isproxy = True
host = self.proxy_host
port = self.proxy_port
if self.scheme == 'https':
print "HTTPS connections over proxy are currently not supported"
return False
else:
host = self.host
port = self.port
self.status = 0
context_opts = {}
if self.scheme == 'https':
# if cafile or capath is specified, enable certificate
# verification (including name checks)
context_opts['ssl'] = {
'verify_peer' : True,
'CN_match' : self.host,
'disable_compression' : True}
context_opts['ssl']['cafile'] = self.cafile
context_opts['ssl']['capath'] = self.capath
host = 'ssl://' + host
#context = stream_context_create(context_opts)
try:
sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM) #tcp socket
sock.connect((host,port))
fp = sock.makefile()
except socket.error, msg:
self.error = msg
"""
if (version_compare(PHP_VERSION, '5.0.0', '>')) {
if(self.scheme == 'http')
host = "tcp://" . host
fp = stream_socket_client(
"host:port",
errno,
errmsg,
self._fp_timeout,
STREAM_CLIENT_CONNECT,
context)
} else {
fp = fsockopen(
host,
port,
errno,
errstr,
self._fp_timeout,
context)
}"""
if fp:
# socket connection succeeded
return fp
else:
return False
"""
Function: _disconnect
Purpose: disconnect a socket connection
Input: fp file pointer
"""
def _disconnect(self, fp):
fp.close()
"""
Function: _prepare_post_body
Purpose: Prepare post body according to encoding type
Input: formvars - form variables
formfiles - form upload files
Output: post body
"""
def _prepare_post_body(self, formvars, formfiles):
postdata = ''
if not formvars and not formfiles:
return ""
switch = self._submit_type
if switch == "application/x-www-form-urlencoded":
for i in formvars.keys():
if is_array(formvars[i]):
for ii in formvars[i]:
postdata += urllib.unquote_plus(i) + "[]=" + urllib.unquote_plus(ii) + "&"
else:
postdata += urllib.unquote_plus(i) + "=" + urllib.unquote_plus(formvars[i]) + "&"
elif switch == "multipart/form-data":
self._mime_boundary = "PYSnoopy" + hashlib.md5(uuid.uuid1()).hexdigest()
for i in formvars.keys():
if is_array(formvars[i]):
for ii in formvars[i]:
postdata += "--" + self._mime_boundary + "\r\n"
postdata += "Content-Disposition: form-data name=\"%s\[\]\"\r\n\r\n"%i
postdata += "%s\r\n"%ii
else:
postdata += "--" + self._mime_boundary + "\r\n"
postdata += "Content-Disposition: form-data name=\"%s\"\r\n\r\n"%i
postdata += "%s\r\n"%formvars[i]
for i in formfiles.keys():
for ii in formfiles[i]:
if not os.access(ii, os.R_OK):
continue
fp = open(ii, "r")
file_content = fp.read()
fp.close()
base_name = os.path.basename(ii)
postdata += "--" + self._mime_boundary + "\r\n"
postdata += "Content-Disposition: form-data name=\"%s\" filename=\"%s\"\r\n\r\n"%(i, base_name)
postdata += "%s\r\n"%file_content
postdata += "--" + self._mime_boundary + "--\r\n"
return postdata
"""
Function: getResults
Purpose: return the results of a request
Output: string results
"""
def getResults(self):
return self.results
if __name__ == "__main__":
test = Snoopy()
test.expandlinks = True
test.submit("http://www.douban.com/")
print test.getResults()