更新时间:2022-10-03 16:33:19
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
|
#!/usr/bin/python # -*-coding:utf-8-*- # Python: 2.7 # Program: 爬取豆瓣电影 from bs4 import BeautifulSoup
import urllib2, json, random, sys
reload (sys)
sys.setdefaultencoding( 'utf-8' )
def get_data(url):
my_headers = [
'Mozilla/5.0 (Windows NT 5.2) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.122 Safari/534.30' ,
'Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0' ,
'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.2; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET4.0E; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET4.0C)' ,
'Opera/9.80 (Windows NT 5.1; U; zh-cn) Presto/2.9.168 Version/11.50' ,
'Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1' ,
'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET4.0E; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET4.0C)' ]
header = { "User-Agent" : random.choice(my_headers)}
req = urllib2.Request(url, headers = header)
html = urllib2.urlopen(req).read()
data = json.loads(html)[ 'data' ]
return data
def get_movieInfo(url):
my_headers = [
'Mozilla/5.0 (Windows NT 5.2) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.122 Safari/534.30' ,
'Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0' ,
'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.2; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET4.0E; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET4.0C)' ,
'Opera/9.80 (Windows NT 5.1; U; zh-cn) Presto/2.9.168 Version/11.50' ,
'Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1' ,
'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET4.0E; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET4.0C)' ]
header = { "User-Agent" : random.choice(my_headers)}
req = urllib2.Request(url, headers = header)
html = urllib2.urlopen(req).read()
soup = BeautifulSoup(html, 'html.parser' )
movie = {}
movie[ 'Name' ] = soup.find( 'span' , property = "v:itemreviewed" ).text
movie[ 'Year' ] = soup.find( 'span' , class_ = 'year' ).text
movie[ 'Rate' ] = soup.find( 'strong' , property = "v:average" ).text
movie[ 'Runtime' ] = soup.find( 'span' , property = "v:runtime" ).text
movie[ 'Summary' ] = soup.find( 'span' , property = 'v:summary' ).text
movie[ 'URL' ] = url
movie[ 'Directors' ] = ''
directors = soup.find_all( 'a' , rel = "v:directedBy" )
for director in directors:
movie[ 'Directors' ] + = director.text
movie[ 'Directors' ] + = ' '
movie[ 'Stars' ] = ''
stars = soup.find_all( 'a' , rel = "v:starring" )
for star in stars:
movie[ 'Stars' ] + = star.text
movie[ 'Stars' ] + = ' '
movie[ 'Category' ] = ''
categorys = soup.find_all( 'span' , property = "v:genre" )
for category in categorys:
movie[ 'Category' ] + = category.text
movie[ 'Category' ] + = ' '
return movie
def get_urls():
base_url = 'https://movie.douban.com/j/new_search_subjects?sort=R&range=1,10&tags=%E7%94%B5%E5%BD%B1&start='
urls = []
nu = 0
while True :
print nu
url = base_url + str (nu)
data = get_data(url)
if len (data) = = 0 :
break
for i in data:
urls.append(i[ 'url' ])
nu + = 20
return urls
if __name__ = = '__main__' :
urls = get_urls()
f = open ( 'movieinfo.txt' , 'w+' )
for url in urls:
try :
movie = get_movieInfo(url)
movie_str = json.dumps(movie,ensure_ascii = False , encoding = 'UTF-8' )
f.write(movie_str)
f.write( '\n' )
f.flush()
except :
print url
continue
f.close()
|