movies.py

#coding:utf-8

import requests
from bs4 import BeautifulSoup
import time
from selenium  import webdriver

headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Encoding": "gzip, deflate",
    "Accept-Language": "en-US,en;q=0.5",
    "Connection": "keep-alive",
    "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:39.0) Gecko/20100101 Firefox/39.0"}

def get_movies():
    f=open('data_movies2013.txt','a')
    start=1
    while start<8519:
        try:
            html=requests.get('http://www.imdb.com/search/title?at=0&sort=boxoffice_gross_us&start=%s&title_type=feature&year=2013,2013'%start,headers=headers,timeout=30).text
        except:
            continue
        items=parser(html)
        for item in items:
            f.write(item+'\n')
        start+=50
        print(start)

def parser(html):
    items=[]
    table=BeautifulSoup(html,'lxml').find('table',attrs={'class':'results'}).find_all('tr')[1:]
    for item in table:
        td=item.find('td',attrs={'class':'title'})
        title=item.find('a').get('title')
        try:
            score=td.find('span',attrs={'class':'rating-rating'}).get_text()
        except:
            score='-'
        try:
            col=item.find('td',attrs={'class':'sort_col'}).get_text()
        except:
            col='-'
        text=title+'||'+score+'||'+col
        items.append(text)
    return items

get_movies()