Skip to content

Commit

Permalink
修改路径表示方式
Browse files Browse the repository at this point in the history
  • Loading branch information
chenjiandongx committed May 3, 2017
1 parent 3339dcd commit 47a950f
Showing 1 changed file with 5 additions and 5 deletions.
10 changes: 5 additions & 5 deletions mm_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
headers = {'X-Requested-With': 'XMLHttpRequest',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/56.0.2924.87 Safari/537.36'}

dir_path = r"E:\mmjpg" # 下载图片保存路径

def save_pic(pic_src, pic_cnt):
""" 将图片下载到本地文件夹 """
Expand All @@ -27,7 +27,7 @@ def save_pic(pic_src, pic_cnt):
def make_dir(folder_name):
""" 新建套图文件夹并切换到该目录下 """

path = os.path.join(r"E:\mmjpg", folder_name)
path = os.path.join(dir_path, folder_name)

# 如果目录已经存在就不用再次爬取了,去重,提高效率。存在返回 False,否则反之
if not os.path.exists(path):
Expand Down Expand Up @@ -55,7 +55,7 @@ def delete_empty_dir(dir):
os.rmdir(dir)
print("remove the empty dir: {}".format(dir))
else:
print("Please start your performance!")
print("Please start your performance!") # 请开始你的表演


lock = threading.Lock() # 全局资源锁
Expand Down Expand Up @@ -100,9 +100,9 @@ def urls_crawler(url):
urls = ['http://mmjpg.com/mm/{cnt}'.format(cnt=cnt) for cnt in range(1, 953)]
pool = Pool(processes=cpu_count())
try:
delete_empty_dir(r"E:\mmjpg")
delete_empty_dir(dir_path)
results = pool.map(urls_crawler, urls)
except Exception as e:
time.sleep(30)
delete_empty_dir(r"E:\mmjpg")
delete_empty_dir(dir_path)
results = pool.map(urls_crawler, urls)

0 comments on commit 47a950f

Please sign in to comment.