Skip to content

Commit

Permalink
clean code
Browse files Browse the repository at this point in the history
  • Loading branch information
chenjiandongx committed Jun 18, 2017
1 parent e47671f commit c68485b
Showing 1 changed file with 4 additions and 6 deletions.
10 changes: 4 additions & 6 deletions mm_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,8 @@ def make_dir(folder_name):
print(path)
os.chdir(path)
return True
else:
print("This folder have been created!")
return False
print("Folder has existed!")
return False


def delete_empty_dir(dir):
Expand Down Expand Up @@ -83,14 +82,13 @@ def urls_crawler(url):
except Exception as e:
print(e)


if __name__ == "__main__":
urls = ['http://mmjpg.com/mm/{cnt}'.format(cnt=cnt) for cnt in range(1, 953)]
pool = Pool(processes=cpu_count())
try:
delete_empty_dir(dir_path)
results = pool.map(urls_crawler, urls)
pool.map(urls_crawler, urls)
except Exception as e:
time.sleep(30)
delete_empty_dir(dir_path)
results = pool.map(urls_crawler, urls)
pool.map(urls_crawler, urls)

0 comments on commit c68485b

Please sign in to comment.