多线程与多进程爬取数据（多线程python爬虫）

爬取的网站为：https://materialsweb.org/

代码：

#导入相关库
import os
import numpy as np
import urllib
import re
from urllib.request import urlopen
from bs4 import BeautifulSoup
import time
from multiprocessing.pool import ThreadPool
from multiprocessing import Pool as ProcessPool
from urllib.request import urlretrieve

#获取要下载的url，存入urls。
urls=
for i in range(1,781):
url="https://materialsweb.org/static/inputs/mw-"+str(i)+"_POSCAR"
urls.append(url)

#undownloaded存储未被下载的链接，可后面下再次尝试下载。
undownloaded=
def download(url):
#下载url文件，如出现异常，输出“Not Found”。
try:
urlretrieve(url, url[39:])
except:
print('Not Found')
undownloaded.append(url)


def thread_pool(num):
#初始化线程池p，参数num为欲采用的线程数。
p = ThreadPool(num)
start_time = time.time
#p.map的第一个参数为函数名，第二个参数为存储函数对应的参数，应可迭代。
ret = p.map(download, urls)
p.close
p.join
print("thread_pool %d, costTime: %fs ret.size: %d" % (num, (time.time - start_time), len(ret)))

def process_pool(num):
#初始化进程池p，参数num为欲采用的进程数。
p = ProcessPool(num)
start_time = time.time
#p.map的第一个参数为函数名，第二个参数为存储函数对应的参数，应可迭代。 ret = p.map(download, urls)
p.close
p.join
print("process_pool %d, costTime: %fs ret.size: %d" % (num, (time.time - start_time), len(ret)))

if __name__ == "__main__":
#分别采用4,5,6,7,8个进程和线程进行爬取，可以比较结果，看谁最快。
for i in range(4, 9):
thread_pool(i)
process_pool(i)
print("=====")

略去"Not FOund", 运行结果如下：

thread_pool 4, costTime: 311.579127s ret.size: 780

process_pool 4, costTime: 340.535949s ret.size: 780

thread_pool 5, costTime: 251.317034s ret.size: 780

process_pool 5, costTime: 481.923806s ret.size: 780

thread_pool 6, costTime: 259.729788s ret.size: 780

process_pool 6, costTime: 214.018717s ret.size: 780

thread_pool 7, costTime: 187.721970s ret.size: 780

process_pool 7, costTime: 184.847098s ret.size: 780

thread_pool 8, costTime: 162.505511s ret.size: 780

process_pool 8, costTime: 190.380221s ret.size: 780