9.7 池
为什么要有池：
-
预先的开启固定个数的进程数，当任务来临的时候，直接提交给已经开好的进程
-
这些已经开好的进程直接去执行程序
-
节省了进程、线程开启、关闭、切换的时间
-
并且减轻了操作系统的调度负担
9.7.1 进程池
import os import timeimport randomfrom concurrent.futures import ProcessPoolExecutordf func(): print('start',os.gitpid()) time.sleep(random.random()) print('edf',os.getpid()) if __name__ == '__main__': p = ProcessPollExcutor(5) for i in range(10): p.submit() p.shutdown() #关闭池之后就不能继续提交任务，并且会阻塞，直到已经提交的任务完成 print('main',os.getpid())
任务参数+返回值
from concurrent.futures import ProcessPoolExecutordef func(i,name): print('start',os.getpid()) time.sleep(random.randint(1,3)) print('end', os.getpid()) return '%s * %s'%(i,os.getpid())if __name__ =='__main__': p = ProcessPoolExcutor(5) ret_l = [] for i in range(10): ret = p.submit(func,i,'liujia') ret_l.append(ret) for ret in ret_l: print('ret---->',ret.result())#ret.result()同步阻塞 print('main',os.getpid())
9.7.2 线程池
from concurrent.futures import ThreadPoolExecutordef func(i): print('start', os.getpid()) time.sleep(random.randint(1,3)) print('end', os.getpid()) return '%s * %s'%(i,os.getpid())tp = ThreadPoolExecutor(20)ret_l = []for i in range(10): ret = tp.submit(func,i) ret_l.append(ret)tp.shutdown()print('main')for ret in ret_l: print('------>',ret.result())
利用map简化
from concurrent.futures import ThreadPoolExecutordef func(i): print('start', os.getpid()) time.sleep(random.randint(1,3)) print('end', os.getpid()) return '%s * %s'%(i,os.getpid())tp = ThreadPoolExecutor(20)ret = tp.map(func,range(20)) #0-20都去执行func函数for i in ret: #生成每一个任务的返回值 print(i)ret_l = []for i in range(10): ret = tp.submit(func,i) ret_l.append(ret)tp.shutdown()print('main')
回调函数
import requestsfrom concurrent.cutures import ThreadPoolExecutordef get_page(url): res = requests.get(url) return {'url':url,'content':res.txt}def paeserpage(ret): #ret其实就是get_page函数的返回值 dic = ret.result() print(dic['url']) tp = ThreadPoolExecutor(5):url_list = ['http://www.baidu.com', # 3 'http://www.cnblogs.com', # 1 'http://www.douban.com', # 1 'http://www.tencent.com', 'http://www.cnblogs.com/Eva-J/articles/8306047.html', 'http://www.cnblogs.com/Eva-J/articles/7206498.html',]ret = [] #中转站for url in url_list: ret = tp.submit(get_page,url) ret_l.append(ret) ret.add_done_callback(parserpage) #哪一个线程先回来就先执行