1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57
| @staticmethod def pool(callback, lists,threadNum=10): import threadpool pool = threadpool.ThreadPool(threadNum) requests = threadpool.makeRequests(callback, lists) [pool.putRequest(req) for req in requests] pool.wait()
@staticmethod def bPool(arg): from multiprocessing.dummy import Pool as ThreadPool tpool = ThreadPool(arg['tnum']) arr=list(map(lambda i:{'cnum':arg['cnum'],'tnum':i,'arg':arg['arg']},range(arg['tnum']))) tpool.map(arg['callback'], arr) tpool.close() tpool.join()
@staticmethod def sPool(callback,tnum=20,cnum='',arg=[]): from multiprocessing import Pool as ProcessPool from multiprocessing import cpu_count if cnum=='': spool = ProcessPool(cpu_count()) else: spool = ProcessPool(cnum) arr=list(map(lambda i:{'cnum':i,'tnum':tnum,'callback':callback,'arg':arg},range(cnum))) spool.map(fleader.bPool, arr) spool.close() spool.join()
def Manager(): from multiprocessing import Manager manager = Manager() q = manager.Queue() lock = manager.Lock() return q,lock
@staticmethod def gPool(callback,urls=[],pnum=800): from gevent import monkey; monkey.patch_all(socket=True,select=True) from gevent.pool import Pool gpool = Pool(pnum) gpool.map(callback, urls)
@staticmethod def getGevent(): from gevent import monkey; monkey.patch_all(socket=True,select=True) from gevent.queue import Queue from gevent.local import local try: from gevent.lock import BoundedSemaphore except: from gevent.coros import BoundedSemaphore sem = BoundedSemaphore(2) return local,Queue,sem
|