python多进程编程
# -*- coding: UTF-8 -*- '''
Created on 2013-9-3
'''
import os
import threading
import multiprocessing
import time
# worker function
def worker(sign, lock):
'''使用Lock同步,在一个任务输出完成之后,再允许另一个任务输出,可以避免多个任务同时向终端输出。'''
lock.acquire()
'''Process.PID中保存有PID,如果进程还没有start(),则PID为None。'''
print(sign, os.getpid())
lock.release()
def proc1(pipe):
pipe.send('hello')
print('proc1 rec:',pipe.recv())
def proc2(pipe):
print('proc2 rec:',pipe.recv())
pipe.send('hello, too')
# input worker
def inputQ(queue):
info = str(os.getpid()) + '(put):' + str(time.time())
queue.put(info)
# output worker
def outputQ(queue,lock):
info = queue.get()
lock.acquire()
print (str(os.getpid()) + '(get):' + info)
lock.release()
if __name__ == '__main__':
'''所有Thread的PID都与主程序相同,而每个Process都有一个不同的PID。'''
# 1、Multi-thread
record = []
lock = threading.Lock()
for i in range(5):
thread = threading.Thread(target=worker,args=('thread',lock))
thread.start()
record.append(thread)
for thread in record:
thread.join()
# Main
print('Main:',os.getpid())
#2、 Multi-process
record = []
lock = multiprocessing.Lock()
for i in range(5):
'''利用multiprocessing.Process对象来创建一个进程
Process对象与Thread对象的用法相同,也有start(), run(), join()的方法。
此外multiprocessing包中也有Lock/Event/Semaphore/Condition类
(这些对象可以像多线程那样,通过参数传递给各个进程),用以同步进程'''
process = multiprocessing.Process(target=worker,args=('process',lock))
process.start()
record.append(process)
for process in record:
process.join()
# 3、Build a pipe
'''Pipe可以是单向(half-duplex)pipe有一个,也可以是双向(duplex)pipe有两个。
我们通过mutiprocessing.Pipe(duplex=False)创建单向管道 (默认为双向)。'''
pipe = multiprocessing.Pipe()
# Pass an end of the pipe to process 1
p1 = multiprocessing.Process(target=proc1, args=(pipe[0],))
# Pass the other end of the pipe to process 2
p2 = multiprocessing.Process(target=proc2, args=(pipe[1],))
p1.start()
p2.start()
p1.join()
p2.join()
#3、Build Queue
'''Queue允许多个进程放入,多个进程从队列取出对象。
Queue使用mutiprocessing.Queue(maxsize)创建,maxsize表示队列中可以存放对象的最大数量。'''
record1 = [] # store input processes
record2 = [] # store output processes
lock = multiprocessing.Lock() # To prevent messy print
queue = multiprocessing.Queue(3)
'''一些进程使用put()在Queue中放入字符串,这个字符串中包含PID和时间。
另一些进程从Queue中取出,并打印自己的PID以及get()的字符串。'''
# input processes
for i in range(10):
process = multiprocessing.Process(target=inputQ,args=(queue,))
process.start()
record1.append(process)
# output processes
for i in range(10):
process = multiprocessing.Process(target=outputQ,args=(queue,lock))
process.start()
record2.append(process)
for p in record1:
p.join()
queue.close() # No more object will come, close the queue
for p in record2:
p.join()