多进程系列:计算各子进程时间
了解各个子进程耗时对于负载均衡至关重要,下面展示代码示例和结果示例说明如何计算各子进程时间。
代码示例
import multiprocessing
import time
from pprint import pprint
# 假设以下是五个分类模型函数
def classify_model_1(image_path, queue):
# 模拟分类操作
time.sleep(1)
print("classify_model_1")
# return f"模型1分类结果: {image_path}"
queue.put(f"模型1分类结果: {image_path}")
def classify_model_2(image_path, queue):
# 模拟分类操作
time.sleep(1)
print("classify_model_2")
# return f"模型2分类结果: {image_path}"
queue.put(f"模型2分类结果: {image_path}")
# 假设以下是五个分割模型函数
def segment_model_1(image_path, queue):
# 模拟分割操作
time.sleep(1)
print("segment_model_1")
# return f"模型1分割结果: {image_path}"
queue.put(f"模型1分割结果: {image_path}")
def segment_model_2(image_path, queue):
# 模拟分割操作
time.sleep(1)
print("segment_model_2")
# return f"模型2分割结果: {image_path}"
queue.put(f"模型2分割结果: {image_path}")
# 进程执行函数
def process_image(image_path, result_dict, result_key):
# 创建用于进程间通信的队列
classify_queue = multiprocessing.Queue()
segment_queue = multiprocessing.Queue()
# 创建分类模型和分割模型的进程
classify_processes = [
multiprocessing.Process(
target=classify_model_1, args=(image_path, classify_queue)
),
multiprocessing.Process(
target=classify_model_2, args=(image_path, classify_queue)
),
]
segment_processes = [
multiprocessing.Process(
target=segment_model_1, args=(image_path, segment_queue)
),
multiprocessing.Process(
target=segment_model_2, args=(image_path, segment_queue)
),
]
model_processes = classify_processes + segment_processes
# 启动所有进程并记录开始时间
start_time_dict = {}
run_time_dict = {}
for p in model_processes:
p.start()
start_time_dict[p.pid] = time.time()
print(f"==>> start_time_dict: {start_time_dict}")
# 等待所有分类模型进程完成并记录结束时间
for p in classify_processes:
p.join()
run_time_dict[p.pid] = str(time.time() - start_time_dict[p.pid])
# 等待所有分割模型进程完成并记录结束时间
for p in segment_processes:
p.join()
run_time_dict[p.pid] = str(time.time() - start_time_dict[p.pid])
print(f"==>> run_time_dict: {run_time_dict}")
# 方法一
# 按照顺序等待所有进程完成,比较耗时,
# 适用于确保分类模型在分割模型之前完成
# for p in model_processes:
# p.join()
# 方法二
# 分类模型进程和分割模型进程同时运行
for p in classify_processes:
p.join()
# 等待所有分割模型进程完成
for p in segment_processes:
p.join()
# 收集所有分类和分割模型的结果
classify_results = []
while not classify_queue.empty():
classify_results.append(classify_queue.get())
segment_results = []
while not segment_queue.empty():
segment_results.append(segment_queue.get())
# return classify_results, segment_results
# 将结果存储在共享字典中
result_dict[result_key] = (classify_results, segment_results, run_time_dict)
# 主函数
if __name__ == "__main__":
tic = time.time()
# 假设我们有一个图像路径列表
image_paths = [
"image1.jpg",
"image2.jpg",
]
# 创建一个Manager来共享结果字典
manager = multiprocessing.Manager()
result_dict = manager.dict()
processes = []
for i, image_path in enumerate(image_paths):
p = multiprocessing.Process(target=process_image, args=(image_path, result_dict, i))
processes.append(p)
p.start()
# 等待所有图像处理进程完成
for p in processes:
p.join()
# 输出结果
for key, value in result_dict.items():
pprint(value)
toc = time.time()
run_time = toc - tic
print("程序运行时间:", run_time)
结果示例
classify_model_1
classify_model_2
classify_model_1
segment_model_1
segment_model_2
==>> start_time_dict: {203732: 1723085307.806911, 204608: 1723085307.9630837, 202820: 1723085307.9887679, 203004: 1723085308.0356302}
==>> run_time_dict: {203732: '2.909015655517578', 204608: '2.8934247493743896', 202820: '3.10205078125', 203004: '3.1345412731170654'}
classify_model_2
segment_model_1
segment_model_2
==>> start_time_dict: {184112: 1723085308.0356302, 200372: 1723085308.1605997, 204200: 1723085308.207462, 203284: 1723085308.4573998}
==>> run_time_dict: {184112: '3.023956775665283', 200372: '3.478219985961914', 204200: '3.510946035385132', 203284: '3.3078691959381104'}
(['模型1分类结果: image1.jpg', '模型2分类结果: image1.jpg'],
['模型1分割结果: image1.jpg', '模型2分割结果: image1.jpg'],
{202820: '3.10205078125',
203004: '3.1345412731170654',
203732: '2.909015655517578',
204608: '2.8934247493743896'})
(['模型1分类结果: image2.jpg', '模型2分类结果: image2.jpg'],
['模型1分割结果: image2.jpg', '模型2分割结果: image2.jpg'],
{184112: '3.023956775665283',
200372: '3.478219985961914',
203284: '3.3078691959381104',
204200: '3.510946035385132'})
程序运行时间: 5.741686582565308
标签:系列,image,dict,各子,time,进程,model,segment,classify
From: https://blog.csdn.net/familytaijun/article/details/141021741