aiohttp模块,也就是async io http操作
# 1. 创建一个对象
aioreq = aiohttp.ClientSession() >>> 即对应异步的requests
# 2. 用这个异步requests来发送请求
resp = aioreq.get(url)
# 3. 异步写入文件,用到aiofiles模块,pip安装,可学习:https://www.w3cschool.cn/article/86236403.html
async with aiofiles.open(...) as f:
await f.write()
"""需要注意的是,完成后需要close来释放资源,最好的办法是用with来帮助管理。
而异步中规定必须在with前面加上async,即 async with..."""
案例示范:
import asyncio
import aiohttp
import aiofiles
"""整体思路就是:
1. 异步发送请求
2. 异步保存信息
3. 异步写入文档
"""
async def download(url):
"""第一种写法是推荐写法,用async with来管理会话、清理资源"""
# 1. 异步发送请求 >>> aiohttp.ClientSession() = 异步requests
async with aiohttp.ClientSession() as aioreq:
# # 2. 获得响应,保存信息
async with aioreq.get(url) as resp:
r_content = await resp.content.read()
# # 3. 异步写入文件
name = url.split('/')[-1]
async with aiofiles.open(rf"F:\异步\{name}", mode="wb") as f:
await f.write(r_content)
print(f'{name}', '完成')
"""第二种写法是手动关闭close,非常不推荐,需要手动关闭,有时候容易导致内存泄漏"""
# # 1. 异步发送请求 >>> aiohttp.ClientSession() = requests
# aioreq = aiohttp.ClientSession()
# # 2. 获得响应,保存信息
# resp = await aioreq.get(url) # 异步挂起
# r_content = await resp.content.read()
# # 3. 异步写入文件
# name = url.split('/')[-1]
# async with aiofiles.open(rf"F:\异步\{name}", mode="wb") as f:
# await f.write(r_content)
#
# await resp.release() >>> 文心一言说没有这种操作
# await aioreq.close()
async def main():
urls = [
"https://i1.huishahe.com/uploads/tu/201911/9999/85c8311a57.jpg",
"https://i1.huishahe.com/uploads/allimg/202205/9999/dd321482f1.jpg",
"https://i1.huishahe.com/uploads/allimg/202205/9999/dd06ff32f2.jpg"
]
tasks = []
for url in urls:
task = asyncio.create_task(download(url))
tasks.append(task)
await asyncio.wait(tasks)
if __name__ == "__main__":
asyncio.run(main())
标签:异步,aioreq,aiohttp,url,await,爬虫,async
From: https://www.cnblogs.com/abloger/p/18249813