https://mp.weixin.qq.com/s/VOgNPEcDhmhMuDdy_HL0BA
from deepspeed.ops.zero_offload import FP16ZeROOffloadEngine
# Initialize the ZeRO-Offload engine
zero_offload_engine = FP16ZeROOffloadEngine()
# Wrap the model with the ZeRO-Offload engine
model, _, _, _ = zero_offload_engine.initialize(model=model, optimizer=optimizer)
# Train the model
for batch in data:
loss = model(batch)
loss.backward()
optimizer.step()
optimizer.zero_grad()
