docker部署d2l环境

编写dockerfile

# 使用NVIDIA提供的CUDA基础镜像，包含CUDA 11.8.0和cuDNN 8，基于Ubuntu 22.04
FROM nvidia/cuda:11.8.0-cudnn8-devel-ubuntu22.04
# 设置维护者信息
MAINTAINER watcherprime <woma@126.com>

# 设置环境变量，包括时区、非交互式前端和PATH变量
ENV TZ=Asia/Shanghai \
    DEBIAN_FRONTEND=noninteractive \
    PATH=/opt/miniconda3/bin:$PATH

# 设置工作目录为/root
WORKDIR /root

# 更新Ubuntu软件包索引
RUN apt-get update

# 安装SSH服务并配置
RUN DEBIAN_FRONTEND=noninteractive apt-get install -y ssh \
    && mkdir /var/run/sshd && mkdir /root/.ssh \
    && echo 'root:root' | chpasswd \
    && sed -i 's/^#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config \
    && sed -i 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config

# 创建并设置自启动SSH服务的脚本
RUN echo 'service ssh start >>/root/start_ssh.log' >> ~/start_ssh.sh \
    && chmod 777 ~/start_ssh.sh \
    && echo '#<<< ssh <<<\nif [ -f /root/start_ssh.sh ]; then\n/root/start_ssh.sh\nfi\n#<<< ssh <<<' >> ~/.bashrc

# 创建并设置自启动TensorBoard的脚本
RUN echo '#!/bin/bash\na=$(netstat -anp | grep 6006)\nif [ -z "$a" ]\nthen\nsource /opt/miniconda3/etc/profile.d/conda.sh\nconda activate d2l\nnohup python /opt/miniconda3/envs/d2l/lib/python3.10/site-packages/tensorboard/main.py --logdir /gemini/logs --bind_all > /root/start_tensorboard.log 2>&1 &\nfi' >> ./start_tensorboard.sh \
    && chmod 777 ./start_tensorboard.sh \
    && echo '#<<< tensorboard <<<\nif [ -f /root/start_tensorboard.sh ]; then\n/root/start_tensorboard.sh\nfi\n#<<< tensorboard <<<' >> ~/.bashrc

# 安装一些常用的工具
RUN apt-get install -y vim && apt-get install -y git && apt-get install -y net-tools

# 下载并安装Miniconda3
RUN wget -O /opt/Miniconda3-py310_24.3.0-0-Linux-x86_64.sh "https://mirrors.bfsu.edu.cn/anaconda/miniconda/Miniconda3-py310_24.3.0-0-Linux-x86_64.sh"  \
    && chmod +x /opt/Miniconda3-py310_24.3.0-0-Linux-x86_64.sh \
    && sh -c '/bin/echo -e "\nyes\n\nyes" | sh /opt/Miniconda3-py310_24.3.0-0-Linux-x86_64.sh -b -p /opt/miniconda3'
ENV PATH=/opt/miniconda3/bin:$PATH

# 调整时区设置
RUN apt install -y tzdata \
    && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \
    && echo ${TZ} > /etc/timezone \
    && dpkg-reconfigure --frontend noninteractive tzdata

# 更新conda源为清华大学镜像源
RUN conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/  \
    && conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main/  \
    && conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/  \
    && conda config --set show_channel_urls yes \
    && conda config --set auto_activate_base no

# 创建conda环境d2l，并设置为默认环境
RUN conda create --name d2l python=3.10.13
SHELL ["/bin/bash", "--login", "-c"]

# 在新创建的conda环境中安装基础依赖项
RUN export PATH=/opt/miniconda3/bin:$PATH \
    && conda update --name base conda \
    && conda init bash \
    && source activate d2l \
    && conda activate d2l \
    && conda install -y --quiet numpy pyyaml mkl mkl-include setuptools cmake cffi typing \
    && conda install -y --quiet -c mingfeima mkldnn

# 升级pip并设置pip源为清华大学镜像源
RUN python -m pip install --upgrade pip -i https://pypi.tuna.tsinghua.edu.cn/simple  \
    && pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple

# 安装PyTorch和相关库
RUN pip install torch==2.1.1 torchvision==0.16.1 torchaudio==2.1.1 -i https://pypi.tuna.tsinghua.edu.cn/simple  \
    && pip install d2l jupyter

# 安装额外的依赖项，如ffmpeg等
RUN apt-get install -y ffmpeg libsm6 libxext6

# 创建项目目录结构
RUN mkdir /womacode && mkdir /womacode/code && mkdir /womacode/output

# 清理不必要的文件，减小镜像大小
RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

# 暴露SSH、TensorBoard和Jupyter Notebook的端口
EXPOSE 22
EXPOSE 6006
EXPOSE 8888

# 设置容器启动时执行的命令为bash
CMD ["bash"]

构建镜像

docker build -t torch2.1.1_cu118_py310_ubuntu22.04 -f Dockerfile .

打标签

docker tag torch2.1.1_cu118_py310_ubuntu22.04 watcherprime/deeplearn:Torch2.1.1_cu118_py310_ubuntu22.04

推送

docker push watcherprime/deeplearn:Torch2.1.1_cu118_py310_ubuntu22.04

启动

挂载宿主磁盘:/opt/project/code/python

docker run -it --gpus all -v /opt/project/code/python:/remote-home/python -p 9998:22 --name d2l_env --restart=always torch2.1.1_cu118_py310_ubuntu22.04

docker run -it --gpus all -p 9999:8080 --name d2l torch2.1.1_cu118_py310_ubuntu22.04

docker 进入 d2l 容器

docker exec -it d2l /bin/bash

测试环境是否生效

import torch
import d2l
print(torch.__version__)
print(torch.cuda.is_available())
print(d2l.__version__)

conda命令

#列出所有环境
conda env list

#激活环境
conda activate d2l

下载相关包

 pip install torch==2.1.1 torchvision==0.16.1 torchaudio==2.1.1

pip install d2l jupyter

运行resNet测试

import torchvision
from torchvision import transforms
from torch.utils import data

def get_dataloader_workers():
    """使用6个进程来读取的数据"""
    return 6

trans = transforms.ToTensor()
def load_data_fashion_mnist(batch_size, resize=None):
    """下载Fashion-MNIST数据集，然后将其加载到内存中"""
    trans = [transforms.ToTensor()]
    if resize:
        trans.insert(0,transforms.Resize(resize)) # 如果有Resize参数传进来，就进行resize操作
    trans = transforms.Compose(trans)
    mnist_train = torchvision.datasets.FashionMNIST(root="01_data/01_DataSet_FashionMNIST",train=True,transform=trans,download=True)
    mnist_test = torchvision.datasets.FashionMNIST(root="01_data/01_DataSet_FashionMNIST",train=False,transform=trans,download=True)            
    return (data.DataLoader(mnist_train, batch_size, shuffle=True, num_workers=get_dataloader_workers()),
           data.DataLoader(mnist_train, batch_size, shuffle=True, num_workers=get_dataloader_workers()))               

# Fashion-MNIST图像的分辨率 低于ImageNet图像。将它们增加到96×96
batch_size = 256
train_iter, test_iter = load_data_fashion_mnist(batch_size, resize=96) # 返回训练集、测试集的迭代器   

# 训练模型
lr, num_epochs, batch_size = 0.05, 10, 256
# train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size, resize=96)  
d2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())

调用GPU成功运行!