InternLM2 Demo初体验-书生浦语大模型实战营学习笔记2

#! /bin/bash

# ~/.bashrc: executed by bash(1) for non-login shells.
# see /usr/share/doc/bash/examples/startup-files (in the package bash-doc)
# for examples

# If not running interactively, don't do anything
case $- in
    *i*) ;;
      *) return;;
esac

# don't put duplicate lines or lines starting with space in the history.
# See bash(1) for more options
HISTCONTROL=ignoreboth

# append to the history file, don't overwrite it
shopt -s histappend

# for setting history length see HISTSIZE and HISTFILESIZE in bash(1)
HISTSIZE=1000
HISTFILESIZE=2000

# check the window size after each command and, if necessary,
# update the values of LINES and COLUMNS.
shopt -s checkwinsize

# If set, the pattern "**" used in a pathname expansion context will
# match all files and zero or more directories and subdirectories.
#shopt -s globstar

# make less more friendly for non-text input files, see lesspipe(1)
[ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)"

# set variable identifying the chroot you work in (used in the prompt below)
if [ -z "${debian_chroot:-}" ] && [ -r /etc/debian_chroot ]; then
    debian_chroot=$(cat /etc/debian_chroot)
fi

# set a fancy prompt (non-color, unless we know we "want" color)
case "$TERM" in
    xterm-color|*-256color) color_prompt=yes;;
esac

# uncomment for a colored prompt, if the terminal has the capability; turned
# off by default to not distract the user: the focus in a terminal window
# should be on the output of commands, not on the prompt
#force_color_prompt=yes

if [ -n "$force_color_prompt" ]; then
    if [ -x /usr/bin/tput ] && tput setaf 1 >&/dev/null; then
	# We have color support; assume it's compliant with Ecma-48
	# (ISO/IEC-6429). (Lack of such support is extremely rare, and such
	# a case would tend to support setf rather than setaf.)
	color_prompt=yes
    else
	color_prompt=
    fi
fi

if [ "$color_prompt" = yes ]; then
    PS1='${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '
else
    PS1='${debian_chroot:+($debian_chroot)}\u@\h:\w\$ '
fi
unset color_prompt force_color_prompt

# If this is an xterm set the title to user@host:dir
case "$TERM" in
xterm*|rxvt*)
    PS1="\[\e]0;${debian_chroot:+($debian_chroot)}\u@\h: \w\a\]$PS1"
    ;;
*)
    ;;
esac

# enable color support of ls and also add handy aliases
if [ -x /usr/bin/dircolors ]; then
    test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)"
    alias ls='ls --color=auto'
    #alias dir='dir --color=auto'
    #alias vdir='vdir --color=auto'

    alias grep='grep --color=auto'
    alias fgrep='fgrep --color=auto'
    alias egrep='egrep --color=auto'
fi

# colored GCC warnings and errors
#export GCC_COLORS='error=01;31:warning=01;35:note=01;36:caret=01;32:locus=01:quote=01'

# some more ls aliases
alias ll='ls -alF'
alias la='ls -A'
alias l='ls -CF'

# Add an "alert" alias for long running commands.  Use like so:
#   sleep 10; alert
alias alert='notify-send --urgency=low -i "$([ $? = 0 ] && echo terminal || echo error)" "$(history|tail -n1|sed -e '\''s/^\s*[0-9]\+\s*//;s/[;&|]\s*alert$//'\'')"'

# Alias definitions.
# You may want to put all your additions into a separate file like
# ~/.bash_aliases, instead of adding them here directly.
# See /usr/share/doc/bash-doc/examples in the bash-doc package.

if [ -f ~/.bash_aliases ]; then
    . ~/.bash_aliases
fi

# enable programmable completion features (you don't need to enable
# this, if it's already enabled in /etc/bash.bashrc and /etc/profile
# sources /etc/bash.bashrc).
if ! shopt -oq posix; then
  if [ -f /usr/share/bash-completion/bash_completion ]; then
    . /usr/share/bash-completion/bash_completion
  elif [ -f /etc/bash_completion ]; then
    . /etc/bash_completion
  fi
fi

# >>> conda initialize >>>
# !! Contents within this block are managed by 'conda init' !!
__conda_setup="$('/root/.conda/condabin/conda' 'shell.bash' 'hook' 2> /dev/null)"
if [ $? -eq 0 ]; then
    eval "$__conda_setup"
else
    if [ -f "/root/.conda/etc/profile.d/conda.sh" ]; then
        . "/root/.conda/etc/profile.d/conda.sh"
    else
        export PATH="/root/.conda/condabin:$PATH"
    fi
fi
unset __conda_setup
# <<< conda initialize <<<

if [ -d "/root/.conda/envs/xtuner" ]; then
  CONDA_ENV=xtuner
else
  CONDA_ENV=base
fi

source activate $CONDA_ENV

cat /share/.aide/config/welcome_vgpu

#if [ $CONDA_ENV != "xtuner" ]; then
#  echo -e """
#  \033[31m 检测到您尚未初始化xtuner环境, 建议执行> source init_xtuner_env.sh \033[0m
#  """
#fi
export https_proxy=http://proxy.intern-ai.org.cn:50000
export http_proxy=http://proxy.intern-ai.org.cn:50000
export no_proxy='localhost,127.0.0.1,0.0.0.0,172.18.47.140'
export PATH=/root/.local/bin:$PATH
export HF_ENDPOINT='https://hf-mirror.com'
alias studio-conda="/share/install_conda_env.sh"
alias studio-smi="/share/studio-smi"

#!/bin/bash
# clone internlm-base conda env to user's conda env
# created by xj on 01.07.2024
# modifed by xj on 01.19.2024 to fix bug of conda env clone
# modified by ljy on 01.26.2024 to extend

XTUNER_UPDATE_DATE=`cat /share/repos/UPDATE | grep xtuner |awk -F= '{print $2}'`
HOME_DIR=/root
CONDA_HOME=$HOME_DIR/.conda
SHARE_CONDA_HOME=/share/conda_envs
SHARE_HOME=/share

list() {
    cat <<-EOF
  预设环境          描述

  internlm-base    pytorch:2.0.1, pytorch-cuda:11.7
  xtuner           Xtuner(源码安装: main $(echo -e "\033[4mhttps://github.com/InternLM/xtuner/tree/main\033[0m"), 更新日期：$XTUNER_UPDATE_DATE)
  pytorch-2.1.2    pytorch:2.1.2, pytorch-cuda:11.8
EOF
}


help() {
    cat <<-EOF
  说明: 用于快速clone预设的conda环境

  使用: 
    1. studio-conda env -l/list 打印预设的conda环境列表
    2. studio-conda <target-conda-name> 快速clone: 默认拷贝internlm-base conda环境
    3. studio-conda -t <target-conda-name> -o <origin-conda-name> 将预设的conda环境拷贝到指定的conda环境
        
EOF
}

clone() {
    source=$1
    target=$2

    if [[ -z "$source" || -z "$target" ]]; then
        echo -e "\033[31m 输入不符合规范 \033[0m"
        help
        exit 1
    fi

    if [ ! -d "${SHARE_CONDA_HOME}/$source" ]; then
        echo -e "\033[34m 指定的预设环境: $source不存在\033[0m"
        list
        exit 1
    fi

    if [ -d "${CONDA_HOME}/envs/$target" ]; then
        echo -e "\033[34m 指定conda环境的目录: ${CONDA_HOME}/envs/$target已存在, 将清空原目录安装 \033[0m"
        wait_echo&
        wait_pid=$!
        rm -rf "${CONDA_HOME}/envs/$target"
        kill $wait_pid
    fi

    echo -e "\033[34m [1/2] 开始安装conda环境: <$target>. \033[0m"
    sleep 3
    tar --skip-old-files -xzvf /share/pkgs.tar.gz -C ${CONDA_HOME}
    wait_echo&
    wait_pid=$!
    conda create -n $target --clone ${SHARE_CONDA_HOME}/${source}
    if [ $? -ne 0 ]; then
        echo -e "\033[31m 初始化conda环境: ${target}失败 \033[0m"
        exit 10
    fi

    kill $wait_pid

    # for xtuner, re-install dependencies
    case "$source" in
    xtuner)
        source_install_xtuner $target
        ;;
    esac

    echo -e "\033[34m [2/2] 同步当前conda环境至jupyterlab kernel \033[0m"
    lab add $target
    source $CONDA_HOME/bin/activate $target
    cd $HOME_DIR

    echo -e "\033[32m conda环境: $target安装成功! \033[0m"

    echo """
    ============================================
                    ALL DONE!
    ============================================
    """
}

source_install_xtuner() {
    conda_env=$1
    echo -e "\033[34m 源码安装xtuner... \033[0m"
    sleep 2
    pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple

    install=0
    if [ -d "${HOME_DIR}/xtuner" ]; then
        read -r -p "$HOME_DIR中已存在目录xtuner: 是否清空目录? [Y/N][yes/no]" input
	case $input in
          [yY][eE][sS]|[yY])
            echo -e "\033[34m 清空目录: $HOME_DIR/xtuner, 并同步源码至该目录进行源码安装... \033[0m"
	    install=1
	    ;;
	  *)
	    echo -e "\033[34m 尝试使用: $HOME_DIR/xtuner目录进行源码安装... \033[0m" 
	    ;;
        esac
    else
        install=1
    fi
    
    if [ $install -eq 1 ]; then
        rm -rf $HOME_DIR/xtuner
	mkdir -p $HOME_DIR/xtuner
	cp -rf $SHARE_HOME/repos/xtuner/* $HOME_DIR/xtuner/
    fi

    cd $HOME_DIR/xtuner

    $CONDA_HOME/envs/$conda_env/bin/pip install -e '.[all]'
    if [ $? -ne 0 ]; then
        echo -e "\033[31m 源码安装xtuner失败 \033[0m"
	exit 10
    fi
    $CONDA_HOME/envs/$conda_env/bin/pip install cchardet
    $CONDA_HOME/envs/$conda_env/bin/pip install -U datasets
}


wait_echo() {
    local i=0
    local sp='/-\|'
    local n=${#sp}
    printf ' '
    while sleep 0.1; do
        printf '\b%s' "${sp:i++%n:1}"
    done
}

dispatch() {

    if [ $# -lt 1 ]; then
        help
        exit -2
    fi

    if [ $1 == "env" ]; then
        list
        exit 0
    fi

    if [[ $1 == "-h" || $1 == "help" ]]; then
        help
        exit 0
    fi

    origin_env=
    target_env=
    if [ $# -eq 1 ]; then
        origin_env=internlm-base
        target_env=$1
    else
        while getopts t:o: flag; do
            case "${flag}" in
            t) target_env=${OPTARG} ;;
            o) origin_env=${OPTARG} ;;
            esac
        done
    fi

    echo -e "\033[32m 预设环境: $origin_env \033[0m"
    echo -e "\033[32m 目标conda环境名称: $target_env \033[0m"
    sleep 3
    clone $origin_env $target_env
}

dispatch $@