Hadoop-3.3.5 自动安装脚本
为了方便数据科学与大数据技术的 hadoop-3.3.5 安装与配置
目前适用于新安装的纯净虚拟机,未配置冲突检测..
所以安装的话请使用全新创建的机器以避免错误...
如何使用
ssh localhost
输入你目前用户的密码exit
退出当前ssh终端(此举是为自动建立.ssh文件夹并记录known_hosts文件)- 然后将下列代码复制进
hadoop_create.sh
中 (vim hadoop_create.sh
) chmod +x hadoop_create.sh
(为该文件添加可执行权限)- 在你保存该文件的文件夹里输入
./hadoop_create.sh
,启动该脚本。
#!/bin/bash
# Set up Java下载链接、目录和Java安装目录
download_url_java="https://aka.ms/download-jdk/microsoft-jdk-21.0.0-linux-x64.tar.gz"
download_dir="$HOME/Download"
java_install_dir="$HOME/JavaRuntime"
# 设置SSH密钥和目录
ssh_key_dir="$HOME/.ssh"
ssh_key_file="$ssh_key_dir/id_rsa"
# 设置Hadoop下载链接、目录和Hadoop安装目录
hadoop_download_url="https://archive.apache.org/dist/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz"
hadoop_download_dir="$download_dir"
hadoop_install_dir="$HOME/HadoopRuntime"
# 创建Hadoop数据目录结构
hadoop_tmp_dir="$hadoop_install_dir/data/tmp"
hadoop_datanode_dir="$hadoop_install_dir/data/datanode"
hadoop_namenode_dir="$hadoop_install_dir/data/namenode"
mkdir -p "$hadoop_tmp_dir"
mkdir -p "$hadoop_datanode_dir"
mkdir -p "$hadoop_namenode_dir"
# 生成SSH密钥并添加到authorized_keys
echo '========================================='
echo '= ='
echo '= ='
echo '= Now is Configure the SSH ='
echo '= ='
echo '= ='
echo '= --- Utree ='
echo '= ='
echo '========================================='
if [ ! -f "$ssh_key_file" ]; then
mkdir -p "$ssh_key_dir"
ssh-keygen -t rsa -f "$ssh_key_file" -N ""
cat "$ssh_key_file.pub" >> "$ssh_key_dir/authorized_keys"
echo "SSH密钥对已生成并添加到authorized_keys."
else
echo "SSH密钥对已存在,未执行生成步骤."
fi
# 确保Download目录存在
mkdir -p "$download_dir"
echo '========================================='
echo '= ='
echo '= ='
echo '= Now is Configure the Java Runtime ='
echo '= ='
echo '= ='
echo '= --- Utree ='
echo '= ='
echo '========================================='
# 使用wget下载Java文件
wget "$download_url_java" -P "$download_dir" -O "$download_dir/jdk-21_linux-x64_bin.tar.gz"
# 检查Java下载是否成功
if [ $? -eq 0 ]; then
echo "Java 21下载成功,已保存在$download_dir 目录下."
# 创建Java安装目录
mkdir -p "$java_install_dir"
# 解压Java文件
tar -xzvf "$download_dir/jdk-21_linux-x64_bin.tar.gz" -C "$java_install_dir"
# 配置Java环境变量
echo "export JAVA_HOME=$java_install_dir/jdk-21+35" >> "$HOME/.bashrc"
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> "$HOME/.bashrc"
# 使配置生效
source "$HOME/.bashrc"
echo "Java 21已安装到$java_install_dir 目录,并已配置到~/.bashrc中."
else
echo "Java 21下载失败."
fi
# 下载和安装Hadoop
# 使用wget下载Hadoop文件
echo '========================================='
echo '= ='
echo '= ='
echo '= Now is Configure the Hadoop. ='
echo '= ='
echo '= ='
echo '= --- Utree ='
echo '= ='
echo '========================================='
wget "$hadoop_download_url" -P "$hadoop_download_dir"
# 检查Hadoop下载是否成功
if [ $? -eq 0 ]; then
echo "Hadoop 3.3.5下载成功,已保存在$download_dir 目录下."
# 创建Hadoop安装目录
mkdir -p "$hadoop_install_dir"
# 解压Hadoop文件
tar -xzvf "$hadoop_download_dir/hadoop-3.3.5.tar.gz" -C "$hadoop_install_dir" --strip-components=1
echo "Hadoop 3.3.5已安装到$hadoop_install_dir 目录."
# 配置core-site.xml
core_site_file="$hadoop_install_dir/etc/hadoop/core-site.xml"
cat <<EOL > "$core_site_file"
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>hadoop.tmp.dir</name>
<value>$hadoop_tmp_dir</value>
<description>A base for other temporary directories.</description>
</property>
<property>
<name>fs.defaultFS</name>
<value>hdfs://localhost:9000</value>
</property>
</configuration>
EOL
# 配置hdfs-site.xml
hdfs_site_file="$hadoop_install_dir/etc/hadoop/hdfs-site.xml"
cat <<EOL > "$hdfs_site_file"
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:$hadoop_namenode_dir</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:$hadoop_datanode_dir</value>
</property>
</configuration>
EOL
echo '========================================='
echo '= ='
echo '= ='
echo '= Now is Configure namenode and ='
echo '= datanode.... ='
echo '= ='
echo '= --- Utree ='
echo '= ='
echo '========================================='
# 配置hadoop-env.sh
hadoop_env_file="$hadoop_install_dir/etc/hadoop/hadoop-env.sh"
echo "export JAVA_HOME=$java_install_dir/jdk-21+35" >> "$hadoop_env_file"
echo "Hadoop配置已完成."
# 格式化HDFS名称节点
$hadoop_install_dir/bin/hdfs namenode -format
echo "Hadoop配置已完成,HDFS名称节点已格式化."
else
echo "Hadoop 3.3.5下载失败."
fi
标签:Hadoop,hadoop,echo,3.3,install,download,安装,dir
From: https://www.cnblogs.com/qiantaosama/p/17764065.html