环境变量
tee /etc/profile.d/hadoopENV.sh <<-'EOF'
HADOOP_HOME=/usr/local/hadoop-3.2.4/
export PATH=$PATH:$HADOOP_HOME/bin
EOF
source /etc/profile
tee /usr/local/hadoop-3.2.4/etc/hadoop/hadoop-env.sh <<-'EOF'
export JAVA_HOME=/usr/local/jdk1.8.0_202
export HDFS_DATANODE_USER=root
export HDFS_NAMENODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
export YARN_NODEMANAGER_USER=root
export YARN_RESOURCEMANAGER_USER=root
export HDFS_ZKFC_USER=root
export HDFS_JOURNALNODE_USER=root
EOF
tee /etc/security/limits.d/hadoop.conf <<-'EOF'
root - nofile 65536
root - nproc 65536
root - memlock unlimited
root - nice -10
EOF
编译工具安装
apt-get update
apt-get -y install build-essential autoconf automake libtool cmake zlib1g-dev pkg-config libsnappy-dev bzip2 libbz2-dev libjansson-dev libxrender-dev libxtst-dev libsasl2-dev libssl-dev
编译snappy
cd /data/setup/
tar xvf snappy-1.1.1.tar.gz
cd snappy-1.1.1 && \
./configure && \
make -j 4 && \
make install
#验证是否成功
ll /usr/local/lib/|grep snappy
protobuf
cd /data/setup/
tar xvf protobuf-2.5.0.tar.gz
cd protobuf-2.5.0 && \
./configure && \
make -j 4 && \
make install && \
ldconfig
#验证
protoc --version
openssl
cd /data/setup/
tar xvf openssl-1.1.1t.tar.gz
cd openssl-1.1.1t
./config
make -j 4
make install
echo "/usr/local/lib" | tee -a /etc/ld.so.conf.d/openssl-1.1.conf
ldconfig
#验证
/usr/local/bin/openssl version
hadoop部署
cd /data/setup/
tar xvf hadoop-3.2.4.tar.gz -C /usr/local/
验证hadoop部署的library的版本
cd /usr/local/hadoop-3.2.4/bin
./hadoop checknative -a
2025-02-27 17:14:49,799 INFO bzip2.Bzip2Factory: Successfully loaded & initialized native-bzip2 library system-native
2025-02-27 17:14:49,804 INFO zlib.ZlibFactory: Successfully loaded & initialized native-zlib library
2025-02-27 17:14:49,843 WARN erasurecode.ErasureCodeNative: ISA-L support is not available in your platform... using builtin-java codec where applicable
2025-02-27 17:14:49,951 INFO nativeio.NativeIO: The native code was built without PMDK support.
Native library checking:
hadoop: true /usr/local/hadoop-3.2.4/lib/native/libhadoop.so.1.0.0
zlib: true /lib/x86_64-linux-gnu/libz.so.1
zstd : false
snappy: true /usr/local/hadoop-3.2.4/lib/native/libsnappy.so.1
lz4: true revision:10301
bzip2: true /lib/x86_64-linux-gnu/libbz2.so.1
openssl: true /usr/local/lib/libcrypto.so
ISA-L: false libhadoop was built without ISA-L support
PMDK: false The native code was built without PMDK support.
2025-02-27 17:14:50,019 INFO util.ExitUtil: Exiting with status 1: ExitException
其他配置
# 在 /etc/pam.d/common-session 中添加：
session required pam_limits.so
# 在 /etc/pam.d/common-session-noninteractive 中添加：
session required pam_limits.so
#重启生效
检查当前限制
用以下命令检查 ulimit 的值：
ulimit -a
重点检查以下项：
max user processes (nproc)
open files (nofile)
real-time priority (rtprio)