搭建Hadoop Yarn环境 (ARM)

[Yikun]

---

date: 2020-08-15

ARM上搭建Hadoop Yarn的指导。

1. 安装依赖

   yum update -y
   yum install -y \
   autoconf \
   automake \
   libtool \
   cmake \
   pkg-config \
   curl \
   sudo \
   git \
   hostname \
   bind-utils \
   openssh-server \
   snappy \
   snappy-devel \
   bzip2 \
   libzip-devel \
   java-1.8.0-openjdk \
   java-1.8.0-openjdk-devel

2. 新增Hadoop用户

   groupadd hadoop
   useradd -m -d /home/hadoop -s /bin/bash hadoop -g hadoop
   echo "hadoop ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers

3. 下载

   curl -L https://downloads.apache.org/hadoop/common/hadoop-3.3.0/hadoop-3.3.0-aarch64.tar.gz | tar zx

4. 环境变量准备

   export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.265.b01-6.oe1.aarch64
   export HADOOP_HOME=/home/hadoop/hadoop-3.3.0

5. 配置添加 cd $HADOOP_HOME/etc/hadoop/ core-site.xml

   <configuration>
   <property>
       <name>fs.defaultFS</name>
       <value>hdfs://localhost:9000</value>
   </property>
   </configuration>

   hdfs-site.xml

   <configuration>
   <property>
       <name>dfs.replication</name>
       <value>1</value>
   </property>
   <property>
       <name>dfs.name.dir</name>
       <value>/home/hadoop/dfs/name</value>
   </property>
   <property>
       <name>dfs.data.dir</name>
       <value>/home/hadoop/dfs/data</value>
   </property>
   </configuration>

   mapred-site.xml

   <configuration>
   <property>
       <name>mapreduce.framework.name</name>
       <value>yarn</value>
   </property>
   <!--Deploy Hadoop on K8s, we need to specify a certain port range and expose by k8s Service-->
   <property>
       <name>yarn.app.mapreduce.am.job.client.port-range</name>
       <value>60000-60010</value>
   </property>
   <property>
       <name>yarn.app.mapreduce.am.webapp.port-range</name>
       <value>60011-60020</value>
   </property>
   <property>
       <name>mapreduce.application.classpath</name>
       <value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
   </property>
   <!--
   <property>
     <name>mapreduce.map.output.compress</name>
     <value>true</value>
   </property>
   <property>
     <name>mapreduce.map.output.compress.codec</name>
     <value>org.apache.hadoop.io.compress.ZStandardCodec</value>
   </property>
   -->
   </configuration>

   yarn-site.xml

   <configuration>
   <property>
       <name>yarn.resourcemanager.hostname</name>
       <value>localhost</value>
   </property>
   <property>
       <name>yarn.nodemanager.address</name>
       <value>0.0.0.0:40555</value>
   </property>
   <property>
       <name>yarn.nodemanager.aux-services</name>
       <value>mapreduce_shuffle</value>
   </property>
   <property>
       <name>yarn.nodemanager.env-whitelist</name>
       <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
   </property>
   <property>
       <name>yarn.log-aggregation-enable</name>
       <value>true</value>
   </property>
   </configuration>

6. 启动服务 先格式化hdfs

   hdfs namenode -format

   启动所有服务

   start-all.sh