Configurando o Hadoop Namenode no Ubuntu 16.04

Dando continuidade na configuração do Cluster Hadoop, vamos configurar alguns servicos de nosso lab.

Editando o hadoop-env.sh e alterar conforme abaixo:

$ vi $HADOOP_HOME/etc/hadoop/hadoop-env.sh
Alterar de
[...]
# The java implementation to use.
export JAVA_HOME=${JAVA_HOME}
[...]
para
[...]
# The java implementation to use.
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-powerpc
[...]

Adicionar ao final do arquivo as linhas
[...]
export HADOOP_HOME_WARN_SUPPRESS=1
export HADOOP_ROOT_LOGGER="WARN,DRFA"

Configuracao do core-site.xml

$ cat > $HADOOP_HOME/etc/hadoop/core-site.xml << "EOF"
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->
<configuration>
   <property>
      <name>fs.defaultFS</name>
      <value>hdfs://namenode01:8020/</value>
   </property>
   <property>
      <name>io.file.buffer.size</name>
      <value>131072</value>
   </property>
</configuration>
EOF

Configuração do mapred-site.xml

$ cat > $HADOOP_HOME/etc/hadoop/mapred-site.xml << "EOF"
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  ou may not use this file except in compliance with the License.
  You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->
<configuration>
   <property>
      <name>mapreduce.framework.name</name>
      <value>yarn</value>
   </property>
   <property>
      <name>mapreduce.jobhistory.address</name>
      <value>namenode01:10020</value>
   </property>
   <property>
      <name>mapreduce.jobhistory.webapp.address</name>
      <value>namenode01:19888</value>
   </property>
   <property>
      <name>yarn.app.mapreduce.am.staging-dir</name>
      <value>/user/app</value>
   </property>
   <property>
      <name>mapred.child.java.opts</name>
      <value>-Djava.security.egd=file:/dev/../dev/urandom</value>
   </property>
</configuration>
EOF

Configuração do hdfs-site.xml

$ cat > $HADOOP_HOME/etc/hadoop/hdfs-site.xml << "EOF"
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->
<configuration>
   <property>
       <name>dfs.namenode.name.dir</name>
       <value>file:/srv/hadoop/hdfs/namenode</value>
    </property>
    <property>
       <name>dfs.datanode.data.dir</name>
       <value>file:/srv/hadoop/hdfs/datanode</value>
    </property>
    <property>
       <name>dfs.namenode.checkpoint.dir</name>
       <value>file:/srv/hadoop/hdfs/namesecondary</value>
    </property>
    <property>
       <name>dfs.replication</name>
       <value>3</value>
    </property>
    <property>
       <name>dfs.block.size</name>
       <value>134217728</value>
    </property>
</configuration>
EOF

Configuração do yarn-site.xml

$ cat > $HADOOP_HOME/etc/hadoop/yarn-site.xml << "EOF"
<?xml version="1.0"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<configuration>
<!-- Site specific YARN configuration properties -->
   <property>
      <name>yarn.resourcemanager.hostname</name>
      <value>namenode01</value>
   </property>
   <property>
      <name>yarn.resourcemanager.bind-host</name>
      <value>0.0.0.0</value>
   </property>
   <property>
      <name>yarn.nodemanager.bind-host</name>
      <value>0.0.0.0</value>
   </property>
   <property>
      <name>yarn.nodemanager.aux-services</name>
      <value>mapreduce_shuffle</value>
   </property>
   <property>
      <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
      <value>org.apache.hadoop.mapred.ShuffleHandler</value>
   </property>
   <property>
      <name>yarn.log-aggregation-enable</name>
      <value>true</value>
   </property>
   <property>
      <name>yarn.nodemanager.local-dirs</name>
      <value>file:/srv/hadoop/yarn/local</value>
   </property>
   <property>
      <name>yarn.nodemanager.log-dirs</name>
      <value>file:/srv/hadoop/yarn/log</value>
   </property>
   <property>
      <name>yarn.nodemanager.remote-app-log-dir</name>
      <value>hdfs://namenode01:8020/var/log/hadoop-yarn/apps</value>
   </property>
</configuration>
EOF

Configurando o master

$ cat > $HADOOP_HOME/etc/hadoop/masters << "EOF"
namenode01
EOF

Formatando o namenode

$ mkdir -p $HADOOP_HOME/hdfs/namenode
$ mkdir -p $HADOOP_HOME/hdfs/namesecondary
$ mkdir -p $HADOOP_HOME/hdfs/namesecondary
$ $HADOOP_HOME/bin/hadoop namenode -format
DEPRECATED: Use of this script to execute hdfs command is deprecated.
Instead use the hdfs command for it.

Formatting using clusterid: CID-1a1cf0ea-d2cd-4950-a17d-6009631a029b
$

 

Douglas Ribas de Mattos
E-mail: douglasmattos0@gmail.comdouglasmattos0@gmail.com
Github: https://github.com/douglasmattos0
LinkedIn: https://www.linkedin.com/in/douglasmattos0/

Deixe uma resposta

O seu endereço de e-mail não será publicado. Campos obrigatórios são marcados com *