Hadoop setup

hadoop.md

sudo apt update
# Install JDK
sudo apt install python-software-properties
sudo add-apt-repository ppa:openjdk-r/ppa
sudo apt-get update
sudo apt install openjdk-11-jdk
sudo apt-get update
java -version

# Create haddop group and user
sudo addgroup hadoop
sudo adduser --ingroup hadoop hduser
sudo adduser hduser sudo

# SSH setup
sudo apt install openssh-server openssh-client
su - hduser
ssh-keygen -t rsa -P ""
cat $HOME/.ssh/id_rsa.pub >> $HOME/.ssh/authorized_keys
chmod 0600 $HOME/.ssh/authorized_keys

# Download Hadoop
cd $HOME
wget https://downloads.apache.org/hadoop/common/hadoop-3.2.2/hadoop-3.2.2-src.tar.gz -O hadoop.tar.gz
sudo tar xzf hadoop.tar.gz
sudo chown -R hduser:hadoop hadoop

# Set environmente variables of Hadoop
cat <<EOT >> .bashrc
#Set HADOOP_HOME
export HADOOP_HOME=$HOME/hadoop
# Add bin/ directory of Hadoop to PATH
export PATH=$PATH:$HADOOP_HOME/bin
EOT

source .bashrc

# Create hadoop temporary directory
sudo mkdir -p  /app/hadoop/tmp
sudo chown -R hduser:hadoop /app/hadoop/tmp
sudo chmod 750 /app/hadoop/tmp

Set JAVA_HOME inside file $HADOOP_HOME/etc/hadoop/hadoop-env.sh to the path of jdk installation in your system

Open the file $HADOOP_HOME/etc/hadoop/core-site.xml and add the following properties between tags <configuration></configuration>

<property>
  <name>hadoop.tmp.dir</name>
  <value>/app/hadoop/tmp</value>
  <description>Parent directory for other temporary directories.</description>
</property>
<property>
  <name>fs.defaultFS </name>
  <value>hdfs://localhost:54310</value>
  <description>The name of the default file system. </description>
</property>