docker 安装hbase修改配置文件 docker 安装hadoop

groupadd hadoop
useradd -g hadoop hdfs
passwd hdfs <<EOF
hdfs
hdfs
EOF
useradd -g hadoop yarn
passwd yarn <<EOF
yarn
yarn
EOF
useradd -g hadoop zookeeper
passwd zookeeper <<EOF
zookeeper
zookeeper
EOF
useradd -g hadoop hive
passwd hive <<EOF
hive
hive
EOF
useradd -g hadoop hbase
passwd hbase <<EOF
hbase
hbase
EOF
echo user added!
 

echo "127.0.0.1 localhost localhost">/etc/hosts
echo "172.17.0.6 9321a27a2b91 hadoop1">>/etc/hosts
echo "172.17.0.7 7c3a3c9cd595 hadoop2">>/etc/hosts
echo "172.17.0.8 f89eaf2a2548 hadoop3">>/etc/hosts
echo "172.17.0.9 28620eee1426 hadoop4">>/etc/hosts
echo "172.17.0.10 ae1f06bd04c8 hadoop5">>/etc/hosts
echo "172.17.0.11 11c433a003b6 hadoop6">>/etc/hosts

su hdfs
ssh-copy-id -i ~/.ssh/id_rsa.pub 172.17.0.6
ssh-copy-id -i ~/.ssh/id_rsa.pub 172.17.0.7
ssh-copy-id -i ~/.ssh/id_rsa.pub 172.17.0.8
ssh-copy-id -i ~/.ssh/id_rsa.pub 172.17.0.9
ssh-copy-id -i ~/.ssh/id_rsa.pub 172.17.0.10
ssh-copy-id -i ~/.ssh/id_rsa.pub 172.17.0.11
 

[hdfs@9321a27a2b91 root]$ ulimit -a
core file size (blocks,-c) unlimited
data seg size (kbytes,-d) unlimited
scheduling priority (-e)0
file size (blocks,-f) unlimited
pending signals (-i)95612
max locked memory (kbytes,-l)64
max memory size (kbytes,-m) unlimited
open files (-n)65536
pipe size (512 bytes,-p)8
POSIX message queues (bytes,-q)819200
real-time priority (-r)0
stack size (kbytes,-s)8192
cpu time (seconds,-t) unlimited
max user processes (-u)1024
virtual memory (kbytes,-v) unlimited
file locks (-x) unlimited

hdfs hard nfile 65536
hdfs soft nfile 65536
yarn hard nfile 65536
yarn soft nfile 65536
......

[root@9321a27a2b91 ~]#mkdir /usr/local/java
[root@9321a27a2b91 ~]#cp jdk-8u121-linux-x64.tar.gz /usr/local/java/
[root@9321a27a2b91 ~]#chown -R hdfs:hadoop /usr/local/java/
[root@9321a27a2b91 ~]#su hdfs
[hdfs@9321a27a2b91 root]$ cd /usr/local/java/jdk-8u121-linux-x64.tar.gz
[hdfs@9321a27a2b91 java]$ tar -zxvf jdk-8u121-linux-x64.tar.gz

mkdir /usr/local/java
chown hdfs:hadoop /usr/local/java
su hdfs
scp -r hdfs@hadoop1:/usr/local/java/jdk1.8.0_121 /usr/local/java

[root@9321a27a2b91 ~]# mkdir /opt/hadoop
[root@9321a27a2b91 ~]# chown hdfs:hadoop hadoop-2.7.3.tar.gz
[root@9321a27a2b91 ~]# chown hdfs:hadoop /opt/hadoop
[root@9321a27a2b91 ~]# cp hadoop-2.7.3.tar.gz /opt/hadoop
[root@9321a27a2b91 ~]# su hdfs
[hdfs@9321a27a2b91 root]$ cd /opt/hadoop/
[hdfs@9321a27a2b91 hadoop]$ tar -zxvf hadoop-2.7.3.tar.gz

#本子网内主机都可以同步
restrict 172.17.0.0 mask 255.255.0.0 nomodify
#优先时间服务器
server 172.17.0.10 prefer
#日志文件位置
logfile /var/log/ntp.log

[root@9321a27a2b91 hadoop]# service ntpd start
Starting ntpd:[ OK ]
[root@9321a27a2b91 hadoop]# service ntpd status
ntpd dead but pid file exists

3Apr11:20:08 ntpd[732]: ntp_io: estimated max descriptors:65536, initial socket boundary:16
3Apr11:20:08 ntpd[732]:Listen and drop on 0 v4wildcard 0.0.0.0 UDP 123
3Apr11:20:08 ntpd[732]:Listen and drop on 1 v6wildcard :: UDP 123
3Apr11:20:08 ntpd[732]:Listen normally on 2 lo 127.0.0.1 UDP 123
3Apr11:20:08 ntpd[732]:Listen normally on 3 eth0 172.17.0.10 UDP 123
3Apr11:20:08 ntpd[732]:Listen normally on 4 lo ::1 UDP 123
3Apr11:20:08 ntpd[732]:Listen normally on 5 eth0 fe80::42:acff:fe11:a UDP 123
3Apr11:20:08 ntpd[732]:Listening on routing socket on fd #22 for interface updates
3Apr11:20:08 ntpd[732]:0.0.0.0 c016 06 restart
3Apr11:20:08 ntpd[732]: ntp_adjtime() failed:Operation not permitted
3Apr11:20:08 ntpd[732]:0.0.0.0 c012 02 freq_set kernel 0.000 PPM
3Apr11:20:08 ntpd[732]:0.0.0.0 c011 01 freq_not_set
3Apr11:20:08 ntpd[732]: cap_set_proc() failed to drop root privileges:Operation not permitted

OPTIONS="-u ntp:ntp -p /var/run/ntpd.pid -g"
echo "# Drop root to id 'ntp:ntp' by default.">/etc/sysconfig/ntpd
echo "#OPTIONS="-u ntp:ntp -p /var/run/ntpd.pid -g" ">>/etc/sysconfig/ntpd

[root@9321a27a2b91 hadoop]# service ntpd start
Starting ntpd:[ OK ]
[root@9321a27a2b91 hadoop]# service ntpd status
ntpd (pid 796) is running..

su hdfs
vi ~.bash_profile
JAVA_HOME=/usr/local/java/jdk1.8.0_121
CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib:$CLASS
HADOOP_HOME=/opt/hadoop/hadoop-2.7.3
HADOOP_PREFIX=/opt/hadoop/hadoop-2.7.3
HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
HADOOP_YARN_HOME=$HADOOP_HOME
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_HOME/jre/lib/amd64/server
PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
export PATH

export JAVA_HOME=/usr/local/java/jdk1.8.0_121
export HADOOP_HOME=/opt/hadoop/hadoop-2.7.3
#hadoop进程的最大heapsize包括namenode/datanode/ secondarynamenode等,默认1000M
#export HADOOP_HEAPSIZE=
#namenode的初始heapsize,默认取上面的值,按需要分配
#export HADOOP_NAMENODE_INIT_HEAPSIZE=""
#JVM启动参数,默认为空
export HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true"
#还可以单独配置各个组件的内存:
export HADOOP_NAMENODE_OPTS=
export HADOOP_DATANODE_OPTS
export HADOOP_SECONDARYNAMENODE_OPTS
#设置hadoop日志,默认是$HADOOP_HOME/log
export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER

export JAVA_HOME=/usr/local/java/jdk1.8.0_121
JAVA_HEAP_MAX=-Xmx1000m
# YARN_HEAPSIZE=1000 #yarn 守护进程heapsize
#export YARN_RESOURCEMANAGER_HEAPSIZE=1000 #单独设置RESOURCEMANAGER的HEAPSIZE
#export YARN_TIMELINESERVER_HEAPSIZE=1000 #单独设置TIMELINESERVER(jobhistoryServer)的HEAPSIZE
#export YARN_RESOURCEMANAGER_OPTS= #单独设置RESOURCEMANAGER的JVM选项
#export YARN_NODEMANAGER_HEAPSIZE=1000 #单独设置NODEMANAGER的HEAPSIZE
#export YARN_NODEMANAGER_OPTS= #单独设置NODEMANAGER的JVM选项

<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://hadoop1:9000</value>
             <description>HDFS 端口</description>
</property>
<property>
<name>io.file.buffer.size</name>
<value>131072</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/opt/hadoop/hadoop-2.7.3/tmp</value>
<description>默认值/tmp/hadoop-${user.name},修改成持久化的目录</description>
</property>
</configuration>

<configuration>
<property>
<name>dfs.replication</name>
<value>3</value>              
               <description>数据块的备份数量</description>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>/opt/hadoop/hadoop-2.7.3/namenodedir</value>
               <description>保存namenode元数据的目录,要自己创建</description>
</property>
<property>
<name>dfs.blocksize</name>
<value>134217728</value>
<description>数据块大小,128M</description>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>/opt/hadoop/hadoop-2.7.3/datadir</value>
           <description>datanode 数据目录</description>
</property>
</configuration>

<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
              <description>使用yarn来管理mr</description>
</property>
<property>
<name>mapreduce.jobhistory.address</name>
<value>hadoop2</value>
</property>
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>hadoop2</value>
</property>
<property>
<name>mapreduce.jobhistory.intermediate-done-dir</name>
<value>/opt/hadoop/hadoop-2.7.3/mrHtmp</value>
</property>
<property>
<name>mapreduce.jobhistory.done-dir</name>
<value>/opt/hadoop/hadoop-2.7.3/mrhHdone</value>
</property>
</configuration>

<configuration>
<!-- Site specific YARN configuration properties -->
<property>
<name>yarn.resourcemanager.hostname</name>
<value>hadoop1</value>
<description>设置resourcemanager节点</description>
</property>
<!-- Site specific YARN configuration properties -->
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
<description>设置nodemanager的aux服务</description>
</property>
</configuration>

vi $HADOOP_HOME/et/hadoop/slaves
hadoop3
hadoop4
hadoop5
hadoop6

mkdir /opt/hadoop
chown hdfs:hadoop /opt/hadoop
su hdfs
scp -r hdfs@hadoop1:/opt/hadoop/hadoop-2.7.3 /opt/hadoop

[hdfs@9321a27a2b91 hadoop-2.7.3]$ start-dfs.sh
Starting namenodes on [hadoop1]
hadoop1: starting namenode, logging to /opt/hadoop/hadoop-2.7.3/logs/hadoop-hdfs-namenode-9321a27a2b91.out
hadoop3: starting datanode, logging to /opt/hadoop/hadoop-2.7.3/logs/hadoop-hdfs-datanode-f89eaf2a2548.out
hadoop4: starting datanode, logging to /opt/hadoop/hadoop-2.7.3/logs/hadoop-hdfs-datanode-28620eee1426.out
hadoop5: starting datanode, logging to /opt/hadoop/hadoop-2.7.3/logs/hadoop-hdfs-datanode-ae1f06bd04c8.out
hadoop6: starting datanode, logging to /opt/hadoop/hadoop-2.7.3/logs/hadoop-hdfs-datanode-11c433a003b6.out
Starting secondary namenodes [0.0.0.0]
0.0.0.0: starting secondarynamenode, logging to /opt/hadoop/hadoop-2.7.3/logs/hadoop-hdfs-secondarynamenode-9321a27a2b91.out

[hdfs@9321a27a2b91 hadoop]$ jps
11105 Jps
10981 SecondaryNameNode
10777 NameNode

[hdfs@9321a27a2b91 hadoop-2.7.3]$ hdfs dfs -put NOTICE.txt /
[hdfs@9321a27a2b91 hadoop-2.7.3]$ hdfs dfs -ls /
Found 1 items
-rw-r--r-- 3 hdfs supergroup 14978 2017-04-03 19:15 /NOTICE.txt

[root@9321a27a2b91 hdfs]# curl hadoop1:50070
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
................

[hdfs@9321a27a2b91 hadoop]$ start-yarn.sh
starting yarn daemons
starting resourcemanager, logging to /opt/hadoop/hadoop-2.7.3/logs/yarn-hdfs-resourcemanager-9321a27a2b91.out
hadoop5: starting nodemanager, logging to /opt/hadoop/hadoop-2.7.3/logs/yarn-hdfs-nodemanager-ae1f06bd04c8.out
hadoop6: starting nodemanager, logging to /opt/hadoop/hadoop-2.7.3/logs/yarn-hdfs-nodemanager-11c433a003b6.out
hadoop3: starting nodemanager, logging to /opt/hadoop/hadoop-2.7.3/logs/yarn-hdfs-nodemanager-f89eaf2a2548.out
hadoop4: starting nodemanager, logging to /opt/hadoop/hadoop-2.7.3/logs/yarn-hdfs-nodemanager-28620eee1426.out

[hdfs@9321a27a2b91 hadoop]$ jps
11105 Jps
10981 SecondaryNameNode
10777 NameNode
10383 ResourceManager

[hdfs@9321a27a2b91 hadoop-2.7.3]$ bin/hdfs dfs -mkdir /user
[hdfs@9321a27a2b91 hadoop-2.7.3]$ bin/hdfs dfs -mkdir /user/hdfs
[hdfs@9321a27a2b91 hadoop-2.7.3]$ bin/hdfs dfs -put etc/hadoop input
...............
17/04/12 12:38:24 INFO mapreduce.JobSubmitter: number of splits:30
17/04/12 12:38:24 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1491968887469_0003
17/04/12 12:38:24 INFO mapreduce.JobSubmitter: Cleaning up the staging area /tmp/hadoop-yarn/staging/hdfs/.staging/job_1491968887469_0003
java.lang.IllegalArgumentException: Does not contain a valid host:port authority: hadoop2
at org.apache.hadoop.net.NetUtils.createSocketAddr(NetUtils.java:213)
........................

[hdfs@9321a27a2b91 hadoop]$ start-dfs.sh
Starting namenodes on [9321a27a2b91]
The authenticity of host '9321a27a2b91 (172.17.0.10)' can't be established.
RSA key fingerprint is 60:0c:61:73:2c:49:ef:e3:f7:61:c9:27:93:5a:1d:c7.
Are you sure you want to continue connecting (yes/no)?

[hdfs@11c433a003b6 hadoop-2.7.3]$ $HADOOP_HOME/sbin/hadoop-daemons.sh start datanode
The authenticity of host '28620eee1426 (172.17.0.7)' can't be established.
RSA key fingerprint is 60:0c:61:73:2c:49:ef:e3:f7:61:c9:27:93:5a:1d:c7.
Are you sure you want to continue connecting (yes/no)? The authenticity of host '11c433a003b6 (172.17.0.5)' can't be established.
RSA key fingerprint is 60:0c:61:73:2c:49:ef:e3:f7:61:c9:27:93:5a:1d:c7.
Are you sure you want to continue connecting (yes/no)? The authenticity of host 'ae1f06bd04c8 (172.17.0.6)' can't be established.
RSA key fingerprint is 60:0c:61:73:2c:49:ef:e3:f7:61:c9:27:93:5a:1d:c7.
Are you sure you want to continue connecting (yes/no)? f89eaf2a2548: datanode running as process 5764. Stop it first.

[hdfs@11c433a003b6 hadoop-2.7.3]$ $HADOOP_HOME/sbin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script hdfs start datanode
cat: /opt/hadoop/hadoop-2.7.3/etc/hadoop/slaves: No such file or directory

usage="Usage: hadoop-daemons.sh [--config confdir] [--hosts hostlistfile] [start|stop] command args..."
# if no args specified, show usage
if[ $# -le 1 ]; then
echo $usage
exit 1
fi
bin=`dirname "${BASH_SOURCE-[hdfs@9321a27a2b91 hadoop]$ $HADOOP_PREFIX/sbin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs start namenode
starting namenode, logging to /opt/hadoop/hadoop-2.7.3/logs/hadoop-hdfs-namenode-9321a27a2b91.out}"`
bin=`cd "$bin"; pwd`
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
. $HADOOP_LIBEXEC_DIR/hadoop-config.sh
exec "$bin/slaves.sh"--config $HADOOP_CONF_DIR cd "$HADOOP_PREFIX" \; "$bin/hadoop-daemon.sh"--config $HADOOP_CONF_DIR "$@"

[hdfs@f89eaf2a2548 hadoop-2.7.3]$ $HADOOP_YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR start resourcemanager
starting resourcemanager, logging to /opt/hadoop/hadoop-2.7.3/logs/yarn-hdfs-resourcemanager-f89eaf2a2548.out

[hdfs@9321a27a2b91 hadoop-2.7.3]$ bin/hdfs dfs -put etc/hadoop input
...............
17/04/1212:38:24 INFO mapreduce.JobSubmitter: number of splits:30
17/04/1212:38:24 INFO mapreduce.JobSubmitter:Submitting tokens for job: job_1491968887469_0003
17/04/1212:38:24 INFO mapreduce.JobSubmitter:Cleaning up the staging area /tmp/hadoop-yarn/staging/hdfs/.staging/job_1491968887469_0003
java.lang.IllegalArgumentException:Does not contain a valid host:port authority: hadoop2
at org.apache.hadoop.net.NetUtils.createSocketAddr(NetUtils.java:213)
........................

<property>
<name>mapreduce.jobhistory.address</name>
<value>7c3a3c9cd595</value>
</property>
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>7c3a3c9cd595</value>
</property>

<property>
<name>mapreduce.jobhistory.address</name>
<value>7c3a3c9cd595:10020</value>
</property>
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>7c3a3c9cd595:19888</value>
</property>

2017-04-0319:13:12,328 WARN org.apache.hadoop.hdfs.server.datanode.DataNode:IOExceptionin offerService
java.io.EOFException:End of FileException between local host is:"ae1f06bd04c8/172.17.0.6"; destination host is:"hadoop1":9000;: java.io.EOFException;For more details see: http://wiki.a
pache.org/hadoop/EOFException
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(NativeMethod)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)

四.总结