用于启动/关停 Hadoop 生态集群的 Shell 脚本
目录
用于启动/关停 Hadoop 生态集群的 Shell 脚本
1 用途
用于启动、关停 Hadoop 集群及 Hadoop 生态下的各种软件,提高效率
2 集群启停脚本
#!/bin/sh
# all.sh —— 20210914
hosts="centos7-1 centos7-2 centos7-3";
date=$(date "+%Y%m%d")
#echo ${date}
# status0,简单状态查看
function status0(){
echo "___查看状态";
for host in $hosts
do
echo -e "\n___${host}_status";
ssh $host "source /etc/profile; $ZK_HOME/bin/zkServer.sh status; echo ' '; jps";
done
exit;
}
# status1,全部状态查看
function status1(){
echo "___查看状态";
ssh centos7-1 "source /etc/profile; \
hdfs haadmin -getAllServiceState; \
hdfs dfsadmin -report";
# ssh centos7-3 "source /etc/profile; yarn rmadmin -getAllServiceState";
#低版本不支持这条命令
ssh centos7-3 "source /etc/profile; yarn node -list -all";
echo " ";
ssh centos7-1 "ps -ef | grep hive";
echo " ";
ssh centos7-3 "ps -ef | grep hive";
echo " ";
ssh centos7-2 "ps -ef | grep hue";
echo " ";
for host in $hosts
do
ssh $host "service impala-server status";
done
ssh $host "service impala-catalog status; service impala-state-store status;";
echo " ";
status0;
exit;
}
# start,启动
function start(){
echo "___启动集群";
for host in $hosts
do
ssh $host "source /etc/profile; $ZK_HOME/bin/zkServer.sh start";
done
sleep 5s;
ssh centos7-1 "source /etc/profile; $HADOOP_HOME/sbin/start-dfs.sh";
ssh centos7-3 "source /etc/profile; $HADOOP_HOME/sbin/start-yarn.sh";
ssh centos7-2 "source /etc/profile; \
$HADOOP_HOME/sbin/yarn-daemon.sh start resourcemanager; \
$HADOOP_HOME/sbin/mr-jobhistory-daemon.sh start historyserver";
sleep 5s;
ssh centos7-1 "source /etc/profile; \
screen -dmS hmeta ; \
sleep 5s ; \
screen -S hmeta -X stuff '$HIVE_HOME/bin/hive --service metastore >> ~/hmeta_out_${date} ^M'";
ssh centos7-3 "source /etc/profile; \
screen -dmS hmeta ; \
sleep 5s ; \
screen -S hmeta -X stuff '$HIVE_HOME/bin/hive --service metastore >> ~/hmeta_out_${date} ^M' ; \
sleep 5s ; \
screen -dmS h2 ; \
sleep 5s ; \
screen -S h2 -X stuff '$HIVE_HOME/bin/hive --service hiveserver2 >> ~/h2_out_${date} ^M'";
sleep 5s;
ssh centos7-2 "source /etc/profile; \
screen -dmS hue ; \
sleep 5s ; \
screen -S hue -X stuff '/opt/lagou/servers/hue/build/env/bin/supervisor >> ~/hue_out_${date} ^M'";
ssh centos7-3 "service impala-state-store start; \
sleep 5s; \
service impala-catalog start; \
sleep 5s; \
service impala-server start";
ssh centos7-1 "service impala-server start";
ssh centos7-2 "service impala-server start";
ssh centos7-1 "source /etc/profile; $HBASE_HOME/bin/start-hbase.sh";
ssh centos7-1 "source /etc/profile; \
cd /opt/lagou/servers/azkaban-exec-server-3.85.1/; \
bash ./bin/start-exec.sh && \
sleep 5s; \
curl -G \"centos7-1:12321/executor?action=activate\" && echo";
ssh centos7-3 "source /etc/profile; \
cd /opt/lagou/servers/azkaban-exec-server-3.85.1/; \
bash ./bin/start-exec.sh && \
sleep 5s; \
curl -G \"centos7-3:12321/executor?action=activate\" && echo";
ssh centos7-2 "source /etc/profile; \
cd /opt/lagou/servers/azkaban-web-server-3.85.1; \
bash ./bin/start-web.sh ";
status1;
exit;
}
# stop,关停
function stop(){
echo "___关闭集群";
ssh centos7-2 "bash /opt/lagou/servers/azkaban-web-server-3.85.1/bin/shutdown-web.sh";
sleep 5s;
ssh centos7-3 "bash /opt/lagou/servers/azkaban-exec-server-3.85.1/bin/shutdown-exec.sh";
ssh centos7-1 "bash /opt/lagou/servers/azkaban-exec-server-3.85.1/bin/shutdown-exec.sh";
ssh centos7-1 "source /etc/profile; $HBASE_HOME/bin/stop-hbase.sh";
ssh centos7-1 "service impala-server stop";
ssh centos7-2 "service impala-server stop";
ssh centos7-3 "service impala-server stop; \
sleep 5s; \
service impala-catalog stop; \
sleep 5s; \
service impala-state-store stop";
ssh centos7-2 "screen -S hue -X stuff '^C'; sleep 5s; screen -X -S hue quit";
sleep 5s;
ssh centos7-3 "screen -S h2 -X stuff '^C'; \
sleep 5s; \
screen -X -S h2 quit; \
screen -S hmeta -X stuff '^C'\
sleep 5s; \
screen -X -S hmeta quit";
ssh centos7-1 "screen -S hmeta -X stuff '^C' sleep 5s; screen -X -S hmeta quit";
ssh centos7-2 "source /etc/profile; \
$HADOOP_HOME/sbin/mr-jobhistory-daemon.sh stop historyserver; \
$HADOOP_HOME/sbin/yarn-daemon.sh stop resourcemanager";
sleep 5s;
ssh centos7-3 "source /etc/profile; $HADOOP_HOME/sbin/stop-yarn.sh";
sleep 5s;
ssh centos7-1 "source /etc/profile; $HADOOP_HOME/sbin/stop-dfs.sh";
sleep 5s;
for host in $hosts
do
ssh $host "source /etc/profile; $ZK_HOME/bin/zkServer.sh stop";
done
status0;
exit;
}
if(($#!=1));then
echo "___params count error";
exit;
elif [ "$1" == 'status0' ];then
status0;
exit;
elif [ "$1" == 'status1' ];then
status1;
exit;
elif [ "$1" == 'start' ];then
start;
exit;
elif [ "$1" == 'stop' ];then
stop;
exit;
else
echo "___wrong param";
exit;
fi
exit;
3 清理 /tmp/hsperfdata_* 脚本
(处理 jps 命令显示异常问题,集群开启后执行一次就可以)
#!/bin/sh
# cleanjps.sh
echo "___cleaning hsperfdata..."
hosts="centos7-1 centos7-2 centos7-3"
for host in $hosts
do
echo "___${host}"
ssh $host "source /etc/profile; rm -rf /tmp/hsperfdata_*; jps;"
done
4 使用说明
# 启动
all.sh start
# 关停
all.sh stop
# 查看状态
all.sh status1