Have the good day
Have the good day

目录

用于启动/关停 Hadoop 生态集群的 Shell 脚本

 

用于启动/关停 Hadoop 生态集群的 Shell 脚本

 


用于启动、关停 Hadoop 集群及 Hadoop 生态下的各种软件,提高效率

 


#!/bin/sh
# all.sh —— 20210914

hosts="centos7-1 centos7-2 centos7-3";
date=$(date "+%Y%m%d")
#echo ${date}

# status0,简单状态查看
function status0(){
    echo "___查看状态";
    for host in $hosts
    do
      	echo -e "\n___${host}_status";
        ssh $host "source /etc/profile; $ZK_HOME/bin/zkServer.sh status; echo ' '; jps";
    done
    exit;
}

# status1,全部状态查看
function status1(){
    echo "___查看状态";
    ssh centos7-1 "source /etc/profile; \
        hdfs haadmin -getAllServiceState; \
        hdfs dfsadmin -report";
#    ssh centos7-3 "source /etc/profile; yarn rmadmin -getAllServiceState";
#低版本不支持这条命令
    ssh centos7-3 "source /etc/profile; yarn node -list -all";

    echo " ";
    ssh centos7-1 "ps -ef | grep hive";
    echo " ";
    ssh centos7-3 "ps -ef | grep hive";
    echo " ";
    ssh centos7-2 "ps -ef | grep hue";

    echo " ";
    for host in $hosts
    do
        ssh $host "service impala-server status";
    done
    ssh $host "service impala-catalog status; service impala-state-store status;";

    echo " ";
    status0;
    exit;
}

# start,启动
function start(){
    echo "___启动集群";
    for host in $hosts
    do
        ssh $host "source /etc/profile; $ZK_HOME/bin/zkServer.sh start";
    done
    sleep 5s;

    ssh centos7-1 "source /etc/profile; $HADOOP_HOME/sbin/start-dfs.sh";
    ssh centos7-3 "source /etc/profile; $HADOOP_HOME/sbin/start-yarn.sh";
    ssh centos7-2 "source /etc/profile; \
        $HADOOP_HOME/sbin/yarn-daemon.sh start resourcemanager; \
        $HADOOP_HOME/sbin/mr-jobhistory-daemon.sh start historyserver";
    sleep 5s;

    ssh centos7-1 "source /etc/profile; \
        screen -dmS hmeta ; \
        sleep 5s ; \
        screen -S hmeta -X stuff '$HIVE_HOME/bin/hive --service metastore >> ~/hmeta_out_${date} ^M'";
    ssh centos7-3 "source /etc/profile; \
        screen -dmS hmeta ; \
        sleep 5s ; \
        screen -S hmeta -X stuff '$HIVE_HOME/bin/hive --service metastore >> ~/hmeta_out_${date} ^M' ; \
        sleep 5s ; \
        screen -dmS h2 ; \
        sleep 5s ; \
        screen -S h2 -X stuff '$HIVE_HOME/bin/hive --service hiveserver2 >> ~/h2_out_${date} ^M'";
    sleep 5s;

    ssh centos7-2 "source /etc/profile; \
        screen -dmS hue ; \
        sleep 5s ; \
        screen -S hue -X stuff '/opt/lagou/servers/hue/build/env/bin/supervisor >> ~/hue_out_${date} ^M'";

    ssh centos7-3 "service impala-state-store start; \
        sleep 5s; \
        service impala-catalog start; \
        sleep 5s; \
        service impala-server start";
    ssh centos7-1 "service impala-server start";
    ssh centos7-2 "service impala-server start";

    ssh centos7-1 "source /etc/profile; $HBASE_HOME/bin/start-hbase.sh";

    ssh centos7-1 "source /etc/profile; \
        cd /opt/lagou/servers/azkaban-exec-server-3.85.1/; \
        bash ./bin/start-exec.sh && \
        sleep 5s; \
        curl -G \"centos7-1:12321/executor?action=activate\" && echo";
    ssh centos7-3 "source /etc/profile; \
        cd /opt/lagou/servers/azkaban-exec-server-3.85.1/; \
        bash ./bin/start-exec.sh && \
        sleep 5s; \
        curl -G \"centos7-3:12321/executor?action=activate\" && echo";
    ssh centos7-2 "source /etc/profile; \
        cd /opt/lagou/servers/azkaban-web-server-3.85.1; \
        bash ./bin/start-web.sh ";

    status1;
    exit;
}

# stop,关停
function stop(){
    echo "___关闭集群";

    ssh centos7-2 "bash /opt/lagou/servers/azkaban-web-server-3.85.1/bin/shutdown-web.sh";
    sleep 5s;
    ssh centos7-3 "bash /opt/lagou/servers/azkaban-exec-server-3.85.1/bin/shutdown-exec.sh";
    ssh centos7-1 "bash /opt/lagou/servers/azkaban-exec-server-3.85.1/bin/shutdown-exec.sh";

    ssh centos7-1 "source /etc/profile; $HBASE_HOME/bin/stop-hbase.sh";
    
    ssh centos7-1 "service impala-server stop";
    ssh centos7-2 "service impala-server stop";
    ssh centos7-3 "service impala-server stop; \
        sleep 5s; \
        service impala-catalog stop; \
        sleep 5s; \
        service impala-state-store stop";
    
    ssh centos7-2 "screen -S hue -X stuff '^C'; sleep 5s; screen -X -S hue quit";
    sleep 5s;

    ssh centos7-3 "screen -S h2 -X stuff '^C'; \
        sleep 5s; \
        screen -X -S h2 quit; \
        screen -S hmeta -X stuff '^C'\
        sleep 5s; \
        screen -X -S hmeta quit";
    ssh centos7-1 "screen -S hmeta -X stuff '^C' sleep 5s; screen -X -S hmeta quit";
    
    ssh centos7-2 "source /etc/profile; \
        $HADOOP_HOME/sbin/mr-jobhistory-daemon.sh stop historyserver; \
        $HADOOP_HOME/sbin/yarn-daemon.sh stop resourcemanager";
    sleep 5s;
    ssh centos7-3 "source /etc/profile; $HADOOP_HOME/sbin/stop-yarn.sh";
    sleep 5s;
    ssh centos7-1 "source /etc/profile; $HADOOP_HOME/sbin/stop-dfs.sh";
    sleep 5s;
    for host in $hosts
    do
      	ssh $host "source /etc/profile; $ZK_HOME/bin/zkServer.sh stop";
    done
    status0;
    exit;
}


if(($#!=1));then
    echo "___params count error";
    exit;
elif [ "$1" == 'status0' ];then
    status0;
    exit;
elif [ "$1" == 'status1' ];then
    status1;
    exit;
elif [ "$1" == 'start' ];then
    start;
    exit;
elif [ "$1" == 'stop' ];then
    stop;
    exit;
else
    echo "___wrong param";
    exit;
fi

exit;

 


(处理 jps 命令显示异常问题,集群开启后执行一次就可以)

#!/bin/sh
# cleanjps.sh

echo "___cleaning hsperfdata..."

hosts="centos7-1 centos7-2 centos7-3"

for host in $hosts
do
	echo "___${host}"
	ssh $host "source /etc/profile; rm -rf /tmp/hsperfdata_*; jps;"
done

 


# 启动
all.sh start

# 关停
all.sh stop

# 查看状态
all.sh status1

相关内容