用于启动、关停 Hadoop 集群及 Hadoop 生态下的各种软件，提高效率
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
|
#!/bin/sh
# all.sh —— 20210914
hosts="centos7-1 centos7-2 centos7-3";
date=$(date "+%Y%m%d")
#echo ${date}
# status0，简单状态查看
function status0(){
echo "___查看状态";
for host in $hosts
do
echo -e "\n___${host}_status";
ssh $host "source /etc/profile; $ZK_HOME/bin/zkServer.sh status; echo ' '; jps";
done
exit;
}
# status1，全部状态查看
function status1(){
echo "___查看状态";
ssh centos7-1 "source /etc/profile; \
hdfs haadmin -getAllServiceState; \
hdfs dfsadmin -report";
# ssh centos7-3 "source /etc/profile; yarn rmadmin -getAllServiceState";
#低版本不支持这条命令
ssh centos7-3 "source /etc/profile; yarn node -list -all";
echo " ";
ssh centos7-1 "ps -ef | grep hive";
echo " ";
ssh centos7-3 "ps -ef | grep hive";
echo " ";
ssh centos7-2 "ps -ef | grep hue";
echo " ";
for host in $hosts
do
ssh $host "service impala-server status";
done
ssh $host "service impala-catalog status; service impala-state-store status;";
echo " ";
status0;
exit;
}
# start，启动
function start(){
echo "___启动集群";
for host in $hosts
do
ssh $host "source /etc/profile; $ZK_HOME/bin/zkServer.sh start";
done
sleep 5s;
ssh centos7-1 "source /etc/profile; $HADOOP_HOME/sbin/start-dfs.sh";
ssh centos7-3 "source /etc/profile; $HADOOP_HOME/sbin/start-yarn.sh";
ssh centos7-2 "source /etc/profile; \
$HADOOP_HOME/sbin/yarn-daemon.sh start resourcemanager; \
$HADOOP_HOME/sbin/mr-jobhistory-daemon.sh start historyserver";
sleep 5s;
ssh centos7-1 "source /etc/profile; \
screen -dmS hmeta ; \
sleep 5s ; \
screen -S hmeta -X stuff '$HIVE_HOME/bin/hive --service metastore >> ~/hmeta_out_${date} ^M'";
ssh centos7-3 "source /etc/profile; \
screen -dmS hmeta ; \
sleep 5s ; \
screen -S hmeta -X stuff '$HIVE_HOME/bin/hive --service metastore >> ~/hmeta_out_${date} ^M' ; \
sleep 5s ; \
screen -dmS h2 ; \
sleep 5s ; \
screen -S h2 -X stuff '$HIVE_HOME/bin/hive --service hiveserver2 >> ~/h2_out_${date} ^M'";
sleep 5s;
ssh centos7-2 "source /etc/profile; \
screen -dmS hue ; \
sleep 5s ; \
screen -S hue -X stuff '/opt/lagou/servers/hue/build/env/bin/supervisor >> ~/hue_out_${date} ^M'";
ssh centos7-3 "service impala-state-store start; \
sleep 5s; \
service impala-catalog start; \
sleep 5s; \
service impala-server start";
ssh centos7-1 "service impala-server start";
ssh centos7-2 "service impala-server start";
ssh centos7-1 "source /etc/profile; $HBASE_HOME/bin/start-hbase.sh";
ssh centos7-1 "source /etc/profile; \
cd /opt/lagou/servers/azkaban-exec-server-3.85.1/; \
bash ./bin/start-exec.sh && \
sleep 5s; \
curl -G \"centos7-1:12321/executor?action=activate\" && echo";
ssh centos7-3 "source /etc/profile; \
cd /opt/lagou/servers/azkaban-exec-server-3.85.1/; \
bash ./bin/start-exec.sh && \
sleep 5s; \
curl -G \"centos7-3:12321/executor?action=activate\" && echo";
ssh centos7-2 "source /etc/profile; \
cd /opt/lagou/servers/azkaban-web-server-3.85.1; \
bash ./bin/start-web.sh ";
status1;
exit;
}
# stop，关停
function stop(){
echo "___关闭集群";
ssh centos7-2 "bash /opt/lagou/servers/azkaban-web-server-3.85.1/bin/shutdown-web.sh";
sleep 5s;
ssh centos7-3 "bash /opt/lagou/servers/azkaban-exec-server-3.85.1/bin/shutdown-exec.sh";
ssh centos7-1 "bash /opt/lagou/servers/azkaban-exec-server-3.85.1/bin/shutdown-exec.sh";
ssh centos7-1 "source /etc/profile; $HBASE_HOME/bin/stop-hbase.sh";
ssh centos7-1 "service impala-server stop";
ssh centos7-2 "service impala-server stop";
ssh centos7-3 "service impala-server stop; \
sleep 5s; \
service impala-catalog stop; \
sleep 5s; \
service impala-state-store stop";
ssh centos7-2 "screen -S hue -X stuff '^C'; sleep 5s; screen -X -S hue quit";
sleep 5s;
ssh centos7-3 "screen -S h2 -X stuff '^C'; \
sleep 5s; \
screen -X -S h2 quit; \
screen -S hmeta -X stuff '^C'\
sleep 5s; \
screen -X -S hmeta quit";
ssh centos7-1 "screen -S hmeta -X stuff '^C' sleep 5s; screen -X -S hmeta quit";
ssh centos7-2 "source /etc/profile; \
$HADOOP_HOME/sbin/mr-jobhistory-daemon.sh stop historyserver; \
$HADOOP_HOME/sbin/yarn-daemon.sh stop resourcemanager";
sleep 5s;
ssh centos7-3 "source /etc/profile; $HADOOP_HOME/sbin/stop-yarn.sh";
sleep 5s;
ssh centos7-1 "source /etc/profile; $HADOOP_HOME/sbin/stop-dfs.sh";
sleep 5s;
for host in $hosts
do
ssh $host "source /etc/profile; $ZK_HOME/bin/zkServer.sh stop";
done
status0;
exit;
}
if(($#!=1));then
echo "___params count error";
exit;
elif [ "$1" == 'status0' ];then
status0;
exit;
elif [ "$1" == 'status1' ];then
status1;
exit;
elif [ "$1" == 'start' ];then
start;
exit;
elif [ "$1" == 'stop' ];then
stop;
exit;
else
echo "___wrong param";
exit;
fi
exit;
|
（处理 jps 命令显示异常问题，集群开启后执行一次就可以）
1
2
3
4
5
6
7
8
9
10
11
12
|
#!/bin/sh
# cleanjps.sh
echo "___cleaning hsperfdata..."
hosts="centos7-1 centos7-2 centos7-3"
for host in $hosts
do
echo "___${host}"
ssh $host "source /etc/profile; rm -rf /tmp/hsperfdata_*; jps;"
done
|
1
2
3
4
5
6
7
8
|
# 启动
all.sh start
# 关停
all.sh stop
# 查看状态
all.sh status1
|
