1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
| # ssh master1.etcd.com
# ETCD_ALL_ENDPOINTS=` etcdctl3 --write-out=fields member list | awk '/ClientURL/{printf "%s%s",sep,$3; sep=","}'`
# etcdctl3 --endpoints=$ETCD_ALL_ENDPOINTS endpoint status --write-out=table
+-----------------------------------+------------------+---------+---------+-----------+-----------+------------+
| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | RAFT TERM | RAFT INDEX |
+-----------------------------------+------------------+---------+---------+-----------+-----------+------------+
| https://master1.etcd.com:2379 | d91b1c20df818655 | 3.2.22 | 17 MB | true | 6 | 42 |
| https://10.0.88.33:2379 | d35cfd2fedc078f | 3.2.22 | 17 MB | false | 6 | 42 |
| https://10.0.88.22:2379 | c9624828ed10ae36 | 3.2.22 | 17 MB | false | 6 | 42 |
| https://10.0.88.11:2379 | d91b1c20df818655 | 3.2.22 | 17 MB | true | 6 | 42 |
+-----------------------------------+------------------+---------+---------+-----------+-----------+------------+
# etcdctl3 snapshot save /var/lib/etcd/snapshot.db
# cp /var/lib/etcd/snapshot.db /tmp/snapshot.db
# cp /var/lib/etcd/member/snap/db /tmp/db
# scp /tmp/snapshot.db master2.etcd.com:/tmp/snapshot.db
# scp /tmp/snapshot.db master3.etcd.com:/tmp/snapshot.db
# mkdir -p /etc/origin/node/pods-stopped/
# mv /etc/origin/node/pods/* /etc/origin/node/pods-stopped/
# mv /var/lib/etcd/member /tmp/etcd-backup-$(date +%d-%m-%y)
# rm -rf /var/lib/etcd
# ssh master2.etcd.com
# mkdir -p /etc/origin/node/pods-stopped/
# mv /etc/origin/node/pods/* /etc/origin/node/pods-stopped/
# mv /var/lib/etcd/member /tmp/etcd-backup-$(date +%d-%m-%y)
# rm -rf /var/lib/etcd
# ssh master3.etcd.com
# mkdir -p /etc/origin/node/pods-stopped/
# mv /etc/origin/node/pods/* /etc/origin/node/pods-stopped/
# mv /var/lib/etcd/member /tmp/etcd-backup-$(date +%d-%m-%y)
# rm -rf /var/lib/etcd
# ssh master1.etcd.com
# source /etc/etcd/etcd.conf
# export ETCDCTL_API=3
# echo -e "$ETCD_INITIAL_CLUSTER \n$ETCD_INITIAL_CLUSTER_TOKEN"
master1.etcd.com=https://10.0.88.11:2380,master2.etcd.com=https://10.0.88.22:2380,master3.etcd.com=https://10.0.88.33:2380
etcd-cluster-1
# ETCDCTL_API=3 etcdctl snapshot restore /tmp/snapshot.db \
--name master1.etcd.com \
--initial-cluster master1.etcd.com=https://10.0.88.11:2380,master2.etcd.com=https://10.0.88.22:2380,master3.etcd.com=https://10.0.88.33:2380 \
--initial-cluster-token etcd-cluster-1 \
--initial-advertise-peer-urls https://10.0.88.11:2380 \
--data-dir /var/lib/etcd
2019-02-05 12:49:04.103233 I | mvcc: restore compact to 2361744
2019-02-05 12:49:04.135995 I | etcdserver/membership: added member d35cfd2fedc078f [https://10.0.88.33:2380] to cluster 1a196dd3442fbe59
2019-02-05 12:49:04.136161 I | etcdserver/membership: added member c9624828ed10ae36 [https://10.0.88.22:2380] to cluster 1a196dd3442fbe59
2019-02-05 12:49:04.136267 I | etcdserver/membership: added member d91b1c20df818655 [https://10.0.88.11:2380] to cluster 1a196dd3442fbe59
# restorecon -Rv /var/lib/etcd
# ssh master2.etcd.com
# ETCDCTL_API=3 etcdctl snapshot restore snapshot.db \
--name master2.etcd.com \
--initial-cluster master1.etcd.com=https://10.0.88.11:2380,master2.etcd.com=https://10.0.88.22:2380,master3.etcd.com=https://10.0.88.33:2380 \
--initial-cluster-token etcd-cluster-1 \
--initial-advertise-peer-urls https://10.0.88.22:2380 \
--data-dir /var/lib/etcd
2019-02-05 12:51:25.179801 I | mvcc: restore compact to 2356950
2019-02-05 12:51:25.193709 I | etcdserver/membership: added member d35cfd2fedc078f [https://10.0.88.33:2380] to cluster 1a196dd3442fbe59
2019-02-05 12:51:25.193745 I | etcdserver/membership: added member c9624828ed10ae36 [https://10.0.88.22:2380] to cluster 1a196dd3442fbe59
2019-02-05 12:51:25.193759 I | etcdserver/membership: added member d91b1c20df818655 [https://10.0.88.11:2380] to cluster 1a196dd3442fbe59
# restorecon -Rv /var/lib/etcd
# ssh master3.etcd.com
# ETCDCTL_API=3 etcdctl snapshot restore snapshot.db \
--name master3.etcd.com \
--initial-cluster master1.etcd.com=https://10.0.88.11:2380,master2.etcd.com=https://10.0.88.22:2380,master3.etcd.com=https://10.0.88.33:2380 \
--initial-cluster-token etcd-cluster-1 \
--initial-advertise-peer-urls https://10.0.88.33:2380 \
--data-dir /var/lib/etcd
2019-02-05 12:53:06.612149 I | mvcc: restore compact to 2356950
2019-02-05 12:53:06.634761 I | etcdserver/membership: added member d35cfd2fedc078f [https://10.0.88.33:2380] to cluster 1a196dd3442fbe59
2019-02-05 12:53:06.634905 I | etcdserver/membership: added member c9624828ed10ae36 [https://10.0.88.22:2380] to cluster 1a196dd3442fbe59
2019-02-05 12:53:06.635001 I | etcdserver/membership: added member d91b1c20df818655 [https://10.0.88.11:2380] to cluster 1a196dd3442fbe59
# restorecon -Rv /var/lib/etcd
# ssh master1.etcd.com
# mv /etc/origin/node/pods-stopped/etcd.yaml /etc/origin/node/pods/
# ssh master2.etcd.com
# mv /etc/origin/node/pods-stopped/etcd.yaml /etc/origin/node/pods/
# ssh master3.etcd.com
# mv /etc/origin/node/pods-stopped/etcd.yaml /etc/origin/node/pods/
# ssh master1.etcd.com
# ETCD_ALL_ENDPOINTS=` etcdctl3 --write-out=fields member list | awk '/ClientURL/{printf "%s%s",sep,$3; sep=","}'`
# etcdctl3 --endpoints=$ETCD_ALL_ENDPOINTS endpoint status --write-out=table
+-----------------------------------+------------------+---------+---------+-----------+-----------+------------+
| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | RAFT TERM | RAFT INDEX |
+-----------------------------------+------------------+---------+---------+-----------+-----------+------------+
| https://master1.etcd.com:2379 | d91b1c20df818655 | 3.2.22 | 17 MB | true | 6 | 42 |
| https://10.0.88.33:2379 | d35cfd2fedc078f | 3.2.22 | 17 MB | false | 6 | 42 |
| https://10.0.88.22:2379 | c9624828ed10ae36 | 3.2.22 | 17 MB | false | 6 | 42 |
| https://10.0.88.11:2379 | d91b1c20df818655 | 3.2.22 | 17 MB | true | 6 | 42 |
+-----------------------------------+------------------+---------+---------+-----------+-----------+------------+
# ssh master1.etcd.com
# mv /etc/origin/node/pods-stopped/* /etc/origin/node/pods/
# ssh master2.etcd.com
# mv /etc/origin/node/pods-stopped/* /etc/origin/node/pods/
# ssh master3.etcd.com
# mv /etc/origin/node/pods-stopped/* /etc/origin/node/pods/
# oc get nodes,pods -n kube-system
|
