1.官网提供snap快照备份恢复

https:/
/www.elastic.co/guide/en/elasticsearch/reference/7.9/snapshot-restore.html

环境要求

要求所有es节点，挂载同一个共享目录，可以使用NFS。


注意背景

[
root@devops01 ~]
#id elasticsearch 
uid=
998(elasticsearch) gid=
996(elasticsearch) groups=
996(elasticsearch)

[
root@devops01 ~]
#

搭建NFS机器，服务端

yum install nfs-utils -y


# 根据es的用户信息，创建挂载用户
groupadd elasticsearch -g 996

useradd elasticsearch -g 996 -u 998 -M -s /sbin/nologin


cat > /etc/exports << 
'EOF'
/es-nfs-data 10.0.0.0/24(rw,
sync,all_squash,anonuid=998,anongid=996)

EOF


# 目录创建授权
[root yc-k8s ~]
#mkdir -p /es-nfs-data
[root yc-k8s ~]
#chown -R elasticsearch.elasticsearch /es-nfs-data/

[root yc-k8s ~]
#ll -d /es-nfs-data/
drwxr-xr-x. 2 elasticsearch elasticsearch 6 Dec 3 02:16 /es-nfs-data/

[root yc-k8s ~]
#


[root yc-k8s ~]
#systemctl restart nfs


[root yc-k8s ~]
#showmount -e 10.0.0.122
Export list 
for 10.0.0.122:

/es-nfs-data 10.0.0.0/24

es节点安装nfs插件

cat > nfs-client.sh <<
'EOF'
yum install nfs-utils -y

mkdir /es-client-data -p

mount -t nfs 10.0.0.122:/es-nfs-data /es-client-data

EOF



sh nfs-client.sh


# 检查挂载
df -h |grep es-client-data

es节点修改config，开启snapshot

每个节点config.yml添加配置

path.repo: /es-client-data/



# 如
[es-node3 root ~]
#cat /etc/elasticsearch/elasticsearch.yml 
cluster.name: yuchao_es

node.name: es-node3

path.data: /var/lib/elasticsearch/

path.logs: /var/log/elasticsearch/

bootstrap.memory_lock: true

network.host:127.0.0.1,
10.0.0.20
http.port:9200
discovery.seed_hosts: [
"10.0.0.18",
"10.0.0.19",
"10.0.0.20"]

cluster.initial_master_nodes: [
"10.0.0.18"]

path.repo: /es-client-data/

[es-node3 root ~]
#

# 重启
systemctl restart elasticsearch.service

确保重启正确

1.先注册快照仓库

PUT /_snapshot/my_backup

{

"type": 
"fs",

"settings": {

"location": 
"/es-client-data/my_backup_location",

"compress": 
true
 }

}


# 可以查询仓库信息
GET /_snapshot/my_backup

2.创建一个快照

PUT /_snapshot/my_backup/snapshot_1?wait_for_completion=
true

检查快照

图解nfs快照

3.针对具体index的快照

# 创建第二个快照，名字snapshot_2 

PUT /_snapshot/my_backup/snapshot_2?wait_for_completion=
true
{

"indices": 
"t1,t2",

"ignore_unavailable": 
true,

"include_global_state": 
false
 }

}

4.查看快照信息


GET /_snapshot


GET /_snapshot/my_backup/



GET /_snapshot/my_backup/snapshot_1


GET /_snapshot/my_backup/snapshot_2

5.查看正在运行的快照

GET /_snapshot/my_backup/_current


{

"snapshots" : [ ]

}

6.演练恢复索引

1. 删除 t2库


2.基于有该t2 库的快照

POST /_snapshot/my_backup/snapshot_2/_restore


3.默认恢复，发现有同名index，导致恢复错误


4.可以单独还原某个index，或者删除es数据，全库恢复。


GET /_snapshot


GET /_snapshot/my_backup/



GET /_snapshot/my_backup/snapshot_1


GET /_snapshot/my_backup/snapshot_2



GET /_snapshot/my_backup/_current


POST /_snapshot/my_backup/snapshot_2/_restore


# 恢复指定的索引
POST /_snapshot/my_backup/snapshot_2/_restore

{

"indices": 
"t2",

"ignore_unavailable": 
true,

"include_global_state": 
false, 

"rename_pattern": 
"index_(.+)", 

"rename_replacement": 
"restored_index_$1", 

"include_aliases": 
false
}


# 参数解释
上面的indices, 表示只恢复索引’index_1’

rename_pattern: 表示重命名索引以’index_’开头的索引.（注意我们这里索引叫做t2，所以没作用）

rename_replacement: 表示将所有的索引重命名为’restored_index_xxx’.如index_1会被重命名为restored_index_1.

指定index恢复

恢复且修改index名


# 如果想修改恢复后的index名字
POST /_snapshot/my_backup/snapshot_2/_restore

{

"indices": 
"t2",

"ignore_unavailable": 
true,

"include_global_state": 
false, 

"rename_pattern": 
"t(.+)", 

"rename_replacement": 
"restored_index_$1", 

"include_aliases": 
false
}

7.用日期命名快照

不建议去用。

PUT /_snapshot/my_backup/<snapshot-{now/d}>

PUT /_snapshot/my_backup/%3Csnapshot-%7Bnow%2Fd%7D%3E

8.记录es练习接口

PUT /_snapshot/my_backup

{

"type": 
"fs",

"settings": {

"location": 
"/es-client-data/my_backup_location",

"compress": 
true
 }

}



GET /_snapshot/my_backup



PUT /_snapshot/my_backup/snapshot_1?wait_for_completion=
true

PUT /_snapshot/my_backup/snapshot_2?wait_for_completion=
true
{

"indices": 
"t1,t2",

"ignore_unavailable": 
true,

"include_global_state": 
false
 }

}



GET /_snapshot/


GET /_snapshot/my_backup/



GET /_snapshot/my_backup/_all



GET /_snapshot/my_backup/snapshot_1


GET /_snapshot/my_backup/snapshot_2



GET /_snapshot/my_backup/_current


POST /_snapshot/my_backup/snapshot_2/_restore


POST /_snapshot/my_backup/snapshot_2/_restore

{

"indices": 
"t2",

"ignore_unavailable": 
true,

"include_global_state": 
false, 

"rename_pattern": 
"index_(.+)", 

"rename_replacement": 
"restored_index_$1", 

"include_aliases": 
false
}



POST /_snapshot/my_backup/snapshot_2/_restore

{

"indices": 
"t2",

"ignore_unavailable": 
true,

"include_global_state": 
false, 

"rename_pattern": 
"t(.+)", 

"rename_replacement": 
"restored_index_$1", 

"include_aliases": 
false
}



PUT /_snapshot/my_backup/%3Csnapshot-%7Bnow%2Fd%7D%3E

2.第三方备份工具

1.安装node环境，基于nodejs开发的备份工具

# 注意node版本要求和于超老师一致

2.软件官网

https:
//www.npmjs.com/
package/elasticdump

2.1 安装node

wget https://nodejs.org/dist/v10.16.3/node-v10.16.3-linux-x64.tar.xz


tar -xf node-v10.16.3-linux-x64.tar.xz 



[root yc-k8s /opt]
#
[root yc-k8s /opt]
#ln -s node-v10.16.3-linux-x64/ node
[root yc-k8s /opt]
#
[root yc-k8s /opt]
#echo 'export PATH=/opt/node/bin:$PATH' >> /etc/profile
[root yc-k8s /opt]
#
[root yc-k8s /opt]
#source /etc/profile
[root yc-k8s /opt]
#
[root yc-k8s /opt]
#node -v
v10.16.3

[root yc-k8s /opt]
#
[root yc-k8s /opt]
#npm -v
6.9.0

[root yc-k8s /opt]
#

# 设置淘宝源
# 于超老师这里测试，腾讯源快一点

npm config 
set registry https://registry.npm.taobao.org

npm config 
set registry http://mirrors.cloud.tencent.com/npm/



npm config get registry


# 安装elasticdump备份工具

[root yc-k8s /opt]
#npm install elasticdump -g

[root yc-k8s /opt]
#elasticdump --version
6.94.1

2.2 备份命令

备份es数据为可读的json文件



[root yc-k8s /opt]
#elasticdump \
> --input=http://10.0.0.18:9200/t1 \

> --output=/es-nfs-data/t1.json \

> --
type=data

Sat, 03 Dec 2022 14:35:45 GMT | starting dump

Sat, 03 Dec 2022 14:35:45 GMT | got 8 objects from 
source elasticsearch (offset: 0)

Sat, 03 Dec 2022 14:35:45 GMT | sent 8 objects to destination file, wrote 8

Sat, 03 Dec 2022 14:35:45 GMT | got 0 objects from 
source elasticsearch (offset: 8)

Sat, 03 Dec 2022 14:35:45 GMT | Total Writes: 8

Sat, 03 Dec 2022 14:35:45 GMT | dump complete

[root yc-k8s /opt]
#


[root yc-k8s /es-nfs-data]
#cat t1.json 
{
"_index":
"t1",
"_type":
"_doc",
"_id":
"7HNg1IQBRcFJVNOeWVxR",
"_score":1,
"_source":{
"name":
"yu1",
"age":
"28",
"address":
"JS",
"job":
"dev"}}

{
"_index":
"t1",
"_type":
"_doc",
"_id":
"7XNg1IQBRcFJVNOeWVyL",
"_score":1,
"_source":{
"name":
"yu2",
"age":
"27",
"address":
"BJ",
"job":
"dev"}}

{
"_index":
"t1",
"_type":
"_doc",
"_id":
"7nNg1IQBRcFJVNOeWVyd",
"_score":1,
"_source":{
"name":
"yu3",
"age":
"26",
"address":
"SD",
"job":
"ops"}}

{
"_index":
"t1",
"_type":
"_doc",
"_id":
"73Ng1IQBRcFJVNOeWVyq",
"_score":1,
"_source":{
"name":
"yu4",
"age":
"25",
"address":
"JX",
"job":
"ops"}}

{
"_index":
"t1",
"_type":
"_doc",
"_id":
"8HNg1IQBRcFJVNOeWVy5",
"_score":1,
"_source":{
"name":
"jack01",
"age":
"19",
"address":
"YN",
"job":
"test"}}

{
"_index":
"t1",
"_type":
"_doc",
"_id":
"8XNg1IQBRcFJVNOeWVzD",
"_score":1,
"_source":{
"name":
"tom02",
"age":
"30",
"address":
"DB",
"job":
"test"}}

{
"_index":
"t1",
"_type":
"_doc",
"_id":
"8nNg1IQBRcFJVNOeWVzN",
"_score":1,
"_source":{
"name":
"david03",
"age":
"30",
"address":
"BJ",
"job":
"test"}}

{
"_index":
"t1",
"_type":
"_doc",
"_id":
"83Ng1IQBRcFJVNOeWVzY",
"_score":1,
"_source":{
"name":
"xiaohei01",
"age":
"17",
"address":
"BJ",
"job":
"ops"}}

[root yc-k8s /es-nfs-data]
#

备份且压缩

elasticdump \

--input=http://10.0.0.18:9200/t2 \
--output=$ \
 | gzip > /es-nfs-
data/t2.json.gz


[root yc-k8s /es-nfs-
data]#gzip -d t2.json.gz 
[root yc-k8s /es-nfs-
data]#ll
total12
drwxr-xr-x. 
3 elasticsearch elasticsearch 
4096Dec304:
50 my_backup_location

-rw-r
--r--. 1 root root 1090 Dec 3 22:35 t1.json
-rw-r
--r--. 1 root root 402 Dec 3 22:37 t2.json
[root yc-k8s /es-nfs-
data]#cat t2.json 
{
"_index":
"t2",
"_type":
"_doc",
"_id":
"9HNg1IQBRcFJVNOe-1xE",
"_score":
1,
"_source":{
"name":
"yu1",
"age":
"28",
"address":
"JS",
"job":
"dev"}}

{
"_index":
"t2",
"_type":
"_doc",
"_id":
"9XNg1IQBRcFJVNOe-1yA",
"_score":
1,
"_source":{
"name":
"yu2",
"age":
"27",
"address":
"BJ",
"job":
"dev"}}

{
"_index":
"t2",
"_type":
"_doc",
"_id":
"9nNg1IQBRcFJVNOe-1yS",
"_score":
1,
"_source":{
"name":
"yu3",
"age":
"26",
"address":
"SD",
"job":
"ops"}}

[root yc-k8s /es-nfs-
data]#

2.3 恢复数据

# 恢复数据，就是input，output反过来


[root yc
-k8s 
/es
-nfs
-data]#elasticdump \

>--input=/es-nfs-data/t2.json \
>--output=http://10.0.0.18:9200/t2
Sat, 
03Dec202214:
44:
23 GMT 
| starting dump

Sat, 
03Dec202214:
44:
23 GMT 
| got 
3 objects 
from source file (
offset: 
0)

Sat, 
03Dec202214:
44:
23 GMT 
| sent 
3 objects 
to destination elasticsearch, wrote 
3
Sat, 
03Dec202214:
44:
23 GMT 
| got 
0 objects 
from source file (
offset: 
3)

Sat, 
03Dec202214:
44:
23 GMT 
| Total Writes: 
3
Sat, 
03Dec202214:
44:
23 GMT 
| dump complete

[root yc
-k8s 
/es
-nfs
-data]#

2.4 批量备份

# 去掉以点开头的index 
curl -s 10.0.0.18:9200/_cat/indices | awk 
'{print $3}' | grep -v 
'^\.'

# 脚本
#!/bin/bash
# author : www.yuchaoit.cn

indexs=$(curl -s 10.0.0.18:9200/_cat/indices | awk 
'{print $3}' | grep -v 
'^\.')


for i 
in$indexs
do
 elasticdump \

 --input=http://10.0.0.18:9200/
${i} \

 --output=/es-nfs-data/
${i}.json \

 --
type=data

done

备份结果

rootyc-k8s/es-nfs-data]#
[
rootyc-k8s/es-nfs-data]
#sh for-index.sh 
Sat,03Dec2022 14:54:57 GMT|startingdump
Sat,03Dec2022 14:54:57 GMT|got3objectsfromsourceelasticsearch(offset:0)
Sat,03Dec2022 14:54:57 GMT|sent3objectstodestinationfile,wrote3
Sat,03Dec2022 14:54:57 GMT|got0objectsfromsourceelasticsearch(offset:3)
Sat,03Dec2022 14:54:57 GMT|Total Writes:3
Sat,03Dec2022 14:54:57 GMT|dumpcomplete
Sat,03Dec2022 14:54:57 GMT|startingdump
Sat,03Dec2022 14:54:57 GMT|got8objectsfromsourceelasticsearch(offset:0)
Sat,03Dec2022 14:54:57 GMT|sent8objectstodestinationfile,wrote8
Sat,03Dec2022 14:54:57 GMT|got0objectsfromsourceelasticsearch(offset:8)
Sat,03Dec2022 14:54:57 GMT|Total Writes:8
Sat,03Dec2022 14:54:57 GMT|dumpcomplete
Sat,03Dec2022 14:54:57 GMT|startingdump
Sat,03Dec2022 14:54:57 GMT|got3objectsfromsourceelasticsearch(offset:0)
Sat,03Dec2022 14:54:57 GMT|sent3objectstodestinationfile,wrote3
Sat,03Dec2022 14:54:57 GMT|got0objectsfromsourceelasticsearch(offset:3)
Sat,03Dec2022 14:54:57 GMT|Total Writes:3
Sat,03Dec2022 14:54:57 GMT|dumpcomplete
[
rootyc-k8s/es-nfs-data]
#ll
\total20
-rw-r--r--.1rootroot248Dec322:54for-index.sh
drwxr-xr-x.3elasticsearchelasticsearch4096 Dec304:50my_backup_location
-rw-r--r--.1rootroot444Dec322:54restored_index_2.json
-rw-r--r--.1rootroot1090 Dec322:54t1.json
-rw-r--r--.1rootroot402Dec322:54t2.json
[
rootyc-k8s/es-nfs-data]
#

2.5 密码认证

es有密码时，备份命令


 elasticdump \

 --input=http://name:[email protected]:9200/t2 \

 --output=/es-nfs-data/t2.json \

 --
type=data

2.6 建议用法

1. 想分析es数据，用elasticdump导出json

2. 仅仅备份数据，用es官网工具

3. 恢复es数据，如果数据冲突会被覆盖

3.es安全认证

逻辑图

官网资料

https://www.elastic.co/guide/en/elasticsearch/reference/current/configuring-stack-security.html

1.创建证书

1.创建证书，输入密码
[
root@es-node1~]
#
[
root@es-node1~]
#
[
root@es-node1~]
#/usr/share/elasticsearch/bin/elasticsearch-certutil ca
ThistoolassistsyouinthegenerationofX.509certificatesandcertificate
signingrequestsforusewithSSL/TLSintheElasticstack.

The'ca'modegeneratesanew'certificate authority'
ThiswillcreateanewX.509certificateandprivatekeythatcanbeused
tosigncertificatewhenrunningin'cert'mode.

Usethe'ca-dn'optionifyouwishtoconfigurethe'distinguished name'
ofthecertificateauthority

Bydefaultthe'ca'modeproducesasinglePKCS#12output file which holds:
*TheCAcertificate
*TheCA'sprivatekey

IfyouelecttogeneratePEMformatcertificates(the-pemoption),thentheoutputwill
beazipfilecontainingindividualfilesfortheCAcertificateandprivatekey

Pleaseenterthedesiredoutputfile [
elastic-stack-ca.p12]
:
Enter password for elastic-stack-ca.p12 :
[
root@es-node1~]
#
[
root@es-node1~]
#
[
root@es-node1~]
#


[
root@es-node1~]
#file /usr/share/elasticsearch/elastic-stack-ca.p12 
/usr/share/elasticsearch/elastic-stack-ca.p12:data



2.根据ca整数，创建私钥，也可以二次设置密码
/usr/share/elasticsearch/bin/elasticsearch-certutilcert--caelastic-stack-ca.p12


3.检查
[
root@es-node1/usr/share/elasticsearch]
#ll
total576
drwxr-xr-x2rootroot4096 Nov1823:53bin
-rw-------1rootroot3443 Dec323:10elastic-certificates.p12
-rw-------1rootroot2527 Dec323:08elastic-stack-ca.p12
drwxr-xr-x9rootroot107Nov1823:53jdk
drwxr-xr-x3rootroot4096 Nov1823:53lib
-rw-r--r--1rootroot13675Sep22020 LICENSE.txt
drwxr-xr-x52rootroot4096 Nov1823:53modules
-rw-rw-r--1rootroot544318Sep22020 NOTICE.txt
drwxr-xr-x2rootroot6Sep22020 plugins
-rw-r--r--1rootroot7007 Sep22020 README.asciidoc


4.复制证书到固定位置，同步es节点

[
root@es-node1/usr/share/elasticsearch]
#scp -r /etc/elasticsearch/certs [email protected]:/etc/elasticsearch/
elastic-certificates.p12100%3443 5.1MB/s00:00
elastic-stack-ca.p12100%2527 3.1MB/s00:00
[
root@es-node1/usr/share/elasticsearch]
#scp -r /etc/elasticsearch/certs [email protected]:/etc/elasticsearch/
elastic-certificates.p12100%3443 5.3MB/s00:00
elastic-stack-ca.p12100%2527 2.8MB/s00:00
[
root@es-node1/usr/share/elasticsearch]
#


5.修改es节点配置文件，开启安全功能
# es安全组件，xpack，开启即可。
# 来自于官网的教程，复制粘贴即可，有机会再阅读es数据，逐步深入学习。



xpack.security.enabled:true
xpack.security.transport.ssl.enabled:true
xpack.security.transport.ssl.verification_mode:certificate
xpack.security.transport.ssl.keystore.path:certs/elastic-stack-ca.p12
xpack.security.transport.ssl.truststore.path:certs/elastic-stack-ca.p12

6.注意修改es目录权限，重启进程
chown-Relasticsearch.elasticsearch/etc/elasticsearch/
systemctlrestartelasticsearch.service

7.检查es状态
netstat-tunlp|grep9200

此时就已连不上了，需要账密了

创建账户，密码

默认会给一些列的内置用户，创建密码，统一123123

[root@es-node1 ~]#/
usr/
share/
elasticsearch/
bin/
elasticsearch-setup-passwordsinteractive
Initiatingthesetupofpasswordsforreserveduserselastic,
apm_system,
kibana,
kibana_system,
logstash_system,
beats_system,
remote_monitoring_user.

Youwillbepromptedtoenterpasswordsastheprocessprogresses.

Pleaseconfirmthatyouwouldliketocontinue[y/N]y


Enterpasswordfor[elastic]: 

passwordsmustbeatleast[6]characterslong
Tryagain.

Enterpasswordfor[elastic]: 

Reenterpasswordfor[elastic]: 

Enterpasswordfor[apm_system]: 

Reenterpasswordfor[apm_system]: 

Enterpasswordfor[kibana_system]: 

Reenterpasswordfor[kibana_system]: 

Enterpasswordfor[logstash_system]: 

Reenterpasswordfor[logstash_system]: 

Enterpasswordfor[beats_system]: 

Reenterpasswordfor[beats_system]: 

Enterpasswordfor[remote_monitoring_user]: 

Reenterpasswordfor[remote_monitoring_user]: 

Changedpasswordforuser[apm_system]
Changedpasswordforuser[kibana_system]
Changedpasswordforuser[kibana]
Changedpasswordforuser[logstash_system]
Changedpasswordforuser[beats_system]
Changedpasswordforuser[remote_monitoring_user]
Changedpasswordforuser[elastic]

内置角色解释

账户elastic为elasticsearch超级管理员，拥有所有权限

账户kibana用于kibana组件获取相关信息用于web展示

账户logstash_system用于logstash服务获取elasticsearch的监控数据

账户beats_system用于存储es监控信息时用

账户kibana_system，用于kibana安全与es集群通讯用。



es官网，内置角色

https:/
/www.elastic.co/guide/en/elasticsearch/reference/7.9/built-
in-roles.html

2.创建kibana登录用户

务必注意，这里kibana使用刚才es创建的账户，密码，来连接es集群。
密码保证一致，否则kibana，无法和es通信了。

[root@es-node1 ~]
#vim /etc/kibana/kibana.yml 

42# If your Elasticsearch is protected with basic authentication, these settings provide
43# the username and password that the Kibana server uses to perform maintenance on the Kibana
44# index at startup. Your Kibana users still need to authenticate with Elasticsearch, which
45# is proxied through the Kibana server.
46 elasticsearch.username: 
"kibana_system"
47 elasticsearch.password: 
"123123"


重启kibana

[root@es-node1 ~]
#systemctl restart kibana

kibana正确安全连接es

elastic

123123


普通用户kibana 123123 也可以登录

es-head登录认证

3.创建用户

kibana先创建es的索引

便于查询数据。

kibana功能space

需求：1.限制开发小于，登录kibana，只能使用某个功能，如discover,且只能看t2 索引


实践：

1. 创建space ，dev，允许只能看某些内容

创建roles

创建用户，开发小于，绑定dev角色

yu

yu123123

开发小于登录kibana

只能访问t1 
index数据

4.角色访问index区别

链接：https://www.cnblogs.com/sxy-blog/p/18128248

（版权归原作者所有，侵删）

继续阅读

阅读原文

ES备份恢复

1.官网提供snap快照备份恢复

环境要求

搭建NFS机器，服务端

es节点安装nfs插件

es节点修改config，开启snapshot

确保重启正确

1.先注册快照仓库

2.创建一个快照

检查快照

图解nfs快照

3.针对具体index的快照

4.查看快照信息

5.查看正在运行的快照

6.演练恢复索引

指定index恢复

恢复且修改index名

7.用日期命名快照

8.记录es练习接口

2.第三方备份工具

2.1 安装node

2.2 备份命令

备份es数据为可读的json文件

备份且压缩

2.3 恢复数据

2.4 批量备份

2.5 密码认证

2.6 建议用法

3.es安全认证

官网资料

1.创建证书

此时就已连不上了，需要账密了

创建账户，密码

内置角色解释

2.创建kibana登录用户

kibana正确安全连接es

es-head登录认证

3.创建用户

kibana先创建es的索引

kibana功能space

创建roles

创建用户，开发小于，绑定dev角色

开发小于登录kibana

4.角色访问index区别