首页 > 其他分享 >es索引迁移Snapshot-迁移部分索引

es索引迁移Snapshot-迁移部分索引

时间：2022-12-05 20:06:00浏览次数：36  
标签：http 9200 elastic backup 192.168 索引 Snapshot 迁移 type

Snapshot-迁移部分索引

源集群 192.168.40.180 192.168.40.181 192.168.40.182

目标集群 192.168.40.61 192.168.40.62 192.168.40.63

生产 需要额外，挂载硬盘

Snapshot API 是 Elasticsearch 用于对数据进行备份和恢复的一组 API 接口，可以通过 Snapshot API 进行跨集群的数据迁移，原理就是从源 Elasticsearch 集群创建数据快照，然后在目标 Elasticsearch 集群中进行恢复。

第一步：在源集群注册 Repository

创建快照前先要注册 Repository , 一个 Repository 可以包含多份快照文件， Repository 主要有以下几种类型：

fs: 共享文件系统，将快照文件存放于文件系统中
 url: 指定文件系统的URL路径，支持协议：http,https,ftp,file,jar
 s3: AWS S3对象存储,快照存放于S3中，以插件形式支持
 hdfs: 快照存放于hdfs中，以插件形式支持
 azure: 快照存放于azure对象存储中，以插件形式支持
 gcs: 快照存放于google cloud对象存储中，以插件形式支持

1模拟数据

curl -H 'Content-type: application/json' -XPUT 'http://192.168.40.180:9200/_template/blog-template' -d '{
  "index_patterns": [
   "blog-*"
  ],
  "settings": {
   "number_of_shards": 3,
   "number_of_replicas": 1
  },
  "mappings": {
   "dynamic_templates": [
    {
     "integers": {
      "match": "int_*",
      "mapping": {
       "type": "integer"
      }
     }
    },
    {
     "strings": {
      "match_mapping_type": "string",
      "mapping": {
       "type": "keyword",
       "ignore_above": 256
      }
     }
    }
   ],
   "properties": {
    "title": {
     "type": "text"
    }
   }
  }
 }'
 
 
 
 
 
 
 
 
 
 
 curl -H 'Content-type: application/json' -XPUT 'http://192.168.40.180:9200/blog-test2' -d '{}'
 
 
 
 
 
 
 
 
 
 curl -H 'Content-type: application/x-ndjson' -XPOST 'http://192.168.40.180:9200/blog-test/_bulk' -d '{"index":{}}
 {"title":"elasticsearch best practice","author_firstname":"tony","author_lastname":"song","tags":"elasticsearch ,logstash","int_age":18,"locale":"zh,en"}
 {"index":{}}
 {"title":"elastic stack release V5","author_firstname":"shay","author_lastname":"banon","tags":"elk ,elasticsearch,release","int_age":28,"locale":"zhc,en"}
 {"index":{}}
 {"title":"kibana tutorial","author_firstname":"zen","author_lastname":"wei","tags":"","int_age":38,"locale":"zhc,en"}
 
 '
 
 
 
 
 
 curl -H 'Content-type: application/x-ndjson' -XPOST 'http://192.168.40.180:9200/blog-test2/_bulk' -d '{"index":{}}
 {"title":"elasticsearch best practice","author_firstname":"tony","author_lastname":"song","tags":"elasticsearch ,logstash","int_age":18,"locale":"zh,en"}
 {"index":{}}
 {"title":"elastic stack release V5","author_firstname":"shay","author_lastname":"banon","tags":"elk ,elasticsearch,release","int_age":28,"locale":"zhc,en"}
 {"index":{}}
 {"title":"kibana tutorial","author_firstname":"zen","author_lastname":"wei","tags":"","int_age":38,"locale":"zhc,en"}
 
 '

2查看索引信息

[es@master elasticsearch-7.2.0]$ curl --user elastic:1qaz1qaz -XGET 'http://192.168.40.180:9200/_cat/indices?pretty'
 green open blog-test   lREj4IwtS9-rHuIdiTMmwA 3 1 0 0  1.4kb   743b
 green open emp         ZAEKZhqJS5i7zt-L8MkB-Q 1 1 1 0  7.5kb  3.7kb
 green open test10      ifIzAccwTc290xcfuxifJA 5 0 1 0  4.8kb  4.8kb
 green open test        4655Yhv1QPGRb-W78qVm4g 1 1 2 3   18kb    9kb
 green open blog-test2  znzIobHlQ5in6dxAr4L_eg 3 1 0 0  1.4kb   743b
 green open test4       5XnbLGmoTo-mPzIT-FpFDA 3 1 0 0  1.6kb   849b
 green open .security-7 iwX-Pv_MSRWb7I1MoP6j4Q 1 1 6 0 39.6kb 19.8kb
 green open test3       KS7T5LvPQ4WhLyihz4AJ_g 3 1 0 0  1.6kb   849b

3安装sshfs

yum install -y epel-release
 yum -y install fuse-sshfs

老集群挂载新集群目录

sshfs -o allow_other -o nonempty 192.168.40.61:/elastic/backup /elastic/backup
 （所有节点执行）除了40.61

4修改elasticsearch配置文件

修改 elasticsearch.yml 配置文件，增加如下配置:

path.repo: ["/elastic/backup"]
 
 mkdir -p /elastic/backup
 chown  es.es /elastic/backup -R
 
  chmod  777 /elastic/ -R
 chmod  777 /elastic/backup/ -R

5重启elasticsearch进程，查看创建的repo

curl --user elastic:1qaz1qaz  -H 'Content-type: application/json' -XGET 'http://192.168.40.180:9200/_cluster/settings?include_defaults&filter_path=*.path.repo&pretty'
 
[es@master elasticsearch-7.2.0]$ curl --user elastic:1qaz1qaz  -H 'Content-type: application/json' -XGET 'http://192.168.40.180:9200/_cluster/settings?include_defaults&filter_path=*.path.repo&pretty'
 {
   "defaults" : {
     "path" : {
       "repo" : [
         "/elastic/backup"
       ]
     }
   }
 }

6创建仓库

---会报错--忽略 -查看仓库（报错原因 需要共享文件服务"type": "fs",）我用的sshfs

curl --user elastic:1qaz1qaz  -H "Content-Type: application/json" -XPUT  http://192.168.40.182:9200/_snapshot/my_fs_backup3 -d '
{
    "type": "fs",
    "settings": {
        "location": "/elastic/backup",
        "max_restore_bytes_per_sec" : "50m",
        "compress": true,
        "max_snapshot_bytes_per_sec" : "50m"
    }
}
'

7查看仓库

[root@node1 ~]# curl --user elastic:1qaz1qaz -XGET 'http://192.168.40.180:9200/_snapshot/_all?pretty'
{
  "my_fs_backup" : {
    "type" : "fs",
    "settings" : {
      "location" : "/elastic/backup",
      "max_restore_bytes_per_sec" : "50m",
      "compress" : "true",
      "max_snapshot_bytes_per_sec" : "50m"
    }
  }
}


GET _snapshot/my_fs_backup
#输出结果:
{
  "my_fs_backup" : {
    "type" : "fs",
    "settings" : {
      "compress" : "true",
      "location" : "/home/elastic/backup/my_fs_backup"
    }
  }
}
查看在哪个node上创建了关联：
[root@master backup]# curl --user elastic:1qaz1qaz  -H 'Content-type: application/json' -XPOST 'http://192.168.40.180:9200/_snapshot/my_fs_backup/_verify?pretty' -d 'undefined'

{
  "nodes" : {
    "I-au_rvbQRKj1xrjvWZPnA" : {
      "name" : "node2"
    },
    "opk7TQctTPaacJ42if-dbg" : {
      "name" : "master"
    },
    "H55oT25BS0-K-Z_WadWxSg" : {
      "name" : "node1"
    }
  }
}

8备份

emp开头索引,test开头索引,blog开头索引, 除了-.security

[root@master backup]# curl --user elastic:1qaz1qaz -H 'Content-type: application/json' -XPUT 'http://192.168.40.180:9200/_snapshot/my_fs_backup/es_bak_20220917' -d '{"indices": "emp*,test*,blog*,-.security*,","ignore_unavailable": "true","include_global_state": false}'

{"accepted":true}[root@master backup]#

查看所有快照信息

curl --user elastic:1qaz1qaz -H 'Content-type: application/json' -XGET 'http://192.168.40.180:9200/_snapshot/my_fs_backup/_all?pretty'

查看指定快照信息

[root@master backup]# curl --user elastic:1qaz1qaz -H 'Content-type: application/json' -XGET 'http://192.168.40.180:9200/_snapshot/my_fs_backup/es_bak*?pretty'
{
  "snapshots" : [
    {
      "snapshot" : "es_bak_20220917",
      "uuid" : "t4HY5qfZRu2wzSzXnn6NaA",
      "version_id" : 7020099,
      "version" : "7.2.0",
      "indices" : [
        "blog-test",
        "emp",
        "test10",
        "test",
        "blog-test2",
        "test4",
        "test3"
      ],
      "include_global_state" : false,
      "state" : "SUCCESS",
      "start_time" : "2022-09-16T17:41:48.692Z",
      "start_time_in_millis" : 1663350108692,
      "end_time" : "2022-09-16T17:41:49.092Z",
      "end_time_in_millis" : 1663350109092,
      "duration_in_millis" : 400,
      "failures" : [ ],
      "shards" : {
        "total" : 19,
        "failed" : 0,
        "successful" : 19
      }
    }
  ]
}

查看快照状态

curl  --user elastic:1qaz1qaz -H 'Content-type: application/json' -XGET 'http://192.168.40.180:9200/_snapshot/my_fs_backup/es_bak_20220917/_status?pretty'

第二步：目标集群 

path.repo: ["/elastic/backup"]

重启

查看repo

curl --user elastic:1qaz1qaz  -H 'Content-type: application/json' -XGET 'http://192.168.40.61:9200/_cluster/settings?include_defaults&filter_path=*.path.repo&pretty'

1新集群仓库创建为只读

curl --user elastic:1qaz1qaz  -H "Content-Type: application/json" -XPUT  http://192.168.40.61:9200/_snapshot/my_fs_backupNEW -d '
{
    "type": "fs",
    "settings": {
        "location": "/elastic/backup",
        "max_restore_bytes_per_sec" : "50m",
        "compress": true,
        "max_snapshot_bytes_per_sec" : "50m",
        "readonly": true
    }
}
'

查看在那个node 上创建了关联

curl --user elastic:1qaz1qaz  -H 'Content-type: application/json' -XPOST 'http://192.168.40.61:9200/_snapshot/my_fs_backupNEW/_verify?pretty' -d 'undefined'
{
  "nodes" : {
    "FmhibG9HTAqfd98tykZqEA" : {
      "name" : "node1"
    },
    "SwOuMfpiRLKECWtfDx2ImA" : {
      "name" : "node2"
    },
    "a5vPcQRSRR-f1fqnZxPIkQ" : {
      "name" : "master"
    }
  }
}

2查看快照信息

[es@master ~]$ curl --user elastic:1qaz1qaz -H 'Content-type: application/json' -XGET 'http://192.168.40.61:9200/_snapshot/my_fs_backupNEW/_all?pretty'

{
  "snapshots" : [
    {
      "snapshot" : "es_bak_20220917",
      "uuid" : "t4HY5qfZRu2wzSzXnn6NaA",
      "version_id" : 7020099,
      "version" : "7.2.0",
      "indices" : [
        "blog-test",
        "emp",
        "test10",
        "test",
        "blog-test2",
        "test4",
        "test3"
      ],
      "include_global_state" : false,
      "state" : "SUCCESS",
      "start_time" : "2022-09-16T17:41:48.692Z",
      "start_time_in_millis" : 1663350108692,
      "end_time" : "2022-09-16T17:41:49.092Z",
      "end_time_in_millis" : 1663350109092,
      "duration_in_millis" : 400,
      "failures" : [ ],
      "shards" : {
        "total" : 19,
        "failed" : 0,
        "successful" : 19
      }
    }
  ]
}
[es@master ~]$

3恢复索引

curl --user elastic:1qaz1qaz -H 'Content-type: application/json' -XPOST 'http://192.168.40.61:9200/_snapshot/my_fs_backupNEW/es_bak_20220917/_restore' -d '{
 "indices": "*",
 "ignore_unavailable": true
}'

4查看索引 集群 节点

curl --user elastic:1qaz1qaz  "http://192.168.40.61:9200/_cat/health?v"

curl --user elastic:1qaz1qaz  "http://192.168.40.61:9200/_cat/nodes?v"
curl --user elastic:1qaz1qaz  "http://192.168.40.61:9200/_cat/indices?v"

查看恢复进度
 curl --user elastic:1qaz1qaz -H 'Content-type: application/json' -XGET 'http://192.168.40.61:9200/_recovery?pretty'

集群迁移部分索引成功

第三步：迁移索引模板

1导出集群中所有模板

curl  --user elastic:1qaz1qaz http://192.168.40.61:9200/_cat/templates | awk '{print $1}'| while read line ;do curl -s -u elastic:xrRWYqr2QEXZ8h\!@6  -XGET http://192.168.40.61:9200/_template/$line?pretty >template-$line.json;done
 
导出集群中所有模板

2.模板json文件处理一遍，删除第二行和最后一行：

sed -i '2d' ./*json

sed -i '$d' ./*json

3新集群导入【需要迁移】的索引es模板：

ls | grep json | awk -F"template-" '{print $2}' | awk -F".json" '{print $1}' | while read line;do curl -XPUT http://localhost:9200/_template/$line -H 'Content-Type: application/json' -d@template-${line}.json;done

第四步

sshfs进程停掉

标签：http,9200,elastic,backup,192.168,索引,Snapshot,迁移,type
From： https://blog.51cto.com/zhaochengsheng/5908695

相关文章

赞助商

阅读排行

网站主页关于我们联系我们网站地图

本网站内容转载自其他媒体，侵权联系[admin##ips99.com]。

Copyright © 2020-2023 IPS99 版权所有 IPS99