分布式搜索服务与日志中心

标签：apps elasticsearch node1 搜索日志 root logstash es 分布式

分布式搜索服务与日志中心

ElasticSearch集群组件及工作机制

root@es-node3:~# cat /etc/hosts

10.4.7.137 es-node1
10.4.7.136 es-node2
10.4.7.134 es-node3

root@es-node1:~# cat /etc/security/limits.conf
root soft core unlimited
root hard core unlimited
root soft nproc 1000000
root hard nproc 1000000
root soft nofile 1000000
root hard nofile 1000000
root soft memlock 32000
root hard memlock 32000
root soft msgqueue 8192000
root hard msgqueue 8192000

* soft core unlimited
* hard core unlimited
* soft nproc 1000000
* hard nproc 1000000
* soft nofile 1000000
* hard nofile 1000000
* soft memlock 32000
* hard memlock 32000
* soft msgqueue 8192000
* hard msgqueue 8192000

root@es-node3:/# mkdir /data/esdata /data/eslogs /apps -pv

root@es-node3:/# chown elasticsearch.elasticsearch /data /apps/ -R

root@es-node3:~# groupadd -g 2888 elasticsearch && useradd -u 2888 -g 2888 -r -m -s /bin/bash elasticsearch
root@es-node3:~# passwd elasticsearch

root@es-node1:/apps# tar xf elasticsearch-8.5.1-linux-x86_64.tar.gz
root@es-node1:/apps# ln -sv /apps/elasticsearch-8.5.1 /apps/elasticsearch

#⽣成CA私钥，默认名字为elastic-stack-ca.p12

elasticsearch@es-node1:/apps/elasticsearch$ ./bin/elasticsearch-certutil ca

#⽣产CA公钥，默认名称为elastic-certificates.p12 elasticsearch@es-node1:/apps/elasticsearch$ bin/elasticsearch-certutil cert --ca elastic-stack-ca.p12

#签发elasticsearch集群主机证书： --silent 静默模式 --in 指定文件instances.yml --out 这里会签发好多证书放在certs.zip这里 --pass 指定证书密码为 --ca 使用那个ca去签发
elasticsearch@es-node1:/apps/elasticsearch$ bin/elasticsearch-certutil cert --silent --in instances.yml --out certs.zip --pass magedu123 --ca elastic-stack-ca.p12

解压证书

elasticsearch@es-node1:/apps/elasticsearch$ unzip certs.zip
Archive: certs.zip
creating: es-node1/
inflating: es-node1/es-node1.p12
creating: es-node2/
inflating: es-node2/es-node2.p12
creating: es-node3/
inflating: es-node3/es-node3.p12

三台主机创建config/certs

elasticsearch@es-node1:/apps/elasticsearch$ mkdir config/certs

拷贝证书

elasticsearch@es-node1:/apps/elasticsearch$ cp es-node1/es-node1.p12 config/certs/

elasticsearch@es-node1:/apps/elasticsearch$ scp es-node2/es-node2.p12 10.4.7.136:/apps/elasticsearch/config/certs

elasticsearch@es-node1:/apps/elasticsearch$ scp es-node3/es-node3.p12 10.4.7.134:/apps/elasticsearch/config/certs

#⽣成 keystore ⽂件(keystore是保存了证书密码的认证⽂件000000) 一台主机生成拷贝到其他主机

现在是空文件

elasticsearch@es-node1:/apps/elasticsearch$ ./bin/elasticsearch-keystore create
warning: ignoring JAVA_HOME=/usr/lib/jvm/jdk-11; using bundled JDK
Created elasticsearch keystore in /apps/elasticsearch/config/elasticsearch.keystore

添加密码

elasticsearch@es-node1:/apps/elasticsearch$ ./bin/elasticsearch-keystore add xpack.security.transport.ssl.keystore.secure_password
warning: ignoring JAVA_HOME=/usr/lib/jvm/jdk-11; using bundled JDK
Enter value for xpack.security.transport.ssl.keystore.secure_password: magedu123

elasticsearch@es-node1:/apps/elasticsearch$ ./bin/elasticsearch-keystore add xpack.security.transport.ssl.truststore.secure_password
warning: ignoring JAVA_HOME=/usr/lib/jvm/jdk-11; using bundled JDK
Enter value for xpack.security.transport.ssl.truststore.secure_password: magedu123

拷贝证书到其他两个服务器

elasticsearch@es-node1:/apps/elasticsearch$ scp /apps/elasticsearch/config/elasticsearch.keystore 10.4.7.136:/apps/elasticsearch/config/elasticsearch.keystore

elasticsearch@es-node1:/apps/elasticsearch$ scp /apps/elasticsearch/config/elasticsearch.keystore 10.4.7.134:/apps/elasticsearch/config/elasticsearch.keystore

配置文件

创建集群后会通告那些主机

discovery.seed_hosts:

初始化的时候那些主机会选举为master

cluster.initial_master_nodes

删除elasticsearch索引时，是不是要传递完整名称不允许使用正则做模糊匹配

action.destructive_requires_name: true

xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.keystore.path: /apps/elasticsearch/config/certs/es-node1.p12

root@es-node2:~# vi /lib/systemd/system/elasticsearch.service
[Unit]
Description=Elasticsearch
Documentation=http://www.elastic.co
Wants=network-online.target
After=network-online.target
[Service]
RuntimeDirectory=elasticsearch
Environment=ES_HOME=/apps/elasticsearch
Environment=ES_PATH_CONF=/apps/elasticsearch/config
Environment=PID_DIR=/apps/elasticsearch
WorkingDirectory=/apps/elasticsearch
User=elasticsearch
Group=elasticsearch
ExecStart=/apps/elasticsearch/bin/elasticsearch --quiet
# StandardOutput is configured to redirect to journalctl since
# some error messages may be logged in standard output before
1.6：⽤户管理：
# elasticsearch logging system is initialized. Elasticsearch
# stores its logs in /var/log/elasticsearch and does not use
# journalctl by default. If you also want to enable journalctl
# logging, you can simply remove the "quiet" option from ExecStart.
StandardOutput=journal
StandardError=inherit
# Specifies the maximum file descriptor number that can be opened by this process
LimitNOFILE=65536
# Specifies the maximum number of processes
LimitNPROC=4096
# Specifies the maximum size of virtual memory
LimitAS=infinity
# Specifies the maximum file size
LimitFSIZE=infinity
# Disable timeout logic and wait until process is stopped
TimeoutStopSec=0
# SIGTERM signal is used to stop the Java process
KillSignal=SIGTERM
# Send the signal only to the JVM rather than its control group
KillMode=process
# Java process is never killed
SendSIGKILL=no
# When a JVM receives a SIGTERM signal it exits with code 143
SuccessExitStatus=143
[Install]
WantedBy=multi-user.target

root@es-node1:~# systemctl daemon-reload && systemctl start elasticsearch.service && systemctl enable elasticsearch.service

es有好多默认密码

⽤户管理：批量修改默认账户密码：批量设置密码： elasticsearch@es-node1:/apps/elasticsearch$ bin/elasticsearch-setup-passwords interactive

创建超级管理员账户： elasticsearch@es-node1:/apps/elasticsearch$ ./bin/elasticsearch-users useradd awen -p123456 -r superuser elasticsearch@es-node1:/apps/elasticsearch$ curl -u awen:123456 http://10.4.7.137:9200 验证集群状态

elasticsearch@es-node3:/apps/elasticsearch$ ./bin/elasticsearch-users useradd awen -p123456 -r superuser

elasticsearch@es-node2:/apps/elasticsearch$ ./bin/elasticsearch-users useradd awen -p123456 -r superuser

查看索引

root@es-node1:~# curl -u awen:123456 -X GET http://10.4.7.137:9200/awen_index?pretty
{
"awen_index" : {
"aliases" : { },
"mappings" : { },
"settings" : {
"index" : {
"routing" : {
"allocation" : {
"include" : {
"_tier_preference" : "data_content"
}
}
},
"number_of_shards" : "1",
"provided_name" : "awen_index",
"creation_date" : "1669426420445",
"number_of_replicas" : "1",
"uuid" : "Orq06b2YSemT4usQhnYZeA",
"version" : {
"created" : "8050199"
}
}
}
}
}

root@es-node1:~# curl -u awen:123456 -X PUT http://10.4.7.137:9200/test_index?pretty
{
"acknowledged" : true,
"shards_acknowledged" : true,
"index" : "test_index"
}
root@es-node1:~# curl -u awen:123456 -X PUT http://10.4.7.137:9200/awen_index
{"acknowledged":true,"shards_acknowledged":true,"index":"awen_index"}root@es-node1:~#

ElasticSearch的常用ETL工具栈及LogStash和各Beats组件功能简介及使用场景 ETL简介： ETL 是大数据世界中的一种常见模式，用于收集和整合数据以进行存储、分析及展示，基本流程为： Extract：数据提取、基于不同的工具从数据源提取数据 Transform：数据转换，通过自定义流程将数据进行内容转换、格式转换、数据字段提取或删除等 Load：数据加载，将数据存储到外部数据库或数据仓库安装logstash root@logstash:/usr/local/src# dpkg -i logstash-8.5.1-amd64.deb

root@logstash:/etc/logstash/conf.d# vim stdin-stout-test.conf
input {
stdin {}
}

output {
stdout {}
}

检查配置文件

root@logstash:/etc/logstash/conf.d# /usr/share/logstash/bin/logstash -f stdin-stout-test.conf -t

启动

root@logstash:/etc/logstash/conf.d# /usr/share/logstash/bin/logstash -e 'input { stdin{} } output { stdout{ codec => rubydebug }}'

root@logstash:/etc/logstash/conf.d# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/stdin-stout-test.conf

root@logstash:/etc/logstash/conf.d# vim log-file.conf
input {
stdin {}
}

output {
file {
path => "/tmp/logstash-test.log"
}
}

root@logstash:/etc/logstash/conf.d# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/log-file.conf

交互式输入文字

查看日志输出

root@logstash:~# tail -f /tmp/logstash-test.log
{"message":"awen","@version":"1","host":{"hostname":"logstash"},"event":{"original":"awen"},"@timestamp":"2022-11-26T09:53:23.373107046Z"}

可以同时输出不同的输出源