Orchestrator使用讲解

标签：false 讲解 orchestrator echo VIP orch 使用 Orchestrator true

1、提前部署Mysql主从（一主两从）

2、配置orch到所有机器的免密，配置好主机名/etc/hosts

3、安装orch

配置文件

{
  "Debug": true,
  "EnableSyslog": false,     //是否把日志输出到系统日志里
  "ListenAddress": ":3000",  
  "MySQLTopologyUser": "root",    //orch管理mysql的用户
  "MySQLTopologyPassword": "abc123",  //orch管理mysql的密码
  "MySQLTopologyCredentialsConfigFile": "",
  "MySQLTopologySSLPrivateKeyFile": "",
  "MySQLTopologySSLCertFile": "",
  "MySQLTopologySSLCAFile": "",
  "MySQLTopologySSLSkipVerify": true,
  "MySQLTopologyUseMutualTLS": false,
  "MySQLOrchestratorHost": "10.211.55.2",  //orch元数据信息存放数据库host
  "MySQLOrchestratorPort": 4000,          //orch元数据信息存放数据库port
  "MySQLOrchestratorDatabase": "orchestrator", //orch元数据信息存放数据库database
  "MySQLOrchestratorUser": "orchestrator",    //orch元数据信息存放数据库user
  "MySQLOrchestratorPassword": "orchestrator",  //orch元数据信息存放数据库password
  "MySQLOrchestratorCredentialsConfigFile": "",
  "MySQLOrchestratorSSLPrivateKeyFile": "",
  "MySQLOrchestratorSSLCertFile": "",
  "MySQLOrchestratorSSLCAFile": "",
  "MySQLOrchestratorSSLSkipVerify": true,
  "MySQLOrchestratorUseMutualTLS": false,
  "MySQLConnectTimeoutSeconds": 1,
  "DefaultInstancePort": 3306,
  "DiscoverByShowSlaveHosts": false,   //通过show slave hosts自动发现slave,需在mysql配置文件中设置正确的report_host和report_port
  "InstancePollSeconds": 5,   //每隔5s获取状态
  "DiscoveryIgnoreReplicaHostnameFilters": [
    "a_host_i_want_to_ignore[.]example[.]com",
    ".*[.]ignore_all_hosts_from_this_domain[.]example[.]com",
    "a_host_with_extra_port_i_want_to_ignore[.]example[.]com:3307"
  ],
  "UnseenInstanceForgetHours": 240, 
  "SnapshotTopologiesIntervalHours": 0,
  "InstanceBulkOperationsWaitTimeoutSeconds": 10, //在进行批量操作时, 等待单个实例的最大时间(秒)
  "HostnameResolveMethod": "default",  //如何解析主机名,none: 直接返回传入的hostname，default: 直接返回传入hostname，cname: resolves an IP or hostname into a normalized valid CNAME，ip: 返回ip
  "MySQLHostnameResolveMethod": "@@hostname",
  "SkipBinlogServerUnresolveCheck": true,
  "ExpiryHostnameResolvesMinutes": 60,
  "RejectHostnameResolvePattern": "",
  "ReasonableReplicationLagSeconds": 0,  //合理的复制滞后秒数. 高于这个值就是有问题的
  "ProblemIgnoreHostnameFilters": [],
  "VerifyReplicationFilters": false,
  "ReasonableMaintenanceReplicationLagSeconds": 20,
  "CandidateInstanceExpireMinutes": 60,
  "AuditLogFile": "",
  "AuditToSyslog": false,
  "RemoveTextFromHostnameDisplay": ".mydomain.com:3306",
  "ReadOnly": false,
  "AuthenticationMethod": "basic",
  "HTTPAuthUser": "admin",
  "HTTPAuthPassword": "admin",
  "AuthUserHeader": "",
  "PowerAuthUsers": [
    "*"
  ],
  "ClusterNameToAlias": {
    "127.0.0.1": "test suite"
  },
  "ReplicationLagQuery": "",   //提供ReplicationLagQuery 时, 计算出的复制延迟. 否则与SecondsBehindMaster 相同
  "DetectClusterAliasQuery": "SELECT SUBSTRING_INDEX(@@hostname, '.', 1)",
  "DetectClusterDomainQuery": "",
  "DetectInstanceAliasQuery": "",
  "DetectPromotionRuleQuery": "",
  "DataCenterPattern": "[.]([^.]+)[.][^.]+[.]mydomain[.]com",
  "PhysicalEnvironmentPattern": "[.]([^.]+[.][^.]+)[.]mydomain[.]com",
  "PromotionIgnoreHostnameFilters": [],
  "DetectSemiSyncEnforcedQuery": "",
  "ServeAgentsHttp": false,
  "AgentsServerPort": ":3001",
  "AgentsUseSSL": false,
  "AgentsUseMutualTLS": false,
  "AgentSSLSkipVerify": false,
  "AgentSSLPrivateKeyFile": "",
  "AgentSSLCertFile": "",
  "AgentSSLCAFile": "",
  "AgentSSLValidOUs": [],
  "UseSSL": false,
  "UseMutualTLS": false,
  "SSLSkipVerify": false,
  "SSLPrivateKeyFile": "",
  "SSLCertFile": "",
  "SSLCAFile": "",
  "SSLValidOUs": [],
  "URLPrefix": "",
  "StatusEndpoint": "/api/status",
  "StatusSimpleHealth": true,
  "StatusOUVerify": false,
  "AgentPollMinutes": 60,
  "UnseenAgentForgetHours": 6,
  "StaleSeedFailMinutes": 60,
  "SeedAcceptableBytesDiff": 8192,
  "PseudoGTIDPattern": "",
  "PseudoGTIDPatternIsFixedSubstring": false,
  "PseudoGTIDMonotonicHint": "asc:",
  "DetectPseudoGTIDQuery": "",
  "BinlogEventsChunkSize": 10000,
  "SkipBinlogEventsContaining": [],
  "ReduceReplicationAnalysisCount": true,
  "FailureDetectionPeriodBlockMinutes": 1,
  "FailMasterPromotionOnLagMinutes": 0,
  "RecoveryPeriodBlockSeconds": 10, //一旦集群经历了恢复，那么在这段时间内将阻止自动恢复，以避免抖动
  "FailMasterPromotionIfSQLThreadNotUpToDate": false,
  "DelayMasterPromotionIfSQLThreadNotUpToDate": true, //等slave relay log apply完成
  "RecoveryIgnoreHostnameFilters": [],
  "RecoverMasterClusterFilters": [   
    "*"
  ],  //只对列表中正则表达式匹配的集群做故障恢复操作
  "RecoverIntermediateMasterClusterFilters": [
    "*"
  ],  //只对列表中正则表达式匹配的IntermediateMaster做故障恢复操作
  "OnFailureDetectionProcesses": [
    "echo 'Detected {failureType} on {failureCluster}. Affected replicas: {countSlaves}' >> /tmp/recovery.log"
  ], //检测出故障时执行（在决定是否进行故障转移之前）
  "PreGracefulTakeoverProcesses": [
    "echo 'Planned takeover about to take place on {failureCluster}. Master will switch to read_only' >> /tmp/recovery.log" //在主变为只读之前立即执行
  ],
  "PreFailoverProcesses": [
    "echo 'Will recover from {failureType} on {failureCluster}' >> /tmp/recovery.log"
  ], //在执行恢复操作之前立即执行。任何这些进程的失败（非零退出代码）都会中止恢复。
  "PostFailoverProcesses": [
    "echo '(for all types) Recovered from {failureType} on {failureCluster}. Failed: {failedHost}:{failedPort}; Successor: {successorHost}:{successorPort}' >> /tmp/recovery.log",
    "sh -x /usr/local/orchestrator/orch_hook.sh {failureType} {failureClusterAlias} {failedHost} {successorHost} >> /tmp/orch.log"
  ],   //成功恢复结束后，执行的脚本，这里调用切换vip的脚本
  "PostUnsuccessfulFailoverProcesses": [],
  "PostMasterFailoverProcesses": [
    "echo 'Recovered from {failureType} on {failureCluster}. Failed: {failedHost}:{failedPort}; Promoted: {successorHost}:{successorPort}' >> /tmp/recovery.log"
  ],
  "PostIntermediateMasterFailoverProcesses": [
    "echo 'Recovered from {failureType} on {failureCluster}. Failed: {failedHost}:{failedPort}; Successor: {successorHost}:{successorPort}' >> /tmp/recovery.log"
  ],  //在intermediate master（新主）恢复成功后执行的脚本
  "PostGracefulTakeoverProcesses": [
    "echo 'Planned takeover complete' >> /tmp/recovery.log",
    "sh -x /usr/local/orchestrator/orch_hook.sh {failureType} {failureClusterAlias} {failedHost} {successorHost} >> /tmp/orch.log"
  ], //在手动主从切换，旧的主节点挂在新的主节点上之后执行
  "CoMasterRecoveryMustPromoteOtherCoMaster": true, //双主的场景. 如果一个主库故障, 当该参数为true时, 只会提升另一个"主库"为新主库(看起来怪怪的), 否则集群中所有节点都可能提升为新主库
  "DetachLostSlavesAfterMasterFailover": true, //某些从库再恢复过程中可能会丢失。当为true时，Orchestrator将通过detach-replica命令将其强行中断复制
  "ApplyMySQLPromotionAfterMasterFailover": true, //当为true时，Orchestrator将会在新的master上执行：reset slave all 和 set read_only=0两条命令
  "PreventCrossDataCenterMasterFailover": false, //默认false，当为true时，Orchestrator只会使用同一个DC下的服务器进行恢复。如果在同一DC中找不到，那么恢复就会失败
  "PreventCrossRegionMasterFailover": false,  //默认false，当为true时，Orchestrator将会在同一区域内的服务器上进行故障恢复，如果找不到，那么恢复就会失败
  "FailMasterPromotionOnLagMinutes": 0,        
  "MasterFailoverDetachReplicaMasterHost": false, //当是true，Orchestrator将发出一个detach-replica-master-host命令到新的master上(确保新的master将不会复制旧的master)。 orchestrator将发出一个detach-replica-master-host升级的母版（这可以确保新的母版不会重生旧的母版）。默认值：false。如果ApplyMySQLPromotionAfterMasterFailover是true，那么该参数是无意义的。MasterFailoverDetachSlaveMasterHost是别名
  "MasterFailoverLostInstancesDowntimeMinutes": 0, //故障转移之后，所有的服务器的停机分钟数。默认为0
  "PostponeReplicaRecoveryOnLagMinutes": 0, //在故障恢复过程中，复制的延迟超过给定时间的从库，只有在选出master并且执行了所有process流程之后，才会在恢复的后期进行恢复。0为禁用。PostponeSlaveRecoveryOnLagMinutes是别名
  "OSCIgnoreHostnameFilters": [],
  "GraphiteAddr": "",
  "GraphitePath": "",
  "GraphiteConvertHostnameDotsToUnderscores": true,
  "ConsulAddress": "",
  "ConsulAclToken": "",
  "ConsulKVStoreProvider": "consul"
}

切换逻辑如下

逻辑如下：
1、分析mysql实例的故障类型并得到DeadCoMaster的故障
2、获取DeadCoMaster对应的处理函数，为checkAndRecoverDeadCoMaster()
3、登记failure detection记录到后端数据库并执行OnFailureDetectionProcesses的hook。执行成功则执行下面的流程，否则不执行下面所有流程
   1、执行PreFailoverProcesses的hook，成功则执行下面的子流程
     1、选择新主
     2、无论选择新主成功与否，清理failure detection的登记，将记录active设为0。
   2、选择新主成功且开启ApplyMySQLPromotionAfterMasterFailover配置时，将新主的read_only设为false
   3、选择新主成功情况下，执行PostMasterFailoverProcesses的hook
   4、如果上述的恢复成功，执行PostFailoverProcesses的hook。上述恢复执行失败则执行PostUnsuccessfulFailoverProcesses的hook

4、准备脚本

起停脚本

start.sh

#!/bin/bash

nohup ./orchestrator --debug --config orchestrator.conf.json http > arch.log &

stop.sh

#!/bin/bash

ps -ef|grep orchestrator|grep http|awk '{print $2}'|xargs -i kill -9 {}

vip切换脚本

orch_hook.sh

#!/bin/bash
isitdead=$1
#cluster=$2
oldmaster=$3
newmaster=$4

ssh=$(which ssh)

logfile="/home/mysql/usr/local/orchestrator/orch.log"
#interface='enp0s3'
interface='eth0'
user=mysql
#VIP=$($ssh -tt ${user}@${oldmaster} "sudo ip address show dev enp0s3|grep -w 'inet'|tail -n 1|awk '{print \$2}'|awk -F/ '{print \$1}'")
VIP='10.37.129.10'
VIP_TEMP=$($ssh -tt ${user}@${oldmaster} "sudo ip address|sed -nr 's#^.*inet (.*)/32.*#\1#gp'")
#remove '\r' at the end $'192.168.56.200\r'
VIP_TEMP=$(echo $VIP_TEMP|awk -F"\\r" '{print $1}')

if [ ${#VIP_TEMP} -gt 0 ]; then
    VIP=$VIP_TEMP
fi
echo ${VIP}
echo ${interface}

if [[ $isitdead == "DeadMaster" ]]; then
        if [ !-z ${!VIP} ] ; then

                echo $(date)
                echo "Revocering from: $isitdead"
                echo "New master is: $newmaster"
                echo "/home/mysql/usr/local/orchestrator/orch_vip.sh -d 1 -n $newmaster -i ${interface} -I $VIP -u ${user} -o ${oldmaster}"
                /home/mysql/usr/local/orchestrator/orch_vip.sh -d 1 -n $newmaster -i ${interface} -I $VIP -u ${user} -o ${oldmaster}
        else

                echo "Cluster does not exist!" | tee $logfile

        fi
fi
[mysql@tidb-4 orchestrator]$ cat orch_hook.sh
#!/bin/bash
isitdead=$1
#cluster=$2
oldmaster=$3
newmaster=$4

ssh=$(which ssh)

logfile="/home/mysql/usr/local/orchestrator/orch.log"
interface='eth0'
user=mysql
VIP='10.37.129.10'
VIP_TEMP=$($ssh -tt ${user}@${oldmaster} "sudo ip address|sed -nr 's#^.*inet (.*)/32.*#\1#gp'")
#remove '\r' at the end $'192.168.56.200\r'
VIP_TEMP=$(echo $VIP_TEMP|awk -F"\\r" '{print $1}')

if [ ${#VIP_TEMP} -gt 0 ]; then
    VIP=$VIP_TEMP
fi
echo ${VIP}
echo ${interface}

if [[ $isitdead == "DeadMaster" ]]; then
        if [ !-z ${!VIP} ] ; then

                echo $(date)
                echo "Revocering from: $isitdead"
                echo "New master is: $newmaster"
                echo "/home/mysql/usr/local/orchestrator/orch_vip.sh -d 1 -n $newmaster -i ${interface} -I $VIP -u ${user} -o ${oldmaster}"
                /home/mysql/usr/local/orchestrator/orch_vip.sh -d 1 -n $newmaster -i ${interface} -I $VIP -u ${user} -o ${oldmaster}
        else

                echo "Cluster does not exist!" | tee $logfile

        fi
fi
[mysql@tidb-4 orchestrator]$ cat orch_vip.sh
#!/bin/bash

function usage {
  cat << EOF
 usage: $0 [-h] [-d master is dead] [-o old master ] [-s ssh options] [-n new master] [-i interface] [-I] [-u SSH user]

 OPTIONS:
    -h        Show this message
    -o string Old master hostname or IP address
    -d int    If master is dead should be 1 otherweise it is 0
    -s string SSH options
    -n string New master hostname or IP address
    -i string Interface exmple eth0:1
    -I string Virtual IP
    -u string SSH user
EOF

}

while getopts ho:d:s:n:i:I:u: flag; do
  case $flag in
    o)
      orig_master="$OPTARG";
      ;;
    d)
      isitdead="${OPTARG}";
      ;;
    s)
      ssh_options="${OPTARG}";
      ;;
    n)
      new_master="$OPTARG";
      ;;
    i)
      interface="$OPTARG";
      ;;
    I)
      vip="$OPTARG";
      ;;
    u)
      ssh_user="$OPTARG";
      ;;
    h)
      usage;
      exit 0;
      ;;
    *)
      usage;
      exit 1;
      ;;
  esac
done


if [ $OPTIND -eq 1 ]; then
    echo "No options were passed";
    usage;
fi

shift $(( OPTIND - 1 ));

# discover commands from our path
ssh=$(which ssh)
arping=$(which arping)
ip2util=$(which ip)
#ip2util='ip'

# command for adding our vip
cmd_vip_add="sudo -n $ip2util address add $vip dev $interface"
# command for deleting our vip
cmd_vip_del="sudo -n $ip2util address del $vip/32 dev $interface"
# command for discovering if our vip is enabled
cmd_vip_chk="sudo -n $ip2util address show dev $interface to ${vip%/*}/32"
# command for sending gratuitous arp to announce ip move
cmd_arp_fix="sudo -n $arping -c 1 -I ${interface} ${vip%/*}"
# command for sending gratuitous arp to announce ip move on current server
#cmd_local_arp_fix="sudo -n $arping -c 1 -I ${interface} ${vip%/*}"
cmd_local_arp_fix="$arping -c 1 -I ${interface} ${vip%/*}"

vip_stop() {
    rc=0
    echo $?

    echo "$ssh ${ssh_options} -tt ${ssh_user}@${orig_master} \
       \"[ -n \"\$(${cmd_vip_chk})\" ] && ${cmd_vip_del} && \
       sudo -n ${ip2util} route flush cache || [ -z \"\$(${cmd_vip_chk})\" ]\""

    # ensure the vip is removed
    $ssh ${ssh_options} -tt ${ssh_user}@${orig_master} \
       "[ -n \"\$(${cmd_vip_chk})\" ] && ${cmd_vip_del} && \
       sudo -n ${ip2util} route flush cache || [ -z \"\$(${cmd_vip_chk})\" ]"
    rc=$?
    return $rc
}

vip_start() {
    rc=0

    # ensure the vip is added
    # this command should exit with failure if we are unable to add the vip
    # if the vip already exists always exit 0 (whether or not we added it)
    echo "$ssh ${ssh_options} -tt ${ssh_user}@${new_master} \
     \"[ -z \"\$(${cmd_vip_chk})\" ] && ${cmd_vip_add} && ${cmd_arp_fix} || [ -n \"\$(${cmd_vip_chk})\" ]\""

    $ssh ${ssh_options} -tt ${ssh_user}@${new_master} \
     "[ -z \"\$(${cmd_vip_chk})\" ] && ${cmd_vip_add} && ${cmd_arp_fix} || [ -n \"\$(${cmd_vip_chk})\" ]"
    rc=$?
    echo "vip started"
    #$cmd_local_arp_fix
    return $rc
}

vip_status() {
    $arping -c 1 -I ${interface} ${vip%/*}
    echo "$arping -c 1 -I ${interface} ${vip%/*}"
    if ping -c 1 -W 1 "$vip"; then
        return 0
    else
        return 1
    fi
}
if [[ $isitdead == 0 ]]; then
    echo "Online failover"
    if vip_stop; then
        if vip_start; then
            echo "$vip is moved to $new_master."
        else
            echo "Can't add $vip on $new_master!"
            exit 1
        fi
    else
        echo $rc
        echo "Can't remove the $vip from orig_master!"
        exit 1
    fi
elif [[ $isitdead == 1 ]]; then
    echo "Master is dead, failover"
    # make sure the vip is not available
    if vip_status; then
        if vip_stop; then
            echo "$vip is removed from orig_master."
        else
            echo $rc
            echo "Couldn't remove $vip from orig_master."
            exit 1
        fi
    fi

    if vip_start; then
          echo "$vip is moved to $new_master."

    else
          echo "Can't add $vip on $new_master!"
          exit 1
    fi
else
    echo "Wrong argument, the master is dead or live?"

fi

5、启动orch并检查拓扑

./start.sh

$ orchestrator-client -c topology -i tidb-4.0-control:3306
 tidb-4-1:3306           [0s,ok,5.7.41-log,rw,ROW,>>,GTID,semi:master]
+ tidb-4-2:3306         [0s,ok,5.7.41-log,ro,ROW,>>,GTID,semi:master,semi:replica]
+ tidb-4.0-control:3306 [0s,ok,5.7.41-log,rw,ROW,>>,GTID,semi:master,semi:replica]

6、手动平滑切换

orchestrator-client -c graceful-master-takeover -a tidb-4.0-control:3306 -d tidb-4-1:3306

注： -a 为旧主， -d为新主。切换后旧主需手动start slave。

7、强制关闭主库切换

确保旧主库已经提前配置vip，如没配置需执行：

/usr/sbin/ip address add 10.37.129.10 dev eth0

停止主库：

mysql > shutdown;

自动重新change master，并且飘移vip.

该场景下的问题：在主从延迟的情况下，从库会立即提升为主库，新主库有数据不一致的问题。

标签：false,讲解,orchestrator,echo,VIP,orch,使用,Orchestrator,true
From： https://www.cnblogs.com/longfeij/p/18279466