千级边缘节点应用分钟级部署测试验证

1.测试组网

一台主节点master部署k8s,mef-center组件，cloud core

7台服务器模拟边缘节点部署docker,edge core

2.模拟边缘节点步骤

2.1 制作边缘容器镜像

拉取官网ubuntu镜像，更新镜像源，安装docker.io,安装 systemd 所需要的 Init 重新提交docker镜像

2.2 执行特权容器

docker run -itd  --privileged myubuntu:v1.0  /usr/sbin/init

2.3 执行编写的脚本

#!/bin/bash
systemctl start docker
docker load -i /mnt/kubeedge-pause.tar
docker load -i /mnt/busybox-131.tar
hostname=$(cat /etc/hostname)
sed -i "s/myubuntu/$hostname/g"  /etc/kubeedge/config/edgecore.yaml
echo $(pwd)
/mnt/edgecore >> edgecore.log 2>&1

2.3 执行批量部署容器脚本

#!/bin/bash
imageName="myubuntu:v1.0" 
total=$(docker ps -a|grep $imageName|wc -l)
echo $total
while(($dockerCount<=10))
do
    totalCnt=$(($total+$dockerCount))
    createName="node1-"$totalCnt
    docker run -itd  --privileged --name="${createName}" $imageName /usr/sbin/init 
    containerID=$(docker ps|grep $imageName|grep "$createName"|awk '{print $1}') 
    let "dockerCount++"
done

2.4 批量单独部署脚本

#!/bin/bash

imageName="myubuntu:v1.0"
containerIDs=$(docker ps|grep $imageName|awk '{print $1}')
for containerID in $containerIDs
do 
	docker exec -itd $containerID /bin/bash -c run.sh
done;

faq

1.Failed to allocate directory watch: Too many open files

vim /etc/sysctl.conf

fs.inotify.max_user_instances=512
fs.inotify.max_user_watches=262144

sysctl -psysctl fs.inotify

2 ws.go:32] 2023/04/06 01:02:23 http: Accept error: accept tcp [::]:10000: accept4: too many open files; retrying in 1s

/etc/security/limits.conf
* soft nofile 65535
* hard nofile 65535

ulimit -n

[Service] 
LimitNOFILE=10000 

3.批量纳管节点

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math"
	"net/http"
	"strconv"
	"strings"
)

func main() {

	client := &http.Client{}

	unmanagedUrl := "http://10.44.115.108:30034/edgemanager/v1/node/list/unmanaged?pageNum=1&pageSize=100"
	unmanagedMethod := "GET"

	unmanagedReq, err := http.NewRequest(unmanagedMethod, unmanagedUrl, nil)

	if err != nil {
		fmt.Println(err)
		return
	}
	unmanagedRes, err := client.Do(unmanagedReq)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer unmanagedRes.Body.Close()

	unmanagedBody, err := ioutil.ReadAll(unmanagedRes.Body)
	if err != nil {
		fmt.Println(err)
		return
	}
	type Result struct {
		Status int32                  `json:"status"`
		Msg    string                 `json:"msg"`
		Data   map[string]interface{} `json:"data"`
	}
	unmanagedSlice := &Result{}
	json.Unmarshal(unmanagedBody, &unmanagedSlice)
	unmanagedSliceData, _ := unmanagedSlice.Data["nodes"].([]interface{})
	for _, item := range unmanagedSliceData {
		s, _ := item.(map[string]interface{})
		if math.Max(s["id"].(float64), 3) != 3 {
			addId, _ := s["id"].(float64)
			addNodeName, _ := s["nodeName"].(string)
			addUrl := "http://10.44.115.108:30034/edgemanager/v1/node/add"
			addMethod := "POST"

			addNode := strconv.Itoa(int(addId))
			addName := addNodeName + "-" + addNode

			postStr := `{"nodeID":` + addNode + `, "name":"` + addName + `", "groupIDs":[2]}`
			fmt.Println(postStr)
			payload := strings.NewReader(postStr)
			addReq, err := http.NewRequest(addMethod, addUrl, payload)

			if err != nil {
				fmt.Println(err)
				return
			}
			addReq.Header.Add("Content-Type", "text/plain")

			addRes, err := client.Do(addReq)
			if err != nil {
				fmt.Println(err)
				return
			}
			defer addRes.Body.Close()

			addBody, err := ioutil.ReadAll(addRes.Body)
			if err != nil {
				fmt.Println(err)
				return
			}
			fmt.Println(string(addBody))
		}
	}
}

4.边缘节点包地址

10.44.115.106@:/root/gwx/edge.zip 解压即可

5.maxNode个数现在修改点

5.1 cloudcore.yaml 修改

参考资料

https://kubeedge.io/zh/blog/scalability-test-report/

5.2 edgemanager 修改

pkg/nodemanger/node_const.go

 目前是1024根据部署模拟节点进行修改
const ( 
	maxNode            = 1024 
) 

修改后重新编译部署即可