docker 多核环境上批量创建容器慢

多核系统批量创建容器导致容器创建时间长

package main

import (
	"fmt"
	"os/exec"

	"sync"
	"time"
)

func main() {
	cmd := exec.Command("sh", "-c", "docker rm -f  `docker ps -qa`")
	cmd.Output()
	var wg sync.WaitGroup
	for i := 0; i <= 200; i++ {
		wg.Add(1)
		go func() {
			t1 := time.Now().UnixMilli()
			cmd := exec.Command("sh", "-c", "docker run -d busybox top")
			cmd.Output()
			wg.Done()
			t2 := time.Now().UnixMilli()
			fmt.Println(t2 - t1)
		}()
	}

	chDone := make(chan struct{})
	go func() {
		wg.Wait()
		close(chDone)
	}()

	select {
	case <-chDone:
	}
	cmd = exec.Command("sh", "-c", "docker rm -f  `docker ps -qa`")
	cmd.Output()
}

查看pprof 占用资源高的地方如下

137 @ 0x119f176 0x11b02d3 0x11b02ad 0x11cc2a5 0x11dd3c5 0x1cd82d5 0x1cd82bb 0x1cd82ba 0x20c52ed 0x2940d4e 0x293f9c5 0x294272c 0x29a6ed2 0x29a67a5 0x24e8ac8 0x1b7289b 0x1b73097 0x15803e9 0x1ba405a 0x152aeef 0x158d90f 0x152e8db 0x1529997 0x11d04e1
# 0x11cc2a4 sync.runtime_SemacquireMutex+0x24 /usr/local/go/src/runtime/sema.go:71
# 0x11dd3c4 sync.(*Mutex).lockSlow+0x164 /usr/local/go/src/sync/mutex.go:162
# 0x1cd82d4 sync.(*Mutex).Lock+0x154 /usr/local/go/src/sync/mutex.go:81
# 0x1cd82ba github.com/docker/docker/vendor/github.com/moby/locker.(*lockCtr).Lock+0x13a /root/go/src/github.com/docker/docker/vendor/github.com/moby/locker/locker.go:56
# 0x1cd82b9 github.com/docker/docker/vendor/github.com/moby/locker.(*Locker).Lock+0x139 /root/go/src/github.com/docker/docker/vendor/github.com/moby/locker/locker.go:91
# 0x20c52ec github.com/docker/docker/libnetwork.(*network).CreateEndpoint+0xec /root/go/src/github.com/docker/docker/libnetwork/network.go:1167
# 0x2940d4d github.com/docker/docker/daemon.(*Daemon).connectToNetwork+0x3ed /root/go/src/github.com/docker/docker/daemon/container_operations.go:784
# 0x293f9c4 github.com/docker/docker/daemon.(*Daemon).allocateNetwork+0x3c4 /root/go/src/github.com/docker/docker/daemon/container_operations.go:562
# 0x294272b github.com/docker/docker/daemon.(*Daemon).initializeNetworking+0xcb /root/go/src/github.com/docker/docker/daemon/container_operations.go:987
# 0x29a6ed1 github.com/docker/docker/daemon.(*Daemon).containerStart+0x351 /root/go/src/github.com/docker/docker/daemon/start.go:149
# 0x29a67a4 github.com/docker/docker/daemon.(*Daemon).ContainerStart+0x2e4 /root/go/src/github.com/docker/docker/daemon/start.go:94
# 0x24e8ac7 github.com/docker/docker/api/server/router/container.(*containerRouter).postContainersStart+0x247 /root/go/src/github.com/docker/docker/api/server/router/container/container_routes.go:217
# 0x1b7289a github.com/docker/docker/api/server/middleware.ExperimentalMiddleware.WrapHandler.func1+0x15a /root/go/src/github.com/docker/docker/api/server/middleware/experimental.go:26
# 0x1b73096 github.com/docker/docker/api/server/middleware.VersionMiddleware.WrapHandler.func1+0x4d6 /root/go/src/github.com/docker/docker/api/server/middleware/version.go:62
# 0x15803e8 github.com/docker/docker/pkg/authorization.(*Middleware).WrapHandler.func1+0x648 /root/go/src/github.com/docker/docker/pkg/authorization/middleware.go:59
# 0x1ba4059 github.com/docker/docker/api/server.(*Server).makeHTTPHandler.func1+0x2f9 /root/go/src/github.com/docker/docker/api/server/server.go:142
# 0x152aeee net/http.HandlerFunc.ServeHTTP+0x2e /usr/local/go/src/net/http/server.go:2084
# 0x158d90e github.com/docker/docker/vendor/github.com/gorilla/mux.(*Router).ServeHTTP+0x1ce /root/go/src/github.com/docker/docker/vendor/github.com/gorilla/mux/mux.go:210
# 0x152e8da net/http.serverHandler.ServeHTTP+0x43a /usr/local/go/src/net/http/server.go:2916
# 0x1529996 net/http.(*conn).serve+0x5d6 /usr/local/go/src/net/http/server.go:1966

耗时卡在n.ctrlr.networkLocker.Lock(n.id) （全局锁，所有容器创建都会去抢这把锁），当大批量创建容器时候大量容器都去等待这把锁导致容器创建慢。这个全局锁引入的原因是为了避免创建网络和删除网络导致竞争关系。

在5.10 内核上慢的原因是 ，5.10内核创建veth速率比较快，会导致更多的锁去等待获取。如果在创建bridge网络那里加上延时20ms，性能就有提升（加了延时后会让等锁的容器个数下降，锁竞争不是那么激烈），这个内核补丁 https://github.com/torvalds/linux/commit/9d3684c24a52 有效的原因是合入该补丁后创建veth设备对的时间变长了，导致等锁的容器个数变多了，锁竞争加剧导致性能下降。

解决方法：

    1.移除全局锁n.ctrlr.networkLocker.Lock，这个锁和删除网络有竞争关系，不容易移除。

    2.使用该锁的函数前增加一个全局的锁，让锁竞争不是那么激烈。

    3.合入内核补丁  https://github.com/torvalds/linux/commit/9d3684c24a52 不能根本解决问题

func (daemon *Daemon) connectToNetwork(container *container.Container, idOrName string, endpointConfig *networktypes.EndpointSettings, updateSettings bool) (err error) {
	mutex.Lock()
	defer mutex.Unlock()

优化后性能提升40%左右

200 容器并行创建从 16523ms减少到 10171ms