psatck
pstack命令是一个在Linux系统中用于查看进程堆栈信息的工具。
写了一个服务端死锁程序，如下：
#include <iostream>
#include <thread>
#include <mutex>
#include <chrono>
#include <sys/socket.h>
#include <netinet/in.h>
#include <unistd.h>
#include <cstring>
// 定义两个互斥锁
std::mutex mutex1;
std::mutex mutex2;
// 处理客户端请求的函数
void handle_client(int client_socket, int client_id) {
std::cout << "Client " << client_id << ": Connected" << std::endl;
// 模拟客户端请求处理
if (client_id == 1) {
// 客户端 1：先获取 mutex1，再获取 mutex2
std::cout << "Client " << client_id << ": Trying to lock mutex1..." << std::endl;
std::lock_guard<std::mutex> lock1(mutex1);
std::this_thread::sleep_for(std::chrono::seconds(5)); // 增加锁的持有时间
std::cout << "Client " << client_id << ": Locked mutex1, now trying to lock mutex2..." << std::endl;
// 尝试获取 mutex2
std::lock_guard<std::mutex> lock2(mutex2); // 死锁发生点
std::this_thread::sleep_for(std::chrono::seconds(5)); // 模拟更多工作
std::cout << "Client " << client_id << ": Locked both mutex1 and mutex2" << std::endl;
} else if (client_id == 2) {
// 客户端 2：先获取 mutex2，再获取 mutex1
std::cout << "Client " << client_id << ": Trying to lock mutex2..." << std::endl;
std::lock_guard<std::mutex> lock2(mutex2);
std::this_thread::sleep_for(std::chrono::seconds(5)); // 增加锁的持有时间
std::cout << "Client " << client_id << ": Locked mutex2, now trying to lock mutex1..." << std::endl;
// 尝试获取 mutex1
std::lock_guard<std::mutex> lock1(mutex1); // 死锁发生点
std::this_thread::sleep_for(std::chrono::seconds(5)); // 模拟更多工作
std::cout << "Client " << client_id << ": Locked both mutex1 and mutex2" << std::endl;
}
// 关闭客户端连接
close(client_socket);
std::cout << "Client " << client_id << ": Disconnected" << std::endl;
}
// TCP 服务器主函数
void start_server(int port) {
int server_fd, new_socket;
struct sockaddr_in address;
int opt = 1;
int addrlen = sizeof(address);
// 创建 socket 文件描述符
if ((server_fd = socket(AF_INET, SOCK_STREAM, 0)) == 0) {
perror("socket failed");
exit(EXIT_FAILURE);
}
// 设置 SO_REUSEADDR 选项
if (setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR | SO_REUSEPORT, &opt, sizeof(opt))) {
perror("setsockopt failed");
exit(EXIT_FAILURE);
}
// 绑定 socket 到指定端口
address.sin_family = AF_INET;
address.sin_addr.s_addr = INADDR_ANY;
address.sin_port = htons(port);
if (bind(server_fd, (struct sockaddr *)&address, sizeof(address)) < 0) {
perror("bind failed");
exit(EXIT_FAILURE);
}
// 监听连接
if (listen(server_fd, 3) < 0) {
perror("listen failed");
exit(EXIT_FAILURE);
}
std::cout << "Server started on port " << port << ". Waiting for connections..." << std::endl;
int client_id = 1; // 用于区分不同客户端
while (true) {
// 接受新的客户端连接
if ((new_socket = accept(server_fd, (struct sockaddr *)&address, (socklen_t*)&addrlen)) < 0) {
perror("accept failed");
continue;
}
// 为每个客户端创建一个新线程
std::thread client_thread(handle_client, new_socket, client_id++);
client_thread.detach(); // 分离线程，允许其独立运行
}
}
int main() {
int port = 8080;
start_server(port);
return 0;
}
tcp_deadlock_server.cpp
编译：g++ -std=c++11 -pthread -o tcp_deadlock_server tcp_deadlock_server.cpp -g
用telnet(telnet 127.1 8080)连两次就会死锁，服务端输出如下：
Server started on port 8080. Waiting for connections...
Client 1: Connected
Client 1: Trying to lock mutex1...
Client 2: Connected
Client 2: Trying to lock mutex2...
Client 1: Locked mutex1, now trying to lock mutex2...
Client 2: Locked mutex2, now trying to lock mutex1...
pstack调试死锁
ps查看进程ID，然后pstack + 进程ID : pstack 915 > pstack_out，将输出重定向到文件，好看一些：
Thread 3 (LWP 919): #0 0x0000fffcc23821dc in ?? () from /lib64/libpthread.so.0 #1 0x0000fffcc237b060 in pthread_mutex_lock () from /lib64/libpthread.so.0 #2 0x00000000004012c4 in __gthread_mutex_lock (__mutex=0x420240 <mutex1>) at /usr/include/c++/7.3.0/aarch64-linux-gnu/bits/gthr-default.h:748 #3 0x0000000000401a88 in std::mutex::lock (this=0x420240 <mutex1>) at /usr/include/c++/7.3.0/bits/std_mutex.h:103 #4 0x0000000000401b34 in std::lock_guard<std::mutex>::lock_guard (this=0xfffcc19ce810, __m=...) at /usr/include/c++/7.3.0/bits/std_mutex.h:162 #5 0x00000000004015ac in handle_client (client_socket=5, client_id=2) at tcp_deadlock_server.cpp:38 #6 0x0000000000402308 in std::__invoke_impl<void, void (*)(int, int), int, int> (__f=@0x248c23e0: 0x401310 <handle_client(int, int)>, __args#0=@0x248c23dc: 5, __args#1=@0x248c23d8: 2) at /usr/include/c++/7.3.0/bits/invoke.h:60 #7 0x0000000000401e18 in std::__invoke<void (*)(int, int), int, int> (__fn=@0x248c23e0: 0x401310 <handle_client(int, int)>, __args#0=@0x248c23dc: 5, __args#1=@0x248c23d8: 2) at /usr/include/c++/7.3.0/bits/invoke.h:95 #8 0x00000000004029cc in std::thread::_Invoker<std::tuple<void (*)(int, int), int, int> >::_M_invoke<0ul, 1ul, 2ul> (this=0x248c23d8) at /usr/include/c++/7.3.0/thread:234 #9 0x0000000000402970 in std::thread::_Invoker<std::tuple<void (*)(int, int), int, int> >::operator() (this=0x248c23d8) at /usr/include/c++/7.3.0/thread:243 #10 0x0000000000402950 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (*)(int, int), int, int> > >::_M_run (this=0x248c23d0) at /usr/include/c++/7.3.0/thread:186 #11 0x0000fffcc257e134 in ?? () from /lib64/libstdc++.so.6 #12 0x0000fffcc23788cc in ?? () from /lib64/libpthread.so.0 #13 0x0000fffcc22ba1ec in ?? () from /lib64/libc.so.6 Thread 2 (LWP 917): #0 0x0000fffcc23821dc in ?? () from /lib64/libpthread.so.0 #1 0x0000fffcc237b060 in pthread_mutex_lock () from /lib64/libpthread.so.0 #2 0x00000000004012c4 in __gthread_mutex_lock (__mutex=0x420270 <mutex2>) at /usr/include/c++/7.3.0/aarch64-linux-gnu/bits/gthr-default.h:748 #3 0x0000000000401a88 in std::mutex::lock (this=0x420270 <mutex2>) at /usr/include/c++/7.3.0/bits/std_mutex.h:103 #4 0x0000000000401b34 in std::lock_guard<std::mutex>::lock_guard (this=0xfffcc21de820, __m=...) at /usr/include/c++/7.3.0/bits/std_mutex.h:162 #5 0x0000000000401450 in handle_client (client_socket=4, client_id=1) at tcp_deadlock_server.cpp:27 #6 0x0000000000402308 in std::__invoke_impl<void, void (*)(int, int), int, int> (__f=@0x248c2290: 0x401310 <handle_client(int, int)>, __args#0=@0x248c228c: 4, __args#1=@0x248c2288: 1) at /usr/include/c++/7.3.0/bits/invoke.h:60 #7 0x0000000000401e18 in std::__invoke<void (*)(int, int), int, int> (__fn=@0x248c2290: 0x401310 <handle_client(int, int)>, __args#0=@0x248c228c: 4, __args#1=@0x248c2288: 1) at /usr/include/c++/7.3.0/bits/invoke.h:95 #8 0x00000000004029cc in std::thread::_Invoker<std::tuple<void (*)(int, int), int, int> >::_M_invoke<0ul, 1ul, 2ul> (this=0x248c2288) at /usr/include/c++/7.3.0/thread:234 #9 0x0000000000402970 in std::thread::_Invoker<std::tuple<void (*)(int, int), int, int> >::operator() (this=0x248c2288) at /usr/include/c++/7.3.0/thread:243 #10 0x0000000000402950 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (*)(int, int), int, int> > >::_M_run (this=0x248c2280) at /usr/include/c++/7.3.0/thread:186 #11 0x0000fffcc257e134 in ?? () from /lib64/libstdc++.so.6 #12 0x0000fffcc23788cc in ?? () from /lib64/libpthread.so.0 #13 0x0000fffcc22ba1ec in ?? () from /lib64/libc.so.6 Thread 1 (LWP 915): #0 0x0000fffcc23827c4 in accept () from /lib64/libpthread.so.0 #1 0x0000000000401868 in start_server (port=8080) at tcp_deadlock_server.cpp:89 #2 0x00000000004018f8 in main () at tcp_deadlock_server.cpp:102pstack 输出
能看到一共三个线程，Thread 3 (LWP 919)卡在pthread_mutex_lock：#5 0x00000000004015ac in handle_client (client_socket=5, client_id=2) at tcp_deadlock_server.cpp:38，
Thread 2 (LWP 917)卡在pthread_mutex_lock：#5 0x0000000000401450 in handle_client (client_socket=4, client_id=1) at tcp_deadlock_server.cpp:27
就发现了死锁在的位置
gdb
不用c++11又写了一个程序，如下：
#include <iostream>
#include <cstring>
#include <unistd.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <pthread.h>
#include <errno.h>
#include <cstdlib>
#include <fcntl.h>
#include <stdio.h>
// 定义两个互斥锁
pthread_mutex_t mutex1 = PTHREAD_MUTEX_INITIALIZER;
pthread_mutex_t mutex2 = PTHREAD_MUTEX_INITIALIZER;
// 处理客户端请求的函数
void* handle_client(void* arg) {
int client_socket = *(static_cast<int*>(arg));
free(arg); // 释放传递给线程的动态分配的内存
std::cout << "Client connected with socket: " << client_socket << std::endl;
// 模拟客户端请求处理
if (client_socket == 4) {
// 客户端 1：先获取 mutex1，再获取 mutex2
std::cout << "Client " << client_socket << ": Trying to lock mutex1..." << std::endl;
pthread_mutex_lock(&mutex1);
usleep(5000000); // 休眠 5 秒
std::cout << "Client " << client_socket << ": Locked mutex1, now trying to lock mutex2..." << std::endl;
// 尝试获取 mutex2
pthread_mutex_lock(&mutex2); // 死锁发生点
usleep(5000000); // 休眠 5 秒
std::cout << "Client " << client_socket << ": Locked both mutex1 and mutex2" << std::endl;
// 释放互斥锁
pthread_mutex_unlock(&mutex2);
pthread_mutex_unlock(&mutex1);
} else if (client_socket == 5) {
// 客户端 2：先获取 mutex2，再获取 mutex1
std::cout << "Client " << client_socket << ": Trying to lock mutex2..." << std::endl;
pthread_mutex_lock(&mutex2);
usleep(5000000); // 休眠 5 秒
std::cout << "Client " << client_socket << ": Locked mutex2, now trying to lock mutex1..." << std::endl;
// 尝试获取 mutex1
pthread_mutex_lock(&mutex1); // 死锁发生点
usleep(5000000); // 休眠 5 秒
std::cout << "Client " << client_socket << ": Locked both mutex1 and mutex2" << std::endl;
// 释放互斥锁
pthread_mutex_unlock(&mutex1);
pthread_mutex_unlock(&mutex2);
}
// 关闭客户端连接
close(client_socket);
std::cout << "Client disconnected with socket: " << client_socket << std::endl;
pthread_exit(NULL);
}
// TCP 服务器主函数
void start_server(int port) {
int server_fd, new_socket;
struct sockaddr_in address;
int opt = 1;
int addrlen = sizeof(address);
// 创建 socket 文件描述符
if ((server_fd = socket(AF_INET, SOCK_STREAM, 0)) == 0) {
perror("socket failed");
exit(EXIT_FAILURE);
}
// 设置 SO_REUSEADDR 选项
if (setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt))) {
perror("setsockopt failed");
exit(EXIT_FAILURE);
}
// 绑定 socket 到指定端口
address.sin_family = AF_INET;
address.sin_addr.s_addr = INADDR_ANY;
address.sin_port = htons(port);
if (bind(server_fd, (struct sockaddr *)&address, sizeof(address)) < 0) {
perror("bind failed");
exit(EXIT_FAILURE);
}
// 监听连接
if (listen(server_fd, 3) < 0) {
perror("listen failed");
exit(EXIT_FAILURE);
}
std::cout << "Server started on port " << port << ". Waiting for connections..." << std::endl;
int client_id = 1; // 用于区分不同客户端
while (true) {
// 接受新的客户端连接
if ((new_socket = accept(server_fd, (struct sockaddr *)&address, (socklen_t*)&addrlen)) < 0) {
perror("accept failed");
continue;
}
// 为每个客户端创建一个新线程
pthread_t thread;
int* client_socket_ptr = new int(new_socket); // 动态分配存储套接字描述符的内存
if (pthread_create(&thread, NULL, handle_client, static_cast<void*>(client_socket_ptr)) != 0) {
perror("pthread_create failed");
delete client_socket_ptr; // 如果线程创建失败，释放内存
close(new_socket);
continue;
}
// 分离线程，允许其独立运行
pthread_detach(thread);
// 为了测试死锁，只接受前两个客户端连接
if (client_id >= 3) {
close(new_socket); // 关闭多余的连接
continue;
}
client_id++;
}
}
int main() {
int port = 8080;
start_server(port);
return 0;
}
tcp_deadlock_server_c++0x.cpp
编译运行telnet测试跟上面一样
gdb调试死锁
ps查看进程ID，然后gdb跟进程：gdb -p 11560
查看所有线程：info threads，进入线程：thread 2 ,然后bt查看线程堆栈，切换另一个线程如上，就能看到两个线程都卡在了lock，具体调试步骤如下：
(gdb) info threads 3 Thread 0x7fb6c0115700 (LWP 11562) 0x0000003b5200dff4 in __lll_lock_wait () from /lib64/libpthread.so.0 2 Thread 0x7fb6bf714700 (LWP 11564) 0x0000003b5200dff4 in __lll_lock_wait () from /lib64/libpthread.so.0 * 1 Thread 0x7fb6c0117720 (LWP 11560) 0x0000003b5200e7ed in accept () from /lib64/libpthread.so.0 (gdb) thread 2 [Switching to thread 2 (Thread 0x7fb6bf714700 (LWP 11564))]#0 0x0000003b5200dff4 in __lll_lock_wait () from /lib64/libpthread.so.0 (gdb) bt #0 0x0000003b5200dff4 in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x0000003b52009328 in _L_lock_854 () from /lib64/libpthread.so.0 #2 0x0000003b520091f7 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000000400f9f in handle_client (arg=0x13a8010) at tcp_deadlock_server.cpp:48 #4 0x0000003b520077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003b51ce570d in clone () from /lib64/libc.so.6 (gdb) thread 3 [Switching to thread 3 (Thread 0x7fb6c0115700 (LWP 11562))]#0 0x0000003b5200dff4 in __lll_lock_wait () from /lib64/libpthread.so.0 (gdb) bt #0 0x0000003b5200dff4 in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x0000003b52009328 in _L_lock_854 () from /lib64/libpthread.so.0 #2 0x0000003b520091f7 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000000400eb2 in handle_client (arg=0x13a8010) at tcp_deadlock_server.cpp:33 #4 0x0000003b520077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003b51ce570d in clone () from /lib64/libc.so.6gdb调试死锁
总结
pstack和gdb都使用 ptrace() 系统调用来附着到目标进程。ptrace() 允许 GDB 暂停目标进程的执行，读取和修改其内存及寄存器，并捕获系统调用。