dcrubro
4d39614cb5
Found by running two nodes under -fsanitize=thread through connect -> mine -> broadcast -> clean
exit. None are in consensus code; all are connection setup/teardown.
Stop flags were plain or volatile ints written by one thread and read as a loop condition by
another. volatile stops the compiler hoisting the load but provides neither atomicity nor
ordering, which on arm64 is a real visibility gap, not just a sanitizer complaint. Now _Atomic:
- net_node_t.maintenanceRunning (Node_Destroy vs Node_MaintenanceThread)
- tcp_server_t.isRunning (TcpServer_Stop vs both accept threads) -- was a bare int, so
the accept loop could legally be hoisted and never see the stop
- udp_node_t.isRunning (UdpNode_Stop vs the recv and retry threads)
TcpServer_Stop had a use-after-free, not merely a race: it took clientsMutex only long enough to
read the array pointer, then walked the slots unlocked. An exiting client thread clears its own
slot under that mutex and immediately destroys and frees the connection, so Stop could
RequestClose and pthread_join a freed pointer on any shutdown with an active peer.
- Stop now requests the close and copies each pthread_t under the mutex, then joins from the
copied handles, so the connection is never dereferenced outside the lock
- the client thread's TcpConnection_Destroy/free moved inside the same critical section, which
closes the window entirely
Client threads that disconnected normally were never joined and leaked their thread resources:
Stop only joins clients still present in the array, and a normal exit removes itself first. Stop
now claims each slot as it copies the handle, so the client thread can tell who owns its join --
it detaches itself if it successfully removed its own slot, and stays joinable if Stop already
claimed it. Both decisions happen under clientsMutex so the cases cannot interleave.
Node_Destroy cleared outbound slots with no lock (via TcpClient_Disconnect) while live inbound
client threads read the same field correctly under outboundLock in Node_HasLiveConnectionTo. The
lock cannot just be held across the destroy, because that path joins an io thread whose
on_disconnect callback takes outboundLock itself. Reworked to detach the connections from their
slots under the lock and tear them down outside it -- the pattern Node_ReapDeadOutbound already
uses in this file. This one only surfaced once the other five were fixed.
Verified: TSan clean over the same run; nodes still converge (height 25, identical tip); shallow
fork still adopts, depth-8 fork still defers on the reorg penalty, and the forced-orphan
regression still reaches full height with zero coinbase rejections.
101 lines
4.1 KiB
C
101 lines
4.1 KiB
C
#ifndef NET_NODE_H
|
|
#define NET_NODE_H
|
|
|
|
#ifndef _WIN32
|
|
// POSIX
|
|
#include <tcpd/tcpconnection.h>
|
|
#include <tcpd/tcpclient.h>
|
|
#include <tcpd/tcpserver.h>
|
|
#endif
|
|
|
|
#include <constants.h>
|
|
#include <packettype.h>
|
|
#include <udpd/udpnode.h>
|
|
|
|
// Forward declaration - the discovery state is defined in nodediscovery.c (opaque here).
|
|
typedef struct node_discovery node_discovery_t;
|
|
|
|
#include <stddef.h>
|
|
|
|
#include <dynarr.h>
|
|
#include <dynset.h>
|
|
|
|
#include <pthread.h>
|
|
|
|
#include <block/block.h>
|
|
#include <block/chain.h>
|
|
#include <block/transaction.h>
|
|
#include <stdatomic.h>
|
|
|
|
typedef struct {
|
|
tcp_server_t* server;
|
|
tcp_client_t outboundClients[MAX_CONS];
|
|
size_t outboundCount;
|
|
// Dedup cache for recently seen block hashes (canonical 32-byte hash)
|
|
DynSet* seenBlocks;
|
|
// Protects seenBlocks
|
|
pthread_mutex_t seenLock;
|
|
// Protects outboundClients snapshots and peerBlockHeight writes
|
|
pthread_mutex_t outboundLock;
|
|
void (*on_connect)(tcp_connection_t* conn, void* user);
|
|
void (*on_data)(tcp_connection_t* conn, const unsigned char* data, size_t len, void* user);
|
|
void (*on_disconnect)(tcp_connection_t* conn, void* user);
|
|
void* callbackUser;
|
|
// Maintenance thread for periodic tasks (orphan attach, pruning, metrics)
|
|
pthread_t maintenanceThread;
|
|
// Cross-thread stop flag: written by Node_Destroy on the main thread, read by the maintenance
|
|
// thread's loop condition. `volatile` stops the compiler hoisting the load but provides neither
|
|
// atomicity nor ordering, so this has to be a real atomic (and TSan rightly flagged it).
|
|
_Atomic int maintenanceRunning;
|
|
int maintenanceIntervalMs;
|
|
// UDP ping/pong daemon (latency oracle) and peer discovery state
|
|
udp_node_t* udpNode;
|
|
node_discovery_t* discovery;
|
|
} net_node_t;
|
|
|
|
net_node_t* Node_Create();
|
|
void Node_Destroy(net_node_t* node);
|
|
|
|
void Node_SetCallbacks(
|
|
net_node_t* node,
|
|
void (*on_connect)(tcp_connection_t* conn, void* user),
|
|
void (*on_data)(tcp_connection_t* conn, const unsigned char* data, size_t len, void* user),
|
|
void (*on_disconnect)(tcp_connection_t* conn, void* user),
|
|
void* user
|
|
);
|
|
|
|
int Node_ConnectPeer(net_node_t* node, const char* ip, unsigned short port);
|
|
int Node_ConnectStartupPeers(net_node_t* node, const char** ips, const unsigned short* ports, size_t peersCount);
|
|
|
|
int Node_SendPacket(net_node_t* node, tcp_connection_t* conn, packet_type_t packetType, const void* payload, size_t payloadLen);
|
|
int Node_BroadcastTransaction(net_node_t* node, signed_transaction_t* tx, tcp_connection_t* excludeNode);
|
|
|
|
// Helpers for outbound peer selection and block broadcast
|
|
int Node_GetBestOutboundPeer(net_node_t* node, tcp_connection_t** outConn, uint64_t* outHeight);
|
|
void Node_BroadcastChainRange(net_node_t* node, size_t startHeightInclusive, tcp_connection_t* sourceConn);
|
|
|
|
// Callback logic
|
|
void Node_Server_OnConnect(tcp_connection_t* client);
|
|
void Node_Server_OnData(tcp_connection_t* client);
|
|
void Node_Server_OnDisconnect(tcp_connection_t* client);
|
|
void Node_Client_OnConnect(tcp_connection_t* client);
|
|
void Node_Client_OnData(tcp_connection_t* client);
|
|
void Node_Client_OnDisconnect(tcp_connection_t* client);
|
|
|
|
void Node_GetClientList(net_node_t* node, tcp_connection_t** outClients, size_t* outCount);
|
|
|
|
// Computes a connection's peer listen endpoint (IP + advertised/dialed listen port) into *out.
|
|
// Outbound: the dialed peerAddr port already is the listen port. Inbound: uses peerListenPort.
|
|
// Returns non-zero on success (usable endpoint with a known, non-zero port), zero otherwise.
|
|
int Node_ConnListenEndpoint(const tcp_connection_t* conn, struct sockaddr_storage* out);
|
|
|
|
// Returns the node identity advertised by a connection's peer, or 0 if it is not known yet.
|
|
uint64_t Node_ConnPeerNodeId(const tcp_connection_t* conn);
|
|
|
|
// Fills outEndpoints with the listen endpoints of all current connections (inbound + outbound),
|
|
// deduped by IP+port, and outNodeIds (optional, may be NULL) with the matching peer identities.
|
|
// Returns the number of endpoints written (<= maxOut).
|
|
size_t Node_GetPeerEndpoints(net_node_t* node, struct sockaddr_storage* outEndpoints, uint64_t* outNodeIds, size_t maxOut);
|
|
|
|
#endif
|