dcrubro
0e90f7d5db
Root cause: peers were keyed only by (IP, listen port). An IPv6 host holds several addresses at once, so one node appeared as several peers and could not recognise its own addresses. Protocol: - Added a random per-run node identity (localNodeId), advertised as a length-guarded trailing field in HELLO and ACK_HELLO — backwards compatible with peers that omit it - Added peerNodeId to tcp_connection_t; peers are now identified by this rather than by an endpoint - ACK_HELLO is now sent before any decision to drop the connection, so a rejected dialer learns whose address it reached instead of retrying forever Self-connection: - Identity match → close the connection and record that endpoint permanently as our own - Self-endpoint set preemptively seeded from getifaddrs() at startup, so a node knows its own addresses before ever dialing one - Node_ConnectPeer refuses self endpoints, which also kills the echo-back chain that could exhaust connection slots Duplicate connections and churn: - Dedup by identity per direction — one inbound and one outbound per physical peer, regardless of how many addresses it has - Moved dial history out of the peer table, so striking a peer no longer resets its connect-retry cooldown (this was the loop engine: strike → re-learn via gossip → redial next tick) - Node_HasOtherInboundFrom / Node_HasLiveConnectionTo now ignore connections that are already tearing down, and match on identity as well as endpoint Gossip hygiene: - Never hand a peer its own other addresses in a PEERS reply (matched on identity, not just the socket address) - Reject unusable endpoints: link-local without scope id, unspecified, multicast, site-local (loopback stays allowed for local testing) - Normalise IPv4-mapped IPv6 so one host cannot occupy two entries Two bugs found along the way: - All nodes drew the same identity — random_eight_byte() comes from srand(time(NULL)), so processes started in the same second produced identical values. Added random_secure_eight_byte() (/dev/urandom) for the identity, and mixed the pid into the seed so connection IDs stop colliding too - Identity dedup initially left inbound-only nodes mute — broadcasts traverse outbound connections only, so suppressing a dial-back because an inbound existed would have silenced such a node. Corrected to per-direction Other: - peers output now shows each entry's node identity and the node's own endpoints - Added _DEFAULT_SOURCE to the build so getifaddrs() stays visible on glibc
95 lines
3.4 KiB
C
95 lines
3.4 KiB
C
#ifndef TCPCONNECTION_H
|
|
#define TCPCONNECTION_H
|
|
|
|
#include <arpa/inet.h>
|
|
#include <pthread.h>
|
|
#include <stdatomic.h>
|
|
#include <stdbool.h>
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
#include <sys/socket.h>
|
|
|
|
#define TCP_IO_BUFFER_SIZE 1500
|
|
#define TCP_FRAME_HEADER_SIZE 4U
|
|
#define TCP_MAX_FRAME_PAYLOAD (1024U * 1024U)
|
|
|
|
typedef enum {
|
|
TCP_CONNECTION_ROLE_INBOUND = 0,
|
|
TCP_CONNECTION_ROLE_OUTBOUND = 1
|
|
} tcp_connection_role_t;
|
|
|
|
typedef struct tcp_connection_t tcp_connection_t;
|
|
|
|
struct tcp_connection_t {
|
|
int sockFd;
|
|
sa_family_t addrFamily;
|
|
struct sockaddr_storage peerAddr;
|
|
uint32_t connectionId;
|
|
tcp_connection_role_t role;
|
|
|
|
// Peer's advertised TCP/UDP listen port (learned from HELLO/ACK_HELLO). 0 until known.
|
|
// For OUTBOUND connections the peerAddr port already is the listen port; this matters for INBOUND peers.
|
|
uint16_t peerListenPort;
|
|
|
|
// Peer's advertised node identity (learned from HELLO/ACK_HELLO). 0 until known / peer too old
|
|
// to advertise one. Unlike the peer address, this is stable across all of a multi-homed peer's
|
|
// endpoints, so it is what identifies the node behind this connection.
|
|
uint64_t peerNodeId;
|
|
|
|
pthread_t ioThread;
|
|
pthread_mutex_t sendLock;
|
|
pthread_mutex_t stateLock;
|
|
|
|
bool closing;
|
|
bool disconnectedNotified;
|
|
|
|
// Non-zero while another thread holds a raw pointer to this connection taken from a
|
|
// lock-protected snapshot and used after releasing the lock. The reaper must not free a
|
|
// pinned connection. See TcpConnection_Pin/Unpin.
|
|
atomic_int pinCount;
|
|
|
|
unsigned char* dataBuf;
|
|
size_t dataBufLen;
|
|
size_t dataBufCap;
|
|
|
|
unsigned char headerBuf[TCP_FRAME_HEADER_SIZE];
|
|
size_t headerBytesRead;
|
|
uint32_t expectedPayloadLen;
|
|
unsigned char* frameBuf;
|
|
size_t frameBytesRead;
|
|
|
|
void (*on_data)(tcp_connection_t* conn);
|
|
void (*on_disconnect)(tcp_connection_t* conn);
|
|
void* owner;
|
|
};
|
|
|
|
int TcpConnection_Init(tcp_connection_t* conn, int sockFd, const struct sockaddr_storage* peerAddr, tcp_connection_role_t role);
|
|
void TcpConnection_Destroy(tcp_connection_t* conn);
|
|
|
|
int TcpConnection_SetDataBuffer(tcp_connection_t* conn, const unsigned char* data, size_t len);
|
|
|
|
void TcpConnection_ResetFramingState(tcp_connection_t* conn);
|
|
int TcpConnection_FeedFramedData(tcp_connection_t* conn, const unsigned char* input, size_t inputLen);
|
|
|
|
// Returns the peer's canonical IP string (strips ::ffff: IPv4-mapped prefix).
|
|
// Writes at most bufLen bytes to buf. Returns buf on success, NULL on failure.
|
|
const char* TcpConnection_GetPeerAddrStr(const tcp_connection_t* conn, char* buf, size_t bufLen);
|
|
|
|
// Returns non-zero if both connections have the same peer IP address.
|
|
// Handles AF_INET vs AF_INET6 mismatches via IPv4-mapped normalisation.
|
|
int TcpConnection_PeerAddrEqual(const tcp_connection_t* a, const tcp_connection_t* b);
|
|
|
|
int TcpConnection_SendRaw(int sockFd, const void* data, size_t len);
|
|
int TcpConnection_SendFramed(tcp_connection_t* conn, const void* payload, size_t payloadLen);
|
|
|
|
void TcpConnection_RequestClose(tcp_connection_t* conn);
|
|
void TcpConnection_MarkDisconnectNotified(tcp_connection_t* conn);
|
|
bool TcpConnection_IsDisconnectNotified(tcp_connection_t* conn);
|
|
|
|
// Pin/unpin a connection so a background reaper won't free it while a caller still holds a raw
|
|
// pointer to it (e.g. across a blocking operation after releasing the collection lock).
|
|
void TcpConnection_Pin(tcp_connection_t* conn);
|
|
void TcpConnection_Unpin(tcp_connection_t* conn);
|
|
|
|
#endif
|