Fix multi-homed / IPv6 peer identity: nodes adding themselves and reconnect churn

Root cause: peers were keyed only by (IP, listen port). An IPv6 host holds several addresses at once, so one node appeared as several peers and could not recognise its own addresses.

Protocol:
 - Added a random per-run node identity (localNodeId), advertised as a length-guarded trailing field in HELLO and ACK_HELLO — backwards compatible with peers that omit it
 - Added peerNodeId to tcp_connection_t; peers are now identified by this rather than by an endpoint
 - ACK_HELLO is now sent before any decision to drop the connection, so a rejected dialer learns whose address it reached instead of retrying forever

Self-connection:
 - Identity match → close the connection and record that endpoint permanently as our own
 - Self-endpoint set preemptively seeded from getifaddrs() at startup, so a node knows its own addresses before ever dialing one
 - Node_ConnectPeer refuses self endpoints, which also kills the echo-back chain that could exhaust connection slots

Duplicate connections and churn:
 - Dedup by identity per direction — one inbound and one outbound per physical peer, regardless of how many addresses it has
 - Moved dial history out of the peer table, so striking a peer no longer resets its connect-retry cooldown (this was the loop engine: strike → re-learn via gossip → redial next tick)
 - Node_HasOtherInboundFrom / Node_HasLiveConnectionTo now ignore connections that are already tearing down, and match on identity as well as endpoint

Gossip hygiene:
 - Never hand a peer its own other addresses in a PEERS reply (matched on identity, not just the socket address)
 - Reject unusable endpoints: link-local without scope id, unspecified, multicast, site-local (loopback stays allowed for local testing)
 - Normalise IPv4-mapped IPv6 so one host cannot occupy two entries

Two bugs found along the way:
 - All nodes drew the same identity — random_eight_byte() comes from srand(time(NULL)), so processes started in the same second produced identical values. Added random_secure_eight_byte() (/dev/urandom) for the identity, and mixed the pid into the seed so connection IDs stop colliding too
 - Identity dedup initially left inbound-only nodes mute — broadcasts traverse outbound connections only, so suppressing a dial-back because an inbound existed would have silenced such a node. Corrected to per-direction

Other:
 - peers output now shows each entry's node identity and the node's own endpoints
 - Added _DEFAULT_SOURCE to the build so getifaddrs() stays visible on glibc
This commit is contained in:
2026-07-27 14:08:57 +02:00
parent 5aa99ecb01
commit 0e90f7d5db
10 changed files with 531 additions and 48 deletions
+3
View File
@@ -230,5 +230,8 @@ target_compile_definitions(node PRIVATE
CHAIN_DATA_DIR="${CMAKE_BINARY_DIR}/chain_data" CHAIN_DATA_DIR="${CMAKE_BINARY_DIR}/chain_data"
$<$<BOOL:${SKALACOIN_AUTOLYKOS2_REF_AVAILABLE}>:SKALACOIN_AUTOLYKOS2_REF_AVAILABLE> $<$<BOOL:${SKALACOIN_AUTOLYKOS2_REF_AVAILABLE}>:SKALACOIN_AUTOLYKOS2_REF_AVAILABLE>
$<$<BOOL:1>:_POSIX_C_SOURCE=200809L> $<$<BOOL:1>:_POSIX_C_SOURCE=200809L>
# getifaddrs() (used to learn our own addresses) is a BSD extension, not POSIX; glibc hides it
# unless the default set is requested as well.
$<$<BOOL:1>:_DEFAULT_SOURCE>
) )
set_target_properties(node PROPERTIES OUTPUT_NAME "skalacoin_node") set_target_properties(node PROPERTIES OUTPUT_NAME "skalacoin_node")
+6 -2
View File
@@ -85,8 +85,12 @@ void Node_GetClientList(net_node_t* node, tcp_connection_t** outClients, size_t*
// Returns non-zero on success (usable endpoint with a known, non-zero port), zero otherwise. // Returns non-zero on success (usable endpoint with a known, non-zero port), zero otherwise.
int Node_ConnListenEndpoint(const tcp_connection_t* conn, struct sockaddr_storage* out); int Node_ConnListenEndpoint(const tcp_connection_t* conn, struct sockaddr_storage* out);
// Returns the node identity advertised by a connection's peer, or 0 if it is not known yet.
uint64_t Node_ConnPeerNodeId(const tcp_connection_t* conn);
// Fills outEndpoints with the listen endpoints of all current connections (inbound + outbound), // Fills outEndpoints with the listen endpoints of all current connections (inbound + outbound),
// deduped by IP+port. Returns the number of endpoints written (<= maxOut). // deduped by IP+port, and outNodeIds (optional, may be NULL) with the matching peer identities.
size_t Node_GetPeerEndpoints(net_node_t* node, struct sockaddr_storage* outEndpoints, size_t maxOut); // Returns the number of endpoints written (<= maxOut).
size_t Node_GetPeerEndpoints(net_node_t* node, struct sockaddr_storage* outEndpoints, uint64_t* outNodeIds, size_t maxOut);
#endif #endif
+12
View File
@@ -27,6 +27,18 @@ void NodeDiscovery_OnPeersReceived(node_discovery_t* disc, tcp_connection_t* fro
// logically disconnected (no remaining connection to it). // logically disconnected (no remaining connection to it).
void NodeDiscovery_RemovePeer(node_discovery_t* disc, const struct sockaddr_storage* endpoint); void NodeDiscovery_RemovePeer(node_discovery_t* disc, const struct sockaddr_storage* endpoint);
// Record the node identity behind an endpoint (learned from a completed HELLO/ACK_HELLO). Entries
// carrying an identity we are already connected to are skipped by the connect picker, which is what
// stops a multi-homed peer from being dialed once per address it is reachable on.
void NodeDiscovery_NoteIdentity(node_discovery_t* disc, const struct sockaddr_storage* endpoint, uint64_t nodeId);
// Mark an endpoint as one of our own, permanently. Self endpoints are never added to the known-peer
// table, never pinged and never dialed. Seeded from the local interface addresses at creation and
// extended whenever a handshake turns out to come from ourselves.
void NodeDiscovery_MarkSelfEndpoint(node_discovery_t* disc, const struct sockaddr_storage* endpoint);
// Returns non-zero if the endpoint is known to be one of our own.
int NodeDiscovery_IsSelfEndpoint(node_discovery_t* disc, const struct sockaddr_storage* endpoint);
// Dump the known-peer table to stdout (for the CLI `peers` command). // Dump the known-peer table to stdout (for the CLI `peers` command).
void NodeDiscovery_PrintPeers(node_discovery_t* disc); void NodeDiscovery_PrintPeers(node_discovery_t* disc);
+5
View File
@@ -11,4 +11,9 @@ uint16_t random_two_byte(void);
uint32_t random_four_byte(void); uint32_t random_four_byte(void);
uint64_t random_eight_byte(void); uint64_t random_eight_byte(void);
// Draws from the OS entropy pool instead of the srand()-seeded generator, which repeats across
// processes started within the same second. Use this wherever a value must be unique between nodes
// (e.g. the node identity). Never returns 0.
uint64_t random_secure_eight_byte(void);
#endif #endif
+4
View File
@@ -17,6 +17,10 @@ extern const char* chainDataDir;
extern unsigned short listenPort; extern unsigned short listenPort;
extern bool echoPeersEnabled; extern bool echoPeersEnabled;
extern bool forceOrphanReorgEnabled; extern bool forceOrphanReorgEnabled;
// Random per-run identity of this node, advertised in HELLO/ACK_HELLO. A host can be reachable
// under many addresses (especially over IPv6), so an (ip, port) endpoint is not a peer identity:
// this nonce is what lets us recognise our own connections and a peer we already talk to.
extern uint64_t localNodeId;
// Global synchronization primitives for runtime state // Global synchronization primitives for runtime state
extern pthread_rwlock_t chainLock; // protects chain structure and related mutations extern pthread_rwlock_t chainLock; // protects chain structure and related mutations
+5
View File
@@ -31,6 +31,11 @@ struct tcp_connection_t {
// For OUTBOUND connections the peerAddr port already is the listen port; this matters for INBOUND peers. // For OUTBOUND connections the peerAddr port already is the listen port; this matters for INBOUND peers.
uint16_t peerListenPort; uint16_t peerListenPort;
// Peer's advertised node identity (learned from HELLO/ACK_HELLO). 0 until known / peer too old
// to advertise one. Unlike the peer address, this is stable across all of a multi-homed peer's
// endpoints, so it is what identifies the node behind this connection.
uint64_t peerNodeId;
pthread_t ioThread; pthread_t ioThread;
pthread_mutex_t sendLock; pthread_mutex_t sendLock;
pthread_mutex_t stateLock; pthread_mutex_t stateLock;
+13 -1
View File
@@ -12,6 +12,8 @@
#include <balance_sheet.h> #include <balance_sheet.h>
#include <unistd.h> #include <unistd.h>
#include <errno.h> #include <errno.h>
#include <inttypes.h>
#include <numgen.h>
#include <txmempool.h> #include <txmempool.h>
#include <constants.h> #include <constants.h>
@@ -34,6 +36,7 @@ bool echoPeersEnabled = ECHO_PEERS != 0;
bool forceOrphanReorgEnabled = false; bool forceOrphanReorgEnabled = false;
uint256_t currentSupply = {{0, 0, 0, 0}}; uint256_t currentSupply = {{0, 0, 0, 0}};
uint64_t currentReward = 750000000000ULL; uint64_t currentReward = 750000000000ULL;
uint64_t localNodeId = 0; // Randomised in main() before the node comes up
// Define the synchronization primitives declared in runtime_state.h // Define the synchronization primitives declared in runtime_state.h
pthread_rwlock_t chainLock; pthread_rwlock_t chainLock;
@@ -678,7 +681,16 @@ int main(int argc, char* argv[]) {
// (handled by the send paths) instead of terminating the whole process. Peers connecting and // (handled by the send paths) instead of terminating the whole process. Peers connecting and
// disconnecting is normal p2p behaviour and must never take the node down. // disconnecting is normal p2p behaviour and must never take the node down.
signal(SIGPIPE, SIG_IGN); signal(SIGPIPE, SIG_IGN);
srand((unsigned int)time(NULL)); // Mix the pid into the seed: nodes launched within the same second would otherwise draw
// identical sequences, so every rand()-derived value (connection ids and the like) would
// collide across them.
srand((unsigned int)time(NULL) ^ ((unsigned int)getpid() << 16));
// Pick this run's node identity before the node (and with it the listener) comes up, so every
// handshake can carry it. Peers are identified by this nonce rather than by an (ip, port)
// endpoint, which a multi-homed host has several of.
localNodeId = random_secure_eight_byte();
printf("Node identity: %016" PRIx64 "\n", localNodeId);
// Initialize runtime locks before any thread or helper can touch chain state. // Initialize runtime locks before any thread or helper can touch chain state.
pthread_rwlock_init(&chainLock, NULL); pthread_rwlock_init(&chainLock, NULL);
+186 -34
View File
@@ -102,13 +102,15 @@ static int Node_HasOutboundTo(net_node_t* node, const struct sockaddr_storage* e
// Returns non-zero if some inbound connection OTHER than `self` already has the given listen // Returns non-zero if some inbound connection OTHER than `self` already has the given listen
// endpoint (used to reject a duplicate inbound once we learn the peer's advertised listen port). // endpoint (used to reject a duplicate inbound once we learn the peer's advertised listen port).
// Connections that are already tearing down do not count - otherwise a peer reconnecting from the
// same endpoint gets its fresh inbound rejected by the corpse of the previous one.
static int Node_HasOtherInboundFrom(net_node_t* node, const tcp_connection_t* self, const struct sockaddr_storage* endpoint) { static int Node_HasOtherInboundFrom(net_node_t* node, const tcp_connection_t* self, const struct sockaddr_storage* endpoint) {
if (!node->server) return 0; if (!node->server) return 0;
int found = 0; int found = 0;
pthread_mutex_lock(&node->server->clientsMutex); pthread_mutex_lock(&node->server->clientsMutex);
for (size_t i = 0; i < node->server->maxClients; ++i) { for (size_t i = 0; i < node->server->maxClients; ++i) {
tcp_connection_t* other = node->server->clientsArrPtr ? node->server->clientsArrPtr[i] : NULL; tcp_connection_t* other = node->server->clientsArrPtr ? node->server->clientsArrPtr[i] : NULL;
if (!other || other == self) continue; if (!other || other == self || TcpConnection_IsDisconnectNotified(other)) continue;
struct sockaddr_storage ep; struct sockaddr_storage ep;
if (Node_ConnListenEndpoint(other, &ep) && NetNode_EndpointEqual(&ep, endpoint)) { if (Node_ConnListenEndpoint(other, &ep) && NetNode_EndpointEqual(&ep, endpoint)) {
found = 1; found = 1;
@@ -119,16 +121,47 @@ static int Node_HasOtherInboundFrom(net_node_t* node, const tcp_connection_t* se
return found; return found;
} }
// Returns non-zero if a connection OTHER than `exclude` to `endpoint` is still live. A connection // Returns non-zero if a live connection OTHER than `self` with the same role already belongs to the
// that is itself mid-disconnect (disconnectedNotified) does not count as live - this is what lets // node identified by nodeId. This is the endpoint-independent duplicate check: a multi-homed peer
// us decide a peer is fully gone even when both its inbound and outbound drop simultaneously. // reaches us from several addresses, so comparing endpoints alone lets the same node in twice.
static int Node_HasLiveConnectionTo(net_node_t* node, const struct sockaddr_storage* endpoint, const tcp_connection_t* exclude) { static int Node_HasOtherConnectionToNode(net_node_t* node, const tcp_connection_t* self, uint64_t nodeId) {
if (nodeId == 0) return 0;
int found = 0;
pthread_mutex_lock(&node->outboundLock);
for (size_t i = 0; i < MAX_CONS && !found; ++i) {
tcp_connection_t* c = node->outboundClients[i].connection;
if (!c || c == self || TcpConnection_IsDisconnectNotified(c)) continue;
if (c->role == self->role && c->peerNodeId == nodeId) found = 1;
}
pthread_mutex_unlock(&node->outboundLock);
if (found) return 1;
if (node->server) {
pthread_mutex_lock(&node->server->clientsMutex);
for (size_t i = 0; i < node->server->maxClients && !found; ++i) {
tcp_connection_t* c = node->server->clientsArrPtr ? node->server->clientsArrPtr[i] : NULL;
if (!c || c == self || TcpConnection_IsDisconnectNotified(c)) continue;
if (c->role == self->role && c->peerNodeId == nodeId) found = 1;
}
pthread_mutex_unlock(&node->server->clientsMutex);
}
return found;
}
// Returns non-zero if a connection OTHER than `exclude` to the same peer is still live - matched
// either on the listen endpoint or, when known, on the peer's identity (which also covers its other
// addresses). A connection that is itself mid-disconnect (disconnectedNotified) does not count as
// live - this is what lets us decide a peer is fully gone even when both its inbound and outbound
// drop simultaneously.
static int Node_HasLiveConnectionTo(net_node_t* node, const struct sockaddr_storage* endpoint, uint64_t nodeId, const tcp_connection_t* exclude) {
int found = 0; int found = 0;
pthread_mutex_lock(&node->outboundLock); pthread_mutex_lock(&node->outboundLock);
for (size_t i = 0; i < MAX_CONS && !found; ++i) { for (size_t i = 0; i < MAX_CONS && !found; ++i) {
tcp_connection_t* c = node->outboundClients[i].connection; tcp_connection_t* c = node->outboundClients[i].connection;
if (!c || c == exclude || TcpConnection_IsDisconnectNotified(c)) continue; if (!c || c == exclude || TcpConnection_IsDisconnectNotified(c)) continue;
if (nodeId != 0 && c->peerNodeId == nodeId) { found = 1; break; }
struct sockaddr_storage ep; struct sockaddr_storage ep;
if (Node_ConnListenEndpoint(c, &ep) && NetNode_EndpointEqual(&ep, endpoint)) found = 1; if (Node_ConnListenEndpoint(c, &ep) && NetNode_EndpointEqual(&ep, endpoint)) found = 1;
} }
@@ -140,6 +173,7 @@ static int Node_HasLiveConnectionTo(net_node_t* node, const struct sockaddr_stor
for (size_t i = 0; i < node->server->maxClients && !found; ++i) { for (size_t i = 0; i < node->server->maxClients && !found; ++i) {
tcp_connection_t* c = node->server->clientsArrPtr ? node->server->clientsArrPtr[i] : NULL; tcp_connection_t* c = node->server->clientsArrPtr ? node->server->clientsArrPtr[i] : NULL;
if (!c || c == exclude || TcpConnection_IsDisconnectNotified(c)) continue; if (!c || c == exclude || TcpConnection_IsDisconnectNotified(c)) continue;
if (nodeId != 0 && c->peerNodeId == nodeId) { found = 1; break; }
struct sockaddr_storage ep; struct sockaddr_storage ep;
if (Node_ConnListenEndpoint(c, &ep) && NetNode_EndpointEqual(&ep, endpoint)) found = 1; if (Node_ConnListenEndpoint(c, &ep) && NetNode_EndpointEqual(&ep, endpoint)) found = 1;
} }
@@ -150,13 +184,14 @@ static int Node_HasLiveConnectionTo(net_node_t* node, const struct sockaddr_stor
// Called when a connection to a peer drops. Strikes the peer from the discovery table, but only // Called when a connection to a peer drops. Strikes the peer from the discovery table, but only
// once it is logically disconnected - i.e. no other live connection (inbound or outbound) to the // once it is logically disconnected - i.e. no other live connection (inbound or outbound) to the
// same listen endpoint remains. Must be called from the disconnect callback while `conn` is still // same node remains. Must be called from the disconnect callback while `conn` is still valid and
// valid and outside outboundLock/clientsMutex. // outside outboundLock/clientsMutex.
static void Node_HandlePeerDisconnect(net_node_t* node, tcp_connection_t* conn) { static void Node_HandlePeerDisconnect(net_node_t* node, tcp_connection_t* conn) {
if (!node || !node->discovery || !conn) return; if (!node || !node->discovery || !conn) return;
struct sockaddr_storage ep; struct sockaddr_storage ep;
if (!Node_ConnListenEndpoint(conn, &ep)) return; // never advertised an endpoint -> not tracked if (!Node_ConnListenEndpoint(conn, &ep)) return; // never advertised an endpoint -> not tracked
if (Node_HasLiveConnectionTo(node, &ep, conn)) return; // still reachable via another connection // Still reachable via another connection (possibly on one of its other addresses).
if (Node_HasLiveConnectionTo(node, &ep, conn->peerNodeId, conn)) return;
NodeDiscovery_RemovePeer(node->discovery, &ep); NodeDiscovery_RemovePeer(node->discovery, &ep);
} }
@@ -203,7 +238,11 @@ int Node_ConnListenEndpoint(const tcp_connection_t* conn, struct sockaddr_storag
return 0; return 0;
} }
size_t Node_GetPeerEndpoints(net_node_t* node, struct sockaddr_storage* outEndpoints, size_t maxOut) { uint64_t Node_ConnPeerNodeId(const tcp_connection_t* conn) {
return conn ? conn->peerNodeId : 0;
}
size_t Node_GetPeerEndpoints(net_node_t* node, struct sockaddr_storage* outEndpoints, uint64_t* outNodeIds, size_t maxOut) {
if (!node || !outEndpoints || maxOut == 0) return 0; if (!node || !outEndpoints || maxOut == 0) return 0;
size_t count = 0; size_t count = 0;
@@ -218,7 +257,9 @@ size_t Node_GetPeerEndpoints(net_node_t* node, struct sockaddr_storage* outEndpo
for (size_t k = 0; k < count; ++k) { for (size_t k = 0; k < count; ++k) {
if (NetNode_EndpointEqual(&outEndpoints[k], &ep)) { dup = 1; break; } if (NetNode_EndpointEqual(&outEndpoints[k], &ep)) { dup = 1; break; }
} }
if (!dup) outEndpoints[count++] = ep; if (dup) continue;
if (outNodeIds) outNodeIds[count] = c->peerNodeId;
outEndpoints[count++] = ep;
} }
pthread_mutex_unlock(&node->outboundLock); pthread_mutex_unlock(&node->outboundLock);
@@ -234,7 +275,9 @@ size_t Node_GetPeerEndpoints(net_node_t* node, struct sockaddr_storage* outEndpo
for (size_t k = 0; k < count; ++k) { for (size_t k = 0; k < count; ++k) {
if (NetNode_EndpointEqual(&outEndpoints[k], &ep)) { dup = 1; break; } if (NetNode_EndpointEqual(&outEndpoints[k], &ep)) { dup = 1; break; }
} }
if (!dup) outEndpoints[count++] = ep; if (dup) continue;
if (outNodeIds) outNodeIds[count] = c->peerNodeId;
outEndpoints[count++] = ep;
} }
pthread_mutex_unlock(&node->server->clientsMutex); pthread_mutex_unlock(&node->server->clientsMutex);
} }
@@ -242,6 +285,44 @@ size_t Node_GetPeerEndpoints(net_node_t* node, struct sockaddr_storage* outEndpo
return count; return count;
} }
// Outcome of the identity check run once a connection's HELLO/ACK_HELLO has been parsed.
typedef enum {
NODE_IDENTITY_OK = 0,
NODE_IDENTITY_SELF, // the peer is this very node, reached through one of its own addresses
NODE_IDENTITY_DUPLICATE // we already hold a connection of this role to that node
} node_identity_result_t;
// Records the identity a peer advertised and decides whether the connection should survive.
// `conn->peerNodeId` and `conn->peerListenPort` must already be set from the handshake.
static node_identity_result_t Node_CheckPeerIdentity(net_node_t* node, tcp_connection_t* conn) {
if (!node || !conn || conn->peerNodeId == 0) return NODE_IDENTITY_OK; // peer too old to advertise one
struct sockaddr_storage ep;
int haveEp = Node_ConnListenEndpoint(conn, &ep);
if (conn->peerNodeId == localNodeId) {
// We dialled ourselves (or accepted our own dial). Remember the endpoint as our own so
// discovery stops offering it back to us, and drop the connection.
if (haveEp && node->discovery) {
NodeDiscovery_MarkSelfEndpoint(node->discovery, &ep);
}
return NODE_IDENTITY_SELF;
}
// Record the identity behind this endpoint even when the connection is about to be dropped as a
// duplicate: that is what lets discovery skip the peer's other addresses while we are connected
// to it, instead of dialling each of them in turn.
if (haveEp && node->discovery) {
NodeDiscovery_NoteIdentity(node->discovery, &ep, conn->peerNodeId);
}
if (Node_HasOtherConnectionToNode(node, conn, conn->peerNodeId)) {
return NODE_IDENTITY_DUPLICATE;
}
return NODE_IDENTITY_OK;
}
// Thunks routing UDP ping/pong events into the discovery state. // Thunks routing UDP ping/pong events into the discovery state.
static void Node_OnPongThunk(udp_node_t* udp, const struct sockaddr_storage* from, static void Node_OnPongThunk(udp_node_t* udp, const struct sockaddr_storage* from,
uint64_t nonce, int protoVersion, uint64_t rttMs, void* user) { uint64_t nonce, int protoVersion, uint64_t rttMs, void* user) {
@@ -641,11 +722,18 @@ int Node_ConnectPeer(net_node_t* node, const char* ip, unsigned short port) {
return -1; return -1;
} }
// Never dial ourselves. Without this an echo-back (or a gossiped copy of one of our own
// addresses) can chain into a self-connection per maintenance tick until the slots run out.
struct sockaddr_storage target;
int haveTarget = NetNode_MakeEndpoint(ip, port, &target);
if (haveTarget && node->discovery && NodeDiscovery_IsSelfEndpoint(node->discovery, &target)) {
return -1;
}
// Enforce a single outbound connection per endpoint: if we already have an outbound to this // Enforce a single outbound connection per endpoint: if we already have an outbound to this
// (ip, port), do not open a second one. (Inbound from the same endpoint is still allowed - that // (ip, port), do not open a second one. (Inbound from the same endpoint is still allowed - that
// is the peer's own outbound to us.) // is the peer's own outbound to us.)
struct sockaddr_storage target; if (haveTarget && Node_HasOutboundTo(node, &target)) {
if (NetNode_MakeEndpoint(ip, port, &target) && Node_HasOutboundTo(node, &target)) {
return 0; // already connected outbound to this endpoint return 0; // already connected outbound to this endpoint
} }
@@ -807,27 +895,23 @@ void Node_Server_OnData(tcp_connection_t* client) {
client->peerListenPort = peerListenPort; client->peerListenPort = peerListenPort;
} }
printf("Received HELLO from node %u: protoVersion=%u, blockHeight=%" PRIu64 ", listenPort=%u\n", // Optional trailing node identity, same length-guarded deal.
client ? client->connectionId : 0U, protoVersion, blockHeight, if (client && payloadLen >= sizeof(uint32_t) + sizeof(uint64_t) + sizeof(uint16_t) + sizeof(uint64_t)) {
client ? client->peerListenPort : 0U); uint64_t peerNodeId;
memcpy(&peerNodeId, payload + sizeof(protoVersion) + sizeof(blockHeight) + sizeof(uint16_t), sizeof(peerNodeId));
// Enforce a single inbound connection per endpoint. Now that we know this peer's listen client->peerNodeId = peerNodeId;
// port, drop this connection if another inbound from the same endpoint already exists
// (keep the established one). An outbound to the same endpoint is unaffected - that is
// this node's own connection to the peer.
if (client && client->peerListenPort != 0) {
net_node_t* dupNode = Node_FromConnection(client);
struct sockaddr_storage myEp;
if (dupNode && Node_ConnListenEndpoint(client, &myEp) &&
Node_HasOtherInboundFrom(dupNode, client, &myEp)) {
printf("Rejecting duplicate inbound connection %u (already have an inbound from this endpoint)\n",
client->connectionId);
TcpConnection_RequestClose(client);
return;
}
} }
// Craft and send ACK_HELLO (echo protoVersion, our height, and our own listen port) printf("Received HELLO from node %u: protoVersion=%u, blockHeight=%" PRIu64 ", listenPort=%u, nodeId=%016" PRIx64 "\n",
client ? client->connectionId : 0U, protoVersion, blockHeight,
client ? client->peerListenPort : 0U, client ? client->peerNodeId : 0ULL);
// Craft and send ACK_HELLO (echo protoVersion, our height, our own listen port and our
// identity). This goes out before any decision to drop the connection: the ACK is what
// tells the dialer whose address it just reached, so an endpoint that turns out to be
// another address of a peer it already talks to (or one of its own) is recognised as
// such instead of being redialled forever. shutdown() flushes what is already queued,
// so the peer still receives this even though we close immediately after.
uint8_t ackBuf[100]; uint8_t ackBuf[100];
uint8_t* ackData = ackBuf; uint8_t* ackData = ackBuf;
size_t ackOffset = 0; size_t ackOffset = 0;
@@ -839,9 +923,47 @@ void Node_Server_OnData(tcp_connection_t* client) {
uint16_t myListenPort = (uint16_t)listenPort; uint16_t myListenPort = (uint16_t)listenPort;
memcpy(ackData + ackOffset, &myListenPort, sizeof(myListenPort)); memcpy(ackData + ackOffset, &myListenPort, sizeof(myListenPort));
ackOffset += sizeof(myListenPort); ackOffset += sizeof(myListenPort);
uint64_t myNodeId = localNodeId;
memcpy(ackData + ackOffset, &myNodeId, sizeof(myNodeId));
ackOffset += sizeof(myNodeId);
Node_SendPacket(Node_FromConnection(client), client, PACKET_TYPE_ACK_HELLO, ackData, ackOffset); Node_SendPacket(Node_FromConnection(client), client, PACKET_TYPE_ACK_HELLO, ackData, ackOffset);
// Enforce one connection per node, identified by the advertised nodeId rather than by
// the address it happens to reach us from.
if (client) {
net_node_t* idNode = Node_FromConnection(client);
node_identity_result_t identity = Node_CheckPeerIdentity(idNode, client);
if (identity == NODE_IDENTITY_SELF) {
printf("Rejecting inbound connection %u: it is this node talking to itself\n",
client->connectionId);
TcpConnection_RequestClose(client);
return;
}
if (identity == NODE_IDENTITY_DUPLICATE) {
printf("Rejecting duplicate inbound connection %u (already connected to node %016" PRIx64 ")\n",
client->connectionId, client->peerNodeId);
TcpConnection_RequestClose(client);
return;
}
}
// Endpoint-level fallback for peers that advertise no identity: drop this connection if
// another inbound from the same endpoint already exists (keep the established one). An
// outbound to the same endpoint is unaffected - that is this node's own connection to
// the peer.
if (client && client->peerNodeId == 0 && client->peerListenPort != 0) {
net_node_t* dupNode = Node_FromConnection(client);
struct sockaddr_storage myEp;
if (dupNode && Node_ConnListenEndpoint(client, &myEp) &&
Node_HasOtherInboundFrom(dupNode, client, &myEp)) {
printf("Rejecting duplicate inbound connection %u (already have an inbound from this endpoint)\n",
client->connectionId);
TcpConnection_RequestClose(client);
return;
}
}
break; break;
} }
case PACKET_TYPE_ACK_HELLO: { case PACKET_TYPE_ACK_HELLO: {
@@ -1087,6 +1209,10 @@ void Node_Client_OnConnect(tcp_connection_t* client) {
uint16_t myListenPort = (uint16_t)listenPort; uint16_t myListenPort = (uint16_t)listenPort;
memcpy((unsigned char*)data + offset, &myListenPort, sizeof(myListenPort)); memcpy((unsigned char*)data + offset, &myListenPort, sizeof(myListenPort));
offset += sizeof(myListenPort); offset += sizeof(myListenPort);
// ...and who we are, so the peer can tell this connection apart from our other addresses
uint64_t myNodeId = localNodeId;
memcpy((unsigned char*)data + offset, &myNodeId, sizeof(myNodeId));
offset += sizeof(myNodeId);
Node_SendPacket(node, client, PACKET_TYPE_HELLO, data, offset); Node_SendPacket(node, client, PACKET_TYPE_HELLO, data, offset);
} }
@@ -1131,10 +1257,36 @@ void Node_Client_OnData(tcp_connection_t* client) {
client->peerListenPort = peerListenPort; client->peerListenPort = peerListenPort;
} }
printf("Received ACK_HELLO from node %u with protoVersion %u and blockHeight %" PRIu64 "\n", client ? client->connectionId : 0U, protoVersion, blockHeight); // Optional trailing node identity, same length-guarded deal.
if (client && payloadLen >= sizeof(uint32_t) + sizeof(uint64_t) + sizeof(uint16_t) + sizeof(uint64_t)) {
uint64_t peerNodeId;
memcpy(&peerNodeId, payload + sizeof(protoVersion) + sizeof(blockHeight) + sizeof(uint16_t), sizeof(peerNodeId));
client->peerNodeId = peerNodeId;
}
printf("Received ACK_HELLO from node %u with protoVersion %u, blockHeight %" PRIu64 " and nodeId %016" PRIx64 "\n",
client ? client->connectionId : 0U, protoVersion, blockHeight, client ? client->peerNodeId : 0ULL);
// Store peer-advertised height on matching outbound client // Store peer-advertised height on matching outbound client
net_node_t* node = Node_FromConnection(client); net_node_t* node = Node_FromConnection(client);
// The dialed endpoint may well be one of our own addresses, or another address of a
// peer we already talk to - neither is worth a connection.
if (client) {
node_identity_result_t identity = Node_CheckPeerIdentity(node, client);
if (identity == NODE_IDENTITY_SELF) {
printf("Closing outbound connection %u: it loops back to this node\n", client->connectionId);
TcpConnection_RequestClose(client);
return;
}
if (identity == NODE_IDENTITY_DUPLICATE) {
printf("Closing outbound connection %u: already connected to node %016" PRIx64 " on another address\n",
client->connectionId, client->peerNodeId);
TcpConnection_RequestClose(client);
return;
}
}
if (node) { if (node) {
pthread_mutex_lock(&node->outboundLock); pthread_mutex_lock(&node->outboundLock);
for (size_t i = 0; i < MAX_CONS; ++i) { for (size_t i = 0; i < MAX_CONS; ++i) {
+266 -11
View File
@@ -8,9 +8,12 @@
#include <netinet/in.h> #include <netinet/in.h>
#include <sys/socket.h> #include <sys/socket.h>
#include <ifaddrs.h>
#include <constants.h> #include <constants.h>
#include <dynarr.h> #include <dynarr.h>
#include <numgen.h> #include <numgen.h>
#include <runtime_state.h>
#include <utils.h> #include <utils.h>
// Wire layout of a single peer endpoint inside a PEERS payload: // Wire layout of a single peer endpoint inside a PEERS payload:
@@ -29,19 +32,29 @@ typedef enum {
typedef struct { typedef struct {
struct sockaddr_storage addr; // listen endpoint (port already set to the peer's listen port) struct sockaddr_storage addr; // listen endpoint (port already set to the peer's listen port)
uint64_t pingMs; // measured UDP RTT, UINT64_MAX if unknown uint64_t pingMs; // measured UDP RTT, UINT64_MAX if unknown
uint64_t nodeId; // identity of the node behind this endpoint, 0 while unknown
uint32_t hop; // distance from us (0 = directly connected) uint32_t hop; // distance from us (0 = directly connected)
discovery_state_t state; discovery_state_t state;
int pingPending; // 1 while a ping is outstanding (matched by address on pong/timeout). int pingPending; // 1 while a ping is outstanding (matched by address on pong/timeout).
// The UDP layer generates its own nonce, so we can't match by nonce here. // The UDP layer generates its own nonce, so we can't match by nonce here.
uint64_t lastPingMs; // when we last sent a ping uint64_t lastPingMs; // when we last sent a ping
uint64_t lastQueryMs; // when we last sent GET_PEERS to it uint64_t lastQueryMs; // when we last sent GET_PEERS to it
uint64_t lastConnectMs; // when we last attempted a connect to it
} discovered_peer_t; } discovered_peer_t;
// When we last dialed an endpoint. Kept outside the peer table on purpose: a peer entry is struck
// the moment its connection drops, and if the dial history went with it, an endpoint that hangs up
// on us would be re-learned through gossip and redialed on every single tick.
typedef struct {
struct sockaddr_storage addr;
uint64_t lastMs;
} discovery_attempt_t;
struct node_discovery { struct node_discovery {
net_node_t* node; net_node_t* node;
udp_node_t* udpNode; udp_node_t* udpNode;
DynArr* peers; // of discovered_peer_t DynArr* peers; // of discovered_peer_t
DynArr* selfEndpoints; // of struct sockaddr_storage - our own listen endpoints
DynArr* connectAttempts; // of discovery_attempt_t
pthread_mutex_t lock; pthread_mutex_t lock;
}; };
@@ -64,6 +77,99 @@ static int Discovery_AddrEqual(const struct sockaddr_storage* a, const struct so
return 0; return 0;
} }
// Rejects endpoints that can never be dialed as written. IPv6 in particular hands us plenty of
// these: link-local addresses are meaningless without the scope id (which the wire format does not
// carry), and the unspecified/multicast ranges are never a peer. Loopback stays allowed so several
// nodes can still be run on one machine on different ports.
static int Discovery_IsUsableAddr(const struct sockaddr_storage* addr) {
if (addr->ss_family == AF_INET) {
const struct sockaddr_in* a = (const struct sockaddr_in*)addr;
if (a->sin_port == 0) return 0;
uint32_t host = ntohl(a->sin_addr.s_addr);
if (host == INADDR_ANY || host == INADDR_BROADCAST) return 0;
if ((host >> 28) == 0xE) return 0; // 224.0.0.0/4 multicast
if ((host & 0xFFFF0000u) == 0xA9FE0000u) return 0; // 169.254.0.0/16 link-local
return 1;
}
if (addr->ss_family == AF_INET6) {
const struct sockaddr_in6* a = (const struct sockaddr_in6*)addr;
if (a->sin6_port == 0) return 0;
if (IN6_IS_ADDR_UNSPECIFIED(&a->sin6_addr)) return 0;
if (IN6_IS_ADDR_MULTICAST(&a->sin6_addr)) return 0;
if (IN6_IS_ADDR_LINKLOCAL(&a->sin6_addr)) return 0; // unusable without a scope id
if (IN6_IS_ADDR_SITELOCAL(&a->sin6_addr)) return 0; // deprecated fec0::/10
return 1;
}
return 0;
}
// Rewrites an IPv4-mapped IPv6 endpoint (::ffff:a.b.c.d) as plain IPv4, so the same host never
// occupies two entries. Matches the normalisation Node_ConnListenEndpoint does.
static void Discovery_NormaliseAddr(struct sockaddr_storage* addr) {
if (addr->ss_family != AF_INET6) return;
struct sockaddr_in6* a = (struct sockaddr_in6*)addr;
if (!IN6_IS_ADDR_V4MAPPED(&a->sin6_addr)) return;
struct in_addr v4;
memcpy(&v4, ((const uint8_t*)&a->sin6_addr) + 12, sizeof(v4));
uint16_t port = a->sin6_port;
memset(addr, 0, sizeof(*addr));
struct sockaddr_in* o = (struct sockaddr_in*)addr;
o->sin_family = AF_INET;
o->sin_addr = v4;
o->sin_port = port;
}
// Returns non-zero if addr is one of our own listen endpoints. Caller holds disc->lock.
static int Discovery_IsSelfUnlocked(node_discovery_t* disc, const struct sockaddr_storage* addr) {
size_t n = DynArr_size(disc->selfEndpoints);
for (size_t i = 0; i < n; ++i) {
const struct sockaddr_storage* self = (const struct sockaddr_storage*)DynArr_at(disc->selfEndpoints, i);
if (Discovery_AddrEqual(self, addr)) return 1;
}
return 0;
}
// Adds addr to the self set if not already there. Caller holds disc->lock.
static void Discovery_AddSelfUnlocked(node_discovery_t* disc, const struct sockaddr_storage* addr) {
if (Discovery_IsSelfUnlocked(disc, addr)) return;
DynArr_push_back(disc->selfEndpoints, (void*)addr);
}
// Seeds the self set with (local interface address, our listen port) for every address this host
// carries. A multi-homed host - the normal case under IPv6, where a machine holds a global, a
// temporary privacy and a link-local address at once - is otherwise unable to tell its own
// endpoints from a peer's when they come back around through peer exchange.
static void Discovery_SeedSelfEndpoints(node_discovery_t* disc) {
struct ifaddrs* ifa = NULL;
if (getifaddrs(&ifa) != 0 || !ifa) return;
for (struct ifaddrs* it = ifa; it; it = it->ifa_next) {
if (!it->ifa_addr) continue;
struct sockaddr_storage ep;
memset(&ep, 0, sizeof(ep));
if (it->ifa_addr->sa_family == AF_INET) {
struct sockaddr_in* o = (struct sockaddr_in*)&ep;
memcpy(o, it->ifa_addr, sizeof(struct sockaddr_in));
o->sin_port = htons(listenPort);
} else if (it->ifa_addr->sa_family == AF_INET6) {
struct sockaddr_in6* o = (struct sockaddr_in6*)&ep;
memcpy(o, it->ifa_addr, sizeof(struct sockaddr_in6));
o->sin6_port = htons(listenPort);
o->sin6_scope_id = 0; // endpoints on the wire are scopeless; compare them the same way
} else {
continue;
}
Discovery_NormaliseAddr(&ep);
Discovery_AddSelfUnlocked(disc, &ep);
}
freeifaddrs(ifa);
}
static discovered_peer_t* Discovery_FindPtr(node_discovery_t* disc, const struct sockaddr_storage* addr) { static discovered_peer_t* Discovery_FindPtr(node_discovery_t* disc, const struct sockaddr_storage* addr) {
size_t n = DynArr_size(disc->peers); size_t n = DynArr_size(disc->peers);
for (size_t i = 0; i < n; ++i) { for (size_t i = 0; i < n; ++i) {
@@ -73,9 +179,13 @@ static discovered_peer_t* Discovery_FindPtr(node_discovery_t* disc, const struct
return NULL; return NULL;
} }
// Insert addr if not already present. Returns a pointer to the (existing or new) entry, or NULL // Insert addr if not already present. Returns a pointer to the (existing or new) entry, or NULL if
// if the table is full. Note: the returned pointer is invalidated by any later push_back. // the address is unusable, is one of our own, or the table is full. Note: the returned pointer is
// invalidated by any later push_back.
static discovered_peer_t* Discovery_Upsert(node_discovery_t* disc, const struct sockaddr_storage* addr, uint32_t hop) { static discovered_peer_t* Discovery_Upsert(node_discovery_t* disc, const struct sockaddr_storage* addr, uint32_t hop) {
if (!Discovery_IsUsableAddr(addr)) return NULL;
if (Discovery_IsSelfUnlocked(disc, addr)) return NULL; // never track, ping or dial ourselves
discovered_peer_t* existing = Discovery_FindPtr(disc, addr); discovered_peer_t* existing = Discovery_FindPtr(disc, addr);
if (existing) { if (existing) {
if (hop < existing->hop) existing->hop = hop; // keep the shortest known distance if (hop < existing->hop) existing->hop = hop; // keep the shortest known distance
@@ -87,12 +197,71 @@ static discovered_peer_t* Discovery_Upsert(node_discovery_t* disc, const struct
memset(&np, 0, sizeof(np)); memset(&np, 0, sizeof(np));
np.addr = *addr; np.addr = *addr;
np.pingMs = UINT64_MAX; np.pingMs = UINT64_MAX;
np.nodeId = 0;
np.hop = hop; np.hop = hop;
np.state = DISCOVERY_STATE_NEW; np.state = DISCOVERY_STATE_NEW;
DynArr_push_back(disc->peers, &np); DynArr_push_back(disc->peers, &np);
return (discovered_peer_t*)DynArr_at(disc->peers, DynArr_size(disc->peers) - 1); return (discovered_peer_t*)DynArr_at(disc->peers, DynArr_size(disc->peers) - 1);
} }
// Returns non-zero if addr may be dialed again, i.e. we have not tried it within the retry window.
// Caller holds disc->lock.
static int Discovery_ConnectCooledDown(node_discovery_t* disc, const struct sockaddr_storage* addr, uint64_t now) {
size_t n = DynArr_size(disc->connectAttempts);
for (size_t i = 0; i < n; ++i) {
const discovery_attempt_t* a = (const discovery_attempt_t*)DynArr_at(disc->connectAttempts, i);
if (Discovery_AddrEqual(&a->addr, addr)) {
return (now - a->lastMs) >= DISCOVERY_CONNECT_RETRY_MS;
}
}
return 1; // never dialed
}
// Stamps a dial attempt against addr, evicting the stalest record once the table is full.
// Caller holds disc->lock.
static void Discovery_NoteConnectAttempt(node_discovery_t* disc, const struct sockaddr_storage* addr, uint64_t now) {
size_t n = DynArr_size(disc->connectAttempts);
size_t oldestIdx = 0;
uint64_t oldestMs = UINT64_MAX;
for (size_t i = 0; i < n; ++i) {
discovery_attempt_t* a = (discovery_attempt_t*)DynArr_at(disc->connectAttempts, i);
if (Discovery_AddrEqual(&a->addr, addr)) {
a->lastMs = now;
return;
}
if (a->lastMs < oldestMs) {
oldestMs = a->lastMs;
oldestIdx = i;
}
}
if (n >= DISCOVERY_MAX_KNOWN_PEERS) {
discovery_attempt_t* victim = (discovery_attempt_t*)DynArr_at(disc->connectAttempts, oldestIdx);
victim->addr = *addr;
victim->lastMs = now;
return;
}
discovery_attempt_t na;
memset(&na, 0, sizeof(na));
na.addr = *addr;
na.lastMs = now;
DynArr_push_back(disc->connectAttempts, &na);
}
// Drops the entry for addr, if any. Caller holds disc->lock.
static void Discovery_RemoveUnlocked(node_discovery_t* disc, const struct sockaddr_storage* addr) {
size_t n = DynArr_size(disc->peers);
for (size_t i = 0; i < n; ++i) {
discovered_peer_t* p = (discovered_peer_t*)DynArr_at(disc->peers, i);
if (Discovery_AddrEqual(&p->addr, addr)) {
DynArr_remove(disc->peers, i);
return;
}
}
}
static int Discovery_AddrToWire(const struct sockaddr_storage* addr, unsigned char out[DISCOVERY_WIRE_ENTRY_SIZE]) { static int Discovery_AddrToWire(const struct sockaddr_storage* addr, unsigned char out[DISCOVERY_WIRE_ENTRY_SIZE]) {
memset(out, 0, DISCOVERY_WIRE_ENTRY_SIZE); memset(out, 0, DISCOVERY_WIRE_ENTRY_SIZE);
if (addr->ss_family == AF_INET) { if (addr->ss_family == AF_INET) {
@@ -131,6 +300,7 @@ static int Discovery_WireToAddr(const unsigned char in[DISCOVERY_WIRE_ENTRY_SIZE
a->sin6_family = AF_INET6; a->sin6_family = AF_INET6;
memcpy(&a->sin6_addr, in + 1, sizeof(struct in6_addr)); memcpy(&a->sin6_addr, in + 1, sizeof(struct in6_addr));
a->sin6_port = htons(port); a->sin6_port = htons(port);
Discovery_NormaliseAddr(out); // a v4-mapped sender must not become a second entry
return port != 0; return port != 0;
} }
return 0; return 0;
@@ -166,13 +336,31 @@ node_discovery_t* NodeDiscovery_Create(net_node_t* node, udp_node_t* udpNode) {
free(disc); free(disc);
return NULL; return NULL;
} }
disc->selfEndpoints = DYNARR_CREATE(struct sockaddr_storage, 8);
if (!disc->selfEndpoints) {
DynArr_destroy(disc->peers);
free(disc);
return NULL;
}
disc->connectAttempts = DYNARR_CREATE(discovery_attempt_t, 16);
if (!disc->connectAttempts) {
DynArr_destroy(disc->selfEndpoints);
DynArr_destroy(disc->peers);
free(disc);
return NULL;
}
pthread_mutex_init(&disc->lock, NULL); pthread_mutex_init(&disc->lock, NULL);
// Nothing else is running yet, so the self set can be seeded without taking the lock.
Discovery_SeedSelfEndpoints(disc);
return disc; return disc;
} }
void NodeDiscovery_Destroy(node_discovery_t* disc) { void NodeDiscovery_Destroy(node_discovery_t* disc) {
if (!disc) return; if (!disc) return;
if (disc->peers) DynArr_destroy(disc->peers); if (disc->peers) DynArr_destroy(disc->peers);
if (disc->selfEndpoints) DynArr_destroy(disc->selfEndpoints);
if (disc->connectAttempts) DynArr_destroy(disc->connectAttempts);
pthread_mutex_destroy(&disc->lock); pthread_mutex_destroy(&disc->lock);
free(disc); free(disc);
} }
@@ -211,12 +399,14 @@ void NodeDiscovery_OnPingTimeout(node_discovery_t* disc, const struct sockaddr_s
void NodeDiscovery_OnGetPeers(node_discovery_t* disc, tcp_connection_t* fromConn) { void NodeDiscovery_OnGetPeers(node_discovery_t* disc, tcp_connection_t* fromConn) {
if (!disc || !fromConn) return; if (!disc || !fromConn) return;
// Snapshot our current peers' listen endpoints (inbound + outbound). // Snapshot our current peers' listen endpoints (inbound + outbound) and their identities.
struct sockaddr_storage all[MAX_CONS * 2]; struct sockaddr_storage all[MAX_CONS * 2];
size_t total = Node_GetPeerEndpoints(disc->node, all, sizeof(all) / sizeof(all[0])); uint64_t allIds[MAX_CONS * 2];
size_t total = Node_GetPeerEndpoints(disc->node, all, allIds, sizeof(all) / sizeof(all[0]));
struct sockaddr_storage reqEndpoint; struct sockaddr_storage reqEndpoint;
int haveReq = Node_ConnListenEndpoint(fromConn, &reqEndpoint); int haveReq = Node_ConnListenEndpoint(fromConn, &reqEndpoint);
uint64_t reqNodeId = Node_ConnPeerNodeId(fromConn);
// Build the response payload: [uint16 count][entries...], capped and sampled for spread. // Build the response payload: [uint16 count][entries...], capped and sampled for spread.
unsigned char payload[sizeof(uint16_t) + DISCOVERY_PEERS_RESPONSE_CAP * DISCOVERY_WIRE_ENTRY_SIZE]; unsigned char payload[sizeof(uint16_t) + DISCOVERY_PEERS_RESPONSE_CAP * DISCOVERY_WIRE_ENTRY_SIZE];
@@ -226,7 +416,11 @@ void NodeDiscovery_OnGetPeers(node_discovery_t* disc, tcp_connection_t* fromConn
size_t startIdx = total ? (size_t)(random_four_byte() % total) : 0; size_t startIdx = total ? (size_t)(random_four_byte() % total) : 0;
for (size_t k = 0; k < total && count < DISCOVERY_PEERS_RESPONSE_CAP; ++k) { for (size_t k = 0; k < total && count < DISCOVERY_PEERS_RESPONSE_CAP; ++k) {
size_t idx = (startIdx + k) % total; size_t idx = (startIdx + k) % total;
if (haveReq && Discovery_AddrEqual(&all[idx], &reqEndpoint)) continue; // don't tell them about themselves // Don't tell them about themselves. Matching on identity as well as on the endpoint they
// reached us from matters: a multi-homed peer is known to us under several addresses, and
// handing one of its own back to it is what makes it discover, ping and dial itself.
if (haveReq && Discovery_AddrEqual(&all[idx], &reqEndpoint)) continue;
if (reqNodeId != 0 && allIds[idx] == reqNodeId) continue;
unsigned char entry[DISCOVERY_WIRE_ENTRY_SIZE]; unsigned char entry[DISCOVERY_WIRE_ENTRY_SIZE];
if (!Discovery_AddrToWire(&all[idx], entry)) continue; if (!Discovery_AddrToWire(&all[idx], entry)) continue;
memcpy(payload + offset, entry, DISCOVERY_WIRE_ENTRY_SIZE); memcpy(payload + offset, entry, DISCOVERY_WIRE_ENTRY_SIZE);
@@ -292,6 +486,40 @@ void NodeDiscovery_RemovePeer(node_discovery_t* disc, const struct sockaddr_stor
pthread_mutex_unlock(&disc->lock); pthread_mutex_unlock(&disc->lock);
} }
void NodeDiscovery_NoteIdentity(node_discovery_t* disc, const struct sockaddr_storage* endpoint, uint64_t nodeId) {
if (!disc || !endpoint || nodeId == 0) return;
pthread_mutex_lock(&disc->lock);
if (nodeId == localNodeId) {
// The peer on the other end is us under one of our own addresses. Record it and drop it so
// discovery stops treating it as a peer.
Discovery_AddSelfUnlocked(disc, endpoint);
Discovery_RemoveUnlocked(disc, endpoint);
} else {
// Learn the endpoint if we did not already know it - a peer that dialled us is a perfectly
// good discovery candidate, and we now know both its listen endpoint and its identity.
discovered_peer_t* p = Discovery_Upsert(disc, endpoint, 0);
if (p) p->nodeId = nodeId;
}
pthread_mutex_unlock(&disc->lock);
}
void NodeDiscovery_MarkSelfEndpoint(node_discovery_t* disc, const struct sockaddr_storage* endpoint) {
if (!disc || !endpoint) return;
pthread_mutex_lock(&disc->lock);
Discovery_AddSelfUnlocked(disc, endpoint);
Discovery_RemoveUnlocked(disc, endpoint);
pthread_mutex_unlock(&disc->lock);
}
int NodeDiscovery_IsSelfEndpoint(node_discovery_t* disc, const struct sockaddr_storage* endpoint) {
if (!disc || !endpoint) return 0;
pthread_mutex_lock(&disc->lock);
int isSelf = Discovery_IsSelfUnlocked(disc, endpoint);
pthread_mutex_unlock(&disc->lock);
return isSelf;
}
// ---- periodic tick ----------------------------------------------------------------------- // ---- periodic tick -----------------------------------------------------------------------
void NodeDiscovery_Iterate(node_discovery_t* disc) { void NodeDiscovery_Iterate(node_discovery_t* disc) {
@@ -305,10 +533,12 @@ void NodeDiscovery_Iterate(node_discovery_t* disc) {
Node_GetClientList(disc->node, outConns, &outCount); Node_GetClientList(disc->node, outConns, &outCount);
struct sockaddr_storage outEndpoints[MAX_CONS]; struct sockaddr_storage outEndpoints[MAX_CONS];
uint64_t outNodeIds[MAX_CONS];
size_t outEpCount = 0; size_t outEpCount = 0;
for (size_t i = 0; i < outCount; ++i) { for (size_t i = 0; i < outCount; ++i) {
struct sockaddr_storage ep; struct sockaddr_storage ep;
if (Node_ConnListenEndpoint(outConns[i], &ep)) { if (Node_ConnListenEndpoint(outConns[i], &ep)) {
outNodeIds[outEpCount] = Node_ConnPeerNodeId(outConns[i]);
outEndpoints[outEpCount++] = ep; outEndpoints[outEpCount++] = ep;
} }
} }
@@ -328,6 +558,7 @@ void NodeDiscovery_Iterate(node_discovery_t* disc) {
if (p) { if (p) {
p->hop = 0; p->hop = 0;
p->state = DISCOVERY_STATE_CONNECTED; p->state = DISCOVERY_STATE_CONNECTED;
if (outNodeIds[i] != 0) p->nodeId = outNodeIds[i];
} }
} }
// Demote entries still marked CONNECTED that are no longer in the outbound set. // Demote entries still marked CONNECTED that are no longer in the outbound set.
@@ -426,17 +657,25 @@ void NodeDiscovery_Iterate(node_discovery_t* disc) {
for (size_t i = 0; i < n; ++i) { for (size_t i = 0; i < n; ++i) {
discovered_peer_t* p = (discovered_peer_t*)DynArr_at(disc->peers, i); discovered_peer_t* p = (discovered_peer_t*)DynArr_at(disc->peers, i);
if (p->state != DISCOVERY_STATE_REACHABLE) continue; if (p->state != DISCOVERY_STATE_REACHABLE) continue;
if (p->lastConnectMs != 0 && (now - p->lastConnectMs) < DISCOVERY_CONNECT_RETRY_MS) continue; if (!Discovery_ConnectCooledDown(disc, &p->addr, now)) continue;
int already = 0; int already = 0;
for (size_t j = 0; j < outEpCount; ++j) { for (size_t j = 0; j < outEpCount; ++j) {
if (Discovery_AddrEqual(&p->addr, &outEndpoints[j])) { already = 1; break; } if (Discovery_AddrEqual(&p->addr, &outEndpoints[j])) { already = 1; break; }
} }
// Skip other addresses of a node we already have an outbound connection to. Only
// outbound counts: an inbound connection from a peer is its own dial, and we still
// want one of our own to it (broadcasts only travel outbound).
if (!already && p->nodeId != 0) {
for (size_t j = 0; j < outEpCount; ++j) {
if (outNodeIds[j] == p->nodeId) { already = 1; break; }
}
}
if (already) continue; if (already) continue;
if (!best || p->pingMs < best->pingMs) best = p; if (!best || p->pingMs < best->pingMs) best = p;
} }
if (!best) break; if (!best) break;
best->lastConnectMs = now; // reserve so it isn't picked again this tick Discovery_NoteConnectAttempt(disc, &best->addr, now); // reserve so it isn't picked again this tick
char ip[INET6_ADDRSTRLEN]; char ip[INET6_ADDRSTRLEN];
unsigned short port = 0; unsigned short port = 0;
if (Discovery_AddrToIpPort(&best->addr, ip, sizeof(ip), &port) && port != 0) { if (Discovery_AddrToIpPort(&best->addr, ip, sizeof(ip), &port) && port != 0) {
@@ -479,12 +718,28 @@ void NodeDiscovery_PrintPeers(node_discovery_t* disc) {
unsigned short port = 0; unsigned short port = 0;
Discovery_AddrToIpPort(&p->addr, ip, sizeof(ip), &port); Discovery_AddrToIpPort(&p->addr, ip, sizeof(ip), &port);
const char* stateStr = (p->state <= DISCOVERY_STATE_UNREACHABLE) ? stateNames[p->state] : "?"; const char* stateStr = (p->state <= DISCOVERY_STATE_UNREACHABLE) ? stateNames[p->state] : "?";
if (p->pingMs == UINT64_MAX) { char idStr[19];
printf(" %-46s hop=%u state=%-11s ping=--\n", ip, p->hop, stateStr); if (p->nodeId != 0) {
snprintf(idStr, sizeof(idStr), "%016" PRIx64, p->nodeId);
} else { } else {
printf(" %-46s hop=%u state=%-11s ping=%" PRIu64 "ms\n", ip, p->hop, stateStr, p->pingMs); snprintf(idStr, sizeof(idStr), "%-16s", "?");
}
if (p->pingMs == UINT64_MAX) {
printf(" %-46s hop=%u state=%-11s id=%s ping=--\n", ip, p->hop, stateStr, idStr);
} else {
printf(" %-46s hop=%u state=%-11s id=%s ping=%" PRIu64 "ms\n", ip, p->hop, stateStr, idStr, p->pingMs);
} }
(void)port; // port is part of ip endpoint identity; shown via connect logs (void)port; // port is part of ip endpoint identity; shown via connect logs
} }
size_t selfCount = DynArr_size(disc->selfEndpoints);
printf("Own endpoints (%zu):\n", selfCount);
for (size_t i = 0; i < selfCount; ++i) {
const struct sockaddr_storage* self = (const struct sockaddr_storage*)DynArr_at(disc->selfEndpoints, i);
char ip[INET6_ADDRSTRLEN] = {0};
unsigned short port = 0;
Discovery_AddrToIpPort(self, ip, sizeof(ip), &port);
printf(" %-46s port=%u\n", ip, port);
}
pthread_mutex_unlock(&disc->lock); pthread_mutex_unlock(&disc->lock);
} }
+31
View File
@@ -1,5 +1,8 @@
#include <numgen.h> #include <numgen.h>
#include <stdio.h>
#include <unistd.h>
unsigned char random_byte(void) { unsigned char random_byte(void) {
return (unsigned char)(rand() % 256); return (unsigned char)(rand() % 256);
} }
@@ -39,3 +42,31 @@ uint64_t random_eight_byte(void) {
return x; return x;
} }
uint64_t random_secure_eight_byte(void) {
uint64_t x = 0;
FILE* urandom = fopen("/dev/urandom", "rb");
if (urandom) {
size_t got = fread(&x, 1, sizeof(x), urandom);
fclose(urandom);
if (got == sizeof(x) && x != 0) {
return x;
}
}
// Fallback: srand() is seeded from the wall clock in whole seconds, so two nodes launched
// together would draw identical values. Mix in the pid and the sub-second clock to separate them.
struct timespec ts;
if (clock_gettime(CLOCK_REALTIME, &ts) != 0) {
ts.tv_sec = 0;
ts.tv_nsec = 0;
}
x = random_eight_byte();
x ^= (uint64_t)ts.tv_nsec;
x ^= ((uint64_t)ts.tv_sec) << 16;
x ^= ((uint64_t)getpid()) << 40;
return x ? x : 1; // 0 means "no identity advertised" on the wire
}