diff --git a/CMakeLists.txt b/CMakeLists.txt index b75f9c9..3fe59bb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -230,5 +230,8 @@ target_compile_definitions(node PRIVATE CHAIN_DATA_DIR="${CMAKE_BINARY_DIR}/chain_data" $<$:SKALACOIN_AUTOLYKOS2_REF_AVAILABLE> $<$:_POSIX_C_SOURCE=200809L> + # getifaddrs() (used to learn our own addresses) is a BSD extension, not POSIX; glibc hides it + # unless the default set is requested as well. + $<$:_DEFAULT_SOURCE> ) set_target_properties(node PROPERTIES OUTPUT_NAME "skalacoin_node") diff --git a/include/nets/net_node.h b/include/nets/net_node.h index 22ff261..b0d7ede 100644 --- a/include/nets/net_node.h +++ b/include/nets/net_node.h @@ -85,8 +85,12 @@ void Node_GetClientList(net_node_t* node, tcp_connection_t** outClients, size_t* // Returns non-zero on success (usable endpoint with a known, non-zero port), zero otherwise. int Node_ConnListenEndpoint(const tcp_connection_t* conn, struct sockaddr_storage* out); +// Returns the node identity advertised by a connection's peer, or 0 if it is not known yet. +uint64_t Node_ConnPeerNodeId(const tcp_connection_t* conn); + // Fills outEndpoints with the listen endpoints of all current connections (inbound + outbound), -// deduped by IP+port. Returns the number of endpoints written (<= maxOut). -size_t Node_GetPeerEndpoints(net_node_t* node, struct sockaddr_storage* outEndpoints, size_t maxOut); +// deduped by IP+port, and outNodeIds (optional, may be NULL) with the matching peer identities. +// Returns the number of endpoints written (<= maxOut). +size_t Node_GetPeerEndpoints(net_node_t* node, struct sockaddr_storage* outEndpoints, uint64_t* outNodeIds, size_t maxOut); #endif diff --git a/include/nets/nodediscovery.h b/include/nets/nodediscovery.h index 21bb314..41e5cc8 100644 --- a/include/nets/nodediscovery.h +++ b/include/nets/nodediscovery.h @@ -27,6 +27,18 @@ void NodeDiscovery_OnPeersReceived(node_discovery_t* disc, tcp_connection_t* fro // logically disconnected (no remaining connection to it). void NodeDiscovery_RemovePeer(node_discovery_t* disc, const struct sockaddr_storage* endpoint); +// Record the node identity behind an endpoint (learned from a completed HELLO/ACK_HELLO). Entries +// carrying an identity we are already connected to are skipped by the connect picker, which is what +// stops a multi-homed peer from being dialed once per address it is reachable on. +void NodeDiscovery_NoteIdentity(node_discovery_t* disc, const struct sockaddr_storage* endpoint, uint64_t nodeId); + +// Mark an endpoint as one of our own, permanently. Self endpoints are never added to the known-peer +// table, never pinged and never dialed. Seeded from the local interface addresses at creation and +// extended whenever a handshake turns out to come from ourselves. +void NodeDiscovery_MarkSelfEndpoint(node_discovery_t* disc, const struct sockaddr_storage* endpoint); +// Returns non-zero if the endpoint is known to be one of our own. +int NodeDiscovery_IsSelfEndpoint(node_discovery_t* disc, const struct sockaddr_storage* endpoint); + // Dump the known-peer table to stdout (for the CLI `peers` command). void NodeDiscovery_PrintPeers(node_discovery_t* disc); diff --git a/include/numgen.h b/include/numgen.h index ad3207e..6a7b21e 100644 --- a/include/numgen.h +++ b/include/numgen.h @@ -11,4 +11,9 @@ uint16_t random_two_byte(void); uint32_t random_four_byte(void); uint64_t random_eight_byte(void); +// Draws from the OS entropy pool instead of the srand()-seeded generator, which repeats across +// processes started within the same second. Use this wherever a value must be unique between nodes +// (e.g. the node identity). Never returns 0. +uint64_t random_secure_eight_byte(void); + #endif diff --git a/include/runtime_state.h b/include/runtime_state.h index 1f904df..ad4cbdd 100644 --- a/include/runtime_state.h +++ b/include/runtime_state.h @@ -17,6 +17,10 @@ extern const char* chainDataDir; extern unsigned short listenPort; extern bool echoPeersEnabled; extern bool forceOrphanReorgEnabled; +// Random per-run identity of this node, advertised in HELLO/ACK_HELLO. A host can be reachable +// under many addresses (especially over IPv6), so an (ip, port) endpoint is not a peer identity: +// this nonce is what lets us recognise our own connections and a peer we already talk to. +extern uint64_t localNodeId; // Global synchronization primitives for runtime state extern pthread_rwlock_t chainLock; // protects chain structure and related mutations diff --git a/include/tcpd/tcpconnection.h b/include/tcpd/tcpconnection.h index 1e407c5..74e09fb 100644 --- a/include/tcpd/tcpconnection.h +++ b/include/tcpd/tcpconnection.h @@ -31,6 +31,11 @@ struct tcp_connection_t { // For OUTBOUND connections the peerAddr port already is the listen port; this matters for INBOUND peers. uint16_t peerListenPort; + // Peer's advertised node identity (learned from HELLO/ACK_HELLO). 0 until known / peer too old + // to advertise one. Unlike the peer address, this is stable across all of a multi-homed peer's + // endpoints, so it is what identifies the node behind this connection. + uint64_t peerNodeId; + pthread_t ioThread; pthread_mutex_t sendLock; pthread_mutex_t stateLock; diff --git a/src/main.c b/src/main.c index 7e6a101..6b09bb9 100644 --- a/src/main.c +++ b/src/main.c @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include @@ -34,6 +36,7 @@ bool echoPeersEnabled = ECHO_PEERS != 0; bool forceOrphanReorgEnabled = false; uint256_t currentSupply = {{0, 0, 0, 0}}; uint64_t currentReward = 750000000000ULL; +uint64_t localNodeId = 0; // Randomised in main() before the node comes up // Define the synchronization primitives declared in runtime_state.h pthread_rwlock_t chainLock; @@ -678,7 +681,16 @@ int main(int argc, char* argv[]) { // (handled by the send paths) instead of terminating the whole process. Peers connecting and // disconnecting is normal p2p behaviour and must never take the node down. signal(SIGPIPE, SIG_IGN); - srand((unsigned int)time(NULL)); + // Mix the pid into the seed: nodes launched within the same second would otherwise draw + // identical sequences, so every rand()-derived value (connection ids and the like) would + // collide across them. + srand((unsigned int)time(NULL) ^ ((unsigned int)getpid() << 16)); + + // Pick this run's node identity before the node (and with it the listener) comes up, so every + // handshake can carry it. Peers are identified by this nonce rather than by an (ip, port) + // endpoint, which a multi-homed host has several of. + localNodeId = random_secure_eight_byte(); + printf("Node identity: %016" PRIx64 "\n", localNodeId); // Initialize runtime locks before any thread or helper can touch chain state. pthread_rwlock_init(&chainLock, NULL); diff --git a/src/nets/net_node.c b/src/nets/net_node.c index fcfeeff..496d493 100644 --- a/src/nets/net_node.c +++ b/src/nets/net_node.c @@ -102,13 +102,15 @@ static int Node_HasOutboundTo(net_node_t* node, const struct sockaddr_storage* e // Returns non-zero if some inbound connection OTHER than `self` already has the given listen // endpoint (used to reject a duplicate inbound once we learn the peer's advertised listen port). +// Connections that are already tearing down do not count - otherwise a peer reconnecting from the +// same endpoint gets its fresh inbound rejected by the corpse of the previous one. static int Node_HasOtherInboundFrom(net_node_t* node, const tcp_connection_t* self, const struct sockaddr_storage* endpoint) { if (!node->server) return 0; int found = 0; pthread_mutex_lock(&node->server->clientsMutex); for (size_t i = 0; i < node->server->maxClients; ++i) { tcp_connection_t* other = node->server->clientsArrPtr ? node->server->clientsArrPtr[i] : NULL; - if (!other || other == self) continue; + if (!other || other == self || TcpConnection_IsDisconnectNotified(other)) continue; struct sockaddr_storage ep; if (Node_ConnListenEndpoint(other, &ep) && NetNode_EndpointEqual(&ep, endpoint)) { found = 1; @@ -119,16 +121,47 @@ static int Node_HasOtherInboundFrom(net_node_t* node, const tcp_connection_t* se return found; } -// Returns non-zero if a connection OTHER than `exclude` to `endpoint` is still live. A connection -// that is itself mid-disconnect (disconnectedNotified) does not count as live - this is what lets -// us decide a peer is fully gone even when both its inbound and outbound drop simultaneously. -static int Node_HasLiveConnectionTo(net_node_t* node, const struct sockaddr_storage* endpoint, const tcp_connection_t* exclude) { +// Returns non-zero if a live connection OTHER than `self` with the same role already belongs to the +// node identified by nodeId. This is the endpoint-independent duplicate check: a multi-homed peer +// reaches us from several addresses, so comparing endpoints alone lets the same node in twice. +static int Node_HasOtherConnectionToNode(net_node_t* node, const tcp_connection_t* self, uint64_t nodeId) { + if (nodeId == 0) return 0; + int found = 0; + + pthread_mutex_lock(&node->outboundLock); + for (size_t i = 0; i < MAX_CONS && !found; ++i) { + tcp_connection_t* c = node->outboundClients[i].connection; + if (!c || c == self || TcpConnection_IsDisconnectNotified(c)) continue; + if (c->role == self->role && c->peerNodeId == nodeId) found = 1; + } + pthread_mutex_unlock(&node->outboundLock); + if (found) return 1; + + if (node->server) { + pthread_mutex_lock(&node->server->clientsMutex); + for (size_t i = 0; i < node->server->maxClients && !found; ++i) { + tcp_connection_t* c = node->server->clientsArrPtr ? node->server->clientsArrPtr[i] : NULL; + if (!c || c == self || TcpConnection_IsDisconnectNotified(c)) continue; + if (c->role == self->role && c->peerNodeId == nodeId) found = 1; + } + pthread_mutex_unlock(&node->server->clientsMutex); + } + return found; +} + +// Returns non-zero if a connection OTHER than `exclude` to the same peer is still live - matched +// either on the listen endpoint or, when known, on the peer's identity (which also covers its other +// addresses). A connection that is itself mid-disconnect (disconnectedNotified) does not count as +// live - this is what lets us decide a peer is fully gone even when both its inbound and outbound +// drop simultaneously. +static int Node_HasLiveConnectionTo(net_node_t* node, const struct sockaddr_storage* endpoint, uint64_t nodeId, const tcp_connection_t* exclude) { int found = 0; pthread_mutex_lock(&node->outboundLock); for (size_t i = 0; i < MAX_CONS && !found; ++i) { tcp_connection_t* c = node->outboundClients[i].connection; if (!c || c == exclude || TcpConnection_IsDisconnectNotified(c)) continue; + if (nodeId != 0 && c->peerNodeId == nodeId) { found = 1; break; } struct sockaddr_storage ep; if (Node_ConnListenEndpoint(c, &ep) && NetNode_EndpointEqual(&ep, endpoint)) found = 1; } @@ -140,6 +173,7 @@ static int Node_HasLiveConnectionTo(net_node_t* node, const struct sockaddr_stor for (size_t i = 0; i < node->server->maxClients && !found; ++i) { tcp_connection_t* c = node->server->clientsArrPtr ? node->server->clientsArrPtr[i] : NULL; if (!c || c == exclude || TcpConnection_IsDisconnectNotified(c)) continue; + if (nodeId != 0 && c->peerNodeId == nodeId) { found = 1; break; } struct sockaddr_storage ep; if (Node_ConnListenEndpoint(c, &ep) && NetNode_EndpointEqual(&ep, endpoint)) found = 1; } @@ -150,13 +184,14 @@ static int Node_HasLiveConnectionTo(net_node_t* node, const struct sockaddr_stor // Called when a connection to a peer drops. Strikes the peer from the discovery table, but only // once it is logically disconnected - i.e. no other live connection (inbound or outbound) to the -// same listen endpoint remains. Must be called from the disconnect callback while `conn` is still -// valid and outside outboundLock/clientsMutex. +// same node remains. Must be called from the disconnect callback while `conn` is still valid and +// outside outboundLock/clientsMutex. static void Node_HandlePeerDisconnect(net_node_t* node, tcp_connection_t* conn) { if (!node || !node->discovery || !conn) return; struct sockaddr_storage ep; - if (!Node_ConnListenEndpoint(conn, &ep)) return; // never advertised an endpoint -> not tracked - if (Node_HasLiveConnectionTo(node, &ep, conn)) return; // still reachable via another connection + if (!Node_ConnListenEndpoint(conn, &ep)) return; // never advertised an endpoint -> not tracked + // Still reachable via another connection (possibly on one of its other addresses). + if (Node_HasLiveConnectionTo(node, &ep, conn->peerNodeId, conn)) return; NodeDiscovery_RemovePeer(node->discovery, &ep); } @@ -203,7 +238,11 @@ int Node_ConnListenEndpoint(const tcp_connection_t* conn, struct sockaddr_storag return 0; } -size_t Node_GetPeerEndpoints(net_node_t* node, struct sockaddr_storage* outEndpoints, size_t maxOut) { +uint64_t Node_ConnPeerNodeId(const tcp_connection_t* conn) { + return conn ? conn->peerNodeId : 0; +} + +size_t Node_GetPeerEndpoints(net_node_t* node, struct sockaddr_storage* outEndpoints, uint64_t* outNodeIds, size_t maxOut) { if (!node || !outEndpoints || maxOut == 0) return 0; size_t count = 0; @@ -218,7 +257,9 @@ size_t Node_GetPeerEndpoints(net_node_t* node, struct sockaddr_storage* outEndpo for (size_t k = 0; k < count; ++k) { if (NetNode_EndpointEqual(&outEndpoints[k], &ep)) { dup = 1; break; } } - if (!dup) outEndpoints[count++] = ep; + if (dup) continue; + if (outNodeIds) outNodeIds[count] = c->peerNodeId; + outEndpoints[count++] = ep; } pthread_mutex_unlock(&node->outboundLock); @@ -234,7 +275,9 @@ size_t Node_GetPeerEndpoints(net_node_t* node, struct sockaddr_storage* outEndpo for (size_t k = 0; k < count; ++k) { if (NetNode_EndpointEqual(&outEndpoints[k], &ep)) { dup = 1; break; } } - if (!dup) outEndpoints[count++] = ep; + if (dup) continue; + if (outNodeIds) outNodeIds[count] = c->peerNodeId; + outEndpoints[count++] = ep; } pthread_mutex_unlock(&node->server->clientsMutex); } @@ -242,6 +285,44 @@ size_t Node_GetPeerEndpoints(net_node_t* node, struct sockaddr_storage* outEndpo return count; } +// Outcome of the identity check run once a connection's HELLO/ACK_HELLO has been parsed. +typedef enum { + NODE_IDENTITY_OK = 0, + NODE_IDENTITY_SELF, // the peer is this very node, reached through one of its own addresses + NODE_IDENTITY_DUPLICATE // we already hold a connection of this role to that node +} node_identity_result_t; + +// Records the identity a peer advertised and decides whether the connection should survive. +// `conn->peerNodeId` and `conn->peerListenPort` must already be set from the handshake. +static node_identity_result_t Node_CheckPeerIdentity(net_node_t* node, tcp_connection_t* conn) { + if (!node || !conn || conn->peerNodeId == 0) return NODE_IDENTITY_OK; // peer too old to advertise one + + struct sockaddr_storage ep; + int haveEp = Node_ConnListenEndpoint(conn, &ep); + + if (conn->peerNodeId == localNodeId) { + // We dialled ourselves (or accepted our own dial). Remember the endpoint as our own so + // discovery stops offering it back to us, and drop the connection. + if (haveEp && node->discovery) { + NodeDiscovery_MarkSelfEndpoint(node->discovery, &ep); + } + return NODE_IDENTITY_SELF; + } + + // Record the identity behind this endpoint even when the connection is about to be dropped as a + // duplicate: that is what lets discovery skip the peer's other addresses while we are connected + // to it, instead of dialling each of them in turn. + if (haveEp && node->discovery) { + NodeDiscovery_NoteIdentity(node->discovery, &ep, conn->peerNodeId); + } + + if (Node_HasOtherConnectionToNode(node, conn, conn->peerNodeId)) { + return NODE_IDENTITY_DUPLICATE; + } + + return NODE_IDENTITY_OK; +} + // Thunks routing UDP ping/pong events into the discovery state. static void Node_OnPongThunk(udp_node_t* udp, const struct sockaddr_storage* from, uint64_t nonce, int protoVersion, uint64_t rttMs, void* user) { @@ -641,11 +722,18 @@ int Node_ConnectPeer(net_node_t* node, const char* ip, unsigned short port) { return -1; } + // Never dial ourselves. Without this an echo-back (or a gossiped copy of one of our own + // addresses) can chain into a self-connection per maintenance tick until the slots run out. + struct sockaddr_storage target; + int haveTarget = NetNode_MakeEndpoint(ip, port, &target); + if (haveTarget && node->discovery && NodeDiscovery_IsSelfEndpoint(node->discovery, &target)) { + return -1; + } + // Enforce a single outbound connection per endpoint: if we already have an outbound to this // (ip, port), do not open a second one. (Inbound from the same endpoint is still allowed - that // is the peer's own outbound to us.) - struct sockaddr_storage target; - if (NetNode_MakeEndpoint(ip, port, &target) && Node_HasOutboundTo(node, &target)) { + if (haveTarget && Node_HasOutboundTo(node, &target)) { return 0; // already connected outbound to this endpoint } @@ -807,27 +895,23 @@ void Node_Server_OnData(tcp_connection_t* client) { client->peerListenPort = peerListenPort; } - printf("Received HELLO from node %u: protoVersion=%u, blockHeight=%" PRIu64 ", listenPort=%u\n", - client ? client->connectionId : 0U, protoVersion, blockHeight, - client ? client->peerListenPort : 0U); - - // Enforce a single inbound connection per endpoint. Now that we know this peer's listen - // port, drop this connection if another inbound from the same endpoint already exists - // (keep the established one). An outbound to the same endpoint is unaffected - that is - // this node's own connection to the peer. - if (client && client->peerListenPort != 0) { - net_node_t* dupNode = Node_FromConnection(client); - struct sockaddr_storage myEp; - if (dupNode && Node_ConnListenEndpoint(client, &myEp) && - Node_HasOtherInboundFrom(dupNode, client, &myEp)) { - printf("Rejecting duplicate inbound connection %u (already have an inbound from this endpoint)\n", - client->connectionId); - TcpConnection_RequestClose(client); - return; - } + // Optional trailing node identity, same length-guarded deal. + if (client && payloadLen >= sizeof(uint32_t) + sizeof(uint64_t) + sizeof(uint16_t) + sizeof(uint64_t)) { + uint64_t peerNodeId; + memcpy(&peerNodeId, payload + sizeof(protoVersion) + sizeof(blockHeight) + sizeof(uint16_t), sizeof(peerNodeId)); + client->peerNodeId = peerNodeId; } - // Craft and send ACK_HELLO (echo protoVersion, our height, and our own listen port) + printf("Received HELLO from node %u: protoVersion=%u, blockHeight=%" PRIu64 ", listenPort=%u, nodeId=%016" PRIx64 "\n", + client ? client->connectionId : 0U, protoVersion, blockHeight, + client ? client->peerListenPort : 0U, client ? client->peerNodeId : 0ULL); + + // Craft and send ACK_HELLO (echo protoVersion, our height, our own listen port and our + // identity). This goes out before any decision to drop the connection: the ACK is what + // tells the dialer whose address it just reached, so an endpoint that turns out to be + // another address of a peer it already talks to (or one of its own) is recognised as + // such instead of being redialled forever. shutdown() flushes what is already queued, + // so the peer still receives this even though we close immediately after. uint8_t ackBuf[100]; uint8_t* ackData = ackBuf; size_t ackOffset = 0; @@ -839,9 +923,47 @@ void Node_Server_OnData(tcp_connection_t* client) { uint16_t myListenPort = (uint16_t)listenPort; memcpy(ackData + ackOffset, &myListenPort, sizeof(myListenPort)); ackOffset += sizeof(myListenPort); + uint64_t myNodeId = localNodeId; + memcpy(ackData + ackOffset, &myNodeId, sizeof(myNodeId)); + ackOffset += sizeof(myNodeId); Node_SendPacket(Node_FromConnection(client), client, PACKET_TYPE_ACK_HELLO, ackData, ackOffset); + // Enforce one connection per node, identified by the advertised nodeId rather than by + // the address it happens to reach us from. + if (client) { + net_node_t* idNode = Node_FromConnection(client); + node_identity_result_t identity = Node_CheckPeerIdentity(idNode, client); + if (identity == NODE_IDENTITY_SELF) { + printf("Rejecting inbound connection %u: it is this node talking to itself\n", + client->connectionId); + TcpConnection_RequestClose(client); + return; + } + if (identity == NODE_IDENTITY_DUPLICATE) { + printf("Rejecting duplicate inbound connection %u (already connected to node %016" PRIx64 ")\n", + client->connectionId, client->peerNodeId); + TcpConnection_RequestClose(client); + return; + } + } + + // Endpoint-level fallback for peers that advertise no identity: drop this connection if + // another inbound from the same endpoint already exists (keep the established one). An + // outbound to the same endpoint is unaffected - that is this node's own connection to + // the peer. + if (client && client->peerNodeId == 0 && client->peerListenPort != 0) { + net_node_t* dupNode = Node_FromConnection(client); + struct sockaddr_storage myEp; + if (dupNode && Node_ConnListenEndpoint(client, &myEp) && + Node_HasOtherInboundFrom(dupNode, client, &myEp)) { + printf("Rejecting duplicate inbound connection %u (already have an inbound from this endpoint)\n", + client->connectionId); + TcpConnection_RequestClose(client); + return; + } + } + break; } case PACKET_TYPE_ACK_HELLO: { @@ -1087,6 +1209,10 @@ void Node_Client_OnConnect(tcp_connection_t* client) { uint16_t myListenPort = (uint16_t)listenPort; memcpy((unsigned char*)data + offset, &myListenPort, sizeof(myListenPort)); offset += sizeof(myListenPort); + // ...and who we are, so the peer can tell this connection apart from our other addresses + uint64_t myNodeId = localNodeId; + memcpy((unsigned char*)data + offset, &myNodeId, sizeof(myNodeId)); + offset += sizeof(myNodeId); Node_SendPacket(node, client, PACKET_TYPE_HELLO, data, offset); } @@ -1131,10 +1257,36 @@ void Node_Client_OnData(tcp_connection_t* client) { client->peerListenPort = peerListenPort; } - printf("Received ACK_HELLO from node %u with protoVersion %u and blockHeight %" PRIu64 "\n", client ? client->connectionId : 0U, protoVersion, blockHeight); + // Optional trailing node identity, same length-guarded deal. + if (client && payloadLen >= sizeof(uint32_t) + sizeof(uint64_t) + sizeof(uint16_t) + sizeof(uint64_t)) { + uint64_t peerNodeId; + memcpy(&peerNodeId, payload + sizeof(protoVersion) + sizeof(blockHeight) + sizeof(uint16_t), sizeof(peerNodeId)); + client->peerNodeId = peerNodeId; + } + + printf("Received ACK_HELLO from node %u with protoVersion %u, blockHeight %" PRIu64 " and nodeId %016" PRIx64 "\n", + client ? client->connectionId : 0U, protoVersion, blockHeight, client ? client->peerNodeId : 0ULL); // Store peer-advertised height on matching outbound client net_node_t* node = Node_FromConnection(client); + + // The dialed endpoint may well be one of our own addresses, or another address of a + // peer we already talk to - neither is worth a connection. + if (client) { + node_identity_result_t identity = Node_CheckPeerIdentity(node, client); + if (identity == NODE_IDENTITY_SELF) { + printf("Closing outbound connection %u: it loops back to this node\n", client->connectionId); + TcpConnection_RequestClose(client); + return; + } + if (identity == NODE_IDENTITY_DUPLICATE) { + printf("Closing outbound connection %u: already connected to node %016" PRIx64 " on another address\n", + client->connectionId, client->peerNodeId); + TcpConnection_RequestClose(client); + return; + } + } + if (node) { pthread_mutex_lock(&node->outboundLock); for (size_t i = 0; i < MAX_CONS; ++i) { diff --git a/src/nets/nodediscovery.c b/src/nets/nodediscovery.c index 8e0c43d..12a72c5 100644 --- a/src/nets/nodediscovery.c +++ b/src/nets/nodediscovery.c @@ -8,9 +8,12 @@ #include #include +#include + #include #include #include +#include #include // Wire layout of a single peer endpoint inside a PEERS payload: @@ -29,19 +32,29 @@ typedef enum { typedef struct { struct sockaddr_storage addr; // listen endpoint (port already set to the peer's listen port) uint64_t pingMs; // measured UDP RTT, UINT64_MAX if unknown + uint64_t nodeId; // identity of the node behind this endpoint, 0 while unknown uint32_t hop; // distance from us (0 = directly connected) discovery_state_t state; int pingPending; // 1 while a ping is outstanding (matched by address on pong/timeout). // The UDP layer generates its own nonce, so we can't match by nonce here. uint64_t lastPingMs; // when we last sent a ping uint64_t lastQueryMs; // when we last sent GET_PEERS to it - uint64_t lastConnectMs; // when we last attempted a connect to it } discovered_peer_t; +// When we last dialed an endpoint. Kept outside the peer table on purpose: a peer entry is struck +// the moment its connection drops, and if the dial history went with it, an endpoint that hangs up +// on us would be re-learned through gossip and redialed on every single tick. +typedef struct { + struct sockaddr_storage addr; + uint64_t lastMs; +} discovery_attempt_t; + struct node_discovery { net_node_t* node; udp_node_t* udpNode; DynArr* peers; // of discovered_peer_t + DynArr* selfEndpoints; // of struct sockaddr_storage - our own listen endpoints + DynArr* connectAttempts; // of discovery_attempt_t pthread_mutex_t lock; }; @@ -64,6 +77,99 @@ static int Discovery_AddrEqual(const struct sockaddr_storage* a, const struct so return 0; } +// Rejects endpoints that can never be dialed as written. IPv6 in particular hands us plenty of +// these: link-local addresses are meaningless without the scope id (which the wire format does not +// carry), and the unspecified/multicast ranges are never a peer. Loopback stays allowed so several +// nodes can still be run on one machine on different ports. +static int Discovery_IsUsableAddr(const struct sockaddr_storage* addr) { + if (addr->ss_family == AF_INET) { + const struct sockaddr_in* a = (const struct sockaddr_in*)addr; + if (a->sin_port == 0) return 0; + uint32_t host = ntohl(a->sin_addr.s_addr); + if (host == INADDR_ANY || host == INADDR_BROADCAST) return 0; + if ((host >> 28) == 0xE) return 0; // 224.0.0.0/4 multicast + if ((host & 0xFFFF0000u) == 0xA9FE0000u) return 0; // 169.254.0.0/16 link-local + return 1; + } + if (addr->ss_family == AF_INET6) { + const struct sockaddr_in6* a = (const struct sockaddr_in6*)addr; + if (a->sin6_port == 0) return 0; + if (IN6_IS_ADDR_UNSPECIFIED(&a->sin6_addr)) return 0; + if (IN6_IS_ADDR_MULTICAST(&a->sin6_addr)) return 0; + if (IN6_IS_ADDR_LINKLOCAL(&a->sin6_addr)) return 0; // unusable without a scope id + if (IN6_IS_ADDR_SITELOCAL(&a->sin6_addr)) return 0; // deprecated fec0::/10 + return 1; + } + return 0; +} + +// Rewrites an IPv4-mapped IPv6 endpoint (::ffff:a.b.c.d) as plain IPv4, so the same host never +// occupies two entries. Matches the normalisation Node_ConnListenEndpoint does. +static void Discovery_NormaliseAddr(struct sockaddr_storage* addr) { + if (addr->ss_family != AF_INET6) return; + struct sockaddr_in6* a = (struct sockaddr_in6*)addr; + if (!IN6_IS_ADDR_V4MAPPED(&a->sin6_addr)) return; + + struct in_addr v4; + memcpy(&v4, ((const uint8_t*)&a->sin6_addr) + 12, sizeof(v4)); + uint16_t port = a->sin6_port; + + memset(addr, 0, sizeof(*addr)); + struct sockaddr_in* o = (struct sockaddr_in*)addr; + o->sin_family = AF_INET; + o->sin_addr = v4; + o->sin_port = port; +} + +// Returns non-zero if addr is one of our own listen endpoints. Caller holds disc->lock. +static int Discovery_IsSelfUnlocked(node_discovery_t* disc, const struct sockaddr_storage* addr) { + size_t n = DynArr_size(disc->selfEndpoints); + for (size_t i = 0; i < n; ++i) { + const struct sockaddr_storage* self = (const struct sockaddr_storage*)DynArr_at(disc->selfEndpoints, i); + if (Discovery_AddrEqual(self, addr)) return 1; + } + return 0; +} + +// Adds addr to the self set if not already there. Caller holds disc->lock. +static void Discovery_AddSelfUnlocked(node_discovery_t* disc, const struct sockaddr_storage* addr) { + if (Discovery_IsSelfUnlocked(disc, addr)) return; + DynArr_push_back(disc->selfEndpoints, (void*)addr); +} + +// Seeds the self set with (local interface address, our listen port) for every address this host +// carries. A multi-homed host - the normal case under IPv6, where a machine holds a global, a +// temporary privacy and a link-local address at once - is otherwise unable to tell its own +// endpoints from a peer's when they come back around through peer exchange. +static void Discovery_SeedSelfEndpoints(node_discovery_t* disc) { + struct ifaddrs* ifa = NULL; + if (getifaddrs(&ifa) != 0 || !ifa) return; + + for (struct ifaddrs* it = ifa; it; it = it->ifa_next) { + if (!it->ifa_addr) continue; + + struct sockaddr_storage ep; + memset(&ep, 0, sizeof(ep)); + if (it->ifa_addr->sa_family == AF_INET) { + struct sockaddr_in* o = (struct sockaddr_in*)&ep; + memcpy(o, it->ifa_addr, sizeof(struct sockaddr_in)); + o->sin_port = htons(listenPort); + } else if (it->ifa_addr->sa_family == AF_INET6) { + struct sockaddr_in6* o = (struct sockaddr_in6*)&ep; + memcpy(o, it->ifa_addr, sizeof(struct sockaddr_in6)); + o->sin6_port = htons(listenPort); + o->sin6_scope_id = 0; // endpoints on the wire are scopeless; compare them the same way + } else { + continue; + } + + Discovery_NormaliseAddr(&ep); + Discovery_AddSelfUnlocked(disc, &ep); + } + + freeifaddrs(ifa); +} + static discovered_peer_t* Discovery_FindPtr(node_discovery_t* disc, const struct sockaddr_storage* addr) { size_t n = DynArr_size(disc->peers); for (size_t i = 0; i < n; ++i) { @@ -73,9 +179,13 @@ static discovered_peer_t* Discovery_FindPtr(node_discovery_t* disc, const struct return NULL; } -// Insert addr if not already present. Returns a pointer to the (existing or new) entry, or NULL -// if the table is full. Note: the returned pointer is invalidated by any later push_back. +// Insert addr if not already present. Returns a pointer to the (existing or new) entry, or NULL if +// the address is unusable, is one of our own, or the table is full. Note: the returned pointer is +// invalidated by any later push_back. static discovered_peer_t* Discovery_Upsert(node_discovery_t* disc, const struct sockaddr_storage* addr, uint32_t hop) { + if (!Discovery_IsUsableAddr(addr)) return NULL; + if (Discovery_IsSelfUnlocked(disc, addr)) return NULL; // never track, ping or dial ourselves + discovered_peer_t* existing = Discovery_FindPtr(disc, addr); if (existing) { if (hop < existing->hop) existing->hop = hop; // keep the shortest known distance @@ -87,12 +197,71 @@ static discovered_peer_t* Discovery_Upsert(node_discovery_t* disc, const struct memset(&np, 0, sizeof(np)); np.addr = *addr; np.pingMs = UINT64_MAX; + np.nodeId = 0; np.hop = hop; np.state = DISCOVERY_STATE_NEW; DynArr_push_back(disc->peers, &np); return (discovered_peer_t*)DynArr_at(disc->peers, DynArr_size(disc->peers) - 1); } +// Returns non-zero if addr may be dialed again, i.e. we have not tried it within the retry window. +// Caller holds disc->lock. +static int Discovery_ConnectCooledDown(node_discovery_t* disc, const struct sockaddr_storage* addr, uint64_t now) { + size_t n = DynArr_size(disc->connectAttempts); + for (size_t i = 0; i < n; ++i) { + const discovery_attempt_t* a = (const discovery_attempt_t*)DynArr_at(disc->connectAttempts, i); + if (Discovery_AddrEqual(&a->addr, addr)) { + return (now - a->lastMs) >= DISCOVERY_CONNECT_RETRY_MS; + } + } + return 1; // never dialed +} + +// Stamps a dial attempt against addr, evicting the stalest record once the table is full. +// Caller holds disc->lock. +static void Discovery_NoteConnectAttempt(node_discovery_t* disc, const struct sockaddr_storage* addr, uint64_t now) { + size_t n = DynArr_size(disc->connectAttempts); + size_t oldestIdx = 0; + uint64_t oldestMs = UINT64_MAX; + + for (size_t i = 0; i < n; ++i) { + discovery_attempt_t* a = (discovery_attempt_t*)DynArr_at(disc->connectAttempts, i); + if (Discovery_AddrEqual(&a->addr, addr)) { + a->lastMs = now; + return; + } + if (a->lastMs < oldestMs) { + oldestMs = a->lastMs; + oldestIdx = i; + } + } + + if (n >= DISCOVERY_MAX_KNOWN_PEERS) { + discovery_attempt_t* victim = (discovery_attempt_t*)DynArr_at(disc->connectAttempts, oldestIdx); + victim->addr = *addr; + victim->lastMs = now; + return; + } + + discovery_attempt_t na; + memset(&na, 0, sizeof(na)); + na.addr = *addr; + na.lastMs = now; + DynArr_push_back(disc->connectAttempts, &na); +} + +// Drops the entry for addr, if any. Caller holds disc->lock. +static void Discovery_RemoveUnlocked(node_discovery_t* disc, const struct sockaddr_storage* addr) { + size_t n = DynArr_size(disc->peers); + for (size_t i = 0; i < n; ++i) { + discovered_peer_t* p = (discovered_peer_t*)DynArr_at(disc->peers, i); + if (Discovery_AddrEqual(&p->addr, addr)) { + DynArr_remove(disc->peers, i); + return; + } + } +} + static int Discovery_AddrToWire(const struct sockaddr_storage* addr, unsigned char out[DISCOVERY_WIRE_ENTRY_SIZE]) { memset(out, 0, DISCOVERY_WIRE_ENTRY_SIZE); if (addr->ss_family == AF_INET) { @@ -131,6 +300,7 @@ static int Discovery_WireToAddr(const unsigned char in[DISCOVERY_WIRE_ENTRY_SIZE a->sin6_family = AF_INET6; memcpy(&a->sin6_addr, in + 1, sizeof(struct in6_addr)); a->sin6_port = htons(port); + Discovery_NormaliseAddr(out); // a v4-mapped sender must not become a second entry return port != 0; } return 0; @@ -166,13 +336,31 @@ node_discovery_t* NodeDiscovery_Create(net_node_t* node, udp_node_t* udpNode) { free(disc); return NULL; } + disc->selfEndpoints = DYNARR_CREATE(struct sockaddr_storage, 8); + if (!disc->selfEndpoints) { + DynArr_destroy(disc->peers); + free(disc); + return NULL; + } + disc->connectAttempts = DYNARR_CREATE(discovery_attempt_t, 16); + if (!disc->connectAttempts) { + DynArr_destroy(disc->selfEndpoints); + DynArr_destroy(disc->peers); + free(disc); + return NULL; + } pthread_mutex_init(&disc->lock, NULL); + + // Nothing else is running yet, so the self set can be seeded without taking the lock. + Discovery_SeedSelfEndpoints(disc); return disc; } void NodeDiscovery_Destroy(node_discovery_t* disc) { if (!disc) return; if (disc->peers) DynArr_destroy(disc->peers); + if (disc->selfEndpoints) DynArr_destroy(disc->selfEndpoints); + if (disc->connectAttempts) DynArr_destroy(disc->connectAttempts); pthread_mutex_destroy(&disc->lock); free(disc); } @@ -211,12 +399,14 @@ void NodeDiscovery_OnPingTimeout(node_discovery_t* disc, const struct sockaddr_s void NodeDiscovery_OnGetPeers(node_discovery_t* disc, tcp_connection_t* fromConn) { if (!disc || !fromConn) return; - // Snapshot our current peers' listen endpoints (inbound + outbound). + // Snapshot our current peers' listen endpoints (inbound + outbound) and their identities. struct sockaddr_storage all[MAX_CONS * 2]; - size_t total = Node_GetPeerEndpoints(disc->node, all, sizeof(all) / sizeof(all[0])); + uint64_t allIds[MAX_CONS * 2]; + size_t total = Node_GetPeerEndpoints(disc->node, all, allIds, sizeof(all) / sizeof(all[0])); struct sockaddr_storage reqEndpoint; int haveReq = Node_ConnListenEndpoint(fromConn, &reqEndpoint); + uint64_t reqNodeId = Node_ConnPeerNodeId(fromConn); // Build the response payload: [uint16 count][entries...], capped and sampled for spread. unsigned char payload[sizeof(uint16_t) + DISCOVERY_PEERS_RESPONSE_CAP * DISCOVERY_WIRE_ENTRY_SIZE]; @@ -226,7 +416,11 @@ void NodeDiscovery_OnGetPeers(node_discovery_t* disc, tcp_connection_t* fromConn size_t startIdx = total ? (size_t)(random_four_byte() % total) : 0; for (size_t k = 0; k < total && count < DISCOVERY_PEERS_RESPONSE_CAP; ++k) { size_t idx = (startIdx + k) % total; - if (haveReq && Discovery_AddrEqual(&all[idx], &reqEndpoint)) continue; // don't tell them about themselves + // Don't tell them about themselves. Matching on identity as well as on the endpoint they + // reached us from matters: a multi-homed peer is known to us under several addresses, and + // handing one of its own back to it is what makes it discover, ping and dial itself. + if (haveReq && Discovery_AddrEqual(&all[idx], &reqEndpoint)) continue; + if (reqNodeId != 0 && allIds[idx] == reqNodeId) continue; unsigned char entry[DISCOVERY_WIRE_ENTRY_SIZE]; if (!Discovery_AddrToWire(&all[idx], entry)) continue; memcpy(payload + offset, entry, DISCOVERY_WIRE_ENTRY_SIZE); @@ -292,6 +486,40 @@ void NodeDiscovery_RemovePeer(node_discovery_t* disc, const struct sockaddr_stor pthread_mutex_unlock(&disc->lock); } +void NodeDiscovery_NoteIdentity(node_discovery_t* disc, const struct sockaddr_storage* endpoint, uint64_t nodeId) { + if (!disc || !endpoint || nodeId == 0) return; + + pthread_mutex_lock(&disc->lock); + if (nodeId == localNodeId) { + // The peer on the other end is us under one of our own addresses. Record it and drop it so + // discovery stops treating it as a peer. + Discovery_AddSelfUnlocked(disc, endpoint); + Discovery_RemoveUnlocked(disc, endpoint); + } else { + // Learn the endpoint if we did not already know it - a peer that dialled us is a perfectly + // good discovery candidate, and we now know both its listen endpoint and its identity. + discovered_peer_t* p = Discovery_Upsert(disc, endpoint, 0); + if (p) p->nodeId = nodeId; + } + pthread_mutex_unlock(&disc->lock); +} + +void NodeDiscovery_MarkSelfEndpoint(node_discovery_t* disc, const struct sockaddr_storage* endpoint) { + if (!disc || !endpoint) return; + pthread_mutex_lock(&disc->lock); + Discovery_AddSelfUnlocked(disc, endpoint); + Discovery_RemoveUnlocked(disc, endpoint); + pthread_mutex_unlock(&disc->lock); +} + +int NodeDiscovery_IsSelfEndpoint(node_discovery_t* disc, const struct sockaddr_storage* endpoint) { + if (!disc || !endpoint) return 0; + pthread_mutex_lock(&disc->lock); + int isSelf = Discovery_IsSelfUnlocked(disc, endpoint); + pthread_mutex_unlock(&disc->lock); + return isSelf; +} + // ---- periodic tick ----------------------------------------------------------------------- void NodeDiscovery_Iterate(node_discovery_t* disc) { @@ -305,10 +533,12 @@ void NodeDiscovery_Iterate(node_discovery_t* disc) { Node_GetClientList(disc->node, outConns, &outCount); struct sockaddr_storage outEndpoints[MAX_CONS]; + uint64_t outNodeIds[MAX_CONS]; size_t outEpCount = 0; for (size_t i = 0; i < outCount; ++i) { struct sockaddr_storage ep; if (Node_ConnListenEndpoint(outConns[i], &ep)) { + outNodeIds[outEpCount] = Node_ConnPeerNodeId(outConns[i]); outEndpoints[outEpCount++] = ep; } } @@ -328,6 +558,7 @@ void NodeDiscovery_Iterate(node_discovery_t* disc) { if (p) { p->hop = 0; p->state = DISCOVERY_STATE_CONNECTED; + if (outNodeIds[i] != 0) p->nodeId = outNodeIds[i]; } } // Demote entries still marked CONNECTED that are no longer in the outbound set. @@ -426,17 +657,25 @@ void NodeDiscovery_Iterate(node_discovery_t* disc) { for (size_t i = 0; i < n; ++i) { discovered_peer_t* p = (discovered_peer_t*)DynArr_at(disc->peers, i); if (p->state != DISCOVERY_STATE_REACHABLE) continue; - if (p->lastConnectMs != 0 && (now - p->lastConnectMs) < DISCOVERY_CONNECT_RETRY_MS) continue; + if (!Discovery_ConnectCooledDown(disc, &p->addr, now)) continue; int already = 0; for (size_t j = 0; j < outEpCount; ++j) { if (Discovery_AddrEqual(&p->addr, &outEndpoints[j])) { already = 1; break; } } + // Skip other addresses of a node we already have an outbound connection to. Only + // outbound counts: an inbound connection from a peer is its own dial, and we still + // want one of our own to it (broadcasts only travel outbound). + if (!already && p->nodeId != 0) { + for (size_t j = 0; j < outEpCount; ++j) { + if (outNodeIds[j] == p->nodeId) { already = 1; break; } + } + } if (already) continue; if (!best || p->pingMs < best->pingMs) best = p; } if (!best) break; - best->lastConnectMs = now; // reserve so it isn't picked again this tick + Discovery_NoteConnectAttempt(disc, &best->addr, now); // reserve so it isn't picked again this tick char ip[INET6_ADDRSTRLEN]; unsigned short port = 0; if (Discovery_AddrToIpPort(&best->addr, ip, sizeof(ip), &port) && port != 0) { @@ -479,12 +718,28 @@ void NodeDiscovery_PrintPeers(node_discovery_t* disc) { unsigned short port = 0; Discovery_AddrToIpPort(&p->addr, ip, sizeof(ip), &port); const char* stateStr = (p->state <= DISCOVERY_STATE_UNREACHABLE) ? stateNames[p->state] : "?"; - if (p->pingMs == UINT64_MAX) { - printf(" %-46s hop=%u state=%-11s ping=--\n", ip, p->hop, stateStr); + char idStr[19]; + if (p->nodeId != 0) { + snprintf(idStr, sizeof(idStr), "%016" PRIx64, p->nodeId); } else { - printf(" %-46s hop=%u state=%-11s ping=%" PRIu64 "ms\n", ip, p->hop, stateStr, p->pingMs); + snprintf(idStr, sizeof(idStr), "%-16s", "?"); + } + if (p->pingMs == UINT64_MAX) { + printf(" %-46s hop=%u state=%-11s id=%s ping=--\n", ip, p->hop, stateStr, idStr); + } else { + printf(" %-46s hop=%u state=%-11s id=%s ping=%" PRIu64 "ms\n", ip, p->hop, stateStr, idStr, p->pingMs); } (void)port; // port is part of ip endpoint identity; shown via connect logs } + + size_t selfCount = DynArr_size(disc->selfEndpoints); + printf("Own endpoints (%zu):\n", selfCount); + for (size_t i = 0; i < selfCount; ++i) { + const struct sockaddr_storage* self = (const struct sockaddr_storage*)DynArr_at(disc->selfEndpoints, i); + char ip[INET6_ADDRSTRLEN] = {0}; + unsigned short port = 0; + Discovery_AddrToIpPort(self, ip, sizeof(ip), &port); + printf(" %-46s port=%u\n", ip, port); + } pthread_mutex_unlock(&disc->lock); } diff --git a/src/numgen.c b/src/numgen.c index e8b07a5..6880ab4 100644 --- a/src/numgen.c +++ b/src/numgen.c @@ -1,5 +1,8 @@ #include +#include +#include + unsigned char random_byte(void) { return (unsigned char)(rand() % 256); } @@ -39,3 +42,31 @@ uint64_t random_eight_byte(void) { return x; } + +uint64_t random_secure_eight_byte(void) { + uint64_t x = 0; + + FILE* urandom = fopen("/dev/urandom", "rb"); + if (urandom) { + size_t got = fread(&x, 1, sizeof(x), urandom); + fclose(urandom); + if (got == sizeof(x) && x != 0) { + return x; + } + } + + // Fallback: srand() is seeded from the wall clock in whole seconds, so two nodes launched + // together would draw identical values. Mix in the pid and the sub-second clock to separate them. + struct timespec ts; + if (clock_gettime(CLOCK_REALTIME, &ts) != 0) { + ts.tv_sec = 0; + ts.tv_nsec = 0; + } + + x = random_eight_byte(); + x ^= (uint64_t)ts.tv_nsec; + x ^= ((uint64_t)ts.tv_sec) << 16; + x ^= ((uint64_t)getpid()) << 40; + + return x ? x : 1; // 0 means "no identity advertised" on the wire +}