Add SQLite handler (refdb) and hook into master/workers/job -> commands

CMakeLists.txt

@@ -552,6 +552,15 @@ file(COPY ${_libs} DESTINATION "@CONCORD_PREFIX@/lib")
     set(HAMMY_CONCORD_EXTERNAL concord_external)
 endif()
 
+# ---------------------------------------------------------
+# SQLite3 (hammy's own dependency, unlike the Concord ones above)
+#
+# CMake's bundled FindSQLite3 module (3.14+) exports the SQLite::SQLite3
+# imported target and covers both a system package and a manually-specified
+# SQLITE3_INCLUDE_DIR/SQLITE3_LIBRARY, so no vendoring is needed here.
+# ---------------------------------------------------------
+find_package(SQLite3 REQUIRED)
+
 # ---------------------------------------------------------
 # Output directories
 # ---------------------------------------------------------
@@ -584,7 +593,7 @@ target_include_directories(hammy SYSTEM PRIVATE
 # target's interface includes as -isystem, so upstream's headers never trip our
 # warning set and no second copy under third_party/ is needed.
 
-target_link_libraries(hammy PRIVATE concord::concord)
+target_link_libraries(hammy PRIVATE concord::concord SQLite::SQLite3)
 if(HAMMY_CONCORD_EXTERNAL)
     add_dependencies(hammy ${HAMMY_CONCORD_EXTERNAL})
 endif()
@@ -630,6 +639,7 @@ message(STATUS "hammy: static analyzer [Analyzer]     ${HAMMY_ANALYZER_FLAGS}")
 message(STATUS "hammy: hardening       ${HAMMY_ENABLE_HARDENING}")
 message(STATUS "hammy: LTO             ${HAMMY_ENABLE_LTO}")
 message(STATUS "hammy: concord         ${concord_SOURCE_DIR} (v3.0.1)")
+message(STATUS "hammy: sqlite3         ${SQLite3_LIBRARIES} (v${SQLite3_VERSION})")
 if(CMAKE_BUILD_TYPE STREQUAL "Analyzer" AND NOT HAMMY_ANALYZER_FLAGS)
     message(STATUS "hammy: NOTE - Analyzer config has no static analyzer on "
                    "${CMAKE_C_COMPILER_ID}; use scan-build over this build tree")

include/hammy/bot.h

@@ -7,9 +7,11 @@
 
 #include <hammy/types.h>
 #include <hammy/command.h>
+#include <hammy/refdb.h>
 
 struct hammy_bot_t {
     struct discord* client; // Owning reference to the client - handoff from main.c
+    hammy_refdb_t* refdb; // Owning reference to sqlite - the master and each worker has their own
     bool commandsRegistered; // A flag if commands have been registered. Avoid re-registering every reconnect.
     vector_t* commands; // vector_t of commands. Owning the vector, NOT the elements.
     hammy_pool_t* pool; // Owning reference to the thread pool.

include/hammy/command.h

@@ -6,6 +6,7 @@
 #include <stdbool.h>
 
 #include <hammy/types.h>
+#include <hammy/refdb.h>
 
 // A command handler. Runs either on the gateway thread (instant commands) or on
 // a worker thread (deferred ones), so it must not assume either. client is
@@ -16,7 +17,7 @@
 // or discord_edit_original_interaction_response().
 //
 // The handler does NOT own the job and must not destroy it.
-typedef void (*hammy_command_fn)(const hammy_job_t* job, struct discord* client);
+typedef void (*hammy_command_fn)(const hammy_job_t* job, struct discord* client, const hammy_refdb_t* refdb);
  
 // Plain old data. Every field points at static storage, so this struct owns
 // nothing, copies freely, and needs no destructor - the vector can be created

include/hammy/job.h

@@ -7,6 +7,7 @@
 #include <stdint.h>
 
 #include <hammy/types.h>
+#include <hammy/refdb.h>
 
 // A single slash-command option. Flattened out of the interaction event.
 // Both strings are owned by the job.
@@ -52,7 +53,7 @@ int64_t hammy_job_age_ms(const hammy_job_t* job, struct discord* client);
 // Runs the job to completion and sends the reply. Called from a worker thread,
 // so client MUST be that worker's own clone, never the gateway client.
 // Does not destroy the job.
-void hammy_job_run(hammy_job_t* job, struct discord* client);
+void hammy_job_run(hammy_job_t* job, struct discord* client, hammy_refdb_t* refdb);
  
 // Edits the (already deferred) interaction response with a plain text body.
 // Used by hammy_job_run() and by the pool's error paths.

include/hammy/refdb.h

@@ -0,0 +1,59 @@
+#ifndef HAMMY_REFDB_H
+#define HAMMY_REFDB_H
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <sqlite3.h>
+
+#include <hammy/types.h>
+
+// Read-only handle to the reference bundle. Owned by exactly one thread.
+// Theading rules: Do NOT share a hammy_refdb_t between threads. Instead, a worker should
+// open it's own in hammy_worker_start(), and the gateway should open one for instant
+// commands. The bundle is read-only, so no writers for sync. Seperate sqlite3 connections
+// never contend.
+
+#define HAMMY_CALLSIGN_MAX 32
+#define HAMMY_ENTITY_NAME_MAX 64
+
+struct hammy_refdb_t {
+    sqlite3* handle;
+    sqlite3_stmt* stExact;
+    sqlite3_stmt* stPrefix;
+    char version[64];
+};
+
+struct hammy_dxcc_t {
+    int entityId; // ADIF DXCC entity code
+    char name[HAMMY_ENTITY_NAME_MAX];
+    char continent[4];
+    int cqZone; // Override applied if present
+    int ituZone;
+    double latitude;  // North-positive
+    double longitude; // East-positive
+    double utcOffset; // UTC + utcOffset = local
+    char matchedPrefix[HAMMY_CALLSIGN_MAX];
+    bool exact;
+};
+
+// Opens the bundle read-only and prepares the hot statements.
+// Returns NULL and logs on failure.
+hammy_refdb_t* hammy_refdb_open(const char* path);
+ 
+// Finalises statements and closes the connection. NULLs the passing reference.
+bool hammy_refdb_close(hammy_refdb_t** db);
+
+// Resolves a callsign to a DXCC entity.
+//
+// Uses candidate-prefix equality seeks rather than "? GLOB prefix || '*'". The
+// GLOB form puts the indexed column on the wrong side of the comparison, so
+// SQLite scans all 7000 prefix rows; generating the candidates and seeking by
+// equality is roughly 40x faster on the current bundle.
+//
+// Returns false if nothing matched.
+bool hammy_refdb_dxcc(hammy_refdb_t* db, const char* callsign, hammy_dxcc_t* out);
+ 
+// Bundle version string from ref_meta, or NULL. Owned by the refdb.
+const char* hammy_refdb_version(hammy_refdb_t* db);
+
+#endif

include/hammy/types.h

@@ -6,5 +6,7 @@ typedef struct hammy_job_t hammy_job_t;
 typedef struct hammy_worker_t hammy_worker_t;
 typedef struct hammy_command_t hammy_command_t;
 typedef struct hammy_bot_t hammy_bot_t;
+typedef struct hammy_refdb_t hammy_refdb_t;
+typedef struct hammy_dxcc_t hammy_dxcc_t;
 
 #endif

include/hammy/worker.h

@@ -5,10 +5,13 @@
 #include <hammy/types.h>
 #include <pthread.h>
 
+#include <hammy/refdb.h>
+
 struct hammy_worker_t {
     pthread_t thread;
     struct discord* clientCopy; // Clone of the client for concord threading safety - owning
     hammy_pool_t* pool; // Non-owning back-reference
+    hammy_refdb_t* refdb; // Owning sqlite ref
     int id; // Log logging mainly
     bool started; // For joining
 };

src/commands/ping.c

@@ -13,7 +13,9 @@ static int64_t hammy_snowflake_to_ms(u64snowflake id) {
 }
  
 // Instant command: runs on the gateway thread, sends a fresh response.
-void hammy_cmd_ping(const hammy_job_t* job, struct discord* client) {
+void hammy_cmd_ping(const hammy_job_t* job, struct discord* client, const hammy_refdb_t* refdb) {
+    (void)refdb;
+
     // Gateway heartbeat round trip, as measured by Concord.
     int gatewayMs = discord_get_ping(client);
  

src/hammy/bot.c

@@ -73,7 +73,7 @@ static void hammy_bot_on_interaction(struct discord* client, const struct discor
 
     // Instant path: no defer or queue, answered on the spot
     if (command->instant) {
-        command->handler(job, client);
+        command->handler(job, client, bot->refdb);
         hammy_job_destroy(&job);
         return;
     }
@@ -121,6 +121,12 @@ hammy_bot_t* hammy_bot_create() {
         return NULL;
     }
 
+    // Open a reference to sqlite
+    bot->refdb = hammy_refdb_open("hammy-ref.sqlite"); // TODO: Probably don't hardcode this?
+    if (!bot->refdb) {
+        log_error("[bot] Failed to open ref to sqlite. Commands requiring it will be unavailable!"); // TODO: Consider making this a hard-fail
+    }
+
     return bot;
 }
 
@@ -311,6 +317,10 @@ bool hammy_bot_destroy(hammy_bot_t** bot) {
         b->client = NULL;
     }
 
+    if (b->refdb) {
+        hammy_refdb_close(&b->refdb);
+    }
+
     free(*bot);
     *bot = NULL;
 

src/hammy/command.c

@@ -6,7 +6,7 @@
 // Handlers live in src/hammy/commands/. Declared here rather than in a header
 // so adding a command touches exactly two places: its own .c file and this
 // table.
-void hammy_cmd_ping(const hammy_job_t* job, struct discord* client);
+void hammy_cmd_ping(const hammy_job_t* job, struct discord* client, const hammy_refdb_t* refdb);
 
 // The command table. Pure data - no allocation, no lifetime, no destructor.
 // Copying an entry into the bot's vector is a plain struct copy, and the vector

src/hammy/job.c

@@ -190,8 +190,12 @@ void hammy_job_respond(const hammy_job_t* job, struct discord* client, const cha
     }
 }
 
-void hammy_job_run(hammy_job_t* job, struct discord* client) {
+void hammy_job_run(hammy_job_t* job, struct discord* client, hammy_refdb_t* refdb) {
     if (!job || !client) { return; }
+    if (!refdb) {
+        // TODO: Consider making this a hard-fail
+        log_warn("[job] Ran command '%s' for interaction %" PRIu64 " without a valid refdb! SQLite functions will be unavailable!", job->command ? job->command : "unknown", job->id);
+    }
 
     log_info("[job] Running command '%s' for interaction %" PRIu64, job->command ? job->command : "unknown", job->id);
     const hammy_command_t* command = hammy_bot_find_command(job->bot, job->command);
@@ -201,5 +205,5 @@ void hammy_job_run(hammy_job_t* job, struct discord* client) {
         return;
     }
 
-    command->handler(job, client);
+    command->handler(job, client, refdb);
 }

src/hammy/refdb.c

@@ -0,0 +1,262 @@
+#include <concord/log.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <hammy/refdb.h>
+
+// Whole-callsign rules (cty.dat's "=CALL" entries) must match the entire string.
+static const char* SQL_EXACT =
+    "SELECT e.id, e.name, e.continent,"
+    "       COALESCE(p.cq_zone, e.cq_zone), COALESCE(p.itu_zone, e.itu_zone),"
+    "       e.latitude, e.longitude, e.utc_offset, p.prefix"
+    "  FROM dxcc_prefixes p JOIN dxcc_entities e ON e.id = p.entity_id"
+    " WHERE p.prefix = ?1 AND p.exact = 1"
+    " LIMIT 1";
+ 
+// One candidate at a time, called longest-first by the caller.
+static const char* SQL_PREFIX =
+    "SELECT e.id, e.name, e.continent,"
+    "       COALESCE(p.cq_zone, e.cq_zone), COALESCE(p.itu_zone, e.itu_zone),"
+    "       e.latitude, e.longitude, e.utc_offset, p.prefix"
+    "  FROM dxcc_prefixes p JOIN dxcc_entities e ON e.id = p.entity_id"
+    " WHERE p.prefix = ?1 AND p.exact = 0"
+    " LIMIT 1";
+ 
+static const char* SQL_VERSION =
+    "SELECT value FROM ref_meta WHERE key = 'bundle_version'";
+
+// Authorizer: the connection may read, and nothing else.
+//
+// SQLITE_OPEN_READONLY protects the MAIN database file. It does not stop
+// "ATTACH DATABASE 'somewhere.db' AS x" - an attached database is a separate
+// file with its own flags, so a read-only connection is not automatically a
+// read-only process. SQLITE_LIMIT_ATTACHED below closes that, and this
+// authorizer closes it again, because defence in depth costs nothing here.
+static int hammy_refdb_authorizer(void* unused, int action,
+                                  const char* a1, const char* a2,
+                                  const char* dbname, const char* trigger) {
+    (void)unused; (void)a1; (void)a2; (void)dbname; (void)trigger;
+ 
+    switch (action) {
+        case SQLITE_SELECT:
+        case SQLITE_READ:
+        case SQLITE_FUNCTION:       // COALESCE, length(), upper() and friends
+            return SQLITE_OK;
+ 
+        default:
+            // INSERT, UPDATE, DELETE, ATTACH, DETACH, CREATE/DROP anything,
+            // PRAGMA, transactions - all refused at prepare time.
+            return SQLITE_DENY;
+    }
+}
+
+static bool prepare(sqlite3* h, const char* sql, sqlite3_stmt** out) {
+    if (sqlite3_prepare_v2(h, sql, -1, out, NULL) != SQLITE_OK) {
+        log_error("[refdb] prepare failed: %s", sqlite3_errmsg(h));
+        return false;
+    }
+
+    return true;
+}
+
+hammy_refdb_t* hammy_refdb_open(const char* path) {
+    if (!path) { return NULL; }
+
+    hammy_refdb_t* db = (hammy_refdb_t*)calloc(1, sizeof(*db));
+    if (!db) { return NULL; }
+
+    // immutable=1 promises SQLite the fill won't change while it's open. Skips locks and change-counter checks
+    // Faster and stricter than plain read-only
+    char uri[1024];
+    snprintf(uri, sizeof(uri), "file:%s?mode=ro&immutable=1", path);
+
+    // NOMUTEX - exactly one thread uses this handle, avoid serialization overhead. Safe only because of the one-per-thread rule.
+    int flags = SQLITE_OPEN_READONLY | SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_URI;
+
+    if (sqlite3_open_v2(uri, &db->handle, flags, NULL) != SQLITE_OK) {
+        log_error("[refdb] cannot open %s: %s", path, db->handle ? sqlite3_errmsg(db->handle) : "out of memory");
+        goto fail;
+    }
+
+    // No ATTACH; closes a way a read-only connection could still open writable files.
+    sqlite3_limit(db->handle, SQLITE_LIMIT_ATTACHED, 0);
+
+    // Bound the damage of a pathological query. Turns a runaway into an error instead of a stall. Keeps plenty of headroom for actual lookups.
+    sqlite3_limit(db->handle, SQLITE_LIMIT_SQL_LENGTH, 8192);
+    sqlite3_limit(db->handle, SQLITE_LIMIT_EXPR_DEPTH, 100);
+    sqlite3_limit(db->handle, SQLITE_LIMIT_LIKE_PATTERN_LENGTH, 256);
+    sqlite3_limit(db->handle, SQLITE_LIMIT_VARIABLE_NUMBER, 32);
+
+    // Refuse schema-corrupting tricks (e.g. PRAGMA writable_schema) and stop the schema itself from invoking unvetted functions.
+    sqlite3_db_config(db->handle, SQLITE_DBCONFIG_DEFENSIVE, 1, NULL);
+    sqlite3_db_config(db->handle, SQLITE_DBCONFIG_TRUSTED_SCHEMA, 0, NULL);
+
+    // The bundle is small enough to just map memory-whole. Avoids a read() per page. Set before authorizer since it denies PRAGMA.
+    sqlite3_exec(db->handle, "PRAGMA mmap_size = 67108864;", NULL, NULL, NULL);
+
+    // TODO: PLEASE chmod 444 OR SOMETHING AND DON'T RUN AS THE SQLITE DB FILE OWNER - THAT'S THE ONLY REAL PROTECTION
+
+    sqlite3_set_authorizer(db->handle, &hammy_refdb_authorizer, NULL);
+
+    if (!prepare(db->handle, SQL_EXACT, &db->stExact)) { goto fail; }
+    if (!prepare(db->handle, SQL_PREFIX, &db->stPrefix)) { goto fail; }
+
+    sqlite3_stmt* st = NULL;
+    if (prepare(db->handle, SQL_VERSION, &st)) {
+        if (sqlite3_step(st) == SQLITE_ROW) {
+            const unsigned char* v = sqlite3_column_text(st, 0);
+            if (v) {
+                snprintf(db->version, sizeof(db->version), "%s", (const char*)v);
+            }
+        }
+
+        sqlite3_finalize(st);
+    }
+    
+    return db;
+
+fail:
+    hammy_refdb_close(&db);
+    return NULL;
+}
+
+bool hammy_refdb_close(hammy_refdb_t** db) {
+    if (!db || !(*db)) { return false; }
+
+    hammy_refdb_t* d = *db;
+
+    // Statements must be finalized before the connection closes, or we're in deep shit
+    // (sqlite3_close() returns SQLITE_BUSY and the handle leaks)
+    if (d->stExact) { sqlite3_finalize(d->stExact); }
+    if (d->stPrefix) { sqlite3_finalize(d->stPrefix); }
+    
+    if (d->handle) { sqlite3_close(d->handle); }
+
+    free(d);
+    *db = NULL;
+
+    return true;
+}
+
+const char* hammy_refdb_version(hammy_refdb_t* db) {
+    return (db && db->version[0]) ? db->version : NULL;
+}
+
+// Uppercase, strip whitespace, keep only characters that appear in callsigns (voodoo).
+static void normalise(const char* in, char* out, size_t cap) {
+    size_t j = 0;
+ 
+    for (size_t i = 0; in[i] && j + 1 < cap; i++) {
+        unsigned char c = (unsigned char)in[i];
+ 
+        if (c >= 'a' && c <= 'z') { c = (unsigned char)(c - 'a' + 'A'); }
+ 
+        if ((c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '/') {
+            out[j++] = (char)c;
+        }
+    }
+ 
+    out[j] = '\0';
+}
+
+static void fill(sqlite3_stmt* st, hammy_dxcc_t* out, bool exact) {
+    out->entityId = sqlite3_column_int(st, 0);
+ 
+    const unsigned char* name = sqlite3_column_text(st, 1);
+    const unsigned char* cont = sqlite3_column_text(st, 2);
+    const unsigned char* pfx = sqlite3_column_text(st, 8);
+ 
+    snprintf(out->name, sizeof(out->name), "%s", name ? (const char*)name : "");
+    snprintf(out->continent, sizeof(out->continent), "%s", cont ? (const char*)cont : "");
+    snprintf(out->matchedPrefix, sizeof(out->matchedPrefix), "%s", pfx ? (const char*)pfx : "");
+ 
+    out->cqZone = sqlite3_column_int(st, 3);
+    out->ituZone = sqlite3_column_int(st, 4);
+    out->latitude = sqlite3_column_double(st, 5);
+    out->longitude = sqlite3_column_double(st, 6);
+    out->utcOffset = sqlite3_column_double(st, 7);
+    out->exact = exact;
+}
+
+// Runs one candidate against a prepared statement. Returns true on a hit.
+static bool try_one(sqlite3_stmt* st, const char* candidate, size_t len, hammy_dxcc_t* out, bool exact) {
+    sqlite3_reset(st);
+    sqlite3_clear_bindings(st);
+    sqlite3_bind_text(st, 1, candidate, (int)len, SQLITE_STATIC);
+ 
+    bool hit = (sqlite3_step(st) == SQLITE_ROW);
+    if (hit) { fill(st, out, exact); }
+ 
+    sqlite3_reset(st);
+ 
+    return hit;
+}
+ 
+// Longest-first prefix search over one string.
+static bool search_prefixes(hammy_refdb_t* db, const char* call, hammy_dxcc_t* out) {
+    size_t len = strlen(call);
+ 
+    for (size_t n = len; n > 0; n--) {
+        if (try_one(db->stPrefix, call, n, out, false)) { return true; }
+    }
+ 
+    return false;
+}
+
+bool hammy_refdb_dxcc(hammy_refdb_t* db, const char* callsign, hammy_dxcc_t* out) {
+    if (!db || !callsign || !out) { return false; }
+ 
+    char call[HAMMY_CALLSIGN_MAX];
+    normalise(callsign, call, sizeof(call));
+ 
+    if (!call[0]) { return false; }
+ 
+    memset(out, 0, sizeof(*out));
+ 
+    // 1. Whole-callsign rules win outright. cty.dat carries a couple of thousand
+    //    of these for stations that do not follow their entity's prefix pattern.
+    if (try_one(db->stExact, call, strlen(call), out, true))
+        return true;
+ 
+    // 2. Portable designators. "DL/W1AW" is Germany, "W1AW/4" is still the US.
+    //    The rule of thumb is that the SHORTER side of the slash is the location
+    //    indicator, and a purely numeric tail is a call-area change rather than
+    //    an entity change.
+    //
+    //    This is a heuristic, not a specification - real callsign parsing has
+    //    genuine ambiguities and every logging program handles them slightly
+    //    differently. Good enough for a lookup command; revisit before using it
+    //    to award DXCC credit. TODO
+    const char* slash = strchr(call, '/');
+    if (slash) {
+        char left[HAMMY_CALLSIGN_MAX] = {0};
+        char right[HAMMY_CALLSIGN_MAX] = {0};
+ 
+        size_t llen = (size_t)(slash - call);
+        snprintf(left, sizeof(left), "%.*s", (int)llen, call);
+        snprintf(right, sizeof(right), "%s", slash + 1);
+ 
+        bool right_numeric = true;
+        for (const char* p = right; *p; p++) {
+            if (*p < '0' || *p > '9') {
+                right_numeric = false;
+                break;
+            }
+        }
+ 
+        // A numeric tail keeps the home entity: try the left side only.
+        if (right_numeric && right[0]) { return search_prefixes(db, left, out); }
+ 
+        // Otherwise the shorter side is the location indicator.
+        const char* location = (strlen(right) && strlen(right) < llen) ? right : left;
+        const char* fallback = (location == right) ? left : right;
+ 
+        if (search_prefixes(db, location, out)) { return true; }
+ 
+        return search_prefixes(db, fallback, out);
+    }
+ 
+    // 3. Plain callsign: longest prefix wins.
+    return search_prefixes(db, call, out);
+}
+

src/hammy/worker.c

@@ -55,7 +55,7 @@ static void* hammy_worker_main(void* arg) {
             log_warn("[worker %d] Dropping stale job '%s' (age %lld ms)", worker->id, job->command ? job->command : "unknown", (long long)age);
             hammy_job_reply(job, worker->clientCopy, "Command Timeout", "Sorry, your command took too long to process and was dropped. Please try again.", true);
         } else {
-            hammy_job_run(job, worker->clientCopy);
+            hammy_job_run(job, worker->clientCopy, worker->refdb);
         }
 
         hammy_job_destroy(&job);
@@ -89,6 +89,12 @@ bool hammy_worker_start(hammy_worker_t* worker, hammy_pool_t* pool, struct disco
         return false;
     }
 
+    // Open a reference to sqlite
+    worker->refdb = hammy_refdb_open("hammy-ref.sqlite"); // TODO: Probably don't hardcode this?
+    if (!worker->refdb) {
+        log_error("[worker %d] Failed to open ref to sqlite. Commands requiring it will be unavailable!", id); // TODO: Consider making this a hard-fail
+    }
+
     if (pthread_create(&worker->thread, NULL, &hammy_worker_main, worker) != 0) {
         log_error("[worker %d] Failed to create thread", id);
         hammy_worker_cleanup_clone(worker->clientCopy);
@@ -114,4 +120,8 @@ void hammy_worker_join(hammy_worker_t* worker) {
         hammy_worker_cleanup_clone(worker->clientCopy);
         worker->clientCopy = NULL;
     }
+
+    if (worker->refdb) {
+        hammy_refdb_close(&worker->refdb);
+    }
 }