diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9d47fc6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +/hammyd +/bin/ +config.json +*.env +.env diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4396986 --- /dev/null +++ b/Makefile @@ -0,0 +1,16 @@ +.PHONY: build run gen test tidy + +build: + go build -o bin/hammyd ./cmd/hammyd + +run: + go run ./cmd/hammyd + +gen: + sqlc generate + +test: + go test ./... + +tidy: + go mod tidy \ No newline at end of file diff --git a/cmd/hammyd/main.go b/cmd/hammyd/main.go new file mode 100644 index 0000000..3e6807f --- /dev/null +++ b/cmd/hammyd/main.go @@ -0,0 +1,37 @@ +// cmd/hammyd/main.go +package main + +import ( + "context" + "log" + "os" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func main() { + ctx := context.Background() + + dsn := os.Getenv("HAMMY_DSN") + if dsn == "" { + log.Fatal("HAMMY_DSN is not set!") + } + + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + log.Fatalf("connect: %v", err) + } + defer pool.Close() + + if err := pool.Ping(ctx); err != nil { + log.Fatalf("ping: %v", err) + } + + var n int + err = pool.QueryRow(ctx, "SELECT count(*) FROM core.users").Scan(&n) + if err != nil { + log.Fatalf("query: %v", err) + } + + log.Printf("connected, %d users", n) +} \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..861d173 --- /dev/null +++ b/go.mod @@ -0,0 +1,16 @@ +module git.dcrubro.com/dcrubro/hammy-backend + +go 1.27.0 + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.10.0 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/redis/go-redis/v9 v9.22.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + golang.org/x/sync v0.17.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/text v0.29.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..e86a51d --- /dev/null +++ b/go.sum @@ -0,0 +1,27 @@ +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0= +github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.22.0 h1:laDvpYXTJtZLloinw1fA5Kqd6HAEH2XKxOkG/PDq2F0= +github.com/redis/go-redis/v9 v9.22.0/go.mod h1:y2g0Wj8rQvuK0ELM+oxSudcLtC09JScs98I/X9gRWY4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/migrations/001-Types.sql b/migrations/001-Types.sql new file mode 100644 index 0000000..c9851b9 --- /dev/null +++ b/migrations/001-Types.sql @@ -0,0 +1,102 @@ +-- Hammy backend bootstrap. Run ONCE, as a superuser, before any migration: +-- +-- psql -U postgres -f scripts/bootstrap.sql +-- +-- Deliberately NOT in migrations/. Three reasons: +-- +-- * CREATE DATABASE cannot run inside a transaction block, and migration +-- tools wrap each file in one. +-- * \c is a psql meta-command, not SQL. A migration tool has no idea what it +-- means. +-- * Roles are cluster-wide, not per-database, so they are not part of any one +-- database's schema history. +-- +-- Order matters: the schemas and ALTER DEFAULT PRIVILEGES must exist BEFORE any +-- table is created, because default privileges only apply to objects created +-- afterwards. Get this right here and no migration ever needs a GRANT. + +-- --------------------------------------------------------------------------- +-- Roles (cluster-wide) +-- --------------------------------------------------------------------------- + +-- Owns every object and runs migrations. Never connects at runtime. +CREATE ROLE hammy_owner NOLOGIN; + +-- What the backend connects as. DML only: it can read and write rows but not +-- create, alter or drop anything, so a compromised backend cannot drop the +-- logbook. +CREATE ROLE hammy_app LOGIN PASSWORD 'CHANGE_ME'; + +-- Bulk loaders for FCC ULS, 44net allocations and the like. +CREATE ROLE hammy_ingest LOGIN PASSWORD 'CHANGE_ME'; + +-- Analytics and debugging. Deliberately no access to the api schema - a +-- reporting query has no business reading key material. +CREATE ROLE hammy_readonly LOGIN PASSWORD 'CHANGE_ME'; + +-- --------------------------------------------------------------------------- +-- Database +-- --------------------------------------------------------------------------- + +CREATE DATABASE hammy OWNER hammy_owner ENCODING 'UTF8'; + +\c hammy + +-- --------------------------------------------------------------------------- +-- Schemas +-- --------------------------------------------------------------------------- + +-- Before PostgreSQL 15, PUBLIC held CREATE on the public schema, so any role +-- could add objects to it. Revoke it and do not use public for anything. +REVOKE ALL ON SCHEMA public FROM PUBLIC; + +CREATE SCHEMA core AUTHORIZATION hammy_owner; +CREATE SCHEMA api AUTHORIZATION hammy_owner; +CREATE SCHEMA logbook AUTHORIZATION hammy_owner; +CREATE SCHEMA ingest AUTHORIZATION hammy_owner; + +COMMENT ON SCHEMA core IS 'Discord-linked user identity and callsign verification.'; +COMMENT ON SCHEMA api IS 'API tenancy: owners, keys, usage. Most sensitive schema.'; +COMMENT ON SCHEMA logbook IS 'Per-user QSO logs and import batches.'; +COMMENT ON SCHEMA ingest IS 'Bulk-loaded third-party reference data.'; + +GRANT USAGE ON SCHEMA core, api, logbook, ingest TO hammy_app; +GRANT USAGE ON SCHEMA ingest TO hammy_ingest; +GRANT USAGE ON SCHEMA core, logbook TO hammy_readonly; + +-- --------------------------------------------------------------------------- +-- Default privileges +-- +-- Apply to objects hammy_owner creates FROM NOW ON. Every migration runs as +-- hammy_owner, so tables pick these up automatically. +-- --------------------------------------------------------------------------- + +ALTER DEFAULT PRIVILEGES FOR ROLE hammy_owner IN SCHEMA core, api, logbook + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO hammy_app; + +ALTER DEFAULT PRIVILEGES FOR ROLE hammy_owner IN SCHEMA core, api, logbook + GRANT USAGE ON SEQUENCES TO hammy_app; + +ALTER DEFAULT PRIVILEGES FOR ROLE hammy_owner IN SCHEMA ingest + GRANT SELECT ON TABLES TO hammy_app; + +ALTER DEFAULT PRIVILEGES FOR ROLE hammy_owner IN SCHEMA ingest + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO hammy_ingest; + +ALTER DEFAULT PRIVILEGES FOR ROLE hammy_owner IN SCHEMA core, logbook + GRANT SELECT ON TABLES TO hammy_readonly; + +-- Postgres grants EXECUTE on new functions to PUBLIC by default, which would +-- make forget_user() callable by anyone who can connect. Revoke it; 005 grants +-- it back to hammy_app explicitly. +ALTER DEFAULT PRIVILEGES FOR ROLE hammy_owner IN SCHEMA core, api + REVOKE EXECUTE ON FUNCTIONS FROM PUBLIC; + +-- --------------------------------------------------------------------------- +-- search_path per role, so queries need not schema-qualify everything. +-- --------------------------------------------------------------------------- + +ALTER ROLE hammy_owner SET search_path = core, api, logbook, ingest; +ALTER ROLE hammy_app SET search_path = core, api, logbook, ingest; +ALTER ROLE hammy_ingest SET search_path = ingest; +ALTER ROLE hammy_readonly SET search_path = core, logbook; diff --git a/migrations/002-Core.sql b/migrations/002-Core.sql new file mode 100644 index 0000000..2e6862e --- /dev/null +++ b/migrations/002-Core.sql @@ -0,0 +1,121 @@ +-- core: Discord-linked identity and callsign verification. + +-- --------------------------------------------------------------------------- +-- users +-- --------------------------------------------------------------------------- + +CREATE TABLE core.users ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + + -- Discord snowflakes are unsigned 64-bit, but the timestamp component keeps + -- real values under 2^63 until roughly 2084, so bigint is safe and indexes + -- far better than text. + discord_id bigint NOT NULL UNIQUE, + + -- NULL until the user claims one. Verification is a separate step: a claim + -- is not a verification, and roles must key off verified_at, not this. + callsign core.callsign, + callsign_verified_at timestamptz, + verification_method text + CHECK (verification_method IN ('arrl_email', 'qrz_profile', 'admin', 'lotw')), + + grid core.gridsquare, + timezone text, -- IANA name, e.g. 'Europe/Ljubljana' + + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + + -- A verified callsign must record how and when. + CONSTRAINT users_verification_complete CHECK ( + (callsign_verified_at IS NULL AND verification_method IS NULL) + OR (callsign_verified_at IS NOT NULL AND verification_method IS NOT NULL + AND callsign IS NOT NULL) + ) +); + +COMMENT ON TABLE core.users IS 'One row per Discord user who has interacted with a stateful command. Deliberately no soft-delete column: a deletion request must actually remove the row, so there is nothing left to resurrect.'; + +-- Two users may not both hold a VERIFIED claim on the same callsign. Unverified +-- claims are unconstrained, since anyone can type anything. +CREATE UNIQUE INDEX users_verified_callsign_uniq + ON core.users (callsign) + WHERE callsign_verified_at IS NOT NULL; + +CREATE INDEX users_callsign_idx ON core.users (callsign) WHERE callsign IS NOT NULL; + +CREATE TRIGGER users_touch + BEFORE UPDATE ON core.users + FOR EACH ROW EXECUTE FUNCTION core.touch_updated_at(); + +-- --------------------------------------------------------------------------- +-- verifications +-- --------------------------------------------------------------------------- + +CREATE TABLE core.verifications ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + user_id bigint NOT NULL REFERENCES core.users(id) ON DELETE CASCADE, + + method text NOT NULL + CHECK (method IN ('arrl_email', 'qrz_profile', 'admin', 'lotw')), + callsign core.callsign NOT NULL, + + status text NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending', 'succeeded', 'failed', 'expired', 'cancelled')), + + -- SHA-256 of the challenge token, never the token. Same reasoning as API + -- keys: a database dump must not let anyone complete a verification. + challenge_hash bytea, + + -- Diagnostics only. Must NOT contain the email address or any other + -- identifier - the whole point of hashing the challenge is undone if the + -- address is sitting in a jsonb column next to it. + detail jsonb NOT NULL DEFAULT '{}'::jsonb, + + created_at timestamptz NOT NULL DEFAULT now(), + expires_at timestamptz, + completed_at timestamptz +); + +CREATE INDEX verifications_user_idx ON core.verifications (user_id, created_at DESC); + +CREATE INDEX verifications_pending_idx + ON core.verifications (expires_at) + WHERE status = 'pending'; + +-- --------------------------------------------------------------------------- +-- subscriptions +-- --------------------------------------------------------------------------- + +-- Discord-shaped, but persistent and worth backing up, so it belongs here +-- rather than in Redis. +CREATE TABLE core.subscriptions ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + + guild_id bigint, -- NULL for a DM subscription + channel_id bigint NOT NULL, + user_id bigint REFERENCES core.users(id) ON DELETE CASCADE, + + kind text NOT NULL + CHECK (kind IN ('pota', 'sota', 'wwff', 'cluster', 'rbn', 'propagation', 'net')), + + -- Band, mode, region, distance, specific refs. Open-ended by nature, and + -- the filter shape differs per kind, so jsonb rather than twenty columns + -- that are NULL for most rows. + filters jsonb NOT NULL DEFAULT '{}'::jsonb, + + enabled boolean NOT NULL DEFAULT true, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX subscriptions_channel_idx ON core.subscriptions (channel_id) WHERE enabled; +CREATE INDEX subscriptions_kind_idx ON core.subscriptions (kind) WHERE enabled; +CREATE INDEX subscriptions_filters_idx ON core.subscriptions USING gin (filters); + +-- Postgres does not index the referencing side of a foreign key. Without this, +-- deleting a user sequentially scans subscriptions to enforce the cascade. +CREATE INDEX subscriptions_user_idx ON core.subscriptions (user_id) WHERE user_id IS NOT NULL; + +CREATE TRIGGER subscriptions_touch + BEFORE UPDATE ON core.subscriptions + FOR EACH ROW EXECUTE FUNCTION core.touch_updated_at(); diff --git a/migrations/003-API.sql b/migrations/003-API.sql new file mode 100644 index 0000000..01dac16 --- /dev/null +++ b/migrations/003-API.sql @@ -0,0 +1,134 @@ +-- api: shall-issue tenancy. Owners, keys, usage. +-- +-- Most sensitive schema in the database. hammy_readonly is deliberately NOT +-- granted USAGE on it - a reporting query has no business reading key material. + +-- --------------------------------------------------------------------------- +-- owners +-- --------------------------------------------------------------------------- + +CREATE TABLE api.owners ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + + -- The abuse contact, and the only field that is genuinely required. It is + -- what makes shall-issue workable: a channel to say "your key is doing + -- something odd, fix it or I revoke". + email core.email NOT NULL UNIQUE, + email_verified_at timestamptz, + + -- Optional, and a scope multiplier rather than a gate. An unlicensed + -- developer building a study tool against the exam endpoints has no + -- callsign and no reason to get one yet. + callsign core.callsign, + callsign_verified_at timestamptz, + + status text NOT NULL DEFAULT 'active' + CHECK (status IN ('active', 'suspended', 'closed')), + + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +COMMENT ON TABLE api.owners IS 'Registrants for API keys. Holds PII from an international user base, so the deletion path in 005_privacy.sql applies here too, not just to Discord users.'; + +CREATE UNIQUE INDEX owners_verified_callsign_uniq + ON api.owners (callsign) + WHERE callsign_verified_at IS NOT NULL; + +CREATE TRIGGER owners_touch + BEFORE UPDATE ON api.owners + FOR EACH ROW EXECUTE FUNCTION core.touch_updated_at(); + +-- --------------------------------------------------------------------------- +-- keys +-- --------------------------------------------------------------------------- + +CREATE TABLE api.keys ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + owner_id bigint NOT NULL REFERENCES api.owners(id) ON DELETE CASCADE, + + -- SHA-256 of the key, computed in the application. The key itself is shown + -- once at issue and never stored: a database dump must not be a set of + -- working credentials. Hashing in the app rather than via pgcrypto keeps + -- the plaintext out of the query log. + key_hash bytea NOT NULL UNIQUE, + CONSTRAINT keys_hash_is_sha256 CHECK (octet_length(key_hash) = 32), + + -- Enough of the key to identify it in a UI and in a leak report, e.g. + -- 'hmy_live_a1b2c3d4'. Not secret and not sufficient to authenticate. + key_prefix text NOT NULL, + label text, + + -- Tier 0 reference data can be near-unlimited; callsign lookup and 44net + -- probing need much tighter ceilings. Scoping from day one means a scope + -- can be tightened later without breaking every key. + scopes text[] NOT NULL DEFAULT '{}', + CONSTRAINT keys_scopes_known CHECK ( + scopes <@ ARRAY[ + 'reference', -- band plans, prefixes, calculators + 'callsign', -- callsign lookup, rate limited + 'spots', -- POTA/SOTA/cluster/RBN feeds + 'propagation', + 'logbook', -- read/write a user's own log + 'net44', -- reachability probes from the 44net allocation + 'admin' + ]::text[] + ), + + quota_tier text NOT NULL DEFAULT 'default' + CHECK (quota_tier IN ('default', 'verified', 'trusted', 'internal')), + + status text NOT NULL DEFAULT 'active' + CHECK (status IN ('active', 'revoked', 'suspended')), + + created_at timestamptz NOT NULL DEFAULT now(), + expires_at timestamptz, + last_used_at timestamptz, + revoked_at timestamptz, + revoked_reason text, + + CONSTRAINT keys_revoked_consistent CHECK ( + (status = 'revoked') = (revoked_at IS NOT NULL) + ) +); + +COMMENT ON TABLE api.keys IS 'Several active keys per owner is intentional: people leak keys into public repositories, and rotating needs an overlap window where both the old and new key work.'; + +CREATE INDEX keys_owner_idx ON api.keys (owner_id); + +-- No separate index on key_hash: the UNIQUE constraint above already builds a +-- btree over exactly that column, and the planner uses it for the lookup. + +CREATE INDEX keys_expiring_idx + ON api.keys (expires_at) + WHERE status = 'active' AND expires_at IS NOT NULL; + +-- --------------------------------------------------------------------------- +-- usage +-- --------------------------------------------------------------------------- + +-- Rolled up from Redis, which is authoritative for the live rate-limit window. +-- Redis holds the current counters; this holds completed days. A Redis flush +-- loses at most the window in progress, which for abuse detection is fine. +CREATE TABLE api.usage_daily ( + key_id bigint NOT NULL REFERENCES api.keys(id) ON DELETE CASCADE, + day date NOT NULL, + endpoint text NOT NULL, + calls bigint NOT NULL DEFAULT 0, + errors bigint NOT NULL DEFAULT 0, + + -- Distinct end users seen, not who they were. Counting them is the + -- enumeration signal - one Discord user driving 90% of a key's traffic + -- looks very different from fifty users sharing it - but the identities + -- themselves are not needed and so are not kept. + distinct_users integer NOT NULL DEFAULT 0, + + PRIMARY KEY (key_id, day, endpoint) +); + +-- The flush job is an upsert: +-- INSERT INTO api.usage_daily (key_id, day, endpoint, calls) +-- VALUES ($1, $2, $3, $4) +-- ON CONFLICT (key_id, day, endpoint) +-- DO UPDATE SET calls = api.usage_daily.calls + EXCLUDED.calls; +CREATE INDEX usage_daily_day_idx ON api.usage_daily (day); diff --git a/migrations/004-Logbook.sql b/migrations/004-Logbook.sql new file mode 100644 index 0000000..897ed4f --- /dev/null +++ b/migrations/004-Logbook.sql @@ -0,0 +1,121 @@ +-- logbook: per-user QSO logs and import batches. + +-- --------------------------------------------------------------------------- +-- imports +-- --------------------------------------------------------------------------- + +-- Every QSO belongs to an import batch, so a bad ADIF file can be rolled back +-- as a unit rather than picked apart row by row afterwards. +CREATE TABLE logbook.imports ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + user_id bigint NOT NULL REFERENCES core.users(id) ON DELETE CASCADE, + + filename text, + source text, -- 'adif', 'lotw', 'clublog', 'manual' + + status text NOT NULL DEFAULT 'running' + CHECK (status IN ('running', 'completed', 'failed', 'rolled_back')), + + rows_total integer NOT NULL DEFAULT 0, + rows_accepted integer NOT NULL DEFAULT 0, + rows_duplicate integer NOT NULL DEFAULT 0, + rows_rejected integer NOT NULL DEFAULT 0, + + error text, + + started_at timestamptz NOT NULL DEFAULT now(), + finished_at timestamptz +); + +CREATE INDEX imports_user_idx ON logbook.imports (user_id, started_at DESC); + +-- --------------------------------------------------------------------------- +-- qsos +-- --------------------------------------------------------------------------- + +CREATE TABLE logbook.qsos ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + user_id bigint NOT NULL REFERENCES core.users(id) ON DELETE CASCADE, + + -- SET NULL rather than CASCADE: deleting an import record should not delete + -- the QSOs it brought in, only the audit trail of how they arrived. + import_id bigint REFERENCES logbook.imports(id) ON DELETE SET NULL, + + callsign core.callsign NOT NULL, + + -- ADIF gives QSO_DATE and TIME_ON separately, both UTC. Combined here. + qso_at timestamptz NOT NULL, + + -- Dedup key. Generated columns must be IMMUTABLE, and date_trunc over a + -- timestamptz is only STABLE because it depends on the TimeZone setting. + -- timezone('UTC', tstz) -> timestamp IS immutable, and date_trunc over a + -- plain timestamp is immutable, so the chain works. + -- + -- Minute granularity because two operators logging the same contact + -- routinely differ by a few seconds. + qso_minute timestamp GENERATED ALWAYS AS + (date_trunc('minute', timezone('UTC', qso_at))) STORED, + + band text, + mode text, + submode text, + freq_hz bigint, + + rst_sent text, + rst_rcvd text, + + grid core.gridsquare, + dxcc integer, -- ADIF entity code, resolved at import + cq_zone smallint, + itu_zone smallint, + + tx_power_w numeric(8,2), + comment text, + + -- ADIF defines a few hundred fields and every logging program invents more. + -- Normalising the columns above and keeping the rest here is the main + -- reason for choosing Postgres: MariaDB's JSON is LONGTEXT with validation, + -- with no binary storage and no functional index on a path. + extra jsonb NOT NULL DEFAULT '{}'::jsonb, + + created_at timestamptz NOT NULL DEFAULT now(), + + CONSTRAINT qsos_freq_sane CHECK (freq_hz IS NULL OR (freq_hz > 0 AND freq_hz < 300000000000)), + CONSTRAINT qsos_zones_sane CHECK ( + (cq_zone IS NULL OR cq_zone BETWEEN 1 AND 40) + AND (itu_zone IS NULL OR itu_zone BETWEEN 1 AND 90) + ) +); + +COMMENT ON COLUMN logbook.qsos.qso_minute IS 'Truncated copy of qso_at used only for duplicate detection. Do not query it directly; use qso_at.'; + +-- Duplicate detection. A repeat import of the same ADIF must not double the +-- log, and this is what makes ON CONFLICT DO NOTHING work on import. +-- +-- NULLS NOT DISTINCT matters: band and mode are nullable, and in a unique index +-- NULLs are distinct by default. Without it, two identical QSOs that both have +-- band IS NULL do not collide, so an ADIF with no BAND field imports twice, and +-- again on every re-import. Requires PostgreSQL 15+. +CREATE UNIQUE INDEX qsos_dedup_uniq + ON logbook.qsos (user_id, callsign, band, mode, qso_minute) + NULLS NOT DISTINCT; + +-- The four access patterns that actually happen. +CREATE INDEX qsos_user_time_idx ON logbook.qsos (user_id, qso_at DESC); +CREATE INDEX qsos_user_call_idx ON logbook.qsos (user_id, callsign); +CREATE INDEX qsos_user_band_mode_idx ON logbook.qsos (user_id, band, mode); + +-- Award progress is DERIVED, never stored. DXCC and WAS counts are aggregates +-- over this table; a stored counter would need maintaining on every insert, +-- edit and rollback, and would drift. This index is what makes that cheap. +CREATE INDEX qsos_user_dxcc_idx ON logbook.qsos (user_id, dxcc) WHERE dxcc IS NOT NULL; + +CREATE INDEX qsos_extra_idx ON logbook.qsos USING gin (extra); + +-- Referencing side of the import FK. Without it, rolling back one import +-- sequentially scans the whole qsos table to apply ON DELETE SET NULL. +CREATE INDEX qsos_import_idx ON logbook.qsos (import_id) WHERE import_id IS NOT NULL; + +-- Worth revisiting if a single user's log passes a few million rows: partition +-- by user_id hash, or by qso_at range. Not now - the indexes above carry a +-- normal log comfortably, and partitioning early costs flexibility. diff --git a/migrations/004-Privacy.sql b/migrations/004-Privacy.sql new file mode 100644 index 0000000..a833445 --- /dev/null +++ b/migrations/004-Privacy.sql @@ -0,0 +1,102 @@ +-- Deletion paths. +-- +-- Discord's developer policy requires deleting user data on request, and API +-- registrants are an international user base whose data protection expectations +-- apply regardless. Build this before you need it: retrofitting a deletion path +-- across three stores is miserable, and being asked for one you do not have is +-- worse. +-- +-- These functions cover Postgres. The caller must ALSO clear Redis - cached +-- callsign lookups, rate-limit counters keyed by hashed Discord ID, and any +-- verification challenge in flight. Postgres is not the whole story. + +-- --------------------------------------------------------------------------- + +-- Removes a Discord user and everything referencing them. The ON DELETE CASCADE +-- chain reaches verifications, subscriptions, imports and qsos, so this is one +-- statement rather than a list that can fall out of date as tables are added. +-- +-- Returns the number of QSOs removed, purely so the caller can tell the user +-- what was deleted. +CREATE FUNCTION core.forget_user(p_discord_id bigint) +RETURNS TABLE (deleted_user boolean, deleted_qsos bigint) +LANGUAGE plpgsql +-- Pinned: the body qualifies its tables, but now(), count() and friends resolve +-- through search_path. Matters more if this ever becomes SECURITY DEFINER. +SET search_path = pg_catalog, core, logbook +AS $$ +DECLARE + v_user_id bigint; + v_qsos bigint := 0; +BEGIN + SELECT id INTO v_user_id FROM core.users WHERE discord_id = p_discord_id; + + IF v_user_id IS NULL THEN + RETURN QUERY SELECT false, 0::bigint; + RETURN; + END IF; + + SELECT count(*) INTO v_qsos FROM logbook.qsos WHERE user_id = v_user_id; + + -- Cascades do the rest. + DELETE FROM core.users WHERE id = v_user_id; + + RETURN QUERY SELECT true, v_qsos; +END; +$$; + +COMMENT ON FUNCTION core.forget_user(bigint) IS 'Hard-deletes a Discord user and all dependent rows. Caller must also purge Redis keys for this user.'; + +-- --------------------------------------------------------------------------- + +-- The API-side equivalent. Keys and usage rows cascade from the owner. +CREATE FUNCTION api.forget_owner(p_email core.email) +RETURNS TABLE (deleted_owner boolean, deleted_keys bigint) +LANGUAGE plpgsql +SET search_path = pg_catalog, core, api +AS $$ +DECLARE + v_owner_id bigint; + v_keys bigint := 0; +BEGIN + SELECT id INTO v_owner_id FROM api.owners WHERE email = p_email; + + IF v_owner_id IS NULL THEN + RETURN QUERY SELECT false, 0::bigint; + RETURN; + END IF; + + SELECT count(*) INTO v_keys FROM api.keys WHERE owner_id = v_owner_id; + + DELETE FROM api.owners WHERE id = v_owner_id; + + RETURN QUERY SELECT true, v_keys; +END; +$$; + +-- --------------------------------------------------------------------------- + +-- Retention. Verification records carry a hashed challenge and a timestamp, and +-- are useful for a short while when investigating a failed verification. They +-- are not useful a year later. Run this from cron. +CREATE FUNCTION core.prune_verifications(p_keep_days integer DEFAULT 90) +RETURNS bigint +LANGUAGE plpgsql +SET search_path = pg_catalog, core +AS $$ +DECLARE + v_removed bigint; +BEGIN + DELETE FROM core.verifications + WHERE created_at < now() - make_interval(days => p_keep_days) + AND status <> 'succeeded'; + + GET DIAGNOSTICS v_removed = ROW_COUNT; + + RETURN v_removed; +END; +$$; + +GRANT EXECUTE ON FUNCTION core.forget_user(bigint) TO hammy_app; +GRANT EXECUTE ON FUNCTION api.forget_owner(core.email) TO hammy_app; +GRANT EXECUTE ON FUNCTION core.prune_verifications(integer) TO hammy_app; diff --git a/migrations/README.md b/migrations/README.md new file mode 100644 index 0000000..5ab3026 --- /dev/null +++ b/migrations/README.md @@ -0,0 +1,58 @@ +# Database setup + +## Once, as a superuser + + psql -U postgres -f scripts/bootstrap.sql + +Creates the roles, the `hammy` database, the four schemas, and the default +privileges. Change the three `CHANGE_ME` passwords first. + +## Migrations, as hammy_owner + + for f in migrations/*.sql; do + psql -U hammy_owner -d hammy -v ON_ERROR_STOP=1 -f "$f" + done + +Running these **as hammy_owner is not optional**. `ALTER DEFAULT PRIVILEGES FOR +ROLE hammy_owner` only fires for objects that role creates; run them as a +superuser and the tables end up owned by the superuser with no grants, so +`hammy_app` sees four empty schemas. + +`hammy_owner` is `NOLOGIN`, so connect as a superuser and switch: + + psql -U postgres -d hammy -c 'SET ROLE hammy_owner' -f migrations/001_types.sql + +or give it `LOGIN` for the duration of the setup and revoke it after. + +Verify afterwards: + + SELECT relname, relacl FROM pg_class + WHERE relnamespace = 'core'::regnamespace AND relkind = 'r'; + +Every row should show `hammy_app=arwd/hammy_owner`. All NULL means the +migrations ran as the wrong role. + +## Files + +| File | Contents | +|---|---| +| `scripts/bootstrap.sql` | Roles, database, schemas, default privileges. Superuser, once. | +| `migrations/001_types.sql` | `touch_updated_at()`, the `callsign`/`email`/`gridsquare` domains. | +| `migrations/002_core.sql` | `users`, `verifications`, `subscriptions`. | +| `migrations/003_api.sql` | `owners`, `keys`, `usage_daily`. | +| `migrations/004_logbook.sql` | `imports`, `qsos`. | +| `migrations/005_privacy.sql` | `forget_user()`, `forget_owner()`, `prune_verifications()`. | + +## sqlc + +`sqlc.yaml` points only at 001-004. It parses SQL to learn the schema and will +choke on `CREATE ROLE`, `ALTER DEFAULT PRIVILEGES` and `DO $$` blocks, which is +another reason those live in `scripts/` rather than `migrations/`. + +## Already have the old numbering? + +The previous layout was `001_bootstrap` through `006_fixes`, applied by hand. If +that database exists and has no data worth keeping, `DROP DATABASE hammy` plus +`DROP ROLE` the four roles, then start from `bootstrap.sql`. The corrections +from the old `006_fixes.sql` are folded into 002-005 here, so there is no +separate fixes file any more. diff --git a/migrations/bootstrap.sql b/migrations/bootstrap.sql new file mode 100644 index 0000000..c9851b9 --- /dev/null +++ b/migrations/bootstrap.sql @@ -0,0 +1,102 @@ +-- Hammy backend bootstrap. Run ONCE, as a superuser, before any migration: +-- +-- psql -U postgres -f scripts/bootstrap.sql +-- +-- Deliberately NOT in migrations/. Three reasons: +-- +-- * CREATE DATABASE cannot run inside a transaction block, and migration +-- tools wrap each file in one. +-- * \c is a psql meta-command, not SQL. A migration tool has no idea what it +-- means. +-- * Roles are cluster-wide, not per-database, so they are not part of any one +-- database's schema history. +-- +-- Order matters: the schemas and ALTER DEFAULT PRIVILEGES must exist BEFORE any +-- table is created, because default privileges only apply to objects created +-- afterwards. Get this right here and no migration ever needs a GRANT. + +-- --------------------------------------------------------------------------- +-- Roles (cluster-wide) +-- --------------------------------------------------------------------------- + +-- Owns every object and runs migrations. Never connects at runtime. +CREATE ROLE hammy_owner NOLOGIN; + +-- What the backend connects as. DML only: it can read and write rows but not +-- create, alter or drop anything, so a compromised backend cannot drop the +-- logbook. +CREATE ROLE hammy_app LOGIN PASSWORD 'CHANGE_ME'; + +-- Bulk loaders for FCC ULS, 44net allocations and the like. +CREATE ROLE hammy_ingest LOGIN PASSWORD 'CHANGE_ME'; + +-- Analytics and debugging. Deliberately no access to the api schema - a +-- reporting query has no business reading key material. +CREATE ROLE hammy_readonly LOGIN PASSWORD 'CHANGE_ME'; + +-- --------------------------------------------------------------------------- +-- Database +-- --------------------------------------------------------------------------- + +CREATE DATABASE hammy OWNER hammy_owner ENCODING 'UTF8'; + +\c hammy + +-- --------------------------------------------------------------------------- +-- Schemas +-- --------------------------------------------------------------------------- + +-- Before PostgreSQL 15, PUBLIC held CREATE on the public schema, so any role +-- could add objects to it. Revoke it and do not use public for anything. +REVOKE ALL ON SCHEMA public FROM PUBLIC; + +CREATE SCHEMA core AUTHORIZATION hammy_owner; +CREATE SCHEMA api AUTHORIZATION hammy_owner; +CREATE SCHEMA logbook AUTHORIZATION hammy_owner; +CREATE SCHEMA ingest AUTHORIZATION hammy_owner; + +COMMENT ON SCHEMA core IS 'Discord-linked user identity and callsign verification.'; +COMMENT ON SCHEMA api IS 'API tenancy: owners, keys, usage. Most sensitive schema.'; +COMMENT ON SCHEMA logbook IS 'Per-user QSO logs and import batches.'; +COMMENT ON SCHEMA ingest IS 'Bulk-loaded third-party reference data.'; + +GRANT USAGE ON SCHEMA core, api, logbook, ingest TO hammy_app; +GRANT USAGE ON SCHEMA ingest TO hammy_ingest; +GRANT USAGE ON SCHEMA core, logbook TO hammy_readonly; + +-- --------------------------------------------------------------------------- +-- Default privileges +-- +-- Apply to objects hammy_owner creates FROM NOW ON. Every migration runs as +-- hammy_owner, so tables pick these up automatically. +-- --------------------------------------------------------------------------- + +ALTER DEFAULT PRIVILEGES FOR ROLE hammy_owner IN SCHEMA core, api, logbook + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO hammy_app; + +ALTER DEFAULT PRIVILEGES FOR ROLE hammy_owner IN SCHEMA core, api, logbook + GRANT USAGE ON SEQUENCES TO hammy_app; + +ALTER DEFAULT PRIVILEGES FOR ROLE hammy_owner IN SCHEMA ingest + GRANT SELECT ON TABLES TO hammy_app; + +ALTER DEFAULT PRIVILEGES FOR ROLE hammy_owner IN SCHEMA ingest + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO hammy_ingest; + +ALTER DEFAULT PRIVILEGES FOR ROLE hammy_owner IN SCHEMA core, logbook + GRANT SELECT ON TABLES TO hammy_readonly; + +-- Postgres grants EXECUTE on new functions to PUBLIC by default, which would +-- make forget_user() callable by anyone who can connect. Revoke it; 005 grants +-- it back to hammy_app explicitly. +ALTER DEFAULT PRIVILEGES FOR ROLE hammy_owner IN SCHEMA core, api + REVOKE EXECUTE ON FUNCTIONS FROM PUBLIC; + +-- --------------------------------------------------------------------------- +-- search_path per role, so queries need not schema-qualify everything. +-- --------------------------------------------------------------------------- + +ALTER ROLE hammy_owner SET search_path = core, api, logbook, ingest; +ALTER ROLE hammy_app SET search_path = core, api, logbook, ingest; +ALTER ROLE hammy_ingest SET search_path = ingest; +ALTER ROLE hammy_readonly SET search_path = core, logbook; diff --git a/queries/keys.sql b/queries/keys.sql new file mode 100644 index 0000000..1ed6d5f --- /dev/null +++ b/queries/keys.sql @@ -0,0 +1,6 @@ +-- name: GetKeyByHash :one +SELECT k.id, k.owner_id, k.scopes, k.quota_tier, k.status, k.expires_at, + o.status AS owner_status +FROM api.keys k +JOIN api.owners o ON o.id = k.owner_id +WHERE k.key_hash = $1 AND k.status = 'active'; diff --git a/sqlc.yaml b/sqlc.yaml new file mode 100644 index 0000000..7ed1be5 --- /dev/null +++ b/sqlc.yaml @@ -0,0 +1,16 @@ +# sqlc.yaml +version: "2" +sql: + - engine: "postgresql" + queries: "queries" + schema: + - "migrations/002_types.sql" + - "migrations/003_core.sql" + - "migrations/004_api.sql" + - "migrations/005_logbook.sql" + gen: + go: + package: "db" + out: "internal/db" + sql_package: "pgx/v5" + emit_pointers_for_null_types: true