initial, copy postgres stuff for migrations, main.go test
This commit is contained in:
@@ -0,0 +1,102 @@
|
||||
-- Hammy backend bootstrap. Run ONCE, as a superuser, before any migration:
|
||||
--
|
||||
-- psql -U postgres -f scripts/bootstrap.sql
|
||||
--
|
||||
-- Deliberately NOT in migrations/. Three reasons:
|
||||
--
|
||||
-- * CREATE DATABASE cannot run inside a transaction block, and migration
|
||||
-- tools wrap each file in one.
|
||||
-- * \c is a psql meta-command, not SQL. A migration tool has no idea what it
|
||||
-- means.
|
||||
-- * Roles are cluster-wide, not per-database, so they are not part of any one
|
||||
-- database's schema history.
|
||||
--
|
||||
-- Order matters: the schemas and ALTER DEFAULT PRIVILEGES must exist BEFORE any
|
||||
-- table is created, because default privileges only apply to objects created
|
||||
-- afterwards. Get this right here and no migration ever needs a GRANT.
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Roles (cluster-wide)
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
-- Owns every object and runs migrations. Never connects at runtime.
|
||||
CREATE ROLE hammy_owner NOLOGIN;
|
||||
|
||||
-- What the backend connects as. DML only: it can read and write rows but not
|
||||
-- create, alter or drop anything, so a compromised backend cannot drop the
|
||||
-- logbook.
|
||||
CREATE ROLE hammy_app LOGIN PASSWORD 'CHANGE_ME';
|
||||
|
||||
-- Bulk loaders for FCC ULS, 44net allocations and the like.
|
||||
CREATE ROLE hammy_ingest LOGIN PASSWORD 'CHANGE_ME';
|
||||
|
||||
-- Analytics and debugging. Deliberately no access to the api schema - a
|
||||
-- reporting query has no business reading key material.
|
||||
CREATE ROLE hammy_readonly LOGIN PASSWORD 'CHANGE_ME';
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Database
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE DATABASE hammy OWNER hammy_owner ENCODING 'UTF8';
|
||||
|
||||
\c hammy
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Schemas
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
-- Before PostgreSQL 15, PUBLIC held CREATE on the public schema, so any role
|
||||
-- could add objects to it. Revoke it and do not use public for anything.
|
||||
REVOKE ALL ON SCHEMA public FROM PUBLIC;
|
||||
|
||||
CREATE SCHEMA core AUTHORIZATION hammy_owner;
|
||||
CREATE SCHEMA api AUTHORIZATION hammy_owner;
|
||||
CREATE SCHEMA logbook AUTHORIZATION hammy_owner;
|
||||
CREATE SCHEMA ingest AUTHORIZATION hammy_owner;
|
||||
|
||||
COMMENT ON SCHEMA core IS 'Discord-linked user identity and callsign verification.';
|
||||
COMMENT ON SCHEMA api IS 'API tenancy: owners, keys, usage. Most sensitive schema.';
|
||||
COMMENT ON SCHEMA logbook IS 'Per-user QSO logs and import batches.';
|
||||
COMMENT ON SCHEMA ingest IS 'Bulk-loaded third-party reference data.';
|
||||
|
||||
GRANT USAGE ON SCHEMA core, api, logbook, ingest TO hammy_app;
|
||||
GRANT USAGE ON SCHEMA ingest TO hammy_ingest;
|
||||
GRANT USAGE ON SCHEMA core, logbook TO hammy_readonly;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Default privileges
|
||||
--
|
||||
-- Apply to objects hammy_owner creates FROM NOW ON. Every migration runs as
|
||||
-- hammy_owner, so tables pick these up automatically.
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
ALTER DEFAULT PRIVILEGES FOR ROLE hammy_owner IN SCHEMA core, api, logbook
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO hammy_app;
|
||||
|
||||
ALTER DEFAULT PRIVILEGES FOR ROLE hammy_owner IN SCHEMA core, api, logbook
|
||||
GRANT USAGE ON SEQUENCES TO hammy_app;
|
||||
|
||||
ALTER DEFAULT PRIVILEGES FOR ROLE hammy_owner IN SCHEMA ingest
|
||||
GRANT SELECT ON TABLES TO hammy_app;
|
||||
|
||||
ALTER DEFAULT PRIVILEGES FOR ROLE hammy_owner IN SCHEMA ingest
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO hammy_ingest;
|
||||
|
||||
ALTER DEFAULT PRIVILEGES FOR ROLE hammy_owner IN SCHEMA core, logbook
|
||||
GRANT SELECT ON TABLES TO hammy_readonly;
|
||||
|
||||
-- Postgres grants EXECUTE on new functions to PUBLIC by default, which would
|
||||
-- make forget_user() callable by anyone who can connect. Revoke it; 005 grants
|
||||
-- it back to hammy_app explicitly.
|
||||
ALTER DEFAULT PRIVILEGES FOR ROLE hammy_owner IN SCHEMA core, api
|
||||
REVOKE EXECUTE ON FUNCTIONS FROM PUBLIC;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- search_path per role, so queries need not schema-qualify everything.
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
ALTER ROLE hammy_owner SET search_path = core, api, logbook, ingest;
|
||||
ALTER ROLE hammy_app SET search_path = core, api, logbook, ingest;
|
||||
ALTER ROLE hammy_ingest SET search_path = ingest;
|
||||
ALTER ROLE hammy_readonly SET search_path = core, logbook;
|
||||
@@ -0,0 +1,121 @@
|
||||
-- core: Discord-linked identity and callsign verification.
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- users
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE TABLE core.users (
|
||||
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
|
||||
-- Discord snowflakes are unsigned 64-bit, but the timestamp component keeps
|
||||
-- real values under 2^63 until roughly 2084, so bigint is safe and indexes
|
||||
-- far better than text.
|
||||
discord_id bigint NOT NULL UNIQUE,
|
||||
|
||||
-- NULL until the user claims one. Verification is a separate step: a claim
|
||||
-- is not a verification, and roles must key off verified_at, not this.
|
||||
callsign core.callsign,
|
||||
callsign_verified_at timestamptz,
|
||||
verification_method text
|
||||
CHECK (verification_method IN ('arrl_email', 'qrz_profile', 'admin', 'lotw')),
|
||||
|
||||
grid core.gridsquare,
|
||||
timezone text, -- IANA name, e.g. 'Europe/Ljubljana'
|
||||
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
|
||||
-- A verified callsign must record how and when.
|
||||
CONSTRAINT users_verification_complete CHECK (
|
||||
(callsign_verified_at IS NULL AND verification_method IS NULL)
|
||||
OR (callsign_verified_at IS NOT NULL AND verification_method IS NOT NULL
|
||||
AND callsign IS NOT NULL)
|
||||
)
|
||||
);
|
||||
|
||||
COMMENT ON TABLE core.users IS 'One row per Discord user who has interacted with a stateful command. Deliberately no soft-delete column: a deletion request must actually remove the row, so there is nothing left to resurrect.';
|
||||
|
||||
-- Two users may not both hold a VERIFIED claim on the same callsign. Unverified
|
||||
-- claims are unconstrained, since anyone can type anything.
|
||||
CREATE UNIQUE INDEX users_verified_callsign_uniq
|
||||
ON core.users (callsign)
|
||||
WHERE callsign_verified_at IS NOT NULL;
|
||||
|
||||
CREATE INDEX users_callsign_idx ON core.users (callsign) WHERE callsign IS NOT NULL;
|
||||
|
||||
CREATE TRIGGER users_touch
|
||||
BEFORE UPDATE ON core.users
|
||||
FOR EACH ROW EXECUTE FUNCTION core.touch_updated_at();
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- verifications
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE TABLE core.verifications (
|
||||
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
user_id bigint NOT NULL REFERENCES core.users(id) ON DELETE CASCADE,
|
||||
|
||||
method text NOT NULL
|
||||
CHECK (method IN ('arrl_email', 'qrz_profile', 'admin', 'lotw')),
|
||||
callsign core.callsign NOT NULL,
|
||||
|
||||
status text NOT NULL DEFAULT 'pending'
|
||||
CHECK (status IN ('pending', 'succeeded', 'failed', 'expired', 'cancelled')),
|
||||
|
||||
-- SHA-256 of the challenge token, never the token. Same reasoning as API
|
||||
-- keys: a database dump must not let anyone complete a verification.
|
||||
challenge_hash bytea,
|
||||
|
||||
-- Diagnostics only. Must NOT contain the email address or any other
|
||||
-- identifier - the whole point of hashing the challenge is undone if the
|
||||
-- address is sitting in a jsonb column next to it.
|
||||
detail jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
expires_at timestamptz,
|
||||
completed_at timestamptz
|
||||
);
|
||||
|
||||
CREATE INDEX verifications_user_idx ON core.verifications (user_id, created_at DESC);
|
||||
|
||||
CREATE INDEX verifications_pending_idx
|
||||
ON core.verifications (expires_at)
|
||||
WHERE status = 'pending';
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- subscriptions
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
-- Discord-shaped, but persistent and worth backing up, so it belongs here
|
||||
-- rather than in Redis.
|
||||
CREATE TABLE core.subscriptions (
|
||||
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
|
||||
guild_id bigint, -- NULL for a DM subscription
|
||||
channel_id bigint NOT NULL,
|
||||
user_id bigint REFERENCES core.users(id) ON DELETE CASCADE,
|
||||
|
||||
kind text NOT NULL
|
||||
CHECK (kind IN ('pota', 'sota', 'wwff', 'cluster', 'rbn', 'propagation', 'net')),
|
||||
|
||||
-- Band, mode, region, distance, specific refs. Open-ended by nature, and
|
||||
-- the filter shape differs per kind, so jsonb rather than twenty columns
|
||||
-- that are NULL for most rows.
|
||||
filters jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
|
||||
enabled boolean NOT NULL DEFAULT true,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX subscriptions_channel_idx ON core.subscriptions (channel_id) WHERE enabled;
|
||||
CREATE INDEX subscriptions_kind_idx ON core.subscriptions (kind) WHERE enabled;
|
||||
CREATE INDEX subscriptions_filters_idx ON core.subscriptions USING gin (filters);
|
||||
|
||||
-- Postgres does not index the referencing side of a foreign key. Without this,
|
||||
-- deleting a user sequentially scans subscriptions to enforce the cascade.
|
||||
CREATE INDEX subscriptions_user_idx ON core.subscriptions (user_id) WHERE user_id IS NOT NULL;
|
||||
|
||||
CREATE TRIGGER subscriptions_touch
|
||||
BEFORE UPDATE ON core.subscriptions
|
||||
FOR EACH ROW EXECUTE FUNCTION core.touch_updated_at();
|
||||
@@ -0,0 +1,134 @@
|
||||
-- api: shall-issue tenancy. Owners, keys, usage.
|
||||
--
|
||||
-- Most sensitive schema in the database. hammy_readonly is deliberately NOT
|
||||
-- granted USAGE on it - a reporting query has no business reading key material.
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- owners
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE TABLE api.owners (
|
||||
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
|
||||
-- The abuse contact, and the only field that is genuinely required. It is
|
||||
-- what makes shall-issue workable: a channel to say "your key is doing
|
||||
-- something odd, fix it or I revoke".
|
||||
email core.email NOT NULL UNIQUE,
|
||||
email_verified_at timestamptz,
|
||||
|
||||
-- Optional, and a scope multiplier rather than a gate. An unlicensed
|
||||
-- developer building a study tool against the exam endpoints has no
|
||||
-- callsign and no reason to get one yet.
|
||||
callsign core.callsign,
|
||||
callsign_verified_at timestamptz,
|
||||
|
||||
status text NOT NULL DEFAULT 'active'
|
||||
CHECK (status IN ('active', 'suspended', 'closed')),
|
||||
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
COMMENT ON TABLE api.owners IS 'Registrants for API keys. Holds PII from an international user base, so the deletion path in 005_privacy.sql applies here too, not just to Discord users.';
|
||||
|
||||
CREATE UNIQUE INDEX owners_verified_callsign_uniq
|
||||
ON api.owners (callsign)
|
||||
WHERE callsign_verified_at IS NOT NULL;
|
||||
|
||||
CREATE TRIGGER owners_touch
|
||||
BEFORE UPDATE ON api.owners
|
||||
FOR EACH ROW EXECUTE FUNCTION core.touch_updated_at();
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- keys
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE TABLE api.keys (
|
||||
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
owner_id bigint NOT NULL REFERENCES api.owners(id) ON DELETE CASCADE,
|
||||
|
||||
-- SHA-256 of the key, computed in the application. The key itself is shown
|
||||
-- once at issue and never stored: a database dump must not be a set of
|
||||
-- working credentials. Hashing in the app rather than via pgcrypto keeps
|
||||
-- the plaintext out of the query log.
|
||||
key_hash bytea NOT NULL UNIQUE,
|
||||
CONSTRAINT keys_hash_is_sha256 CHECK (octet_length(key_hash) = 32),
|
||||
|
||||
-- Enough of the key to identify it in a UI and in a leak report, e.g.
|
||||
-- 'hmy_live_a1b2c3d4'. Not secret and not sufficient to authenticate.
|
||||
key_prefix text NOT NULL,
|
||||
label text,
|
||||
|
||||
-- Tier 0 reference data can be near-unlimited; callsign lookup and 44net
|
||||
-- probing need much tighter ceilings. Scoping from day one means a scope
|
||||
-- can be tightened later without breaking every key.
|
||||
scopes text[] NOT NULL DEFAULT '{}',
|
||||
CONSTRAINT keys_scopes_known CHECK (
|
||||
scopes <@ ARRAY[
|
||||
'reference', -- band plans, prefixes, calculators
|
||||
'callsign', -- callsign lookup, rate limited
|
||||
'spots', -- POTA/SOTA/cluster/RBN feeds
|
||||
'propagation',
|
||||
'logbook', -- read/write a user's own log
|
||||
'net44', -- reachability probes from the 44net allocation
|
||||
'admin'
|
||||
]::text[]
|
||||
),
|
||||
|
||||
quota_tier text NOT NULL DEFAULT 'default'
|
||||
CHECK (quota_tier IN ('default', 'verified', 'trusted', 'internal')),
|
||||
|
||||
status text NOT NULL DEFAULT 'active'
|
||||
CHECK (status IN ('active', 'revoked', 'suspended')),
|
||||
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
expires_at timestamptz,
|
||||
last_used_at timestamptz,
|
||||
revoked_at timestamptz,
|
||||
revoked_reason text,
|
||||
|
||||
CONSTRAINT keys_revoked_consistent CHECK (
|
||||
(status = 'revoked') = (revoked_at IS NOT NULL)
|
||||
)
|
||||
);
|
||||
|
||||
COMMENT ON TABLE api.keys IS 'Several active keys per owner is intentional: people leak keys into public repositories, and rotating needs an overlap window where both the old and new key work.';
|
||||
|
||||
CREATE INDEX keys_owner_idx ON api.keys (owner_id);
|
||||
|
||||
-- No separate index on key_hash: the UNIQUE constraint above already builds a
|
||||
-- btree over exactly that column, and the planner uses it for the lookup.
|
||||
|
||||
CREATE INDEX keys_expiring_idx
|
||||
ON api.keys (expires_at)
|
||||
WHERE status = 'active' AND expires_at IS NOT NULL;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- usage
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
-- Rolled up from Redis, which is authoritative for the live rate-limit window.
|
||||
-- Redis holds the current counters; this holds completed days. A Redis flush
|
||||
-- loses at most the window in progress, which for abuse detection is fine.
|
||||
CREATE TABLE api.usage_daily (
|
||||
key_id bigint NOT NULL REFERENCES api.keys(id) ON DELETE CASCADE,
|
||||
day date NOT NULL,
|
||||
endpoint text NOT NULL,
|
||||
calls bigint NOT NULL DEFAULT 0,
|
||||
errors bigint NOT NULL DEFAULT 0,
|
||||
|
||||
-- Distinct end users seen, not who they were. Counting them is the
|
||||
-- enumeration signal - one Discord user driving 90% of a key's traffic
|
||||
-- looks very different from fifty users sharing it - but the identities
|
||||
-- themselves are not needed and so are not kept.
|
||||
distinct_users integer NOT NULL DEFAULT 0,
|
||||
|
||||
PRIMARY KEY (key_id, day, endpoint)
|
||||
);
|
||||
|
||||
-- The flush job is an upsert:
|
||||
-- INSERT INTO api.usage_daily (key_id, day, endpoint, calls)
|
||||
-- VALUES ($1, $2, $3, $4)
|
||||
-- ON CONFLICT (key_id, day, endpoint)
|
||||
-- DO UPDATE SET calls = api.usage_daily.calls + EXCLUDED.calls;
|
||||
CREATE INDEX usage_daily_day_idx ON api.usage_daily (day);
|
||||
@@ -0,0 +1,121 @@
|
||||
-- logbook: per-user QSO logs and import batches.
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- imports
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
-- Every QSO belongs to an import batch, so a bad ADIF file can be rolled back
|
||||
-- as a unit rather than picked apart row by row afterwards.
|
||||
CREATE TABLE logbook.imports (
|
||||
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
user_id bigint NOT NULL REFERENCES core.users(id) ON DELETE CASCADE,
|
||||
|
||||
filename text,
|
||||
source text, -- 'adif', 'lotw', 'clublog', 'manual'
|
||||
|
||||
status text NOT NULL DEFAULT 'running'
|
||||
CHECK (status IN ('running', 'completed', 'failed', 'rolled_back')),
|
||||
|
||||
rows_total integer NOT NULL DEFAULT 0,
|
||||
rows_accepted integer NOT NULL DEFAULT 0,
|
||||
rows_duplicate integer NOT NULL DEFAULT 0,
|
||||
rows_rejected integer NOT NULL DEFAULT 0,
|
||||
|
||||
error text,
|
||||
|
||||
started_at timestamptz NOT NULL DEFAULT now(),
|
||||
finished_at timestamptz
|
||||
);
|
||||
|
||||
CREATE INDEX imports_user_idx ON logbook.imports (user_id, started_at DESC);
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- qsos
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE TABLE logbook.qsos (
|
||||
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
user_id bigint NOT NULL REFERENCES core.users(id) ON DELETE CASCADE,
|
||||
|
||||
-- SET NULL rather than CASCADE: deleting an import record should not delete
|
||||
-- the QSOs it brought in, only the audit trail of how they arrived.
|
||||
import_id bigint REFERENCES logbook.imports(id) ON DELETE SET NULL,
|
||||
|
||||
callsign core.callsign NOT NULL,
|
||||
|
||||
-- ADIF gives QSO_DATE and TIME_ON separately, both UTC. Combined here.
|
||||
qso_at timestamptz NOT NULL,
|
||||
|
||||
-- Dedup key. Generated columns must be IMMUTABLE, and date_trunc over a
|
||||
-- timestamptz is only STABLE because it depends on the TimeZone setting.
|
||||
-- timezone('UTC', tstz) -> timestamp IS immutable, and date_trunc over a
|
||||
-- plain timestamp is immutable, so the chain works.
|
||||
--
|
||||
-- Minute granularity because two operators logging the same contact
|
||||
-- routinely differ by a few seconds.
|
||||
qso_minute timestamp GENERATED ALWAYS AS
|
||||
(date_trunc('minute', timezone('UTC', qso_at))) STORED,
|
||||
|
||||
band text,
|
||||
mode text,
|
||||
submode text,
|
||||
freq_hz bigint,
|
||||
|
||||
rst_sent text,
|
||||
rst_rcvd text,
|
||||
|
||||
grid core.gridsquare,
|
||||
dxcc integer, -- ADIF entity code, resolved at import
|
||||
cq_zone smallint,
|
||||
itu_zone smallint,
|
||||
|
||||
tx_power_w numeric(8,2),
|
||||
comment text,
|
||||
|
||||
-- ADIF defines a few hundred fields and every logging program invents more.
|
||||
-- Normalising the columns above and keeping the rest here is the main
|
||||
-- reason for choosing Postgres: MariaDB's JSON is LONGTEXT with validation,
|
||||
-- with no binary storage and no functional index on a path.
|
||||
extra jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
|
||||
CONSTRAINT qsos_freq_sane CHECK (freq_hz IS NULL OR (freq_hz > 0 AND freq_hz < 300000000000)),
|
||||
CONSTRAINT qsos_zones_sane CHECK (
|
||||
(cq_zone IS NULL OR cq_zone BETWEEN 1 AND 40)
|
||||
AND (itu_zone IS NULL OR itu_zone BETWEEN 1 AND 90)
|
||||
)
|
||||
);
|
||||
|
||||
COMMENT ON COLUMN logbook.qsos.qso_minute IS 'Truncated copy of qso_at used only for duplicate detection. Do not query it directly; use qso_at.';
|
||||
|
||||
-- Duplicate detection. A repeat import of the same ADIF must not double the
|
||||
-- log, and this is what makes ON CONFLICT DO NOTHING work on import.
|
||||
--
|
||||
-- NULLS NOT DISTINCT matters: band and mode are nullable, and in a unique index
|
||||
-- NULLs are distinct by default. Without it, two identical QSOs that both have
|
||||
-- band IS NULL do not collide, so an ADIF with no BAND field imports twice, and
|
||||
-- again on every re-import. Requires PostgreSQL 15+.
|
||||
CREATE UNIQUE INDEX qsos_dedup_uniq
|
||||
ON logbook.qsos (user_id, callsign, band, mode, qso_minute)
|
||||
NULLS NOT DISTINCT;
|
||||
|
||||
-- The four access patterns that actually happen.
|
||||
CREATE INDEX qsos_user_time_idx ON logbook.qsos (user_id, qso_at DESC);
|
||||
CREATE INDEX qsos_user_call_idx ON logbook.qsos (user_id, callsign);
|
||||
CREATE INDEX qsos_user_band_mode_idx ON logbook.qsos (user_id, band, mode);
|
||||
|
||||
-- Award progress is DERIVED, never stored. DXCC and WAS counts are aggregates
|
||||
-- over this table; a stored counter would need maintaining on every insert,
|
||||
-- edit and rollback, and would drift. This index is what makes that cheap.
|
||||
CREATE INDEX qsos_user_dxcc_idx ON logbook.qsos (user_id, dxcc) WHERE dxcc IS NOT NULL;
|
||||
|
||||
CREATE INDEX qsos_extra_idx ON logbook.qsos USING gin (extra);
|
||||
|
||||
-- Referencing side of the import FK. Without it, rolling back one import
|
||||
-- sequentially scans the whole qsos table to apply ON DELETE SET NULL.
|
||||
CREATE INDEX qsos_import_idx ON logbook.qsos (import_id) WHERE import_id IS NOT NULL;
|
||||
|
||||
-- Worth revisiting if a single user's log passes a few million rows: partition
|
||||
-- by user_id hash, or by qso_at range. Not now - the indexes above carry a
|
||||
-- normal log comfortably, and partitioning early costs flexibility.
|
||||
@@ -0,0 +1,102 @@
|
||||
-- Deletion paths.
|
||||
--
|
||||
-- Discord's developer policy requires deleting user data on request, and API
|
||||
-- registrants are an international user base whose data protection expectations
|
||||
-- apply regardless. Build this before you need it: retrofitting a deletion path
|
||||
-- across three stores is miserable, and being asked for one you do not have is
|
||||
-- worse.
|
||||
--
|
||||
-- These functions cover Postgres. The caller must ALSO clear Redis - cached
|
||||
-- callsign lookups, rate-limit counters keyed by hashed Discord ID, and any
|
||||
-- verification challenge in flight. Postgres is not the whole story.
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
-- Removes a Discord user and everything referencing them. The ON DELETE CASCADE
|
||||
-- chain reaches verifications, subscriptions, imports and qsos, so this is one
|
||||
-- statement rather than a list that can fall out of date as tables are added.
|
||||
--
|
||||
-- Returns the number of QSOs removed, purely so the caller can tell the user
|
||||
-- what was deleted.
|
||||
CREATE FUNCTION core.forget_user(p_discord_id bigint)
|
||||
RETURNS TABLE (deleted_user boolean, deleted_qsos bigint)
|
||||
LANGUAGE plpgsql
|
||||
-- Pinned: the body qualifies its tables, but now(), count() and friends resolve
|
||||
-- through search_path. Matters more if this ever becomes SECURITY DEFINER.
|
||||
SET search_path = pg_catalog, core, logbook
|
||||
AS $$
|
||||
DECLARE
|
||||
v_user_id bigint;
|
||||
v_qsos bigint := 0;
|
||||
BEGIN
|
||||
SELECT id INTO v_user_id FROM core.users WHERE discord_id = p_discord_id;
|
||||
|
||||
IF v_user_id IS NULL THEN
|
||||
RETURN QUERY SELECT false, 0::bigint;
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
SELECT count(*) INTO v_qsos FROM logbook.qsos WHERE user_id = v_user_id;
|
||||
|
||||
-- Cascades do the rest.
|
||||
DELETE FROM core.users WHERE id = v_user_id;
|
||||
|
||||
RETURN QUERY SELECT true, v_qsos;
|
||||
END;
|
||||
$$;
|
||||
|
||||
COMMENT ON FUNCTION core.forget_user(bigint) IS 'Hard-deletes a Discord user and all dependent rows. Caller must also purge Redis keys for this user.';
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
-- The API-side equivalent. Keys and usage rows cascade from the owner.
|
||||
CREATE FUNCTION api.forget_owner(p_email core.email)
|
||||
RETURNS TABLE (deleted_owner boolean, deleted_keys bigint)
|
||||
LANGUAGE plpgsql
|
||||
SET search_path = pg_catalog, core, api
|
||||
AS $$
|
||||
DECLARE
|
||||
v_owner_id bigint;
|
||||
v_keys bigint := 0;
|
||||
BEGIN
|
||||
SELECT id INTO v_owner_id FROM api.owners WHERE email = p_email;
|
||||
|
||||
IF v_owner_id IS NULL THEN
|
||||
RETURN QUERY SELECT false, 0::bigint;
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
SELECT count(*) INTO v_keys FROM api.keys WHERE owner_id = v_owner_id;
|
||||
|
||||
DELETE FROM api.owners WHERE id = v_owner_id;
|
||||
|
||||
RETURN QUERY SELECT true, v_keys;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
-- Retention. Verification records carry a hashed challenge and a timestamp, and
|
||||
-- are useful for a short while when investigating a failed verification. They
|
||||
-- are not useful a year later. Run this from cron.
|
||||
CREATE FUNCTION core.prune_verifications(p_keep_days integer DEFAULT 90)
|
||||
RETURNS bigint
|
||||
LANGUAGE plpgsql
|
||||
SET search_path = pg_catalog, core
|
||||
AS $$
|
||||
DECLARE
|
||||
v_removed bigint;
|
||||
BEGIN
|
||||
DELETE FROM core.verifications
|
||||
WHERE created_at < now() - make_interval(days => p_keep_days)
|
||||
AND status <> 'succeeded';
|
||||
|
||||
GET DIAGNOSTICS v_removed = ROW_COUNT;
|
||||
|
||||
RETURN v_removed;
|
||||
END;
|
||||
$$;
|
||||
|
||||
GRANT EXECUTE ON FUNCTION core.forget_user(bigint) TO hammy_app;
|
||||
GRANT EXECUTE ON FUNCTION api.forget_owner(core.email) TO hammy_app;
|
||||
GRANT EXECUTE ON FUNCTION core.prune_verifications(integer) TO hammy_app;
|
||||
@@ -0,0 +1,58 @@
|
||||
# Database setup
|
||||
|
||||
## Once, as a superuser
|
||||
|
||||
psql -U postgres -f scripts/bootstrap.sql
|
||||
|
||||
Creates the roles, the `hammy` database, the four schemas, and the default
|
||||
privileges. Change the three `CHANGE_ME` passwords first.
|
||||
|
||||
## Migrations, as hammy_owner
|
||||
|
||||
for f in migrations/*.sql; do
|
||||
psql -U hammy_owner -d hammy -v ON_ERROR_STOP=1 -f "$f"
|
||||
done
|
||||
|
||||
Running these **as hammy_owner is not optional**. `ALTER DEFAULT PRIVILEGES FOR
|
||||
ROLE hammy_owner` only fires for objects that role creates; run them as a
|
||||
superuser and the tables end up owned by the superuser with no grants, so
|
||||
`hammy_app` sees four empty schemas.
|
||||
|
||||
`hammy_owner` is `NOLOGIN`, so connect as a superuser and switch:
|
||||
|
||||
psql -U postgres -d hammy -c 'SET ROLE hammy_owner' -f migrations/001_types.sql
|
||||
|
||||
or give it `LOGIN` for the duration of the setup and revoke it after.
|
||||
|
||||
Verify afterwards:
|
||||
|
||||
SELECT relname, relacl FROM pg_class
|
||||
WHERE relnamespace = 'core'::regnamespace AND relkind = 'r';
|
||||
|
||||
Every row should show `hammy_app=arwd/hammy_owner`. All NULL means the
|
||||
migrations ran as the wrong role.
|
||||
|
||||
## Files
|
||||
|
||||
| File | Contents |
|
||||
|---|---|
|
||||
| `scripts/bootstrap.sql` | Roles, database, schemas, default privileges. Superuser, once. |
|
||||
| `migrations/001_types.sql` | `touch_updated_at()`, the `callsign`/`email`/`gridsquare` domains. |
|
||||
| `migrations/002_core.sql` | `users`, `verifications`, `subscriptions`. |
|
||||
| `migrations/003_api.sql` | `owners`, `keys`, `usage_daily`. |
|
||||
| `migrations/004_logbook.sql` | `imports`, `qsos`. |
|
||||
| `migrations/005_privacy.sql` | `forget_user()`, `forget_owner()`, `prune_verifications()`. |
|
||||
|
||||
## sqlc
|
||||
|
||||
`sqlc.yaml` points only at 001-004. It parses SQL to learn the schema and will
|
||||
choke on `CREATE ROLE`, `ALTER DEFAULT PRIVILEGES` and `DO $$` blocks, which is
|
||||
another reason those live in `scripts/` rather than `migrations/`.
|
||||
|
||||
## Already have the old numbering?
|
||||
|
||||
The previous layout was `001_bootstrap` through `006_fixes`, applied by hand. If
|
||||
that database exists and has no data worth keeping, `DROP DATABASE hammy` plus
|
||||
`DROP ROLE` the four roles, then start from `bootstrap.sql`. The corrections
|
||||
from the old `006_fixes.sql` are folded into 002-005 here, so there is no
|
||||
separate fixes file any more.
|
||||
@@ -0,0 +1,102 @@
|
||||
-- Hammy backend bootstrap. Run ONCE, as a superuser, before any migration:
|
||||
--
|
||||
-- psql -U postgres -f scripts/bootstrap.sql
|
||||
--
|
||||
-- Deliberately NOT in migrations/. Three reasons:
|
||||
--
|
||||
-- * CREATE DATABASE cannot run inside a transaction block, and migration
|
||||
-- tools wrap each file in one.
|
||||
-- * \c is a psql meta-command, not SQL. A migration tool has no idea what it
|
||||
-- means.
|
||||
-- * Roles are cluster-wide, not per-database, so they are not part of any one
|
||||
-- database's schema history.
|
||||
--
|
||||
-- Order matters: the schemas and ALTER DEFAULT PRIVILEGES must exist BEFORE any
|
||||
-- table is created, because default privileges only apply to objects created
|
||||
-- afterwards. Get this right here and no migration ever needs a GRANT.
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Roles (cluster-wide)
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
-- Owns every object and runs migrations. Never connects at runtime.
|
||||
CREATE ROLE hammy_owner NOLOGIN;
|
||||
|
||||
-- What the backend connects as. DML only: it can read and write rows but not
|
||||
-- create, alter or drop anything, so a compromised backend cannot drop the
|
||||
-- logbook.
|
||||
CREATE ROLE hammy_app LOGIN PASSWORD 'CHANGE_ME';
|
||||
|
||||
-- Bulk loaders for FCC ULS, 44net allocations and the like.
|
||||
CREATE ROLE hammy_ingest LOGIN PASSWORD 'CHANGE_ME';
|
||||
|
||||
-- Analytics and debugging. Deliberately no access to the api schema - a
|
||||
-- reporting query has no business reading key material.
|
||||
CREATE ROLE hammy_readonly LOGIN PASSWORD 'CHANGE_ME';
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Database
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE DATABASE hammy OWNER hammy_owner ENCODING 'UTF8';
|
||||
|
||||
\c hammy
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Schemas
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
-- Before PostgreSQL 15, PUBLIC held CREATE on the public schema, so any role
|
||||
-- could add objects to it. Revoke it and do not use public for anything.
|
||||
REVOKE ALL ON SCHEMA public FROM PUBLIC;
|
||||
|
||||
CREATE SCHEMA core AUTHORIZATION hammy_owner;
|
||||
CREATE SCHEMA api AUTHORIZATION hammy_owner;
|
||||
CREATE SCHEMA logbook AUTHORIZATION hammy_owner;
|
||||
CREATE SCHEMA ingest AUTHORIZATION hammy_owner;
|
||||
|
||||
COMMENT ON SCHEMA core IS 'Discord-linked user identity and callsign verification.';
|
||||
COMMENT ON SCHEMA api IS 'API tenancy: owners, keys, usage. Most sensitive schema.';
|
||||
COMMENT ON SCHEMA logbook IS 'Per-user QSO logs and import batches.';
|
||||
COMMENT ON SCHEMA ingest IS 'Bulk-loaded third-party reference data.';
|
||||
|
||||
GRANT USAGE ON SCHEMA core, api, logbook, ingest TO hammy_app;
|
||||
GRANT USAGE ON SCHEMA ingest TO hammy_ingest;
|
||||
GRANT USAGE ON SCHEMA core, logbook TO hammy_readonly;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Default privileges
|
||||
--
|
||||
-- Apply to objects hammy_owner creates FROM NOW ON. Every migration runs as
|
||||
-- hammy_owner, so tables pick these up automatically.
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
ALTER DEFAULT PRIVILEGES FOR ROLE hammy_owner IN SCHEMA core, api, logbook
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO hammy_app;
|
||||
|
||||
ALTER DEFAULT PRIVILEGES FOR ROLE hammy_owner IN SCHEMA core, api, logbook
|
||||
GRANT USAGE ON SEQUENCES TO hammy_app;
|
||||
|
||||
ALTER DEFAULT PRIVILEGES FOR ROLE hammy_owner IN SCHEMA ingest
|
||||
GRANT SELECT ON TABLES TO hammy_app;
|
||||
|
||||
ALTER DEFAULT PRIVILEGES FOR ROLE hammy_owner IN SCHEMA ingest
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO hammy_ingest;
|
||||
|
||||
ALTER DEFAULT PRIVILEGES FOR ROLE hammy_owner IN SCHEMA core, logbook
|
||||
GRANT SELECT ON TABLES TO hammy_readonly;
|
||||
|
||||
-- Postgres grants EXECUTE on new functions to PUBLIC by default, which would
|
||||
-- make forget_user() callable by anyone who can connect. Revoke it; 005 grants
|
||||
-- it back to hammy_app explicitly.
|
||||
ALTER DEFAULT PRIVILEGES FOR ROLE hammy_owner IN SCHEMA core, api
|
||||
REVOKE EXECUTE ON FUNCTIONS FROM PUBLIC;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- search_path per role, so queries need not schema-qualify everything.
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
ALTER ROLE hammy_owner SET search_path = core, api, logbook, ingest;
|
||||
ALTER ROLE hammy_app SET search_path = core, api, logbook, ingest;
|
||||
ALTER ROLE hammy_ingest SET search_path = ingest;
|
||||
ALTER ROLE hammy_readonly SET search_path = core, logbook;
|
||||
Reference in New Issue
Block a user