SQLite building for data
This commit is contained in:
@@ -4,3 +4,4 @@ build/
|
||||
config.json
|
||||
.vscode
|
||||
.vscode/
|
||||
sqlite/*.sqlite
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
-- Hammy reference bundle schema
|
||||
--
|
||||
-- Read-only at runtime. Regenerated as a whole file, never migrated in place.
|
||||
-- Open with SQLITE_OPEN_READONLY.
|
||||
|
||||
PRAGMA foreign_keys = ON;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Bundle metadata
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE TABLE ref_meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- Where each dataset came from and when. Shown by /about and used to decide
|
||||
-- whether a newer bundle is worth downloading.
|
||||
CREATE TABLE ref_sources (
|
||||
dataset TEXT PRIMARY KEY,
|
||||
source_name TEXT NOT NULL,
|
||||
source_url TEXT,
|
||||
license TEXT,
|
||||
retrieved TEXT, -- ISO 8601 date
|
||||
notes TEXT
|
||||
);
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Bands and privileges
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
-- Named bands, region-independent. edge_low/high are the widest extent across
|
||||
-- all regions; per-region reality lives in band_segments.
|
||||
CREATE TABLE bands (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE, -- '20m', '70cm'
|
||||
edge_low_hz INTEGER NOT NULL,
|
||||
edge_high_hz INTEGER NOT NULL,
|
||||
wavelength_m REAL,
|
||||
sort_order INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE license_classes (
|
||||
id INTEGER PRIMARY KEY,
|
||||
country TEXT NOT NULL, -- ISO 3166-1 alpha-2
|
||||
code TEXT NOT NULL, -- 'E', 'G', 'T'
|
||||
name TEXT NOT NULL,
|
||||
rank INTEGER NOT NULL, -- higher = more privileges
|
||||
notes TEXT,
|
||||
UNIQUE (country, code)
|
||||
);
|
||||
|
||||
-- One row per (country, class, contiguous frequency range, mode group).
|
||||
-- A class with split phone/CW privileges on a band gets several rows.
|
||||
CREATE TABLE band_segments (
|
||||
id INTEGER PRIMARY KEY,
|
||||
band_id INTEGER NOT NULL REFERENCES bands(id),
|
||||
country TEXT NOT NULL,
|
||||
iaru_region INTEGER, -- 1, 2, 3, or NULL if not region-scoped
|
||||
class_id INTEGER REFERENCES license_classes(id),
|
||||
low_hz INTEGER NOT NULL,
|
||||
high_hz INTEGER NOT NULL,
|
||||
modes TEXT NOT NULL, -- 'CW', 'CW,DATA', 'PHONE,IMAGE'
|
||||
max_power_w INTEGER, -- NULL = national default
|
||||
notes TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX idx_band_segments_freq ON band_segments (country, low_hz, high_hz);
|
||||
CREATE INDEX idx_band_segments_class ON band_segments (class_id);
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- DXCC
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
-- id is the ADIF/ARRL DXCC entity code, supplied by cty2sql.py's DXCC_IDS map
|
||||
-- (cty.dat itself carries no entity numbers). WAE and other non-DXCC entities
|
||||
-- are excluded by default since they have no code.
|
||||
--
|
||||
-- latitude is north-positive and longitude is east-positive; utc_offset is the
|
||||
-- usual "UTC + this = local". All three are flipped relative to cty.dat's own
|
||||
-- positive-west conventions by the importer.
|
||||
CREATE TABLE dxcc_entities (
|
||||
id INTEGER PRIMARY KEY, -- ADIF DXCC entity code
|
||||
name TEXT NOT NULL,
|
||||
primary_prefix TEXT NOT NULL UNIQUE,
|
||||
continent TEXT, -- 'EU', 'NA', ...
|
||||
cq_zone INTEGER, -- record default
|
||||
itu_zone INTEGER, -- record default
|
||||
latitude REAL,
|
||||
longitude REAL,
|
||||
utc_offset REAL,
|
||||
deleted INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE INDEX idx_dxcc_entities_prefix ON dxcc_entities (primary_prefix);
|
||||
|
||||
-- Prefixes are matched longest-first:
|
||||
-- SELECT ... WHERE ? GLOB prefix || '*' ORDER BY length(prefix) DESC LIMIT 1
|
||||
-- cq_zone/itu_zone here OVERRIDE the entity default when non-NULL. About 75% of
|
||||
-- rows carry one, so COALESCE them in every query:
|
||||
-- COALESCE(p.cq_zone, e.cq_zone)
|
||||
CREATE TABLE dxcc_prefixes (
|
||||
prefix TEXT NOT NULL,
|
||||
entity_id INTEGER NOT NULL REFERENCES dxcc_entities(id),
|
||||
exact INTEGER NOT NULL DEFAULT 0, -- 1 = whole callsign must match
|
||||
cq_zone INTEGER,
|
||||
itu_zone INTEGER,
|
||||
PRIMARY KEY (prefix, entity_id)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_dxcc_prefixes_len ON dxcc_prefixes (length(prefix) DESC);
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Operating reference
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE TABLE qcodes (
|
||||
code TEXT PRIMARY KEY,
|
||||
question TEXT NOT NULL,
|
||||
answer TEXT NOT NULL,
|
||||
category TEXT -- 'general', 'aeronautical', 'maritime'
|
||||
);
|
||||
|
||||
CREATE TABLE prosigns (
|
||||
symbol TEXT PRIMARY KEY, -- 'AR', 'SK', 'BT'
|
||||
morse TEXT NOT NULL,
|
||||
meaning TEXT NOT NULL,
|
||||
usage TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE phonetics (
|
||||
letter TEXT PRIMARY KEY,
|
||||
word TEXT NOT NULL,
|
||||
pronunciation TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE morse (
|
||||
character TEXT PRIMARY KEY,
|
||||
code TEXT NOT NULL, -- '.-' with . and -
|
||||
category TEXT NOT NULL -- 'letter', 'digit', 'punctuation'
|
||||
);
|
||||
|
||||
CREATE TABLE abbreviations (
|
||||
abbr TEXT PRIMARY KEY,
|
||||
meaning TEXT NOT NULL,
|
||||
context TEXT -- 'CW', 'general', 'contest'
|
||||
);
|
||||
|
||||
-- RST readability/strength/tone scale
|
||||
CREATE TABLE rst_scale (
|
||||
component TEXT NOT NULL, -- 'R', 'S', 'T'
|
||||
value INTEGER NOT NULL,
|
||||
meaning TEXT NOT NULL,
|
||||
PRIMARY KEY (component, value)
|
||||
);
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Engineering reference
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
-- Matched loss follows the usual two-term model:
|
||||
-- loss_dB_per_100ft = k1 * sqrt(f_MHz) + k2 * f_MHz
|
||||
CREATE TABLE coax_types (
|
||||
name TEXT PRIMARY KEY,
|
||||
impedance_ohm REAL NOT NULL,
|
||||
velocity_factor REAL NOT NULL,
|
||||
capacitance_pf_per_ft REAL,
|
||||
outer_diameter_mm REAL,
|
||||
loss_k1 REAL,
|
||||
loss_k2 REAL,
|
||||
max_power_hf_w INTEGER,
|
||||
notes TEXT
|
||||
);
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- NCDXF/IARU beacon network
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
-- 18 stations, 10 s per slot, 3 min for a full cycle on each of 5 frequencies.
|
||||
-- Which station is on which band at time t is pure arithmetic from slot_index.
|
||||
CREATE TABLE ncdxf_beacons (
|
||||
slot_index INTEGER PRIMARY KEY, -- 0..17, order within the cycle
|
||||
callsign TEXT NOT NULL,
|
||||
location TEXT NOT NULL,
|
||||
grid TEXT,
|
||||
latitude REAL,
|
||||
longitude REAL,
|
||||
dxcc_id INTEGER REFERENCES dxcc_entities(id),
|
||||
active INTEGER NOT NULL DEFAULT 1
|
||||
);
|
||||
|
||||
CREATE TABLE ncdxf_frequencies (
|
||||
band_id INTEGER PRIMARY KEY REFERENCES bands(id),
|
||||
freq_hz INTEGER NOT NULL,
|
||||
slot_offset INTEGER NOT NULL -- slots to add for this band
|
||||
);
|
||||
@@ -0,0 +1,33 @@
|
||||
INSERT INTO ref_meta (key, value) VALUES
|
||||
('schema_version', '1'),
|
||||
('bundle_version', '2026.08.30-dev'),
|
||||
('generated_at', '2026-08-30'),
|
||||
('project', 'Hammy'),
|
||||
('project_url', 'https://hammybot.org');
|
||||
|
||||
INSERT INTO ref_sources (dataset, source_name, source_url, license, retrieved, notes) VALUES
|
||||
('band_segments', 'FCC Part 97 Subpart D', 'https://www.ecfr.gov/current/title-47/part-97',
|
||||
'US Government work, public domain', NULL,
|
||||
'HAND-ENTERED STARTER DATA. Must be verified against the current Part 97 text before shipping. Wrong band edges can put an operator out of band.'),
|
||||
|
||||
('band_segments_iaru', 'IARU Region 1/2/3 band plans', 'https://www.iaru.org/', 'See IARU', NULL,
|
||||
'Only allocation extents are seeded, not the full mode/bandwidth plans. Incomplete.'),
|
||||
|
||||
('dxcc_entities', 'ARRL DXCC list / cty.dat', 'https://www.country-files.com/',
|
||||
'Free for amateur radio use, see country-files.com', NULL,
|
||||
'DEMO SUBSET ONLY - about 25 entities to exercise the schema. Real bundle must be generated by parsing cty.dat.'),
|
||||
|
||||
('qcodes', 'ITU Q-code series', NULL, 'Public domain', NULL,
|
||||
'Amateur-relevant subset of the QRA-QUZ series.'),
|
||||
|
||||
('phonetics', 'ITU/NATO phonetic alphabet', NULL, 'Public domain', NULL, NULL),
|
||||
|
||||
('morse', 'ITU-R M.1677-1', NULL, 'Public domain', NULL,
|
||||
'International Morse. Does not include non-Latin extensions.'),
|
||||
|
||||
('coax_types', 'Manufacturer datasheets', NULL, 'Various', NULL,
|
||||
'Impedance, velocity factor and capacitance are nominal published figures. loss_k1/loss_k2 are deliberately NULL - populate from the actual datasheet for each cable rather than a generic approximation, since loss is what the calculator returns.'),
|
||||
|
||||
('ncdxf_beacons', 'NCDXF/IARU International Beacon Project', 'https://www.ncdxf.org/beacon/',
|
||||
'See NCDXF', NULL,
|
||||
'Station order is stable; individual beacons go offline for extended periods. The active flag needs refreshing against the NCDXF status page at build time.');
|
||||
@@ -0,0 +1,177 @@
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Bands
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
INSERT INTO bands (id, name, edge_low_hz, edge_high_hz, wavelength_m, sort_order) VALUES
|
||||
(1, '2200m', 135700, 137800, 2200, 10),
|
||||
(2, '630m', 472000, 479000, 630, 20),
|
||||
(3, '160m', 1800000, 2000000, 160, 30),
|
||||
(4, '80m', 3500000, 4000000, 80, 40),
|
||||
(5, '60m', 5330500, 5406400, 60, 50),
|
||||
(6, '40m', 7000000, 7300000, 40, 60),
|
||||
(7, '30m', 10100000, 10150000, 30, 70),
|
||||
(8, '20m', 14000000, 14350000, 20, 80),
|
||||
(9, '17m', 18068000, 18168000, 17, 90),
|
||||
(10, '15m', 21000000, 21450000, 15, 100),
|
||||
(11, '12m', 24890000, 24990000, 12, 110),
|
||||
(12, '10m', 28000000, 29700000, 10, 120),
|
||||
(13, '6m', 50000000, 54000000, 6, 130),
|
||||
(14, '2m', 144000000, 148000000, 2, 140),
|
||||
(15, '1.25m', 222000000, 225000000, 1.25, 150),
|
||||
(16, '70cm', 420000000, 450000000, 0.70, 160),
|
||||
(17, '33cm', 902000000, 928000000, 0.33, 170),
|
||||
(18, '23cm', 1240000000, 1300000000, 0.23, 180);
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- US license classes
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
INSERT INTO license_classes (id, country, code, name, rank, notes) VALUES
|
||||
(1, 'US', 'E', 'Amateur Extra', 50, NULL),
|
||||
(2, 'US', 'A', 'Advanced', 40, 'Closed to new issue since 1 April 2000; existing licences remain valid.'),
|
||||
(3, 'US', 'G', 'General', 30, NULL),
|
||||
(4, 'US', 'T', 'Technician', 20, NULL),
|
||||
(5, 'US', 'N', 'Novice', 10, 'Closed to new issue since 1 April 2000; existing licences remain valid.');
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- US band segments (47 CFR Part 97)
|
||||
--
|
||||
-- VERIFY BEFORE SHIPPING. Hand-entered from the Part 97 privilege tables.
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
-- 2200m and 630m: all classes, strict EIRP limits, notification required
|
||||
INSERT INTO band_segments (band_id, country, class_id, low_hz, high_hz, modes, max_power_w, notes) VALUES
|
||||
(1, 'US', 1, 135700, 137800, 'CW,DATA', NULL, '1 W EIRP maximum'),
|
||||
(1, 'US', 3, 135700, 137800, 'CW,DATA', NULL, '1 W EIRP maximum'),
|
||||
(1, 'US', 4, 135700, 137800, 'CW,DATA', NULL, '1 W EIRP maximum'),
|
||||
(2, 'US', 1, 472000, 479000, 'CW,DATA', NULL, '5 W EIRP maximum'),
|
||||
(2, 'US', 3, 472000, 479000, 'CW,DATA', NULL, '5 W EIRP maximum'),
|
||||
(2, 'US', 4, 472000, 479000, 'CW,DATA', NULL, '5 W EIRP maximum');
|
||||
|
||||
-- 160m: General and above, full band
|
||||
INSERT INTO band_segments (band_id, country, class_id, low_hz, high_hz, modes, max_power_w, notes) VALUES
|
||||
(3, 'US', 1, 1800000, 2000000, 'CW,DATA,PHONE,IMAGE', NULL, NULL),
|
||||
(3, 'US', 2, 1800000, 2000000, 'CW,DATA,PHONE,IMAGE', NULL, NULL),
|
||||
(3, 'US', 3, 1800000, 2000000, 'CW,DATA,PHONE,IMAGE', NULL, NULL);
|
||||
|
||||
-- 80m / 75m
|
||||
INSERT INTO band_segments (band_id, country, class_id, low_hz, high_hz, modes, max_power_w, notes) VALUES
|
||||
(4, 'US', 1, 3500000, 3600000, 'CW,DATA', NULL, NULL),
|
||||
(4, 'US', 1, 3600000, 4000000, 'PHONE,IMAGE', NULL, NULL),
|
||||
(4, 'US', 2, 3525000, 3600000, 'CW,DATA', NULL, NULL),
|
||||
(4, 'US', 2, 3700000, 4000000, 'PHONE,IMAGE', NULL, NULL),
|
||||
(4, 'US', 3, 3525000, 3600000, 'CW,DATA', NULL, NULL),
|
||||
(4, 'US', 3, 3800000, 4000000, 'PHONE,IMAGE', NULL, NULL),
|
||||
(4, 'US', 4, 3525000, 3600000, 'CW', 200, 'CW only'),
|
||||
(4, 'US', 5, 3525000, 3600000, 'CW', 200, 'CW only');
|
||||
|
||||
-- 60m: five discrete channels, General and above.
|
||||
-- Stored as channel-width ranges; centre frequency is the USB dial + 1.5 kHz.
|
||||
INSERT INTO band_segments (band_id, country, class_id, low_hz, high_hz, modes, max_power_w, notes) VALUES
|
||||
(5, 'US', 1, 5330500, 5333300, 'CW,DATA,PHONE', NULL, 'Channel 1, 100 W ERP, USB'),
|
||||
(5, 'US', 1, 5346500, 5349300, 'CW,DATA,PHONE', NULL, 'Channel 2, 100 W ERP, USB'),
|
||||
(5, 'US', 1, 5357000, 5359800, 'CW,DATA,PHONE', NULL, 'Channel 3, 100 W ERP, USB'),
|
||||
(5, 'US', 1, 5371500, 5374300, 'CW,DATA,PHONE', NULL, 'Channel 4, 100 W ERP, USB'),
|
||||
(5, 'US', 1, 5403500, 5406300, 'CW,DATA,PHONE', NULL, 'Channel 5, 100 W ERP, USB'),
|
||||
(5, 'US', 3, 5330500, 5333300, 'CW,DATA,PHONE', NULL, 'Channel 1, 100 W ERP, USB'),
|
||||
(5, 'US', 3, 5346500, 5349300, 'CW,DATA,PHONE', NULL, 'Channel 2, 100 W ERP, USB'),
|
||||
(5, 'US', 3, 5357000, 5359800, 'CW,DATA,PHONE', NULL, 'Channel 3, 100 W ERP, USB'),
|
||||
(5, 'US', 3, 5371500, 5374300, 'CW,DATA,PHONE', NULL, 'Channel 4, 100 W ERP, USB'),
|
||||
(5, 'US', 3, 5403500, 5406300, 'CW,DATA,PHONE', NULL, 'Channel 5, 100 W ERP, USB');
|
||||
|
||||
-- 40m
|
||||
INSERT INTO band_segments (band_id, country, class_id, low_hz, high_hz, modes, max_power_w, notes) VALUES
|
||||
(6, 'US', 1, 7000000, 7125000, 'CW,DATA', NULL, NULL),
|
||||
(6, 'US', 1, 7125000, 7300000, 'PHONE,IMAGE', NULL, NULL),
|
||||
(6, 'US', 2, 7025000, 7125000, 'CW,DATA', NULL, NULL),
|
||||
(6, 'US', 2, 7125000, 7300000, 'PHONE,IMAGE', NULL, NULL),
|
||||
(6, 'US', 3, 7025000, 7125000, 'CW,DATA', NULL, NULL),
|
||||
(6, 'US', 3, 7175000, 7300000, 'PHONE,IMAGE', NULL, NULL),
|
||||
(6, 'US', 4, 7025000, 7125000, 'CW', 200, 'CW only'),
|
||||
(6, 'US', 5, 7025000, 7125000, 'CW', 200, 'CW only');
|
||||
|
||||
-- 30m: no phone or image anywhere, 200 W limit
|
||||
INSERT INTO band_segments (band_id, country, class_id, low_hz, high_hz, modes, max_power_w, notes) VALUES
|
||||
(7, 'US', 1, 10100000, 10150000, 'CW,DATA', 200, 'No phone or image permitted'),
|
||||
(7, 'US', 2, 10100000, 10150000, 'CW,DATA', 200, 'No phone or image permitted'),
|
||||
(7, 'US', 3, 10100000, 10150000, 'CW,DATA', 200, 'No phone or image permitted');
|
||||
|
||||
-- 20m
|
||||
INSERT INTO band_segments (band_id, country, class_id, low_hz, high_hz, modes, max_power_w, notes) VALUES
|
||||
(8, 'US', 1, 14000000, 14150000, 'CW,DATA', NULL, NULL),
|
||||
(8, 'US', 1, 14150000, 14350000, 'PHONE,IMAGE', NULL, NULL),
|
||||
(8, 'US', 2, 14025000, 14150000, 'CW,DATA', NULL, NULL),
|
||||
(8, 'US', 2, 14175000, 14350000, 'PHONE,IMAGE', NULL, NULL),
|
||||
(8, 'US', 3, 14025000, 14150000, 'CW,DATA', NULL, NULL),
|
||||
(8, 'US', 3, 14225000, 14350000, 'PHONE,IMAGE', NULL, NULL);
|
||||
|
||||
-- 17m
|
||||
INSERT INTO band_segments (band_id, country, class_id, low_hz, high_hz, modes, max_power_w, notes) VALUES
|
||||
(9, 'US', 1, 18068000, 18110000, 'CW,DATA', NULL, NULL),
|
||||
(9, 'US', 1, 18110000, 18168000, 'PHONE,IMAGE', NULL, NULL),
|
||||
(9, 'US', 2, 18068000, 18110000, 'CW,DATA', NULL, NULL),
|
||||
(9, 'US', 2, 18110000, 18168000, 'PHONE,IMAGE', NULL, NULL),
|
||||
(9, 'US', 3, 18068000, 18110000, 'CW,DATA', NULL, NULL),
|
||||
(9, 'US', 3, 18110000, 18168000, 'PHONE,IMAGE', NULL, NULL);
|
||||
|
||||
-- 15m
|
||||
INSERT INTO band_segments (band_id, country, class_id, low_hz, high_hz, modes, max_power_w, notes) VALUES
|
||||
(10, 'US', 1, 21000000, 21200000, 'CW,DATA', NULL, NULL),
|
||||
(10, 'US', 1, 21200000, 21450000, 'PHONE,IMAGE', NULL, NULL),
|
||||
(10, 'US', 2, 21025000, 21200000, 'CW,DATA', NULL, NULL),
|
||||
(10, 'US', 2, 21225000, 21450000, 'PHONE,IMAGE', NULL, NULL),
|
||||
(10, 'US', 3, 21025000, 21200000, 'CW,DATA', NULL, NULL),
|
||||
(10, 'US', 3, 21275000, 21450000, 'PHONE,IMAGE', NULL, NULL),
|
||||
(10, 'US', 4, 21025000, 21200000, 'CW', 200, 'CW only'),
|
||||
(10, 'US', 5, 21025000, 21200000, 'CW', 200, 'CW only');
|
||||
|
||||
-- 12m
|
||||
INSERT INTO band_segments (band_id, country, class_id, low_hz, high_hz, modes, max_power_w, notes) VALUES
|
||||
(11, 'US', 1, 24890000, 24930000, 'CW,DATA', NULL, NULL),
|
||||
(11, 'US', 1, 24930000, 24990000, 'PHONE,IMAGE', NULL, NULL),
|
||||
(11, 'US', 2, 24890000, 24930000, 'CW,DATA', NULL, NULL),
|
||||
(11, 'US', 2, 24930000, 24990000, 'PHONE,IMAGE', NULL, NULL),
|
||||
(11, 'US', 3, 24890000, 24930000, 'CW,DATA', NULL, NULL),
|
||||
(11, 'US', 3, 24930000, 24990000, 'PHONE,IMAGE', NULL, NULL);
|
||||
|
||||
-- 10m: the only HF band with Technician phone privileges
|
||||
INSERT INTO band_segments (band_id, country, class_id, low_hz, high_hz, modes, max_power_w, notes) VALUES
|
||||
(12, 'US', 1, 28000000, 28300000, 'CW,DATA', NULL, NULL),
|
||||
(12, 'US', 1, 28300000, 29700000, 'PHONE,IMAGE', NULL, NULL),
|
||||
(12, 'US', 2, 28000000, 28300000, 'CW,DATA', NULL, NULL),
|
||||
(12, 'US', 2, 28300000, 29700000, 'PHONE,IMAGE', NULL, NULL),
|
||||
(12, 'US', 3, 28000000, 28300000, 'CW,DATA', NULL, NULL),
|
||||
(12, 'US', 3, 28300000, 29700000, 'PHONE,IMAGE', NULL, NULL),
|
||||
(12, 'US', 4, 28000000, 28300000, 'CW,DATA', 200, NULL),
|
||||
(12, 'US', 4, 28300000, 28500000, 'PHONE', 200, 'Technician phone privileges'),
|
||||
(12, 'US', 5, 28100000, 28300000, 'CW,DATA', 200, NULL),
|
||||
(12, 'US', 5, 28300000, 28500000, 'PHONE', 200, NULL);
|
||||
|
||||
-- VHF/UHF and above: Technician and higher, full allocations
|
||||
INSERT INTO band_segments (band_id, country, class_id, low_hz, high_hz, modes, max_power_w, notes) VALUES
|
||||
(13, 'US', 4, 50000000, 50100000, 'CW', NULL, 'CW only below 50.1 MHz'),
|
||||
(13, 'US', 4, 50100000, 54000000, 'CW,DATA,PHONE,IMAGE', NULL, NULL),
|
||||
(14, 'US', 4, 144000000, 144100000, 'CW', NULL, 'CW only below 144.1 MHz'),
|
||||
(14, 'US', 4, 144100000, 148000000, 'CW,DATA,PHONE,IMAGE', NULL, NULL),
|
||||
(15, 'US', 4, 222000000, 225000000, 'CW,DATA,PHONE,IMAGE', NULL, NULL),
|
||||
(16, 'US', 4, 420000000, 450000000, 'CW,DATA,PHONE,IMAGE', NULL, 'Geographic restrictions apply near some radar sites'),
|
||||
(17, 'US', 4, 902000000, 928000000, 'CW,DATA,PHONE,IMAGE', NULL, 'Secondary allocation, shared with Part 15 devices'),
|
||||
(18, 'US', 4, 1240000000, 1300000000, 'CW,DATA,PHONE,IMAGE', NULL, NULL);
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- IARU allocation extents (class_id NULL = the allocation, not privileges)
|
||||
-- Partial. Enough to answer "is 7.250 legal here" for Region 1 vs 2.
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
INSERT INTO band_segments (band_id, country, iaru_region, class_id, low_hz, high_hz, modes, notes) VALUES
|
||||
(3, '', 1, NULL, 1810000, 2000000, 'ALL', 'Varies by country within R1'),
|
||||
(4, '', 1, NULL, 3500000, 3800000, 'ALL', NULL),
|
||||
(6, '', 1, NULL, 7000000, 7200000, 'ALL', NULL),
|
||||
(8, '', 1, NULL, 14000000, 14350000, 'ALL', NULL),
|
||||
(10, '', 1, NULL, 21000000, 21450000, 'ALL', NULL),
|
||||
(12, '', 1, NULL, 28000000, 29700000, 'ALL', NULL),
|
||||
(14, '', 1, NULL, 144000000, 146000000, 'ALL', NULL),
|
||||
(16, '', 1, NULL, 430000000, 440000000, 'ALL', 'Varies by country within R1'),
|
||||
(4, '', 3, NULL, 3500000, 3900000, 'ALL', 'Varies by country within R3'),
|
||||
(6, '', 3, NULL, 7000000, 7200000, 'ALL', NULL),
|
||||
(14, '', 3, NULL, 144000000, 148000000, 'ALL', NULL);
|
||||
@@ -0,0 +1,171 @@
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Morse (ITU-R M.1677-1)
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
INSERT INTO morse (character, code, category) VALUES
|
||||
('A', '.-', 'letter'), ('B', '-...', 'letter'), ('C', '-.-.', 'letter'),
|
||||
('D', '-..', 'letter'), ('E', '.', 'letter'), ('F', '..-.', 'letter'),
|
||||
('G', '--.', 'letter'), ('H', '....', 'letter'), ('I', '..', 'letter'),
|
||||
('J', '.---', 'letter'), ('K', '-.-', 'letter'), ('L', '.-..', 'letter'),
|
||||
('M', '--', 'letter'), ('N', '-.', 'letter'), ('O', '---', 'letter'),
|
||||
('P', '.--.', 'letter'), ('Q', '--.-', 'letter'), ('R', '.-.', 'letter'),
|
||||
('S', '...', 'letter'), ('T', '-', 'letter'), ('U', '..-', 'letter'),
|
||||
('V', '...-', 'letter'), ('W', '.--', 'letter'), ('X', '-..-', 'letter'),
|
||||
('Y', '-.--', 'letter'), ('Z', '--..', 'letter'),
|
||||
('0', '-----', 'digit'), ('1', '.----', 'digit'), ('2', '..---', 'digit'),
|
||||
('3', '...--', 'digit'), ('4', '....-', 'digit'), ('5', '.....', 'digit'),
|
||||
('6', '-....', 'digit'), ('7', '--...', 'digit'), ('8', '---..', 'digit'),
|
||||
('9', '----.', 'digit'),
|
||||
('.', '.-.-.-', 'punctuation'), (',', '--..--', 'punctuation'),
|
||||
('?', '..--..', 'punctuation'), ('''', '.----.', 'punctuation'),
|
||||
('!', '-.-.--', 'punctuation'), ('/', '-..-.', 'punctuation'),
|
||||
('(', '-.--.', 'punctuation'), (')', '-.--.-', 'punctuation'),
|
||||
('&', '.-...', 'punctuation'), (':', '---...', 'punctuation'),
|
||||
(';', '-.-.-.', 'punctuation'), ('=', '-...-', 'punctuation'),
|
||||
('+', '.-.-.', 'punctuation'), ('-', '-....-', 'punctuation'),
|
||||
('_', '..--.-', 'punctuation'), ('"', '.-..-.', 'punctuation'),
|
||||
('$', '...-..-','punctuation'), ('@', '.--.-.', 'punctuation');
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- ITU/NATO phonetics
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
INSERT INTO phonetics (letter, word, pronunciation) VALUES
|
||||
('A', 'Alfa', 'AL-FAH'), ('B', 'Bravo', 'BRAH-VOH'),
|
||||
('C', 'Charlie', 'CHAR-LEE'), ('D', 'Delta', 'DELL-TAH'),
|
||||
('E', 'Echo', 'ECK-OH'), ('F', 'Foxtrot', 'FOKS-TROT'),
|
||||
('G', 'Golf', 'GOLF'), ('H', 'Hotel', 'HOH-TELL'),
|
||||
('I', 'India', 'IN-DEE-AH'), ('J', 'Juliett', 'JEW-LEE-ETT'),
|
||||
('K', 'Kilo', 'KEY-LOH'), ('L', 'Lima', 'LEE-MAH'),
|
||||
('M', 'Mike', 'MIKE'), ('N', 'November', 'NO-VEM-BER'),
|
||||
('O', 'Oscar', 'OSS-CAH'), ('P', 'Papa', 'PAH-PAH'),
|
||||
('Q', 'Quebec', 'KEH-BECK'), ('R', 'Romeo', 'ROW-ME-OH'),
|
||||
('S', 'Sierra', 'SEE-AIR-RAH'), ('T', 'Tango', 'TANG-GO'),
|
||||
('U', 'Uniform', 'YOU-NEE-FORM'),('V', 'Victor', 'VIK-TAH'),
|
||||
('W', 'Whiskey', 'WISS-KEY'), ('X', 'X-ray', 'ECKS-RAY'),
|
||||
('Y', 'Yankee', 'YANG-KEY'), ('Z', 'Zulu', 'ZOO-LOO'),
|
||||
('0', 'Zero', 'ZEE-RO'), ('1', 'One', 'WUN'),
|
||||
('2', 'Two', 'TOO'), ('3', 'Three', 'TREE'),
|
||||
('4', 'Four', 'FOW-ER'), ('5', 'Five', 'FIFE'),
|
||||
('6', 'Six', 'SIX'), ('7', 'Seven', 'SEV-EN'),
|
||||
('8', 'Eight', 'AIT'), ('9', 'Nine', 'NIN-ER');
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Q-codes (amateur-relevant subset)
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
INSERT INTO qcodes (code, question, answer, category) VALUES
|
||||
('QRA', 'What is the name of your station?', 'The name of my station is ...', 'general'),
|
||||
('QRG', 'Will you tell me my exact frequency?', 'Your exact frequency is ... kHz', 'general'),
|
||||
('QRK', 'What is the readability of my signals?', 'The readability of your signals is ... (1 to 5)', 'general'),
|
||||
('QRL', 'Are you busy?', 'I am busy, please do not interfere', 'general'),
|
||||
('QRM', 'Is my transmission being interfered with?', 'Your transmission is being interfered with', 'general'),
|
||||
('QRN', 'Are you troubled by static?', 'I am troubled by static', 'general'),
|
||||
('QRO', 'Shall I increase transmit power?', 'Increase transmit power', 'general'),
|
||||
('QRP', 'Shall I decrease transmit power?', 'Decrease transmit power', 'general'),
|
||||
('QRQ', 'Shall I send faster?', 'Send faster (... words per minute)', 'general'),
|
||||
('QRS', 'Shall I send more slowly?', 'Send more slowly (... words per minute)', 'general'),
|
||||
('QRT', 'Shall I stop sending?', 'Stop sending', 'general'),
|
||||
('QRU', 'Have you anything for me?', 'I have nothing for you', 'general'),
|
||||
('QRV', 'Are you ready?', 'I am ready', 'general'),
|
||||
('QRX', 'When will you call me again?', 'I will call you again at ...', 'general'),
|
||||
('QRZ', 'Who is calling me?', 'You are being called by ...', 'general'),
|
||||
('QSA', 'What is the strength of my signals?', 'The strength of your signals is ... (1 to 5)', 'general'),
|
||||
('QSB', 'Are my signals fading?', 'Your signals are fading', 'general'),
|
||||
('QSK', 'Can you hear me between your signals?', 'I can hear you between my signals', 'general'),
|
||||
('QSL', 'Can you acknowledge receipt?', 'I acknowledge receipt', 'general'),
|
||||
('QSO', 'Can you communicate with ... directly?', 'I can communicate with ... directly', 'general'),
|
||||
('QSP', 'Will you relay to ...?', 'I will relay to ...', 'general'),
|
||||
('QSY', 'Shall I change frequency?', 'Change frequency to ...', 'general'),
|
||||
('QTC', 'How many messages have you to send?', 'I have ... messages for you', 'general'),
|
||||
('QTH', 'What is your position?', 'My position is ...', 'general'),
|
||||
('QTR', 'What is the correct time?', 'The correct time is ...', 'general');
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Prosigns
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
INSERT INTO prosigns (symbol, morse, meaning, usage) VALUES
|
||||
('AR', '.-.-.', 'End of message', 'Sent at the end of a transmission to a specific station'),
|
||||
('AS', '.-...', 'Wait / stand by', 'Asking the other station to hold'),
|
||||
('BK', '-...-.-', 'Break', 'Interrupting to hand over quickly'),
|
||||
('BT', '-...-', 'Separator', 'Between paragraphs or sections; sent as a long dash'),
|
||||
('CL', '-.-..-..', 'Closing station', 'Going off the air entirely'),
|
||||
('CT', '-.-.-', 'Attention / start', 'Marks the beginning of a transmission'),
|
||||
('HH', '........', 'Error', 'Eight dits, meaning the last word was wrong'),
|
||||
('KN', '-.--.', 'Go ahead, named station only', 'Invites only the station being worked to reply'),
|
||||
('SK', '...-.-', 'End of contact', 'Final transmission of a QSO'),
|
||||
('SN', '...-.', 'Understood', 'Also written VE'),
|
||||
('SOS', '...---...', 'Distress', 'International distress signal; sent as one symbol');
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- CW abbreviations
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
INSERT INTO abbreviations (abbr, meaning, context) VALUES
|
||||
('73', 'Best regards', 'general'),
|
||||
('88', 'Love and kisses', 'general'),
|
||||
('ABT', 'About', 'CW'),
|
||||
('AGN', 'Again', 'CW'),
|
||||
('ANT', 'Antenna', 'CW'),
|
||||
('BURO', 'QSL bureau', 'general'),
|
||||
('CFM', 'Confirm', 'CW'),
|
||||
('CQ', 'Calling any station', 'general'),
|
||||
('CUL', 'See you later', 'CW'),
|
||||
('DE', 'From (this is)', 'CW'),
|
||||
('DX', 'Distance / long-distance station', 'general'),
|
||||
('ES', 'And', 'CW'),
|
||||
('FB', 'Fine business (excellent)', 'CW'),
|
||||
('GA', 'Good afternoon / go ahead', 'CW'),
|
||||
('GE', 'Good evening', 'CW'),
|
||||
('GM', 'Good morning', 'CW'),
|
||||
('HI', 'Laughter', 'CW'),
|
||||
('HW', 'How copy?', 'CW'),
|
||||
('K', 'Invitation to transmit', 'CW'),
|
||||
('OM', 'Old man (any operator)', 'CW'),
|
||||
('PSE', 'Please', 'CW'),
|
||||
('PWR', 'Power', 'CW'),
|
||||
('RIG', 'Station equipment', 'general'),
|
||||
('RPT', 'Repeat', 'CW'),
|
||||
('RST', 'Readability, strength, tone', 'general'),
|
||||
('RX', 'Receiver', 'general'),
|
||||
('SIG', 'Signal', 'CW'),
|
||||
('SK', 'Silent key (deceased operator)', 'general'),
|
||||
('SRI', 'Sorry', 'CW'),
|
||||
('TNX', 'Thanks', 'CW'),
|
||||
('TU', 'Thank you', 'CW'),
|
||||
('TX', 'Transmitter', 'general'),
|
||||
('UR', 'Your / you are', 'CW'),
|
||||
('VY', 'Very', 'CW'),
|
||||
('WX', 'Weather', 'CW'),
|
||||
('XYL', 'Wife', 'CW'),
|
||||
('YL', 'Young lady (female operator)','CW');
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- RST scale
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
INSERT INTO rst_scale (component, value, meaning) VALUES
|
||||
('R', 1, 'Unreadable'),
|
||||
('R', 2, 'Barely readable, occasional words distinguishable'),
|
||||
('R', 3, 'Readable with considerable difficulty'),
|
||||
('R', 4, 'Readable with practically no difficulty'),
|
||||
('R', 5, 'Perfectly readable'),
|
||||
('S', 1, 'Faint, signals barely perceptible'),
|
||||
('S', 2, 'Very weak'),
|
||||
('S', 3, 'Weak'),
|
||||
('S', 4, 'Fair'),
|
||||
('S', 5, 'Fairly good'),
|
||||
('S', 6, 'Good'),
|
||||
('S', 7, 'Moderately strong'),
|
||||
('S', 8, 'Strong'),
|
||||
('S', 9, 'Extremely strong'),
|
||||
('T', 1, 'Extremely rough hissing note'),
|
||||
('T', 2, 'Very rough AC note, no trace of musicality'),
|
||||
('T', 3, 'Rough, low-pitched AC note, slightly musical'),
|
||||
('T', 4, 'Rather rough AC note, moderately musical'),
|
||||
('T', 5, 'Musically modulated note'),
|
||||
('T', 6, 'Modulated note, slight trace of whistle'),
|
||||
('T', 7, 'Near DC note, smooth ripple'),
|
||||
('T', 8, 'Good DC note, trace of ripple'),
|
||||
('T', 9, 'Purest DC note');
|
||||
+1962
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,57 @@
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Coax
|
||||
--
|
||||
-- Impedance, velocity factor and capacitance are nominal published figures.
|
||||
-- loss_k1/loss_k2 are intentionally NULL: matched loss varies enough between
|
||||
-- manufacturers of nominally the same cable that a generic approximation would
|
||||
-- give the calculator false precision. Populate from the datasheet of the cable
|
||||
-- you actually mean.
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
INSERT INTO coax_types (name, impedance_ohm, velocity_factor, capacitance_pf_per_ft, outer_diameter_mm, loss_k1, loss_k2, notes) VALUES
|
||||
('RG-58C/U', 50.0, 0.66, 30.8, 4.95, NULL, NULL, 'Thin, lossy, common for short VHF jumpers'),
|
||||
('RG-8X', 50.0, 0.82, 25.0, 6.10, NULL, NULL, 'Foam dielectric, a middle ground'),
|
||||
('RG-213/U', 50.0, 0.66, 30.8, 10.30, NULL, NULL, 'Workhorse HF cable'),
|
||||
('RG-214/U', 50.0, 0.66, 30.8, 10.80, NULL, NULL, 'Double-shielded RG-213 equivalent'),
|
||||
('LMR-400', 50.0, 0.85, 23.9, 10.29, NULL, NULL, 'Low loss, common for VHF/UHF runs'),
|
||||
('LMR-600', 50.0, 0.87, 23.4, 14.99, NULL, NULL, 'Lower loss, stiffer'),
|
||||
('RG-6/U', 75.0, 0.83, 16.2, 6.90, NULL, NULL, '75 ohm, cheap, fine for receive'),
|
||||
('RG-11/U', 75.0, 0.66, 20.6, 10.30, NULL, NULL, '75 ohm, lower loss than RG-6'),
|
||||
('RG-174/U', 50.0, 0.66, 30.8, 2.55, NULL, NULL, 'Very thin, very lossy, patch use only'),
|
||||
('Belden 9913', 50.0, 0.84, 24.6, 10.30, NULL, NULL, 'Air dielectric, needs care against water ingress'),
|
||||
('Hardline LDF4-50A', 50.0, 0.88, 25.9, 15.90, NULL, NULL, '1/2 inch corrugated hardline');
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- NCDXF/IARU beacons
|
||||
--
|
||||
-- 18 stations, one 10 s slot each, 180 s for a full cycle per band.
|
||||
-- Station transmitting on a given band at UTC time t:
|
||||
-- slot = (floor(t / 10) - slot_offset) mod 18
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
INSERT INTO ncdxf_beacons (slot_index, callsign, location, grid, latitude, longitude, dxcc_id) VALUES
|
||||
(0, '4U1UN', 'United Nations, New York', 'FN30AS', 40.75, -73.97, NULL),
|
||||
(1, 'VE8AT', 'Inuvik, NT, Canada', 'CP38GJ', 68.38, -133.72, 1),
|
||||
(2, 'W6WX', 'Mt Umunhum, CA, USA', 'CM97BD', 37.16, -121.90, 291),
|
||||
(3, 'KH6RS', 'Maui, Hawaii', 'BL10TS', 20.79, -156.46, NULL),
|
||||
(4, 'ZL6B', 'Masterton, New Zealand', 'RE78TW', -40.92, 175.61, 170),
|
||||
(5, 'VK6RBP', 'Rolystone, WA, Australia', 'OF87AV', -32.11, 116.05, 150),
|
||||
(6, 'JA2IGY', 'Mt Asama, Japan', 'PM84JK', 34.45, 136.79, 339),
|
||||
(7, 'RR9O', 'Novosibirsk, Russia', 'NO14KX', 54.98, 82.89, NULL),
|
||||
(8, 'VR2B', 'Hong Kong', 'OL72BG', 22.28, 114.16, NULL),
|
||||
(9, '4S7B', 'Colombo, Sri Lanka', 'MJ96WV', 6.91, 79.87, NULL),
|
||||
(10, 'ZS6DN', 'Pretoria, South Africa', 'KG33XI', -25.90, 28.26, 462),
|
||||
(11, '5Z4B', 'Kariobangi, Kenya', 'KI88HR', -1.24, 36.89, NULL),
|
||||
(12, '4X6TU', 'Tel Aviv, Israel', 'KM72JB', 32.05, 34.78, NULL),
|
||||
(13, 'OH2B', 'Lohja, Finland', 'KP20EH', 60.25, 24.40, 224),
|
||||
(14, 'CS3B', 'Madeira', 'IM12JT', 32.72, -16.99, NULL),
|
||||
(15, 'LU4AA', 'Buenos Aires, Argentina', 'GF05TJ', -34.62, -58.37, 100),
|
||||
(16, 'OA4B', 'Lima, Peru', 'FH17MW', -12.07, -77.03, NULL),
|
||||
(17, 'YV5B', 'Caracas, Venezuela', 'FJ69CC', 10.50, -66.92, NULL);
|
||||
|
||||
INSERT INTO ncdxf_frequencies (band_id, freq_hz, slot_offset) VALUES
|
||||
(8, 14100000, 0),
|
||||
(9, 18110000, 1),
|
||||
(10, 21150000, 2),
|
||||
(11, 24930000, 3),
|
||||
(12, 28200000, 4);
|
||||
+3661
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,13 @@
|
||||
-- Generated by audit.py --indexes. Runs last in the build.
|
||||
-- The bundle is read-only, so indexes cost file size and nothing else.
|
||||
|
||||
-- candidate-prefix equality lookup; leading PK column, but stated explicitly because everything depends on it
|
||||
CREATE INDEX IF NOT EXISTS idx_dxcc_prefixes_prefix ON "dxcc_prefixes" ("prefix");
|
||||
-- joins back to dxcc_entities
|
||||
CREATE INDEX IF NOT EXISTS idx_dxcc_prefixes_entity_id ON "dxcc_prefixes" ("entity_id");
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
ANALYZE;
|
||||
Executable
+342
@@ -0,0 +1,342 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
adif2sql.py -- fetch the current ADIF release's exported data files and turn the
|
||||
enumerations into SQL for the Hammy reference bundle.
|
||||
|
||||
Usage:
|
||||
python3 adif2sql.py --list # show what the zip contains
|
||||
python3 adif2sql.py -o 07-adif.sql # fetch, parse, emit SQL
|
||||
python3 adif2sql.py --zip local.zip --list # work from an already-downloaded zip
|
||||
python3 adif2sql.py --version 316 -o out.sql
|
||||
|
||||
How the site works
|
||||
------------------
|
||||
* https://adif.org.uk/adiflatestrelease.txt returns the release as three
|
||||
digits, e.g. "317" meaning ADIF 3.1.7.
|
||||
* Most filenames can be constructed from that: 317/adx317.xsd and so on.
|
||||
* The resources zip CANNOT: its name embeds a release date that is not derivable
|
||||
from the version number (ADIF_317_resources_2026_03_22.zip). So this script
|
||||
scrapes /317/index.htm to find it rather than guessing.
|
||||
* The hosting provider blocks unrecognised User-Agent strings. Python's default
|
||||
("Python-urllib/3.x") is one of the blocked ones, so a UA is set explicitly
|
||||
below. Do not remove it.
|
||||
|
||||
Be polite: the ADIF site asks that applications download and keep local copies
|
||||
rather than fetching on every run. This script is a build-time tool, not a
|
||||
runtime dependency - run it when you cut a new bundle, commit the output.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import io
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from datetime import date, timezone
|
||||
import zipfile
|
||||
|
||||
BASE = "https://adif.org.uk"
|
||||
LATEST_URL = BASE + "/adiflatestrelease.txt"
|
||||
|
||||
# The hosting provider blocks Python's default UA. Any plausible string works;
|
||||
# this one identifies the tool honestly, which is the polite option.
|
||||
USER_AGENT = "Hammy-refbundle/1.0 (+https://hammybot.org)"
|
||||
|
||||
RETRIEVED = date.today().isoformat()
|
||||
|
||||
# The resources zip ships every enumeration in six formats:
|
||||
# exports/{csv,json,ods,tsv,xlsx,xml}/enumerations_<name>.<ext>
|
||||
# Only one text format is wanted. ods and xlsx are themselves zip archives and
|
||||
# json/xml are not delimited, so feeding them to a CSV reader produces garbage.
|
||||
FORMATS = ("tsv", "csv")
|
||||
|
||||
# Members are matched with an anchored pattern rather than a substring test.
|
||||
# Substring matching collapsed enumerations_mode, enumerations_submode and
|
||||
# enumerations_propagation_mode into one table.
|
||||
def member_pattern(fmt):
|
||||
return re.compile(
|
||||
rf"(?:^|/)exports/{fmt}/enumerations_(?P<name>[A-Za-z0-9_]+)\.{fmt}$",
|
||||
re.IGNORECASE)
|
||||
|
||||
|
||||
# Optional prettier names. Anything not listed becomes adif_<enumeration_name>.
|
||||
TABLE_ALIASES = {
|
||||
"dxcc_entity_code": "adif_dxcc",
|
||||
"primary_administrative_subdivision": "adif_subdivisions",
|
||||
"secondary_administrative_subdivision": "adif_subdivisions_secondary",
|
||||
"secondary_administrative_subdivision_alt": "adif_subdivisions_secondary_alt",
|
||||
}
|
||||
|
||||
|
||||
def table_for(enum_name):
|
||||
key = enum_name.lower()
|
||||
|
||||
return TABLE_ALIASES.get(key, "adif_" + key)
|
||||
|
||||
|
||||
def fetch(url, binary=False):
|
||||
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=60) as resp:
|
||||
data = resp.read()
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code == 403:
|
||||
sys.exit(f"403 from {url} - the User-Agent is being blocked. "
|
||||
f"Current UA: {USER_AGENT!r}")
|
||||
raise
|
||||
|
||||
return data if binary else data.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def latest_version():
|
||||
return fetch(LATEST_URL).strip()
|
||||
|
||||
|
||||
def find_resources_zip(version):
|
||||
"""Scrape the version index for the resources zip, whose name carries a date."""
|
||||
index = fetch(f"{BASE}/{version}/index.htm")
|
||||
|
||||
m = re.search(rf"ADIF_{version}_resources[_\d]*\.zip", index, re.IGNORECASE)
|
||||
if not m:
|
||||
sys.exit(f"no resources zip linked from {BASE}/{version}/index.htm - "
|
||||
f"the page layout may have changed, check it by hand")
|
||||
|
||||
return f"{BASE}/{version}/{m.group(0)}"
|
||||
|
||||
|
||||
def load_zip(path_or_url):
|
||||
if os.path.exists(path_or_url):
|
||||
return zipfile.ZipFile(path_or_url)
|
||||
|
||||
return zipfile.ZipFile(io.BytesIO(fetch(path_or_url, binary=True)))
|
||||
|
||||
|
||||
def sniff_rows(zf, name):
|
||||
"""Read a member as delimited text, guessing the delimiter."""
|
||||
raw = zf.read(name).decode("utf-8-sig", errors="replace")
|
||||
|
||||
# ADIF's exports ship with CRLF. io.StringIO translates newlines by default,
|
||||
# which leaves the \r attached to the last field of every row and makes csv
|
||||
# raise "new-line character seen in unquoted field". Normalise first, then
|
||||
# open with newline="" so csv does its own line splitting.
|
||||
raw = raw.replace("\r\n", "\n").replace("\r", "\n")
|
||||
|
||||
first_line = raw.split("\n", 1)[0]
|
||||
|
||||
# Prefer an explicit tab check over Sniffer. These files are tab-separated
|
||||
# and their description columns contain commas, which Sniffer sometimes
|
||||
# mistakes for the delimiter.
|
||||
if "\t" in first_line:
|
||||
dialect = csv.excel_tab
|
||||
else:
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(raw[:4096], delimiters="\t,;")
|
||||
except csv.Error:
|
||||
dialect = csv.excel_tab
|
||||
|
||||
reader = csv.reader(io.StringIO(raw, newline=""), dialect)
|
||||
|
||||
try:
|
||||
rows = [r for r in reader if any(cell.strip() for cell in r)]
|
||||
except csv.Error as exc:
|
||||
# Not a delimited file at all - a .adi test QSO file, say. Skip it
|
||||
# rather than taking the whole run down.
|
||||
sys.stderr.write(f"skipping {name}: {exc}\n")
|
||||
return [], []
|
||||
|
||||
if not rows:
|
||||
return [], []
|
||||
|
||||
return rows[0], rows[1:]
|
||||
|
||||
|
||||
def ident(name):
|
||||
"""Turn an arbitrary header cell into a safe SQL column name."""
|
||||
col = re.sub(r"[^0-9a-zA-Z]+", "_", name.strip().lower()).strip("_")
|
||||
|
||||
return col or "col"
|
||||
|
||||
|
||||
def q(value):
|
||||
if value is None or value == "":
|
||||
return "NULL"
|
||||
|
||||
return "'" + str(value).replace("'", "''") + "'"
|
||||
|
||||
|
||||
# Pure ADIF bookkeeping: identical on every row of every file, and recorded once
|
||||
# in ref_sources instead. Everything else is kept even when constant, because a
|
||||
# constant can be meaningful - adif_award.import_only is 'Import-only' on all 29
|
||||
# rows, which says something real about those awards.
|
||||
METADATA_COLUMNS = {"enumeration_name", "adif_version", "adif_status"}
|
||||
|
||||
|
||||
def prune_columns(cols, rows, keep_all=False):
|
||||
"""Drop ADIF bookkeeping and columns that are empty in this file.
|
||||
|
||||
Returns (kept_column_names, kept_rows, dropped_report).
|
||||
"""
|
||||
if keep_all:
|
||||
return cols, rows, []
|
||||
|
||||
keep, dropped = [], []
|
||||
|
||||
for i, c in enumerate(cols):
|
||||
values = {(r[i].strip() if i < len(r) and r[i] is not None else "") for r in rows}
|
||||
|
||||
# Compare on the normalised name: headers arrive as "Enumeration Name",
|
||||
# "ADIF Version" and so on, and are only identifier-ised later.
|
||||
if ident(c) in METADATA_COLUMNS:
|
||||
sample = next(iter(values)) if len(values) == 1 else None
|
||||
dropped.append((c, f"metadata{'=' + sample if sample else ''}"))
|
||||
continue
|
||||
|
||||
if values <= {""}:
|
||||
dropped.append((c, "empty"))
|
||||
continue
|
||||
|
||||
keep.append(i)
|
||||
|
||||
kept_cols = [cols[i] for i in keep]
|
||||
kept_rows = [[(r[i] if i < len(r) else "") for i in keep] for r in rows]
|
||||
|
||||
return kept_cols, kept_rows, dropped
|
||||
|
||||
|
||||
def emit_table(table, header, rows, out):
|
||||
cols = []
|
||||
seen = {}
|
||||
for h in header:
|
||||
c = ident(h)
|
||||
if c in seen:
|
||||
seen[c] += 1
|
||||
c = f"{c}_{seen[c]}"
|
||||
else:
|
||||
seen[c] = 0
|
||||
cols.append(c)
|
||||
|
||||
out.write(f"\nDROP TABLE IF EXISTS {table};\n")
|
||||
out.write(f"CREATE TABLE {table} (\n")
|
||||
out.write(",\n".join(f" {c} TEXT" for c in cols))
|
||||
out.write("\n);\n\n")
|
||||
|
||||
out.write(f"INSERT INTO {table} ({', '.join(cols)}) VALUES\n")
|
||||
|
||||
lines = []
|
||||
for r in rows:
|
||||
# Pad or trim to the header width; ADIF exports occasionally have
|
||||
# ragged trailing columns.
|
||||
r = (list(r) + [""] * len(cols))[:len(cols)]
|
||||
lines.append(" (" + ", ".join(q(cell.strip()) for cell in r) + ")")
|
||||
|
||||
out.write(",\n".join(lines) + ";\n")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--version", help="3-digit ADIF version (default: whatever is current)")
|
||||
ap.add_argument("--zip", dest="zip_path", help="use a local zip instead of downloading")
|
||||
ap.add_argument("--list", action="store_true", help="list zip contents and exit")
|
||||
ap.add_argument("-o", "--output", help="write SQL here (default: stdout)")
|
||||
ap.add_argument("--format", choices=FORMATS, default="tsv",
|
||||
help="which export format to read (default: tsv). The zip also "
|
||||
"contains json, xml, ods and xlsx copies, which are not "
|
||||
"delimited text and are ignored")
|
||||
ap.add_argument("--keep-all-columns", action="store_true",
|
||||
help="keep ADIF bookkeeping columns (enumeration_name, "
|
||||
"adif_version, adif_status) and columns that are empty "
|
||||
"in this release")
|
||||
ap.add_argument("--only", nargs="+", metavar="ENUM",
|
||||
help="import just these enumerations by name, e.g. "
|
||||
"--only mode band dxcc_entity_code")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.zip_path:
|
||||
source = args.zip_path
|
||||
version = args.version or "local"
|
||||
else:
|
||||
version = args.version or latest_version()
|
||||
sys.stderr.write(f"ADIF version {version}\n")
|
||||
source = find_resources_zip(version)
|
||||
sys.stderr.write(f"resources: {source}\n")
|
||||
|
||||
zf = load_zip(source)
|
||||
members = [n for n in zf.namelist() if not n.endswith("/")]
|
||||
|
||||
if args.list:
|
||||
for n in sorted(members):
|
||||
info = zf.getinfo(n)
|
||||
print(f" {info.file_size:9d} {n}")
|
||||
print(f"\n{len(members)} files")
|
||||
|
||||
return 0
|
||||
|
||||
pat = member_pattern(args.format)
|
||||
only = {o.lower() for o in args.only} if args.only else None
|
||||
|
||||
picked = []
|
||||
for n in members:
|
||||
m = pat.search(n)
|
||||
if not m:
|
||||
continue
|
||||
|
||||
enum_name = m.group("name").lower()
|
||||
if only and enum_name not in only:
|
||||
continue
|
||||
|
||||
picked.append((n, enum_name, table_for(enum_name)))
|
||||
|
||||
picked.sort(key=lambda x: x[2])
|
||||
|
||||
if not picked:
|
||||
sys.exit(f"no exports/{args.format}/enumerations_*.{args.format} members found. "
|
||||
f"Run with --list to see the zip layout.")
|
||||
|
||||
# One member per table, or a later file silently clobbers an earlier one.
|
||||
by_table = {}
|
||||
for n, enum_name, table in picked:
|
||||
if table in by_table:
|
||||
sys.exit(f"{table} claimed by both {by_table[table]} and {n} - "
|
||||
f"add an entry to TABLE_ALIASES to disambiguate")
|
||||
by_table[table] = n
|
||||
|
||||
out = open(args.output, "w", encoding="utf-8") if args.output else sys.stdout
|
||||
try:
|
||||
out.write("-- Generated by adif2sql.py. Do not edit by hand.\n")
|
||||
out.write(f"-- ADIF version {version}, {args.format} exports, from {source}\n")
|
||||
out.write("-- Columns are all TEXT: these are enumerations, and ADIF's own\n")
|
||||
out.write("-- exports carry version-dependent extra columns. Cast at query time.\n")
|
||||
|
||||
for name, enum_name, table in picked:
|
||||
header, rows = sniff_rows(zf, name)
|
||||
if not header:
|
||||
sys.stderr.write(f"skipping empty {name}\n")
|
||||
continue
|
||||
|
||||
header, rows, dropped = prune_columns(header, rows, args.keep_all_columns)
|
||||
|
||||
note = f" (-{len(dropped)} cols)" if dropped else ""
|
||||
sys.stderr.write(f"{enum_name:44} -> {table:32} {len(rows):5d} rows{note}\n")
|
||||
|
||||
emit_table(table, header, rows, out)
|
||||
|
||||
# Provenance, so audit.py stops complaining about undocumented tables.
|
||||
out.write(
|
||||
"\nINSERT OR REPLACE INTO ref_sources "
|
||||
"(dataset, source_name, source_url, license, retrieved, notes) VALUES\n"
|
||||
f" ({q(table)}, {q('ADIF ' + version + ' ' + enum_name)}, "
|
||||
f"{q(source)}, 'ADIF specification, openly published', "
|
||||
f"{q(RETRIEVED)}, "
|
||||
f"{q('Generated by adif2sql.py. Dropped columns: ' + (', '.join(c for c, _ in dropped) or 'none'))});\n")
|
||||
finally:
|
||||
if args.output:
|
||||
out.close()
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+451
@@ -0,0 +1,451 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
audit.py -- consistency checks for the Hammy reference bundle.
|
||||
|
||||
Usage:
|
||||
python3 audit.py hammy-ref.sqlite # run every check
|
||||
python3 audit.py hammy-ref.sqlite --quiet # only failures
|
||||
python3 audit.py hammy-ref.sqlite --indexes # emit CREATE INDEX DDL
|
||||
|
||||
Exits non-zero if any check fails, so it drops straight into CI.
|
||||
|
||||
The checks fall into three groups:
|
||||
|
||||
structural SQLite's own integrity and declared foreign keys.
|
||||
relational Joins that SHOULD hold but are not declared as foreign keys,
|
||||
mostly because the ADIF tables are generated with TEXT columns.
|
||||
Orphans here mean two datasets disagree.
|
||||
domain Amateur-radio specific invariants. Band edges that cross, gaps in
|
||||
the beacon cycle, prefixes that resolve to nothing. These are the
|
||||
ones that catch a bad hand-entered row.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sqlite3
|
||||
import sys
|
||||
|
||||
FAILURES = []
|
||||
WARNINGS = []
|
||||
|
||||
|
||||
def fail(check, detail):
|
||||
FAILURES.append((check, detail))
|
||||
|
||||
|
||||
def warn(check, detail):
|
||||
WARNINGS.append((check, detail))
|
||||
|
||||
|
||||
def tables(db):
|
||||
return [r[0] for r in db.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' "
|
||||
"AND name NOT LIKE 'sqlite_%' ORDER BY name")]
|
||||
|
||||
|
||||
def columns(db, table):
|
||||
return [r[1] for r in db.execute(f"PRAGMA table_info({table})")]
|
||||
|
||||
|
||||
def indexed_columns(db, table):
|
||||
"""Columns usable as the LEADING column of some index or the primary key.
|
||||
|
||||
In a composite key (prefix, entity_id) only 'prefix' is reachable; a query
|
||||
filtering on entity_id alone still scans. PRAGMA table_info's pk field is
|
||||
the 1-based position within the key, so pk == 1 is the leading column.
|
||||
"""
|
||||
covered = set()
|
||||
|
||||
for row in db.execute(f"PRAGMA table_info({table})"):
|
||||
if row[5] == 1:
|
||||
covered.add(row[1])
|
||||
|
||||
for idx in db.execute(f"PRAGMA index_list({table})"):
|
||||
info = list(db.execute(f"PRAGMA index_info({idx[1]})"))
|
||||
if info:
|
||||
covered.add(info[0][2])
|
||||
|
||||
return covered
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Structural
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def check_structural(db, verbose):
|
||||
result = db.execute("PRAGMA integrity_check").fetchone()[0]
|
||||
if result != "ok":
|
||||
fail("integrity_check", result)
|
||||
elif verbose:
|
||||
print(" integrity_check ok")
|
||||
|
||||
violations = db.execute("PRAGMA foreign_key_check").fetchall()
|
||||
if violations:
|
||||
for v in violations[:10]:
|
||||
fail("foreign_key_check", f"{v[0]} rowid {v[1]} -> {v[2]}")
|
||||
elif verbose:
|
||||
print(" foreign_key_check ok")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Relational: undeclared joins that should still hold
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# (child table, child column, parent table, parent column, description)
|
||||
RELATIONS = [
|
||||
("dxcc_prefixes", "entity_id", "dxcc_entities", "id",
|
||||
"every prefix resolves to an entity"),
|
||||
("band_segments", "band_id", "bands", "id",
|
||||
"every segment belongs to a band"),
|
||||
("band_segments", "class_id", "license_classes", "id",
|
||||
"every segment's licence class exists"),
|
||||
("ncdxf_beacons", "dxcc_id", "dxcc_entities", "id",
|
||||
"beacon entities resolve"),
|
||||
("ncdxf_frequencies", "band_id", "bands", "id",
|
||||
"beacon frequencies map to a band"),
|
||||
# ADIF tables are all TEXT, so these need a CAST to join against integers.
|
||||
("adif_subdivisions", "dxcc_entity_code", "adif_dxcc", "entity_code",
|
||||
"subdivisions point at a real ADIF entity"),
|
||||
]
|
||||
|
||||
|
||||
def check_relations(db, verbose):
|
||||
present = set(tables(db))
|
||||
|
||||
for child, ccol, parent, pcol, desc in RELATIONS:
|
||||
if child not in present or parent not in present:
|
||||
continue
|
||||
if ccol not in columns(db, child) or pcol not in columns(db, parent):
|
||||
warn(f"{child}.{ccol}", f"column missing, skipped ({desc})")
|
||||
continue
|
||||
|
||||
# TRIM both sides: the ADIF exports pad some cells.
|
||||
q = (f"SELECT COUNT(*) FROM {child} c "
|
||||
f"WHERE c.{ccol} IS NOT NULL AND TRIM(c.{ccol}) <> '' "
|
||||
f"AND NOT EXISTS (SELECT 1 FROM {parent} p "
|
||||
f" WHERE TRIM(p.{pcol}) = TRIM(c.{ccol}))")
|
||||
orphans = db.execute(q).fetchone()[0]
|
||||
|
||||
if orphans:
|
||||
sample = db.execute(
|
||||
f"SELECT DISTINCT c.{ccol} FROM {child} c "
|
||||
f"WHERE NOT EXISTS (SELECT 1 FROM {parent} p "
|
||||
f"WHERE TRIM(p.{pcol}) = TRIM(c.{ccol})) "
|
||||
f"AND TRIM(c.{ccol}) <> '' LIMIT 5").fetchall()
|
||||
vals = ", ".join(repr(s[0]) for s in sample)
|
||||
fail(f"{child}.{ccol} -> {parent}.{pcol}",
|
||||
f"{orphans} orphan rows ({desc}); e.g. {vals}")
|
||||
elif verbose:
|
||||
print(f" {child}.{ccol} -> {parent}.{pcol}".ljust(66) + "ok")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Domain invariants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def check_domain(db, verbose):
|
||||
present = set(tables(db))
|
||||
|
||||
def q1(sql, *args):
|
||||
return db.execute(sql, args).fetchone()[0]
|
||||
|
||||
if "band_segments" in present:
|
||||
n = q1("SELECT COUNT(*) FROM band_segments WHERE low_hz >= high_hz")
|
||||
if n:
|
||||
fail("band_segments", f"{n} rows where low_hz >= high_hz")
|
||||
elif verbose:
|
||||
print(" band_segments edges ordered".ljust(66) + "ok")
|
||||
|
||||
# A segment outside its own band's extent is almost always a typo.
|
||||
rows = db.execute("""
|
||||
SELECT b.name, s.low_hz, s.high_hz, b.edge_low_hz, b.edge_high_hz
|
||||
FROM band_segments s JOIN bands b ON b.id = s.band_id
|
||||
WHERE s.low_hz < b.edge_low_hz OR s.high_hz > b.edge_high_hz""").fetchall()
|
||||
if rows:
|
||||
for r in rows[:5]:
|
||||
fail("band_segments", f"{r[0]} segment {r[1]}-{r[2]} outside band {r[3]}-{r[4]}")
|
||||
elif verbose:
|
||||
print(" band_segments within band edges".ljust(66) + "ok")
|
||||
|
||||
# Two segments for the same country+region+class+mode should not overlap.
|
||||
# iaru_region must be part of the key: the Region 1 and Region 3
|
||||
# allocation rows share country='' and legitimately cover the same
|
||||
# frequencies.
|
||||
rows = db.execute("""
|
||||
SELECT a.country, a.iaru_region, a.class_id, a.modes,
|
||||
a.low_hz, a.high_hz, b.low_hz, b.high_hz
|
||||
FROM band_segments a JOIN band_segments b
|
||||
ON a.id < b.id AND a.country = b.country
|
||||
AND IFNULL(a.iaru_region,-1) = IFNULL(b.iaru_region,-1)
|
||||
AND IFNULL(a.class_id,-1) = IFNULL(b.class_id,-1)
|
||||
AND a.modes = b.modes
|
||||
AND a.low_hz < b.high_hz AND b.low_hz < a.high_hz""").fetchall()
|
||||
if rows:
|
||||
for r in rows[:5]:
|
||||
warn("band_segments", f"{r[0] or 'IARU R' + str(r[1])} class {r[2]} {r[3]}: "
|
||||
f"{r[4]}-{r[5]} overlaps {r[6]}-{r[7]}")
|
||||
elif verbose:
|
||||
print(" band_segments no overlaps".ljust(66) + "ok")
|
||||
|
||||
if "ncdxf_beacons" in present:
|
||||
slots = [r[0] for r in db.execute("SELECT slot_index FROM ncdxf_beacons ORDER BY slot_index")]
|
||||
if slots != list(range(18)):
|
||||
fail("ncdxf_beacons", f"expected slots 0..17, got {len(slots)}: {slots}")
|
||||
elif verbose:
|
||||
print(" ncdxf_beacons slots 0..17 complete".ljust(66) + "ok")
|
||||
|
||||
if "dxcc_prefixes" in present:
|
||||
# Longest-prefix matching breaks if a prefix is empty or has whitespace.
|
||||
n = q1("SELECT COUNT(*) FROM dxcc_prefixes WHERE prefix IS NULL OR TRIM(prefix) <> prefix OR prefix = ''")
|
||||
if n:
|
||||
fail("dxcc_prefixes", f"{n} prefixes empty or with surrounding whitespace")
|
||||
elif verbose:
|
||||
print(" dxcc_prefixes clean".ljust(66) + "ok")
|
||||
|
||||
# Zone overrides that merely restate the entity default are noise.
|
||||
n = q1("""SELECT COUNT(*) FROM dxcc_prefixes p JOIN dxcc_entities e ON e.id = p.entity_id
|
||||
WHERE p.cq_zone = e.cq_zone AND p.itu_zone = e.itu_zone""")
|
||||
if n:
|
||||
warn("dxcc_prefixes", f"{n} rows whose zone override equals the entity default")
|
||||
|
||||
if "morse" in present:
|
||||
n = q1("SELECT COUNT(*) FROM (SELECT code FROM morse GROUP BY code HAVING COUNT(*) > 1)")
|
||||
if n:
|
||||
fail("morse", f"{n} duplicate codes - decoding would be ambiguous")
|
||||
elif verbose:
|
||||
print(" morse codes unique".ljust(66) + "ok")
|
||||
|
||||
if "ref_sources" in present:
|
||||
# Every populated table should say where it came from.
|
||||
documented = {r[0] for r in db.execute("SELECT dataset FROM ref_sources")}
|
||||
undocumented = []
|
||||
for t in tables(db):
|
||||
if t.startswith(("ref_", "sqlite_")):
|
||||
continue
|
||||
if q1(f"SELECT COUNT(*) FROM {t}") == 0:
|
||||
continue
|
||||
if t not in documented and not any(d in t or t in d for d in documented):
|
||||
undocumented.append(t)
|
||||
if undocumented:
|
||||
warn("ref_sources", f"no provenance row for: {', '.join(undocumented)}")
|
||||
elif verbose:
|
||||
print(" ref_sources covers every table".ljust(66) + "ok")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dead weight
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def check_dead_columns(db, verbose, group=True):
|
||||
"""Constant or empty columns.
|
||||
|
||||
A constant column is not automatically a bug: adif_award.import_only is
|
||||
'Import-only' on every row because every award in that enumeration is, and
|
||||
adif_subdivisions_secondary.dxcc_entity_code is '6' because Alaska is the
|
||||
only entity with secondary subdivisions. Empty columns are always dead
|
||||
weight. Both are reported, but repeated findings across many tables collapse
|
||||
into one line so the signal is not buried.
|
||||
"""
|
||||
empties = []
|
||||
constants = []
|
||||
|
||||
for t in tables(db):
|
||||
total = db.execute(f"SELECT COUNT(*) FROM {t}").fetchone()[0]
|
||||
if total < 2:
|
||||
continue
|
||||
|
||||
for c in columns(db, t):
|
||||
distinct = db.execute(
|
||||
f'SELECT COUNT(DISTINCT IFNULL("{c}", char(0))) FROM "{t}"').fetchone()[0]
|
||||
|
||||
if distinct != 1:
|
||||
continue
|
||||
|
||||
val = db.execute(f'SELECT "{c}" FROM "{t}" LIMIT 1').fetchone()[0]
|
||||
|
||||
if val is None or str(val).strip() == "":
|
||||
empties.append((t, c, total))
|
||||
else:
|
||||
constants.append((t, c, val, total))
|
||||
|
||||
if group:
|
||||
# Collapse by column name: the same finding across 20 ADIF tables is one
|
||||
# fact about the export format, not 20 problems.
|
||||
by_col = {}
|
||||
for t, c, total in empties:
|
||||
by_col.setdefault(c, []).append(t)
|
||||
for c, ts in sorted(by_col.items()):
|
||||
if len(ts) > 2:
|
||||
warn(f"*.{c}", f"entirely empty in {len(ts)} tables - drop it "
|
||||
f"({', '.join(ts[:3])}, ...)")
|
||||
else:
|
||||
for t in ts:
|
||||
warn(f"{t}.{c}", "entirely empty - drop it")
|
||||
|
||||
by_col = {}
|
||||
for t, c, val, total in constants:
|
||||
by_col.setdefault((c, str(val)), []).append(t)
|
||||
for (c, val), ts in sorted(by_col.items()):
|
||||
if len(ts) > 2:
|
||||
warn(f"*.{c}", f"constant {val!r} in {len(ts)} tables - redundant "
|
||||
f"({', '.join(ts[:3])}, ...)")
|
||||
else:
|
||||
for t in ts:
|
||||
warn(f"{t}.{c}", f"constant {val!r} - check it is meaningful")
|
||||
else:
|
||||
for t, c, total in empties:
|
||||
warn(f"{t}.{c}", f"entirely empty across {total} rows - drop it")
|
||||
for t, c, val, total in constants:
|
||||
warn(f"{t}.{c}", f"constant {val!r} across {total} rows - redundant")
|
||||
|
||||
|
||||
def check_duplicates(db, verbose):
|
||||
"""Exact duplicate rows, which in a reference table are always a mistake."""
|
||||
for t in tables(db):
|
||||
cols = columns(db, t)
|
||||
if not cols:
|
||||
continue
|
||||
|
||||
collist = ", ".join(f'"{c}"' for c in cols)
|
||||
n = db.execute(
|
||||
f"SELECT COUNT(*) FROM (SELECT {collist}, COUNT(*) AS n "
|
||||
f'FROM "{t}" GROUP BY {collist} HAVING n > 1)').fetchone()[0]
|
||||
|
||||
if n:
|
||||
warn(t, f"{n} groups of exactly duplicated rows")
|
||||
elif verbose:
|
||||
print(f" {t} no duplicate rows".ljust(66) + "ok")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Index generation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Columns worth indexing, by suffix or exact name.
|
||||
#
|
||||
# An index only helps a query that filters with equality or a range on its
|
||||
# leading column. It does NOT help "? GLOB prefix || '*'", because the indexed
|
||||
# column is on the wrong side of the comparison - SQLite still scans every row
|
||||
# and evaluates the GLOB. Worse, an index can make such a query slower by
|
||||
# tempting the planner into a nested loop.
|
||||
#
|
||||
# The fix for callsign lookup is on the query side, not here: generate the
|
||||
# candidate prefixes from the callsign and do equality seeks, longest first.
|
||||
# See docs in the repo. That turns a 7000-row scan into a handful of index
|
||||
# seeks, measured at roughly 40x on the current bundle.
|
||||
INDEX_HINTS = ("code", "_code", "_id", "name", "prefix", "callsign", "band",
|
||||
"mode", "abbr", "symbol", "letter", "character", "grid",
|
||||
"country", "continent", "cq_zone", "itu_zone", "dataset")
|
||||
|
||||
|
||||
def emit_indexes(db, out):
|
||||
out.write("-- Generated by audit.py --indexes. Runs last in the build.\n")
|
||||
out.write("-- The bundle is read-only, so indexes cost file size and nothing else.\n\n")
|
||||
|
||||
# Indexes the heuristic cannot infer but the bot's hot paths need.
|
||||
ESSENTIAL = [
|
||||
("dxcc_prefixes", "prefix",
|
||||
"candidate-prefix equality lookup; leading PK column, but stated "
|
||||
"explicitly because everything depends on it"),
|
||||
("dxcc_prefixes", "entity_id", "joins back to dxcc_entities"),
|
||||
]
|
||||
|
||||
made = 0
|
||||
present = set(tables(db))
|
||||
for t, c, why in ESSENTIAL:
|
||||
if t in present and c in columns(db, t):
|
||||
out.write(f"-- {why}\n")
|
||||
out.write(f'CREATE INDEX IF NOT EXISTS idx_{t}_{c} ON "{t}" ("{c}");\n')
|
||||
made += 1
|
||||
out.write("\n")
|
||||
|
||||
for t in tables(db):
|
||||
if t.startswith("ref_"):
|
||||
continue
|
||||
|
||||
total = db.execute(f"SELECT COUNT(*) FROM {t}").fetchone()[0]
|
||||
if total < 50: # a scan of 50 rows is free
|
||||
continue
|
||||
|
||||
covered = indexed_columns(db, t)
|
||||
|
||||
for c in columns(db, t):
|
||||
if c in covered or (t, c) in {(a, b) for a, b, _ in ESSENTIAL}:
|
||||
continue
|
||||
|
||||
lc = c.lower()
|
||||
if not (lc in INDEX_HINTS or any(lc.endswith(h) for h in INDEX_HINTS)):
|
||||
continue
|
||||
|
||||
# A column with almost no distinct values is not worth an index.
|
||||
distinct = db.execute(f'SELECT COUNT(DISTINCT "{c}") FROM "{t}"').fetchone()[0]
|
||||
if distinct < 2 or distinct < total / 100:
|
||||
continue
|
||||
|
||||
out.write(f'CREATE INDEX IF NOT EXISTS idx_{t}_{lc} ON "{t}" ("{c}");\n')
|
||||
made += 1
|
||||
|
||||
out.write("\n")
|
||||
|
||||
out.write("ANALYZE;\n")
|
||||
sys.stderr.write(f"{made} indexes\n")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="Audit the Hammy reference bundle")
|
||||
ap.add_argument("db", help="path to the bundle")
|
||||
ap.add_argument("--quiet", action="store_true", help="only report problems")
|
||||
ap.add_argument("--ungrouped", action="store_true",
|
||||
help="report every constant/empty column separately instead of "
|
||||
"collapsing repeated findings")
|
||||
ap.add_argument("--indexes", action="store_true",
|
||||
help="emit CREATE INDEX DDL to stdout instead of auditing")
|
||||
ap.add_argument("-o", "--output", help="write index DDL here")
|
||||
args = ap.parse_args()
|
||||
|
||||
db = sqlite3.connect(f"file:{args.db}?mode=ro", uri=True)
|
||||
db.execute("PRAGMA foreign_keys = ON")
|
||||
|
||||
if args.indexes:
|
||||
out = open(args.output, "w") if args.output else sys.stdout
|
||||
try:
|
||||
emit_indexes(db, out)
|
||||
finally:
|
||||
if args.output:
|
||||
out.close()
|
||||
|
||||
return 0
|
||||
|
||||
verbose = not args.quiet
|
||||
|
||||
if verbose:
|
||||
print("structural")
|
||||
check_structural(db, verbose)
|
||||
|
||||
if verbose:
|
||||
print("\nrelational")
|
||||
check_relations(db, verbose)
|
||||
|
||||
if verbose:
|
||||
print("\ndomain")
|
||||
check_domain(db, verbose)
|
||||
|
||||
if verbose:
|
||||
print("\ndead weight and duplicates")
|
||||
check_dead_columns(db, verbose, group=not args.ungrouped)
|
||||
check_duplicates(db, False)
|
||||
|
||||
print()
|
||||
for check, detail in WARNINGS:
|
||||
print(f"WARN {check}: {detail}")
|
||||
for check, detail in FAILURES:
|
||||
print(f"FAIL {check}: {detail}")
|
||||
|
||||
print(f"\n{len(FAILURES)} failures, {len(WARNINGS)} warnings")
|
||||
|
||||
return 1 if FAILURES else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+421
@@ -0,0 +1,421 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
cty2sql.py -- convert AD1C cty.dat (ham radio DXCC country file) into SQL
|
||||
INSERT statements for the dxcc_entities / dxcc_prefixes tables.
|
||||
|
||||
Usage:
|
||||
python3 cty2sql.py cty.dat > dxcc.sql
|
||||
python3 cty2sql.py cty.dat -o dxcc.sql --schema --transaction
|
||||
|
||||
Notes on cty.dat that this script deals with for you:
|
||||
|
||||
* cty.dat has NO DXCC entity numbers. The ADIF entity code (1 = Canada,
|
||||
291 = United States, ...) is supplied by the DXCC_IDS table below, keyed
|
||||
on the record's primary prefix. Override or extend it with --dxcc-map
|
||||
(a JSON file of {"primary_prefix": entity_id}).
|
||||
* cty.dat longitudes are POSITIVE WEST. This script flips them to the
|
||||
conventional positive-east form by default (Ottawa -> -75.0). Use
|
||||
--longitude as-is to keep the raw cty.dat sign.
|
||||
* cty.dat UTC offsets are also positive-west (Canada 5.0, Japan -9.0) and
|
||||
are emitted unchanged by default, matching the target schema. Use
|
||||
--utc-offset standard to flip them into real UTC offsets (Japan +9.0).
|
||||
* Entities whose primary prefix starts with "*" are WAE/CQ-only entities
|
||||
(Sicily, Shetland Is., European Turkey, ...). They are not DXCC entities
|
||||
and have no entity code, so they are skipped unless --include-wae is
|
||||
given (which requires you to supply ids for them via --dxcc-map).
|
||||
* Prefix modifiers are stripped: (cq) [itu] <lat/lon> {cont} ~offset~.
|
||||
A prefix written "=CALL" is a full callsign match and is emitted with
|
||||
exact = 1.
|
||||
|
||||
No third-party dependencies. Python 3.8+.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections import OrderedDict
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# cty.dat primary prefix -> ADIF DXCC entity code (ADIF 3.1.6 enumeration).
|
||||
# Deleted entities are not included. Regenerate/extend with --dxcc-map.
|
||||
# --------------------------------------------------------------------------
|
||||
DXCC_IDS = {
|
||||
"VE": 1, "YA": 3, "3B6": 4, "OH0": 5, "KL": 6, "ZA": 7, "KH8": 9, "FT/z": 10, "VU4": 11,
|
||||
"VP2E": 12, "CE9": 13, "EK": 14, "UA9": 15, "ZL9": 16, "YV0": 17, "4J": 18, "KH1": 20,
|
||||
"EA6": 21, "T8": 22, "3Y/b": 24, "EU": 27, "EA8": 29, "T31": 31, "EA9": 32, "VQ9": 33,
|
||||
"ZL7": 34, "VK9X": 35, "FO/c": 36, "TI9": 37, "VK9C": 38, "SV9": 40, "FT/w": 41, "KP5": 43,
|
||||
"SV5": 45, "9M6": 46, "CE0Y": 47, "T32": 48, "3C": 49, "XE": 50, "E3": 51, "ES": 52,
|
||||
"ET": 53, "UA": 54, "PY0F": 56, "C6": 60, "R1FJ": 61, "8P": 62, "FY": 63, "VP9": 64,
|
||||
"VP2V": 65, "V3": 66, "ZF": 69, "CM": 70, "HC8": 71, "HI": 72, "YS": 74, "4L": 75,
|
||||
"TG": 76, "J3": 77, "HH": 78, "FG": 79, "HR": 80, "6Y": 82, "FM": 84, "YN": 86, "HP": 88,
|
||||
"VP5": 89, "9Y": 90, "P4": 91, "V2": 94, "J7": 95, "VP2M": 96, "J6": 97, "J8": 98,
|
||||
"FT/g": 99, "LU": 100, "KH2": 103, "CP": 104, "KG4": 105, "GU": 106, "3X": 107, "PY": 108,
|
||||
"J5": 109, "KH6": 110, "VK0H": 111, "CE": 112, "GD": 114, "HK": 116, "4U1I": 117,
|
||||
"JX": 118, "HC": 120, "GJ": 122, "KH3": 123, "FT/j": 124, "CE0Z": 125, "UA2": 126,
|
||||
"8R": 129, "UN": 130, "FT/x": 131, "ZP": 132, "ZL8": 133, "EX": 135, "OA": 136, "HL": 137,
|
||||
"KH7K": 138, "PZ": 140, "VP8": 141, "VU7": 142, "XW": 143, "CX": 144, "YL": 145, "LY": 146,
|
||||
"VK9L": 147, "YV": 148, "CU": 149, "VK": 150, "XX9": 152, "VK0M": 153, "C2": 157,
|
||||
"YJ": 158, "8Q": 159, "A3": 160, "HK0/m": 161, "FK": 162, "P2": 163, "3B8": 165,
|
||||
"KH0": 166, "OJ0": 167, "V7": 168, "FH": 169, "ZL": 170, "VK9M": 171, "VP6": 172,
|
||||
"V6": 173, "KH4": 174, "FO": 175, "3D2": 176, "JD/m": 177, "ER": 179, "SV/a": 180,
|
||||
"C9": 181, "KP1": 182, "H4": 185, "5U": 187, "E6": 188, "VK9N": 189, "5W": 190,
|
||||
"E5/n": 191, "JD/o": 192, "3C0": 195, "KH5": 197, "3Y/p": 199, "ZS8": 201, "KP4": 202,
|
||||
"C3": 203, "XF4": 204, "ZD8": 205, "OE": 206, "3B9": 207, "ON": 209, "CY0": 211, "LZ": 212,
|
||||
"FS": 213, "TK": 214, "5B": 215, "HK0/a": 216, "CE0X": 217, "S9": 219, "OZ": 221,
|
||||
"OY": 222, "G": 223, "OH": 224, "IS": 225, "F": 227, "DL": 230, "T5": 232, "ZB": 233,
|
||||
"E5/s": 234, "VP8/g": 235, "SV": 236, "OX": 237, "VP8/o": 238, "HA": 239, "VP8/s": 240,
|
||||
"VP8/h": 241, "TF": 242, "EI": 245, "1A": 246, "1S": 247, "I": 248, "V4": 249, "ZD7": 250,
|
||||
"HB0": 251, "CY9": 252, "PY0S": 253, "LX": 254, "CT3": 256, "9H": 257, "JW": 259,
|
||||
"3A": 260, "EY": 262, "PA": 263, "GI": 265, "LA": 266, "SP": 269, "ZK3": 270, "CT": 272,
|
||||
"PY0T": 273, "ZD9": 274, "YO": 275, "FT/t": 276, "FP": 277, "T7": 278, "GM": 279,
|
||||
"EZ": 280, "EA": 281, "T2": 282, "ZC4": 283, "SM": 284, "KP2": 285, "5X": 286, "HB": 287,
|
||||
"UR": 288, "4U1U": 289, "K": 291, "UK": 292, "3W": 293, "GW": 294, "HV": 295, "YU": 296,
|
||||
"KH9": 297, "FW": 298, "9M2": 299, "T30": 301, "S0": 302, "VK9W": 303, "A9": 304,
|
||||
"S2": 305, "A5": 306, "TI": 308, "XZ": 309, "XU": 312, "4S": 315, "BY": 318, "VR": 321,
|
||||
"VU": 324, "YB": 327, "EP": 330, "YI": 333, "4X": 336, "JA": 339, "JY": 342, "P5": 344,
|
||||
"V8": 345, "9K": 348, "OD": 354, "JT": 363, "9N": 369, "A4": 370, "AP": 372, "DU": 375,
|
||||
"A7": 376, "HZ": 378, "S7": 379, "9V": 381, "J2": 382, "YK": 384, "BV": 386, "HS": 387,
|
||||
"TA": 390, "A6": 391, "7X": 400, "D2": 401, "A2": 402, "9U": 404, "TJ": 406, "TL": 408,
|
||||
"D4": 409, "TT": 410, "D6": 411, "TN": 412, "9Q": 414, "TY": 416, "TR": 420, "C5": 422,
|
||||
"9G": 424, "TU": 428, "5Z": 430, "7P": 432, "EL": 434, "5A": 436, "5R": 438, "7Q": 440,
|
||||
"TZ": 442, "5T": 444, "CN": 446, "5N": 450, "Z2": 452, "FR": 453, "9X": 454, "6W": 456,
|
||||
"9L": 458, "3D2/r": 460, "ZS": 462, "V5": 464, "ST": 466, "3DA": 468, "5H": 470, "3V": 474,
|
||||
"SU": 478, "XT": 480, "9J": 482, "5V": 483, "3D2/c": 489, "T33": 490, "7O": 492, "9A": 497,
|
||||
"S5": 499, "E7": 501, "Z3": 502, "OK": 503, "OM": 504, "BV9P": 505, "BS7": 506, "H40": 507,
|
||||
"FO/a": 508, "FO/m": 509, "E4": 510, "4W": 511, "FK/c": 512, "VP6/d": 513, "4O": 514,
|
||||
"KH8/s": 515, "FJ": 516, "PJ2": 517, "PJ7": 518, "PJ5": 519, "PJ4": 520, "Z8": 521,
|
||||
"Z6": 522
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Parsing
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
# Modifiers that may be attached to a prefix inside a cty.dat prefix list.
|
||||
# These are CAPTURED rather than discarded: about 75% of the prefix tokens in a
|
||||
# current cty.dat carry a zone override, and they exist precisely because the
|
||||
# record default is wrong for that prefix. VK4[55] is Queensland at ITU 55, not
|
||||
# Australia's default 59; drop the modifier and every Queensland call gets the
|
||||
# wrong zone.
|
||||
MOD_CQ = re.compile(r"\(\s*(\d+)\s*\)")
|
||||
MOD_ITU = re.compile(r"\[\s*(\d+)\s*\]")
|
||||
MOD_LATLON = re.compile(r"<\s*([-\d.]+)\s*/\s*([-\d.]+)\s*>")
|
||||
MOD_CONT = re.compile(r"\{([^}]*)\}")
|
||||
MOD_OFFSET = re.compile(r"~([^~]*)~")
|
||||
|
||||
MODIFIER_RE = re.compile(r"""
|
||||
\(\s*\d+\s*\)
|
||||
| \[\s*\d+\s*\]
|
||||
| <[^>]*>
|
||||
| \{[^}]*\}
|
||||
| ~[^~]*~
|
||||
""", re.VERBOSE)
|
||||
|
||||
|
||||
def split_modifiers(token):
|
||||
"""Return (bare_prefix, overrides dict) for one prefix-list token."""
|
||||
ov = {"cq_zone": None, "itu_zone": None, "latitude": None,
|
||||
"longitude": None, "continent": None, "utc_offset": None}
|
||||
|
||||
m = MOD_CQ.search(token)
|
||||
if m:
|
||||
ov["cq_zone"] = int(m.group(1))
|
||||
|
||||
m = MOD_ITU.search(token)
|
||||
if m:
|
||||
ov["itu_zone"] = int(m.group(1))
|
||||
|
||||
m = MOD_LATLON.search(token)
|
||||
if m:
|
||||
ov["latitude"] = float(m.group(1))
|
||||
ov["longitude"] = float(m.group(2)) # still positive-west here
|
||||
|
||||
m = MOD_CONT.search(token)
|
||||
if m:
|
||||
ov["continent"] = m.group(1).strip().upper() or None
|
||||
|
||||
m = MOD_OFFSET.search(token)
|
||||
if m:
|
||||
try:
|
||||
ov["utc_offset"] = float(m.group(1))
|
||||
except ValueError:
|
||||
ov["utc_offset"] = None
|
||||
|
||||
return MODIFIER_RE.sub("", token).strip(), ov
|
||||
|
||||
|
||||
class Entity:
|
||||
__slots__ = ("name", "cq_zone", "itu_zone", "continent", "latitude",
|
||||
"longitude", "utc_offset", "primary", "wae", "prefixes",
|
||||
"entity_id", "line_no")
|
||||
|
||||
def __init__(self, **kw):
|
||||
for k in self.__slots__:
|
||||
setattr(self, k, kw.get(k))
|
||||
|
||||
|
||||
def parse_cty(text):
|
||||
"""Yield Entity objects from the contents of a cty.dat file."""
|
||||
text = text.replace("\r\n", "\n").replace("\r", "\n")
|
||||
|
||||
# Records are terminated by ';'. Track line numbers for error messages.
|
||||
pos = 0
|
||||
line_no = 1
|
||||
for chunk in text.split(";"):
|
||||
start_line = line_no + chunk[:len(chunk) - len(chunk.lstrip("\n"))].count("\n")
|
||||
line_no += chunk.count("\n")
|
||||
record = chunk.strip()
|
||||
if not record:
|
||||
continue
|
||||
|
||||
head, _, body = record.partition("\n")
|
||||
fields = [f.strip() for f in head.split(":")]
|
||||
if len(fields) < 8:
|
||||
raise ValueError(
|
||||
"line %d: expected 8 colon-separated header fields, got %d: %r"
|
||||
% (start_line, len(fields), head))
|
||||
|
||||
primary = fields[7]
|
||||
wae = primary.startswith("*")
|
||||
|
||||
ent = Entity(
|
||||
name=fields[0],
|
||||
cq_zone=int(fields[1]),
|
||||
itu_zone=int(fields[2]),
|
||||
continent=fields[3].upper(),
|
||||
latitude=float(fields[4]),
|
||||
longitude=float(fields[5]),
|
||||
utc_offset=float(fields[6]),
|
||||
primary=primary.lstrip("*"),
|
||||
wae=wae,
|
||||
prefixes=[],
|
||||
entity_id=None,
|
||||
line_no=start_line,
|
||||
)
|
||||
|
||||
seen = set()
|
||||
for token in body.replace("\n", "").split(","):
|
||||
token = token.strip()
|
||||
if not token:
|
||||
continue
|
||||
bare, ov = split_modifiers(token)
|
||||
exact = bare.startswith("=")
|
||||
bare = bare.lstrip("=").strip().upper()
|
||||
if not bare or bare in seen:
|
||||
continue
|
||||
seen.add(bare)
|
||||
ent.prefixes.append((bare, 1 if exact else 0, ov))
|
||||
|
||||
yield ent
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# SQL emission
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
SCHEMA = """\
|
||||
CREATE TABLE IF NOT EXISTS dxcc_entities (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name VARCHAR(64) NOT NULL,
|
||||
continent CHAR(2) NOT NULL,
|
||||
cq_zone INTEGER NOT NULL,
|
||||
itu_zone INTEGER NOT NULL,
|
||||
latitude DECIMAL(6,2) NOT NULL,
|
||||
longitude DECIMAL(7,2) NOT NULL,
|
||||
utc_offset DECIMAL(4,2) NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS dxcc_prefixes (
|
||||
prefix VARCHAR(16) NOT NULL,
|
||||
entity_id INTEGER NOT NULL REFERENCES dxcc_entities (id),
|
||||
exact INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (prefix, entity_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_dxcc_prefixes_entity ON dxcc_prefixes (entity_id);
|
||||
"""
|
||||
|
||||
|
||||
def q(value):
|
||||
"""Quote a string for SQL, doubling embedded single quotes."""
|
||||
return "'" + value.replace("'", "''") + "'"
|
||||
|
||||
|
||||
def emit_entities(entities, out):
|
||||
rows = []
|
||||
for e in entities:
|
||||
rows.append((
|
||||
"%d," % e.entity_id,
|
||||
q(e.name) + ",",
|
||||
q(e.primary) + ",",
|
||||
q(e.continent) + ",",
|
||||
"%d," % e.cq_zone,
|
||||
"%d," % e.itu_zone,
|
||||
"%.2f," % e.latitude,
|
||||
"%.2f," % e.longitude,
|
||||
"%.1f" % e.utc_offset,
|
||||
))
|
||||
widths = [max(len(r[i]) for r in rows) for i in range(9)]
|
||||
# Numeric columns look better right-aligned, text columns left-aligned.
|
||||
align = ["<", "<", "<", "<", "<", "<", ">", ">", ">"]
|
||||
|
||||
out.write("INSERT INTO dxcc_entities "
|
||||
"(id, name, primary_prefix, continent, cq_zone, itu_zone, "
|
||||
"latitude, longitude, utc_offset)"
|
||||
" VALUES\n")
|
||||
for n, row in enumerate(rows):
|
||||
cells = [format(cell, "%s%d" % (align[i], widths[i]))
|
||||
for i, cell in enumerate(row)]
|
||||
line = " (" + " ".join(cells).rstrip() + ")"
|
||||
out.write(line + (",\n" if n < len(rows) - 1 else ";\n"))
|
||||
|
||||
|
||||
def emit_prefixes(entities, out, per_line=3):
|
||||
out.write("INSERT INTO dxcc_prefixes "
|
||||
"(prefix, entity_id, exact, cq_zone, itu_zone) VALUES\n")
|
||||
|
||||
def n(v):
|
||||
return "NULL" if v is None else str(v)
|
||||
|
||||
tuples_by_entity = []
|
||||
for e in entities:
|
||||
if e.prefixes:
|
||||
tuples_by_entity.append(
|
||||
["(%s, %d, %d, %s, %s)" % (q(p), e.entity_id, x,
|
||||
n(ov["cq_zone"]), n(ov["itu_zone"]))
|
||||
for p, x, ov in e.prefixes])
|
||||
|
||||
last = len(tuples_by_entity) - 1
|
||||
for i, group in enumerate(tuples_by_entity):
|
||||
for j in range(0, len(group), per_line):
|
||||
slice_ = group[j:j + per_line]
|
||||
is_final = (i == last) and (j + per_line >= len(group))
|
||||
out.write(" " + ", ".join(slice_) + (";\n" if is_final else ",\n"))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Main
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(
|
||||
description="Convert a cty.dat DXCC country file into SQL INSERT statements.")
|
||||
ap.add_argument("cty", help="path to cty.dat")
|
||||
ap.add_argument("-o", "--output", help="write SQL here (default: stdout)")
|
||||
ap.add_argument("--dxcc-map", metavar="JSON",
|
||||
help='JSON file of {"primary_prefix": entity_id} merged over '
|
||||
"the built-in table")
|
||||
ap.add_argument("--longitude", choices=["east-positive", "as-is"],
|
||||
default="east-positive",
|
||||
help="cty.dat stores longitude positive-west; 'east-positive' "
|
||||
"(default) flips it to the usual convention")
|
||||
ap.add_argument("--utc-offset", choices=["cty", "standard"], default="standard",
|
||||
help="'cty' (default) keeps cty.dat's positive-west offsets; "
|
||||
"'standard' flips them to real UTC offsets")
|
||||
ap.add_argument("--include-wae", action="store_true",
|
||||
help="include WAE/CQ-only entities (needs ids via --dxcc-map)")
|
||||
ap.add_argument("--exclude-exact", action="store_true",
|
||||
help="skip '=CALLSIGN' full-callsign entries entirely")
|
||||
ap.add_argument("--schema", action="store_true",
|
||||
help="emit CREATE TABLE statements first")
|
||||
ap.add_argument("--truncate", action="store_true",
|
||||
help="emit DELETE FROM statements before the inserts")
|
||||
ap.add_argument("--transaction", action="store_true",
|
||||
help="wrap the output in BEGIN / COMMIT")
|
||||
ap.add_argument("--per-line", type=int, default=5, metavar="N",
|
||||
help="prefix tuples per output line (default 5)")
|
||||
ap.add_argument("--strict", action="store_true",
|
||||
help="exit non-zero if any entity has no known DXCC id")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
ids = dict(DXCC_IDS)
|
||||
if args.dxcc_map:
|
||||
with open(args.dxcc_map, encoding="utf-8") as fh:
|
||||
ids.update({k: int(v) for k, v in json.load(fh).items()})
|
||||
|
||||
with open(args.cty, encoding="utf-8", errors="replace") as fh:
|
||||
raw = fh.read()
|
||||
|
||||
entities = []
|
||||
skipped_wae = []
|
||||
unmapped = []
|
||||
for ent in parse_cty(raw):
|
||||
if ent.wae and not args.include_wae:
|
||||
skipped_wae.append(ent)
|
||||
continue
|
||||
ent.entity_id = ids.get(ent.primary)
|
||||
if ent.entity_id is None:
|
||||
unmapped.append(ent)
|
||||
continue
|
||||
if args.longitude == "east-positive":
|
||||
ent.longitude = -ent.longitude or 0.0 # avoid "-0.00"
|
||||
if args.utc_offset == "standard":
|
||||
ent.utc_offset = -ent.utc_offset or 0.0
|
||||
if args.exclude_exact:
|
||||
ent.prefixes = [(p, x, ov) for p, x, ov in ent.prefixes if not x]
|
||||
entities.append(ent)
|
||||
|
||||
entities.sort(key=lambda e: e.entity_id)
|
||||
|
||||
# A prefix must resolve to one entity; cty.dat should not collide, but say
|
||||
# so loudly if a hand-edited file does.
|
||||
owner = OrderedDict()
|
||||
for e in entities:
|
||||
kept = []
|
||||
for p, x, ov in e.prefixes:
|
||||
if p in owner:
|
||||
sys.stderr.write(
|
||||
"warning: prefix %s claimed by both %s and %s; keeping %s\n"
|
||||
% (p, owner[p], e.name, owner[p]))
|
||||
continue
|
||||
owner[p] = e.name
|
||||
kept.append((p, x, ov))
|
||||
e.prefixes = kept
|
||||
|
||||
for e in skipped_wae:
|
||||
sys.stderr.write("note: skipping WAE/CQ-only entity %s (%s)\n"
|
||||
% (e.name, e.primary))
|
||||
for e in unmapped:
|
||||
sys.stderr.write("warning: no DXCC id for %s (primary prefix %s, line %d)\n"
|
||||
% (e.name, e.primary, e.line_no))
|
||||
|
||||
if not entities:
|
||||
sys.stderr.write("error: no entities to write\n")
|
||||
return 2
|
||||
|
||||
out = open(args.output, "w", encoding="utf-8") if args.output else sys.stdout
|
||||
try:
|
||||
src = os.path.basename(args.cty)
|
||||
out.write("-- " + "-" * 73 + "\n")
|
||||
out.write("-- DXCC entities and prefixes generated from %s by cty2sql.py.\n" % src)
|
||||
out.write("-- %d entities, %d prefixes. Longitude is %s; utc_offset uses the %s.\n"
|
||||
% (len(entities), sum(len(e.prefixes) for e in entities),
|
||||
"positive east" if args.longitude == "east-positive"
|
||||
else "positive west (raw cty.dat)",
|
||||
"cty.dat sign (positive west)" if args.utc_offset == "cty"
|
||||
else "standard UTC sign"))
|
||||
out.write("-- " + "-" * 73 + "\n")
|
||||
if args.schema:
|
||||
out.write(SCHEMA + "\n")
|
||||
if args.transaction:
|
||||
out.write("BEGIN;\n")
|
||||
if args.truncate:
|
||||
out.write("DELETE FROM dxcc_prefixes;\nDELETE FROM dxcc_entities;\n")
|
||||
emit_entities(entities, out)
|
||||
emit_prefixes(entities, out, per_line=max(1, args.per_line))
|
||||
if args.transaction:
|
||||
out.write("COMMIT;\n")
|
||||
finally:
|
||||
if args.output:
|
||||
out.close()
|
||||
|
||||
sys.stderr.write("wrote %d entities and %d prefixes\n"
|
||||
% (len(entities), sum(len(e.prefixes) for e in entities)))
|
||||
return 1 if (args.strict and unmapped) else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+57
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Build the Hammy reference bundle from the numbered SQL sources.
|
||||
#
|
||||
# The plain `for f in *.sql; do sqlite3 db < $f; done` loop keeps going after a
|
||||
# failure, so one bad file leaves a half-populated database that looks fine
|
||||
# until something queries the missing rows. -bail plus set -e stops at the first
|
||||
# error instead.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
DB="${1:-hammy-ref.sqlite}"
|
||||
|
||||
# -batch ignores ~/.sqliterc, so output formatting does not depend on whatever
|
||||
# .mode the developer has configured.
|
||||
SQLITE=(sqlite3 -bail -batch)
|
||||
|
||||
# Order matters: 05-dxcc.sql must load before 06-eng-beacons.sql, because
|
||||
# ncdxf_beacons.dxcc_id has a foreign key into dxcc_entities.
|
||||
SOURCES=(*.sql)
|
||||
|
||||
if [ -e "$DB" ]; then
|
||||
echo "removing existing $DB"
|
||||
rm -f "$DB"
|
||||
fi
|
||||
|
||||
for f in "${SOURCES[@]}"; do
|
||||
printf ' %-24s' "$f"
|
||||
"${SQLITE[@]}" "$DB" < "$f"
|
||||
echo "ok"
|
||||
done
|
||||
|
||||
echo
|
||||
echo "integrity"
|
||||
fk=$("${SQLITE[@]}" "$DB" 'PRAGMA foreign_key_check;')
|
||||
if [ -n "$fk" ]; then
|
||||
echo " FOREIGN KEY VIOLATIONS:"
|
||||
echo "$fk" | sed 's/^/ /'
|
||||
exit 1
|
||||
fi
|
||||
echo " foreign keys ok"
|
||||
echo " integrity $("${SQLITE[@]}" "$DB" 'PRAGMA integrity_check;')"
|
||||
|
||||
echo
|
||||
echo "row counts"
|
||||
# pragma_table_info returns one row per COLUMN, so counting it gives column
|
||||
# counts. Real row counts need a query per table; generate them and pipe back in.
|
||||
"${SQLITE[@]}" -noheader "$DB" "
|
||||
SELECT 'SELECT '' ' || name || ''' , COUNT(*) FROM ' || name || ';'
|
||||
FROM sqlite_master WHERE type='table' ORDER BY name;
|
||||
" | "${SQLITE[@]}" -noheader -separator ' ' "$DB" | awk '{printf " %-34s %8s\n", $1, $2}'
|
||||
|
||||
echo
|
||||
echo "compacting"
|
||||
"${SQLITE[@]}" "$DB" 'VACUUM;'
|
||||
|
||||
ls -lh "$DB"
|
||||
Reference in New Issue
Block a user