From fb68f26d829aaf98304c29532d1ce77d2c3923f5 Mon Sep 17 00:00:00 2001 From: Adam Wisher <37659188+mrwish7@users.noreply.github.com> Date: Thu, 12 Jun 2025 09:18:59 +0100 Subject: [PATCH] Local db optimisation and GPS support DB indexing to improve tx search performance with PI and reg PI, re-download db on failure, add websocket for GPS location update support Co-Authored-By: Amateur Audio Dude <168192910+AmateurAudioDude@users.noreply.github.com> Co-Authored-By: Highpoint2000 <168109804+Highpoint2000@users.noreply.github.com> --- server/server_config.js | 1 + server/tx_search.js | 129 +++++++++++++++++++++++++++++++++------- 2 files changed, 107 insertions(+), 23 deletions(-) diff --git a/server/server_config.js b/server/server_config.js index fcf4290..f03419f 100644 --- a/server/server_config.js +++ b/server/server_config.js @@ -58,6 +58,7 @@ let serverConfig = { lat: "", lon: "", broadcastTuner: false, + gpsMode: false, proxyIp: "", contact: null, }, diff --git a/server/tx_search.js b/server/tx_search.js index f00a04a..2958ba5 100644 --- a/server/tx_search.js +++ b/server/tx_search.js @@ -3,14 +3,24 @@ const { serverConfig } = require('./server_config'); const consoleCmd = require('./console'); let localDb = {}; +let lastDownloadTime = 0; // Last DB download attempt time. let lastFetchTime = 0; +let piFreqIndex = {}; // Indexing for speedier PI+Freq combinations const fetchInterval = 1000; +const downloadInterval = 300000; const esSwitchCache = {"lastCheck": null, "esSwitch": false}; const esFetchInterval = 300000; var currentPiCode = ''; var currentRdsPs = ''; const usStatesGeoJsonUrl = "https://raw.githubusercontent.com/PublicaMundi/MappingAPI/master/data/geojson/us-states.json"; let usStatesGeoJson = null; // To cache the GeoJSON data for US states +let Latitude = serverConfig.identification.lat; +let Longitude = serverConfig.identification.lon; + +// Create WebSocket URL for GPS lat/lon update. +const webserverPort = serverConfig.webserver.webserverPort || 8080; // Fallback to port 8080 +const externalWsUrl = `ws://127.0.0.1:${webserverPort}/data_plugins`; +const WebSocket = require('ws'); // Get weighting values based on algorithm setting. // Defaults = algorithm 1 @@ -30,16 +40,76 @@ if (typeof algorithms[algoSetting] !== 'undefined') { // IIFE to build the local TX DB cache from the endpoint. (async () => { + const now = Date.now(); + lastDownloadTime = now; + await buildTxDatabase(); +})(); + +if (serverConfig.identification.gpsMode) { + // 5-second delay before activation of GPS lat/lon websocket + setTimeout(() => { + const websocket = new WebSocket(externalWsUrl); + consoleCmd.logInfo('Set up GPS websocket for lat/lon'); + // Event listener to receive data + websocket.on('message', (data) => { + try { + // Parse the received data + const parsedData = JSON.parse(data); + + // Check if the dataset is of type GPS + if (parsedData.type === "GPS" && parsedData.value) { + const gpsData = parsedData.value; + const { status, time, lat, lon, alt, mode } = gpsData; + + if (status === "active") { + Latitude = parseFloat(lat); + Longitude = parseFloat(lon); + } + } + } catch (error) { + consoleCmd.logError("Error processing WebSocket data:", error); + } + }); + + }, 5000); +} + +// Function to build local TX database from FMDX Maps endpoint. +async function buildTxDatabase() { try { consoleCmd.logInfo('Fetching transmitter database...'); const response = await fetch(`https://maps.fmdx.org/api?qth=${serverConfig.identification.lat},${serverConfig.identification.lon}`); if (!response.ok) throw new Error(`HTTP error! Status: ${response.status}`); localDb = await response.json(); + buildPiFreqIndex(); consoleCmd.logInfo('Transmitter database successfully loaded.'); } catch (error) { consoleCmd.logError("Failed to fetch transmitter database:", error); } -})(); +} + +// Function to build index map of PI+Freq combinations +function buildPiFreqIndex() { + piFreqIndex = {}; // reset + for (const locData of Object.values(localDb.locations || {})) { + for (const station of locData.stations || []) { + if (!station.freq) continue; + const freq = station.freq; + const pi = station.pi?.toUpperCase(); + const pireg = station.pireg?.toUpperCase(); + if (pi) { + const key = `${freq}|${pi}`; + if (!piFreqIndex[key]) piFreqIndex[key] = []; + piFreqIndex[key].push({ ...locData, station }); + } + if (pireg) { + const regKey = `${freq}|${pireg}`; + if (!piFreqIndex[regKey]) piFreqIndex[regKey] = []; + piFreqIndex[regKey].push({ ...locData, station }); + } + } + } +} // Load the US states GeoJSON data async function loadUsStatesGeoJson() { @@ -105,11 +175,16 @@ function getStateForCoordinates(lat, lon) { * If at least three valid matches are found for any token, the function returns true. */ function validPsCompare(rdsPs, stationPs) { + if (typeof stationPs !== 'string' || typeof rdsPs !== 'string') { + consoleCmd.logError(`Invalid TX values. stationPs: ${stationPs}, rdsPs: ${rdsPs}`); + return false; + } + // Standardize the rdsPs string: replace spaces with underscores and convert to lowercase. const standardizedRdsPs = rdsPs.replace(/ /g, '_').toLowerCase(); // Split stationPs into tokens (e.g., "__mdr___ _kultur_" -> ["__mdr___", "_kultur_"]) - const psTokens = stationPs.split(/\s+/).filter(token => token.length > 0).map(token => token.toLowerCase()); + const psTokens = stationPs.split(/\s+/).filter(token => token.length > 0).map(token => { const lower = token.toLowerCase(); return lower.length < 8 ? lower.padEnd(8, '_') : lower; }); // Iterate through all tokens and check if any token yields at least three valid (non "_" ) matches. for (let token of psTokens) { @@ -156,29 +231,37 @@ async function fetchTx(freq, piCode, rdsPs) { const now = Date.now(); freq = parseFloat(freq); - if (isNaN(freq)) return; - if (now - lastFetchTime < fetchInterval - || serverConfig.identification.lat.length < 2 - || freq < 87 - || Object.keys(localDb).length === 0 - || (currentPiCode === piCode && currentRdsPs === rdsPs)) { - return Promise.resolve(); + // If we don't have a local database and the interval has passed, re-try download. + if ( + Object.keys(localDb).length === 0 && + now - lastDownloadTime > downloadInterval + ) { + lastDownloadTime = now; + await buildTxDatabase(); } + if ( + isNaN(freq) || + now - lastFetchTime < fetchInterval || + Latitude.length < 2 || + freq < 87 || + Object.keys(localDb).length === 0 || + (currentPiCode === piCode && currentRdsPs === rdsPs) + ) return Promise.resolve(); + lastFetchTime = now; currentPiCode = piCode; currentRdsPs = rdsPs; if (serverConfig.webserver.rdsMode === true) await loadUsStatesGeoJson(); - let filteredLocations = Object.values(localDb.locations || {}) - .map(locData => ({ - ...locData, - stations: locData.stations.filter(station => - station.freq === freq && - (station.pi === piCode.toUpperCase() || station.pireg === piCode.toUpperCase() ) - ) - })) - .filter(locData => locData.stations.length > 0); // Ensure locations with at least one matching station remain + const key = `${freq}|${piCode.toUpperCase()}`; + let rawMatches = piFreqIndex[key] || []; + + // Format the results into the same structure as before + let filteredLocations = rawMatches.map(({ station, ...locData }) => ({ + ...locData, + stations: [station] + })); // Only check PS if we have more than one match. if (filteredLocations.length > 1) { @@ -191,7 +274,7 @@ async function fetchTx(freq, piCode, rdsPs) { for (let loc of filteredLocations) { loc = Object.assign(loc, loc.stations[0]); delete loc.stations; - const dist = haversine(serverConfig.identification.lat, serverConfig.identification.lon, loc.lat, loc.lon); + const dist = haversine(Latitude, Longitude, loc.lat, loc.lon); loc = Object.assign(loc, dist); loc.detectedByPireg = (loc.pireg === piCode.toUpperCase()); } @@ -206,7 +289,7 @@ async function fetchTx(freq, piCode, rdsPs) { // Have a maximum of 10 extra matches and remove any with less than 1/10 of the winning score multiMatches = filteredLocations .slice(1, 11) - .filter(obj => obj.score >= (match.score/10)); + .filter(obj => obj.score >= (match.score / 10)); } else if (filteredLocations.length === 1) { match = filteredLocations[0]; match.score = 1; @@ -254,7 +337,7 @@ function checkEs() { return esSwitchCache.esSwitch; } - if (serverConfig.identification.lat > 20) { + if (Latitude > 20) { esSwitchCache.lastCheck = now; fetch(url) .then(response => { @@ -262,8 +345,8 @@ function checkEs() { return response.json(); }) .then(data => { - if ((serverConfig.identification.lon < -32 && data.north_america.max_frequency !== "No data") || - (serverConfig.identification.lon >= -32 && data.europe.max_frequency !== "No data")) { + if ((Longitude < -32 && data.north_america.max_frequency !== "No data") || + (Longitude >= -32 && data.europe.max_frequency !== "No data")) { esSwitchCache.esSwitch = true; } })