Local db optimisation and GPS support

DB indexing to improve tx search performance with PI and reg PI, re-download db on failure, add websocket for GPS location update support

Co-Authored-By: Amateur Audio Dude <168192910+AmateurAudioDude@users.noreply.github.com>
Co-Authored-By: Highpoint2000 <168109804+Highpoint2000@users.noreply.github.com>

server/server_config.js

@@ -58,6 +58,7 @@ let serverConfig = {
     lat: "",
     lon: "",
     broadcastTuner: false,
+    gpsMode: false,
     proxyIp: "",
     contact: null,
   },

server/tx_search.js

@@ -3,14 +3,24 @@ const { serverConfig } = require('./server_config');
 const consoleCmd = require('./console');
 
 let localDb = {};
+let lastDownloadTime = 0; // Last DB download attempt time.
 let lastFetchTime = 0;
+let piFreqIndex = {}; // Indexing for speedier PI+Freq combinations
 const fetchInterval = 1000;
+const downloadInterval = 300000;
 const esSwitchCache = {"lastCheck": null, "esSwitch": false};
 const esFetchInterval = 300000;
 var currentPiCode = '';
 var currentRdsPs = '';
 const usStatesGeoJsonUrl = "https://raw.githubusercontent.com/PublicaMundi/MappingAPI/master/data/geojson/us-states.json";
 let usStatesGeoJson = null;  // To cache the GeoJSON data for US states
+let Latitude = serverConfig.identification.lat;
+let Longitude = serverConfig.identification.lon;
+
+// Create WebSocket URL for GPS lat/lon update.
+const webserverPort = serverConfig.webserver.webserverPort || 8080; // Fallback to port 8080
+const externalWsUrl = `ws://127.0.0.1:${webserverPort}/data_plugins`;
+const WebSocket = require('ws'); 
 
 // Get weighting values based on algorithm setting.
 // Defaults = algorithm 1
@@ -30,16 +40,76 @@ if (typeof algorithms[algoSetting] !== 'undefined') {
 
 // IIFE to build the local TX DB cache from the endpoint.
 (async () => {
+    const now = Date.now();
+    lastDownloadTime = now;
+    await buildTxDatabase();
+})();
+
+if (serverConfig.identification.gpsMode) {
+    // 5-second delay before activation of GPS lat/lon websocket
+    setTimeout(() => {
+        const websocket = new WebSocket(externalWsUrl);
+        consoleCmd.logInfo('Set up GPS websocket for lat/lon');
+        // Event listener to receive data
+        websocket.on('message', (data) => {
+            try {
+                // Parse the received data
+                const parsedData = JSON.parse(data);
+
+                // Check if the dataset is of type GPS
+                if (parsedData.type === "GPS" && parsedData.value) {
+                    const gpsData = parsedData.value;
+                    const { status, time, lat, lon, alt, mode } = gpsData;
+
+                    if (status === "active") {
+                        Latitude = parseFloat(lat);
+                        Longitude = parseFloat(lon);
+                    }
+                }
+            } catch (error) {
+                consoleCmd.logError("Error processing WebSocket data:", error);
+            }
+        });
+
+    }, 5000);
+}
+
+// Function to build local TX database from FMDX Maps endpoint.
+async function buildTxDatabase() {
     try {
         consoleCmd.logInfo('Fetching transmitter database...');
         const response = await fetch(`https://maps.fmdx.org/api?qth=${serverConfig.identification.lat},${serverConfig.identification.lon}`);
         if (!response.ok) throw new Error(`HTTP error! Status: ${response.status}`);
         localDb = await response.json();
+        buildPiFreqIndex();
         consoleCmd.logInfo('Transmitter database successfully loaded.');
     } catch (error) {
         consoleCmd.logError("Failed to fetch transmitter database:", error);
     }
-})();
+}
+
+// Function to build index map of PI+Freq combinations
+function buildPiFreqIndex() {
+    piFreqIndex = {}; // reset
+    for (const locData of Object.values(localDb.locations || {})) {
+        for (const station of locData.stations || []) {
+            if (!station.freq) continue;
+            const freq = station.freq;
+            const pi = station.pi?.toUpperCase();
+            const pireg = station.pireg?.toUpperCase();
+            if (pi) {
+                const key = `${freq}|${pi}`;
+                if (!piFreqIndex[key]) piFreqIndex[key] = [];
+                piFreqIndex[key].push({ ...locData, station });
+            }
+            if (pireg) {
+                const regKey = `${freq}|${pireg}`;
+                if (!piFreqIndex[regKey]) piFreqIndex[regKey] = [];
+                piFreqIndex[regKey].push({ ...locData, station });
+            }
+        }
+    }
+}
 
 // Load the US states GeoJSON data
 async function loadUsStatesGeoJson() {
@@ -105,11 +175,16 @@ function getStateForCoordinates(lat, lon) {
  * If at least three valid matches are found for any token, the function returns true.
  */
 function validPsCompare(rdsPs, stationPs) {
+    if (typeof stationPs !== 'string' || typeof rdsPs !== 'string') {
+        consoleCmd.logError(`Invalid TX values. stationPs: ${stationPs}, rdsPs: ${rdsPs}`);
+        return false;
+    }
+
     // Standardize the rdsPs string: replace spaces with underscores and convert to lowercase.
     const standardizedRdsPs = rdsPs.replace(/ /g, '_').toLowerCase();
     
     // Split stationPs into tokens (e.g., "__mdr___ _kultur_" -> ["__mdr___", "_kultur_"])
-    const psTokens = stationPs.split(/\s+/).filter(token => token.length > 0).map(token => token.toLowerCase());
+    const psTokens = stationPs.split(/\s+/).filter(token => token.length > 0).map(token => { const lower = token.toLowerCase(); return lower.length < 8 ? lower.padEnd(8, '_') : lower; });
        
     // Iterate through all tokens and check if any token yields at least three valid (non "_" ) matches.
     for (let token of psTokens) {
@@ -156,29 +231,37 @@ async function fetchTx(freq, piCode, rdsPs) {
     const now = Date.now();
     freq = parseFloat(freq);
 
-    if (isNaN(freq)) return;
+    // If we don't have a local database and the interval has passed, re-try download. 
-    if (now - lastFetchTime < fetchInterval
+    if (
-        || serverConfig.identification.lat.length < 2
+        Object.keys(localDb).length === 0 &&
-        || freq < 87
+        now - lastDownloadTime > downloadInterval
-        || Object.keys(localDb).length === 0
+    ) {
-        || (currentPiCode === piCode && currentRdsPs === rdsPs)) {
+        lastDownloadTime = now;
-        return Promise.resolve();
+        await buildTxDatabase();
     }
 
+    if (
+        isNaN(freq) ||
+        now - lastFetchTime < fetchInterval ||
+        Latitude.length < 2 ||
+        freq < 87 ||
+        Object.keys(localDb).length === 0 ||
+        (currentPiCode === piCode && currentRdsPs === rdsPs)
+    ) return Promise.resolve();
+
     lastFetchTime = now;
     currentPiCode = piCode;
     currentRdsPs = rdsPs;
     if (serverConfig.webserver.rdsMode === true) await loadUsStatesGeoJson();
 
-    let filteredLocations = Object.values(localDb.locations || {})
+    const key = `${freq}|${piCode.toUpperCase()}`;
-        .map(locData => ({
+    let rawMatches = piFreqIndex[key] || [];
-            ...locData,
+
-            stations: locData.stations.filter(station => 
+    // Format the results into the same structure as before
-            station.freq === freq &&
+    let filteredLocations = rawMatches.map(({ station, ...locData }) => ({
-            (station.pi === piCode.toUpperCase() || station.pireg === piCode.toUpperCase() )
+        ...locData,
-            )
+        stations: [station]
-        }))
+    }));
-        .filter(locData => locData.stations.length > 0); // Ensure locations with at least one matching station remain
 
     // Only check PS if we have more than one match.
     if (filteredLocations.length > 1) {
@@ -191,7 +274,7 @@ async function fetchTx(freq, piCode, rdsPs) {
     for (let loc of filteredLocations) {
       loc = Object.assign(loc, loc.stations[0]);
       delete loc.stations;
-      const dist = haversine(serverConfig.identification.lat, serverConfig.identification.lon, loc.lat, loc.lon);
+      const dist = haversine(Latitude, Longitude, loc.lat, loc.lon);
       loc = Object.assign(loc, dist);
       loc.detectedByPireg = (loc.pireg === piCode.toUpperCase());
     }
@@ -206,7 +289,7 @@ async function fetchTx(freq, piCode, rdsPs) {
         // Have a maximum of 10 extra matches and remove any with less than 1/10 of the winning score
         multiMatches = filteredLocations
             .slice(1, 11)
-            .filter(obj => obj.score >= (match.score/10));
+            .filter(obj => obj.score >= (match.score / 10));
     } else if (filteredLocations.length === 1) {
         match = filteredLocations[0];
         match.score = 1;
@@ -254,7 +337,7 @@ function checkEs() {
         return esSwitchCache.esSwitch;
     }
 
-    if (serverConfig.identification.lat > 20) {
+    if (Latitude > 20) {
         esSwitchCache.lastCheck = now;
         fetch(url)
             .then(response => {
@@ -262,8 +345,8 @@ function checkEs() {
                 return response.json();
             })
             .then(data => {
-                if ((serverConfig.identification.lon < -32 && data.north_america.max_frequency !== "No data") ||
+                if ((Longitude < -32 && data.north_america.max_frequency !== "No data") ||
-                    (serverConfig.identification.lon >= -32 && data.europe.max_frequency !== "No data")) {
+                    (Longitude >= -32 && data.europe.max_frequency !== "No data")) {
                     esSwitchCache.esSwitch = true;
                 }
             })