Merge pull request #451 from leiweibau/next_update

Next update

Author: leiweibau

back/pialert-cli

@@ -1,5 +1,5 @@
 #!/bin/bash
-SCRIPTVERSION='v1.30'
+SCRIPTVERSION='v1.31'
 SCRIPT=$(readlink -f $0)
 SCRIPTPATH=`dirname $SCRIPT`
 PIA_CONF_FILE=${SCRIPTPATH}'/../config/pialert.conf'
@@ -653,6 +653,41 @@ EOM
     # ICMP_Mon_Connections
     echo -e "...Insert new table 'ICMP_Mon_Connections' to DB"
     sqlite3 $PIA_DB_FILE "CREATE TABLE IF NOT EXISTS 'ICMP_Mon_Connections' ('icmpeve_ip' NUMERIC NOT NULL,'icmpeve_DateTime' TEXT NOT NULL,'icmpeve_Present' INTEGER,'icmpeve_EventType' STRING(50) NOT NULL COLLATE NOCASE, 'icmpeve_AdditionalInfo' STRING(250) DEFAULT (''));"
+    # Devices.dev_Scan_Validation
+    echo -e "...Insert new column 'dev_Scan_Validation' to table 'Devices' to DB"    
+    existing=$(sqlite3 $PIA_DB_FILE "PRAGMA table_info('Devices');" | awk -F\| '{print $2}' | grep -wci "dev_Scan_Validation")
+    if [ $existing -eq 0 ]; then
+      sqlite3 $PIA_DB_FILE "ALTER TABLE 'Devices' ADD COLUMN 'dev_Scan_Validation' INTEGER NOT NULL DEFAULT 0;"
+    else
+      echo "        Column 'dev_Scan_Validation' already exists in the 'Devices' table."
+    fi
+    # Devices.dev_Scan_Validation_State
+    echo -e "...Insert new column 'dev_Scan_Validation_State' to table 'Devices' to DB"    
+    existing=$(sqlite3 $PIA_DB_FILE "PRAGMA table_info('Devices');" | awk -F\| '{print $2}' | grep -wci "dev_Scan_Validation_State")
+    if [ $existing -eq 0 ]; then
+      sqlite3 $PIA_DB_FILE "ALTER TABLE 'Devices' ADD COLUMN 'dev_Scan_Validation_State' INTEGER NOT NULL DEFAULT 0;"
+    else
+      echo "        Column 'dev_Scan_Validation_State' already exists in the 'Devices' table."
+    fi
+    # ICMP_Mon.icmp_Scan_Validation
+    echo -e "...Insert new column 'icmp_Scan_Validation' to table 'ICMP_Mon' to DB"    
+    existing=$(sqlite3 $PIA_DB_FILE "PRAGMA table_info('ICMP_Mon');" | awk -F\| '{print $2}' | grep -wci "icmp_Scan_Validation")
+    if [ $existing -eq 0 ]; then
+      sqlite3 $PIA_DB_FILE "ALTER TABLE 'ICMP_Mon' ADD COLUMN 'icmp_Scan_Validation' INTEGER NOT NULL DEFAULT 0;"
+    else
+      echo "        Column 'icmp_Scan_Validation' already exists in the 'ICMP_Mon' table."
+    fi
+    # ICMP_Mon.icmp_Scan_Validation_State
+    echo -e "...Insert new column 'icmp_Scan_Validation_State' to table 'ICMP_Mon' to DB"    
+    existing=$(sqlite3 $PIA_DB_FILE "PRAGMA table_info('ICMP_Mon');" | awk -F\| '{print $2}' | grep -wci "icmp_Scan_Validation_State")
+    if [ $existing -eq 0 ]; then
+      sqlite3 $PIA_DB_FILE "ALTER TABLE 'ICMP_Mon' ADD COLUMN 'icmp_Scan_Validation_State' INTEGER NOT NULL DEFAULT 0;"
+    else
+      echo "        Column 'icmp_Scan_Validation_State' already exists in the 'ICMP_Mon' table."
+    fi
+
+
+
 
     echo -e "\nUpdate finished!\n"
 

back/pialert.py

@@ -891,12 +891,6 @@ def scan_network():
     sql_connection.commit()
     closeDB()
 
-    # Issue #370
-    # new column for example "dev_alarm_delay" (True/False) and config parameter (2) scan counter
-    # For devices that are in the “Current_Scan” table and for which “alarm_delay” is set to True, 
-    # “Pending_Alert” is reset if the time interval is less than the desired one.
-    # maybe a new function like "apply_notifivation_delay()""
-
     # Web Service Monitoring
     if SCAN_WEBSERVICES:
         if str(startTime)[15] == "0":
@@ -909,8 +903,6 @@ def scan_network():
         print('\nLooking for Rogue DHCP Servers...')
         rogue_dhcp_detection()
 
-
-
     return 0
 
 #-------------------------------------------------------------------------------
@@ -1385,31 +1377,20 @@ def read_openwrt_clients():
         print('        Missing python package')
         return
 
-    
     try:
         router = OpenWrtRpc(str(OPENWRT_IP), str(OPENWRT_USER), str(OPENWRT_PASS))
         result = router.get_all_connected_devices(only_reachable=True)
 
-        # devices_info = []
-
         for device in result:
             if str(device.hostname) == 'None':
                 hostname = '(unknown)'
             else:
                 hostname = device.hostname
 
-            # device_data = {
-            #     'mac': device.mac,
-            #     'hostname': hostname,
-            #     'ip': device.ip
-            # }
-            # devices_info.append(device_data)
-
             sql.execute ("INSERT INTO Openwrt_Network (OWRT_MAC, OWRT_IP, UF_Name, OWRT_Vendor) "+
-                         "VALUES (?, ?, ?, ?) ", (device.mac, device.ip, hostname, '(unknown)') )
+                         "VALUES (?, ?, ?, ?) ", (device.mac.lower(), device.ip, hostname, '(unknown)') )
 
     except Exception as e:
-        #print(f"Es ist ein Fehler aufgetreten")
         print(f"Error")
 
 #-------------------------------------------------------------------------------
@@ -1566,7 +1547,7 @@ def process_satellites(satellite_list):
 
                 for result in data['scan_results']:
                     if result['cur_ScanMethod'] != 'Internet Check':
-                        sat_MAC = result['cur_MAC']
+                        sat_MAC = result['cur_MAC'].lower()
                         sat_IP = result['cur_IP']
                         sat_hostname = result['cur_hostname']
                         sat_Vendor = result['cur_Vendor']
@@ -1654,6 +1635,42 @@ def cleanup_satellite_scans(satellite_list):
         if os.path.exists(WORKING_DIR+token+".json"):
             os.remove(WORKING_DIR+token+".json") 
 
+#-------------------------------------------------------------------------------
+def update_scan_validation():
+    print('    Update Scan Validation...')
+    # 1. set dev_Scan_Validation_State to 0 for devices that are in CurrentScan and have dev_Scan_Validation > 0
+    sql.execute("""
+        UPDATE Devices
+        SET dev_Scan_Validation_State = 0
+        WHERE dev_Scan_Validation > 0
+        AND dev_MAC IN (SELECT cur_MAC FROM CurrentScan)
+    """)
+    
+    # 2. find devices to be inserted in CurrentScan and save them in a list
+    sql.execute("""
+        SELECT dev_ScanCycle, dev_MAC, dev_LastIP, dev_Vendor, dev_ScanSource
+        FROM Devices
+        WHERE dev_Scan_Validation > 0
+        AND dev_Scan_Validation_State < dev_Scan_Validation
+        AND dev_MAC NOT IN (SELECT cur_MAC FROM CurrentScan)
+    """)
+    devices_to_insert = sql.fetchall()
+    
+    # 3. Add the devices to CurrentScan
+    sql.executemany("""
+        INSERT INTO CurrentScan (cur_ScanCycle, cur_MAC, cur_IP, cur_Vendor, cur_ScanMethod, cur_ScanSource)
+        VALUES (?, ?, ?, ?, NULL, ?)
+    """, devices_to_insert)
+    
+    # 4. increase dev_Scan_Validation_State by 1 for the devices saved in point 2
+    mac_addresses = [device[1] for device in devices_to_insert]
+    if mac_addresses:
+        sql.executemany("""
+            UPDATE Devices
+            SET dev_Scan_Validation_State = dev_Scan_Validation_State + 1
+            WHERE dev_MAC = ?
+        """, [(mac,) for mac in mac_addresses])
+
 #-------------------------------------------------------------------------------
 def save_scanned_devices(p_arpscan_devices, p_cycle_interval):
     # Delete previous scan data
@@ -1694,6 +1711,9 @@ def save_scanned_devices(p_arpscan_devices, p_cycle_interval):
                                       WHERE cur_MAC = Sat_MAC )""",
                     (cycle) )
 
+    # Scan Validation
+    update_scan_validation()
+
     if not OFFLINE_MODE :
         # Check Internet connectivity
         internet_IP = get_internet_IP()
@@ -2537,7 +2557,6 @@ def rogue_dhcp_detection():
     # Flush Table
     sql.execute("DELETE FROM Nmap_DHCP_Server")
     sql_connection.commit()
-
     closeDB()
 
     # Execute 15 probes and insert in list
@@ -2547,7 +2566,6 @@ def rogue_dhcp_detection():
     for _ in range(dhcp_probes):
         stream = os.popen('sudo nmap --script broadcast-dhcp-discover 2>/dev/null | grep "Server Identifier" | awk \'{ print $4 }\'')
         output = stream.read()
-        # dhcp_server_list.append(output.replace("\n", ""))
 
         multiple_dhcp_ips = output.split("\n")
 
@@ -3090,12 +3108,8 @@ def icmp_monitoring():
 
     closeDB()
     scantime = startTime.strftime("%Y-%m-%d %H:%M")
-
     icmp_scan_results = {}
-
     icmphosts_all = len(icmphosts)
-    icmphosts_online = 0
-    icmphosts_offline = 0
 
     try:
         ping_retries = ICMP_ONLINE_TEST
@@ -3108,18 +3122,14 @@ def icmp_monitoring():
         while icmphosts_index < icmphosts_all:
             host_ip = icmphosts[icmphosts_index]
             for i in range(ping_retries):
-                # print("Host %s retry %s" % (host_ip, str(i+1)))
                 icmp_status = ping(host_ip)
                 if icmp_status == "1":
                     break;
 
             if icmp_status == "1":
                 icmp_rtt = ping_avg(host_ip)
-                # print("Host %s RTT %s" % (host_ip, str(icmp_rtt)))
-                icmphosts_online+=1
             else:
                 icmp_rtt = "99999"
-                icmphosts_offline+=1
 
             current_data = {
                 "host_ip": host_ip,
@@ -3133,18 +3143,20 @@ def icmp_monitoring():
 
             icmphosts_index += 1
 
-        print("        Online Host(s)  : " + str(icmphosts_online))
-        print("        Offline Host(s) : " + str(icmphosts_offline))
-
         openDB()
         # Save Scan Results
         icmp_save_scandata(icmp_scan_results)
 
+        update_icmp_validation()
+        online, offline = get_online_offline_hosts()
+        print("        Online Host(s)  : " + str(online))
+        print("        Offline Host(s) : " + str(offline))
+
         print("    Create Events...")
         icmp_create_events()
 
         print("    Calculate Activity History...")
-        calc_activity_history_icmp(icmphosts_online, icmphosts_offline)
+        calc_activity_history_icmp(online, offline)
 
         sql_connection.commit()
         closeDB()
@@ -3153,6 +3165,67 @@ def icmp_monitoring():
         # openDB()
         print("    No Hosts(s) to monitor!")
 
+
+#-------------------------------------------------------------------------------
+def get_online_offline_hosts():
+    sql.execute("""
+        SELECT COUNT(*) 
+        FROM ICMP_Mon_CurrentScan 
+        WHERE cur_Present = 1
+    """)
+    icmphosts_online = sql.fetchone()[0]
+
+    sql.execute("""
+        SELECT COUNT(*) 
+        FROM ICMP_Mon_CurrentScan 
+        WHERE cur_Present = 0
+    """)
+    icmphosts_offline = sql.fetchone()[0]
+
+    return icmphosts_online, icmphosts_offline
+
+#-------------------------------------------------------------------------------
+def update_icmp_validation():
+    print('    Update ICMP Validation...')
+    # 1. Set dev_Scan_Validation_State to 0 for devices that are in Present in CurrentScan and have dev_Scan_Validation > 0
+    sql.execute("""
+        UPDATE ICMP_Mon
+        SET icmp_Scan_Validation_State = 0
+        WHERE icmp_Scan_Validation > 0
+        AND icmp_ip IN (
+            SELECT cur_ip FROM ICMP_Mon_CurrentScan WHERE cur_Present = 1
+        );
+    """)
+    # 2. Find devices in CurrentScan that have activated Scan_Validation and are not currently active
+    sql.execute("""
+        SELECT cur_ip 
+        FROM ICMP_Mon_CurrentScan 
+        WHERE cur_Present = 0 
+        AND cur_ip IN (
+            SELECT icmp_ip 
+            FROM ICMP_Mon 
+            WHERE icmp_Scan_Validation > 0 
+            AND icmp_Scan_Validation_State < icmp_Scan_Validation
+        )
+    """)
+    host_ips = [(row[0],) for row in sql.fetchall()]
+    # 3. Set the relevant devices as online
+    sql.executemany("""
+        UPDATE ICMP_Mon_CurrentScan 
+        SET cur_Present = 1, cur_PresentChanged = 0, cur_avgrrt = 999
+        WHERE cur_ip = ?
+    """, host_ips)
+    # 4. increase dev_Scan_Validation_State by 1 for the devices saved in point 2
+    sql.executemany("""
+        UPDATE ICMP_Mon
+        SET icmp_Scan_Validation_State = icmp_Scan_Validation_State + 1,
+            icmp_PresentLastScan = 1,
+            icmp_avgrtt = 999
+        WHERE icmp_Scan_Validation > 0 AND icmp_ip = ?
+    """, host_ips)
+
+    sql_connection.commit()
+
 # -----------------------------------------------------------------------------
 def icmp_save_scandata(data):
     print("    Save scan results...")
@@ -3163,7 +3236,6 @@ def icmp_save_scandata(data):
 
 # -----------------------------------------------------------------------------
 def icmp_create_events():
-
     # Check new connections
     print_log ('Events - New Connections')
     sql.execute ("""INSERT INTO ICMP_Mon_Connections (icmpeve_ip, icmpeve_DateTime, icmpeve_Present, icmpeve_EventType)
@@ -3646,9 +3718,6 @@ def email_reporting():
         '  <td> <a href="{}{}"> {} </a>  </td><td> {} </td>'+ \
         '  <td> {} </td><td> {} </td><td> {} </td></tr>\n'
 
-    # Issue #370
-    # AND eve_DateTime < datetime('now', '-{DELAY} minutes') for devices where dev_alarm_delay is true
-
     sql.execute ("""SELECT * FROM Events_Devices
                     WHERE eve_PendingAlertEmail = 1
                       AND eve_EventType = 'Device Down'
@@ -3740,19 +3809,6 @@ def email_reporting():
     sql.execute("""UPDATE Events SET eve_PendingAlertEmail = 0
                    WHERE eve_PendingAlertEmail = 1""")
 
-    # Issue #370
-    # Clean Pending Alert Events
-    # sql.execute("""UPDATE Devices SET dev_LastNotification = ?
-    #                WHERE dev_MAC IN (SELECT eve_MAC FROM Events
-    #                   WHERE eve_PendingAlertEmail = 1 AND eve_EventType =='Device Down' 
-    #                 AND eve_DateTime < datetime('now', '-{DELAY} minutes')
-    #         """, (datetime.datetime.now(),))
-    # sql.execute ("""UPDATE Events SET eve_PendingAlertEmail = 0
-    #                 WHERE eve_PendingAlertEmail = 1 
-    #                 AND eve_EventType =='Device Down' 
-    #                 AND eve_DateTime < datetime('now', '-{DELAY} minutes')
-    #         """)
-
     # Set Notification Presets
     sql.execute("""UPDATE Devices SET dev_AlertEvents = ?, dev_AlertDeviceDown = ?
                    WHERE dev_NewDevice = 1

config/version.conf

@@ -1,3 +1,3 @@
 VERSION           = ''
 VERSION_YEAR      = '2025'
-VERSION_DATE      = '2025-02-06'
+VERSION_DATE      = '2025-02-16'

db/pialert.db

@@ -23,6 +23,7 @@ To edit device information:
   - **Serial Number**: Here you can enter any existing serial number for unique identification
   - **Group**: Select a grouper ('Always on', 'Personal', Friends') or type your own Group name
   - **Location**: Select a location ('Hall', 'Kitchen', ....) of the device
+  - **Favorite**: Mark the device as favorite and then it will appears at the begining of the device list
   - **Comments**: Type any comments for the device
 
 ## Session Info
@@ -51,8 +52,9 @@ To edit device information:
       notification to this device for X hours
     - *(Useful to avoid notification saturation on devices that frequently
       connects and disconnects)*
-  - **Favorite**: Mark the device as favorite and then it will appears at the
-    begining of the device list
+  - **Scan Validation**: The "Scan Validation" parameter determines how many consecutive scans 
+      must detect a device as offline before its status is officially changed to "offline" or 
+      "down." If a validation is pending, the device is marked as "online*" (with an asterisk as an indicator).
   - **Show on "Presence"**: The device can be hidden on the “Presence” page, 
     e.g. to increase the overview or because presence tracking is not necessary.
 

docs/DEVICE_MANAGEMENT.md

@@ -24,6 +24,7 @@ Various badges can appear in the "Status" column:
  - "red" = offline, "Down" notification activated
  - "gray" = offline
  - "green" = online
+ - "green" = online*, pending Scan Validation
  - "yellow, green" = new and online
  - "yellow, gray" = new and offline