Optimized background tracker

conduit.sh

@@ -1517,62 +1517,104 @@ AWK_BIN=$(command -v gawk 2>/dev/null || command -v awk 2>/dev/null || echo "awk
 LOCAL_IP=$(ip route get 1.1.1.1 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="src") print $(i+1)}')
 [ -z "$LOCAL_IP" ] && LOCAL_IP=$(hostname -I 2>/dev/null | awk '{print $1}')
 
-# Main capture loop: tcpdump -> awk -> process
+# Batch process: resolve GeoIP + merge into cumulative files in bulk
-while true; do
+process_batch() {
-    # Process tcpdump output in awk, sync every 15 seconds
+    local batch="$1"
-    while IFS= read -r line; do
+    local resolved="$PERSIST_DIR/resolved_batch"
-        if [ "$line" = "SYNC_MARKER" ]; then
+    local geo_map="$PERSIST_DIR/geo_map"
-            continue
+
+    # Step 1: Extract unique IPs and bulk-resolve GeoIP
+    # Read cache once, resolve uncached, produce ip|country mapping
+    $AWK_BIN -F'|' '{print $2}' "$batch" | sort -u > "$PERSIST_DIR/batch_ips"
+
+    # Build geo mapping: read cache + resolve missing
+    > "$geo_map"
+    while IFS= read -r ip; do
+        [ -z "$ip" ] && continue
+        country=""
+        if [ -f "$GEOIP_CACHE" ]; then
+            country=$(grep "^${ip}|" "$GEOIP_CACHE" 2>/dev/null | head -1 | cut -d'|' -f2)
         fi
-        # Parse: direction|IP|bytes
+        if [ -z "$country" ]; then
-        local_dir=$(echo "$line" | cut -d'|' -f1)
+            country=$(geo_lookup "$ip")
-        local_ip_addr=$(echo "$line" | cut -d'|' -f2)
+        fi
-        local_bytes=$(echo "$line" | cut -d'|' -f3)
+        # Normalize
-        [ -z "$local_ip_addr" ] && continue
-
-        # Resolve country
-        country=$(geo_lookup "$local_ip_addr")
-
-        # Normalize country names
         case "$country" in
             *"Iran, Islamic Republic of"*) country="Iran - #FreeIran" ;;
             *"Moldova, Republic of"*) country="Moldova" ;;
         esac
+        echo "${ip}|${country}" >> "$geo_map"
+    done < "$PERSIST_DIR/batch_ips"
 
-        # Update cumulative data
+    # Step 2: Single awk pass — merge batch into cumulative_data + write snapshot
-        if [ -f "$STATS_FILE" ]; then
+    $AWK_BIN -F'|' -v snap="$SNAPSHOT_FILE" '
-            existing=$(grep "^${country}|" "$STATS_FILE" 2>/dev/null | head -1)
+        FILENAME == ARGV[1] { geo[$1] = $2; next }
-            if [ -n "$existing" ]; then
+        FILENAME == ARGV[2] { existing[$1] = $2 "|" $3; next }
-                old_from=$(echo "$existing" | cut -d'|' -f2)
+        FILENAME == ARGV[3] {
-                old_to=$(echo "$existing" | cut -d'|' -f3)
+            dir = $1; ip = $2; bytes = $3 + 0
-                if [ "$local_dir" = "FROM" ]; then
+            c = geo[ip]
-                    new_from=$((old_from + local_bytes))
+            if (c == "") c = "Unknown"
-                    new_to=$old_to
+            if (dir == "FROM") from_bytes[c] += bytes
-                else
+            else to_bytes[c] += bytes
-                    new_from=$old_from
+            # Also collect snapshot lines
-                    new_to=$((old_to + local_bytes))
+            print dir "|" c "|" bytes "|" ip > snap
-                fi
+            next
-                # Update in place using temp file
+        }
-                grep -v "^${country}|" "$STATS_FILE" > "$STATS_FILE.tmp" 2>/dev/null || true
+        END {
-                echo "${country}|${new_from}|${new_to}" >> "$STATS_FILE.tmp"
+            # Merge existing + new
-                mv "$STATS_FILE.tmp" "$STATS_FILE"
+            for (c in existing) {
-            else
+                split(existing[c], v, "|")
-                if [ "$local_dir" = "FROM" ]; then
+                f = v[1] + 0; t = v[2] + 0
-                    echo "${country}|${local_bytes}|0" >> "$STATS_FILE"
+                f += from_bytes[c] + 0
-                else
+                t += to_bytes[c] + 0
-                    echo "${country}|0|${local_bytes}" >> "$STATS_FILE"
+                print c "|" f "|" t
-                fi
+                delete from_bytes[c]
+                delete to_bytes[c]
+            }
+            # New countries not in existing
+            for (c in from_bytes) {
+                f = from_bytes[c] + 0
+                t = to_bytes[c] + 0
+                print c "|" f "|" t
+                delete to_bytes[c]
+            }
+            for (c in to_bytes) {
+                print c "|0|" to_bytes[c] + 0
+            }
+        }
+    ' "$geo_map" "$STATS_FILE" "$batch" > "$STATS_FILE.tmp" && mv "$STATS_FILE.tmp" "$STATS_FILE"
+
+    # Step 3: Single awk pass — merge batch IPs into cumulative_ips
+    $AWK_BIN -F'|' '
+        FILENAME == ARGV[1] { geo[$1] = $2; next }
+        FILENAME == ARGV[2] { seen[$0] = 1; print; next }
+        FILENAME == ARGV[3] {
+            ip = $2; c = geo[ip]
+            if (c == "") c = "Unknown"
+            key = c "|" ip
+            if (!(key in seen)) { seen[key] = 1; print key }
+        }
+    ' "$geo_map" "$IPS_FILE" "$batch" > "$IPS_FILE.tmp" && mv "$IPS_FILE.tmp" "$IPS_FILE"
+
+    rm -f "$PERSIST_DIR/batch_ips" "$geo_map" "$resolved"
+}
+
+# Main capture loop: tcpdump -> awk -> batch process
+while true; do
+    BATCH_FILE="$PERSIST_DIR/batch_tmp"
+    > "$BATCH_FILE"
+
+    while IFS= read -r line; do
+        if [ "$line" = "SYNC_MARKER" ]; then
+            # Process entire batch at once
+            if [ -s "$BATCH_FILE" ]; then
+                > "$SNAPSHOT_FILE"
+                process_batch "$BATCH_FILE"
             fi
+            > "$BATCH_FILE"
+            continue
         fi
-
+        echo "$line" >> "$BATCH_FILE"
-        # Update cumulative IPs
-        if ! grep -q "^${country}|${local_ip_addr}$" "$IPS_FILE" 2>/dev/null; then
-            echo "${country}|${local_ip_addr}" >> "$IPS_FILE"
-        fi
-
-        # Write snapshot for speed calculation
-        echo "${local_dir}|${country}|${local_bytes}|${local_ip_addr}" >> "$SNAPSHOT_FILE"
-
     done < <($TCPDUMP_BIN -tt -l -ni any -n -q "(tcp or udp) and not port 22" 2>/dev/null | $AWK_BIN -v local_ip="$LOCAL_IP" '
     BEGIN { last_sync = 0 }
     {