Squashed commit of the following:

commit 53f6d81
Merge: 8c1ea37 5c08355
Author: Calesthio <celesthioailabs@gmail.com>
Date:   Wed Mar 25 10:21:02 2026 -0700

    Merge pull request calesthio#68 from schergr/fix/osint-signal-truncation

    Fix remaining OSINT signal text truncation

commit 5c08355
Author: calesthio <celesthioailabs@gmail.com>
Date:   Tue Mar 24 18:48:55 2026 -0700

    Fix Telegram dedup identity and legacy Markdown escaping

commit b7322f1
Author: Greg Scher <greg@212southadvisors.com>
Date:   Mon Mar 23 13:01:32 2026 -0400

    Fix HTML entity decoding and broaden OSINT dedup window

    - Replace single &calesthio#39; handler with generic numeric/hex entity decoder
      so &calesthio#39; and other unpadded entities are properly converted
    - Dedup urgent OSINT posts against all hot memory runs (last 3 sweeps)
      instead of only the previous sweep, preventing posts that drop out
      of one sweep from reappearing as "new" in the next

    Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

commit 31c305c
Author: Greg Scher <greg@212southadvisors.com>
Date:   Mon Mar 23 12:57:37 2026 -0400

    Escape Markdown in alert signals and cap OSINT text in ideas prompt

    Addresses PR review: escape Markdown-sensitive characters in
    _formatTieredAlert signal bullets to prevent Telegram Bot API
    rejections, and add a 1500-char budget for URGENT_OSINT in
    compactSweepForLLM to bound prompt size while keeping full text upstream.

    Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

commit 2d166c2
Author: Greg Scher <greg@212southadvisors.com>
Date:   Sat Mar 21 12:59:30 2026 -0400

    Remove remaining text truncation across delta engine, memory, and ideas

    The prior fix (753c676) only removed truncation at source ingestion and
    alert formatting. Signals were still being cut to 120 chars in the delta
    engine, 80 chars in memory snapshots, and 120 chars in the ideas LLM
    context — so OSINT posts arrived at the alerter already truncated.

    Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

commit 753c676
Author: Greg Scher <greg@212southadvisors.com>
Date:   Fri Mar 20 16:49:58 2026 -0400

    Remove text truncation limits from Telegram posts

    Posts were being cut to 300 chars (source ingestion) and 150 chars
    (alert evaluation), losing valuable OSINT context. The sendMessage
    chunker already handles the 4096-char Telegram API limit.

    Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: Matt-Drong

apis/sources/telegram.mjs

@@ -94,7 +94,7 @@ export async function getChat(chatId) {
 // Compact a Bot API message for briefing output
 function compactBotMessage(msg) {
   return {
-    text: (msg.text || msg.caption || '').slice(0, 300),
+    text: msg.text || msg.caption || '',
     date: msg.date ? new Date(msg.date * 1000).toISOString() : null,
     chat: msg.chat?.title || msg.chat?.username || 'unknown',
     views: msg.views || 0,
@@ -169,10 +169,12 @@ function parseWebPreview(html, channelId) {
         .replace(/&lt;/g, '<')
         .replace(/&gt;/g, '>')
         .replace(/&quot;/g, '"')
-        .replace(/&#039;/g, "'")
+        .replace(/&#0*39;/g, "'")
+        .replace(/&#x0*27;/gi, "'")
+        .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)))
+        .replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCharCode(parseInt(h, 16)))
         .replace(/&nbsp;/g, ' ')
-        .trim()
-        .slice(0, 300);
+        .trim();
     }
 
     // Extract view count

lib/alerts/telegram.mjs

@@ -273,7 +273,7 @@ export class TelegramAlerter {
         headline: `OSINT Surge: ${osintNew.length} New Urgent Posts`,
         reason: `${osintNew.length} new urgent OSINT signals detected. Elevated conflict reporting tempo.`,
         actionable: 'Review OSINT stream for pattern. Cross-check with satellite and ACLED data.',
-        signals: osintNew.map(s => (s.text || '').substring(0, 40)).slice(0, 3),
+        signals: osintNew.map(s => s.text || s.label || s.key).slice(0, 5),
         crossCorrelation: 'telegram OSINT',
       };
     }
@@ -684,7 +684,7 @@ Respond with ONLY valid JSON:
     if (osintSignals.length > 0) {
       sections.push('📡 OSINT SIGNALS:\n' + osintSignals.map(s => {
         const post = s.item || s;
-        return `  [${post.channel || 'UNKNOWN'}] ${(post.text || s.reason || '').substring(0, 150)}`;
+        return `  [${post.channel || 'UNKNOWN'}] ${post.text || s.reason || ''}`;
       }).join('\n'));
     }
 
@@ -731,7 +731,10 @@ Respond with ONLY valid JSON:
     }
 
     if (evaluation.signals?.length) {
-      lines.push('', `Signals: ${evaluation.signals.join(' · ')}`);
+      lines.push('', `*Signals:*`);
+      for (const sig of evaluation.signals) {
+        lines.push(`• ${escapeMd(sig)}`);
+      }
     }
 
     lines.push('', `_${new Date().toISOString().replace('T', ' ').substring(0, 19)} UTC_`);
@@ -742,7 +745,14 @@ Respond with ONLY valid JSON:
 
 // ─── Helpers ──────────────────────────────────────────────────────────────
 
-export function parseJSON(text) {
+function escapeMd(text) {
+  if (!text) return '';
+  // The bot sends alerts with legacy Markdown parse mode, not MarkdownV2.
+  // Escape only the characters that legacy Markdown actually treats as markup.
+  return text.replace(/([_*`\[])/g, '\\$1');
+}
+
+function parseJSON(text) {
   if (!text) return null;
   let cleaned = text.trim();
   if (cleaned.startsWith('```')) {

lib/delta/engine.mjs

@@ -66,9 +66,9 @@ const RISK_KEYS = ['vix', 'hy_spread', 'urgent_posts', 'conflict_events', 'therm
 // ─── Semantic Hashing for Telegram Posts ─────────────────────────────────────
 
 /**
- * Produce a normalized hash of a post's content.
- * Strips timestamps, normalizes numbers, lowercases — so "BREAKING: 5 missiles at 14:32"
- * and "Breaking: 7 missiles at 15:01" produce the same hash (both are "missile strike" signals).
+ * Produce a normalized semantic hash of a post's content.
+ * This is intentionally lossy and is only safe as a fallback when a stable
+ * post identity is unavailable.
  */
 function contentHash(text) {
   if (!text) return '';
@@ -83,14 +83,34 @@ function contentHash(text) {
   return createHash('sha256').update(normalized).digest('hex').substring(0, 12);
 }
 
+function stablePostKey(post) {
+  if (!post) return '';
+
+  const sourceId = post.postId || post.messageId || '';
+  const channelId = post.channel || post.chat || '';
+  const date = post.date || '';
+  const text = (post.text || '').trim().substring(0, 200);
+
+  if (sourceId) return `id:${sourceId}`;
+  if (channelId && date) {
+    return createHash('sha256')
+      .update(`${channelId}|${date}|${text}`)
+      .digest('hex')
+      .substring(0, 16);
+  }
+
+  return `semantic:${contentHash(post.text)}`;
+}
+
 // ─── Core Delta Computation ──────────────────────────────────────────────────
 
 /**
  * @param {object} current - current sweep's synthesized data
  * @param {object|null} previous - previous sweep's synthesized data (null on first run)
  * @param {object} [thresholdOverrides] - optional: { numeric: {...}, count: {...} }
+ * @param {Array<object>} [priorRuns] - optional compacted prior runs for broader dedup
  */
-export function computeDelta(current, previous, thresholdOverrides = {}) {
+export function computeDelta(current, previous, thresholdOverrides = {}, priorRuns = []) {
   if (!previous) return null;
   if (!current) return null;
 
@@ -152,16 +172,21 @@ export function computeDelta(current, previous, thresholdOverrides = {}) {
 
   // ─── New urgent Telegram posts (semantic dedup) ──────────────────────
 
+  // Dedup against all recent runs (not just the last one) to catch posts that
+  // drop out of one sweep but reappear in a later one. Use stable post identity
+  // where possible so updated posts are not collapsed into earlier alerts just
+  // because their text is semantically similar.
+  const sources = priorRuns.length > 0 ? priorRuns : [previous];
   const prevHashes = new Set(
-    (previous.tg?.urgent || []).map(p => contentHash(p.text))
+    sources.flatMap(run => (run?.tg?.urgent || []).map(stablePostKey)).filter(Boolean)
   );
 
   for (const post of (current.tg?.urgent || [])) {
-    const hash = contentHash(post.text);
+    const hash = stablePostKey(post);
     if (hash && !prevHashes.has(hash)) {
       signals.new.push({
         key: `tg_urgent:${hash}`,
-        text: post.text?.substring(0, 120),
+        text: post.text,
         item: post,
         reason: 'New urgent OSINT post',
       });

lib/delta/memory.mjs

@@ -74,7 +74,9 @@ export class MemoryManager {
   // Add a new run to hot memory
   addRun(synthesizedData) {
     const previous = this.getLastRun();
-    const delta = computeDelta(synthesizedData, previous);
+    // Collect urgent post hashes from all hot runs for broader dedup window
+    const priorRuns = this.hot.runs.map(r => r.data);
+    const delta = computeDelta(synthesizedData, previous, {}, priorRuns);
 
     // Compact the data for storage (strip large arrays)
     const compact = this._compactForStorage(synthesizedData);
@@ -199,7 +201,15 @@ export class MemoryManager {
       bls: data.bls,
       treasury: data.treasury,
       gscpi: data.gscpi,
-      tg: { posts: data.tg?.posts, urgent: (data.tg?.urgent || []).map(p => ({ text: p.text?.substring(0, 80), date: p.date })) },
+      tg: {
+        posts: data.tg?.posts,
+        urgent: (data.tg?.urgent || []).map(p => ({
+          text: p.text,
+          date: p.date,
+          channel: p.channel || p.chat || null,
+          postId: p.postId || null,
+        })),
+      },
       thermal: (data.thermal || []).map(t => ({ region: t.region, det: t.det, night: t.night, hc: t.hc })),
       air: (data.air || []).map(a => ({ region: a.region, total: a.total })),
       nuke: (data.nuke || []).map(n => ({ site: n.site, anom: n.anom, cpm: n.cpm })),

lib/llm/ideas.mjs

@@ -128,10 +128,20 @@ function compactSweepForLLM(data, delta, previousIdeas) {
     sections.push(`SUPPLY_CHAIN: GSCPI=${data.gscpi.value} (${data.gscpi.interpretation})`);
   }
 
-  // Geopolitical signals
+  // Geopolitical signals (cap total OSINT text to ~1500 chars to keep prompt compact)
   const urgentPosts = (data.tg?.urgent || []).slice(0, 5);
   if (urgentPosts.length) {
-    sections.push(`URGENT_OSINT:\n${urgentPosts.map(p => `- ${(p.text || '').substring(0, 120)}`).join('\n')}`);
+    const MAX_OSINT_CHARS = 1500;
+    let remaining = MAX_OSINT_CHARS;
+    const lines = [];
+    for (const p of urgentPosts) {
+      const text = p.text || '';
+      if (remaining <= 0) break;
+      const trimmed = text.length > remaining ? text.substring(0, remaining) + '…' : text;
+      lines.push(`- ${trimmed}`);
+      remaining -= trimmed.length;
+    }
+    sections.push(`URGENT_OSINT:\n${lines.join('\n')}`);
   }
 
   // Thermal / fire detections