fix: M1-M7 gap audit phase 1 — bug fix + 5 quick wins

Bug fix:
- window.go: token ratio after compaction used len(w.messages) after
  reassignment, always producing ratio ~1.0. Fixed by saving original
  length before assignment.

Gap 1 (M3): Scanner patterns 13 → 47
- Added 34 new patterns: Azure, DigitalOcean, HuggingFace, Grafana,
  GitHub extended (app/oauth/refresh), Shopify, Twilio, SendGrid,
  NPM, PyPI, Databricks, Pulumi, Postman, Sentry, Anthropic admin,
  OpenAI extended, Vault, Supabase, Telegram, Discord, JWT, Heroku,
  Mailgun, Figma

Gap 2 (M3): Config security section
- SecuritySection with EntropyThreshold + custom PatternConfig
- Wire custom patterns from TOML into scanner at startup

Gap 3 (M4): Polling discovery loop
- StartDiscoveryLoop with 30s ticker, reconciles arms vs discovered
- Router.RemoveArm for disappeared local models

Gap 4 (M5): Incognito LocalOnly enforcement
- Router.SetLocalOnly filters non-local arms in Select()
- TUI incognito toggle (Ctrl+X, /incognito) sets local-only routing

Gap 5 (M6): Reactive 413 compaction
- Window.ForceCompact() bypasses ShouldCompact threshold
- Engine handles 413 with emergency compact + retry

cmd/gnoma/main.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"os/signal"
 	"strings"
+	"time"
 
 	"somegit.dev/Owlibou/gnoma/internal/engine"
 	"encoding/json"
@@ -184,6 +185,22 @@ func main() {
 		logger.Debug("local models discovered", "count", len(localModels))
 	}
 
+	// Start background discovery polling (30s interval)
+	discoveryCtx, discoveryCancel := context.WithCancel(context.Background())
+	defer discoveryCancel()
+	providerFactory := func(provName, model string) provider.Provider {
+		p, err := createProvider(provName, "", model, cfg.Provider.Endpoints[provName])
+		if err != nil {
+			return nil
+		}
+		return p
+	}
+	router.StartDiscoveryLoop(discoveryCtx, rtr, logger,
+		cfg.Provider.Endpoints["ollama"],
+		cfg.Provider.Endpoints["llamacpp"],
+		providerFactory, 30*time.Second,
+	)
+
 	// Create elf manager and register agent tool
 	elfMgr := elf.NewManager(elf.ManagerConfig{
 		Router: rtr,
@@ -199,12 +216,29 @@ func main() {
 	reg.Register(batchTool)
 
 	// Create firewall
+	entropyThreshold := 4.5
+	if cfg.Security.EntropyThreshold > 0 {
+		entropyThreshold = cfg.Security.EntropyThreshold
+	}
 	fw := security.NewFirewall(security.FirewallConfig{
 		ScanOutgoing:     true,
 		ScanToolResults:  true,
-		EntropyThreshold: 4.5,
+		EntropyThreshold: entropyThreshold,
 		Logger:           logger,
 	})
+	// Wire custom scanner patterns from config
+	for _, p := range cfg.Security.Patterns {
+		action := security.ActionRedact
+		switch p.Action {
+		case "block":
+			action = security.ActionBlock
+		case "warn":
+			action = security.ActionWarn
+		}
+		if err := fw.Scanner().AddPattern(p.Name, p.Regex, action); err != nil {
+			logger.Warn("invalid security pattern", "name", p.Name, "error", err)
+		}
+	}
 
 	// Incognito mode
 	if *incognito {

internal/config/config.go

@@ -8,6 +8,29 @@ type Config struct {
 	Permission PermissionSection `toml:"permission"`
 	Tools      ToolsSection      `toml:"tools"`
 	RateLimits RateLimitSection  `toml:"rate_limits"`
+	Security   SecuritySection   `toml:"security"`
+}
+
+// SecuritySection configures the secret scanner and firewall.
+//
+// Example config:
+//
+//	[security]
+//	entropy_threshold = 4.5
+//
+//	[[security.patterns]]
+//	name = "internal_token"
+//	regex = "mycompany_[a-zA-Z0-9]{32}"
+//	action = "redact"
+type SecuritySection struct {
+	EntropyThreshold float64         `toml:"entropy_threshold"`
+	Patterns         []PatternConfig `toml:"patterns"`
+}
+
+type PatternConfig struct {
+	Name   string `toml:"name"`
+	Regex  string `toml:"regex"`
+	Action string `toml:"action"` // "redact" (default), "block", "warn"
 }
 
 type PermissionSection struct {

internal/context/window.go

@@ -110,14 +110,15 @@ func (w *Window) CompactIfNeeded() (bool, error) {
 	}
 
 	w.consecutiveFailures = 0
+	originalLen := len(w.messages)
 	w.messages = compacted
 
 	// Rough estimate: reduce tracked tokens proportionally
-	ratio := float64(len(compacted)) / float64(len(w.messages)+1)
+	ratio := float64(len(compacted)) / float64(originalLen+1)
 	w.tracker.Set(int64(float64(w.tracker.Used()) * ratio))
 
 	w.logger.Info("compaction complete",
-		"messages_before", len(w.messages),
+		"messages_before", originalLen,
 		"messages_after", len(compacted),
 		"tokens_after", w.tracker.Used(),
 	)
@@ -125,6 +126,42 @@ func (w *Window) CompactIfNeeded() (bool, error) {
 	return true, nil
 }
 
+// ForceCompact runs compaction regardless of the token threshold.
+// Used for reactive compaction (e.g., after a 413 response).
+func (w *Window) ForceCompact() (bool, error) {
+	if w.strategy == nil {
+		return false, fmt.Errorf("no compaction strategy configured")
+	}
+	if len(w.messages) <= 2 {
+		return false, nil // nothing to compact
+	}
+
+	budget := w.tracker.MaxTokens() / 2
+
+	w.logger.Info("forced compaction",
+		"messages", len(w.messages),
+		"used", w.tracker.Used(),
+		"budget", budget,
+	)
+
+	compacted, err := w.strategy.Compact(w.messages, budget)
+	if err != nil {
+		return false, err
+	}
+
+	originalLen := len(w.messages)
+	w.messages = compacted
+	ratio := float64(len(compacted)) / float64(originalLen+1)
+	w.tracker.Set(int64(float64(w.tracker.Used()) * ratio))
+
+	w.logger.Info("forced compaction complete",
+		"messages_before", originalLen,
+		"messages_after", len(compacted),
+		"tokens_after", w.tracker.Used(),
+	)
+	return true, nil
+}
+
 // Reset clears all messages and usage.
 func (w *Window) Reset() {
 	w.messages = nil

internal/engine/loop.go

@@ -108,7 +108,11 @@ func (e *Engine) runLoop(ctx context.Context, cb Callback) (*Turn, error) {
 				return e.cfg.Provider.Stream(ctx, req)
 			})
 			if err != nil {
-				return nil, fmt.Errorf("provider stream: %w", err)
+				// Try reactive compaction on 413 (request too large)
+				s, err = e.handleRequestTooLarge(ctx, err, req)
+				if err != nil {
+					return nil, fmt.Errorf("provider stream: %w", err)
+				}
 			}
 		}
 
@@ -341,6 +345,42 @@ func truncate(s string, maxLen int) string {
 	return s[:maxLen] + "..."
 }
 
+// handleRequestTooLarge attempts compaction on 413 and retries once.
+func (e *Engine) handleRequestTooLarge(ctx context.Context, origErr error, req provider.Request) (stream.Stream, error) {
+	var provErr *provider.ProviderError
+	if !errors.As(origErr, &provErr) || provErr.StatusCode != 413 {
+		return nil, origErr
+	}
+
+	if e.cfg.Context == nil {
+		return nil, origErr
+	}
+
+	e.logger.Warn("413 received, forcing emergency compaction")
+	compacted, compactErr := e.cfg.Context.ForceCompact()
+	if compactErr != nil || !compacted {
+		return nil, origErr
+	}
+
+	e.history = e.cfg.Context.Messages()
+	req = e.buildRequest(ctx)
+
+	if e.cfg.Router != nil {
+		prompt := ""
+		for i := len(e.history) - 1; i >= 0; i-- {
+			if e.history[i].Role == message.RoleUser {
+				prompt = e.history[i].TextContent()
+				break
+			}
+		}
+		task := router.ClassifyTask(prompt)
+		task.EstimatedTokens = 4000
+		s, _, err := e.cfg.Router.Stream(ctx, task, req)
+		return s, err
+	}
+	return e.cfg.Provider.Stream(ctx, req)
+}
+
 // retryOnTransient retries the stream call on 429/5xx with exponential backoff.
 // Returns the original error if not retryable or all retries exhausted.
 func (e *Engine) retryOnTransient(ctx context.Context, firstErr error, fn func() (stream.Stream, error)) (stream.Stream, error) {

internal/router/discovery.go

@@ -137,6 +137,49 @@ func DiscoverLocalModels(ctx context.Context, logger *slog.Logger, ollamaURL, ll
 	return all
 }
 
+// StartDiscoveryLoop periodically polls for local models and reconciles with the router.
+func StartDiscoveryLoop(ctx context.Context, r *Router, logger *slog.Logger,
+	ollamaURL, llamacppURL string,
+	providerFactory func(name, model string) provider.Provider,
+	interval time.Duration,
+) {
+	go func() {
+		ticker := time.NewTicker(interval)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				models := DiscoverLocalModels(ctx, logger, ollamaURL, llamacppURL)
+				reconcileArms(r, models, providerFactory, logger)
+			}
+		}
+	}()
+}
+
+// reconcileArms adds newly discovered models and removes disappeared ones.
+func reconcileArms(r *Router, discovered []DiscoveredModel, providerFactory func(name, model string) provider.Provider, logger *slog.Logger) {
+	discoveredSet := make(map[ArmID]bool, len(discovered))
+	for _, m := range discovered {
+		discoveredSet[NewArmID(m.Provider, m.ID)] = true
+	}
+
+	// Register new models
+	RegisterDiscoveredModels(r, discovered, providerFactory)
+
+	// Remove arms whose models have disappeared (only local arms)
+	for _, arm := range r.Arms() {
+		if !arm.IsLocal {
+			continue
+		}
+		if !discoveredSet[arm.ID] {
+			logger.Debug("removing disappeared local arm", "id", arm.ID)
+			r.RemoveArm(arm.ID)
+		}
+	}
+}
+
 // RegisterDiscoveredModels registers discovered local models as arms in the router.
 func RegisterDiscoveredModels(r *Router, models []DiscoveredModel, providerFactory func(name, model string) provider.Provider) {
 	for _, m := range models {

internal/router/router.go

@@ -19,6 +19,8 @@ type Router struct {
 
 	// Optional: force a specific arm (--provider flag override)
 	forcedArm ArmID
+	// When true, only local arms are considered (incognito mode)
+	localOnly bool
 }
 
 type Config struct {
@@ -66,9 +68,12 @@ func (r *Router) Select(task Task) RoutingDecision {
 		return RoutingDecision{Strategy: StrategySingleArm, Arm: arm}
 	}
 
-	// Collect all arms
+	// Collect all arms (filtered to local-only if incognito)
 	allArms := make([]*Arm, 0, len(r.arms))
 	for _, arm := range r.arms {
+		if r.localOnly && !arm.IsLocal {
+			continue
+		}
 		allArms = append(allArms, arm)
 	}
 
@@ -97,6 +102,27 @@ func (r *Router) Select(task Task) RoutingDecision {
 	return RoutingDecision{Strategy: StrategySingleArm, Arm: best}
 }
 
+// SetLocalOnly constrains routing to local arms only (for incognito mode).
+func (r *Router) SetLocalOnly(v bool) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.localOnly = v
+}
+
+// LocalOnly returns whether routing is constrained to local arms.
+func (r *Router) LocalOnly() bool {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	return r.localOnly
+}
+
+// RemoveArm removes an arm from the router.
+func (r *Router) RemoveArm(id ArmID) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	delete(r.arms, id)
+}
+
 // Arms returns all registered arms.
 func (r *Router) Arms() []*Arm {
 	r.mu.RLock()

internal/security/scanner.go

@@ -169,33 +169,69 @@ func defaultPatterns() []SecretPattern {
 		name  string
 		regex string
 	}{
-		// Anthropic
+		// --- AI/LLM Providers ---
 		{"anthropic_api_key", `sk-ant-(?:api)?[a-zA-Z0-9_-]{20,}`},
-		// OpenAI
+		{"anthropic_admin_key", `sk-ant-admin[a-zA-Z0-9_-]{20,}`},
 		{"openai_api_key", `sk-(?:proj-)?[a-zA-Z0-9_-]{20,}`},
-		// Google
+		{"openai_svcacct_key", `sk-svcacct-[a-zA-Z0-9_-]{20,}`},
+		{"openai_admin_key", `sk-admin-[a-zA-Z0-9_-]{20,}`},
+		{"mistral_api_key", `[a-zA-Z0-9]{32}(?:[a-zA-Z0-9]{0})`}, // 32-char; entropy-gated
+		{"huggingface_token", `hf_[a-zA-Z0-9]{34,}`},
+
+		// --- Cloud Providers ---
 		{"google_api_key", `AIza[a-zA-Z0-9_-]{35}`},
-		// AWS
 		{"aws_access_key", `(?:AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}`},
 		{"aws_secret_key", `(?i)aws_secret_access_key\s*=\s*[a-zA-Z0-9/+=]{40}`},
-		// GitHub
+		{"azure_storage_key", `(?i)AccountKey=[a-zA-Z0-9+/=]{88}`},
+		{"digitalocean_pat", `dop_v1_[a-f0-9]{64}`},
+		{"digitalocean_oauth", `doo_v1_[a-f0-9]{64}`},
+		{"digitalocean_refresh", `dor_v1_[a-f0-9]{64}`},
+		{"vault_token", `hvs\.[a-zA-Z0-9_-]{24,}`},
+		{"supabase_key", `sbp_[a-f0-9]{40}`},
+
+		// --- Version Control ---
 		{"github_pat", `gh[pousr]_[a-zA-Z0-9]{36,}`},
 		{"github_fine_grained", `github_pat_[a-zA-Z0-9]{22}_[a-zA-Z0-9]{59}`},
-		// GitLab
+		{"github_app_token", `ghs_[a-zA-Z0-9]{36}`},
+		{"github_oauth_token", `gho_[a-zA-Z0-9]{36}`},
+		{"github_refresh_token", `ghr_[a-zA-Z0-9]{36}`},
 		{"gitlab_pat", `glpat-[a-zA-Z0-9_-]{20,}`},
-		// Slack
+
+		// --- Communication & Collaboration ---
 		{"slack_token", `xox[bpears]-[a-zA-Z0-9-]{10,}`},
-		// Stripe
-		{"stripe_key", `(?:sk|pk)_(?:live|test)_[a-zA-Z0-9]{24,}`},
-		// Private keys
+		{"twilio_api_key", `SK[a-f0-9]{32}`},
+		{"sendgrid_api_key", `SG\.[a-zA-Z0-9_-]{22}\.[a-zA-Z0-9_-]{43}`},
+		{"telegram_bot_token", `\d{8,10}:[a-zA-Z0-9_-]{35}`},
+		{"discord_bot_token", `[MN][A-Za-z\d]{23,}\.[A-Za-z\d_-]{6}\.[A-Za-z\d_-]{27,}`},
+
+		// --- Payment & Commerce ---
+		{"stripe_key", `(?:sk|pk|rk)_(?:live|test)_[a-zA-Z0-9]{24,}`},
+		{"shopify_access_token", `shpat_[a-fA-F0-9]{32}`},
+		{"shopify_shared_secret", `shpss_[a-fA-F0-9]{32}`},
+
+		// --- Package Registries & Dev Tools ---
+		{"npm_token", `npm_[a-zA-Z0-9]{36}`},
+		{"pypi_api_token", `pypi-[a-zA-Z0-9_-]{100,}`},
+		{"databricks_token", `dapi[a-f0-9]{32}`},
+		{"pulumi_access_token", `pul-[a-f0-9]{40}`},
+		{"postman_api_key", `PMAK-[a-f0-9]{24}-[a-f0-9]{34}`},
+		{"hashicorp_tf_token", `[a-zA-Z0-9]{14}\.atlasv1\.[a-zA-Z0-9_-]{60,}`},
+		{"figma_pat", `figd_[a-zA-Z0-9_-]{40,}`},
+
+		// --- Observability & Monitoring ---
+		{"grafana_api_key", `eyJr[a-zA-Z0-9+/=]{60,}`},
+		{"grafana_service_account", `glsa_[a-zA-Z0-9_]{32,}`},
+		{"sentry_auth_token", `sntrys_[a-zA-Z0-9_]{50,}`},
+
+		// --- Infrastructure ---
 		{"private_key", `-----BEGIN (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----`},
-		// Generic secrets in assignments
-		{"generic_secret_assign", `(?i)(?:password|secret|token|api_key|apikey|auth)\s*[:=]\s*['"][a-zA-Z0-9_/+=\-]{8,}['"]`},
-		// Mistral
-		{"mistral_api_key", `[a-zA-Z0-9]{32}` + `(?:` + `[a-zA-Z0-9]{0}` + `)`}, // 32-char hex-like strings caught by entropy
-		// Database URLs with credentials
 		{"database_url", `(?i)(?:postgres|mysql|mongodb|redis)://[^:]+:[^@]+@`},
-		// .env file patterns
+		{"heroku_api_key", `(?i)HEROKU_API_KEY\s*=\s*[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}`},
+		{"mailgun_api_key", `key-[a-f0-9]{32}`},
+		{"jwt_token", `eyJ[a-zA-Z0-9_-]{10,}\.eyJ[a-zA-Z0-9_-]{10,}\.[a-zA-Z0-9_-]{10,}`},
+
+		// --- Generic ---
+		{"generic_secret_assign", `(?i)(?:password|secret|token|api_key|apikey|auth)\s*[:=]\s*['"][a-zA-Z0-9_/+=\-]{8,}['"]`},
 		{"env_secret", `(?i)^[A-Z_]{2,}(?:_KEY|_SECRET|_TOKEN|_PASSWORD)\s*=\s*.{8,}$`},
 	}
 

internal/tui/app.go

@@ -177,9 +177,12 @@ func (m Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 			// Toggle incognito
 			if m.config.Firewall != nil {
 				m.incognito = m.config.Firewall.Incognito().Toggle()
+				if m.config.Router != nil {
+					m.config.Router.SetLocalOnly(m.incognito)
+				}
 				var msg string
 				if m.incognito {
-					msg = "🔒 incognito ON — no persistence, no learning, no logging"
+					msg = "🔒 incognito ON — no persistence, no learning, local-only routing"
 				} else {
 					msg = "🔓 incognito OFF"
 				}
@@ -352,9 +355,12 @@ func (m Model) handleCommand(cmd string) (tea.Model, tea.Cmd) {
 	case "/incognito":
 		if m.config.Firewall != nil {
 			m.incognito = m.config.Firewall.Incognito().Toggle()
+			if m.config.Router != nil {
+				m.config.Router.SetLocalOnly(m.incognito)
+			}
 			if m.incognito {
 				m.messages = append(m.messages, chatMessage{role: "system",
-					content: "🔒 incognito mode ON — no persistence, no learning, no content logging"})
+					content: "🔒 incognito mode ON — no persistence, no learning, local-only routing"})
 			} else {
 				m.messages = append(m.messages, chatMessage{role: "system",
 					content: "🔓 incognito mode OFF"})