🐜 Colony Code Explained

Deep dive into the Ouroboros colony codebase

Project Structure

database.js — SQLite + XNOR/POPCOUNT

The foundation: SQLite database with custom binary similarity functions.

// Popcount lookup table — precompute all 256 byte values
const POPCOUNT_TABLE = new Uint8Array(256);
for (let i = 0; i < 256; i++) {
  let count = 0, n = i;
  while (n) { count++; n &= n - 1; }  // Brian Kernighan's algorithm
  POPCOUNT_TABLE[i] = count;
}

function initDb() {
  if (!_db) {
    _db = new Database(DB_FILE);
    
    // High-performance SQLite settings
    _db.pragma('journal_mode = WAL');      // Concurrent reads + writes
    _db.pragma('synchronous = NORMAL');     // Faster, still safe
    _db.pragma('cache_size = -64000');      // 64MB cache
    _db.pragma('temp_store = MEMORY');       // Temp tables in RAM
    
    // === REGISTER XNOR + POPCOUNT ===
    _db.function('hamming_similarity', { deterministic: true }, (a, b) => {
      if (!Buffer.isBuffer(a) || !Buffer.isBuffer(b)) return 0;
      if (a.length !== b.length) return 0;
      
      let matching = 0;
      for (let i = 0; i < a.length; i++) {
        const xnor = ~(a[i] ^ b[i]) & 0xFF;  // XNOR
        matching += POPCOUNT_TABLE[xnor];     // POPCOUNT via lookup
      }
      return matching / (a.length * 8);       // 0.0 to 1.0
    });
    
    initSchema(_db);
  }
  return _db;
}

embeddings.js — Binary Embedding Generator

Creates 128-bit embeddings from text: 64 domain features + 64 SimHash bits.

// 64 domain-specific features (regex patterns)
const DOMAIN_FEATURES = [
  // Architectures (bits 0-15)
  /transformer|attention/i,          // bit 0
  /mamba|ssm|state.?space/i,         // bit 1
  /rnn|lstm|gru|recurrent/i,         // bit 2
  // ... 61 more patterns ...
];

function embed(text) {
  if (!text) return '00000000000000000000000000000000';
  
  const bits = new Uint8Array(16);  // 128 bits = 16 bytes
  
  // FIRST 64 BITS: Domain features (explicit patterns)
  for (let i = 0; i < 64; i++) {
    if (DOMAIN_FEATURES[i]?.test(text)) {
      bits[Math.floor(i / 8)] |= (1 << (i % 8));
    }
  }
  
  // LAST 64 BITS: SimHash of word n-grams
  const words = text.toLowerCase().split(/\W+/);
  const votes = new Int8Array(64);
  
  for (const word of words) {
    const hash = crypto.createHash('md5').update(word).digest();
    for (let i = 0; i < 64; i++) {
      const bit = (hash[i % 16] >> (i % 8)) & 1;
      votes[i] += bit ? 1 : -1;  // Vote for or against this bit
    }
  }
  
  // Set bits 64-127 based on vote majority
  for (let i = 0; i < 64; i++) {
    if (votes[i] > 0) {
      bits[8 + Math.floor(i / 8)] |= (1 << (i % 8));
    }
  }
  
  return Buffer.from(bits).toString('hex');
}

pheromones-db.js — Pheromone System

Stigmergic communication: ants deposit pheromones, others sense them.

// Decay rates per hour — tuned for colony dynamics
const DECAY_RATES = {
  candidate:    0.25,  // 25%/hr — needs quick validation or dies
  breakthrough: 0.04,  // 4%/hr  — earned knowledge persists
  connection:   0.01,  // 1%/hr  — links are stable
  synapse:      0.005, // 0.5%/hr — reinforced connections persist
  action:       0.20,  // 20%/hr — act fast or forget
};

// Deposit a pheromone (ants leave these as signals)
function deposit(type, claim, metadata = {}) {
  const db = initDb();
  const id = generateId();
  const embedding = embeddings.embed(claim);
  
  db.prepare(`
    INSERT INTO pheromones 
    (pheromone_id, type, claim, embedding, strength, decay_rate)
    VALUES (?, ?, ?, ?, 1.0, ?)
  `).run(id, type, claim, embedding, DECAY_RATES[type] || 0.05);
  
  return id;
}

// Sniff for similar pheromones (using XNOR+POPCOUNT!)
function sniffSimilar(queryText, threshold = 0.65, limit = 20) {
  const db = initDb();
  const queryEmb = embeddings.embedToBlob(queryText);
  
  return db.prepare(`
    SELECT *, hamming_similarity(embedding, ?) as similarity
    FROM pheromones
    WHERE embedding IS NOT NULL
      AND hamming_similarity(embedding, ?) > ?
      AND strength > 0.1
    ORDER BY similarity DESC, strength DESC
    LIMIT ?
  `).all(queryEmb, queryEmb, threshold, limit);
}

// Reinforce with diminishing returns: α/√(n+1)
function reinforce(pheromoneId, boost = 0.2) {
  const db = initDb();
  const row = db.prepare('SELECT * FROM pheromones WHERE pheromone_id = ?').get(pheromoneId);
  
  if (!row) return;
  
  const n = row.reinforcement_count || 1;
  const effectiveBoost = boost / Math.sqrt(n + 1);  // Diminishing returns
  const newStrength = Math.min(2.0, row.strength + effectiveBoost);
  
  db.prepare(`
    UPDATE pheromones 
    SET strength = ?, reinforcement_count = ?
    WHERE pheromone_id = ?
  `).run(newStrength, n + 1, pheromoneId);
}

prompt-cache.js — Nick's Question Memory

Remembers questions to avoid re-explaining the same things.

// Check if a question was asked before
function wasAskedBefore(question, threshold = 0.7) {
  const similar = findSimilar(question, threshold, 1);
  
  if (similar.length > 0) {
    return {
      asked: true,
      when: similar[0].asked_at,
      similarity: similar[0].similarity,
      previousAnswer: similar[0].answer
    };
  }
  return { asked: false };
}

// Store a new Q&A pair
function store(question, answer) {
  const emb = embed(question);
  
  db.prepare(`
    INSERT INTO prompts (question, answer, question_embedding)
    VALUES (?, ?, ?)
  `).run(question, answer, emb);
}

All XNOR+POPCOUNT Databases