Semantic Deduplication

from dakera import DakeraClient

client = DakeraClient(base_url="http://localhost:3000", api_key="dk-...")
AGENT = "research-bot"

# Ingest paper summaries; the same abstract can arrive from several sources
papers = [
    ("BERT: Pre-training of Deep Bidirectional Transformers", 0.85),
    ("BERT: Pre-training of Deep Bidirectional Transformers", 0.82),  # same text, second source
    ("Attention Is All You Need: the transformer architecture", 0.90),
    ("The original transformer paper by Vaswani et al.", 0.88),        # paraphrase of the line above
    ("GPT-3: Language Models are Few-Shot Learners", 0.87),
]
for content, importance in papers:
    client.store_memory(agent_id=AGENT, content=content, importance=importance,
                        memory_type="semantic", tags=["paper", "nlp"])

# Dakera: preview, then merge true duplicates (same words, same order, same entity tags)
preview = client.deduplicate(agent_id=AGENT, dry_run=True)
print(preview)
client.deduplicate(agent_id=AGENT, dry_run=False)

# Your application: merge paraphrases of one fact
def same_fact(a: str, b: str) -> bool:
    """Your rule or an LLM call. Dakera's scores rank results; they are not a similarity cut-off."""
    ...

fact = "The original transformer paper (Attention Is All You Need)"
response = client.recall(agent_id=AGENT, query=fact, top_k=10)
same = [m for m in response.memories if same_fact(m.content, fact)]
if len(same) > 1:
    client.store_memory(agent_id=AGENT, content=fact, memory_type="semantic",
                        importance=max(m.importance for m in same), tags=["paper", "nlp"])
    for m in same:
        client.forget(agent_id=AGENT, memory_id=m.id)

import { DakeraClient } from '@dakera-ai/dakera';

const client = new DakeraClient({ baseUrl: 'http://localhost:3000', apiKey: 'dk-...' });
const AGENT = 'research-bot';

const papers = [
  { content: 'BERT: Pre-training of Deep Bidirectional Transformers', importance: 0.85 },
  { content: 'BERT: Pre-training of Deep Bidirectional Transformers', importance: 0.82 }, // same text
  { content: 'Attention Is All You Need: the transformer architecture', importance: 0.9 },
  { content: 'The original transformer paper by Vaswani et al.', importance: 0.88 },      // paraphrase
];
for (const p of papers) {
  await client.storeMemory(AGENT, { ...p, memory_type: 'semantic', tags: ['paper', 'nlp'] });
}

// Dakera: preview, then merge true duplicates
console.log(await client.deduplicate({ agent_id: AGENT, dry_run: true }));
await client.deduplicate({ agent_id: AGENT, dry_run: false });

// Your application: merge paraphrases you judge to be one fact
declare function sameFact(a: string, b: string): Promise<boolean>; // your rule or an LLM call

const fact = 'The original transformer paper (Attention Is All You Need)';
const { memories } = await client.recall(AGENT, fact, { top_k: 10 });
const same = [];
for (const m of memories) if (await sameFact(m.content, fact)) same.push(m);
if (same.length > 1) {
  await client.storeMemory(AGENT, {
    content: fact,
    memory_type: 'semantic',
    importance: Math.max(...same.map(m => m.importance)),
    tags: ['paper', 'nlp'],
  });
  for (const m of same) await client.forget(AGENT, m.id);
}

use dakera_client::{DakeraClient, DeduplicateRequest, ForgetRequest, RecallRequest, StoreMemoryRequest};

let client = DakeraClient::builder("http://localhost:3000").api_key("dk-...").build()?;
let agent = "research-bot";

for (content, importance) in [
    ("BERT: Pre-training of Deep Bidirectional Transformers", 0.85),
    ("BERT: Pre-training of Deep Bidirectional Transformers", 0.82), // same text
    ("The original transformer paper by Vaswani et al.", 0.88),
] {
    client.store_memory(
        StoreMemoryRequest::new(agent, content)
            .with_importance(importance)
            .with_tags(vec!["paper".into(), "nlp".into()]),
    ).await?;
}

// Dakera: preview, then merge true duplicates
let preview = client.deduplicate(DeduplicateRequest {
    agent_id: agent.into(), threshold: None, memory_type: None, dry_run: true,
}).await?;
println!("{} duplicates found", preview.duplicates_found);
client.deduplicate(DeduplicateRequest {
    agent_id: agent.into(), threshold: None, memory_type: None, dry_run: false,
}).await?;

// Your application: replace paraphrases you judge to be one fact
let fact = "The original transformer paper (Attention Is All You Need)";
let found = client.recall(RecallRequest::new(agent, fact).with_top_k(10)).await?;
let same: Vec<String> = found.memories.iter()
    .filter(|m| same_fact(&m.content, fact)) // your rule or an LLM call
    .map(|m| m.id.clone())
    .collect();
if same.len() > 1 {
    client.store_memory(StoreMemoryRequest::new(agent, fact).with_importance(0.88)).await?;
    client.forget(ForgetRequest::by_ids(agent, same)).await?;
}

ctx := context.Background()
client := dakera.NewClientWithOptions(dakera.ClientOptions{
    BaseURL: "http://localhost:3000",
    APIKey:  os.Getenv("DAKERA_API_KEY"),
})
agent := "research-bot"

for _, content := range []string{
    "BERT: Pre-training of Deep Bidirectional Transformers",
    "BERT: Pre-training of Deep Bidirectional Transformers", // same text
    "The original transformer paper by Vaswani et al.",
} {
    imp := float32(0.85)
    client.StoreMemory(ctx, agent, dakera.StoreMemoryRequest{
        Content: content, Importance: &imp, Tags: []string{"paper", "nlp"},
    })
}

// Dakera: preview, then merge true duplicates
preview, _ := client.Deduplicate(ctx, dakera.DeduplicateRequest{AgentID: agent, DryRun: true})
fmt.Printf("%+v\n", preview)
client.Deduplicate(ctx, dakera.DeduplicateRequest{AgentID: agent})

// Your application: replace paraphrases you judge to be one fact
fact := "The original transformer paper (Attention Is All You Need)"
found, _ := client.Recall(ctx, agent, dakera.RecallRequest{Query: fact, TopK: 10})
var same []string
for _, m := range found.Memories {
    if sameFact(m.Content, fact) { // your rule or an LLM call
        same = append(same, m.ID)
    }
}
if len(same) > 1 {
    imp := float32(0.88)
    client.StoreMemory(ctx, agent, dakera.StoreMemoryRequest{Content: fact, Importance: &imp})
    for _, id := range same {
        client.Forget(ctx, agent, id)
    }
}