Knowledge Base Management

Knowledge Base Management for DEVS

The Knowledge Base system allows users to store and manage files, documents, and other knowledge items that agents can reference during conversations.

Core Concepts

KnowledgeItem Interface

interface KnowledgeItem {
  id: string
  name: string
  type: 'file' | 'folder'
  fileType?: 'document' | 'image' | 'text'
  content?: string // Text content or base64 for files
  contentHash?: string // SHA-256 hash for deduplication
  mimeType?: string
  size?: number
  path: string // Virtual path in knowledge base
  parentId?: string // Parent folder ID
  lastModified: Date
  createdAt: Date
  tags?: string[]
  description?: string
  syncSource?: 'manual' | 'filesystem_api' | 'connector'
  fileSystemHandle?: string // For File System API sync
  watchId?: string
  lastSyncCheck?: Date
}

File Type Detection

Supported file types in src/lib/knowledge-utils.ts:

const TEXT_EXTENSIONS = [
  '.txt',
  '.md',
  '.js',
  '.ts',
  '.jsx',
  '.tsx',
  '.css',
  '.html',
  '.xml',
  '.csv',
  '.yaml',
  '.yml',
  '.log',
]

const IMAGE_EXTENSIONS = [
  '.jpg',
  '.jpeg',
  '.png',
  '.gif',
  '.bmp',
  '.webp',
  '.svg',
  '.ico',
  '.tiff',
  '.tif',
]

const DOCUMENT_EXTENSIONS = [
  '.pdf',
  '.doc',
  '.docx',
  '.xls',
  '.xlsx',
  '.ppt',
  '.pptx',
  '.rtf',
  '.epub',
]

function getFileType(
  filename: string,
): 'text' | 'image' | 'document' | 'unknown' {
  const ext = filename.toLowerCase().split('.').pop()
  if (TEXT_EXTENSIONS.includes(`.${ext}`)) return 'text'
  if (IMAGE_EXTENSIONS.includes(`.${ext}`)) return 'image'
  if (DOCUMENT_EXTENSIONS.includes(`.${ext}`)) return 'document'
  return 'unknown'
}

Database Operations

import { db } from '@/lib/db'

// Create knowledge item
async function createKnowledgeItem(
  item: Omit<KnowledgeItem, 'id'>,
): Promise<string> {
  const id = crypto.randomUUID()
  await db.knowledge.add({ ...item, id })
  return id
}

// Get items by path
async function getItemsByPath(path: string): Promise<KnowledgeItem[]> {
  return db.knowledge.where('path').startsWith(path).toArray()
}

// Get folder contents
async function getFolderContents(folderId?: string): Promise<KnowledgeItem[]> {
  if (folderId) {
    return db.knowledge.where('parentId').equals(folderId).toArray()
  }
  return db.knowledge.filter((item) => !item.parentId).toArray()
}

// Search by content hash (deduplication)
async function findDuplicate(
  contentHash: string,
): Promise<KnowledgeItem | undefined> {
  return db.knowledge.where('contentHash').equals(contentHash).first()
}

Content Hashing for Deduplication

async function generateContentHash(
  content: string | ArrayBuffer,
): Promise<string> {
  const data =
    typeof content === 'string'
      ? new TextEncoder().encode(content)
      : new Uint8Array(content)

  const hashBuffer = await crypto.subtle.digest('SHA-256', data)
  const hashArray = Array.from(new Uint8Array(hashBuffer))
  return hashArray.map((b) => b.toString(16).padStart(2, '0')).join('')
}

File Upload Handling

async function handleFileUpload(
  file: File,
  parentId?: string,
): Promise<KnowledgeItem> {
  const content = await readFileContent(file)
  const contentHash = await generateContentHash(content)

  // Check for duplicates
  const existing = await findDuplicate(contentHash)
  if (existing) {
    toast.info('File already exists in knowledge base')
    return existing
  }

  const item: KnowledgeItem = {
    id: crypto.randomUUID(),
    name: file.name,
    type: 'file',
    fileType: getFileType(file.name),
    content: typeof content === 'string' ? content : btoa(content),
    contentHash,
    mimeType: file.type,
    size: file.size,
    path: parentId ? `${getPath(parentId)}/${file.name}` : `/${file.name}`,
    parentId,
    lastModified: new Date(file.lastModified),
    createdAt: new Date(),
    syncSource: 'manual',
  }

  await db.knowledge.add(item)
  return item
}

async function readFileContent(file: File): Promise<string | ArrayBuffer> {
  return new Promise((resolve, reject) => {
    const reader = new FileReader()

    if (file.type.startsWith('text/') || getFileType(file.name) === 'text') {
      reader.onload = () => resolve(reader.result as string)
      reader.onerror = reject
      reader.readAsText(file)
    } else {
      reader.onload = () => resolve(reader.result as ArrayBuffer)
      reader.onerror = reject
      reader.readAsArrayBuffer(file)
    }
  })
}

Folder Watching (File System API)

import { db } from '@/lib/db'

async function watchFolder(handle: FileSystemDirectoryHandle): Promise<void> {
  // Request permission
  const permission = await handle.requestPermission({ mode: 'read' })
  if (permission !== 'granted') {
    throw new Error('Permission denied')
  }

  // Scan and sync folder
  await syncFolder(handle, undefined)

  // Store handle for persistence
  await db.knowledge.add({
    id: crypto.randomUUID(),
    name: handle.name,
    type: 'folder',
    path: `/${handle.name}`,
    createdAt: new Date(),
    lastModified: new Date(),
    syncSource: 'filesystem_api',
    fileSystemHandle: handle.name,
  })
}

async function syncFolder(
  handle: FileSystemDirectoryHandle,
  parentId?: string,
): Promise<void> {
  for await (const entry of handle.values()) {
    if (entry.kind === 'file') {
      const file = await entry.getFile()
      await handleFileUpload(file, parentId)
    } else if (entry.kind === 'directory') {
      // Create folder and recurse
      const folderId = await createFolder(entry.name, parentId)
      await syncFolder(entry, folderId)
    }
  }
}

Context Injection for Agents

When agents need knowledge context:

import { db } from '@/lib/db'

async function getRelevantKnowledge(
  query: string,
  agentId: string,
): Promise<KnowledgeItem[]> {
  // Get agent's assigned knowledge items
  const agentKnowledge = await getAgentKnowledge(agentId)

  // Search for relevant items
  const searchTerms = query.toLowerCase().split(' ')

  return agentKnowledge.filter((item) => {
    const searchText =
      `${item.name} ${item.description || ''} ${item.tags?.join(' ') || ''}`.toLowerCase()
    return searchTerms.some((term) => searchText.includes(term))
  })
}

function formatKnowledgeForContext(items: KnowledgeItem[]): string {
  return items
    .map((item) => {
      return `## ${item.name}
${item.description || ''}
${item.content || '[Binary content]'}
---`
    })
    .join('\n\n')
}

Document Processing

For processing different document types:

// src/lib/document-processor.ts
export async function processDocument(item: KnowledgeItem): Promise<string> {
  switch (item.fileType) {
    case 'text':
      return item.content || ''

    case 'document':
      return await extractTextFromDocument(item)

    case 'image':
      // Could use OCR or image description
      return `[Image: ${item.name}]`

    default:
      return `[Unsupported file type: ${item.mimeType}]`
  }
}

Component Integration

import { useCallback, useState } from 'react'
import { Button, Card, Progress } from '@heroui/react'
import { Icon } from '@/components/Icon'
import { toast } from '@/lib/toast'

function KnowledgeUploader() {
  const [isUploading, setIsUploading] = useState(false)
  const [progress, setProgress] = useState(0)

  const handleDrop = useCallback(async (e: React.DragEvent) => {
    e.preventDefault()
    const files = Array.from(e.dataTransfer.files)

    setIsUploading(true)
    setProgress(0)

    for (let i = 0; i < files.length; i++) {
      await handleFileUpload(files[i])
      setProgress(((i + 1) / files.length) * 100)
    }

    setIsUploading(false)
    toast.success(`Uploaded ${files.length} files`)
  }, [])

  return (
    <Card
     
      => e.preventDefault()}
      className="border-dashed border-2 p-8 text-center"
    >
      <Icon name="Upload" size={48} className="mx-auto mb-4" />
      <p>Drop files here to upload</p>
      {isUploading && <Progress value={progress} className="mt-4" />}
    </Card>
  )
}

Testing

import { describe, it, expect, beforeEach, vi } from 'vitest'
import { db } from '@/lib/db'

vi.mock('@/lib/db')

describe('Knowledge Base', () => {
  beforeEach(() => {
    vi.clearAllMocks()
  })

  it('should detect file types correctly', () => {
    expect(getFileType('document.md')).toBe('text')
    expect(getFileType('image.png')).toBe('image')
    expect(getFileType('file.pdf')).toBe('document')
    expect(getFileType('data.bin')).toBe('unknown')
  })

  it('should generate consistent content hashes', async () => {
    const content = 'test content'
    const hash1 = await generateContentHash(content)
    const hash2 = await generateContentHash(content)
    expect(hash1).toBe(hash2)
  })

  it('should detect duplicates', async () => {
    const hash = 'abc123'
    vi.mocked(db.knowledge.where).mockReturnValue({
      equals: vi.fn().mockReturnValue({
        first: vi.fn().mockResolvedValue({ id: 'existing' }),
      }),
    } as any)

    const result = await findDuplicate(hash)
    expect(result).toEqual({ id: 'existing' })
  })
})

Converted and distributed by TomeVault — claim your Tome and manage your conversions.