import express from 'express' import cors from 'cors' import fs from 'node:fs' import fsp from 'node:fs/promises' import path from 'node:path' import { fileURLToPath } from 'node:url' import { execFile } from 'node:child_process' import { promisify } from 'node:util' const execFileAsync = promisify(execFile) const __filename = fileURLToPath(import.meta.url) const __dirname = path.dirname(__filename) const app = express() app.use(cors()) app.use(express.json({ limit: '1mb' })) const WORKSPACE_DIR = path.join(process.env.HOME || '', '.openclaw/workspace') const MEMORY_DIR = path.join(WORKSPACE_DIR, 'memory') const MAIN_MEMORY_FILE = path.join(WORKSPACE_DIR, 'MEMORY.md') const DIST_DIR = path.join(__dirname, 'dist') const DAILY_MEMORY_PATTERN = /^\d{4}-\d{2}-\d{2}\.md$/ const APPWRITE_ENDPOINT = process.env.APPWRITE_SELF_HOSTED_URL || process.env.APPWRITE_LOCAL_ENDPOINT const APPWRITE_PROJECT_ID = process.env.APPWRITE_SELF_HOSTED_PROJECT_ID || process.env.APPWRITE_LOCAL_PROJECT_ID const APPWRITE_KEY = process.env.APPWRITE_SELF_HOSTED_API_KEY || process.env.APPWRITE_LOCAL_API_KEY const APPWRITE_DATABASE_ID = process.env.APPWRITE_MEMORY_DATABASE_ID || 'freecastle' const APPWRITE_COLLECTION_ID = process.env.APPWRITE_MEMORY_COLLECTION_ID || 'memory_files' const APPWRITE_ENV = { ...process.env, APPWRITE_ENDPOINT, APPWRITE_PROJECT_ID, APPWRITE_KEY, } if (!fs.existsSync(MEMORY_DIR)) { fs.mkdirSync(MEMORY_DIR, { recursive: true }) } function todayDateStamp() { return new Date().toISOString().split('T')[0] } function currentTimeStamp() { return new Date().toTimeString().slice(0, 5) } function isValidDailyMemoryFilename(filename) { return typeof filename === 'string' && DAILY_MEMORY_PATTERN.test(filename) } function isValidMemoryFilename(filename) { return filename === 'MEMORY.md' || isValidDailyMemoryFilename(filename) } function resolveDailyMemoryPath(filename) { if (!isValidDailyMemoryFilename(filename)) return null const resolvedPath = path.resolve(MEMORY_DIR, filename) const memoryDirPrefix = `${path.resolve(MEMORY_DIR)}${path.sep}` return resolvedPath.startsWith(memoryDirPrefix) ? resolvedPath : null } function resolveMemoryPath(filename) { if (filename === 'MEMORY.md') return MAIN_MEMORY_FILE return resolveDailyMemoryPath(filename) } function kindForFilename(filename) { return filename === 'MEMORY.md' ? 'main' : 'daily' } function readUtf8File(filePath) { return fs.readFileSync(filePath, 'utf8') } function wordCount(content) { return content.trim().split(/\s+/).filter(Boolean).length } function getEntryCount(content) { if (!content) return 0 return content.split('\n').filter((line) => line.trim().startsWith('- [')).length } const TAG_STOPWORDS = new Set([ 'about', 'after', 'again', 'also', 'been', 'being', 'between', 'could', 'daily', 'does', 'done', 'from', 'have', 'into', 'just', 'like', 'main', 'make', 'maybe', 'memory', 'more', 'most', 'much', 'need', 'note', 'notes', 'only', 'other', 'over', 'really', 'same', 'some', 'still', 'than', 'that', 'their', 'them', 'then', 'there', 'these', 'they', 'this', 'today', 'very', 'want', 'were', 'what', 'when', 'with', 'work', 'would', 'your', 'yours', 'jimmi', 'rook', 'said', 'todo', 'todos', 'session', 'update', 'updated', 'using', 'used', 'kind', 'actually', 'awesome', 'maybe', 'seems' ]) const SMART_TAG_RULES = [ { tag: 'appwrite', patterns: [/appwrite/i, /tablesdb/i, /database/i, /collection/i] }, { tag: 'agents', patterns: [/openclaw/i, /agent/i, /subagent/i, /codex/i, /claude code/i] }, { tag: 'design', patterns: [/design/i, /ui\b/i, /visual/i, /theme/i, /scandinavian/i, /gradient/i] }, { tag: 'memory', patterns: [/memory\.md/i, /memory app/i, /memories/i, /archive/i] }, { tag: 'infra', patterns: [/cloudflare/i, /systemd/i, /service/i, /deploy/i, /domain/i, /proxy/i] }, { tag: 'auth', patterns: [/auth/i, /login/i, /session cookie/i, /oauth/i, /permission/i] }, { tag: 'product', patterns: [/idea/i, /validation/i, /launch/i, /mvp/i, /portfolio/i] }, { tag: 'kidsstories', patterns: [/kidsstories/i, /stories/i, /child/i] }, { tag: 'dashboard', patterns: [/dashboard/i, /command center/i, /pixel office/i] }, { tag: 'family', patterns: [/kids/i, /family/i, /home/i] }, ] const TAG_ALIASES = { login: 'auth', oauth: 'auth', permissions: 'auth', permission: 'auth', visual: 'design', theme: 'design', ui: 'design', archive: 'memory', memories: 'memory', deploy: 'infra', deployment: 'infra', cloudflare: 'infra', service: 'infra', openclaw: 'agents', agentboard: 'agents', codex: 'agents', child: 'kidsstories', stories: 'kidsstories', home: 'family', } const MOOD_PROFILES = [ { mood: 'focused', score: 0, patterns: [/fix/i, /ship/i, /build/i, /implement/i, /polish/i, /deploy/i, /migrate/i] }, { mood: 'playful', score: 0, patterns: [/fun/i, /mischief/i, /awesome/i, /play/i, /spark/i] }, { mood: 'reflective', score: 0, patterns: [/remember/i, /reflect/i, /journal/i, /learn/i, /insight/i] }, { mood: 'tense', score: 0, patterns: [/blocked/i, /worry/i, /broken/i, /error/i, /issue/i, /frustrat/i] }, { mood: 'warm', score: 0, patterns: [/family/i, /kids/i, /home/i, /story/i, /love/i] }, ] const COLLECTION_RULES = [ { id: 'infra-week', label: 'Infra week', tags: ['infra', 'appwrite', 'auth'] }, { id: 'design-streak', label: 'Design streak', tags: ['design', 'memory'] }, { id: 'family-notes', label: 'Family notes', tags: ['family', 'kidsstories'] }, { id: 'agent-ops', label: 'Agent ops', tags: ['agents', 'appwrite', 'dashboard'] }, { id: 'product-loop', label: 'Product loop', tags: ['product', 'design', 'appwrite'] }, ] const THEME_PROFILES = { nord: { name: 'Nord', accent: '#8fbcbb', accentSoft: 'rgba(143,188,187,0.18)', accentStrong: '#a3d5d3', glow: 'rgba(143,188,187,0.28)' }, plum: { name: 'Plum', accent: '#b48ead', accentSoft: 'rgba(180,142,173,0.18)', accentStrong: '#d1a9ca', glow: 'rgba(180,142,173,0.28)' }, amber: { name: 'Amber', accent: '#d0a86e', accentSoft: 'rgba(208,168,110,0.18)', accentStrong: '#e5bf8c', glow: 'rgba(208,168,110,0.28)' }, ice: { name: 'Ice', accent: '#81a1c1', accentSoft: 'rgba(129,161,193,0.18)', accentStrong: '#a9c2db', glow: 'rgba(129,161,193,0.28)' }, } function isProbablyNoiseTag(value) { return /^\d{4}-\d{2}-\d{2}$/.test(value) || /^\d+$/.test(value) } function normalizeTag(value) { const cleaned = String(value || '').toLowerCase().replace(/[^a-z0-9+-]+/g, '').trim() return TAG_ALIASES[cleaned] || cleaned } function generateTags(content, filename) { const tags = [] const scores = new Map() const addScore = (tag, amount) => { const normalized = normalizeTag(tag) if (!normalized) return scores.set(normalized, (scores.get(normalized) || 0) + amount) } const pushTag = (value) => { const cleaned = normalizeTag(value) if (!cleaned || cleaned.length < 3 || TAG_STOPWORDS.has(cleaned) || tags.includes(cleaned) || isProbablyNoiseTag(cleaned)) return tags.push(cleaned) } if (filename === 'MEMORY.md') addScore('core', 5) for (const rule of SMART_TAG_RULES) { for (const pattern of rule.patterns) { const matches = content.match(new RegExp(pattern.source, pattern.flags.includes('g') ? pattern.flags : `${pattern.flags}g`)) || [] if (matches.length) addScore(rule.tag, matches.length * 3) } } for (const line of content.split('\n')) { const trimmed = line.trim() if (trimmed.startsWith('#')) { trimmed.replace(/^#+\s*/, '').split(/[^a-zA-Z0-9+-]+/).forEach((token) => addScore(token.toLowerCase(), 2)) } const hashtagMatches = trimmed.match(/#[a-zA-Z0-9+-]+/g) || [] hashtagMatches.forEach((tag) => addScore(tag.slice(1).toLowerCase(), 4)) } const frequency = new Map() const normalized = content .toLowerCase() .replace(/\[[^\]]*\]/g, ' ') .replace(/[^a-z0-9+\-\s]/g, ' ') for (const token of normalized.split(/\s+/)) { if (!token || token.length < 4 || TAG_STOPWORDS.has(token) || isProbablyNoiseTag(token)) continue frequency.set(token, (frequency.get(token) || 0) + 1) } for (const [token, count] of frequency.entries()) { if (count >= 2) addScore(token, count) } Array.from(scores.entries()) .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) .slice(0, 8) .forEach(([token]) => pushTag(token)) return tags.slice(0, 5) } function buildThemeProfile(content = '', tags = []) { const haystack = `${content} ${(tags || []).join(' ')}`.toLowerCase() if (/design|theme|visual|scandinavian|style/.test(haystack)) return THEME_PROFILES.plum if (/infra|deploy|cloudflare|service|proxy|appwrite/.test(haystack)) return THEME_PROFILES.ice if (/kidsstories|family|home|story/.test(haystack)) return THEME_PROFILES.amber return THEME_PROFILES.nord } function detectMood(content = '', tags = []) { const haystack = `${content} ${(tags || []).join(' ')}`.toLowerCase() let best = { mood: 'steady', score: 0 } for (const profile of MOOD_PROFILES) { let score = 0 for (const pattern of profile.patterns) { const matches = haystack.match(new RegExp(pattern.source, pattern.flags.includes('g') ? pattern.flags : `${pattern.flags}g`)) || [] score += matches.length } if (score > best.score) best = { mood: profile.mood, score } } return best.mood } function buildCollections(files = []) { return COLLECTION_RULES.map((rule) => { const matchedFiles = files.filter((file) => rule.tags.some((tag) => (file.tags || []).includes(tag))) if (!matchedFiles.length) return null return { id: rule.id, label: rule.label, count: matchedFiles.length, tags: rule.tags, latest: matchedFiles[0]?.dateLabel, filenames: matchedFiles.slice(0, 6).map((file) => file.filename), } }).filter(Boolean) } function formatDateLabel(filename) { const match = filename.match(/^(\d{4})-(\d{2})-(\d{2})\.md$/) if (!match) return filename const [, year, month, day] = match const date = new Date(`${year}-${month}-${day}T12:00:00`) return date.toLocaleDateString(undefined, { weekday: 'short', year: 'numeric', month: 'short', day: 'numeric', }) } async function runAppwrite(args) { const { stdout } = await execFileAsync('appwrite', args, { env: APPWRITE_ENV, maxBuffer: 10 * 1024 * 1024, }) return stdout?.trim() || '' } async function runAppwriteJson(args) { const stdout = await runAppwrite([...args, '--json']) return stdout ? JSON.parse(stdout) : null } async function getDocument(documentId) { try { return await runAppwriteJson([ 'databases', 'get-document', '--database-id', APPWRITE_DATABASE_ID, '--collection-id', APPWRITE_COLLECTION_ID, '--document-id', documentId, ]) } catch (error) { const stderr = `${error.stderr || ''}${error.stdout || ''}${error.message || ''}` if (/document_not_found|404|not found|could not be found/i.test(stderr)) return null throw error } } async function listDocuments() { const result = await runAppwriteJson([ 'databases', 'list-documents', '--database-id', APPWRITE_DATABASE_ID, '--collection-id', APPWRITE_COLLECTION_ID, ]) return result?.documents || [] } async function upsertDocument(filename, content) { const data = JSON.stringify({ filename, kind: kindForFilename(filename), content, }) const existing = await getDocument(filename) if (existing) { return runAppwriteJson([ 'databases', 'update-document', '--database-id', APPWRITE_DATABASE_ID, '--collection-id', APPWRITE_COLLECTION_ID, '--document-id', filename, '--data', data, ]) } return runAppwriteJson([ 'databases', 'create-document', '--database-id', APPWRITE_DATABASE_ID, '--collection-id', APPWRITE_COLLECTION_ID, '--document-id', filename, '--data', data, ]) } async function syncFileToAppwrite(filename) { const filePath = resolveMemoryPath(filename) if (!filePath || !fs.existsSync(filePath)) return null const content = await fsp.readFile(filePath, 'utf8') return upsertDocument(filename, content) } async function syncWorkspaceToAppwrite() { const filenames = [ 'MEMORY.md', ...fs.readdirSync(MEMORY_DIR).filter(isValidDailyMemoryFilename).sort(), ] for (const filename of filenames) { await syncFileToAppwrite(filename) } } let syncPromise = syncWorkspaceToAppwrite().catch((error) => { console.error('Initial Appwrite sync failed:', error) }) async function ensureSync() { await syncPromise } async function readMemoryDocument(filename) { const doc = await getDocument(filename) if (doc?.content != null) { return doc } await syncFileToAppwrite(filename) return getDocument(filename) } function buildFileMeta(doc) { const content = doc.content || '' const tags = generateTags(content, doc.filename) const mtimeMs = doc.$updatedAt ? new Date(doc.$updatedAt).getTime() : Date.now() return { filename: doc.filename, mtimeMs, size: Buffer.byteLength(content, 'utf8'), wordCount: wordCount(content), entryCount: getEntryCount(content), dateLabel: formatDateLabel(doc.filename), tags, mood: detectMood(content, tags), } } app.get('/api/health', async (req, res) => { try { await ensureSync() await runAppwrite(['databases', 'get-collection', '--database-id', APPWRITE_DATABASE_ID, '--collection-id', APPWRITE_COLLECTION_ID]) res.json({ ok: true, appwrite: true, memoryDir: fs.existsSync(MEMORY_DIR), mainMemory: fs.existsSync(MAIN_MEMORY_FILE), }) } catch (error) { res.status(500).json({ ok: false, error: error.message }) } }) app.get('/api/meta', async (req, res) => { try { await ensureSync() const docs = await listDocuments() const mainDoc = docs.find((doc) => doc.filename === 'MEMORY.md') const dailyDocs = docs .filter((doc) => isValidDailyMemoryFilename(doc.filename)) .sort((a, b) => String(b.filename).localeCompare(String(a.filename))) const mainMemory = mainDoc ? { exists: true, mtimeMs: new Date(mainDoc.$updatedAt).getTime(), size: Buffer.byteLength(mainDoc.content || '', 'utf8'), wordCount: wordCount(mainDoc.content || ''), entryCount: getEntryCount(mainDoc.content || ''), tags: generateTags(mainDoc.content || '', 'MEMORY.md'), } : null const dailyFiles = dailyDocs.map(buildFileMeta) const summary = dailyFiles.reduce((acc, file) => { acc.dailyFileCount += 1 acc.totalDailyEntries += file.entryCount acc.totalDailyWords += file.wordCount return acc }, { dailyFileCount: 0, totalDailyEntries: 0, totalDailyWords: 0 }) const theme = buildThemeProfile(mainDoc?.content || '', mainMemory?.tags || []) const collections = buildCollections(dailyFiles) const mood = detectMood(mainDoc?.content || '', mainMemory?.tags || []) res.json({ mainMemory: mainMemory ? { ...mainMemory, mood } : null, dailyFiles, summary, theme, collections }) } catch (error) { res.status(500).json({ error: error.message }) } }) app.get('/api/memories', async (req, res) => { try { await ensureSync() const docs = await listDocuments() const files = docs .map((doc) => doc.filename) .filter(isValidDailyMemoryFilename) .sort() .reverse() res.json(files) } catch (error) { res.status(500).json({ error: error.message }) } }) app.get('/api/memories/:filename', async (req, res) => { try { const { filename } = req.params if (!isValidDailyMemoryFilename(filename)) { return res.status(400).json({ error: 'Invalid memory filename' }) } await ensureSync() const doc = await readMemoryDocument(filename) if (!doc) return res.status(404).json({ error: 'File not found' }) const tags = generateTags(doc.content || '', filename) return res.json({ content: doc.content || '', tags, mood: detectMood(doc.content || '', tags) }) } catch (error) { return res.status(500).json({ error: error.message }) } }) app.get('/api/main-memory', async (req, res) => { try { await ensureSync() const doc = await readMemoryDocument('MEMORY.md') if (!doc) return res.status(404).json({ error: 'MEMORY.md not found' }) const tags = generateTags(doc.content || '', 'MEMORY.md') res.json({ content: doc.content || '', tags, mood: detectMood(doc.content || '', tags) }) } catch (error) { res.status(500).json({ error: error.message }) } }) async function writeThrough(filename, content) { const filePath = resolveMemoryPath(filename) if (!filePath) throw new Error('Invalid memory filename') await fsp.writeFile(filePath, content, 'utf8') await upsertDocument(filename, content) } app.put('/api/main-memory', async (req, res) => { try { const { content } = req.body if (typeof content !== 'string') { return res.status(400).json({ error: 'Content is required' }) } await ensureSync() await writeThrough('MEMORY.md', content) res.json({ success: true }) } catch (error) { res.status(500).json({ error: error.message }) } }) app.put('/api/memories/:filename', async (req, res) => { try { const { filename } = req.params if (!isValidDailyMemoryFilename(filename)) { return res.status(400).json({ error: 'Invalid memory filename' }) } const { content } = req.body if (typeof content !== 'string') { return res.status(400).json({ error: 'Content is required' }) } await ensureSync() await writeThrough(filename, content) res.json({ success: true }) } catch (error) { res.status(500).json({ error: error.message }) } }) app.get('/api/search', async (req, res) => { const query = typeof req.query.q === 'string' ? req.query.q.trim().toLowerCase() : '' if (!query || query.length < 2) { return res.json({ query, results: [], total: 0 }) } try { await ensureSync() const docs = (await listDocuments()) .filter((doc) => isValidDailyMemoryFilename(doc.filename)) .sort((a, b) => String(b.filename).localeCompare(String(a.filename))) const results = [] for (const doc of docs) { const content = doc.content || '' const lower = content.toLowerCase() if (!lower.includes(query)) continue const lines = content.split('\n') const matchingLines = lines .map((line) => line.trim()) .filter((line) => line.toLowerCase().includes(query)) .slice(0, 5) const firstMatchIdx = lower.indexOf(query) const windowStart = Math.max(0, firstMatchIdx - 80) const windowEnd = Math.min(content.length, firstMatchIdx + 120) const snippet = content.slice(windowStart, windowEnd).replace(/\n/g, ' ') const tags = generateTags(content, doc.filename) results.push({ filename: doc.filename, dateLabel: formatDateLabel(doc.filename), entryCount: getEntryCount(content), wordCount: wordCount(content), tags, mood: detectMood(content, tags), matchCount: (lower.match(new RegExp(query.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g')) || []).length, snippet: `${windowStart > 0 ? '…' : ''}${snippet}${windowEnd < content.length ? '…' : ''}`, matchingLines, }) } results.sort((a, b) => b.matchCount - a.matchCount) res.json({ query, results, total: results.length }) } catch (error) { res.status(500).json({ error: error.message }) } }) app.post('/api/log', async (req, res) => { const message = typeof req.body?.message === 'string' ? req.body.message.trim() : '' if (!message) { return res.status(400).json({ error: 'Message is required' }) } try { await ensureSync() const filename = `${todayDateStamp()}.md` const filePath = resolveDailyMemoryPath(filename) const existing = filePath && fs.existsSync(filePath) ? await fsp.readFile(filePath, 'utf8') : '' const prefix = existing && !existing.endsWith('\n') ? '\n' : '' const content = `${existing}${prefix}- [${currentTimeStamp()}] ${message}\n` await writeThrough(filename, content) res.json({ success: true, mode: 'appwrite' }) } catch (error) { res.status(500).json({ error: error.message }) } }) app.get('/api/review', async (req, res) => { try { const { stdout, stderr } = await execFileAsync('claw-mem', ['review']) const output = `${stdout || ''}${stderr || ''}` const REVIEW_PATTERN = /^ File: (.+\.md)$/gm const files = [] let match while ((match = REVIEW_PATTERN.exec(output)) !== null) { const filename = path.basename(match[1].trim()) if (isValidDailyMemoryFilename(filename)) { const doc = await readMemoryDocument(filename) if (doc) { files.push({ filename, dateLabel: formatDateLabel(filename), wordCount: wordCount(doc.content || ''), entryCount: getEntryCount(doc.content || ''), content: doc.content || '', }) } } } res.json({ files, hasCandidates: files.length > 0, instructions: files.length > 0 ? 'Read each file, distill key learnings into MEMORY.md, then prune.' : 'All memory files are recent. Check back in a few days.', }) } catch (error) { res.status(500).json({ error: error.message }) } }) app.get('/api/review-count', async (req, res) => { try { const { stdout, stderr } = await execFileAsync('claw-mem', ['review']) const output = `${stdout || ''}${stderr || ''}` const REVIEW_PATTERN = /^ File: (.+\.md)$/gm let count = 0 let match while ((match = REVIEW_PATTERN.exec(output)) !== null) { const filename = path.basename(match[1].trim()) if (isValidDailyMemoryFilename(filename)) count++ } res.json({ count }) } catch (error) { res.status(500).json({ error: error.message }) } }) app.post('/api/prune', async (req, res) => { try { const { stdout, stderr } = await execFileAsync('claw-mem', ['prune']) const output = `${stdout || ''}${stderr || ''}` const countMatch = output.match(/(\d+)\s+file/i) || output.match(/pruned\s+(\d+)/i) res.json({ success: true, prunedCount: countMatch ? Number.parseInt(countMatch[1], 10) : null, output }) } catch (error) { res.status(500).json({ error: error.message || 'Prune failed', output: `${error.stdout || ''}${error.stderr || ''}` }) } }) app.post('/api/distill', async (req, res) => { const { note, sourceFilename } = req.body || {} if (typeof note !== 'string' || !note.trim()) { return res.status(400).json({ error: 'Distilled note is required' }) } try { await ensureSync() const mainDoc = await readMemoryDocument('MEMORY.md') if (!mainDoc) return res.status(404).json({ error: 'MEMORY.md not found' }) const existing = mainDoc.content || '' const dateStr = todayDateStamp() const sourceTag = sourceFilename ? ` (from ${sourceFilename})` : '' const distillEntry = `\n- **[${dateStr}]**${sourceTag} ${note.trim()}\n` await writeThrough('MEMORY.md', `${existing}${distillEntry}`) res.json({ success: true, appended: distillEntry.trim() }) } catch (error) { res.status(500).json({ error: error.message }) } }) app.use(express.static(DIST_DIR)) app.get(/.*/, (req, res) => { res.sendFile(path.join(DIST_DIR, 'index.html')) }) const PORT = Number(process.env.PORT) || 3333 if (import.meta.url === `file://${process.argv[1]}`) { app.listen(PORT, '0.0.0.0', () => { console.log(`Freeclaw Memory Cloud running on port ${PORT}`) }) } export { app, isValidDailyMemoryFilename, resolveDailyMemoryPath }