import express from 'express' import cors from 'cors' import fs from 'node:fs' import fsp from 'node:fs/promises' import path from 'node:path' import { fileURLToPath } from 'node:url' import { execFile } from 'node:child_process' import { promisify } from 'node:util' const execFileAsync = promisify(execFile) const __filename = fileURLToPath(import.meta.url) const __dirname = path.dirname(__filename) const app = express() app.use(cors()) app.use(express.json({ limit: '1mb' })) const WORKSPACE_DIR = path.join(process.env.HOME || '', '.openclaw/workspace') const MEMORY_DIR = path.join(WORKSPACE_DIR, 'memory') const MAIN_MEMORY_FILE = path.join(WORKSPACE_DIR, 'MEMORY.md') const DIST_DIR = path.join(__dirname, 'dist') const DAILY_MEMORY_PATTERN = /^\d{4}-\d{2}-\d{2}\.md$/ const APPWRITE_ENDPOINT = process.env.APPWRITE_SELF_HOSTED_URL || process.env.APPWRITE_LOCAL_ENDPOINT const APPWRITE_PROJECT_ID = process.env.APPWRITE_SELF_HOSTED_PROJECT_ID || process.env.APPWRITE_LOCAL_PROJECT_ID const APPWRITE_KEY = process.env.APPWRITE_SELF_HOSTED_API_KEY || process.env.APPWRITE_LOCAL_API_KEY const APPWRITE_DATABASE_ID = process.env.APPWRITE_MEMORY_DATABASE_ID || 'freecastle' const APPWRITE_COLLECTION_ID = process.env.APPWRITE_MEMORY_COLLECTION_ID || 'memory_files' const APPWRITE_ENV = { ...process.env, APPWRITE_ENDPOINT, APPWRITE_PROJECT_ID, APPWRITE_KEY, } if (!fs.existsSync(MEMORY_DIR)) { fs.mkdirSync(MEMORY_DIR, { recursive: true }) } function todayDateStamp() { return new Date().toISOString().split('T')[0] } function currentTimeStamp() { return new Date().toTimeString().slice(0, 5) } function isValidDailyMemoryFilename(filename) { return typeof filename === 'string' && DAILY_MEMORY_PATTERN.test(filename) } function isValidMemoryFilename(filename) { return filename === 'MEMORY.md' || isValidDailyMemoryFilename(filename) } function resolveDailyMemoryPath(filename) { if (!isValidDailyMemoryFilename(filename)) return null const resolvedPath = path.resolve(MEMORY_DIR, filename) const memoryDirPrefix = `${path.resolve(MEMORY_DIR)}${path.sep}` return resolvedPath.startsWith(memoryDirPrefix) ? resolvedPath : null } function resolveMemoryPath(filename) { if (filename === 'MEMORY.md') return MAIN_MEMORY_FILE return resolveDailyMemoryPath(filename) } function kindForFilename(filename) { return filename === 'MEMORY.md' ? 'main' : 'daily' } function readUtf8File(filePath) { return fs.readFileSync(filePath, 'utf8') } function wordCount(content) { return content.trim().split(/\s+/).filter(Boolean).length } function getEntryCount(content) { if (!content) return 0 return content.split('\n').filter((line) => line.trim().startsWith('- [')).length } function formatDateLabel(filename) { const match = filename.match(/^(\d{4})-(\d{2})-(\d{2})\.md$/) if (!match) return filename const [, year, month, day] = match const date = new Date(`${year}-${month}-${day}T12:00:00`) return date.toLocaleDateString(undefined, { weekday: 'short', year: 'numeric', month: 'short', day: 'numeric', }) } async function runAppwrite(args) { const { stdout } = await execFileAsync('appwrite', args, { env: APPWRITE_ENV, maxBuffer: 10 * 1024 * 1024, }) return stdout?.trim() || '' } async function runAppwriteJson(args) { const stdout = await runAppwrite([...args, '--json']) return stdout ? JSON.parse(stdout) : null } async function getDocument(documentId) { try { return await runAppwriteJson([ 'databases', 'get-document', '--database-id', APPWRITE_DATABASE_ID, '--collection-id', APPWRITE_COLLECTION_ID, '--document-id', documentId, ]) } catch (error) { const stderr = `${error.stderr || ''}${error.stdout || ''}${error.message || ''}` if (/document_not_found|404|not found|could not be found/i.test(stderr)) return null throw error } } async function listDocuments() { const result = await runAppwriteJson([ 'databases', 'list-documents', '--database-id', APPWRITE_DATABASE_ID, '--collection-id', APPWRITE_COLLECTION_ID, ]) return result?.documents || [] } async function upsertDocument(filename, content) { const data = JSON.stringify({ filename, kind: kindForFilename(filename), content, }) const existing = await getDocument(filename) if (existing) { return runAppwriteJson([ 'databases', 'update-document', '--database-id', APPWRITE_DATABASE_ID, '--collection-id', APPWRITE_COLLECTION_ID, '--document-id', filename, '--data', data, ]) } return runAppwriteJson([ 'databases', 'create-document', '--database-id', APPWRITE_DATABASE_ID, '--collection-id', APPWRITE_COLLECTION_ID, '--document-id', filename, '--data', data, ]) } async function syncFileToAppwrite(filename) { const filePath = resolveMemoryPath(filename) if (!filePath || !fs.existsSync(filePath)) return null const content = await fsp.readFile(filePath, 'utf8') return upsertDocument(filename, content) } async function syncWorkspaceToAppwrite() { const filenames = [ 'MEMORY.md', ...fs.readdirSync(MEMORY_DIR).filter(isValidDailyMemoryFilename).sort(), ] for (const filename of filenames) { await syncFileToAppwrite(filename) } } let syncPromise = syncWorkspaceToAppwrite().catch((error) => { console.error('Initial Appwrite sync failed:', error) }) async function ensureSync() { await syncPromise } async function readMemoryDocument(filename) { const doc = await getDocument(filename) if (doc?.content != null) { return doc } await syncFileToAppwrite(filename) return getDocument(filename) } function buildFileMeta(doc) { const content = doc.content || '' const mtimeMs = doc.$updatedAt ? new Date(doc.$updatedAt).getTime() : Date.now() return { filename: doc.filename, mtimeMs, size: Buffer.byteLength(content, 'utf8'), wordCount: wordCount(content), entryCount: getEntryCount(content), dateLabel: formatDateLabel(doc.filename), } } app.get('/api/health', async (req, res) => { try { await ensureSync() await runAppwrite(['databases', 'get-collection', '--database-id', APPWRITE_DATABASE_ID, '--collection-id', APPWRITE_COLLECTION_ID]) res.json({ ok: true, appwrite: true, memoryDir: fs.existsSync(MEMORY_DIR), mainMemory: fs.existsSync(MAIN_MEMORY_FILE), }) } catch (error) { res.status(500).json({ ok: false, error: error.message }) } }) app.get('/api/meta', async (req, res) => { try { await ensureSync() const docs = await listDocuments() const mainDoc = docs.find((doc) => doc.filename === 'MEMORY.md') const dailyDocs = docs .filter((doc) => isValidDailyMemoryFilename(doc.filename)) .sort((a, b) => String(b.filename).localeCompare(String(a.filename))) const mainMemory = mainDoc ? { exists: true, mtimeMs: new Date(mainDoc.$updatedAt).getTime(), size: Buffer.byteLength(mainDoc.content || '', 'utf8'), wordCount: wordCount(mainDoc.content || ''), entryCount: getEntryCount(mainDoc.content || ''), } : null const dailyFiles = dailyDocs.map(buildFileMeta) const summary = dailyFiles.reduce((acc, file) => { acc.dailyFileCount += 1 acc.totalDailyEntries += file.entryCount acc.totalDailyWords += file.wordCount return acc }, { dailyFileCount: 0, totalDailyEntries: 0, totalDailyWords: 0 }) res.json({ mainMemory, dailyFiles, summary }) } catch (error) { res.status(500).json({ error: error.message }) } }) app.get('/api/memories', async (req, res) => { try { await ensureSync() const docs = await listDocuments() const files = docs .map((doc) => doc.filename) .filter(isValidDailyMemoryFilename) .sort() .reverse() res.json(files) } catch (error) { res.status(500).json({ error: error.message }) } }) app.get('/api/memories/:filename', async (req, res) => { try { const { filename } = req.params if (!isValidDailyMemoryFilename(filename)) { return res.status(400).json({ error: 'Invalid memory filename' }) } await ensureSync() const doc = await readMemoryDocument(filename) if (!doc) return res.status(404).json({ error: 'File not found' }) return res.json({ content: doc.content || '' }) } catch (error) { return res.status(500).json({ error: error.message }) } }) app.get('/api/main-memory', async (req, res) => { try { await ensureSync() const doc = await readMemoryDocument('MEMORY.md') if (!doc) return res.status(404).json({ error: 'MEMORY.md not found' }) res.json({ content: doc.content || '' }) } catch (error) { res.status(500).json({ error: error.message }) } }) async function writeThrough(filename, content) { const filePath = resolveMemoryPath(filename) if (!filePath) throw new Error('Invalid memory filename') await fsp.writeFile(filePath, content, 'utf8') await upsertDocument(filename, content) } app.put('/api/main-memory', async (req, res) => { try { const { content } = req.body if (typeof content !== 'string') { return res.status(400).json({ error: 'Content is required' }) } await ensureSync() await writeThrough('MEMORY.md', content) res.json({ success: true }) } catch (error) { res.status(500).json({ error: error.message }) } }) app.put('/api/memories/:filename', async (req, res) => { try { const { filename } = req.params if (!isValidDailyMemoryFilename(filename)) { return res.status(400).json({ error: 'Invalid memory filename' }) } const { content } = req.body if (typeof content !== 'string') { return res.status(400).json({ error: 'Content is required' }) } await ensureSync() await writeThrough(filename, content) res.json({ success: true }) } catch (error) { res.status(500).json({ error: error.message }) } }) app.get('/api/search', async (req, res) => { const query = typeof req.query.q === 'string' ? req.query.q.trim().toLowerCase() : '' if (!query || query.length < 2) { return res.json({ query, results: [], total: 0 }) } try { await ensureSync() const docs = (await listDocuments()) .filter((doc) => isValidDailyMemoryFilename(doc.filename)) .sort((a, b) => String(b.filename).localeCompare(String(a.filename))) const results = [] for (const doc of docs) { const content = doc.content || '' const lower = content.toLowerCase() if (!lower.includes(query)) continue const lines = content.split('\n') const matchingLines = lines .map((line) => line.trim()) .filter((line) => line.toLowerCase().includes(query)) .slice(0, 5) const firstMatchIdx = lower.indexOf(query) const windowStart = Math.max(0, firstMatchIdx - 80) const windowEnd = Math.min(content.length, firstMatchIdx + 120) const snippet = content.slice(windowStart, windowEnd).replace(/\n/g, ' ') results.push({ filename: doc.filename, dateLabel: formatDateLabel(doc.filename), entryCount: getEntryCount(content), wordCount: wordCount(content), matchCount: (lower.match(new RegExp(query.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g')) || []).length, snippet: `${windowStart > 0 ? '…' : ''}${snippet}${windowEnd < content.length ? '…' : ''}`, matchingLines, }) } results.sort((a, b) => b.matchCount - a.matchCount) res.json({ query, results, total: results.length }) } catch (error) { res.status(500).json({ error: error.message }) } }) app.post('/api/log', async (req, res) => { const message = typeof req.body?.message === 'string' ? req.body.message.trim() : '' if (!message) { return res.status(400).json({ error: 'Message is required' }) } try { await ensureSync() const filename = `${todayDateStamp()}.md` const filePath = resolveDailyMemoryPath(filename) const existing = filePath && fs.existsSync(filePath) ? await fsp.readFile(filePath, 'utf8') : '' const prefix = existing && !existing.endsWith('\n') ? '\n' : '' const content = `${existing}${prefix}- [${currentTimeStamp()}] ${message}\n` await writeThrough(filename, content) res.json({ success: true, mode: 'appwrite' }) } catch (error) { res.status(500).json({ error: error.message }) } }) app.get('/api/review', async (req, res) => { try { const { stdout, stderr } = await execFileAsync('claw-mem', ['review']) const output = `${stdout || ''}${stderr || ''}` const REVIEW_PATTERN = /^ File: (.+\.md)$/gm const files = [] let match while ((match = REVIEW_PATTERN.exec(output)) !== null) { const filename = path.basename(match[1].trim()) if (isValidDailyMemoryFilename(filename)) { const doc = await readMemoryDocument(filename) if (doc) { files.push({ filename, dateLabel: formatDateLabel(filename), wordCount: wordCount(doc.content || ''), entryCount: getEntryCount(doc.content || ''), content: doc.content || '', }) } } } res.json({ files, hasCandidates: files.length > 0, instructions: files.length > 0 ? 'Read each file, distill key learnings into MEMORY.md, then prune.' : 'All memory files are recent. Check back in a few days.', }) } catch (error) { res.status(500).json({ error: error.message }) } }) app.get('/api/review-count', async (req, res) => { try { const { stdout, stderr } = await execFileAsync('claw-mem', ['review']) const output = `${stdout || ''}${stderr || ''}` const REVIEW_PATTERN = /^ File: (.+\.md)$/gm let count = 0 let match while ((match = REVIEW_PATTERN.exec(output)) !== null) { const filename = path.basename(match[1].trim()) if (isValidDailyMemoryFilename(filename)) count++ } res.json({ count }) } catch (error) { res.status(500).json({ error: error.message }) } }) app.post('/api/prune', async (req, res) => { try { const { stdout, stderr } = await execFileAsync('claw-mem', ['prune']) const output = `${stdout || ''}${stderr || ''}` const countMatch = output.match(/(\d+)\s+file/i) || output.match(/pruned\s+(\d+)/i) res.json({ success: true, prunedCount: countMatch ? Number.parseInt(countMatch[1], 10) : null, output }) } catch (error) { res.status(500).json({ error: error.message || 'Prune failed', output: `${error.stdout || ''}${error.stderr || ''}` }) } }) app.post('/api/distill', async (req, res) => { const { note, sourceFilename } = req.body || {} if (typeof note !== 'string' || !note.trim()) { return res.status(400).json({ error: 'Distilled note is required' }) } try { await ensureSync() const mainDoc = await readMemoryDocument('MEMORY.md') if (!mainDoc) return res.status(404).json({ error: 'MEMORY.md not found' }) const existing = mainDoc.content || '' const dateStr = todayDateStamp() const sourceTag = sourceFilename ? ` (from ${sourceFilename})` : '' const distillEntry = `\n- **[${dateStr}]**${sourceTag} ${note.trim()}\n` await writeThrough('MEMORY.md', `${existing}${distillEntry}`) res.json({ success: true, appended: distillEntry.trim() }) } catch (error) { res.status(500).json({ error: error.message }) } }) app.use(express.static(DIST_DIR)) app.get(/.*/, (req, res) => { res.sendFile(path.join(DIST_DIR, 'index.html')) }) const PORT = Number(process.env.PORT) || 3333 if (import.meta.url === `file://${process.argv[1]}`) { app.listen(PORT, '0.0.0.0', () => { console.log(`Freeclaw Memory Cloud running on port ${PORT}`) }) } export { app, isValidDailyMemoryFilename, resolveDailyMemoryPath }