OpenClaw
503 lines
16 KiB
JavaScript
503 lines
16 KiB
JavaScript
import express from 'express'
|
|
import cors from 'cors'
|
|
import fs from 'node:fs'
|
|
import fsp from 'node:fs/promises'
|
|
import path from 'node:path'
|
|
import { fileURLToPath } from 'node:url'
|
|
import { execFile } from 'node:child_process'
|
|
import { promisify } from 'node:util'
|
|
|
|
const execFileAsync = promisify(execFile)
|
|
const __filename = fileURLToPath(import.meta.url)
|
|
const __dirname = path.dirname(__filename)
|
|
|
|
const app = express()
|
|
app.use(cors())
|
|
app.use(express.json({ limit: '1mb' }))
|
|
|
|
const WORKSPACE_DIR = path.join(process.env.HOME || '', '.openclaw/workspace')
|
|
const MEMORY_DIR = path.join(WORKSPACE_DIR, 'memory')
|
|
const MAIN_MEMORY_FILE = path.join(WORKSPACE_DIR, 'MEMORY.md')
|
|
const DIST_DIR = path.join(__dirname, 'dist')
|
|
const DAILY_MEMORY_PATTERN = /^\d{4}-\d{2}-\d{2}\.md$/
|
|
|
|
const APPWRITE_ENDPOINT = process.env.APPWRITE_SELF_HOSTED_URL || process.env.APPWRITE_LOCAL_ENDPOINT
|
|
const APPWRITE_PROJECT_ID = process.env.APPWRITE_SELF_HOSTED_PROJECT_ID || process.env.APPWRITE_LOCAL_PROJECT_ID
|
|
const APPWRITE_KEY = process.env.APPWRITE_SELF_HOSTED_API_KEY || process.env.APPWRITE_LOCAL_API_KEY
|
|
const APPWRITE_DATABASE_ID = process.env.APPWRITE_MEMORY_DATABASE_ID || 'freecastle'
|
|
const APPWRITE_COLLECTION_ID = process.env.APPWRITE_MEMORY_COLLECTION_ID || 'memory_files'
|
|
|
|
const APPWRITE_ENV = {
|
|
...process.env,
|
|
APPWRITE_ENDPOINT,
|
|
APPWRITE_PROJECT_ID,
|
|
APPWRITE_KEY,
|
|
}
|
|
|
|
if (!fs.existsSync(MEMORY_DIR)) {
|
|
fs.mkdirSync(MEMORY_DIR, { recursive: true })
|
|
}
|
|
|
|
function todayDateStamp() {
|
|
return new Date().toISOString().split('T')[0]
|
|
}
|
|
|
|
function currentTimeStamp() {
|
|
return new Date().toTimeString().slice(0, 5)
|
|
}
|
|
|
|
function isValidDailyMemoryFilename(filename) {
|
|
return typeof filename === 'string' && DAILY_MEMORY_PATTERN.test(filename)
|
|
}
|
|
|
|
function isValidMemoryFilename(filename) {
|
|
return filename === 'MEMORY.md' || isValidDailyMemoryFilename(filename)
|
|
}
|
|
|
|
function resolveDailyMemoryPath(filename) {
|
|
if (!isValidDailyMemoryFilename(filename)) return null
|
|
const resolvedPath = path.resolve(MEMORY_DIR, filename)
|
|
const memoryDirPrefix = `${path.resolve(MEMORY_DIR)}${path.sep}`
|
|
return resolvedPath.startsWith(memoryDirPrefix) ? resolvedPath : null
|
|
}
|
|
|
|
function resolveMemoryPath(filename) {
|
|
if (filename === 'MEMORY.md') return MAIN_MEMORY_FILE
|
|
return resolveDailyMemoryPath(filename)
|
|
}
|
|
|
|
function kindForFilename(filename) {
|
|
return filename === 'MEMORY.md' ? 'main' : 'daily'
|
|
}
|
|
|
|
function readUtf8File(filePath) {
|
|
return fs.readFileSync(filePath, 'utf8')
|
|
}
|
|
|
|
function wordCount(content) {
|
|
return content.trim().split(/\s+/).filter(Boolean).length
|
|
}
|
|
|
|
function getEntryCount(content) {
|
|
if (!content) return 0
|
|
return content.split('\n').filter((line) => line.trim().startsWith('- [')).length
|
|
}
|
|
|
|
function formatDateLabel(filename) {
|
|
const match = filename.match(/^(\d{4})-(\d{2})-(\d{2})\.md$/)
|
|
if (!match) return filename
|
|
const [, year, month, day] = match
|
|
const date = new Date(`${year}-${month}-${day}T12:00:00`)
|
|
return date.toLocaleDateString(undefined, {
|
|
weekday: 'short',
|
|
year: 'numeric',
|
|
month: 'short',
|
|
day: 'numeric',
|
|
})
|
|
}
|
|
|
|
async function runAppwrite(args) {
|
|
const { stdout } = await execFileAsync('appwrite', args, {
|
|
env: APPWRITE_ENV,
|
|
maxBuffer: 10 * 1024 * 1024,
|
|
})
|
|
return stdout?.trim() || ''
|
|
}
|
|
|
|
async function runAppwriteJson(args) {
|
|
const stdout = await runAppwrite([...args, '--json'])
|
|
return stdout ? JSON.parse(stdout) : null
|
|
}
|
|
|
|
async function getDocument(documentId) {
|
|
try {
|
|
return await runAppwriteJson([
|
|
'databases', 'get-document',
|
|
'--database-id', APPWRITE_DATABASE_ID,
|
|
'--collection-id', APPWRITE_COLLECTION_ID,
|
|
'--document-id', documentId,
|
|
])
|
|
} catch (error) {
|
|
const stderr = `${error.stderr || ''}${error.stdout || ''}${error.message || ''}`
|
|
if (/document_not_found|404|not found|could not be found/i.test(stderr)) return null
|
|
throw error
|
|
}
|
|
}
|
|
|
|
async function listDocuments() {
|
|
const result = await runAppwriteJson([
|
|
'databases', 'list-documents',
|
|
'--database-id', APPWRITE_DATABASE_ID,
|
|
'--collection-id', APPWRITE_COLLECTION_ID,
|
|
])
|
|
return result?.documents || []
|
|
}
|
|
|
|
async function upsertDocument(filename, content) {
|
|
const data = JSON.stringify({
|
|
filename,
|
|
kind: kindForFilename(filename),
|
|
content,
|
|
})
|
|
|
|
const existing = await getDocument(filename)
|
|
if (existing) {
|
|
return runAppwriteJson([
|
|
'databases', 'update-document',
|
|
'--database-id', APPWRITE_DATABASE_ID,
|
|
'--collection-id', APPWRITE_COLLECTION_ID,
|
|
'--document-id', filename,
|
|
'--data', data,
|
|
])
|
|
}
|
|
|
|
return runAppwriteJson([
|
|
'databases', 'create-document',
|
|
'--database-id', APPWRITE_DATABASE_ID,
|
|
'--collection-id', APPWRITE_COLLECTION_ID,
|
|
'--document-id', filename,
|
|
'--data', data,
|
|
])
|
|
}
|
|
|
|
async function syncFileToAppwrite(filename) {
|
|
const filePath = resolveMemoryPath(filename)
|
|
if (!filePath || !fs.existsSync(filePath)) return null
|
|
const content = await fsp.readFile(filePath, 'utf8')
|
|
return upsertDocument(filename, content)
|
|
}
|
|
|
|
async function syncWorkspaceToAppwrite() {
|
|
const filenames = [
|
|
'MEMORY.md',
|
|
...fs.readdirSync(MEMORY_DIR).filter(isValidDailyMemoryFilename).sort(),
|
|
]
|
|
|
|
for (const filename of filenames) {
|
|
await syncFileToAppwrite(filename)
|
|
}
|
|
}
|
|
|
|
let syncPromise = syncWorkspaceToAppwrite().catch((error) => {
|
|
console.error('Initial Appwrite sync failed:', error)
|
|
})
|
|
|
|
async function ensureSync() {
|
|
await syncPromise
|
|
}
|
|
|
|
async function readMemoryDocument(filename) {
|
|
const doc = await getDocument(filename)
|
|
if (doc?.content != null) {
|
|
return doc
|
|
}
|
|
|
|
await syncFileToAppwrite(filename)
|
|
return getDocument(filename)
|
|
}
|
|
|
|
function buildFileMeta(doc) {
|
|
const content = doc.content || ''
|
|
const mtimeMs = doc.$updatedAt ? new Date(doc.$updatedAt).getTime() : Date.now()
|
|
return {
|
|
filename: doc.filename,
|
|
mtimeMs,
|
|
size: Buffer.byteLength(content, 'utf8'),
|
|
wordCount: wordCount(content),
|
|
entryCount: getEntryCount(content),
|
|
dateLabel: formatDateLabel(doc.filename),
|
|
}
|
|
}
|
|
|
|
app.get('/api/health', async (req, res) => {
|
|
try {
|
|
await ensureSync()
|
|
await runAppwrite(['databases', 'get-collection', '--database-id', APPWRITE_DATABASE_ID, '--collection-id', APPWRITE_COLLECTION_ID])
|
|
res.json({
|
|
ok: true,
|
|
appwrite: true,
|
|
memoryDir: fs.existsSync(MEMORY_DIR),
|
|
mainMemory: fs.existsSync(MAIN_MEMORY_FILE),
|
|
})
|
|
} catch (error) {
|
|
res.status(500).json({ ok: false, error: error.message })
|
|
}
|
|
})
|
|
|
|
app.get('/api/meta', async (req, res) => {
|
|
try {
|
|
await ensureSync()
|
|
const docs = await listDocuments()
|
|
const mainDoc = docs.find((doc) => doc.filename === 'MEMORY.md')
|
|
const dailyDocs = docs
|
|
.filter((doc) => isValidDailyMemoryFilename(doc.filename))
|
|
.sort((a, b) => String(b.filename).localeCompare(String(a.filename)))
|
|
|
|
const mainMemory = mainDoc ? {
|
|
exists: true,
|
|
mtimeMs: new Date(mainDoc.$updatedAt).getTime(),
|
|
size: Buffer.byteLength(mainDoc.content || '', 'utf8'),
|
|
wordCount: wordCount(mainDoc.content || ''),
|
|
entryCount: getEntryCount(mainDoc.content || ''),
|
|
} : null
|
|
|
|
const dailyFiles = dailyDocs.map(buildFileMeta)
|
|
const summary = dailyFiles.reduce((acc, file) => {
|
|
acc.dailyFileCount += 1
|
|
acc.totalDailyEntries += file.entryCount
|
|
acc.totalDailyWords += file.wordCount
|
|
return acc
|
|
}, { dailyFileCount: 0, totalDailyEntries: 0, totalDailyWords: 0 })
|
|
|
|
res.json({ mainMemory, dailyFiles, summary })
|
|
} catch (error) {
|
|
res.status(500).json({ error: error.message })
|
|
}
|
|
})
|
|
|
|
app.get('/api/memories', async (req, res) => {
|
|
try {
|
|
await ensureSync()
|
|
const docs = await listDocuments()
|
|
const files = docs
|
|
.map((doc) => doc.filename)
|
|
.filter(isValidDailyMemoryFilename)
|
|
.sort()
|
|
.reverse()
|
|
res.json(files)
|
|
} catch (error) {
|
|
res.status(500).json({ error: error.message })
|
|
}
|
|
})
|
|
|
|
app.get('/api/memories/:filename', async (req, res) => {
|
|
try {
|
|
const { filename } = req.params
|
|
if (!isValidDailyMemoryFilename(filename)) {
|
|
return res.status(400).json({ error: 'Invalid memory filename' })
|
|
}
|
|
await ensureSync()
|
|
const doc = await readMemoryDocument(filename)
|
|
if (!doc) return res.status(404).json({ error: 'File not found' })
|
|
return res.json({ content: doc.content || '' })
|
|
} catch (error) {
|
|
return res.status(500).json({ error: error.message })
|
|
}
|
|
})
|
|
|
|
app.get('/api/main-memory', async (req, res) => {
|
|
try {
|
|
await ensureSync()
|
|
const doc = await readMemoryDocument('MEMORY.md')
|
|
if (!doc) return res.status(404).json({ error: 'MEMORY.md not found' })
|
|
res.json({ content: doc.content || '' })
|
|
} catch (error) {
|
|
res.status(500).json({ error: error.message })
|
|
}
|
|
})
|
|
|
|
async function writeThrough(filename, content) {
|
|
const filePath = resolveMemoryPath(filename)
|
|
if (!filePath) throw new Error('Invalid memory filename')
|
|
await fsp.writeFile(filePath, content, 'utf8')
|
|
await upsertDocument(filename, content)
|
|
}
|
|
|
|
app.put('/api/main-memory', async (req, res) => {
|
|
try {
|
|
const { content } = req.body
|
|
if (typeof content !== 'string') {
|
|
return res.status(400).json({ error: 'Content is required' })
|
|
}
|
|
await ensureSync()
|
|
await writeThrough('MEMORY.md', content)
|
|
res.json({ success: true })
|
|
} catch (error) {
|
|
res.status(500).json({ error: error.message })
|
|
}
|
|
})
|
|
|
|
app.put('/api/memories/:filename', async (req, res) => {
|
|
try {
|
|
const { filename } = req.params
|
|
if (!isValidDailyMemoryFilename(filename)) {
|
|
return res.status(400).json({ error: 'Invalid memory filename' })
|
|
}
|
|
const { content } = req.body
|
|
if (typeof content !== 'string') {
|
|
return res.status(400).json({ error: 'Content is required' })
|
|
}
|
|
await ensureSync()
|
|
await writeThrough(filename, content)
|
|
res.json({ success: true })
|
|
} catch (error) {
|
|
res.status(500).json({ error: error.message })
|
|
}
|
|
})
|
|
|
|
app.get('/api/search', async (req, res) => {
|
|
const query = typeof req.query.q === 'string' ? req.query.q.trim().toLowerCase() : ''
|
|
if (!query || query.length < 2) {
|
|
return res.json({ query, results: [], total: 0 })
|
|
}
|
|
|
|
try {
|
|
await ensureSync()
|
|
const docs = (await listDocuments())
|
|
.filter((doc) => isValidDailyMemoryFilename(doc.filename))
|
|
.sort((a, b) => String(b.filename).localeCompare(String(a.filename)))
|
|
|
|
const results = []
|
|
for (const doc of docs) {
|
|
const content = doc.content || ''
|
|
const lower = content.toLowerCase()
|
|
if (!lower.includes(query)) continue
|
|
|
|
const lines = content.split('\n')
|
|
const matchingLines = lines
|
|
.map((line) => line.trim())
|
|
.filter((line) => line.toLowerCase().includes(query))
|
|
.slice(0, 5)
|
|
|
|
const firstMatchIdx = lower.indexOf(query)
|
|
const windowStart = Math.max(0, firstMatchIdx - 80)
|
|
const windowEnd = Math.min(content.length, firstMatchIdx + 120)
|
|
const snippet = content.slice(windowStart, windowEnd).replace(/\n/g, ' ')
|
|
|
|
results.push({
|
|
filename: doc.filename,
|
|
dateLabel: formatDateLabel(doc.filename),
|
|
entryCount: getEntryCount(content),
|
|
wordCount: wordCount(content),
|
|
matchCount: (lower.match(new RegExp(query.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g')) || []).length,
|
|
snippet: `${windowStart > 0 ? '…' : ''}${snippet}${windowEnd < content.length ? '…' : ''}`,
|
|
matchingLines,
|
|
})
|
|
}
|
|
|
|
results.sort((a, b) => b.matchCount - a.matchCount)
|
|
res.json({ query, results, total: results.length })
|
|
} catch (error) {
|
|
res.status(500).json({ error: error.message })
|
|
}
|
|
})
|
|
|
|
app.post('/api/log', async (req, res) => {
|
|
const message = typeof req.body?.message === 'string' ? req.body.message.trim() : ''
|
|
if (!message) {
|
|
return res.status(400).json({ error: 'Message is required' })
|
|
}
|
|
|
|
try {
|
|
await ensureSync()
|
|
const filename = `${todayDateStamp()}.md`
|
|
const filePath = resolveDailyMemoryPath(filename)
|
|
const existing = filePath && fs.existsSync(filePath) ? await fsp.readFile(filePath, 'utf8') : ''
|
|
const prefix = existing && !existing.endsWith('\n') ? '\n' : ''
|
|
const content = `${existing}${prefix}- [${currentTimeStamp()}] ${message}\n`
|
|
await writeThrough(filename, content)
|
|
res.json({ success: true, mode: 'appwrite' })
|
|
} catch (error) {
|
|
res.status(500).json({ error: error.message })
|
|
}
|
|
})
|
|
|
|
app.get('/api/review', async (req, res) => {
|
|
try {
|
|
const { stdout, stderr } = await execFileAsync('claw-mem', ['review'])
|
|
const output = `${stdout || ''}${stderr || ''}`
|
|
const REVIEW_PATTERN = /^ File: (.+\.md)$/gm
|
|
const files = []
|
|
let match
|
|
while ((match = REVIEW_PATTERN.exec(output)) !== null) {
|
|
const filename = path.basename(match[1].trim())
|
|
if (isValidDailyMemoryFilename(filename)) {
|
|
const doc = await readMemoryDocument(filename)
|
|
if (doc) {
|
|
files.push({
|
|
filename,
|
|
dateLabel: formatDateLabel(filename),
|
|
wordCount: wordCount(doc.content || ''),
|
|
entryCount: getEntryCount(doc.content || ''),
|
|
content: doc.content || '',
|
|
})
|
|
}
|
|
}
|
|
}
|
|
res.json({
|
|
files,
|
|
hasCandidates: files.length > 0,
|
|
instructions: files.length > 0
|
|
? 'Read each file, distill key learnings into MEMORY.md, then prune.'
|
|
: 'All memory files are recent. Check back in a few days.',
|
|
})
|
|
} catch (error) {
|
|
res.status(500).json({ error: error.message })
|
|
}
|
|
})
|
|
|
|
app.get('/api/review-count', async (req, res) => {
|
|
try {
|
|
const { stdout, stderr } = await execFileAsync('claw-mem', ['review'])
|
|
const output = `${stdout || ''}${stderr || ''}`
|
|
const REVIEW_PATTERN = /^ File: (.+\.md)$/gm
|
|
let count = 0
|
|
let match
|
|
while ((match = REVIEW_PATTERN.exec(output)) !== null) {
|
|
const filename = path.basename(match[1].trim())
|
|
if (isValidDailyMemoryFilename(filename)) count++
|
|
}
|
|
res.json({ count })
|
|
} catch (error) {
|
|
res.status(500).json({ error: error.message })
|
|
}
|
|
})
|
|
|
|
app.post('/api/prune', async (req, res) => {
|
|
try {
|
|
const { stdout, stderr } = await execFileAsync('claw-mem', ['prune'])
|
|
const output = `${stdout || ''}${stderr || ''}`
|
|
const countMatch = output.match(/(\d+)\s+file/i) || output.match(/pruned\s+(\d+)/i)
|
|
res.json({ success: true, prunedCount: countMatch ? Number.parseInt(countMatch[1], 10) : null, output })
|
|
} catch (error) {
|
|
res.status(500).json({ error: error.message || 'Prune failed', output: `${error.stdout || ''}${error.stderr || ''}` })
|
|
}
|
|
})
|
|
|
|
app.post('/api/distill', async (req, res) => {
|
|
const { note, sourceFilename } = req.body || {}
|
|
if (typeof note !== 'string' || !note.trim()) {
|
|
return res.status(400).json({ error: 'Distilled note is required' })
|
|
}
|
|
|
|
try {
|
|
await ensureSync()
|
|
const mainDoc = await readMemoryDocument('MEMORY.md')
|
|
if (!mainDoc) return res.status(404).json({ error: 'MEMORY.md not found' })
|
|
|
|
const existing = mainDoc.content || ''
|
|
const dateStr = todayDateStamp()
|
|
const sourceTag = sourceFilename ? ` (from ${sourceFilename})` : ''
|
|
const distillEntry = `\n- **[${dateStr}]**${sourceTag} ${note.trim()}\n`
|
|
await writeThrough('MEMORY.md', `${existing}${distillEntry}`)
|
|
res.json({ success: true, appended: distillEntry.trim() })
|
|
} catch (error) {
|
|
res.status(500).json({ error: error.message })
|
|
}
|
|
})
|
|
|
|
app.use(express.static(DIST_DIR))
|
|
app.get(/.*/, (req, res) => {
|
|
res.sendFile(path.join(DIST_DIR, 'index.html'))
|
|
})
|
|
|
|
const PORT = Number(process.env.PORT) || 3333
|
|
|
|
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
app.listen(PORT, '0.0.0.0', () => {
|
|
console.log(`Freeclaw Memory Cloud running on port ${PORT}`)
|
|
})
|
|
}
|
|
|
|
export { app, isValidDailyMemoryFilename, resolveDailyMemoryPath }
|