feat: turn memory cloud into llm memory hub
This commit is contained in:
OpenClaw
@@ -26,9 +26,14 @@ const APPWRITE_PROJECT_ID = process.env.APPWRITE_SELF_HOSTED_PROJECT_ID || proce
|
||||
const APPWRITE_KEY = process.env.APPWRITE_SELF_HOSTED_API_KEY || process.env.APPWRITE_LOCAL_API_KEY
|
||||
const APPWRITE_DATABASE_ID = process.env.APPWRITE_MEMORY_DATABASE_ID || 'freecastle'
|
||||
const APPWRITE_COLLECTION_ID = process.env.APPWRITE_MEMORY_COLLECTION_ID || 'memory_files'
|
||||
const MEMORY_SLACK_CHANNEL_ID = process.env.MEMORY_SLACK_CHANNEL_ID || process.env.LLM_MEMORIES_SLACK_CHANNEL_ID || ''
|
||||
const SLACK_BOT_TOKEN = process.env.SLACK_BOT_TOKEN || ''
|
||||
const IMPORT_MAX_CHARS = 80_000
|
||||
const FETCH_MAX_CHARS = 220_000
|
||||
|
||||
const APPWRITE_ENV = {
|
||||
...process.env,
|
||||
PATH: process.env.PATH,
|
||||
HOME: process.env.HOME,
|
||||
APPWRITE_ENDPOINT,
|
||||
APPWRITE_PROJECT_ID,
|
||||
APPWRITE_KEY,
|
||||
@@ -74,6 +79,40 @@ function readUtf8File(filePath) {
|
||||
return fs.readFileSync(filePath, 'utf8')
|
||||
}
|
||||
|
||||
function truncateText(value = '', max = 240) {
|
||||
const clean = String(value || '').replace(/\s+/g, ' ').trim()
|
||||
return clean.length > max ? `${clean.slice(0, max).trim()}…` : clean
|
||||
}
|
||||
|
||||
function stripHtml(value = '') {
|
||||
return String(value || '')
|
||||
.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, ' ')
|
||||
.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, ' ')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
}
|
||||
|
||||
function escapeFence(value = '') {
|
||||
return String(value || '').replace(/```/g, 'ʼʼʼ')
|
||||
}
|
||||
|
||||
function redactSensitive(value = '') {
|
||||
return String(value || '')
|
||||
.replace(/(standard_[a-f0-9]{24,})/gi, '[redacted-appwrite-key]')
|
||||
.replace(/(xox[baprs]-[A-Za-z0-9-]+)/g, '[redacted-slack-token]')
|
||||
.replace(/(ghp_[A-Za-z0-9_]+)/g, '[redacted-github-token]')
|
||||
.replace(/(cf[a-zA-Z0-9_\-]{20,})/g, '[redacted-cloudflare-token]')
|
||||
.replace(/((?:password|pass|token|api[_-]?key|secret)\s*[:=]\s*)([^\n\s`]+)/gi, '$1[redacted]')
|
||||
.replace(/((?:admin user|user|login)\*?\*?:?[^\n]{0,120}\/\s*`?)([^\n\s`]+)/gi, '$1[redacted]')
|
||||
}
|
||||
|
||||
function wordCount(content) {
|
||||
return content.trim().split(/\s+/).filter(Boolean).length
|
||||
}
|
||||
@@ -154,6 +193,15 @@ function normalizeTag(value) {
|
||||
return TAG_ALIASES[cleaned] || cleaned
|
||||
}
|
||||
|
||||
function tokenize(value = '') {
|
||||
return String(value)
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9+\-\s]/g, ' ')
|
||||
.split(/\s+/)
|
||||
.map((entry) => normalizeTag(entry))
|
||||
.filter((entry) => entry && entry.length > 2 && !TAG_STOPWORDS.has(entry) && !isProbablyNoiseTag(entry))
|
||||
}
|
||||
|
||||
function generateTags(content, filename) {
|
||||
const tags = []
|
||||
const scores = new Map()
|
||||
@@ -246,6 +294,10 @@ function buildCollections(files = []) {
|
||||
}).filter(Boolean)
|
||||
}
|
||||
|
||||
function collectionsForFile(file, collections = []) {
|
||||
return collections.filter((collection) => collection.filenames?.includes(file.filename))
|
||||
}
|
||||
|
||||
function formatDateLabel(filename) {
|
||||
const match = filename.match(/^(\d{4})-(\d{2})-(\d{2})\.md$/)
|
||||
if (!match) return filename
|
||||
@@ -262,7 +314,7 @@ function formatDateLabel(filename) {
|
||||
async function runAppwrite(args) {
|
||||
const { stdout } = await execFileAsync('appwrite', args, {
|
||||
env: APPWRITE_ENV,
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
maxBuffer: 12 * 1024 * 1024,
|
||||
})
|
||||
return stdout?.trim() || ''
|
||||
}
|
||||
@@ -272,6 +324,58 @@ async function runAppwriteJson(args) {
|
||||
return stdout ? JSON.parse(stdout) : null
|
||||
}
|
||||
|
||||
function isAppwriteScopeError(error) {
|
||||
const message = `${error?.message || ''} ${error?.response?.message || ''} ${error?.stderr || ''} ${error?.stdout || ''}`
|
||||
return /missing scopes|forbidden|not authorized|unauthorized/i.test(message)
|
||||
}
|
||||
|
||||
|
||||
async function appwriteRest(method, pathname, body = null) {
|
||||
if (!APPWRITE_ENDPOINT || !APPWRITE_PROJECT_ID || !APPWRITE_KEY) {
|
||||
throw new Error('Appwrite REST credentials are not configured')
|
||||
}
|
||||
|
||||
const response = await fetch(`${APPWRITE_ENDPOINT}${pathname}`, {
|
||||
method,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'X-Appwrite-Project': APPWRITE_PROJECT_ID,
|
||||
'X-Appwrite-Key': APPWRITE_KEY,
|
||||
},
|
||||
body: body ? JSON.stringify(body) : undefined,
|
||||
})
|
||||
const payload = await response.json().catch(() => ({}))
|
||||
if (!response.ok) {
|
||||
const error = new Error(payload.message || `Appwrite REST ${method} ${pathname} failed with HTTP ${response.status}`)
|
||||
error.status = response.status
|
||||
error.response = payload
|
||||
throw error
|
||||
}
|
||||
return payload
|
||||
}
|
||||
|
||||
function isAppwriteNotFound(error) {
|
||||
const message = `${error?.message || ''} ${error?.stderr || ''} ${error?.stdout || ''} ${error?.response?.message || ''}`
|
||||
return error?.status === 404 || /document_not_found|404|not found|could not be found/i.test(message)
|
||||
}
|
||||
|
||||
async function getLocalDocument(filename) {
|
||||
const filePath = resolveMemoryPath(filename)
|
||||
if (!filePath || !fs.existsSync(filePath)) return null
|
||||
const stats = await fsp.stat(filePath)
|
||||
return {
|
||||
filename,
|
||||
kind: kindForFilename(filename),
|
||||
content: await fsp.readFile(filePath, 'utf8'),
|
||||
$updatedAt: stats.mtime.toISOString(),
|
||||
}
|
||||
}
|
||||
|
||||
async function listLocalDocuments() {
|
||||
const filenames = ['MEMORY.md', ...fs.readdirSync(MEMORY_DIR).filter(isValidDailyMemoryFilename).sort()]
|
||||
return Promise.all(filenames.map((filename) => getLocalDocument(filename))).then((entries) => entries.filter(Boolean))
|
||||
}
|
||||
|
||||
async function getDocument(documentId) {
|
||||
try {
|
||||
return await runAppwriteJson([
|
||||
@@ -281,46 +385,49 @@ async function getDocument(documentId) {
|
||||
'--document-id', documentId,
|
||||
])
|
||||
} catch (error) {
|
||||
const stderr = `${error.stderr || ''}${error.stdout || ''}${error.message || ''}`
|
||||
if (/document_not_found|404|not found|could not be found/i.test(stderr)) return null
|
||||
const message = `${error?.message || ''} ${error?.stderr || ''} ${error?.stdout || ''}`
|
||||
if (isAppwriteScopeError(error)) return getLocalDocument(documentId)
|
||||
if (/document_not_found|404|not found|could not be found/i.test(message)) return null
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function listDocuments() {
|
||||
const result = await runAppwriteJson([
|
||||
'databases', 'list-documents',
|
||||
'--database-id', APPWRITE_DATABASE_ID,
|
||||
'--collection-id', APPWRITE_COLLECTION_ID,
|
||||
])
|
||||
return result?.documents || []
|
||||
try {
|
||||
const result = await runAppwriteJson([
|
||||
'databases', 'list-documents',
|
||||
'--database-id', APPWRITE_DATABASE_ID,
|
||||
'--collection-id', APPWRITE_COLLECTION_ID,
|
||||
])
|
||||
return result?.documents || []
|
||||
} catch (error) {
|
||||
if (isAppwriteScopeError(error)) return listLocalDocuments()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function upsertDocument(filename, content) {
|
||||
const data = JSON.stringify({
|
||||
const data = {
|
||||
filename,
|
||||
kind: kindForFilename(filename),
|
||||
content,
|
||||
})
|
||||
|
||||
const existing = await getDocument(filename)
|
||||
if (existing) {
|
||||
return runAppwriteJson([
|
||||
'databases', 'update-document',
|
||||
'--database-id', APPWRITE_DATABASE_ID,
|
||||
'--collection-id', APPWRITE_COLLECTION_ID,
|
||||
'--document-id', filename,
|
||||
'--data', data,
|
||||
])
|
||||
}
|
||||
|
||||
return runAppwriteJson([
|
||||
'databases', 'create-document',
|
||||
'--database-id', APPWRITE_DATABASE_ID,
|
||||
'--collection-id', APPWRITE_COLLECTION_ID,
|
||||
'--document-id', filename,
|
||||
'--data', data,
|
||||
])
|
||||
const documentPath = `/databases/${encodeURIComponent(APPWRITE_DATABASE_ID)}/collections/${encodeURIComponent(APPWRITE_COLLECTION_ID)}/documents/${encodeURIComponent(filename)}`
|
||||
try {
|
||||
return await appwriteRest('PATCH', documentPath, { data })
|
||||
} catch (error) {
|
||||
if (isAppwriteNotFound(error)) {
|
||||
return appwriteRest('POST', `/databases/${encodeURIComponent(APPWRITE_DATABASE_ID)}/collections/${encodeURIComponent(APPWRITE_COLLECTION_ID)}/documents`, {
|
||||
documentId: filename,
|
||||
data,
|
||||
})
|
||||
}
|
||||
if (isAppwriteScopeError(error)) {
|
||||
return { filename, content, fallback: true }
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function syncFileToAppwrite(filename) {
|
||||
@@ -337,8 +444,17 @@ async function syncWorkspaceToAppwrite() {
|
||||
]
|
||||
|
||||
for (const filename of filenames) {
|
||||
await syncFileToAppwrite(filename)
|
||||
try {
|
||||
await syncFileToAppwrite(filename)
|
||||
} catch (error) {
|
||||
if (isAppwriteScopeError(error)) {
|
||||
return { fallback: true, reason: 'appwrite-scope-missing' }
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
return { fallback: false }
|
||||
}
|
||||
|
||||
let syncPromise = syncWorkspaceToAppwrite().catch((error) => {
|
||||
@@ -365,6 +481,7 @@ function buildFileMeta(doc) {
|
||||
const mtimeMs = doc.$updatedAt ? new Date(doc.$updatedAt).getTime() : Date.now()
|
||||
return {
|
||||
filename: doc.filename,
|
||||
dateStamp: String(doc.filename || '').replace(/\.md$/, ''),
|
||||
mtimeMs,
|
||||
size: Buffer.byteLength(content, 'utf8'),
|
||||
wordCount: wordCount(content),
|
||||
@@ -375,10 +492,227 @@ function buildFileMeta(doc) {
|
||||
}
|
||||
}
|
||||
|
||||
function dateValueForFile(file) {
|
||||
const parsed = Date.parse(`${file?.dateStamp || String(file?.filename || '').replace(/\.md$/, '')}T12:00:00`)
|
||||
return Number.isFinite(parsed) ? parsed : file?.mtimeMs || Date.now()
|
||||
}
|
||||
|
||||
function pickSemanticSnippet(content = '', tokens = []) {
|
||||
const lines = content
|
||||
.split('\n')
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
|
||||
const exact = lines.find((line) => tokens.some((token) => line.toLowerCase().includes(token)))
|
||||
const candidate = exact || lines.find((line) => !line.startsWith('#')) || ''
|
||||
return candidate.slice(0, 180)
|
||||
}
|
||||
|
||||
function buildSemanticResults(docs = []) {
|
||||
const dailyFiles = docs
|
||||
.filter((doc) => isValidDailyMemoryFilename(doc.filename))
|
||||
.sort((a, b) => String(b.filename).localeCompare(String(a.filename)))
|
||||
.map((doc) => ({
|
||||
...buildFileMeta(doc),
|
||||
content: doc.content || '',
|
||||
}))
|
||||
|
||||
const collections = buildCollections(dailyFiles)
|
||||
|
||||
return function querySemantic(rawQuery = '') {
|
||||
const normalizedQuery = String(rawQuery || '').trim().toLowerCase()
|
||||
const tokens = tokenize(normalizedQuery)
|
||||
if (!normalizedQuery || (!tokens.length && normalizedQuery.length < 2)) return []
|
||||
|
||||
const tokenSet = new Set(tokens)
|
||||
const results = []
|
||||
|
||||
for (const file of dailyFiles) {
|
||||
const collectionHits = collectionsForFile(file, collections)
|
||||
const contentLower = file.content.toLowerCase()
|
||||
let score = 0
|
||||
const reasons = []
|
||||
|
||||
for (const tag of file.tags || []) {
|
||||
if (tokenSet.has(tag)) {
|
||||
score += 36
|
||||
reasons.push(`tag:${tag}`)
|
||||
}
|
||||
}
|
||||
|
||||
for (const collection of collectionHits) {
|
||||
if (normalizedQuery.includes(collection.label.toLowerCase()) || collection.tags.some((tag) => tokenSet.has(tag))) {
|
||||
score += 26
|
||||
reasons.push(`collection:${collection.label}`)
|
||||
}
|
||||
}
|
||||
|
||||
if (normalizedQuery.includes(file.mood)) {
|
||||
score += 12
|
||||
reasons.push(`mood:${file.mood}`)
|
||||
}
|
||||
|
||||
for (const token of tokens) {
|
||||
if (contentLower.includes(token)) {
|
||||
score += 9
|
||||
reasons.push(`text:${token}`)
|
||||
}
|
||||
}
|
||||
|
||||
const matchIndex = normalizedQuery.length >= 2 ? contentLower.indexOf(normalizedQuery) : -1
|
||||
if (matchIndex >= 0) {
|
||||
score += 18
|
||||
}
|
||||
|
||||
if (!score) continue
|
||||
|
||||
const windowStart = Math.max(0, (matchIndex >= 0 ? matchIndex : 0) - 70)
|
||||
const windowEnd = Math.min(file.content.length, (matchIndex >= 0 ? matchIndex : 110) + 130)
|
||||
const snippet = (matchIndex >= 0 ? file.content.slice(windowStart, windowEnd) : pickSemanticSnippet(file.content, tokens))
|
||||
.replace(/\n/g, ' ')
|
||||
.trim()
|
||||
|
||||
score += Math.max(0, 12 - Math.floor((Date.now() - dateValueForFile(file)) / 86400000))
|
||||
|
||||
results.push({
|
||||
filename: file.filename,
|
||||
dateLabel: file.dateLabel,
|
||||
dateStamp: file.dateStamp,
|
||||
tags: file.tags,
|
||||
mood: file.mood,
|
||||
wordCount: file.wordCount,
|
||||
entryCount: file.entryCount,
|
||||
score,
|
||||
snippet: `${windowStart > 0 ? '…' : ''}${snippet}${windowEnd < file.content.length ? '…' : ''}`,
|
||||
why: reasons.slice(0, 2).join(' · ') || 'semantic overlap',
|
||||
})
|
||||
}
|
||||
|
||||
return results.sort((a, b) => b.score - a.score || b.wordCount - a.wordCount).slice(0, 12)
|
||||
}
|
||||
}
|
||||
|
||||
function scoreDocumentForQuery(doc, tokens = []) {
|
||||
const content = doc.content || ''
|
||||
const lower = content.toLowerCase()
|
||||
const tags = generateTags(content, doc.filename)
|
||||
if (!tokens.length) return doc.filename === 'MEMORY.md' ? 30 : 8
|
||||
|
||||
let score = doc.filename === 'MEMORY.md' ? 12 : 0
|
||||
for (const tag of tags) {
|
||||
if (tokens.includes(tag)) score += 22
|
||||
}
|
||||
for (const token of tokens) {
|
||||
const count = lower.split(token).length - 1
|
||||
if (count > 0) score += Math.min(24, count * 6)
|
||||
}
|
||||
return score
|
||||
}
|
||||
|
||||
function snippetForQuery(content = '', tokens = [], max = 900) {
|
||||
const lower = content.toLowerCase()
|
||||
const firstIndex = tokens.map((token) => lower.indexOf(token)).filter((index) => index >= 0).sort((a, b) => a - b)[0] ?? 0
|
||||
const start = Math.max(0, firstIndex - 180)
|
||||
return content.slice(start, start + max).replace(/\n{3,}/g, '\n\n').trim()
|
||||
}
|
||||
|
||||
async function notifyMemoryChannel(event = {}) {
|
||||
if (!SLACK_BOT_TOKEN || !MEMORY_SLACK_CHANNEL_ID) return { skipped: true }
|
||||
|
||||
const title = event.title || 'Memory replicated'
|
||||
const filename = event.filename ? `\n• File: ${event.filename}` : ''
|
||||
const source = event.source ? `\n• Source: ${event.source}` : ''
|
||||
const summary = event.summary ? `\n• ${truncateText(event.summary, 220)}` : ''
|
||||
const text = `🧠 ${title}${filename}${source}${summary}`
|
||||
|
||||
try {
|
||||
const response = await fetch('https://slack.com/api/chat.postMessage', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${SLACK_BOT_TOKEN}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({ channel: MEMORY_SLACK_CHANNEL_ID, text }),
|
||||
})
|
||||
const payload = await response.json().catch(() => ({}))
|
||||
if (!response.ok || payload.ok === false) {
|
||||
console.warn('Memory Slack notification failed:', payload.error || response.statusText)
|
||||
return { ok: false, error: payload.error || response.statusText }
|
||||
}
|
||||
return { ok: true }
|
||||
} catch (error) {
|
||||
console.warn('Memory Slack notification failed:', error.message)
|
||||
return { ok: false, error: error.message }
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchImportSource(sourceUrl) {
|
||||
if (!sourceUrl) return ''
|
||||
let url
|
||||
try {
|
||||
url = new URL(sourceUrl)
|
||||
} catch {
|
||||
throw new Error('Source URL is invalid')
|
||||
}
|
||||
if (!['http:', 'https:'].includes(url.protocol)) {
|
||||
throw new Error('Only http(s) imports are supported')
|
||||
}
|
||||
|
||||
const response = await fetch(url, {
|
||||
headers: { 'User-Agent': 'FreeclawMemoryCloud/1.0 (+memory.friborg.uk)' },
|
||||
})
|
||||
if (!response.ok) throw new Error(`Import fetch failed: HTTP ${response.status}`)
|
||||
const raw = (await response.text()).slice(0, FETCH_MAX_CHARS)
|
||||
const contentType = response.headers.get('content-type') || ''
|
||||
return contentType.includes('html') ? stripHtml(raw) : raw.trim()
|
||||
}
|
||||
|
||||
async function appendDailyEntry({ heading, body, notify }) {
|
||||
const filename = `${todayDateStamp()}.md`
|
||||
const filePath = resolveDailyMemoryPath(filename)
|
||||
const existing = filePath && fs.existsSync(filePath) ? await fsp.readFile(filePath, 'utf8') : ''
|
||||
const prefix = existing && !existing.endsWith('\n') ? '\n' : ''
|
||||
const content = `${existing}${prefix}${body}`
|
||||
await writeThrough(filename, content, notify)
|
||||
return { filename, heading }
|
||||
}
|
||||
|
||||
function buildLlmContextPayload(docs = [], query = '', limit = 6) {
|
||||
const tokens = tokenize(query)
|
||||
const ranked = docs
|
||||
.filter((doc) => isValidMemoryFilename(doc.filename))
|
||||
.map((doc) => {
|
||||
const content = doc.content || ''
|
||||
const tags = generateTags(content, doc.filename)
|
||||
const score = scoreDocumentForQuery(doc, tokens)
|
||||
return {
|
||||
filename: doc.filename,
|
||||
kind: kindForFilename(doc.filename),
|
||||
dateLabel: doc.filename === 'MEMORY.md' ? 'Core memory' : formatDateLabel(doc.filename),
|
||||
tags,
|
||||
mood: detectMood(content, tags),
|
||||
score,
|
||||
snippet: redactSensitive(snippetForQuery(content, tokens, doc.filename === 'MEMORY.md' ? 1200 : 800)),
|
||||
updatedAt: doc.$updatedAt || null,
|
||||
}
|
||||
})
|
||||
.filter((entry) => entry.score > 0 || !tokens.length)
|
||||
.sort((a, b) => b.score - a.score || String(b.updatedAt || '').localeCompare(String(a.updatedAt || '')))
|
||||
.slice(0, Math.max(1, Math.min(Number(limit) || 6, 12)))
|
||||
|
||||
const context = ranked.map((entry) => [
|
||||
`## ${entry.dateLabel} (${entry.filename})`,
|
||||
`Tags: ${entry.tags.join(', ') || 'none'} | Mood: ${entry.mood} | Score: ${entry.score}`,
|
||||
entry.snippet,
|
||||
].join('\n')).join('\n\n---\n\n')
|
||||
|
||||
return { query, generatedAt: new Date().toISOString(), count: ranked.length, results: ranked, context }
|
||||
}
|
||||
|
||||
app.get('/api/health', async (req, res) => {
|
||||
try {
|
||||
await ensureSync()
|
||||
await runAppwrite(['databases', 'get-collection', '--database-id', APPWRITE_DATABASE_ID, '--collection-id', APPWRITE_COLLECTION_ID])
|
||||
await runAppwrite(['databases', 'get-document', '--database-id', APPWRITE_DATABASE_ID, '--collection-id', APPWRITE_COLLECTION_ID, '--document-id', 'MEMORY.md'])
|
||||
res.json({
|
||||
ok: true,
|
||||
appwrite: true,
|
||||
@@ -386,6 +720,15 @@ app.get('/api/health', async (req, res) => {
|
||||
mainMemory: fs.existsSync(MAIN_MEMORY_FILE),
|
||||
})
|
||||
} catch (error) {
|
||||
if (isAppwriteScopeError(error)) {
|
||||
return res.json({
|
||||
ok: true,
|
||||
appwrite: false,
|
||||
fallback: true,
|
||||
memoryDir: fs.existsSync(MEMORY_DIR),
|
||||
mainMemory: fs.existsSync(MAIN_MEMORY_FILE),
|
||||
})
|
||||
}
|
||||
res.status(500).json({ ok: false, error: error.message })
|
||||
}
|
||||
})
|
||||
@@ -469,11 +812,14 @@ app.get('/api/main-memory', async (req, res) => {
|
||||
}
|
||||
})
|
||||
|
||||
async function writeThrough(filename, content) {
|
||||
async function writeThrough(filename, content, notify = null) {
|
||||
const filePath = resolveMemoryPath(filename)
|
||||
if (!filePath) throw new Error('Invalid memory filename')
|
||||
await fsp.writeFile(filePath, content, 'utf8')
|
||||
await upsertDocument(filename, content)
|
||||
if (notify) {
|
||||
await notifyMemoryChannel({ filename, ...notify })
|
||||
}
|
||||
}
|
||||
|
||||
app.put('/api/main-memory', async (req, res) => {
|
||||
@@ -483,7 +829,7 @@ app.put('/api/main-memory', async (req, res) => {
|
||||
return res.status(400).json({ error: 'Content is required' })
|
||||
}
|
||||
await ensureSync()
|
||||
await writeThrough('MEMORY.md', content)
|
||||
await writeThrough('MEMORY.md', content, { title: 'Core memory replicated', summary: 'MEMORY.md was updated from the memory cockpit.' })
|
||||
res.json({ success: true })
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message })
|
||||
@@ -501,7 +847,7 @@ app.put('/api/memories/:filename', async (req, res) => {
|
||||
return res.status(400).json({ error: 'Content is required' })
|
||||
}
|
||||
await ensureSync()
|
||||
await writeThrough(filename, content)
|
||||
await writeThrough(filename, content, { title: 'Daily memory replicated', summary: `${filename} was updated from the memory cockpit.` })
|
||||
res.json({ success: true })
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message })
|
||||
@@ -541,6 +887,7 @@ app.get('/api/search', async (req, res) => {
|
||||
results.push({
|
||||
filename: doc.filename,
|
||||
dateLabel: formatDateLabel(doc.filename),
|
||||
dateStamp: String(doc.filename || '').replace(/\.md$/, ''),
|
||||
entryCount: getEntryCount(content),
|
||||
wordCount: wordCount(content),
|
||||
tags,
|
||||
@@ -558,6 +905,23 @@ app.get('/api/search', async (req, res) => {
|
||||
}
|
||||
})
|
||||
|
||||
app.get('/api/semantic', async (req, res) => {
|
||||
const query = typeof req.query.q === 'string' ? req.query.q.trim() : ''
|
||||
if (!query || query.length < 2) {
|
||||
return res.json({ query, results: [], total: 0 })
|
||||
}
|
||||
|
||||
try {
|
||||
await ensureSync()
|
||||
const docs = await listDocuments()
|
||||
const querySemantic = buildSemanticResults(docs)
|
||||
const results = querySemantic(query)
|
||||
res.json({ query, results, total: results.length })
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message })
|
||||
}
|
||||
})
|
||||
|
||||
app.post('/api/log', async (req, res) => {
|
||||
const message = typeof req.body?.message === 'string' ? req.body.message.trim() : ''
|
||||
if (!message) {
|
||||
@@ -571,7 +935,7 @@ app.post('/api/log', async (req, res) => {
|
||||
const existing = filePath && fs.existsSync(filePath) ? await fsp.readFile(filePath, 'utf8') : ''
|
||||
const prefix = existing && !existing.endsWith('\n') ? '\n' : ''
|
||||
const content = `${existing}${prefix}- [${currentTimeStamp()}] ${message}\n`
|
||||
await writeThrough(filename, content)
|
||||
await writeThrough(filename, content, { title: 'Memory note replicated', summary: message })
|
||||
res.json({ success: true, mode: 'appwrite' })
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message })
|
||||
@@ -655,13 +1019,92 @@ app.post('/api/distill', async (req, res) => {
|
||||
const dateStr = todayDateStamp()
|
||||
const sourceTag = sourceFilename ? ` (from ${sourceFilename})` : ''
|
||||
const distillEntry = `\n- **[${dateStr}]**${sourceTag} ${note.trim()}\n`
|
||||
await writeThrough('MEMORY.md', `${existing}${distillEntry}`)
|
||||
await writeThrough('MEMORY.md', `${existing}${distillEntry}`, { title: 'Distilled note replicated', source: sourceFilename || 'memory review', summary: note.trim() })
|
||||
res.json({ success: true, appended: distillEntry.trim() })
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message })
|
||||
}
|
||||
})
|
||||
|
||||
app.get('/api/platform', async (req, res) => {
|
||||
try {
|
||||
await ensureSync()
|
||||
const docs = await listDocuments()
|
||||
res.json({
|
||||
ok: true,
|
||||
appwrite: true,
|
||||
documents: docs.length,
|
||||
notifications: {
|
||||
slack: Boolean(SLACK_BOT_TOKEN && MEMORY_SLACK_CHANNEL_ID),
|
||||
channelId: MEMORY_SLACK_CHANNEL_ID || null,
|
||||
},
|
||||
llm: {
|
||||
contextEndpoint: '/api/llm/context?q=your+question',
|
||||
semanticEndpoint: '/api/semantic?q=your+question',
|
||||
},
|
||||
imports: {
|
||||
enabled: true,
|
||||
maxChars: IMPORT_MAX_CHARS,
|
||||
supportsUrl: true,
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
res.status(500).json({ ok: false, error: error.message })
|
||||
}
|
||||
})
|
||||
|
||||
app.get('/api/llm/context', async (req, res) => {
|
||||
const query = typeof req.query.q === 'string' ? req.query.q.trim() : ''
|
||||
const limit = Number.parseInt(req.query.limit || '6', 10)
|
||||
|
||||
try {
|
||||
await ensureSync()
|
||||
const docs = await listDocuments()
|
||||
res.json(buildLlmContextPayload(docs, query, limit))
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message })
|
||||
}
|
||||
})
|
||||
|
||||
app.post('/api/import', async (req, res) => {
|
||||
const { sourceTitle, sourceUrl, content, tags } = req.body || {}
|
||||
const title = truncateText(sourceTitle || sourceUrl || 'Imported knowledge', 120)
|
||||
|
||||
try {
|
||||
await ensureSync()
|
||||
const fetched = typeof content === 'string' && content.trim()
|
||||
? content.trim()
|
||||
: await fetchImportSource(sourceUrl)
|
||||
|
||||
const normalized = String(fetched || '').trim().slice(0, IMPORT_MAX_CHARS)
|
||||
if (!normalized || normalized.length < 8) {
|
||||
return res.status(400).json({ error: 'Import content is empty or too short' })
|
||||
}
|
||||
|
||||
const tagLine = Array.isArray(tags) && tags.length
|
||||
? `\n- Tags: ${tags.map((tag) => String(tag).trim()).filter(Boolean).join(', ')}`
|
||||
: ''
|
||||
const sourceLine = sourceUrl ? `\n- Source: ${sourceUrl}` : ''
|
||||
const body = [
|
||||
`\n## Knowledge import — ${title}\n`,
|
||||
`- [${currentTimeStamp()}] Imported into Memory Hub.${sourceLine}${tagLine}\n`,
|
||||
'```text\n',
|
||||
escapeFence(normalized.slice(0, IMPORT_MAX_CHARS)),
|
||||
'\n```\n',
|
||||
].join('')
|
||||
|
||||
const result = await appendDailyEntry({
|
||||
heading: title,
|
||||
body,
|
||||
notify: { title: 'Knowledge import replicated', source: sourceUrl || 'manual paste', summary: title },
|
||||
})
|
||||
|
||||
res.json({ success: true, filename: result.filename, title, importedChars: normalized.length })
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message })
|
||||
}
|
||||
})
|
||||
|
||||
app.use(express.static(DIST_DIR))
|
||||
app.get(/.*/, (req, res) => {
|
||||
res.sendFile(path.join(DIST_DIR, 'index.html'))
|
||||
|
||||
Reference in New Issue
Block a user