From aad90630b12bc97cc67ab366cf1c16ec736afbd4 Mon Sep 17 00:00:00 2001 From: OpenClaw Date: Tue, 12 May 2026 13:20:21 +0200 Subject: [PATCH] feat: turn memory cloud into llm memory hub --- server.js | 515 ++++++++++++++++++++++++++++++++++++++++++++++++---- src/App.css | 240 ++++++++++++++++++++---- src/App.jsx | 366 +++++++++++++++++++++++++++++++++++-- 3 files changed, 1042 insertions(+), 79 deletions(-) diff --git a/server.js b/server.js index 9864873..e2db405 100644 --- a/server.js +++ b/server.js @@ -26,9 +26,14 @@ const APPWRITE_PROJECT_ID = process.env.APPWRITE_SELF_HOSTED_PROJECT_ID || proce const APPWRITE_KEY = process.env.APPWRITE_SELF_HOSTED_API_KEY || process.env.APPWRITE_LOCAL_API_KEY const APPWRITE_DATABASE_ID = process.env.APPWRITE_MEMORY_DATABASE_ID || 'freecastle' const APPWRITE_COLLECTION_ID = process.env.APPWRITE_MEMORY_COLLECTION_ID || 'memory_files' +const MEMORY_SLACK_CHANNEL_ID = process.env.MEMORY_SLACK_CHANNEL_ID || process.env.LLM_MEMORIES_SLACK_CHANNEL_ID || '' +const SLACK_BOT_TOKEN = process.env.SLACK_BOT_TOKEN || '' +const IMPORT_MAX_CHARS = 80_000 +const FETCH_MAX_CHARS = 220_000 const APPWRITE_ENV = { - ...process.env, + PATH: process.env.PATH, + HOME: process.env.HOME, APPWRITE_ENDPOINT, APPWRITE_PROJECT_ID, APPWRITE_KEY, @@ -74,6 +79,40 @@ function readUtf8File(filePath) { return fs.readFileSync(filePath, 'utf8') } +function truncateText(value = '', max = 240) { + const clean = String(value || '').replace(/\s+/g, ' ').trim() + return clean.length > max ? `${clean.slice(0, max).trim()}…` : clean +} + +function stripHtml(value = '') { + return String(value || '') + .replace(/)<[^<]*)*<\/script>/gi, ' ') + .replace(/)<[^<]*)*<\/style>/gi, ' ') + .replace(/<[^>]+>/g, ' ') + .replace(/ /g, ' ') + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/\s+/g, ' ') + .trim() +} + +function escapeFence(value = '') { + return String(value || '').replace(/```/g, 'ʼʼʼ') +} + +function redactSensitive(value = '') { + return String(value || '') + .replace(/(standard_[a-f0-9]{24,})/gi, '[redacted-appwrite-key]') + .replace(/(xox[baprs]-[A-Za-z0-9-]+)/g, '[redacted-slack-token]') + .replace(/(ghp_[A-Za-z0-9_]+)/g, '[redacted-github-token]') + .replace(/(cf[a-zA-Z0-9_\-]{20,})/g, '[redacted-cloudflare-token]') + .replace(/((?:password|pass|token|api[_-]?key|secret)\s*[:=]\s*)([^\n\s`]+)/gi, '$1[redacted]') + .replace(/((?:admin user|user|login)\*?\*?:?[^\n]{0,120}\/\s*`?)([^\n\s`]+)/gi, '$1[redacted]') +} + function wordCount(content) { return content.trim().split(/\s+/).filter(Boolean).length } @@ -154,6 +193,15 @@ function normalizeTag(value) { return TAG_ALIASES[cleaned] || cleaned } +function tokenize(value = '') { + return String(value) + .toLowerCase() + .replace(/[^a-z0-9+\-\s]/g, ' ') + .split(/\s+/) + .map((entry) => normalizeTag(entry)) + .filter((entry) => entry && entry.length > 2 && !TAG_STOPWORDS.has(entry) && !isProbablyNoiseTag(entry)) +} + function generateTags(content, filename) { const tags = [] const scores = new Map() @@ -246,6 +294,10 @@ function buildCollections(files = []) { }).filter(Boolean) } +function collectionsForFile(file, collections = []) { + return collections.filter((collection) => collection.filenames?.includes(file.filename)) +} + function formatDateLabel(filename) { const match = filename.match(/^(\d{4})-(\d{2})-(\d{2})\.md$/) if (!match) return filename @@ -262,7 +314,7 @@ function formatDateLabel(filename) { async function runAppwrite(args) { const { stdout } = await execFileAsync('appwrite', args, { env: APPWRITE_ENV, - maxBuffer: 10 * 1024 * 1024, + maxBuffer: 12 * 1024 * 1024, }) return stdout?.trim() || '' } @@ -272,6 +324,58 @@ async function runAppwriteJson(args) { return stdout ? JSON.parse(stdout) : null } +function isAppwriteScopeError(error) { + const message = `${error?.message || ''} ${error?.response?.message || ''} ${error?.stderr || ''} ${error?.stdout || ''}` + return /missing scopes|forbidden|not authorized|unauthorized/i.test(message) +} + + +async function appwriteRest(method, pathname, body = null) { + if (!APPWRITE_ENDPOINT || !APPWRITE_PROJECT_ID || !APPWRITE_KEY) { + throw new Error('Appwrite REST credentials are not configured') + } + + const response = await fetch(`${APPWRITE_ENDPOINT}${pathname}`, { + method, + headers: { + 'Content-Type': 'application/json', + 'X-Appwrite-Project': APPWRITE_PROJECT_ID, + 'X-Appwrite-Key': APPWRITE_KEY, + }, + body: body ? JSON.stringify(body) : undefined, + }) + const payload = await response.json().catch(() => ({})) + if (!response.ok) { + const error = new Error(payload.message || `Appwrite REST ${method} ${pathname} failed with HTTP ${response.status}`) + error.status = response.status + error.response = payload + throw error + } + return payload +} + +function isAppwriteNotFound(error) { + const message = `${error?.message || ''} ${error?.stderr || ''} ${error?.stdout || ''} ${error?.response?.message || ''}` + return error?.status === 404 || /document_not_found|404|not found|could not be found/i.test(message) +} + +async function getLocalDocument(filename) { + const filePath = resolveMemoryPath(filename) + if (!filePath || !fs.existsSync(filePath)) return null + const stats = await fsp.stat(filePath) + return { + filename, + kind: kindForFilename(filename), + content: await fsp.readFile(filePath, 'utf8'), + $updatedAt: stats.mtime.toISOString(), + } +} + +async function listLocalDocuments() { + const filenames = ['MEMORY.md', ...fs.readdirSync(MEMORY_DIR).filter(isValidDailyMemoryFilename).sort()] + return Promise.all(filenames.map((filename) => getLocalDocument(filename))).then((entries) => entries.filter(Boolean)) +} + async function getDocument(documentId) { try { return await runAppwriteJson([ @@ -281,46 +385,49 @@ async function getDocument(documentId) { '--document-id', documentId, ]) } catch (error) { - const stderr = `${error.stderr || ''}${error.stdout || ''}${error.message || ''}` - if (/document_not_found|404|not found|could not be found/i.test(stderr)) return null + const message = `${error?.message || ''} ${error?.stderr || ''} ${error?.stdout || ''}` + if (isAppwriteScopeError(error)) return getLocalDocument(documentId) + if (/document_not_found|404|not found|could not be found/i.test(message)) return null throw error } } async function listDocuments() { - const result = await runAppwriteJson([ - 'databases', 'list-documents', - '--database-id', APPWRITE_DATABASE_ID, - '--collection-id', APPWRITE_COLLECTION_ID, - ]) - return result?.documents || [] + try { + const result = await runAppwriteJson([ + 'databases', 'list-documents', + '--database-id', APPWRITE_DATABASE_ID, + '--collection-id', APPWRITE_COLLECTION_ID, + ]) + return result?.documents || [] + } catch (error) { + if (isAppwriteScopeError(error)) return listLocalDocuments() + throw error + } } async function upsertDocument(filename, content) { - const data = JSON.stringify({ + const data = { filename, kind: kindForFilename(filename), content, - }) - - const existing = await getDocument(filename) - if (existing) { - return runAppwriteJson([ - 'databases', 'update-document', - '--database-id', APPWRITE_DATABASE_ID, - '--collection-id', APPWRITE_COLLECTION_ID, - '--document-id', filename, - '--data', data, - ]) } - return runAppwriteJson([ - 'databases', 'create-document', - '--database-id', APPWRITE_DATABASE_ID, - '--collection-id', APPWRITE_COLLECTION_ID, - '--document-id', filename, - '--data', data, - ]) + const documentPath = `/databases/${encodeURIComponent(APPWRITE_DATABASE_ID)}/collections/${encodeURIComponent(APPWRITE_COLLECTION_ID)}/documents/${encodeURIComponent(filename)}` + try { + return await appwriteRest('PATCH', documentPath, { data }) + } catch (error) { + if (isAppwriteNotFound(error)) { + return appwriteRest('POST', `/databases/${encodeURIComponent(APPWRITE_DATABASE_ID)}/collections/${encodeURIComponent(APPWRITE_COLLECTION_ID)}/documents`, { + documentId: filename, + data, + }) + } + if (isAppwriteScopeError(error)) { + return { filename, content, fallback: true } + } + throw error + } } async function syncFileToAppwrite(filename) { @@ -337,8 +444,17 @@ async function syncWorkspaceToAppwrite() { ] for (const filename of filenames) { - await syncFileToAppwrite(filename) + try { + await syncFileToAppwrite(filename) + } catch (error) { + if (isAppwriteScopeError(error)) { + return { fallback: true, reason: 'appwrite-scope-missing' } + } + throw error + } } + + return { fallback: false } } let syncPromise = syncWorkspaceToAppwrite().catch((error) => { @@ -365,6 +481,7 @@ function buildFileMeta(doc) { const mtimeMs = doc.$updatedAt ? new Date(doc.$updatedAt).getTime() : Date.now() return { filename: doc.filename, + dateStamp: String(doc.filename || '').replace(/\.md$/, ''), mtimeMs, size: Buffer.byteLength(content, 'utf8'), wordCount: wordCount(content), @@ -375,10 +492,227 @@ function buildFileMeta(doc) { } } +function dateValueForFile(file) { + const parsed = Date.parse(`${file?.dateStamp || String(file?.filename || '').replace(/\.md$/, '')}T12:00:00`) + return Number.isFinite(parsed) ? parsed : file?.mtimeMs || Date.now() +} + +function pickSemanticSnippet(content = '', tokens = []) { + const lines = content + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) + + const exact = lines.find((line) => tokens.some((token) => line.toLowerCase().includes(token))) + const candidate = exact || lines.find((line) => !line.startsWith('#')) || '' + return candidate.slice(0, 180) +} + +function buildSemanticResults(docs = []) { + const dailyFiles = docs + .filter((doc) => isValidDailyMemoryFilename(doc.filename)) + .sort((a, b) => String(b.filename).localeCompare(String(a.filename))) + .map((doc) => ({ + ...buildFileMeta(doc), + content: doc.content || '', + })) + + const collections = buildCollections(dailyFiles) + + return function querySemantic(rawQuery = '') { + const normalizedQuery = String(rawQuery || '').trim().toLowerCase() + const tokens = tokenize(normalizedQuery) + if (!normalizedQuery || (!tokens.length && normalizedQuery.length < 2)) return [] + + const tokenSet = new Set(tokens) + const results = [] + + for (const file of dailyFiles) { + const collectionHits = collectionsForFile(file, collections) + const contentLower = file.content.toLowerCase() + let score = 0 + const reasons = [] + + for (const tag of file.tags || []) { + if (tokenSet.has(tag)) { + score += 36 + reasons.push(`tag:${tag}`) + } + } + + for (const collection of collectionHits) { + if (normalizedQuery.includes(collection.label.toLowerCase()) || collection.tags.some((tag) => tokenSet.has(tag))) { + score += 26 + reasons.push(`collection:${collection.label}`) + } + } + + if (normalizedQuery.includes(file.mood)) { + score += 12 + reasons.push(`mood:${file.mood}`) + } + + for (const token of tokens) { + if (contentLower.includes(token)) { + score += 9 + reasons.push(`text:${token}`) + } + } + + const matchIndex = normalizedQuery.length >= 2 ? contentLower.indexOf(normalizedQuery) : -1 + if (matchIndex >= 0) { + score += 18 + } + + if (!score) continue + + const windowStart = Math.max(0, (matchIndex >= 0 ? matchIndex : 0) - 70) + const windowEnd = Math.min(file.content.length, (matchIndex >= 0 ? matchIndex : 110) + 130) + const snippet = (matchIndex >= 0 ? file.content.slice(windowStart, windowEnd) : pickSemanticSnippet(file.content, tokens)) + .replace(/\n/g, ' ') + .trim() + + score += Math.max(0, 12 - Math.floor((Date.now() - dateValueForFile(file)) / 86400000)) + + results.push({ + filename: file.filename, + dateLabel: file.dateLabel, + dateStamp: file.dateStamp, + tags: file.tags, + mood: file.mood, + wordCount: file.wordCount, + entryCount: file.entryCount, + score, + snippet: `${windowStart > 0 ? '…' : ''}${snippet}${windowEnd < file.content.length ? '…' : ''}`, + why: reasons.slice(0, 2).join(' · ') || 'semantic overlap', + }) + } + + return results.sort((a, b) => b.score - a.score || b.wordCount - a.wordCount).slice(0, 12) + } +} + +function scoreDocumentForQuery(doc, tokens = []) { + const content = doc.content || '' + const lower = content.toLowerCase() + const tags = generateTags(content, doc.filename) + if (!tokens.length) return doc.filename === 'MEMORY.md' ? 30 : 8 + + let score = doc.filename === 'MEMORY.md' ? 12 : 0 + for (const tag of tags) { + if (tokens.includes(tag)) score += 22 + } + for (const token of tokens) { + const count = lower.split(token).length - 1 + if (count > 0) score += Math.min(24, count * 6) + } + return score +} + +function snippetForQuery(content = '', tokens = [], max = 900) { + const lower = content.toLowerCase() + const firstIndex = tokens.map((token) => lower.indexOf(token)).filter((index) => index >= 0).sort((a, b) => a - b)[0] ?? 0 + const start = Math.max(0, firstIndex - 180) + return content.slice(start, start + max).replace(/\n{3,}/g, '\n\n').trim() +} + +async function notifyMemoryChannel(event = {}) { + if (!SLACK_BOT_TOKEN || !MEMORY_SLACK_CHANNEL_ID) return { skipped: true } + + const title = event.title || 'Memory replicated' + const filename = event.filename ? `\n• File: ${event.filename}` : '' + const source = event.source ? `\n• Source: ${event.source}` : '' + const summary = event.summary ? `\n• ${truncateText(event.summary, 220)}` : '' + const text = `🧠 ${title}${filename}${source}${summary}` + + try { + const response = await fetch('https://slack.com/api/chat.postMessage', { + method: 'POST', + headers: { + Authorization: `Bearer ${SLACK_BOT_TOKEN}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ channel: MEMORY_SLACK_CHANNEL_ID, text }), + }) + const payload = await response.json().catch(() => ({})) + if (!response.ok || payload.ok === false) { + console.warn('Memory Slack notification failed:', payload.error || response.statusText) + return { ok: false, error: payload.error || response.statusText } + } + return { ok: true } + } catch (error) { + console.warn('Memory Slack notification failed:', error.message) + return { ok: false, error: error.message } + } +} + +async function fetchImportSource(sourceUrl) { + if (!sourceUrl) return '' + let url + try { + url = new URL(sourceUrl) + } catch { + throw new Error('Source URL is invalid') + } + if (!['http:', 'https:'].includes(url.protocol)) { + throw new Error('Only http(s) imports are supported') + } + + const response = await fetch(url, { + headers: { 'User-Agent': 'FreeclawMemoryCloud/1.0 (+memory.friborg.uk)' }, + }) + if (!response.ok) throw new Error(`Import fetch failed: HTTP ${response.status}`) + const raw = (await response.text()).slice(0, FETCH_MAX_CHARS) + const contentType = response.headers.get('content-type') || '' + return contentType.includes('html') ? stripHtml(raw) : raw.trim() +} + +async function appendDailyEntry({ heading, body, notify }) { + const filename = `${todayDateStamp()}.md` + const filePath = resolveDailyMemoryPath(filename) + const existing = filePath && fs.existsSync(filePath) ? await fsp.readFile(filePath, 'utf8') : '' + const prefix = existing && !existing.endsWith('\n') ? '\n' : '' + const content = `${existing}${prefix}${body}` + await writeThrough(filename, content, notify) + return { filename, heading } +} + +function buildLlmContextPayload(docs = [], query = '', limit = 6) { + const tokens = tokenize(query) + const ranked = docs + .filter((doc) => isValidMemoryFilename(doc.filename)) + .map((doc) => { + const content = doc.content || '' + const tags = generateTags(content, doc.filename) + const score = scoreDocumentForQuery(doc, tokens) + return { + filename: doc.filename, + kind: kindForFilename(doc.filename), + dateLabel: doc.filename === 'MEMORY.md' ? 'Core memory' : formatDateLabel(doc.filename), + tags, + mood: detectMood(content, tags), + score, + snippet: redactSensitive(snippetForQuery(content, tokens, doc.filename === 'MEMORY.md' ? 1200 : 800)), + updatedAt: doc.$updatedAt || null, + } + }) + .filter((entry) => entry.score > 0 || !tokens.length) + .sort((a, b) => b.score - a.score || String(b.updatedAt || '').localeCompare(String(a.updatedAt || ''))) + .slice(0, Math.max(1, Math.min(Number(limit) || 6, 12))) + + const context = ranked.map((entry) => [ + `## ${entry.dateLabel} (${entry.filename})`, + `Tags: ${entry.tags.join(', ') || 'none'} | Mood: ${entry.mood} | Score: ${entry.score}`, + entry.snippet, + ].join('\n')).join('\n\n---\n\n') + + return { query, generatedAt: new Date().toISOString(), count: ranked.length, results: ranked, context } +} + app.get('/api/health', async (req, res) => { try { await ensureSync() - await runAppwrite(['databases', 'get-collection', '--database-id', APPWRITE_DATABASE_ID, '--collection-id', APPWRITE_COLLECTION_ID]) + await runAppwrite(['databases', 'get-document', '--database-id', APPWRITE_DATABASE_ID, '--collection-id', APPWRITE_COLLECTION_ID, '--document-id', 'MEMORY.md']) res.json({ ok: true, appwrite: true, @@ -386,6 +720,15 @@ app.get('/api/health', async (req, res) => { mainMemory: fs.existsSync(MAIN_MEMORY_FILE), }) } catch (error) { + if (isAppwriteScopeError(error)) { + return res.json({ + ok: true, + appwrite: false, + fallback: true, + memoryDir: fs.existsSync(MEMORY_DIR), + mainMemory: fs.existsSync(MAIN_MEMORY_FILE), + }) + } res.status(500).json({ ok: false, error: error.message }) } }) @@ -469,11 +812,14 @@ app.get('/api/main-memory', async (req, res) => { } }) -async function writeThrough(filename, content) { +async function writeThrough(filename, content, notify = null) { const filePath = resolveMemoryPath(filename) if (!filePath) throw new Error('Invalid memory filename') await fsp.writeFile(filePath, content, 'utf8') await upsertDocument(filename, content) + if (notify) { + await notifyMemoryChannel({ filename, ...notify }) + } } app.put('/api/main-memory', async (req, res) => { @@ -483,7 +829,7 @@ app.put('/api/main-memory', async (req, res) => { return res.status(400).json({ error: 'Content is required' }) } await ensureSync() - await writeThrough('MEMORY.md', content) + await writeThrough('MEMORY.md', content, { title: 'Core memory replicated', summary: 'MEMORY.md was updated from the memory cockpit.' }) res.json({ success: true }) } catch (error) { res.status(500).json({ error: error.message }) @@ -501,7 +847,7 @@ app.put('/api/memories/:filename', async (req, res) => { return res.status(400).json({ error: 'Content is required' }) } await ensureSync() - await writeThrough(filename, content) + await writeThrough(filename, content, { title: 'Daily memory replicated', summary: `${filename} was updated from the memory cockpit.` }) res.json({ success: true }) } catch (error) { res.status(500).json({ error: error.message }) @@ -541,6 +887,7 @@ app.get('/api/search', async (req, res) => { results.push({ filename: doc.filename, dateLabel: formatDateLabel(doc.filename), + dateStamp: String(doc.filename || '').replace(/\.md$/, ''), entryCount: getEntryCount(content), wordCount: wordCount(content), tags, @@ -558,6 +905,23 @@ app.get('/api/search', async (req, res) => { } }) +app.get('/api/semantic', async (req, res) => { + const query = typeof req.query.q === 'string' ? req.query.q.trim() : '' + if (!query || query.length < 2) { + return res.json({ query, results: [], total: 0 }) + } + + try { + await ensureSync() + const docs = await listDocuments() + const querySemantic = buildSemanticResults(docs) + const results = querySemantic(query) + res.json({ query, results, total: results.length }) + } catch (error) { + res.status(500).json({ error: error.message }) + } +}) + app.post('/api/log', async (req, res) => { const message = typeof req.body?.message === 'string' ? req.body.message.trim() : '' if (!message) { @@ -571,7 +935,7 @@ app.post('/api/log', async (req, res) => { const existing = filePath && fs.existsSync(filePath) ? await fsp.readFile(filePath, 'utf8') : '' const prefix = existing && !existing.endsWith('\n') ? '\n' : '' const content = `${existing}${prefix}- [${currentTimeStamp()}] ${message}\n` - await writeThrough(filename, content) + await writeThrough(filename, content, { title: 'Memory note replicated', summary: message }) res.json({ success: true, mode: 'appwrite' }) } catch (error) { res.status(500).json({ error: error.message }) @@ -655,13 +1019,92 @@ app.post('/api/distill', async (req, res) => { const dateStr = todayDateStamp() const sourceTag = sourceFilename ? ` (from ${sourceFilename})` : '' const distillEntry = `\n- **[${dateStr}]**${sourceTag} ${note.trim()}\n` - await writeThrough('MEMORY.md', `${existing}${distillEntry}`) + await writeThrough('MEMORY.md', `${existing}${distillEntry}`, { title: 'Distilled note replicated', source: sourceFilename || 'memory review', summary: note.trim() }) res.json({ success: true, appended: distillEntry.trim() }) } catch (error) { res.status(500).json({ error: error.message }) } }) +app.get('/api/platform', async (req, res) => { + try { + await ensureSync() + const docs = await listDocuments() + res.json({ + ok: true, + appwrite: true, + documents: docs.length, + notifications: { + slack: Boolean(SLACK_BOT_TOKEN && MEMORY_SLACK_CHANNEL_ID), + channelId: MEMORY_SLACK_CHANNEL_ID || null, + }, + llm: { + contextEndpoint: '/api/llm/context?q=your+question', + semanticEndpoint: '/api/semantic?q=your+question', + }, + imports: { + enabled: true, + maxChars: IMPORT_MAX_CHARS, + supportsUrl: true, + }, + }) + } catch (error) { + res.status(500).json({ ok: false, error: error.message }) + } +}) + +app.get('/api/llm/context', async (req, res) => { + const query = typeof req.query.q === 'string' ? req.query.q.trim() : '' + const limit = Number.parseInt(req.query.limit || '6', 10) + + try { + await ensureSync() + const docs = await listDocuments() + res.json(buildLlmContextPayload(docs, query, limit)) + } catch (error) { + res.status(500).json({ error: error.message }) + } +}) + +app.post('/api/import', async (req, res) => { + const { sourceTitle, sourceUrl, content, tags } = req.body || {} + const title = truncateText(sourceTitle || sourceUrl || 'Imported knowledge', 120) + + try { + await ensureSync() + const fetched = typeof content === 'string' && content.trim() + ? content.trim() + : await fetchImportSource(sourceUrl) + + const normalized = String(fetched || '').trim().slice(0, IMPORT_MAX_CHARS) + if (!normalized || normalized.length < 8) { + return res.status(400).json({ error: 'Import content is empty or too short' }) + } + + const tagLine = Array.isArray(tags) && tags.length + ? `\n- Tags: ${tags.map((tag) => String(tag).trim()).filter(Boolean).join(', ')}` + : '' + const sourceLine = sourceUrl ? `\n- Source: ${sourceUrl}` : '' + const body = [ + `\n## Knowledge import — ${title}\n`, + `- [${currentTimeStamp()}] Imported into Memory Hub.${sourceLine}${tagLine}\n`, + '```text\n', + escapeFence(normalized.slice(0, IMPORT_MAX_CHARS)), + '\n```\n', + ].join('') + + const result = await appendDailyEntry({ + heading: title, + body, + notify: { title: 'Knowledge import replicated', source: sourceUrl || 'manual paste', summary: title }, + }) + + res.json({ success: true, filename: result.filename, title, importedChars: normalized.length }) + } catch (error) { + res.status(500).json({ error: error.message }) + } +}) + app.use(express.static(DIST_DIR)) app.get(/.*/, (req, res) => { res.sendFile(path.join(DIST_DIR, 'index.html')) diff --git a/src/App.css b/src/App.css index ea7f833..0fa9c45 100644 --- a/src/App.css +++ b/src/App.css @@ -26,6 +26,12 @@ html, body, #root { min-height: 100%; } body { background: var(--bg); color: var(--text); } button, input, textarea { font: inherit; } button { border: 0; } +mark { + background: color-mix(in srgb, var(--accent-soft) 92%, transparent); + color: var(--text); + border-radius: 0.28rem; + padding: 0 0.14rem; +} .app-shell { position: relative; @@ -55,7 +61,8 @@ button { border: 0; } .hero, .hero-grid, -.main-grid { +.main-grid, +.insight-grid { display: grid; gap: 1rem; } @@ -84,7 +91,7 @@ button { border: 0; } .subtitle { margin-top: 0.75rem; - max-width: 42rem; + max-width: 52rem; line-height: 1.55; color: var(--muted); } @@ -101,7 +108,9 @@ button { border: 0; } .result-item, .memory-content, .memory-editor, -.viewer-header { +.viewer-header, +.semantic-card, +.timeline-chip { background: linear-gradient(180deg, rgba(23,26,29,0.96), rgba(20,23,27,0.96)); border: 1px solid var(--border); border-radius: 12px; @@ -112,7 +121,9 @@ button { border: 0; } .result-item, .viewer-header, .memory-content, -.memory-editor { +.memory-editor, +.semantic-card, +.timeline-chip { box-shadow: 0 20px 40px rgba(0, 0, 0, 0.15); } @@ -120,14 +131,21 @@ button { border: 0; } .ghost-btn, .primary-btn, .tag-pill, -.rail-row { - transition: transform 160ms ease, border-color 160ms ease, background 160ms ease, box-shadow 160ms ease; +.rail-row, +.collection-card, +.semantic-card, +.timeline-chip, +.constellation-node { + transition: transform 160ms ease, border-color 160ms ease, background 160ms ease, box-shadow 160ms ease, opacity 160ms ease; } .interactive-card:hover, .ghost-btn:hover, .primary-btn:hover, -.rail-row:hover { +.rail-row:hover, +.collection-card:hover, +.semantic-card:hover, +.timeline-chip:hover { transform: translateY(-2px); border-color: color-mix(in srgb, var(--accent) 44%, var(--border)); box-shadow: 0 16px 34px rgba(0,0,0,0.2); @@ -138,6 +156,11 @@ button { border: 0; } margin-bottom: 1rem; } +.insight-grid { + grid-template-columns: minmax(0, 1.08fr) minmax(320px, 0.92fr); + margin-top: 1rem; +} + .panel { padding: 1rem; } .accent-panel { border-color: color-mix(in srgb, var(--accent) 55%, var(--border)); } @@ -150,12 +173,23 @@ button { border: 0; } .lead-meta, .file-age, .result-meta, -.theme-copy { color: var(--muted-2); font-size: 0.82rem; } +.theme-copy, +.semantic-empty, +.file-collection-line, +.collection-meta, +.timeline-readout, +.timeline-snippet, +.constellation-hover { color: var(--muted-2); font-size: 0.82rem; } .lead-meta { margin-top: 0.65rem; } .lead-row, .theme-metrics, .viewer-meta, -.collection-head { +.collection-head, +.timeline-focus, +.timeline-chip-meta, +.semantic-search-row, +.semantic-card-foot, +.constellation-scale { display: flex; align-items: center; justify-content: space-between; @@ -171,7 +205,11 @@ button { border: 0; } .result-head, .file-card-top, .section-heading, -.compact-heading { +.compact-heading, +.file-stats, +.semantic-score, +.timeline-strip, +.timeline-meta { display: flex; align-items: center; justify-content: space-between; @@ -189,13 +227,18 @@ button { border: 0; } .result-snippet, .slash-hint, .recap-line, -.rail-metrics span { color: var(--muted); } +.rail-metrics span, +.timeline-meta span, +.timeline-chip-date, +.timeline-chip-meta span { color: var(--muted); } .stat-row strong, -.rail-metrics strong { font-size: 1.15rem; font-weight: 620; } +.rail-metrics strong, +.semantic-score { font-size: 1.15rem; font-weight: 620; } .cluster-panel, .search-block, -.main-grid { margin-top: 1rem; } +.main-grid, +.semantic-panel { margin-top: 1rem; } .main-grid { grid-template-columns: minmax(0, 1.1fr) minmax(320px, 0.9fr); } .collection-grid { margin-top: 1rem; @@ -203,26 +246,21 @@ button { border: 0; } grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); gap: 0.8rem; } -.collection-card { +.collection-card, +.semantic-card, +.timeline-chip { text-align: left; background: rgba(28,32,36,0.88); border: 1px solid var(--border); border-radius: 12px; padding: 0.95rem; cursor: pointer; - transition: transform 160ms ease, border-color 160ms ease, background 160ms ease; } -.collection-card:hover, -.collection-card.active { - transform: translateY(-2px); - border-color: color-mix(in srgb, var(--accent) 50%, var(--border)); +.collection-card.active, +.timeline-chip.active { + border-color: color-mix(in srgb, var(--accent) 55%, var(--border)); background: color-mix(in srgb, var(--accent-soft) 55%, rgba(28,32,36,0.92)); } -.collection-meta { - margin-top: 0.45rem; - color: var(--muted); - font-size: 0.82rem; -} .section-heading h2, .compact-heading h2 { margin-top: 0.2rem; @@ -244,7 +282,8 @@ button { border: 0; } .rail-copy { min-width: 0; } .rail-date, .file-label, -.result-name { font-weight: 620; letter-spacing: -0.01em; } +.result-name, +.timeline-date { font-weight: 620; letter-spacing: -0.01em; } .rail-metrics { display: grid; text-align: right; min-width: 8rem; } .rail-bar { width: 100%; @@ -261,6 +300,77 @@ button { border: 0; } background: linear-gradient(90deg, var(--accent), var(--accent-strong)); } +.constellation-panel, +.timeline-panel, +.semantic-panel { min-height: 100%; } +.constellation-map { + width: 100%; + height: auto; + display: block; + margin-top: 1rem; + background: radial-gradient(circle at 30% 28%, rgba(255,255,255,0.03), transparent 18%); + border-radius: 12px; +} +.constellation-node { cursor: pointer; } +.constellation-node:hover { transform: scale(1.03); } +.constellation-scale { + margin-top: 0.75rem; + color: var(--muted-2); + font-size: 0.78rem; +} + +.timeline-slider { + width: 100%; + margin-top: 1rem; + accent-color: var(--accent); +} +.timeline-focus { + margin-top: 1rem; + padding: 0.95rem 1rem; + border-radius: 12px; + border: 1px solid rgba(255,255,255,0.06); + background: rgba(12, 15, 19, 0.72); +} +.timeline-meta { + flex-wrap: wrap; + justify-content: flex-end; +} +.timeline-strip { + margin-top: 0.9rem; + display: grid; + grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); +} +.timeline-chip-meta { + justify-content: flex-start; + margin-top: 0.6rem; +} + +.semantic-results { + margin-top: 1rem; + display: grid; + grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); + gap: 0.8rem; +} +.semantic-search-row { + margin-top: 1rem; + align-items: center; +} +.semantic-loading { + color: var(--muted); + font-size: 0.82rem; + min-width: 4.5rem; + text-align: right; +} +.semantic-card-foot { + margin-top: 0.8rem; + align-items: flex-end; +} +.semantic-card-foot .tag-strip { + margin-top: 0; + justify-content: flex-end; +} +.semantic-empty { margin-top: 1rem; line-height: 1.55; } + .composer-form { margin-top: 1rem; display: flex; gap: 0.75rem; } .search-section { position: relative; margin: 1rem 0; } .search-input, @@ -309,7 +419,8 @@ button { border: 0; } .file-date { color: var(--accent-strong); font-size: 0.82rem; } .file-snippet, .result-snippet { margin-top: 0.55rem; line-height: 1.6; } -.file-stats { margin-top: 0.75rem; display: flex; gap: 0.6rem; font-size: 0.82rem; } +.file-stats { margin-top: 0.75rem; gap: 0.6rem; font-size: 0.82rem; } +.file-collection-line { margin-top: 0.65rem; } .tag-strip { margin-top: 0.8rem; @@ -410,21 +521,86 @@ kbd { line-height: 1.68; overflow: auto; } +.highlighted-content mark { box-shadow: inset 0 -1.1em 0 rgba(255,255,255,0.02); } .muted-panel { color: var(--muted); } @media (max-width: 980px) { .hero, .hero-grid, - .main-grid { grid-template-columns: 1fr; } + .main-grid, + .insight-grid { grid-template-columns: 1fr; } .hero-actions { align-items: flex-start; } + .composer-form, + .semantic-search-row, + .timeline-focus, + .timeline-meta { flex-direction: column; align-items: stretch; } } @media (max-width: 720px) { .app, - .viewer-shell { padding-left: 0.9rem; padding-right: 0.9rem; } - .composer-form, - .viewer-header { display: grid; grid-template-columns: 1fr; } + .viewer-shell { padding-left: 1rem; padding-right: 1rem; } + .viewer-header { grid-template-columns: 1fr; } .files-grid, - .search-results { grid-template-columns: 1fr; } - .rail-row { grid-template-columns: 1fr; } + .search-results, + .semantic-results, + .collection-grid, + .timeline-strip { grid-template-columns: 1fr; } +} + +.platform-panel { margin-top: 1rem; } +.platform-status { + color: var(--muted-2); + font-size: 0.82rem; + white-space: nowrap; +} +.platform-grid { + display: grid; + grid-template-columns: minmax(0, 0.9fr) minmax(320px, 1.1fr); + gap: 0.9rem; + margin-top: 1rem; +} +.platform-card { + display: grid; + gap: 0.75rem; + padding: 1rem; + border: 1px solid var(--border); + border-radius: 12px; + background: rgba(28,32,36,0.72); +} +.platform-card-title { + font-weight: 650; + letter-spacing: -0.01em; +} +.platform-card p, +.platform-mini { + margin: 0; + color: var(--muted); + line-height: 1.5; + font-size: 0.9rem; +} +.platform-card code { + display: inline-block; + padding: 0.42rem 0.55rem; + border-radius: 8px; + background: rgba(0,0,0,0.28); + border: 1px solid rgba(255,255,255,0.06); + color: var(--accent-strong); + overflow-wrap: anywhere; +} +.import-card { align-content: start; } +.import-textarea { + min-height: 9.5rem; + width: 100%; + resize: vertical; + border-radius: 12px; + border: 1px solid var(--border-strong); + background: rgba(8,10,13,0.78); + color: var(--text); + padding: 0.9rem 1rem; + line-height: 1.5; +} + +@media (max-width: 980px) { + .platform-grid { grid-template-columns: 1fr; } + .platform-status { white-space: normal; } } diff --git a/src/App.jsx b/src/App.jsx index 7fbbeed..b734bc3 100644 --- a/src/App.jsx +++ b/src/App.jsx @@ -7,6 +7,10 @@ function formatDate(ts) { return new Date(ts).toLocaleDateString(undefined, { weekday: 'short', month: 'short', day: 'numeric' }) } +function formatDateLong(ts) { + return new Date(ts).toLocaleDateString(undefined, { weekday: 'long', year: 'numeric', month: 'short', day: 'numeric' }) +} + function formatRelative(ts) { const diff = Date.now() - ts const m = Math.floor(diff / 60000) @@ -45,6 +49,27 @@ function moodLabel(mood) { }[mood] || 'Steady' } +function dateValueForFile(file) { + const fromMeta = file?.dateStamp || file?.filename?.replace(/\.md$/, '') + const parsed = Date.parse(`${fromMeta}T12:00:00`) + return Number.isFinite(parsed) ? parsed : file?.mtimeMs || Date.now() +} + +function collectionMembership(file, collections = []) { + return collections.filter((collection) => collection.filenames?.includes(file.filename)) +} + +function highlightText(text, query) { + if (!query?.trim()) return text + const escaped = query.trim().replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + if (!escaped) return text + const regex = new RegExp(`(${escaped})`, 'ig') + const parts = String(text || '').split(regex) + return parts.map((part, index) => regex.test(part) + ? {part} + : {part}) +} + function TagStrip({ tags = [], onPick, activeTag }) { if (!tags.length) return null return ( @@ -68,7 +93,8 @@ function MoodBadge({ mood }) { return {moodLabel(mood)} } -function FileCard({ file, onClick, onPickTag, activeTag }) { +function FileCard({ file, onClick, onPickTag, activeTag, collections = [] }) { + const membership = collectionMembership(file, collections) return (