From d01cc6ede7b3367826ac74f0befb55399612341e Mon Sep 17 00:00:00 2001 From: OpenClaw Bot Date: Tue, 26 May 2026 22:53:43 +0200 Subject: [PATCH] Guard rank feedback with Scattermind non-goals --- README.md | 2 +- scripts/check-rank-feedback.mjs | 37 ++++++++++++++++- server.js | 71 ++++++++++++++++++++++++++++----- 3 files changed, 98 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 4bcaf77..b1d1955 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ Ranker's continuation job is narrow: `POST /api/rank-feedback` accepts a `prioritix-feature-set-v1`-style payload from Scattermind and returns ranked items plus `buildOrder.doFirst / validateNext / defer / park`. It accepts candidate arrays as `features`, `actions`, `nextMoves`, or `candidates` either at the top level or under `featureSet`, and it can consume a nested `conceptMap.nextActions / nextMoves` artifact directly, so Scattermind can hand off Concept Map next actions without renaming them into fake software features. It also returns a `handoff` object (`rank-feedback-result-v1`) with source provenance, item trace rows, and contract warnings for missing artifact IDs, source sections, original prompt provenance, or evidence on active items. Keep this contract action-first; do not use it as a reason to add generic dashboard, auth, billing, or workspace layers before the bridge has proof. -Candidate items may include optional 1–10 `rankerHints` (`value`, `effort`, `confidence`, `urgency`, `revenue`, `novelty`, `risk`). Ranker blends those explicit Scattermind hints with text heuristics; `effort` is inverted into feasibility. Hints improve the defended order, but `recommendedLane: "defer"` or `"park"` remains a safety rail so dashboard-swamp items do not get promoted by flashy wording. For clean bridge handoff, Scattermind should send `sourceSection` and `evidenceNeeded` on each active next move. +Candidate items may include optional 1–10 `rankerHints` (`value`, `effort`, `confidence`, `urgency`, `revenue`, `novelty`, `risk`). Ranker blends those explicit Scattermind hints with text heuristics; `effort` is inverted into feasibility. Hints improve the defended order, but `recommendedLane: "defer"` or `"park"` remains a safety rail so dashboard-swamp items do not get promoted by flashy wording. For clean bridge handoff, Scattermind should send `sourceSection` and `evidenceNeeded` on each active next move. Scattermind can also send `targetAudience`, `constraints`, `assumptions`, and `nonGoals` / `avoid` at the top level, in `featureSet`, or inside `conceptMap.context`; Ranker returns that decision context in `input.decisionContext` and `handoff.decisionContext`, and penalizes candidates that conflict with source non-goals (for example saved workspaces/auth/billing before the continuation proof). Recommended payload shape: diff --git a/scripts/check-rank-feedback.mjs b/scripts/check-rank-feedback.mjs index 2ae0c67..7943c78 100644 --- a/scripts/check-rank-feedback.mjs +++ b/scripts/check-rank-feedback.mjs @@ -169,7 +169,42 @@ try { assert.equal(nestedConcept.ranked.find(item => item.id === 'lesson-library').lane.id, 'park'); assert.deepEqual(nestedConcept.handoff.warnings, []); - console.log(JSON.stringify({ ok: true, top: data.ranked[0].id, hintedTop: hinted.ranked[0].id, actionTop: actions.ranked[0].id, nestedConceptTop: nestedConcept.ranked[0].id, provenance: data.input.provenance, buildOrder: data.buildOrder }, null, 2)); + const nonGoalResponse = await fetch(`${base}/api/rank-feedback`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + sourceName: 'Scattermind', + artifactId: 'concept_map_non_goals', + originalPrompt: 'I clarified a tiny service idea and need the first build order.', + idea: 'Ranker should respect Concept Map non-goals when defending the build order.', + mode: 'mvp', + conceptMap: { + snapshotTitle: 'Non-goal guarded continuation', + context: { + targetAudience: 'Tired non-AI-native solo operator', + constraints: ['No account before first value', 'Manual proof is acceptable'], + nonGoals: ['Avoid saved workspaces', 'No auth dashboard', 'No billing layer before proof'], + }, + nextActions: [ + { id: 'workspace-autopilot', action: 'Saved workspace autopilot', why: 'Build accounts, auth dashboard, saved workspaces, and team sync.', evidence: 'None yet', suggestedLane: 'do-first', rankerHints: { value: 10, effort: 2, confidence: 9, urgency: 9, risk: 2 } }, + { id: 'manual-next-move', action: 'Manual next-move build order preview', why: 'Turn one source artifact into a defended first action and rank-ready build order without accounts.', evidence: 'Can 3 users act on the first move?', validationSteps: ['Create one static brief'], suggestedLane: 'do-first', rankerHints: { value: 9, effort: 1, confidence: 9, urgency: 9, risk: 1 } }, + { id: 'copy-brief', action: 'Copyable build-order brief', why: 'Give the user a plain artifact they can paste into notes.', evidence: 'Does copy/paste preserve the next step?', suggestedLane: 'validate-next' }, + ], + }, + }), + }); + assert.equal(nonGoalResponse.status, 200); + const nonGoal = await nonGoalResponse.json(); + assert.equal(nonGoal.input.decisionContext.targetAudience, 'Tired non-AI-native solo operator'); + assert.deepEqual(nonGoal.input.decisionContext.nonGoals, ['Avoid saved workspaces', 'No auth dashboard', 'No billing layer before proof']); + assert.equal(nonGoal.ranked[0].id, 'manual-next-move', 'non-goal conflicts should beat flashy positive hints'); + const workspace = nonGoal.ranked.find(item => item.id === 'workspace-autopilot'); + assert.ok(workspace.metrics.nonGoalConflicts.length >= 2); + assert.match(workspace.concern, /Source context says not to do this yet/); + assert.deepEqual(nonGoal.handoff.itemTrace.find(item => item.id === 'workspace-autopilot').nonGoalConflicts, workspace.metrics.nonGoalConflicts); + assert.ok(!nonGoal.buildOrder.doFirst.includes('workspace-autopilot')); + + console.log(JSON.stringify({ ok: true, top: data.ranked[0].id, hintedTop: hinted.ranked[0].id, actionTop: actions.ranked[0].id, nestedConceptTop: nestedConcept.ranked[0].id, nonGoalTop: nonGoal.ranked[0].id, provenance: data.input.provenance, buildOrder: data.buildOrder }, null, 2)); } finally { server.kill('SIGTERM'); } diff --git a/server.js b/server.js index 2d6f171..f61b870 100644 --- a/server.js +++ b/server.js @@ -145,6 +145,18 @@ function cleanTextList(value, maxItems = 6, maxText = 180) { return list.map(item => cleanText(item, maxText)).filter(Boolean).slice(0, maxItems); } +function cleanFlexibleTextList(value, maxItems = 8, maxText = 180) { + if (Array.isArray(value)) return value.map(item => cleanText(item, maxText)).filter(Boolean).slice(0, maxItems); + const text = cleanMultiline(value || '', maxItems * maxText); + if (!text) return []; + return text + .split(/\n|;|\|/) + .map(item => item.replace(/^\s*[-*•\d.)]+\s*/, '').trim()) + .filter(Boolean) + .map(item => cleanText(item, maxText)) + .slice(0, maxItems); +} + function cleanMetricHints(item = {}) { const raw = { ...(item.factors && typeof item.factors === 'object' ? item.factors : {}), @@ -410,6 +422,32 @@ function cleanProvenance(input = {}) { }; } +function cleanDecisionContext(input = {}) { + const featureSet = objectFrom(input.featureSet); + const artifact = objectFrom(input.artifact || featureSet.artifact); + const conceptMap = objectFrom(input.conceptMap || featureSet.conceptMap || artifact.conceptMap); + const sourceContext = objectFrom(input.decisionContext || featureSet.decisionContext || artifact.decisionContext || conceptMap.decisionContext || conceptMap.context); + return { + targetAudience: cleanText(input.targetAudience || featureSet.targetAudience || sourceContext.targetAudience || conceptMap.targetAudience || '', 180), + constraints: cleanFlexibleTextList(input.constraints || featureSet.constraints || sourceContext.constraints || conceptMap.constraints, 8, 180), + nonGoals: cleanFlexibleTextList(input.nonGoals || input.avoid || featureSet.nonGoals || featureSet.avoid || sourceContext.nonGoals || sourceContext.avoid || conceptMap.nonGoals || conceptMap.avoid, 8, 180), + assumptions: cleanFlexibleTextList(input.assumptions || featureSet.assumptions || sourceContext.assumptions || conceptMap.assumptions, 6, 180), + }; +} + +function meaningfulTokens(text = '') { + const stop = new Set(['the', 'and', 'with', 'from', 'that', 'this', 'into', 'before', 'after', 'user', 'users', 'idea', 'ideas', 'build', 'first', 'avoid', 'without', 'more', 'full', 'proof', 'value', 'layer']); + return [...new Set(String(text).toLowerCase().match(/[a-z0-9][a-z0-9-]{3,}/g) || [])].filter(token => !stop.has(token)).slice(0, 8); +} + +function nonGoalConflicts(optionText, decisionContext = {}) { + const lower = String(optionText || '').toLowerCase(); + return (decisionContext.nonGoals || []).filter(nonGoal => { + const tokens = meaningfulTokens(nonGoal); + return tokens.length > 0 && tokens.some(token => lower.includes(token)); + }); +} + function normalizeFeatureOption(item, index, fallbackId = 'feature', defaultSourceSection = '') { const title = cleanText(item?.title || item?.name || item?.action || '', 140); const proofSteps = cleanTextList(item?.proofSteps || item?.proof || item?.validationSteps, 5, 180); @@ -470,7 +508,7 @@ function optionsFromBody(body = {}) { return parseOptionsFromText(body.optionsText || featureSet.optionsText || conceptMap.optionsText || ''); } -function scoreOption(option, mode, context = '') { +function scoreOption(option, mode, context = '', decisionContext = {}) { const factors = option.factors || {}; const text = `${option.title} ${option.description} ${factors.userValue || ''} ${factors.evidenceNeeded || ''} ${factors.risk || ''} ${(factors.proofSteps || []).join(' ')} ${context}`; const effortHits = hits(text, wordSets.effort); @@ -482,16 +520,18 @@ function scoreOption(option, mode, context = '') { const dependencyPenalty = Math.min(2.2, (factors.dependencies || []).length * 0.45); const laneHint = factors.recommendedLane || ''; const normalizedLaneHint = normalizeLaneHint(laneHint); + const conflicts = nonGoalConflicts(`${option.title} ${option.description}`, decisionContext); + const nonGoalPenalty = Math.min(14, conflicts.length * 7); const laneBoost = /do|first|now|build/.test(laneHint) ? 0.55 : /validate|test|proof/.test(laneHint) ? 0.25 : /defer|park|cut/.test(laneHint) ? -0.75 : 0; const lanePenalty = normalizedLaneHint === 'park' ? 18 : normalizedLaneHint === 'defer' ? 9 : 0; const heuristicMetrics = { value: Math.min(10, 4.8 + hits(text, wordSets.value) * 0.9 + coreLoopHits * 0.8 + bridgeHits * 0.75 + proofHits * 0.2 + hits(context, wordSets.value) * 0.15), - feasibility: Math.max(1, Math.min(10, 8.2 - effortHits * 0.8 - riskHits * 0.35 - dependencyPenalty + Math.min(1.1, coreLoopHits * 0.28 + bridgeHits * 0.18 + proofHits * 0.12))), + feasibility: Math.max(1, Math.min(10, 8.2 - effortHits * 0.8 - riskHits * 0.35 - dependencyPenalty - conflicts.length * 1.1 + Math.min(1.1, coreLoopHits * 0.28 + bridgeHits * 0.18 + proofHits * 0.12))), confidence: Math.max(1, Math.min(10, 5.8 + coreLoopHits * 0.35 + bridgeHits * 0.28 + proofHits * 0.32 + hits(text, ['manual', 'existing', 'already', 'simple', 'clear', 'known']) * 0.7 - hits(text, ['maybe', 'unknown', 'new market', 'all users']) * 0.9)), urgency: Math.min(10, 4.9 + hits(text, wordSets.urgency) * 0.9), revenue: Math.min(10, 3.8 + hits(text, wordSets.revenue) * 1.05), novelty: Math.min(10, 4.1 + hits(text, wordSets.novelty) * 0.95), - risk: Math.min(10, 2.5 + riskHits * 1.1 + Math.max(0, effortHits - 2) * 0.45 + swampHits * 0.2 + dependencyPenalty), + risk: Math.min(10, 2.5 + riskHits * 1.1 + Math.max(0, effortHits - 2) * 0.45 + swampHits * 0.2 + dependencyPenalty + conflicts.length * 1.25), }; const hinted = factors.metricHints || {}; const hintedFeasibility = Number.isFinite(hinted.effort) ? 11 - hinted.effort : undefined; @@ -507,8 +547,8 @@ function scoreOption(option, mode, context = '') { const weights = mode.weights; const weighted = Object.entries(weights).reduce((sum, [key, weight]) => sum + metrics[key] * weight, 0); const possible = Object.entries(weights).reduce((sum, [_key, weight]) => sum + Math.abs(weight) * 10, 0); - const score = Math.max(0, Math.min(100, Math.round(((weighted + laneBoost) + Math.abs(Math.min(0, weights.risk || 0)) * 10) / possible * 100) - lanePenalty)); - return { ...metrics, score }; + const score = Math.max(0, Math.min(100, Math.round(((weighted + laneBoost) + Math.abs(Math.min(0, weights.risk || 0)) * 10) / possible * 100) - lanePenalty - nonGoalPenalty)); + return { ...metrics, score, nonGoalConflicts: conflicts }; } function normalizeLaneHint(value = '') { @@ -537,6 +577,7 @@ function laneFor(option, rankIndex, total) { function reasonFor(option) { const m = option.metrics; + if (m.nonGoalConflicts?.length) return `it conflicts with the source non-goal “${m.nonGoalConflicts[0]}”, so it should not lead the build order`; if (option.lane?.id === 'do' && /snapshot|concept map|feature set|build order|rank/i.test(`${option.title} ${option.description}`)) return 'it strengthens the Scattermind → Ranker bridge instead of inventing a generic workspace'; if (option.factors?.evidenceNeeded && m.confidence >= 6.4) return 'it names the evidence needed, so the next move can be tested instead of guessed'; if (m.feasibility >= 7.2 && m.value >= 6.2) return 'it has high enough value with low enough delivery drag to create fast signal'; @@ -548,6 +589,7 @@ function reasonFor(option) { function concernFor(option) { const m = option.metrics; + if (m.nonGoalConflicts?.length) return `Source context says not to do this yet: ${m.nonGoalConflicts.join('; ')}.`; if ((option.factors?.dependencies || []).length >= 3) return 'Too many prerequisites. Split the proof slice before treating this as build-ready.'; if (m.risk >= 6.5) return 'The hidden risk is pretending this is ready to build before the core assumption is proven.'; if (m.feasibility <= 4.5) return 'The likely trap is scope creep: this may need too much machinery for an MVP.'; @@ -597,7 +639,7 @@ function createDecisionBrief({ idea, context, mode, ranked, provenance }) { }; } -function createHandoffContract({ ranked, provenance }) { +function createHandoffContract({ ranked, provenance, decisionContext }) { const warnings = []; if (!provenance?.artifactId) warnings.push('missing source artifact id'); if (!provenance?.originalPrompt) warnings.push('missing original prompt provenance'); @@ -605,6 +647,7 @@ function createHandoffContract({ ranked, provenance }) { const itemTrace = ranked.map(item => { if (!item.provenance?.sourceSection) warnings.push(`missing source section for ${item.id}`); if (!item.factors?.evidenceNeeded && ['do', 'test'].includes(item.lane?.id)) warnings.push(`missing evidence needed for active item ${item.id}`); + if (item.metrics?.nonGoalConflicts?.length && ['do', 'test'].includes(item.lane?.id)) warnings.push(`active item ${item.id} conflicts with source non-goals: ${item.metrics.nonGoalConflicts.join('; ')}`); return { id: item.id, title: item.title, @@ -612,6 +655,7 @@ function createHandoffContract({ ranked, provenance }) { sourceSection: item.provenance?.sourceSection || '', sourceId: item.provenance?.sourceId || '', evidenceNeeded: item.factors?.evidenceNeeded || '', + nonGoalConflicts: item.metrics?.nonGoalConflicts || [], }; }); @@ -625,6 +669,12 @@ function createHandoffContract({ ranked, provenance }) { conceptMapId: provenance?.conceptMapId || '', hasOriginalPrompt: Boolean(provenance?.originalPrompt), }, + decisionContext: { + targetAudience: decisionContext?.targetAudience || '', + constraints: decisionContext?.constraints || [], + nonGoals: decisionContext?.nonGoals || [], + assumptions: decisionContext?.assumptions || [], + }, itemTrace, warnings: [...new Set(warnings)], }; @@ -636,10 +686,11 @@ app.post('/api/rank-feedback', (req, res) => { const modeId = cleanText(req.body?.mode || 'progress', 40); const mode = judgementModes[modeId] || judgementModes.progress; const provenance = cleanProvenance(req.body || {}); + const decisionContext = cleanDecisionContext(req.body || {}); let options = optionsFromBody(req.body || {}); if (options.length < 2) return res.status(400).json({ error: 'Paste at least two options, features, ideas, or next moves to rank.' }); - const bridgeContext = `${idea}\n${context}\n${provenance.snapshotTitle}\n${provenance.schema}`; - options = options.map(option => ({ ...option, metrics: scoreOption(option, mode, bridgeContext) })) + const bridgeContext = `${idea}\n${context}\n${provenance.snapshotTitle}\n${provenance.schema}\n${decisionContext.targetAudience}\n${decisionContext.constraints.join('\n')}\n${decisionContext.assumptions.join('\n')}`; + options = options.map(option => ({ ...option, metrics: scoreOption(option, mode, bridgeContext, decisionContext) })) .sort((a, b) => b.metrics.score - a.metrics.score || b.metrics.value - a.metrics.value || a.metrics.risk - b.metrics.risk) .map((option, index, arr) => { const lane = laneFor(option, index, arr.length); @@ -651,11 +702,11 @@ app.post('/api/rank-feedback', (req, res) => { }; }); const brief = createDecisionBrief({ idea, context, mode, ranked: options, provenance }); - const handoff = createHandoffContract({ ranked: options, provenance }); + const handoff = createHandoffContract({ ranked: options, provenance, decisionContext }); res.json({ ok: true, mode: { id: modeId in judgementModes ? modeId : 'progress', label: mode.label }, - input: { idea, context, optionCount: options.length, provenance }, + input: { idea, context, optionCount: options.length, provenance, decisionContext }, ranked: options, brief, handoff,