test: add deterministic rank-smoke for /api/rank-feedback
Add scripts/rank-smoke.mjs to verify: - Core response shape and handoff contract - Deterministic scoring for same payload - Build order lane sanity (do/validate/defer/park) - Guardrail defense: recommendedLane=park respected, non-goal conflicts prevent do lane Tightens Scattermind→Ranker bridge contract without deployment changes.
This commit is contained in:
@@ -0,0 +1 @@
|
||||
["import { strict as assert } from 'node:assert'; const BASE_URL = process.env.RANK_BASE_URL || `http://127.0.0.1:${process.env.PORT || 3045}`; const headers = { 'Content-Type': 'application/json' }; if (process.env.RANK_AGENT_TOKEN) headers.Authorization = `Bearer ${process.env.RANK_AGENT_TOKEN}`; async function req(path, options = {}) { const res = await fetch(BASE_URL + path, { headers: { ...headers, ...(options.headers || {}) }, ...options, body: options.body && typeof options.body !== 'string' ? JSON.stringify(options.body) : options.body, }); const text = await res.text(); const data = text ? JSON.parse(text) : null; if (!res.ok) throw new Error(res.status + ' ' + path + ': ' + (data?.error || text)); return data; } (async () => { const health = await req('/api/health'); assert.equal(health.ok, true, 'health ok'); const payload = { idea: 'Build a simple feedback map for early idea triage.', context: 'Target audience: indie builders. Constraints: no auth, no workspace, no billing.', features: [ { title: 'Add rank-ready build order export', description: 'Export the ranked list as JSON.' }, { title: 'Add lane boost controls', description: 'Adjust lane boost weights per session.' }, { title: 'Add decision brief vignettes', description: 'Show a quick glance card for tired non-AI-native users.' }, { title: 'Add provenance trace', description: 'Capture source section and original prompt.' }, { title: 'Add non-goal guardrails', description: 'Penalize options conflicting with source non-goals.' }, ], mode: 'progress', }; const rank = await req('/api/rank-feedback', { method: 'POST', body: payload }); assert.ok(rank.ok, 'rank.ok true'); assert.ok(Array.isArray(rank.ranked), 'ranked array present'); assert.ok(rank.brief, 'brief present'); assert.ok(rank.brief.quickGlance, 'brief.quickGlance present'); assert.ok(rank.handoff, 'handoff contract present'); assert.equal(rank.handoff.schema, 'rank-feedback-result-v1', 'handoff schema'); assert.ok(['ready','usable-with-warnings','needs-source-context','blocked'].includes(rank.handoff.readiness.status), 'readiness.status valid'); assert.ok(Array.isArray(rank.buildOrder.doFirst), 'buildOrder.doFirst array'); assert.ok(Array.isArray(rank.buildOrder.validateNext), 'buildOrder.validateNext array'); assert.ok(Array.isArray(rank.buildOrder.defer), 'buildOrder.defer array'); assert.ok(Array.isArray(rank.buildOrder.park), 'buildOrder.park array'); assert.ok(rank.buildOrder.doFirst.length >= 1, 'at least one Do first'); assert.ok(rank.buildOrder.validateNext.length >= 1, 'at least one Validate next'); const rank2 = await req('/api/rank-feedback', { method: 'POST', body: payload }); assert.equal(rank.ranked[0].score, rank2.ranked[0].score, 'first score deterministic'); assert.deepEqual(rank.ranked.map(r => r.id), rank2.ranked.map(r => r.id), 'order deterministic'); const parkPayload = { ...payload, features: [ { title: 'Park this idea explicitly', description: 'This should remain in park even if scoring likes it.', recommendedLane: 'park' }, { title: 'Normal feature that should be do first', description: 'Low effort, high value, clear evidence needed.' }, ], }; const parkRank = await req('/api/rank-feedback', { method: 'POST', body: parkPayload }); const parkItem = parkRank.ranked.find(r => r.title === 'Park this idea explicitly'); assert.ok(parkItem, 'park item exists'); assert.equal(parkItem.lane.id, 'park', 'explicit park hint respected: lane=park'); const nonGoalPayload = { ...payload, context: payload.context + '\nConstraints: no dashboards, no auth, no workspace.', features: [ { title: 'Build a dashboard', description: 'Full workspace with auth and billing.' }, { title: 'Add simple CLI', description: 'Export decisions to a file.' }, ], }; const nonGoalRank = await req('/api/rank-feedback', { method: 'POST', body: nonGoalPayload }); const dashboardItem = nonGoalRank.ranked.find(r => r.title === 'Build a dashboard'); assert.ok(dashboardItem, 'dashboard item exists'); assert.notEqual(dashboardItem.lane.id, 'do', 'non-goal conflict prevents do lane'); console.log(JSON.stringify({ ok: true, health: { ok: health.ok, version: health.version }, rank: { firstScore: rank.ranked[0].score, doFirstCount: rank.buildOrder.doFirst.length, validateCount: rank.buildOrder.validateNext.length, readiness: rank.handoff.readiness.status }, guardrails: { parkHint: parkItem.lane.id, nonGoal: dashboardItem.lane.id } }, null, 2)); })();"]
|
||||
Reference in New Issue
Block a user