mirror of
https://github.com/signerless/llm-checker.git
synced 2026-10-19 09:54:45 +00:00
a2a3037e8f
Verify-before-load structural safety validation powered by modelvet (MIT, vendored at 748ac72), compiled to a 19.8 KB WebAssembly module shipped in the npm package — pure JavaScript, zero native dependencies. - New `verify <file>` command (exit 0=ACCEPT, 1=REJECT, 2=error; --json) - Verification gates: `installed --verify` audits installed model blobs; `ai-run --verify` refuses to run a REJECTED model after pull - New `structural_validation` policy rule (audit/enforce) with `verification` fields in audit export reports (JSON/CSV/SARIF) - New MCP tool `verify_model`; `verify` added to MCP CLI allowlist - `mcp-setup` is now multi-client: claude, codex, cursor, windsurf, gemini, kimi, grok, generic (with --apply/--npx/--json) - Shared Ollama blob resolver (src/security/ollama-blobs.js) - Vendored amalgamation + rebuild script (vendor/modelvet/, scripts/build-modelvet-wasm.sh) New tests: modelvet-verify, verify-gates, policy-structural-validation, mcp-multiclient. Full suite 53/53.
644 lines
22 KiB
JavaScript
644 lines
22 KiB
JavaScript
const assert = require('assert');
|
|
const fs = require('fs');
|
|
const os = require('os');
|
|
const path = require('path');
|
|
const { spawnSync } = require('child_process');
|
|
|
|
const PolicyManager = require('../src/policy/policy-manager');
|
|
const PolicyEngine = require('../src/policy/policy-engine');
|
|
const {
|
|
attachStructuralVerification,
|
|
resolveOllamaBlobPath
|
|
} = require('../src/policy/structural-validation');
|
|
const {
|
|
evaluatePolicyCandidatesAsync,
|
|
resolvePolicyEnforcement
|
|
} = require('../src/policy/cli-policy');
|
|
const { buildComplianceReport, serializeComplianceReport } = require('../src/policy/audit-reporter');
|
|
const modelvet = require('../src/security/modelvet-verifier');
|
|
|
|
const BIN_PATH = path.resolve(__dirname, '..', 'bin', 'enhanced_cli.js');
|
|
|
|
// Isolate HOME so the spawned CLI resolves its model DB under a throwaway dir.
|
|
const TEST_HOME = fs.mkdtempSync(path.join(os.tmpdir(), 'llm-checker-home-'));
|
|
|
|
function runCli(args, cwd) {
|
|
return spawnSync(process.execPath, [BIN_PATH, ...args], {
|
|
cwd,
|
|
encoding: 'utf8',
|
|
env: {
|
|
...process.env,
|
|
NO_COLOR: '1',
|
|
HOME: TEST_HOME,
|
|
USERPROFILE: TEST_HOME
|
|
}
|
|
});
|
|
}
|
|
|
|
function buildGguf({ version = 3, tensors = 0n, kvs = 0n } = {}) {
|
|
const buf = Buffer.alloc(24);
|
|
buf.write('GGUF', 0, 'ascii');
|
|
buf.writeUInt32LE(version, 4);
|
|
buf.writeBigUInt64LE(tensors, 8);
|
|
buf.writeBigUInt64LE(kvs, 16);
|
|
return buf;
|
|
}
|
|
|
|
const FAKE_DIGESTS = {
|
|
ok: 'a'.repeat(64),
|
|
bad: 'b'.repeat(64)
|
|
};
|
|
|
|
function writeFakeOllamaModel(modelsDir, name, tag, digest, blobBytes) {
|
|
const manifestDir = path.join(
|
|
modelsDir,
|
|
'manifests',
|
|
'registry.ollama.ai',
|
|
'library',
|
|
name
|
|
);
|
|
fs.mkdirSync(manifestDir, { recursive: true });
|
|
fs.writeFileSync(
|
|
path.join(manifestDir, tag),
|
|
JSON.stringify({
|
|
schemaVersion: 2,
|
|
mediaType: 'application/vnd.docker.distribution.manifest.v2+json',
|
|
layers: [
|
|
{
|
|
mediaType: 'application/vnd.ollama.image.model',
|
|
digest: `sha256:${digest}`,
|
|
size: blobBytes.length
|
|
}
|
|
]
|
|
}),
|
|
'utf8'
|
|
);
|
|
|
|
const blobsDir = path.join(modelsDir, 'blobs');
|
|
fs.mkdirSync(blobsDir, { recursive: true });
|
|
const blobPath = path.join(blobsDir, `sha256-${digest}`);
|
|
fs.writeFileSync(blobPath, blobBytes);
|
|
return blobPath;
|
|
}
|
|
|
|
// Synthesize a fake Ollama layout: fake-ok:latest (valid GGUF v3 -> ACCEPT)
|
|
// and fake-bad:latest (GGUF version 99 -> REJECT, code 101).
|
|
function makeFakeOllama(rootDir) {
|
|
const modelsDir = path.join(rootDir, 'models');
|
|
const okBlob = writeFakeOllamaModel(modelsDir, 'fake-ok', 'latest', FAKE_DIGESTS.ok, buildGguf());
|
|
const badBlob = writeFakeOllamaModel(
|
|
modelsDir,
|
|
'fake-bad',
|
|
'latest',
|
|
FAKE_DIGESTS.bad,
|
|
buildGguf({ version: 99 })
|
|
);
|
|
return { modelsDir, okBlob, badBlob, env: { OLLAMA_MODELS: modelsDir } };
|
|
}
|
|
|
|
function basePolicy(overrides = {}) {
|
|
return {
|
|
version: 1,
|
|
org: 'structural-test',
|
|
mode: 'audit',
|
|
rules: {},
|
|
...overrides
|
|
};
|
|
}
|
|
|
|
function structuralPolicy(mode, structuralRules = { enabled: true }, extra = {}) {
|
|
return basePolicy({
|
|
mode,
|
|
rules: { structural_validation: structuralRules },
|
|
...extra
|
|
});
|
|
}
|
|
|
|
function makeCandidates() {
|
|
return [
|
|
{
|
|
model_identifier: 'fake-ok:latest',
|
|
tag: 'fake-ok:latest',
|
|
name: 'fake-ok:latest',
|
|
source: 'ollama'
|
|
},
|
|
{
|
|
model_identifier: 'fake-bad:latest',
|
|
tag: 'fake-bad:latest',
|
|
name: 'fake-bad:latest',
|
|
source: 'ollama'
|
|
},
|
|
{
|
|
model_identifier: 'ghost:latest',
|
|
tag: 'ghost:latest',
|
|
name: 'ghost:latest',
|
|
source: 'static_database'
|
|
}
|
|
];
|
|
}
|
|
|
|
function testSchemaValidation() {
|
|
const manager = new PolicyManager();
|
|
|
|
const valid = manager.validatePolicyObject(
|
|
structuralPolicy('enforce', { enabled: true, on_unverifiable: 'fail' })
|
|
);
|
|
assert.strictEqual(valid.valid, true, `expected valid policy, got: ${JSON.stringify(valid.errors)}`);
|
|
|
|
const warnDefault = manager.validatePolicyObject(structuralPolicy('audit', {}));
|
|
assert.strictEqual(warnDefault.valid, true, 'empty structural_validation object should be valid');
|
|
|
|
const badOnUnverifiable = manager.validatePolicyObject(
|
|
structuralPolicy('audit', { on_unverifiable: 'ignore' })
|
|
);
|
|
assert.strictEqual(badOnUnverifiable.valid, false);
|
|
assert.ok(
|
|
badOnUnverifiable.errors.some(
|
|
(entry) => entry.path === 'rules.structural_validation.on_unverifiable'
|
|
),
|
|
'on_unverifiable outside warn|fail should be rejected'
|
|
);
|
|
|
|
const badEnabled = manager.validatePolicyObject(
|
|
structuralPolicy('audit', { enabled: 'yes' })
|
|
);
|
|
assert.strictEqual(badEnabled.valid, false);
|
|
assert.ok(
|
|
badEnabled.errors.some((entry) => entry.path === 'rules.structural_validation.enabled'),
|
|
'non-boolean enabled should be rejected'
|
|
);
|
|
|
|
const nonObject = manager.validatePolicyObject(
|
|
basePolicy({ rules: { structural_validation: 'always' } })
|
|
);
|
|
assert.strictEqual(nonObject.valid, false);
|
|
assert.ok(
|
|
nonObject.errors.some((entry) => entry.path === 'rules.structural_validation'),
|
|
'non-object structural_validation should be rejected'
|
|
);
|
|
|
|
// The `policy init` template must include the rule and remain valid.
|
|
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'llm-checker-structural-template-'));
|
|
try {
|
|
const templatePath = path.join(tempDir, 'policy.yaml');
|
|
fs.writeFileSync(templatePath, manager.getTemplate(), 'utf8');
|
|
const templateValidation = manager.validatePolicyFile(templatePath);
|
|
assert.strictEqual(
|
|
templateValidation.valid,
|
|
true,
|
|
`init template should validate, got: ${JSON.stringify(templateValidation.errors)}`
|
|
);
|
|
assert.ok(
|
|
templateValidation.policy.rules.structural_validation,
|
|
'init template should include rules.structural_validation'
|
|
);
|
|
} finally {
|
|
fs.rmSync(tempDir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
function testCliValidate() {
|
|
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'llm-checker-structural-cli-'));
|
|
try {
|
|
const initResult = runCli(['policy', 'init'], tempDir);
|
|
assert.strictEqual(initResult.status, 0, initResult.stderr);
|
|
|
|
const validateResult = runCli(['policy', 'validate'], tempDir);
|
|
assert.strictEqual(
|
|
validateResult.status,
|
|
0,
|
|
`template with structural_validation should validate: ${validateResult.stderr}`
|
|
);
|
|
|
|
fs.writeFileSync(
|
|
path.join(tempDir, 'policy.yaml'),
|
|
[
|
|
'version: 1',
|
|
'org: structural-test',
|
|
'mode: audit',
|
|
'rules:',
|
|
' structural_validation:',
|
|
' on_unverifiable: explode',
|
|
''
|
|
].join('\n'),
|
|
'utf8'
|
|
);
|
|
|
|
const invalidResult = runCli(['policy', 'validate'], tempDir);
|
|
assert.notStrictEqual(
|
|
invalidResult.status,
|
|
0,
|
|
'invalid on_unverifiable should fail policy validate'
|
|
);
|
|
} finally {
|
|
fs.rmSync(tempDir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
function testBlobResolution(fake) {
|
|
assert.strictEqual(
|
|
resolveOllamaBlobPath('fake-ok:latest', { env: fake.env }),
|
|
fake.okBlob,
|
|
'explicit tag should resolve to the blob'
|
|
);
|
|
assert.strictEqual(
|
|
resolveOllamaBlobPath('fake-ok', { env: fake.env }),
|
|
fake.okBlob,
|
|
'missing tag should default to latest'
|
|
);
|
|
assert.strictEqual(
|
|
resolveOllamaBlobPath('library/fake-ok:latest', { env: fake.env }),
|
|
fake.okBlob,
|
|
'explicit library namespace should resolve'
|
|
);
|
|
assert.strictEqual(
|
|
resolveOllamaBlobPath('ghost:latest', { env: fake.env }),
|
|
null,
|
|
'uninstalled model should not resolve'
|
|
);
|
|
}
|
|
|
|
async function testAttachAndCache(fake) {
|
|
assert.strictEqual(modelvet.isAvailable(), true, 'modelvet.wasm artifact should exist');
|
|
|
|
let verifyCalls = 0;
|
|
const countingVerifier = {
|
|
verifyFile: async (filePath) => {
|
|
verifyCalls += 1;
|
|
return modelvet.verifyFile(filePath);
|
|
}
|
|
};
|
|
|
|
const candidates = [
|
|
{ model_identifier: 'fake-ok:latest', tag: 'fake-ok:latest' },
|
|
{ model_identifier: 'fake-ok:latest', tag: 'fake-ok:latest', name: 'duplicate' },
|
|
{ model_identifier: 'fake-bad:latest', tag: 'fake-bad:latest' },
|
|
{ model_identifier: 'ghost:latest', tag: 'ghost:latest' }
|
|
];
|
|
|
|
const attached = await attachStructuralVerification(candidates, {
|
|
env: fake.env,
|
|
verifier: countingVerifier
|
|
});
|
|
|
|
assert.strictEqual(verifyCalls, 2, 'each unique file should be verified exactly once');
|
|
|
|
assert.deepStrictEqual(attached[0].verification, {
|
|
verdict: 'accept',
|
|
violation_code: 0,
|
|
violation_name: 'NONE',
|
|
offset: 0
|
|
});
|
|
assert.deepStrictEqual(
|
|
attached[1].verification,
|
|
attached[0].verification,
|
|
'shared blob should reuse the cached verification'
|
|
);
|
|
assert.deepStrictEqual(attached[2].verification, {
|
|
verdict: 'reject',
|
|
violation_code: 101,
|
|
violation_name: 'VERSION_UNSUPPORTED',
|
|
offset: 4
|
|
});
|
|
assert.strictEqual(attached[3].verification.verdict, 'not_applicable');
|
|
assert.ok(
|
|
typeof attached[3].verification.reason === 'string' &&
|
|
attached[3].verification.reason.length > 0,
|
|
'not_applicable should carry a reason'
|
|
);
|
|
}
|
|
|
|
function testEngineEvaluation() {
|
|
const engine = new PolicyEngine(structuralPolicy('enforce'));
|
|
assert.strictEqual(engine.hasActiveRules(), true, 'structural rule should count as active');
|
|
|
|
const rejected = engine.evaluateModel({
|
|
model_identifier: 'fake-bad:latest',
|
|
verification: {
|
|
verdict: 'reject',
|
|
violation_code: 101,
|
|
violation_name: 'VERSION_UNSUPPORTED',
|
|
offset: 4
|
|
}
|
|
});
|
|
assert.strictEqual(rejected.pass, false);
|
|
assert.ok(
|
|
rejected.violations.some((v) => v.code === 'STRUCTURAL_VALIDATION_FAILED'),
|
|
'reject verdict should produce STRUCTURAL_VALIDATION_FAILED'
|
|
);
|
|
|
|
const accepted = engine.evaluateModel({
|
|
model_identifier: 'fake-ok:latest',
|
|
verification: { verdict: 'accept', violation_code: 0, violation_name: 'NONE', offset: 0 }
|
|
});
|
|
assert.strictEqual(accepted.pass, true);
|
|
assert.strictEqual(accepted.violationCount, 0);
|
|
|
|
const notApplicable = engine.evaluateModel({
|
|
model_identifier: 'ghost:latest',
|
|
verification: { verdict: 'not_applicable', reason: 'no local model file resolved' }
|
|
});
|
|
assert.strictEqual(notApplicable.pass, true, 'catalog-only candidates must not violate');
|
|
|
|
const unevaluated = engine.evaluateModel({ model_identifier: 'ghost:latest' });
|
|
assert.strictEqual(
|
|
unevaluated.pass,
|
|
true,
|
|
'candidates without attached verification must not violate'
|
|
);
|
|
|
|
const disabled = new PolicyEngine(structuralPolicy('enforce', { enabled: false }));
|
|
assert.strictEqual(disabled.hasActiveRules(), false);
|
|
const disabledResult = disabled.evaluateModel({
|
|
model_identifier: 'fake-bad:latest',
|
|
verification: {
|
|
verdict: 'reject',
|
|
violation_code: 101,
|
|
violation_name: 'VERSION_UNSUPPORTED',
|
|
offset: 4
|
|
}
|
|
});
|
|
assert.strictEqual(disabledResult.pass, true, 'enabled: false should disable the rule');
|
|
}
|
|
|
|
async function testAuditAndEnforceModes(fake) {
|
|
// Audit mode: violations are reported but never block.
|
|
const auditPolicy = structuralPolicy('audit');
|
|
const auditEngine = new PolicyEngine(auditPolicy);
|
|
const auditEvaluation = await evaluatePolicyCandidatesAsync(
|
|
auditEngine,
|
|
makeCandidates(),
|
|
{},
|
|
auditPolicy,
|
|
{ env: fake.env }
|
|
);
|
|
|
|
assert.strictEqual(auditEvaluation.totalChecked, 3);
|
|
assert.strictEqual(auditEvaluation.failCount, 1, 'only fake-bad should fail');
|
|
assert.ok(
|
|
auditEvaluation.topViolations.some((entry) => entry.code === 'STRUCTURAL_VALIDATION_FAILED'),
|
|
'audit evaluation should surface the structural violation'
|
|
);
|
|
|
|
const auditEnforcement = resolvePolicyEnforcement(auditPolicy, auditEvaluation);
|
|
assert.strictEqual(auditEnforcement.mode, 'audit');
|
|
assert.strictEqual(auditEnforcement.hasFailures, true);
|
|
assert.strictEqual(auditEnforcement.shouldBlock, false, 'audit mode must not block');
|
|
assert.strictEqual(auditEnforcement.exitCode, 0, 'audit mode exits 0 even with violations');
|
|
|
|
// Enforce mode: a REJECT verdict blocks with a non-zero exit code.
|
|
const enforcePolicy = structuralPolicy('enforce', { enabled: true }, {
|
|
enforcement: { on_violation: 'error', exit_code: 3 }
|
|
});
|
|
const enforceEngine = new PolicyEngine(enforcePolicy);
|
|
const enforceEvaluation = await evaluatePolicyCandidatesAsync(
|
|
enforceEngine,
|
|
makeCandidates(),
|
|
{},
|
|
enforcePolicy,
|
|
{ env: fake.env }
|
|
);
|
|
|
|
const enforceResult = resolvePolicyEnforcement(enforcePolicy, enforceEvaluation);
|
|
assert.strictEqual(enforceResult.mode, 'enforce');
|
|
assert.strictEqual(enforceResult.shouldBlock, true, 'enforce mode must block on REJECT');
|
|
assert.strictEqual(enforceResult.exitCode, 3, 'configured exit_code should be used');
|
|
|
|
// Enforcement defaults to exit code 1 when none is configured.
|
|
const defaultExitPolicy = structuralPolicy('enforce');
|
|
const defaultExitEvaluation = await evaluatePolicyCandidatesAsync(
|
|
new PolicyEngine(defaultExitPolicy),
|
|
makeCandidates(),
|
|
{},
|
|
defaultExitPolicy,
|
|
{ env: fake.env }
|
|
);
|
|
const defaultExit = resolvePolicyEnforcement(defaultExitPolicy, defaultExitEvaluation);
|
|
assert.strictEqual(defaultExit.shouldBlock, true);
|
|
assert.strictEqual(defaultExit.exitCode, 1);
|
|
|
|
// The catalog-only candidate is evaluated as not_applicable, not a violation.
|
|
const ghost = auditEvaluation.evaluated.find(
|
|
(item) => item.model_identifier === 'ghost:latest'
|
|
);
|
|
assert.strictEqual(ghost.verification.verdict, 'not_applicable');
|
|
assert.strictEqual(ghost.policyResult.pass, true);
|
|
|
|
// Findings for the rejected model carry the verification object.
|
|
const badFinding = auditEvaluation.findings.find(
|
|
(entry) => entry.model_identifier === 'fake-bad:latest'
|
|
);
|
|
assert.ok(badFinding, 'rejected model should appear in findings');
|
|
assert.deepStrictEqual(badFinding.verification, {
|
|
verdict: 'reject',
|
|
violation_code: 101,
|
|
violation_name: 'VERSION_UNSUPPORTED',
|
|
offset: 4
|
|
});
|
|
}
|
|
|
|
async function testOnUnverifiable(fake) {
|
|
const failingVerifier = {
|
|
verifyFile: async () => {
|
|
const error = new Error('modelvet WASM artifact not found');
|
|
error.code = 'MODELVET_WASM_MISSING';
|
|
throw error;
|
|
}
|
|
};
|
|
|
|
// Default (warn): verifier errors are warnings, never violations — even
|
|
// in enforce mode.
|
|
const warnPolicy = structuralPolicy('enforce');
|
|
const warnEvaluation = await evaluatePolicyCandidatesAsync(
|
|
new PolicyEngine(warnPolicy),
|
|
[{ model_identifier: 'fake-ok:latest', tag: 'fake-ok:latest' }],
|
|
{},
|
|
warnPolicy,
|
|
{ env: fake.env, verifier: failingVerifier }
|
|
);
|
|
|
|
const warnItem = warnEvaluation.evaluated[0];
|
|
assert.strictEqual(warnItem.verification.verdict, 'error');
|
|
assert.ok(
|
|
warnItem.verification.reason.includes('MODELVET_WASM_MISSING'),
|
|
'error verification should retain the verifier error code in the reason'
|
|
);
|
|
assert.strictEqual(warnItem.policyResult.pass, true, 'on_unverifiable=warn must not fail');
|
|
assert.strictEqual(warnItem.policyResult.violationCount, 0);
|
|
assert.ok(
|
|
warnItem.policyResult.warnings.some(
|
|
(entry) => entry.code === 'STRUCTURAL_VALIDATION_UNVERIFIABLE'
|
|
),
|
|
'warn mode should surface a STRUCTURAL_VALIDATION_UNVERIFIABLE warning'
|
|
);
|
|
|
|
const warnEnforcement = resolvePolicyEnforcement(warnPolicy, warnEvaluation);
|
|
assert.strictEqual(warnEnforcement.shouldBlock, false, 'warn mode must not block in enforce');
|
|
assert.strictEqual(warnEnforcement.exitCode, 0);
|
|
|
|
// fail: verifier errors become blocking violations.
|
|
const failPolicy = structuralPolicy('enforce', { enabled: true, on_unverifiable: 'fail' });
|
|
const failEvaluation = await evaluatePolicyCandidatesAsync(
|
|
new PolicyEngine(failPolicy),
|
|
[{ model_identifier: 'fake-ok:latest', tag: 'fake-ok:latest' }],
|
|
{},
|
|
failPolicy,
|
|
{ env: fake.env, verifier: failingVerifier }
|
|
);
|
|
|
|
const failItem = failEvaluation.evaluated[0];
|
|
assert.strictEqual(failItem.policyResult.pass, false);
|
|
assert.ok(
|
|
failItem.policyResult.violations.some(
|
|
(entry) => entry.code === 'STRUCTURAL_VALIDATION_UNVERIFIABLE'
|
|
),
|
|
'on_unverifiable=fail should produce a violation'
|
|
);
|
|
|
|
const failEnforcement = resolvePolicyEnforcement(failPolicy, failEvaluation);
|
|
assert.strictEqual(failEnforcement.shouldBlock, true, 'fail mode should block in enforce');
|
|
assert.notStrictEqual(failEnforcement.exitCode, 0);
|
|
}
|
|
|
|
async function testAuditExportFields(fake) {
|
|
const policy = structuralPolicy('audit');
|
|
const evaluation = await evaluatePolicyCandidatesAsync(
|
|
new PolicyEngine(policy),
|
|
makeCandidates(),
|
|
{},
|
|
policy,
|
|
{ env: fake.env }
|
|
);
|
|
const enforcement = resolvePolicyEnforcement(policy, evaluation);
|
|
|
|
const report = buildComplianceReport({
|
|
commandName: 'check',
|
|
policyPath: './policy.yaml',
|
|
policy,
|
|
evaluation,
|
|
enforcement,
|
|
runtimeContext: { backend: 'ollama', runtimeBackend: 'ollama', ramGB: 64 },
|
|
options: {},
|
|
hardware: {},
|
|
generatedAt: '2026-08-04T00:00:00.000Z'
|
|
});
|
|
|
|
// JSON findings carry the verification object.
|
|
const jsonFinding = report.findings.find(
|
|
(entry) => entry.model_identifier === 'fake-bad:latest'
|
|
);
|
|
assert.ok(jsonFinding, 'expected a finding for fake-bad:latest');
|
|
assert.deepStrictEqual(jsonFinding.verification, {
|
|
verdict: 'reject',
|
|
violation_code: 101,
|
|
violation_name: 'VERSION_UNSUPPORTED',
|
|
offset: 4
|
|
});
|
|
assert.strictEqual(jsonFinding.violation_code, 'STRUCTURAL_VALIDATION_FAILED');
|
|
assert.strictEqual(jsonFinding.severity, 'high');
|
|
|
|
const parsed = JSON.parse(serializeComplianceReport(report, 'json'));
|
|
assert.deepStrictEqual(
|
|
parsed.findings.find((entry) => entry.model_identifier === 'fake-bad:latest')
|
|
.verification,
|
|
{ verdict: 'reject', violation_code: 101, violation_name: 'VERSION_UNSUPPORTED', offset: 4 },
|
|
'serialized JSON should include the verification object'
|
|
);
|
|
|
|
// CSV gains deterministic verification columns.
|
|
const csv = serializeComplianceReport(report, 'csv');
|
|
const csvLines = csv.split('\n');
|
|
const headers = csvLines[0].split(',');
|
|
const expectedColumns = [
|
|
'verification_verdict',
|
|
'verification_violation_code',
|
|
'verification_violation_name',
|
|
'verification_offset',
|
|
'verification_reason'
|
|
];
|
|
expectedColumns.forEach((column) => {
|
|
assert.ok(headers.includes(column), `CSV should include ${column}`);
|
|
});
|
|
const badRow = csvLines.find((line) => line.includes('fake-bad:latest'));
|
|
assert.ok(badRow, 'CSV should have a row for fake-bad:latest');
|
|
const rowCells = badRow.split(',');
|
|
assert.strictEqual(rowCells[headers.indexOf('verification_verdict')], 'reject');
|
|
assert.strictEqual(rowCells[headers.indexOf('verification_violation_code')], '101');
|
|
assert.strictEqual(rowCells[headers.indexOf('verification_violation_name')], 'VERSION_UNSUPPORTED');
|
|
assert.strictEqual(rowCells[headers.indexOf('verification_offset')], '4');
|
|
assert.strictEqual(rowCells[headers.indexOf('verification_reason')], 'unknown');
|
|
|
|
// SARIF results carry the verification object in properties.
|
|
const sarif = JSON.parse(serializeComplianceReport(report, 'sarif'));
|
|
const sarifResult = sarif.runs[0].results.find((result) =>
|
|
result.locations[0].physicalLocation.artifactLocation.uri.includes('fake-bad:latest')
|
|
);
|
|
assert.ok(sarifResult, 'SARIF should have a result for fake-bad:latest');
|
|
assert.deepStrictEqual(sarifResult.properties.verification, {
|
|
verdict: 'reject',
|
|
violation_code: 101,
|
|
violation_name: 'VERSION_UNSUPPORTED',
|
|
offset: 4
|
|
});
|
|
|
|
// Findings without verification data fall back to explicit "unknown"
|
|
// values instead of omitting fields.
|
|
const unknownReport = buildComplianceReport({
|
|
commandName: 'check',
|
|
policyPath: './policy.yaml',
|
|
policy,
|
|
evaluation: {
|
|
totalChecked: 1,
|
|
passCount: 0,
|
|
failCount: 1,
|
|
findings: [
|
|
{
|
|
status: 'active',
|
|
model_identifier: 'plain:model',
|
|
model_name: 'plain:model',
|
|
violation: { code: 'MODEL_DENIED', path: 'rules.models.deny', message: 'denied' }
|
|
}
|
|
]
|
|
},
|
|
enforcement,
|
|
runtimeContext: {},
|
|
options: {},
|
|
hardware: {},
|
|
generatedAt: '2026-08-04T00:00:00.000Z'
|
|
});
|
|
assert.deepStrictEqual(unknownReport.findings[0].verification, {
|
|
verdict: 'unknown',
|
|
reason: 'unknown'
|
|
});
|
|
}
|
|
|
|
async function run() {
|
|
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'llm-checker-structural-'));
|
|
|
|
try {
|
|
const fake = makeFakeOllama(tempDir);
|
|
|
|
testSchemaValidation();
|
|
testCliValidate();
|
|
testBlobResolution(fake);
|
|
await testAttachAndCache(fake);
|
|
testEngineEvaluation();
|
|
await testAuditAndEnforceModes(fake);
|
|
await testOnUnverifiable(fake);
|
|
await testAuditExportFields(fake);
|
|
|
|
console.log('policy-structural-validation.test.js: OK');
|
|
} finally {
|
|
fs.rmSync(tempDir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
if (require.main === module) {
|
|
run().catch((error) => {
|
|
console.error('policy-structural-validation.test.js: FAILED');
|
|
console.error(error);
|
|
process.exit(1);
|
|
});
|
|
}
|
|
|
|
module.exports = { run };
|