refactor: CI optimization, logging improvements, and codebase formatting (#456)
* fix(db): remove unique constraint on folder to support multi-channel agents * ci: implement automated skill drift detection and self-healing PRs * fix: align registration logic with Gavriel's feedback and fix build/test issues from Daniel Mi * style: conform to prettier standards for CI validation * test: fix branch naming inconsistency in CI (master vs main) * fix(ci): robust module resolution by removing file extensions in scripts * refactor(ci): simplify skill validation by removing redundant combination tests * style: conform skills-engine to prettier, unify logging in index.ts and cleanup unused imports * refactor: extract multi-channel DB changes to separate branch Move channel column, folder suffix logic, and related migrations to feat/multi-channel-db-v2 for independent review. This PR now contains only CI/CD optimizations, Prettier formatting, and logging improvements. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Gabi Simons
GitHub
252
scripts/validate-all-skills.ts
Normal file
252
scripts/validate-all-skills.ts
Normal file
@@ -0,0 +1,252 @@
|
||||
#!/usr/bin/env npx tsx
|
||||
/**
|
||||
* Validate all skills by applying each in isolation against current main.
|
||||
*
|
||||
* For each skill:
|
||||
* 1. Reset working tree to clean state
|
||||
* 2. Initialize .nanoclaw/ (snapshot current source as base)
|
||||
* 3. Apply skill via apply-skill.ts
|
||||
* 4. Run tsc --noEmit (typecheck)
|
||||
* 5. Run the skill's test command (from manifest.yaml)
|
||||
*
|
||||
* Sets GitHub Actions outputs:
|
||||
* drifted — "true" | "false"
|
||||
* drifted_skills — JSON array of drifted skill names, e.g. ["add-telegram"]
|
||||
* results — JSON array of per-skill results
|
||||
*
|
||||
* Exit code 1 if any skill drifted, 0 otherwise.
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx scripts/validate-all-skills.ts # validate all
|
||||
* npx tsx scripts/validate-all-skills.ts add-telegram # validate one
|
||||
*/
|
||||
import { execSync } from 'child_process';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
import { parse } from 'yaml';
|
||||
import type { SkillManifest } from '../skills-engine/types.js';
|
||||
|
||||
interface SkillValidationResult {
|
||||
name: string;
|
||||
success: boolean;
|
||||
failedStep?: 'apply' | 'typecheck' | 'test';
|
||||
error?: string;
|
||||
}
|
||||
|
||||
function discoverSkills(
|
||||
skillsDir: string,
|
||||
): { name: string; dir: string; manifest: SkillManifest }[] {
|
||||
if (!fs.existsSync(skillsDir)) return [];
|
||||
const results: { name: string; dir: string; manifest: SkillManifest }[] = [];
|
||||
|
||||
for (const entry of fs.readdirSync(skillsDir, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
const manifestPath = path.join(skillsDir, entry.name, 'manifest.yaml');
|
||||
if (!fs.existsSync(manifestPath)) continue;
|
||||
const manifest = parse(
|
||||
fs.readFileSync(manifestPath, 'utf-8'),
|
||||
) as SkillManifest;
|
||||
results.push({
|
||||
name: entry.name,
|
||||
dir: path.join(skillsDir, entry.name),
|
||||
manifest,
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/** Restore tracked files and remove untracked skill artifacts. */
|
||||
function resetWorkingTree(): void {
|
||||
execSync('git checkout -- .', { stdio: 'pipe' });
|
||||
// Remove untracked files added by skill application (e.g. src/channels/telegram.ts)
|
||||
// but preserve node_modules to avoid costly reinstalls.
|
||||
execSync('git clean -fd --exclude=node_modules', { stdio: 'pipe' });
|
||||
// Clean skills-system state directory
|
||||
if (fs.existsSync('.nanoclaw')) {
|
||||
fs.rmSync('.nanoclaw', { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function initNanoclaw(): void {
|
||||
execSync(
|
||||
'npx tsx -e "import { initNanoclawDir } from \'./skills-engine/index\'; initNanoclawDir();"',
|
||||
{ stdio: 'pipe', timeout: 30_000 },
|
||||
);
|
||||
}
|
||||
|
||||
/** Append a key=value to $GITHUB_OUTPUT (no-op locally). */
|
||||
function setOutput(key: string, value: string): void {
|
||||
const outputFile = process.env.GITHUB_OUTPUT;
|
||||
if (!outputFile) return;
|
||||
|
||||
if (value.includes('\n')) {
|
||||
const delimiter = `ghadelim_${Date.now()}`;
|
||||
fs.appendFileSync(
|
||||
outputFile,
|
||||
`${key}<<${delimiter}\n${value}\n${delimiter}\n`,
|
||||
);
|
||||
} else {
|
||||
fs.appendFileSync(outputFile, `${key}=${value}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
function truncate(s: string, max = 300): string {
|
||||
return s.length > max ? s.slice(0, max) + '...' : s;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const projectRoot = process.cwd();
|
||||
const skillsDir = path.join(projectRoot, '.claude', 'skills');
|
||||
|
||||
// Allow filtering to specific skills via CLI args
|
||||
const filterSkills = process.argv.slice(2);
|
||||
|
||||
let skills = discoverSkills(skillsDir);
|
||||
if (filterSkills.length > 0) {
|
||||
skills = skills.filter((s) => filterSkills.includes(s.name));
|
||||
}
|
||||
|
||||
if (skills.length === 0) {
|
||||
console.log('No skills found to validate.');
|
||||
setOutput('drifted', 'false');
|
||||
setOutput('drifted_skills', '[]');
|
||||
setOutput('results', '[]');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
console.log(
|
||||
`Validating ${skills.length} skill(s): ${skills.map((s) => s.name).join(', ')}\n`,
|
||||
);
|
||||
|
||||
const results: SkillValidationResult[] = [];
|
||||
|
||||
for (const skill of skills) {
|
||||
console.log(`--- ${skill.name} ---`);
|
||||
|
||||
// Clean slate
|
||||
resetWorkingTree();
|
||||
initNanoclaw();
|
||||
|
||||
// Step 1: Apply skill
|
||||
try {
|
||||
const applyOutput = execSync(
|
||||
`npx tsx scripts/apply-skill.ts "${skill.dir}"`,
|
||||
{
|
||||
encoding: 'utf-8',
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
timeout: 120_000,
|
||||
},
|
||||
);
|
||||
// parse stdout to verify success
|
||||
try {
|
||||
const parsed = JSON.parse(applyOutput);
|
||||
if (!parsed.success) {
|
||||
console.log(` FAIL (apply): ${truncate(parsed.error || 'unknown')}`);
|
||||
results.push({
|
||||
name: skill.name,
|
||||
success: false,
|
||||
failedStep: 'apply',
|
||||
error: parsed.error,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
} catch {
|
||||
// Non-JSON stdout with exit 0 is treated as success
|
||||
}
|
||||
} catch (err: any) {
|
||||
const stderr = err.stderr?.toString() || '';
|
||||
const stdout = err.stdout?.toString() || '';
|
||||
let error = 'Apply failed';
|
||||
try {
|
||||
const parsed = JSON.parse(stdout);
|
||||
error = parsed.error || error;
|
||||
} catch {
|
||||
error = stderr || stdout || err.message;
|
||||
}
|
||||
console.log(` FAIL (apply): ${truncate(error)}`);
|
||||
results.push({
|
||||
name: skill.name,
|
||||
success: false,
|
||||
failedStep: 'apply',
|
||||
error,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
console.log(' apply: OK');
|
||||
|
||||
// Step 2: Typecheck
|
||||
try {
|
||||
execSync('npx tsc --noEmit', {
|
||||
stdio: 'pipe',
|
||||
timeout: 120_000,
|
||||
});
|
||||
} catch (err: any) {
|
||||
const error = err.stdout?.toString() || err.message;
|
||||
console.log(` FAIL (typecheck): ${truncate(error)}`);
|
||||
results.push({
|
||||
name: skill.name,
|
||||
success: false,
|
||||
failedStep: 'typecheck',
|
||||
error,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
console.log(' typecheck: OK');
|
||||
|
||||
// Step 3: Skill's own test command
|
||||
if (skill.manifest.test) {
|
||||
try {
|
||||
execSync(skill.manifest.test, {
|
||||
stdio: 'pipe',
|
||||
timeout: 300_000,
|
||||
});
|
||||
} catch (err: any) {
|
||||
const error =
|
||||
err.stdout?.toString() || err.stderr?.toString() || err.message;
|
||||
console.log(` FAIL (test): ${truncate(error)}`);
|
||||
results.push({
|
||||
name: skill.name,
|
||||
success: false,
|
||||
failedStep: 'test',
|
||||
error,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
console.log(' test: OK');
|
||||
}
|
||||
|
||||
console.log(' PASS');
|
||||
results.push({ name: skill.name, success: true });
|
||||
}
|
||||
|
||||
// Restore clean state
|
||||
resetWorkingTree();
|
||||
|
||||
// Summary
|
||||
const drifted = results.filter((r) => !r.success);
|
||||
const passed = results.filter((r) => r.success);
|
||||
|
||||
console.log('\n=== Summary ===');
|
||||
for (const r of results) {
|
||||
const status = r.success ? 'PASS' : 'FAIL';
|
||||
const detail = r.failedStep ? ` (${r.failedStep})` : '';
|
||||
console.log(` ${status} ${r.name}${detail}`);
|
||||
}
|
||||
console.log(`\n${passed.length} passed, ${drifted.length} failed`);
|
||||
|
||||
// GitHub Actions outputs
|
||||
setOutput('drifted', drifted.length > 0 ? 'true' : 'false');
|
||||
setOutput('drifted_skills', JSON.stringify(drifted.map((d) => d.name)));
|
||||
setOutput('results', JSON.stringify(results));
|
||||
|
||||
if (drifted.length > 0) {
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('Fatal error:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user