feat(core): add QMD-inspired hybrid search pipeline (#40)

* feat(core): add QMD-inspired hybrid search pipeline

Implement a hybrid search system combining vector embeddings with keyword search:

- Vector embeddings using node-llama-cpp with GGUF models (nomic-embed-text)
- RRF (Reciprocal Rank Fusion) for combining multiple rankers
- Query expansion with LLM and synonym fallback
- Position-aware score blending (different weights for top3, top10, rest)
- SQLite storage with sqlite-vec extension (falls back to in-memory)
- Auto-download of local GGUF models (~2GB)

New search module:
- embeddings.ts: Vector embedding generation service
- vector-store.ts: SQLite-based vector storage
- rrf.ts: RRF fusion algorithm
- expansion.ts: Query expansion with caching
- hybrid.ts: Main hybrid search pipeline
- local-models.ts: GGUF model management

CLI integration:
- skillkit recommend --hybrid: Enable hybrid search
- skillkit recommend --expand: Enable query expansion
- skillkit recommend --rerank: Enable LLM re-ranking
- skillkit recommend --build-index: Build embedding index

Tests: 56 new tests for search module (all passing)

* fix(search): address CodeRabbit review findings

- Add missing beforeEach import in embeddings.test.ts
- Update optional dependencies to current versions
- Fix weights/variations length mismatch in expansion.ts
- Avoid mutating input arrays in rrf.ts mergeRankings
- Delete orphaned chunk vectors in vector-store.ts delete() and clear()
- Add comment for empty catch block in vector-store.ts
- Make hybridSearch helper use buildIndex for true hybrid search
- Handle graceful degradation when embeddings unavailable
- Add node-llama-cpp mock in hybrid.test.ts

* fix(search): address CodeRabbit review round 2

- Fix gemma GGUF URL to point to valid community repo (bartowski)
- Add SQL injection protection for table names in vector-store.ts
- Fix resource leak with try/finally in hybridSearch helper
- Add backpressure handling for large model downloads
- Clamp startLine to non-negative in chunk calculation
- Add runtime validation warning for --expand/--rerank without --hybrid
- Update node-llama-cpp API usage (createEmbeddingContext, createContext)
- Update pnpm-lock.yaml for new optional dependency versions

* fix(search): use @ts-ignore for optional node-llama-cpp imports

CI doesn't install optional dependencies, so @ts-ignore is needed
instead of @ts-expect-error to suppress module not found errors.
This commit is contained in:
Rohit Ghumare
2026-02-02 23:28:08 +00:00
committed by GitHub
parent e3d7e0e28f
commit 109f00cd95
18 changed files with 4473 additions and 0 deletions
+162
View File
@@ -52,6 +52,9 @@ export class RecommendCommand extends Command {
['Update skill index', '$0 recommend --update'],
['Search for skills by task', '$0 recommend --task "authentication"'],
['Search for skills (alias)', '$0 recommend --search "testing"'],
['Hybrid search (vector + keyword)', '$0 recommend --search "auth" --hybrid'],
['Hybrid search with query expansion', '$0 recommend --search "auth" --hybrid --expand'],
['Build hybrid search index', '$0 recommend --build-index'],
],
});
@@ -125,6 +128,26 @@ export class RecommendCommand extends Command {
description: 'Show category path for each recommendation',
});
// Hybrid search mode
hybrid = Option.Boolean('--hybrid,-H', false, {
description: 'Use hybrid search (vector + keyword)',
});
// Query expansion
expand = Option.Boolean('--expand,-x', false, {
description: 'Enable query expansion (requires --hybrid)',
});
// Reranking
rerank = Option.Boolean('--rerank', false, {
description: 'Enable LLM reranking (requires --hybrid)',
});
// Build index
buildIndex = Option.Boolean('--build-index', false, {
description: 'Build/rebuild the hybrid search embedding index',
});
async execute(): Promise<number> {
const targetPath = resolve(this.projectPath || process.cwd());
@@ -133,6 +156,16 @@ export class RecommendCommand extends Command {
return await this.updateIndex();
}
// Handle hybrid index building
if (this.buildIndex) {
return await this.buildHybridIndex();
}
// Validate hybrid-dependent options
if ((this.expand || this.rerank) && !this.hybrid) {
warn('--expand and --rerank require --hybrid flag. These options will be ignored.');
}
if (!this.quiet && !this.json) {
header('Skill Recommendations');
}
@@ -168,6 +201,9 @@ export class RecommendCommand extends Command {
// Handle search mode (--search or --task)
const searchQuery = this.search || this.task;
if (searchQuery) {
if (this.hybrid) {
return await this.handleHybridSearch(engine, searchQuery);
}
return this.handleSearch(engine, searchQuery);
}
@@ -446,6 +482,132 @@ export class RecommendCommand extends Command {
console.log(colors.muted('More details: skillkit recommend --explain --verbose'));
}
private async handleHybridSearch(engine: RecommendationEngine, query: string): Promise<number> {
if (!this.quiet && !this.json) {
header(`Hybrid Search: "${query}"`);
}
const s = !this.quiet && !this.json ? spinner() : null;
s?.start('Initializing hybrid search...');
try {
await engine.initHybridSearch();
s?.message('Searching...');
const results = await engine.hybridSearch({
query,
limit: this.limit ? parseInt(this.limit, 10) : 10,
hybrid: true,
enableExpansion: this.expand,
enableReranking: this.rerank,
filters: {
minScore: this.minScore ? parseInt(this.minScore, 10) : undefined,
},
});
s?.stop(`Found ${results.length} results`);
if (this.json) {
console.log(JSON.stringify(results, null, 2));
return 0;
}
if (results.length === 0) {
warn(`No skills found matching "${query}"`);
return 0;
}
console.log('');
console.log(colors.bold(`Hybrid search results for "${query}" (${results.length} found):`));
if (this.expand && results[0]?.expandedTerms?.length) {
console.log(colors.muted(` Expanded: ${results[0].expandedTerms.join(', ')}`));
}
console.log('');
for (const result of results) {
let relevanceColor: (text: string) => string;
if (result.relevance >= 70) {
relevanceColor = colors.success;
} else if (result.relevance >= 50) {
relevanceColor = colors.warning;
} else {
relevanceColor = colors.muted;
}
const relevanceBar = progressBar(result.relevance, 100, 10);
console.log(` ${relevanceColor(`${result.relevance}%`)} ${colors.dim(relevanceBar)} ${colors.bold(result.skill.name)}`);
if (result.skill.description) {
console.log(` ${colors.muted(truncate(result.skill.description, 70))}`);
}
if (this.verbose) {
const scores: string[] = [];
if (typeof result.vectorSimilarity === 'number') {
scores.push(`vector: ${(result.vectorSimilarity * 100).toFixed(0)}%`);
}
if (typeof result.keywordScore === 'number') {
scores.push(`keyword: ${result.keywordScore.toFixed(0)}%`);
}
if (typeof result.rrfScore === 'number') {
scores.push(`rrf: ${result.rrfScore.toFixed(3)}`);
}
if (scores.length > 0) {
console.log(` ${colors.dim('Scores:')} ${scores.join(' | ')}`);
}
}
if (result.matchedTerms.length > 0) {
console.log(` ${colors.dim('Matched:')} ${result.matchedTerms.join(', ')}`);
}
console.log('');
}
return 0;
} catch (err) {
s?.stop(colors.error('Hybrid search failed'));
console.log(colors.muted(err instanceof Error ? err.message : String(err)));
console.log(colors.muted('Falling back to standard search...'));
return this.handleSearch(engine, query);
}
}
private async buildHybridIndex(): Promise<number> {
if (!this.quiet) {
header('Build Hybrid Search Index');
}
const index = this.loadIndex();
if (!index || index.skills.length === 0) {
warn('No skill index found. Run --update first.');
return 1;
}
const s = spinner();
s.start('Initializing...');
try {
const engine = new RecommendationEngine();
engine.loadIndex(index);
await engine.buildHybridIndex((progress) => {
const percentage = Math.round((progress.current / progress.total) * 100);
s.message(`${progress.phase}: ${progress.message || ''} (${percentage}%)`);
});
s.stop(colors.success(`${symbols.success} Built hybrid index for ${index.skills.length} skills`));
console.log(colors.muted(' Index stored in: ~/.skillkit/search.db'));
console.log(colors.muted(' Use --hybrid flag for vector+keyword search\n'));
return 0;
} catch (err) {
s.stop(colors.error('Failed to build hybrid index'));
console.log(colors.muted(err instanceof Error ? err.message : String(err)));
return 1;
}
}
private handleSearch(engine: RecommendationEngine, query: string): number {
if (!this.quiet && !this.json) {
header(`Search: "${query}"`);
+10
View File
@@ -37,6 +37,10 @@
"./learning": {
"import": "./dist/learning/index.js",
"types": "./dist/learning/index.d.ts"
},
"./search": {
"import": "./dist/search/index.js",
"types": "./dist/search/index.d.ts"
}
},
"files": [
@@ -53,7 +57,13 @@
"yaml": "^2.6.1",
"zod": "^3.24.1"
},
"optionalDependencies": {
"node-llama-cpp": "^3.15.0",
"better-sqlite3": "^12.0.0",
"sqlite-vec": "^0.1.6"
},
"devDependencies": {
"@types/better-sqlite3": "^7.6.11",
"@types/node": "^22.10.5",
"tsup": "^8.3.5",
"typescript": "^5.7.2",
+3
View File
@@ -101,3 +101,6 @@ export * from './connectors/index.js';
// Execution Flow (Step Tracking & Metrics - Phase 21)
export * from './execution/index.js';
// Hybrid Search (QMD-Inspired Vector + Keyword Search)
export * from './search/index.js';
+91
View File
@@ -11,8 +11,11 @@ import type {
SearchOptions,
SearchResult,
FreshnessResult,
RecommendHybridSearchOptions,
RecommendHybridSearchResult,
} from './types.js';
import { DEFAULT_SCORING_WEIGHTS, TAG_TO_TECH, getTechTags } from './types.js';
import type { HybridSearchPipeline } from '../search/hybrid.js';
/**
* Recommendation engine for matching skills to project profiles
@@ -20,16 +23,39 @@ import { DEFAULT_SCORING_WEIGHTS, TAG_TO_TECH, getTechTags } from './types.js';
export class RecommendationEngine {
private weights: ScoringWeights;
private index: SkillIndex | null = null;
private hybridPipeline: HybridSearchPipeline | null = null;
constructor(weights?: Partial<ScoringWeights>) {
this.weights = { ...DEFAULT_SCORING_WEIGHTS, ...weights };
}
/**
* Initialize hybrid search pipeline for vector + keyword search
*/
async initHybridSearch(): Promise<void> {
const { createHybridSearchPipeline } = await import('../search/hybrid.js');
this.hybridPipeline = createHybridSearchPipeline();
if (this.index) {
this.hybridPipeline.loadSkillsIndex(this.index);
}
await this.hybridPipeline.initialize();
}
/**
* Check if hybrid search is available
*/
isHybridSearchAvailable(): boolean {
return this.hybridPipeline !== null && this.hybridPipeline.isInitialized();
}
/**
* Load skill index from cache or generate from local skills
*/
loadIndex(index: SkillIndex): void {
this.index = index;
if (this.hybridPipeline) {
this.hybridPipeline.loadSkillsIndex(index);
}
}
/**
@@ -603,6 +629,71 @@ export class RecommendationEngine {
return { relevance, matchedTerms, snippet };
}
/**
* Hybrid search combining vector embeddings and keyword matching
*/
async hybridSearch(options: RecommendHybridSearchOptions): Promise<RecommendHybridSearchResult[]> {
const { query, limit = 10, hybrid = true, enableExpansion = false, enableReranking = false, filters } = options;
if (!hybrid || !this.hybridPipeline) {
const basicResults = this.search({ query, limit, semantic: true, filters });
return basicResults.map((r) => ({
...r,
hybridScore: r.relevance / 100,
}));
}
const response = await this.hybridPipeline.search({
query,
limit,
enableExpansion,
enableReranking,
});
let results = response.results.map((r) => ({
skill: r.skill,
relevance: r.relevance,
matchedTerms: r.matchedTerms,
snippet: r.snippet,
hybridScore: r.hybridScore,
vectorSimilarity: r.vectorSimilarity,
keywordScore: r.keywordScore,
rrfScore: r.rrfScore,
expandedTerms: r.expandedTerms,
}));
if (filters?.tags && filters.tags.length > 0) {
results = results.filter((r) =>
r.skill.tags?.some((t) => filters.tags!.includes(t))
);
}
if (filters?.verified) {
results = results.filter((r) => r.skill.verified);
}
if (filters?.minScore) {
results = results.filter((r) => r.relevance >= filters.minScore!);
}
return results.slice(0, limit);
}
/**
* Build hybrid search index from skills
*/
async buildHybridIndex(
onProgress?: (progress: { phase: string; current: number; total: number; message?: string }) => void
): Promise<void> {
if (!this.index) {
throw new Error('No skill index loaded. Call loadIndex() first.');
}
if (!this.hybridPipeline) {
await this.initHybridSearch();
}
await this.hybridPipeline!.buildIndex(this.index.skills, onProgress);
}
/**
* Check freshness of installed skills against project dependencies
*
+22
View File
@@ -326,3 +326,25 @@ export interface ReasoningRecommendationResult extends RecommendationResult {
strategy: string;
};
}
/**
* Hybrid search options for RecommendationEngine
*/
export interface RecommendHybridSearchOptions extends SearchOptions {
hybrid?: boolean;
enableExpansion?: boolean;
enableReranking?: boolean;
semanticWeight?: number;
keywordWeight?: number;
}
/**
* Hybrid search result with additional metadata for RecommendationEngine
*/
export interface RecommendHybridSearchResult extends SearchResult {
hybridScore?: number;
vectorSimilarity?: number;
keywordScore?: number;
rrfScore?: number;
expandedTerms?: string[];
}
@@ -0,0 +1,217 @@
import { describe, it, expect, beforeEach } from 'vitest';
import { EmbeddingService } from '../embeddings.js';
import type { SkillSummary } from '../../recommend/types.js';
describe('EmbeddingService', () => {
describe('cosineSimilarity', () => {
let service: EmbeddingService;
beforeEach(() => {
service = new EmbeddingService();
});
it('should return 1 for identical vectors', () => {
const a = new Float32Array([1, 0, 0]);
const b = new Float32Array([1, 0, 0]);
const similarity = service.cosineSimilarity(a, b);
expect(similarity).toBeCloseTo(1, 5);
});
it('should return 0 for orthogonal vectors', () => {
const a = new Float32Array([1, 0, 0]);
const b = new Float32Array([0, 1, 0]);
const similarity = service.cosineSimilarity(a, b);
expect(similarity).toBeCloseTo(0, 5);
});
it('should return -1 for opposite vectors', () => {
const a = new Float32Array([1, 0, 0]);
const b = new Float32Array([-1, 0, 0]);
const similarity = service.cosineSimilarity(a, b);
expect(similarity).toBeCloseTo(-1, 5);
});
it('should handle normalized vectors', () => {
const a = new Float32Array([0.5, 0.5, 0.5, 0.5]);
const b = new Float32Array([0.5, 0.5, 0.5, 0.5]);
const similarity = service.cosineSimilarity(a, b);
expect(similarity).toBeCloseTo(1, 5);
});
it('should throw for mismatched dimensions', () => {
const a = new Float32Array([1, 0]);
const b = new Float32Array([1, 0, 0]);
expect(() => service.cosineSimilarity(a, b)).toThrow('Vector dimensions mismatch');
});
it('should return 0 for zero vectors', () => {
const a = new Float32Array([0, 0, 0]);
const b = new Float32Array([1, 0, 0]);
const similarity = service.cosineSimilarity(a, b);
expect(similarity).toBe(0);
});
});
describe('initialization state', () => {
it('should not be initialized by default', () => {
const service = new EmbeddingService();
expect(service.isInitialized()).toBe(false);
});
it('should return default dimensions', () => {
const service = new EmbeddingService();
expect(service.getDimensions()).toBe(768);
});
});
describe('skill text building', () => {
it('should handle skill with all fields', async () => {
const service = new EmbeddingService();
const skill: SkillSummary = {
name: 'test-skill',
description: 'A test skill',
tags: ['tag1', 'tag2'],
compatibility: {
frameworks: ['react'],
languages: ['typescript'],
libraries: ['lodash'],
},
popularity: 100,
quality: 80,
verified: true,
};
const buildSkillText = (s: SkillSummary): string => {
const parts: string[] = [s.name];
if (s.description) parts.push(s.description);
if (s.tags?.length) parts.push(s.tags.join(' '));
if (s.compatibility?.frameworks?.length) parts.push(s.compatibility.frameworks.join(' '));
if (s.compatibility?.languages?.length) parts.push(s.compatibility.languages.join(' '));
if (s.compatibility?.libraries?.length) parts.push(s.compatibility.libraries.join(' '));
return parts.join(' ').toLowerCase();
};
const text = buildSkillText(skill);
expect(text).toContain('test-skill');
expect(text).toContain('a test skill');
expect(text).toContain('tag1');
expect(text).toContain('react');
expect(text).toContain('typescript');
expect(text).toContain('lodash');
});
it('should handle skill with minimal fields', () => {
const skill: SkillSummary = {
name: 'minimal-skill',
tags: [],
popularity: 0,
quality: 50,
verified: false,
};
const buildSkillText = (s: SkillSummary): string => {
const parts: string[] = [s.name];
if (s.description) parts.push(s.description);
if (s.tags?.length) parts.push(s.tags.join(' '));
return parts.join(' ').toLowerCase();
};
const text = buildSkillText(skill);
expect(text).toBe('minimal-skill');
});
});
});
describe('EmbeddingService chunking', () => {
it('should split text into chunks', () => {
const chunkText = (
text: string,
config = { maxTokens: 100, overlapPercent: 15, minChunkSize: 10 }
) => {
const lines = text.split('\n');
const chunks: { content: string; startLine: number; endLine: number }[] = [];
const avgCharsPerToken = 4;
const maxCharsPerChunk = config.maxTokens * avgCharsPerToken;
const overlapChars = Math.floor(maxCharsPerChunk * (config.overlapPercent / 100));
let currentChunk: string[] = [];
let currentChars = 0;
let startLine = 0;
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
const lineChars = line.length + 1;
if (currentChars + lineChars > maxCharsPerChunk && currentChunk.length > 0) {
const content = currentChunk.join('\n');
if (content.length >= config.minChunkSize) {
chunks.push({ content, startLine, endLine: i - 1 });
}
const overlapLines: string[] = [];
let overlapCharsCount = 0;
for (let j = currentChunk.length - 1; j >= 0 && overlapCharsCount < overlapChars; j--) {
overlapLines.unshift(currentChunk[j]);
overlapCharsCount += currentChunk[j].length + 1;
}
currentChunk = overlapLines;
currentChars = overlapCharsCount;
startLine = i - overlapLines.length;
}
currentChunk.push(line);
currentChars += lineChars;
}
if (currentChunk.length > 0) {
const content = currentChunk.join('\n');
if (content.length >= config.minChunkSize) {
chunks.push({ content, startLine, endLine: lines.length - 1 });
}
}
return chunks;
};
const longText = Array(50).fill('This is a line of text for testing.').join('\n');
const chunks = chunkText(longText);
expect(chunks.length).toBeGreaterThan(1);
chunks.forEach((chunk) => {
expect(chunk.content.length).toBeLessThanOrEqual(400);
expect(chunk.startLine).toBeGreaterThanOrEqual(0);
expect(chunk.endLine).toBeGreaterThanOrEqual(chunk.startLine);
});
});
it('should not chunk short text', () => {
const chunkText = (
text: string,
config = { maxTokens: 800, overlapPercent: 15, minChunkSize: 100 }
) => {
const lines = text.split('\n');
if (text.length < config.maxTokens * 4) {
if (text.length >= config.minChunkSize) {
return [{ content: text, startLine: 0, endLine: lines.length - 1 }];
}
return [];
}
return [{ content: text, startLine: 0, endLine: lines.length - 1 }];
};
const shortText = 'This is a short text that should not be chunked.';
const chunks = chunkText(shortText, { maxTokens: 800, overlapPercent: 15, minChunkSize: 10 });
expect(chunks.length).toBe(1);
expect(chunks[0].content).toBe(shortText);
});
});
@@ -0,0 +1,100 @@
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import { QueryExpander, expandQuerySimple } from '../expansion.js';
describe('QueryExpander', () => {
let expander: QueryExpander;
beforeEach(() => {
expander = new QueryExpander();
});
afterEach(() => {
expander.clearCache();
});
describe('expandQuerySimple (without LLM)', () => {
it('should expand "auth" to authentication-related terms', () => {
const result = expandQuerySimple('auth');
expect(result.original).toBe('auth');
expect(result.variations.length).toBeGreaterThan(0);
expect(result.variations.some((v) => v.includes('authentication'))).toBe(true);
});
it('should expand "api" to related terms', () => {
const result = expandQuerySimple('api');
expect(result.original).toBe('api');
expect(result.variations.some((v) => v.includes('rest') || v.includes('endpoint'))).toBe(true);
});
it('should expand "test" to testing-related terms', () => {
const result = expandQuerySimple('testing');
expect(result.original).toBe('testing');
expect(result.variations.some((v) => v.includes('test') || v.includes('unit'))).toBe(true);
});
it('should handle queries without synonyms', () => {
const result = expandQuerySimple('xyz123unique');
expect(result.original).toBe('xyz123unique');
expect(result.variations).toEqual([]);
expect(result.weights).toEqual([2.0]);
});
it('should give original query weight of 2.0', () => {
const result = expandQuerySimple('auth');
expect(result.weights[0]).toBe(2.0);
for (let i = 1; i < result.weights.length; i++) {
expect(result.weights[i]).toBe(1.0);
}
});
it('should limit variations to 3', () => {
const result = expandQuerySimple('auth');
expect(result.variations.length).toBeLessThanOrEqual(3);
});
});
describe('expand (fallback without LLM)', () => {
it('should return expanded query without LLM initialization', async () => {
const result = await expander.expand('auth');
expect(result.original).toBe('auth');
expect(result.variations.length).toBeGreaterThan(0);
});
it('should cache results', async () => {
const result1 = await expander.expand('auth');
const result2 = await expander.expand('auth');
expect(result1).toEqual(result2);
});
it('should handle case-insensitive caching', async () => {
const result1 = await expander.expand('Auth');
const result2 = await expander.expand('AUTH');
expect(result1.original).toBe('Auth');
expect(result2.original).toBe('AUTH');
});
});
describe('clearCache', () => {
it('should clear the expansion cache', async () => {
await expander.expand('auth');
expander.clearCache();
const result = await expander.expand('auth');
expect(result).toBeDefined();
});
});
describe('isInitialized', () => {
it('should return false when not initialized with LLM', () => {
expect(expander.isInitialized()).toBe(false);
});
});
});
@@ -0,0 +1,244 @@
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import type { SkillSummary, SkillIndex } from '../../recommend/types.js';
const mockSkills: SkillSummary[] = [
{
name: 'react-auth',
description: 'Authentication utilities for React applications',
tags: ['react', 'auth', 'authentication'],
compatibility: {
frameworks: ['react'],
languages: ['typescript', 'javascript'],
libraries: ['next-auth'],
},
popularity: 500,
quality: 80,
verified: true,
},
{
name: 'vue-forms',
description: 'Form validation library for Vue',
tags: ['vue', 'forms', 'validation'],
compatibility: {
frameworks: ['vue'],
languages: ['typescript', 'javascript'],
libraries: [],
},
popularity: 300,
quality: 75,
verified: false,
},
{
name: 'api-testing',
description: 'API testing utilities for REST endpoints',
tags: ['testing', 'api', 'rest'],
compatibility: {
frameworks: [],
languages: ['typescript'],
libraries: ['jest', 'supertest'],
},
popularity: 400,
quality: 85,
verified: true,
},
{
name: 'login-helper',
description: 'Helper functions for login and authentication flows',
tags: ['auth', 'login', 'oauth'],
compatibility: {
frameworks: ['react', 'vue'],
languages: ['typescript'],
libraries: [],
},
popularity: 200,
quality: 70,
verified: false,
},
];
const mockIndex: SkillIndex = {
version: 1,
lastUpdated: new Date().toISOString(),
skills: mockSkills,
sources: [],
};
vi.mock('better-sqlite3', () => {
return {
default: vi.fn().mockImplementation(() => {
throw new Error("Cannot find module 'better-sqlite3'");
}),
};
});
vi.mock('sqlite-vec', () => {
return {
load: vi.fn(),
};
});
vi.mock('node-llama-cpp', () => {
return {
getLlama: vi.fn().mockImplementation(() => {
throw new Error("Cannot find package 'node-llama-cpp'");
}),
};
});
describe('HybridSearchPipeline', () => {
let HybridSearchPipeline: typeof import('../hybrid.js').HybridSearchPipeline;
let createHybridSearchPipeline: typeof import('../hybrid.js').createHybridSearchPipeline;
let pipeline: InstanceType<typeof HybridSearchPipeline>;
beforeEach(async () => {
const module = await import('../hybrid.js');
HybridSearchPipeline = module.HybridSearchPipeline;
createHybridSearchPipeline = module.createHybridSearchPipeline;
pipeline = createHybridSearchPipeline();
});
afterEach(async () => {
if (pipeline) {
await pipeline.dispose();
}
});
describe('initialization', () => {
it('should create a pipeline', () => {
expect(pipeline).toBeInstanceOf(HybridSearchPipeline);
});
it('should not be initialized before calling initialize()', () => {
expect(pipeline.isInitialized()).toBe(false);
});
it('should be initialized after calling initialize()', async () => {
await pipeline.initialize();
expect(pipeline.isInitialized()).toBe(true);
});
it('should load skills index', async () => {
pipeline.loadSkillsIndex(mockIndex);
const stats = pipeline.getStats();
expect(stats.skillCount).toBe(mockSkills.length);
});
});
describe('search (keyword-only fallback)', () => {
beforeEach(async () => {
pipeline.loadSkillsIndex(mockIndex);
await pipeline.initialize();
});
it('should find skills matching query', async () => {
const response = await pipeline.search({ query: 'authentication' });
expect(response.results.length).toBeGreaterThan(0);
expect(response.results.some((r) => r.skill.name === 'react-auth')).toBe(true);
});
it('should find skills by tag', async () => {
const response = await pipeline.search({ query: 'react' });
expect(response.results.length).toBeGreaterThan(0);
expect(response.results.some((r) => r.skill.tags?.includes('react'))).toBe(true);
});
it('should respect limit parameter', async () => {
const response = await pipeline.search({ query: 'auth', limit: 2 });
expect(response.results.length).toBeLessThanOrEqual(2);
});
it('should include timing information', async () => {
const response = await pipeline.search({ query: 'test' });
expect(typeof response.timing.totalMs).toBe('number');
expect(response.timing.totalMs).toBeGreaterThanOrEqual(0);
expect(response.timing.keywordSearchMs).toBeGreaterThanOrEqual(0);
});
it('should include stats', async () => {
const response = await pipeline.search({ query: 'auth' });
expect(response.stats.candidatesFromKeyword).toBeGreaterThan(0);
expect(response.stats.totalMerged).toBeGreaterThan(0);
});
it('should return original query in response', async () => {
const response = await pipeline.search({ query: 'authentication' });
expect(response.query.original).toBe('authentication');
});
});
describe('search with expansion', () => {
beforeEach(async () => {
pipeline.loadSkillsIndex(mockIndex);
await pipeline.initialize();
});
it('should expand query and find more results', async () => {
const response = await pipeline.search({
query: 'auth',
enableExpansion: true,
});
expect(response.results.length).toBeGreaterThan(0);
expect(response.query.expanded).toBeDefined();
});
it('should include expanded terms in response', async () => {
const response = await pipeline.search({
query: 'auth',
enableExpansion: true,
});
if (response.query.expanded) {
expect(response.query.expanded.original).toBe('auth');
expect(response.query.expanded.weights[0]).toBe(2.0);
}
});
});
describe('getStats', () => {
it('should return stats before initialization', () => {
const stats = pipeline.getStats();
expect(stats.initialized).toBe(false);
expect(stats.skillCount).toBe(0);
});
it('should return updated stats after loading index', async () => {
pipeline.loadSkillsIndex(mockIndex);
await pipeline.initialize();
const stats = pipeline.getStats();
expect(stats.initialized).toBe(true);
expect(stats.skillCount).toBe(mockSkills.length);
});
});
});
describe('hybridSearch helper', () => {
let hybridSearch: typeof import('../hybrid.js').hybridSearch;
beforeEach(async () => {
const module = await import('../hybrid.js');
hybridSearch = module.hybridSearch;
});
it('should perform search on skills array', async () => {
const response = await hybridSearch(mockSkills, 'react');
expect(response.results.length).toBeGreaterThan(0);
expect(response.query.original).toBe('react');
});
it('should respect options', async () => {
const response = await hybridSearch(mockSkills, 'auth', {
limit: 1,
});
expect(response.results.length).toBe(1);
});
});
@@ -0,0 +1,276 @@
import { describe, it, expect } from 'vitest';
import {
computeRRFScore,
fuseWithRRF,
normalizeScores,
weightedCombine,
applyPositionAwareBlending,
getRankFromScore,
mergeRankings,
type RankerResult,
type RRFInput,
} from '../rrf.js';
describe('RRF (Reciprocal Rank Fusion)', () => {
describe('computeRRFScore', () => {
it('should compute RRF score for a single rank', () => {
const score = computeRRFScore([1], 60);
expect(score).toBeCloseTo(1 / 61, 5);
});
it('should sum RRF scores for multiple ranks', () => {
const score = computeRRFScore([1, 2, 3], 60);
const expected = 1 / 61 + 1 / 62 + 1 / 63;
expect(score).toBeCloseTo(expected, 5);
});
it('should use custom k value', () => {
const scoreK60 = computeRRFScore([1], 60);
const scoreK30 = computeRRFScore([1], 30);
expect(scoreK30).toBeGreaterThan(scoreK60);
});
it('should return 0 for empty ranks', () => {
const score = computeRRFScore([]);
expect(score).toBe(0);
});
});
describe('fuseWithRRF', () => {
it('should fuse results from multiple rankers', () => {
const inputs: RRFInput[] = [
{
source: 'vector',
results: [
{ skillName: 'skill-a', score: 0.9 },
{ skillName: 'skill-b', score: 0.8 },
{ skillName: 'skill-c', score: 0.7 },
],
},
{
source: 'keyword',
results: [
{ skillName: 'skill-b', score: 90 },
{ skillName: 'skill-a', score: 80 },
{ skillName: 'skill-d', score: 70 },
],
},
];
const rankings = fuseWithRRF(inputs, 60);
expect(rankings.length).toBe(4);
expect(rankings[0].skillName).toBe('skill-a');
expect(rankings[1].skillName).toBe('skill-b');
});
it('should include rank information from each source', () => {
const inputs: RRFInput[] = [
{
source: 'vector',
results: [{ skillName: 'skill-a', score: 0.9 }],
},
{
source: 'keyword',
results: [{ skillName: 'skill-a', score: 90 }],
},
];
const rankings = fuseWithRRF(inputs);
expect(rankings[0].ranks).toHaveLength(2);
expect(rankings[0].ranks.map((r) => r.source)).toContain('vector');
expect(rankings[0].ranks.map((r) => r.source)).toContain('keyword');
});
it('should handle skills appearing in only one ranker', () => {
const inputs: RRFInput[] = [
{
source: 'vector',
results: [{ skillName: 'skill-a', score: 0.9 }],
},
{
source: 'keyword',
results: [{ skillName: 'skill-b', score: 90 }],
},
];
const rankings = fuseWithRRF(inputs);
expect(rankings).toHaveLength(2);
const skillAInfo = rankings.find((r) => r.skillName === 'skill-a');
const skillBInfo = rankings.find((r) => r.skillName === 'skill-b');
expect(skillAInfo?.ranks).toHaveLength(1);
expect(skillBInfo?.ranks).toHaveLength(1);
});
});
describe('normalizeScores', () => {
it('should normalize scores to 0-1 range', () => {
const results: RankerResult[] = [
{ skillName: 'a', score: 100 },
{ skillName: 'b', score: 50 },
{ skillName: 'c', score: 0 },
];
const normalized = normalizeScores(results);
expect(normalized[0].score).toBe(1);
expect(normalized[1].score).toBe(0.5);
expect(normalized[2].score).toBe(0);
});
it('should handle all same scores', () => {
const results: RankerResult[] = [
{ skillName: 'a', score: 50 },
{ skillName: 'b', score: 50 },
];
const normalized = normalizeScores(results);
expect(normalized[0].score).toBe(1);
expect(normalized[1].score).toBe(1);
});
it('should return empty array for empty input', () => {
const normalized = normalizeScores([]);
expect(normalized).toEqual([]);
});
});
describe('weightedCombine', () => {
it('should combine results with weights', () => {
const inputs: RRFInput[] = [
{
source: 'vector',
results: [
{ skillName: 'a', score: 100 },
{ skillName: 'b', score: 50 },
],
},
{
source: 'keyword',
results: [
{ skillName: 'b', score: 100 },
{ skillName: 'a', score: 50 },
],
},
];
const combined = weightedCombine(inputs, { vector: 0.6, keyword: 0.4 });
expect(combined).toHaveLength(2);
const skillA = combined.find((r) => r.skillName === 'a')!;
const skillB = combined.find((r) => r.skillName === 'b')!;
expect(skillA.score).toBeCloseTo(0.6 * 1 + 0.4 * 0, 5);
expect(skillB.score).toBeCloseTo(0.6 * 0 + 0.4 * 1, 5);
});
it('should default to weight 1 for unknown sources', () => {
const inputs: RRFInput[] = [
{
source: 'unknown',
results: [{ skillName: 'a', score: 100 }],
},
];
const combined = weightedCombine(inputs, {});
expect(combined[0].score).toBe(1);
});
});
describe('applyPositionAwareBlending', () => {
it('should apply different weights based on position', () => {
const retrievalScores = new Map([
['a', 0.9],
['b', 0.8],
['c', 0.7],
['d', 0.6],
]);
const rerankerScores = new Map([
['a', 0.5],
['b', 0.6],
['c', 0.7],
['d', 0.8],
]);
const sortedSkills = ['a', 'b', 'c', 'd'];
const blended = applyPositionAwareBlending(
retrievalScores,
rerankerScores,
sortedSkills
);
const scoreA = blended.get('a')!;
expect(scoreA).toBeCloseTo(0.9 * 0.75 + 0.5 * 0.25, 5);
const scoreD = blended.get('d')!;
expect(scoreD).toBeCloseTo(0.6 * 0.6 + 0.8 * 0.4, 5);
});
it('should handle missing scores', () => {
const retrievalScores = new Map([['a', 0.9]]);
const rerankerScores = new Map<string, number>();
const sortedSkills = ['a'];
const blended = applyPositionAwareBlending(
retrievalScores,
rerankerScores,
sortedSkills
);
expect(blended.get('a')).toBeCloseTo(0.9 * 0.75, 5);
});
});
describe('getRankFromScore', () => {
it('should return correct rank for a skill', () => {
const results: RankerResult[] = [
{ skillName: 'a', score: 90 },
{ skillName: 'b', score: 80 },
{ skillName: 'c', score: 70 },
];
expect(getRankFromScore('a', results)).toBe(1);
expect(getRankFromScore('b', results)).toBe(2);
expect(getRankFromScore('c', results)).toBe(3);
});
it('should return length+1 for missing skill', () => {
const results: RankerResult[] = [
{ skillName: 'a', score: 90 },
{ skillName: 'b', score: 80 },
];
expect(getRankFromScore('missing', results)).toBe(3);
});
});
describe('mergeRankings', () => {
it('should merge primary and secondary rankings with weights', () => {
const primary = [
{ skillName: 'a', rrfScore: 0.1, ranks: [{ source: 'primary', rank: 1 }] },
{ skillName: 'b', rrfScore: 0.05, ranks: [{ source: 'primary', rank: 2 }] },
];
const secondary = [
{ skillName: 'b', rrfScore: 0.1, ranks: [{ source: 'secondary', rank: 1 }] },
{ skillName: 'c', rrfScore: 0.05, ranks: [{ source: 'secondary', rank: 2 }] },
];
const merged = mergeRankings(primary, secondary, 0.7);
expect(merged).toHaveLength(3);
const skillA = merged.find((r) => r.skillName === 'a')!;
const skillB = merged.find((r) => r.skillName === 'b')!;
const skillC = merged.find((r) => r.skillName === 'c')!;
expect(skillA.rrfScore).toBeCloseTo(0.1 * 0.7, 5);
expect(skillB.rrfScore).toBeCloseTo(0.05 * 0.7 + 0.1 * 0.3, 5);
expect(skillC.rrfScore).toBeCloseTo(0.05 * 0.3, 5);
});
});
});
+294
View File
@@ -0,0 +1,294 @@
import type { SkillSummary } from '../recommend/types.js';
import type {
SkillEmbedding,
IndexBuildCallback,
} from './types.js';
import { DEFAULT_CHUNKING_CONFIG, MODEL_REGISTRY } from './types.js';
import { LocalModelManager } from './local-models.js';
export class EmbeddingService {
private modelManager: LocalModelManager;
private model: unknown = null;
private context: unknown = null;
private initialized = false;
private dimensions: number = 768;
constructor(modelManager?: LocalModelManager) {
this.modelManager = modelManager ?? new LocalModelManager();
}
async initialize(onProgress?: IndexBuildCallback): Promise<void> {
if (this.initialized) return;
onProgress?.({
phase: 'loading',
current: 0,
total: 1,
message: 'Loading embedding model...',
});
const modelPath = await this.modelManager.ensureEmbedModel(onProgress);
try {
// @ts-ignore - node-llama-cpp is an optional dependency
const llama = await import('node-llama-cpp');
const { getLlama } = llama;
const llamaInstance = await getLlama();
this.model = await llamaInstance.loadModel({ modelPath });
this.context = await (this.model as { createEmbeddingContext: () => Promise<unknown> }).createEmbeddingContext();
const modelInfo = MODEL_REGISTRY.embeddings[
this.modelManager.getConfig().embedModel as keyof typeof MODEL_REGISTRY.embeddings
];
this.dimensions = modelInfo?.dimensions ?? 768;
this.initialized = true;
onProgress?.({
phase: 'loading',
current: 1,
total: 1,
message: 'Embedding model loaded successfully',
});
} catch (error) {
if (
error instanceof Error &&
error.message.includes("Cannot find package 'node-llama-cpp'")
) {
throw new Error(
'node-llama-cpp is not installed. Install it with: pnpm add node-llama-cpp'
);
}
throw error;
}
}
async embed(text: string): Promise<Float32Array> {
if (!this.initialized || !this.context) {
throw new Error('EmbeddingService not initialized. Call initialize() first.');
}
const ctx = this.context as {
getEmbeddingFor(text: string): Promise<{ vector: number[] }>;
};
const result = await ctx.getEmbeddingFor(text);
return new Float32Array(result.vector);
}
async embedBatch(texts: string[]): Promise<Float32Array[]> {
const results: Float32Array[] = [];
for (const text of texts) {
results.push(await this.embed(text));
}
return results;
}
async embedSkill(skill: SkillSummary): Promise<SkillEmbedding> {
const textContent = this.buildSkillText(skill);
const vector = await this.embed(textContent);
return {
skillName: skill.name,
vector,
textContent,
generatedAt: new Date().toISOString(),
};
}
async embedSkillWithChunks(
skill: SkillSummary,
fullContent?: string
): Promise<SkillEmbedding> {
const textContent = this.buildSkillText(skill);
const mainVector = await this.embed(textContent);
const chunks = fullContent ? this.chunkText(fullContent) : undefined;
let embeddedChunks: SkillEmbedding['chunks'] | undefined;
if (chunks && chunks.length > 0) {
embeddedChunks = [];
for (const chunk of chunks) {
const chunkVector = await this.embed(chunk.content);
embeddedChunks.push({
content: chunk.content,
vector: chunkVector,
startLine: chunk.startLine,
endLine: chunk.endLine,
});
}
}
return {
skillName: skill.name,
vector: mainVector,
textContent,
chunks: embeddedChunks,
generatedAt: new Date().toISOString(),
};
}
async embedSkills(
skills: SkillSummary[],
onProgress?: IndexBuildCallback
): Promise<SkillEmbedding[]> {
await this.initialize(onProgress);
const embeddings: SkillEmbedding[] = [];
const total = skills.length;
for (let i = 0; i < skills.length; i++) {
const skill = skills[i];
onProgress?.({
phase: 'embedding',
current: i + 1,
total,
skillName: skill.name,
message: `Embedding skill ${i + 1}/${total}: ${skill.name}`,
});
const embedding = await this.embedSkill(skill);
embeddings.push(embedding);
}
onProgress?.({
phase: 'complete',
current: total,
total,
message: `Embedded ${total} skills`,
});
return embeddings;
}
private buildSkillText(skill: SkillSummary): string {
const parts: string[] = [skill.name];
if (skill.description) {
parts.push(skill.description);
}
if (skill.tags && skill.tags.length > 0) {
parts.push(skill.tags.join(' '));
}
if (skill.compatibility) {
if (skill.compatibility.frameworks?.length) {
parts.push(skill.compatibility.frameworks.join(' '));
}
if (skill.compatibility.languages?.length) {
parts.push(skill.compatibility.languages.join(' '));
}
if (skill.compatibility.libraries?.length) {
parts.push(skill.compatibility.libraries.join(' '));
}
}
return parts.join(' ').toLowerCase();
}
private chunkText(
text: string,
config = DEFAULT_CHUNKING_CONFIG
): { content: string; startLine: number; endLine: number }[] {
const lines = text.split('\n');
const chunks: { content: string; startLine: number; endLine: number }[] = [];
const avgCharsPerToken = 4;
const maxCharsPerChunk = config.maxTokens * avgCharsPerToken;
const overlapChars = Math.floor(maxCharsPerChunk * (config.overlapPercent / 100));
let currentChunk: string[] = [];
let currentChars = 0;
let startLine = 0;
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
const lineChars = line.length + 1;
if (currentChars + lineChars > maxCharsPerChunk && currentChunk.length > 0) {
const content = currentChunk.join('\n');
if (content.length >= config.minChunkSize) {
chunks.push({
content,
startLine,
endLine: i - 1,
});
}
const overlapLines: string[] = [];
let overlapCharsCount = 0;
for (let j = currentChunk.length - 1; j >= 0 && overlapCharsCount < overlapChars; j--) {
overlapLines.unshift(currentChunk[j]);
overlapCharsCount += currentChunk[j].length + 1;
}
currentChunk = overlapLines;
currentChars = overlapCharsCount;
startLine = Math.max(0, i - overlapLines.length);
}
currentChunk.push(line);
currentChars += lineChars;
}
if (currentChunk.length > 0) {
const content = currentChunk.join('\n');
if (content.length >= config.minChunkSize) {
chunks.push({
content,
startLine,
endLine: lines.length - 1,
});
}
}
return chunks;
}
cosineSimilarity(a: Float32Array, b: Float32Array): number {
if (a.length !== b.length) {
throw new Error(`Vector dimensions mismatch: ${a.length} vs ${b.length}`);
}
let dotProduct = 0;
let normA = 0;
let normB = 0;
for (let i = 0; i < a.length; i++) {
dotProduct += a[i] * b[i];
normA += a[i] * a[i];
normB += b[i] * b[i];
}
const magnitude = Math.sqrt(normA) * Math.sqrt(normB);
if (magnitude === 0) return 0;
return dotProduct / magnitude;
}
getDimensions(): number {
return this.dimensions;
}
isInitialized(): boolean {
return this.initialized;
}
async dispose(): Promise<void> {
if (this.context && typeof (this.context as { dispose?: () => void }).dispose === 'function') {
(this.context as { dispose: () => void }).dispose();
}
if (this.model && typeof (this.model as { dispose?: () => void }).dispose === 'function') {
(this.model as { dispose: () => void }).dispose();
}
this.context = null;
this.model = null;
this.initialized = false;
}
}
export function createEmbeddingService(modelManager?: LocalModelManager): EmbeddingService {
return new EmbeddingService(modelManager);
}
+236
View File
@@ -0,0 +1,236 @@
import type { ExpandedQuery, IndexBuildCallback } from './types.js';
import { DEFAULT_HYBRID_CONFIG } from './types.js';
import { LocalModelManager } from './local-models.js';
interface CacheEntry {
query: ExpandedQuery;
timestamp: number;
}
const expansionCache = new Map<string, CacheEntry>();
export class QueryExpander {
private modelManager: LocalModelManager;
private model: unknown = null;
private context: unknown = null;
private initialized = false;
private cacheTtlMs: number;
constructor(modelManager?: LocalModelManager, cacheTtlMs?: number) {
this.modelManager = modelManager ?? new LocalModelManager();
this.cacheTtlMs = cacheTtlMs ?? DEFAULT_HYBRID_CONFIG.expansionCacheTtlMs;
}
async initialize(onProgress?: IndexBuildCallback): Promise<void> {
if (this.initialized) return;
onProgress?.({
phase: 'loading',
current: 0,
total: 1,
message: 'Loading LLM model for query expansion...',
});
const modelPath = await this.modelManager.ensureLlmModel(onProgress);
try {
// @ts-ignore - node-llama-cpp is an optional dependency
const llamaModule = await import('node-llama-cpp');
const { getLlama, LlamaChatSession } = llamaModule;
const llama = await getLlama();
this.model = await llama.loadModel({ modelPath });
const ctx = await (this.model as { createContext: (opts?: { contextSize?: number }) => Promise<{ getSequence: () => unknown }> }).createContext({ contextSize: 2048 });
this.context = new LlamaChatSession({ contextSequence: ctx.getSequence() as never });
this.initialized = true;
onProgress?.({
phase: 'loading',
current: 1,
total: 1,
message: 'LLM model loaded successfully',
});
} catch (error) {
if (
error instanceof Error &&
error.message.includes("Cannot find package 'node-llama-cpp'")
) {
throw new Error(
'node-llama-cpp is not installed. Install it with: pnpm add node-llama-cpp'
);
}
throw error;
}
}
async expand(query: string, maxVariations: number = 3): Promise<ExpandedQuery> {
const cached = this.getFromCache(query);
if (cached) {
return cached;
}
if (!this.initialized) {
return this.expandWithoutLLM(query);
}
try {
const variations = await this.generateVariations(query, maxVariations);
const weights = [2.0, ...variations.map(() => 1.0)];
const expanded: ExpandedQuery = {
original: query,
variations,
weights,
};
this.addToCache(query, expanded);
return expanded;
} catch {
return this.expandWithoutLLM(query);
}
}
private async generateVariations(query: string, maxVariations: number): Promise<string[]> {
const session = this.context as {
prompt(text: string): Promise<string>;
};
const prompt = `Generate ${maxVariations} alternative search queries for finding software development skills related to: "${query}"
Rules:
- Each alternative should use different terminology but capture the same intent
- Keep each alternative concise (2-5 words)
- Focus on technical terms and synonyms
- Output only the alternatives, one per line, no numbering
Example:
Query: "auth for react"
Alternatives:
authentication react
login react hooks
OAuth react integration
Query: "${query}"
Alternatives:`;
const response = await session.prompt(prompt);
const variations = response
.split('\n')
.map((line) => line.trim())
.filter((line) => line.length > 0 && line.length < 100)
.filter((line) => !line.startsWith('-') && !line.match(/^\d+\./))
.slice(0, maxVariations);
return variations;
}
private expandWithoutLLM(query: string): ExpandedQuery {
const variations: string[] = [];
const queryLower = query.toLowerCase();
const synonyms: Record<string, string[]> = {
auth: ['authentication', 'login', 'oauth', 'jwt'],
authentication: ['auth', 'login', 'oauth'],
login: ['auth', 'authentication', 'signin'],
api: ['rest', 'endpoint', 'http', 'graphql'],
rest: ['api', 'http', 'endpoint'],
db: ['database', 'sql', 'storage'],
database: ['db', 'sql', 'data storage'],
test: ['testing', 'unit test', 'spec'],
testing: ['test', 'unit test', 'e2e'],
ui: ['user interface', 'frontend', 'component'],
frontend: ['ui', 'client', 'browser'],
backend: ['server', 'api', 'service'],
form: ['input', 'validation', 'submit'],
validation: ['form validation', 'input check', 'verify'],
state: ['state management', 'store', 'data flow'],
animation: ['motion', 'transition', 'animate'],
style: ['styling', 'css', 'design'],
css: ['style', 'styling', 'tailwind'],
};
const terms = queryLower.split(/\s+/);
for (const term of terms) {
const termSynonyms = synonyms[term];
if (termSynonyms) {
for (const syn of termSynonyms.slice(0, 2)) {
const variation = query.replace(new RegExp(term, 'i'), syn);
if (variation !== query && !variations.includes(variation)) {
variations.push(variation);
}
}
}
}
const limitedVariations = variations.slice(0, 3);
const weights = [2.0, ...limitedVariations.map(() => 1.0)];
return {
original: query,
variations: limitedVariations,
weights,
};
}
private getFromCache(query: string): ExpandedQuery | null {
const entry = expansionCache.get(query.toLowerCase());
if (!entry) return null;
const now = Date.now();
if (now - entry.timestamp > this.cacheTtlMs) {
expansionCache.delete(query.toLowerCase());
return null;
}
return entry.query;
}
private addToCache(query: string, expanded: ExpandedQuery): void {
const maxCacheSize = 100;
if (expansionCache.size >= maxCacheSize) {
const oldestKey = expansionCache.keys().next().value;
if (oldestKey) {
expansionCache.delete(oldestKey);
}
}
expansionCache.set(query.toLowerCase(), {
query: expanded,
timestamp: Date.now(),
});
}
clearCache(): void {
expansionCache.clear();
}
isInitialized(): boolean {
return this.initialized;
}
async dispose(): Promise<void> {
if (this.context && typeof (this.context as { dispose?: () => void }).dispose === 'function') {
(this.context as { dispose: () => void }).dispose();
}
if (this.model && typeof (this.model as { dispose?: () => void }).dispose === 'function') {
(this.model as { dispose: () => void }).dispose();
}
this.context = null;
this.model = null;
this.initialized = false;
}
}
export function createQueryExpander(
modelManager?: LocalModelManager,
cacheTtlMs?: number
): QueryExpander {
return new QueryExpander(modelManager, cacheTtlMs);
}
export function expandQuerySimple(query: string): ExpandedQuery {
const expander = new QueryExpander();
return expander['expandWithoutLLM'](query);
}
+384
View File
@@ -0,0 +1,384 @@
import type { SkillSummary, SkillIndex } from '../recommend/types.js';
import type {
HybridSearchOptions,
HybridSearchResult,
HybridSearchResponse,
ExpandedQuery,
IndexBuildCallback,
RerankerInput,
RerankerOutput,
LocalModelConfig,
} from './types.js';
import { DEFAULT_HYBRID_CONFIG } from './types.js';
import { EmbeddingService, createEmbeddingService } from './embeddings.js';
import { VectorStore, createVectorStore } from './vector-store.js';
import { QueryExpander, createQueryExpander } from './expansion.js';
import { LocalModelManager } from './local-models.js';
import {
fuseWithRRF,
applyPositionAwareBlending,
type RankerResult,
type RRFInput,
} from './rrf.js';
import { RecommendationEngine } from '../recommend/engine.js';
export class HybridSearchPipeline {
private embeddingService: EmbeddingService;
private vectorStore: VectorStore;
private queryExpander: QueryExpander;
private modelManager: LocalModelManager;
private recommendationEngine: RecommendationEngine;
private initialized = false;
private skillsMap: Map<string, SkillSummary> = new Map();
constructor(config: Partial<LocalModelConfig> = {}) {
this.modelManager = new LocalModelManager(config);
this.embeddingService = createEmbeddingService(this.modelManager);
this.vectorStore = createVectorStore(undefined, this.embeddingService);
this.queryExpander = createQueryExpander(this.modelManager);
this.recommendationEngine = new RecommendationEngine();
}
async initialize(onProgress?: IndexBuildCallback): Promise<void> {
if (this.initialized) return;
await this.vectorStore.initialize();
this.initialized = true;
onProgress?.({
phase: 'complete',
current: 1,
total: 1,
message: 'Hybrid search pipeline initialized',
});
}
async buildIndex(
skills: SkillSummary[],
onProgress?: IndexBuildCallback
): Promise<void> {
await this.initialize(onProgress);
this.skillsMap.clear();
for (const skill of skills) {
this.skillsMap.set(skill.name, skill);
}
const index: SkillIndex = {
version: 1,
lastUpdated: new Date().toISOString(),
skills,
sources: [],
};
this.recommendationEngine.loadIndex(index);
try {
await this.embeddingService.initialize(onProgress);
const embeddings = await this.embeddingService.embedSkills(skills, onProgress);
await this.vectorStore.storeBatch(embeddings, onProgress);
onProgress?.({
phase: 'complete',
current: skills.length,
total: skills.length,
message: `Indexed ${skills.length} skills with embeddings`,
});
} catch {
// Embeddings not available - will use keyword-only search
onProgress?.({
phase: 'complete',
current: skills.length,
total: skills.length,
message: `Indexed ${skills.length} skills (keyword-only, embeddings unavailable)`,
});
}
}
async search(options: HybridSearchOptions): Promise<HybridSearchResponse> {
const startTime = Date.now();
const timing: HybridSearchResponse['timing'] = { totalMs: 0 };
const stats: HybridSearchResponse['stats'] = {
candidatesFromVector: 0,
candidatesFromKeyword: 0,
totalMerged: 0,
reranked: 0,
};
await this.initialize();
const {
query,
limit = 10,
enableExpansion = false,
enableReranking = false,
rrfK = DEFAULT_HYBRID_CONFIG.rrfK,
positionAwareBlending = true,
} = options;
let expandedQuery: ExpandedQuery | undefined;
const searchQueries: string[] = [query];
if (enableExpansion) {
try {
expandedQuery = await this.queryExpander.expand(query);
searchQueries.push(...expandedQuery.variations);
} catch {
expandedQuery = {
original: query,
variations: [],
weights: [2.0],
};
}
}
const rankerInputs: RRFInput[] = [];
const embeddingStart = Date.now();
let queryVector: Float32Array | null = null;
try {
if (this.embeddingService.isInitialized()) {
queryVector = await this.embeddingService.embed(query);
}
} catch {
}
timing.embeddingMs = Date.now() - embeddingStart;
if (queryVector) {
const vectorStart = Date.now();
const vectorResults = await this.vectorStore.search(queryVector, limit * 3);
timing.vectorSearchMs = Date.now() - vectorStart;
const vectorRankerResults: RankerResult[] = vectorResults.map((r) => ({
skillName: r.skillName,
score: r.similarity,
}));
rankerInputs.push({
source: 'vector',
results: vectorRankerResults,
});
stats.candidatesFromVector = vectorResults.length;
}
const keywordStart = Date.now();
const keywordResults: RankerResult[] = [];
for (let i = 0; i < searchQueries.length; i++) {
const searchQuery = searchQueries[i];
const weight = expandedQuery?.weights[i] ?? 1;
const results = this.recommendationEngine.search({
query: searchQuery,
limit: limit * 3,
semantic: true,
});
for (const result of results) {
const existing = keywordResults.find(
(r) => r.skillName === result.skill.name
);
if (existing) {
existing.score = Math.max(existing.score, result.relevance * weight);
} else {
keywordResults.push({
skillName: result.skill.name,
score: result.relevance * weight,
});
}
}
}
timing.keywordSearchMs = Date.now() - keywordStart;
rankerInputs.push({
source: 'keyword',
results: keywordResults,
});
stats.candidatesFromKeyword = keywordResults.length;
const fusionStart = Date.now();
const rrfRankings = fuseWithRRF(rankerInputs, rrfK);
timing.fusionMs = Date.now() - fusionStart;
stats.totalMerged = rrfRankings.length;
let finalScores = new Map<string, number>();
for (const ranking of rrfRankings) {
finalScores.set(ranking.skillName, ranking.rrfScore);
}
if (enableReranking && rrfRankings.length > 0) {
const rerankStart = Date.now();
const topCandidates = rrfRankings.slice(0, DEFAULT_HYBRID_CONFIG.rerankTopN);
const rerankerInputs: RerankerInput[] = topCandidates
.map((r) => {
const skill = this.skillsMap.get(r.skillName);
if (!skill) return null;
return {
query,
skillName: r.skillName,
skillDescription: skill.description ?? '',
skillTags: skill.tags ?? [],
};
})
.filter((r): r is RerankerInput => r !== null);
const rerankerOutputs = await this.rerank(rerankerInputs);
timing.rerankingMs = Date.now() - rerankStart;
stats.reranked = rerankerOutputs.length;
if (positionAwareBlending && rerankerOutputs.length > 0) {
const retrievalScores = new Map<string, number>();
const maxRrfScore = Math.max(...rrfRankings.map((r) => r.rrfScore));
for (const ranking of topCandidates) {
retrievalScores.set(
ranking.skillName,
ranking.rrfScore / (maxRrfScore || 1)
);
}
const rerankerScores = new Map<string, number>();
const maxRerankerScore = Math.max(...rerankerOutputs.map((r) => r.score));
for (const output of rerankerOutputs) {
rerankerScores.set(
output.skillName,
output.score / (maxRerankerScore || 1)
);
}
const sortedSkillNames = topCandidates.map((r) => r.skillName);
const blendedScores = applyPositionAwareBlending(
retrievalScores,
rerankerScores,
sortedSkillNames
);
for (const [skillName, score] of blendedScores) {
finalScores.set(skillName, score);
}
}
}
const sortedResults = Array.from(finalScores.entries())
.sort((a, b) => b[1] - a[1])
.slice(0, limit);
const vectorInput = rankerInputs.find((r) => r.source === 'vector');
const keywordInput = rankerInputs.find((r) => r.source === 'keyword');
const hybridResults: HybridSearchResult[] = [];
for (const [skillName, hybridScore] of sortedResults) {
const skill = this.skillsMap.get(skillName);
if (!skill) continue;
hybridResults.push({
skill,
relevance: Math.round(hybridScore * 100),
hybridScore,
vectorSimilarity: vectorInput?.results.find((r) => r.skillName === skillName)?.score,
keywordScore: keywordInput?.results.find((r) => r.skillName === skillName)?.score,
rrfScore: rrfRankings.find((r) => r.skillName === skillName)?.rrfScore,
matchedTerms: [],
expandedTerms: expandedQuery?.variations,
});
}
timing.totalMs = Date.now() - startTime;
return {
results: hybridResults,
query: {
original: query,
expanded: expandedQuery,
},
timing,
stats,
};
}
private async rerank(inputs: RerankerInput[]): Promise<RerankerOutput[]> {
const outputs: RerankerOutput[] = [];
for (const input of inputs) {
const baseScore =
(input.skillDescription.toLowerCase().includes(input.query.toLowerCase())
? 0.5
: 0) +
(input.skillTags.some((t) =>
t.toLowerCase().includes(input.query.toLowerCase())
)
? 0.3
: 0) +
(input.skillName.toLowerCase().includes(input.query.toLowerCase())
? 0.2
: 0);
outputs.push({
skillName: input.skillName,
score: Math.min(1, baseScore + 0.3),
});
}
return outputs;
}
loadSkillsIndex(index: SkillIndex): void {
this.skillsMap.clear();
for (const skill of index.skills) {
this.skillsMap.set(skill.name, skill);
}
this.recommendationEngine.loadIndex(index);
}
getStats(): {
vectorStore: ReturnType<VectorStore['getStats']>;
initialized: boolean;
skillCount: number;
} {
return {
vectorStore: this.vectorStore.getStats(),
initialized: this.initialized,
skillCount: this.skillsMap.size,
};
}
isInitialized(): boolean {
return this.initialized;
}
async dispose(): Promise<void> {
await this.embeddingService.dispose();
await this.queryExpander.dispose();
await this.vectorStore.close();
this.initialized = false;
}
}
export function createHybridSearchPipeline(
config?: Partial<LocalModelConfig>
): HybridSearchPipeline {
return new HybridSearchPipeline(config);
}
export async function hybridSearch(
skills: SkillSummary[],
query: string,
options: Partial<HybridSearchOptions> = {}
): Promise<HybridSearchResponse> {
const pipeline = createHybridSearchPipeline();
try {
await pipeline.buildIndex(skills);
return await pipeline.search({
query,
...options,
});
} finally {
await pipeline.dispose();
}
}
+7
View File
@@ -0,0 +1,7 @@
export * from './types.js';
export * from './local-models.js';
export * from './embeddings.js';
export * from './vector-store.js';
export * from './rrf.js';
export * from './expansion.js';
export * from './hybrid.js';
+243
View File
@@ -0,0 +1,243 @@
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import { createWriteStream } from 'node:fs';
import type { LocalModelConfig, IndexBuildCallback } from './types.js';
import { MODEL_REGISTRY } from './types.js';
const DEFAULT_MODEL_DIR = path.join(os.homedir(), '.skillkit', 'models');
type EmbeddingModelName = keyof typeof MODEL_REGISTRY.embeddings;
type LlmModelName = keyof typeof MODEL_REGISTRY.llm;
interface ModelInfo {
url: string;
size: number;
description: string;
dimensions?: number;
}
export class LocalModelManager {
private config: Required<LocalModelConfig>;
private embedModelPath: string | null = null;
private llmModelPath: string | null = null;
constructor(config: Partial<LocalModelConfig> = {}) {
this.config = {
embedModel: config.embedModel ?? 'nomic-embed-text-v1.5.Q4_K_M.gguf',
llmModel: config.llmModel ?? 'gemma-2b-it-Q4_K_M.gguf',
modelDir: config.modelDir ?? DEFAULT_MODEL_DIR,
autoDownload: config.autoDownload ?? true,
gpuLayers: config.gpuLayers ?? 0,
};
}
async ensureModelsDirectory(): Promise<void> {
if (!fs.existsSync(this.config.modelDir)) {
await fs.promises.mkdir(this.config.modelDir, { recursive: true });
}
}
getEmbedModelPath(): string {
return path.join(this.config.modelDir, this.config.embedModel);
}
getLlmModelPath(): string {
return path.join(this.config.modelDir, this.config.llmModel);
}
isEmbedModelAvailable(): boolean {
return fs.existsSync(this.getEmbedModelPath());
}
isLlmModelAvailable(): boolean {
return fs.existsSync(this.getLlmModelPath());
}
private getModelInfo(modelType: 'embed' | 'llm', modelName: string): ModelInfo | undefined {
if (modelType === 'embed') {
return MODEL_REGISTRY.embeddings[modelName as EmbeddingModelName] as ModelInfo | undefined;
}
return MODEL_REGISTRY.llm[modelName as LlmModelName] as ModelInfo | undefined;
}
async downloadModel(
modelType: 'embed' | 'llm',
onProgress?: IndexBuildCallback
): Promise<string> {
await this.ensureModelsDirectory();
const modelName = modelType === 'embed' ? this.config.embedModel : this.config.llmModel;
const modelPath = modelType === 'embed' ? this.getEmbedModelPath() : this.getLlmModelPath();
if (fs.existsSync(modelPath)) {
return modelPath;
}
const modelInfo = this.getModelInfo(modelType, modelName);
if (!modelInfo) {
const availableModels = modelType === 'embed'
? Object.keys(MODEL_REGISTRY.embeddings)
: Object.keys(MODEL_REGISTRY.llm);
throw new Error(`Unknown model: ${modelName}. Available models: ${availableModels.join(', ')}`);
}
onProgress?.({
phase: 'downloading',
current: 0,
total: modelInfo.size,
message: `Downloading ${modelName}...`,
});
const response = await fetch(modelInfo.url);
if (!response.ok) {
throw new Error(`Failed to download model: ${response.statusText}`);
}
const contentLength = Number(response.headers.get('content-length')) || modelInfo.size;
let downloaded = 0;
const tempPath = `${modelPath}.tmp`;
const fileStream = createWriteStream(tempPath);
const reader = response.body?.getReader();
if (!reader) {
throw new Error('Failed to get response body reader');
}
try {
while (true) {
const { done, value } = await reader.read();
if (done) break;
const canContinue = fileStream.write(Buffer.from(value));
if (!canContinue) {
await new Promise<void>((resolve) => fileStream.once('drain', resolve));
}
downloaded += value.length;
onProgress?.({
phase: 'downloading',
current: downloaded,
total: contentLength,
message: `Downloading ${modelName}: ${Math.round((downloaded / contentLength) * 100)}%`,
});
}
fileStream.end();
await new Promise<void>((resolve, reject) => {
fileStream.on('finish', resolve);
fileStream.on('error', reject);
});
await fs.promises.rename(tempPath, modelPath);
return modelPath;
} catch (error) {
if (fs.existsSync(tempPath)) {
await fs.promises.unlink(tempPath);
}
throw error;
}
}
async ensureEmbedModel(onProgress?: IndexBuildCallback): Promise<string> {
if (this.embedModelPath && fs.existsSync(this.embedModelPath)) {
return this.embedModelPath;
}
if (!this.isEmbedModelAvailable()) {
if (!this.config.autoDownload) {
throw new Error(
`Embedding model not found at ${this.getEmbedModelPath()}. ` +
'Set autoDownload: true or manually download the model.'
);
}
this.embedModelPath = await this.downloadModel('embed', onProgress);
} else {
this.embedModelPath = this.getEmbedModelPath();
}
return this.embedModelPath;
}
async ensureLlmModel(onProgress?: IndexBuildCallback): Promise<string> {
if (this.llmModelPath && fs.existsSync(this.llmModelPath)) {
return this.llmModelPath;
}
if (!this.isLlmModelAvailable()) {
if (!this.config.autoDownload) {
throw new Error(
`LLM model not found at ${this.getLlmModelPath()}. ` +
'Set autoDownload: true or manually download the model.'
);
}
this.llmModelPath = await this.downloadModel('llm', onProgress);
} else {
this.llmModelPath = this.getLlmModelPath();
}
return this.llmModelPath;
}
async ensureAllModels(onProgress?: IndexBuildCallback): Promise<{
embedModel: string;
llmModel: string;
}> {
const embedModel = await this.ensureEmbedModel(onProgress);
const llmModel = await this.ensureLlmModel(onProgress);
return { embedModel, llmModel };
}
getPublicModelInfo(modelType: 'embed' | 'llm'): {
name: string;
path: string;
available: boolean;
size: number;
description: string;
} {
const modelName = modelType === 'embed' ? this.config.embedModel : this.config.llmModel;
const modelPath = modelType === 'embed' ? this.getEmbedModelPath() : this.getLlmModelPath();
const modelInfo = this.getModelInfo(modelType, modelName);
return {
name: modelName,
path: modelPath,
available: fs.existsSync(modelPath),
size: modelInfo?.size ?? 0,
description: modelInfo?.description ?? 'Unknown model',
};
}
async clearModels(): Promise<void> {
const embedPath = this.getEmbedModelPath();
const llmPath = this.getLlmModelPath();
if (fs.existsSync(embedPath)) {
await fs.promises.unlink(embedPath);
}
if (fs.existsSync(llmPath)) {
await fs.promises.unlink(llmPath);
}
this.embedModelPath = null;
this.llmModelPath = null;
}
getConfig(): Required<LocalModelConfig> {
return { ...this.config };
}
}
export function formatBytes(bytes: number): string {
if (bytes === 0) return '0 Bytes';
const k = 1024;
const sizes = ['Bytes', 'KB', 'MB', 'GB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return `${parseFloat((bytes / Math.pow(k, i)).toFixed(2))} ${sizes[i]}`;
}
export function getDefaultModelDir(): string {
return DEFAULT_MODEL_DIR;
}
+177
View File
@@ -0,0 +1,177 @@
import type { RRFRanking } from './types.js';
import { DEFAULT_HYBRID_CONFIG } from './types.js';
export interface RankerResult {
skillName: string;
score: number;
}
export interface RRFInput {
source: string;
results: RankerResult[];
}
export function computeRRFScore(ranks: number[], k: number = DEFAULT_HYBRID_CONFIG.rrfK): number {
return ranks.reduce((sum, rank) => sum + 1 / (k + rank), 0);
}
export function fuseWithRRF(
rankerInputs: RRFInput[],
k: number = DEFAULT_HYBRID_CONFIG.rrfK
): RRFRanking[] {
const skillRanks = new Map<string, { source: string; rank: number }[]>();
for (const input of rankerInputs) {
const sortedResults = [...input.results].sort((a, b) => b.score - a.score);
for (let i = 0; i < sortedResults.length; i++) {
const result = sortedResults[i];
const rank = i + 1;
if (!skillRanks.has(result.skillName)) {
skillRanks.set(result.skillName, []);
}
skillRanks.get(result.skillName)!.push({
source: input.source,
rank,
});
}
}
const rankings: RRFRanking[] = [];
for (const [skillName, ranks] of skillRanks) {
const rankValues = ranks.map((r) => r.rank);
const rrfScore = computeRRFScore(rankValues, k);
rankings.push({
skillName,
rrfScore,
ranks,
});
}
rankings.sort((a, b) => b.rrfScore - a.rrfScore);
return rankings;
}
export function normalizeScores(results: RankerResult[]): RankerResult[] {
if (results.length === 0) return [];
const maxScore = Math.max(...results.map((r) => r.score));
const minScore = Math.min(...results.map((r) => r.score));
const range = maxScore - minScore;
if (range === 0) {
return results.map((r) => ({ ...r, score: 1 }));
}
return results.map((r) => ({
...r,
score: (r.score - minScore) / range,
}));
}
export function weightedCombine(
results: RRFInput[],
weights: Record<string, number>
): RankerResult[] {
const combinedScores = new Map<string, number>();
for (const input of results) {
const weight = weights[input.source] ?? 1;
const normalizedResults = normalizeScores(input.results);
for (const result of normalizedResults) {
const current = combinedScores.get(result.skillName) ?? 0;
combinedScores.set(result.skillName, current + result.score * weight);
}
}
return Array.from(combinedScores, ([skillName, score]) => ({ skillName, score }))
.sort((a, b) => b.score - a.score);
}
export function applyPositionAwareBlending(
retrievalScores: Map<string, number>,
rerankerScores: Map<string, number>,
sortedSkillNames: string[],
config = DEFAULT_HYBRID_CONFIG.positionBlending
): Map<string, number> {
const blendedScores = new Map<string, number>();
for (let i = 0; i < sortedSkillNames.length; i++) {
const skillName = sortedSkillNames[i];
const rank = i + 1;
let weights: { retrieval: number; reranker: number };
if (rank <= 3) {
weights = config.top3;
} else if (rank <= 10) {
weights = config.top10;
} else {
weights = config.rest;
}
const retrievalScore = retrievalScores.get(skillName) ?? 0;
const rerankerScore = rerankerScores.get(skillName) ?? 0;
const blendedScore =
retrievalScore * weights.retrieval + rerankerScore * weights.reranker;
blendedScores.set(skillName, blendedScore);
}
return blendedScores;
}
export function getRankFromScore(
skillName: string,
rankedResults: RankerResult[]
): number {
const sortedResults = [...rankedResults].sort((a, b) => b.score - a.score);
const index = sortedResults.findIndex((r) => r.skillName === skillName);
return index === -1 ? sortedResults.length + 1 : index + 1;
}
export function mergeRankings(
primaryRankings: RRFRanking[],
secondaryRankings: RRFRanking[],
primaryWeight: number = 0.7
): RRFRanking[] {
const secondaryWeight = 1 - primaryWeight;
const mergedScores = new Map<string, { score: number; ranks: { source: string; rank: number }[] }>();
for (const ranking of primaryRankings) {
mergedScores.set(ranking.skillName, {
score: ranking.rrfScore * primaryWeight,
ranks: [...ranking.ranks],
});
}
for (const ranking of secondaryRankings) {
const existing = mergedScores.get(ranking.skillName);
if (existing) {
existing.score += ranking.rrfScore * secondaryWeight;
existing.ranks = [...existing.ranks, ...ranking.ranks];
} else {
mergedScores.set(ranking.skillName, {
score: ranking.rrfScore * secondaryWeight,
ranks: [...ranking.ranks],
});
}
}
const merged: RRFRanking[] = [];
for (const [skillName, data] of mergedScores) {
merged.push({
skillName,
rrfScore: data.score,
ranks: data.ranks,
});
}
merged.sort((a, b) => b.rrfScore - a.rrfScore);
return merged;
}
+177
View File
@@ -0,0 +1,177 @@
import { z } from 'zod';
import type { SearchResult } from '../recommend/types.js';
export const LocalModelConfigSchema = z.object({
embedModel: z.string().default('nomic-embed-text-v1.5.Q4_K_M.gguf'),
llmModel: z.string().default('gemma-2b-it-Q4_K_M.gguf'),
modelDir: z.string().optional(),
autoDownload: z.boolean().default(true),
gpuLayers: z.number().default(0),
});
export type LocalModelConfig = z.infer<typeof LocalModelConfigSchema>;
export const SkillChunkSchema = z.object({
content: z.string(),
startLine: z.number(),
endLine: z.number(),
tokenCount: z.number(),
});
export type SkillChunk = z.infer<typeof SkillChunkSchema>;
export interface SkillEmbedding {
skillName: string;
vector: Float32Array;
textContent: string;
chunks?: {
content: string;
vector: Float32Array;
startLine: number;
endLine: number;
}[];
generatedAt: string;
}
export interface VectorSearchResult {
skillName: string;
similarity: number;
matchedChunk?: {
content: string;
startLine: number;
};
}
export interface ExpandedQuery {
original: string;
variations: string[];
weights: number[];
}
export interface RRFRanking {
skillName: string;
rrfScore: number;
ranks: {
source: string;
rank: number;
}[];
}
export interface HybridSearchOptions {
query: string;
limit?: number;
enableExpansion?: boolean;
enableReranking?: boolean;
semanticWeight?: number;
keywordWeight?: number;
rrfK?: number;
positionAwareBlending?: boolean;
}
export interface HybridSearchResult extends SearchResult {
hybridScore: number;
vectorSimilarity?: number;
keywordScore?: number;
rrfScore?: number;
rerankerScore?: number;
expandedTerms?: string[];
blendingWeights?: {
retrieval: number;
reranker: number;
};
}
export interface HybridSearchResponse {
results: HybridSearchResult[];
query: {
original: string;
expanded?: ExpandedQuery;
};
timing: {
totalMs: number;
embeddingMs?: number;
vectorSearchMs?: number;
keywordSearchMs?: number;
fusionMs?: number;
rerankingMs?: number;
};
stats: {
candidatesFromVector: number;
candidatesFromKeyword: number;
totalMerged: number;
reranked: number;
};
}
export interface EmbeddingServiceStats {
totalSkillsIndexed: number;
totalChunks: number;
indexSizeBytes: number;
lastIndexedAt: string;
modelName: string;
embeddingDimensions: number;
}
export interface VectorStoreConfig {
dbPath: string;
tableName?: string;
dimensions?: number;
}
export interface IndexBuildProgress {
phase: 'downloading' | 'loading' | 'embedding' | 'storing' | 'complete';
current: number;
total: number;
skillName?: string;
message?: string;
}
export type IndexBuildCallback = (progress: IndexBuildProgress) => void;
export const MODEL_REGISTRY = {
embeddings: {
'nomic-embed-text-v1.5.Q4_K_M.gguf': {
url: 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.Q4_K_M.gguf',
size: 547000000,
dimensions: 768,
description: 'Nomic Embed Text v1.5 - High quality embeddings',
},
},
llm: {
'gemma-2b-it-Q4_K_M.gguf': {
url: 'https://huggingface.co/bartowski/gemma-2-2b-it-GGUF/resolve/main/gemma-2-2b-it-Q4_K_M.gguf',
size: 1500000000,
description: 'Gemma 2 2B Instruct - Fast query expansion and reranking',
},
},
} as const;
export const DEFAULT_CHUNKING_CONFIG = {
maxTokens: 800,
overlapPercent: 15,
minChunkSize: 100,
} as const;
export const DEFAULT_HYBRID_CONFIG = {
rrfK: 60,
semanticWeight: 0.5,
keywordWeight: 0.5,
rerankTopN: 30,
positionBlending: {
top3: { retrieval: 0.75, reranker: 0.25 },
top10: { retrieval: 0.6, reranker: 0.4 },
rest: { retrieval: 0.4, reranker: 0.6 },
},
expansionCacheTtlMs: 5 * 60 * 1000,
} as const;
export interface RerankerInput {
query: string;
skillName: string;
skillDescription: string;
skillTags: string[];
}
export interface RerankerOutput {
skillName: string;
score: number;
reasoning?: string;
}
+510
View File
@@ -0,0 +1,510 @@
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import type {
SkillEmbedding,
VectorSearchResult,
VectorStoreConfig,
EmbeddingServiceStats,
IndexBuildCallback,
} from './types.js';
import { EmbeddingService } from './embeddings.js';
const DEFAULT_DB_PATH = path.join(os.homedir(), '.skillkit', 'search.db');
const DEFAULT_TABLE_NAME = 'skill_embeddings';
const DEFAULT_DIMENSIONS = 768;
export class VectorStore {
private config: Required<VectorStoreConfig>;
private db: unknown = null;
private initialized = false;
private usingSqliteVec = false;
private embeddings: Map<string, SkillEmbedding> = new Map();
private embeddingService: EmbeddingService;
constructor(config: Partial<VectorStoreConfig> = {}, embeddingService?: EmbeddingService) {
const tableName = config.tableName ?? DEFAULT_TABLE_NAME;
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(tableName)) {
throw new Error(`Invalid table name: ${tableName}. Must be a valid SQL identifier.`);
}
this.config = {
dbPath: config.dbPath ?? DEFAULT_DB_PATH,
tableName,
dimensions: config.dimensions ?? DEFAULT_DIMENSIONS,
};
this.embeddingService = embeddingService ?? new EmbeddingService();
}
async initialize(): Promise<void> {
if (this.initialized) return;
const dbDir = path.dirname(this.config.dbPath);
if (!fs.existsSync(dbDir)) {
await fs.promises.mkdir(dbDir, { recursive: true });
}
try {
const BetterSqlite3Module = await import('better-sqlite3');
const BetterSqlite3 = BetterSqlite3Module.default || BetterSqlite3Module;
this.db = new BetterSqlite3(this.config.dbPath);
try {
const sqliteVec = await import('sqlite-vec');
sqliteVec.load(this.db as Parameters<typeof sqliteVec.load>[0]);
this.usingSqliteVec = true;
await this.initializeSqliteVecTables();
} catch {
this.usingSqliteVec = false;
await this.initializeFallbackTables();
}
await this.loadEmbeddingsFromDb();
this.initialized = true;
} catch (error) {
if (
error instanceof Error &&
(error.message.includes("Cannot find package 'better-sqlite3'") ||
error.message.includes("Cannot find module 'better-sqlite3'"))
) {
this.initialized = true;
this.usingSqliteVec = false;
return;
}
throw error;
}
}
private async initializeSqliteVecTables(): Promise<void> {
const db = this.db as {
exec(sql: string): void;
prepare(sql: string): { run(...args: unknown[]): void };
};
db.exec(`
CREATE TABLE IF NOT EXISTS ${this.config.tableName}_meta (
skill_name TEXT PRIMARY KEY,
text_content TEXT NOT NULL,
generated_at TEXT NOT NULL,
chunk_count INTEGER DEFAULT 0
)
`);
db.exec(`
CREATE VIRTUAL TABLE IF NOT EXISTS ${this.config.tableName}_vec USING vec0(
skill_name TEXT PRIMARY KEY,
embedding float[${this.config.dimensions}]
)
`);
db.exec(`
CREATE TABLE IF NOT EXISTS ${this.config.tableName}_chunks (
id INTEGER PRIMARY KEY AUTOINCREMENT,
skill_name TEXT NOT NULL,
content TEXT NOT NULL,
start_line INTEGER NOT NULL,
end_line INTEGER NOT NULL,
FOREIGN KEY (skill_name) REFERENCES ${this.config.tableName}_meta(skill_name)
)
`);
db.exec(`
CREATE VIRTUAL TABLE IF NOT EXISTS ${this.config.tableName}_chunks_vec USING vec0(
chunk_id INTEGER PRIMARY KEY,
embedding float[${this.config.dimensions}]
)
`);
}
private async initializeFallbackTables(): Promise<void> {
const db = this.db as { exec(sql: string): void };
db.exec(`
CREATE TABLE IF NOT EXISTS ${this.config.tableName} (
skill_name TEXT PRIMARY KEY,
text_content TEXT NOT NULL,
vector BLOB NOT NULL,
generated_at TEXT NOT NULL
)
`);
db.exec(`
CREATE TABLE IF NOT EXISTS ${this.config.tableName}_chunks (
id INTEGER PRIMARY KEY AUTOINCREMENT,
skill_name TEXT NOT NULL,
content TEXT NOT NULL,
vector BLOB NOT NULL,
start_line INTEGER NOT NULL,
end_line INTEGER NOT NULL
)
`);
db.exec(`
CREATE INDEX IF NOT EXISTS idx_${this.config.tableName}_skill
ON ${this.config.tableName}_chunks(skill_name)
`);
}
private async loadEmbeddingsFromDb(): Promise<void> {
if (!this.db) return;
const db = this.db as {
prepare(sql: string): { all(): Array<{
skill_name: string;
text_content: string;
vector?: Buffer;
generated_at: string;
}> };
};
try {
if (this.usingSqliteVec) {
const rows = db.prepare(`
SELECT skill_name, text_content, generated_at
FROM ${this.config.tableName}_meta
`).all();
for (const row of rows) {
this.embeddings.set(row.skill_name, {
skillName: row.skill_name,
textContent: row.text_content,
vector: new Float32Array(this.config.dimensions),
generatedAt: row.generated_at,
});
}
} else {
const rows = db.prepare(`
SELECT skill_name, text_content, vector, generated_at
FROM ${this.config.tableName}
`).all();
for (const row of rows) {
if (row.vector) {
this.embeddings.set(row.skill_name, {
skillName: row.skill_name,
textContent: row.text_content,
vector: new Float32Array(row.vector.buffer),
generatedAt: row.generated_at,
});
}
}
}
} catch {
// Failed to load embeddings from DB - will start with empty in-memory store
}
}
async store(embedding: SkillEmbedding): Promise<void> {
await this.initialize();
this.embeddings.set(embedding.skillName, embedding);
if (!this.db) return;
const db = this.db as {
prepare(sql: string): {
run(...args: unknown[]): { lastInsertRowid?: number };
};
};
if (this.usingSqliteVec) {
db.prepare(`
INSERT OR REPLACE INTO ${this.config.tableName}_meta
(skill_name, text_content, generated_at, chunk_count)
VALUES (?, ?, ?, ?)
`).run(
embedding.skillName,
embedding.textContent,
embedding.generatedAt,
embedding.chunks?.length ?? 0
);
db.prepare(`
INSERT OR REPLACE INTO ${this.config.tableName}_vec
(skill_name, embedding)
VALUES (?, ?)
`).run(embedding.skillName, Buffer.from(embedding.vector.buffer));
if (embedding.chunks) {
db.prepare(`
DELETE FROM ${this.config.tableName}_chunks WHERE skill_name = ?
`).run(embedding.skillName);
for (const chunk of embedding.chunks) {
const result = db.prepare(`
INSERT INTO ${this.config.tableName}_chunks
(skill_name, content, start_line, end_line)
VALUES (?, ?, ?, ?)
`).run(embedding.skillName, chunk.content, chunk.startLine, chunk.endLine);
const rowId = result.lastInsertRowid ?? 0;
db.prepare(`
INSERT INTO ${this.config.tableName}_chunks_vec
(chunk_id, embedding)
VALUES (?, ?)
`).run(rowId, Buffer.from(chunk.vector.buffer));
}
}
} else {
db.prepare(`
INSERT OR REPLACE INTO ${this.config.tableName}
(skill_name, text_content, vector, generated_at)
VALUES (?, ?, ?, ?)
`).run(
embedding.skillName,
embedding.textContent,
Buffer.from(embedding.vector.buffer),
embedding.generatedAt
);
if (embedding.chunks) {
db.prepare(`
DELETE FROM ${this.config.tableName}_chunks WHERE skill_name = ?
`).run(embedding.skillName);
for (const chunk of embedding.chunks) {
db.prepare(`
INSERT INTO ${this.config.tableName}_chunks
(skill_name, content, vector, start_line, end_line)
VALUES (?, ?, ?, ?, ?)
`).run(
embedding.skillName,
chunk.content,
Buffer.from(chunk.vector.buffer),
chunk.startLine,
chunk.endLine
);
}
}
}
}
async storeBatch(
embeddings: SkillEmbedding[],
onProgress?: IndexBuildCallback
): Promise<void> {
await this.initialize();
const total = embeddings.length;
for (let i = 0; i < embeddings.length; i++) {
const embedding = embeddings[i];
onProgress?.({
phase: 'storing',
current: i + 1,
total,
skillName: embedding.skillName,
message: `Storing embedding ${i + 1}/${total}: ${embedding.skillName}`,
});
await this.store(embedding);
}
onProgress?.({
phase: 'complete',
current: total,
total,
message: `Stored ${total} embeddings`,
});
}
async search(
queryVector: Float32Array,
limit: number = 10
): Promise<VectorSearchResult[]> {
await this.initialize();
if (this.usingSqliteVec && this.db) {
return this.searchWithSqliteVec(queryVector, limit);
}
return this.searchInMemory(queryVector, limit);
}
private async searchWithSqliteVec(
queryVector: Float32Array,
limit: number
): Promise<VectorSearchResult[]> {
const db = this.db as {
prepare(sql: string): {
all(...args: unknown[]): Array<{ skill_name: string; distance: number }>;
};
};
const rows = db.prepare(`
SELECT skill_name, distance
FROM ${this.config.tableName}_vec
WHERE embedding MATCH ?
ORDER BY distance
LIMIT ?
`).all(Buffer.from(queryVector.buffer), limit);
return rows.map((row) => ({
skillName: row.skill_name,
similarity: 1 - row.distance,
}));
}
private searchInMemory(
queryVector: Float32Array,
limit: number
): VectorSearchResult[] {
const results: VectorSearchResult[] = [];
for (const [skillName, embedding] of this.embeddings) {
const similarity = this.embeddingService.cosineSimilarity(queryVector, embedding.vector);
results.push({ skillName, similarity });
}
results.sort((a, b) => b.similarity - a.similarity);
return results.slice(0, limit);
}
async searchChunks(
queryVector: Float32Array,
limit: number = 10
): Promise<VectorSearchResult[]> {
await this.initialize();
const results: VectorSearchResult[] = [];
for (const [skillName, embedding] of this.embeddings) {
if (!embedding.chunks || embedding.chunks.length === 0) {
const similarity = this.embeddingService.cosineSimilarity(queryVector, embedding.vector);
results.push({ skillName, similarity });
continue;
}
let bestChunk: { similarity: number; content: string; startLine: number } | null = null;
for (const chunk of embedding.chunks) {
const similarity = this.embeddingService.cosineSimilarity(queryVector, chunk.vector);
if (!bestChunk || similarity > bestChunk.similarity) {
bestChunk = {
similarity,
content: chunk.content,
startLine: chunk.startLine,
};
}
}
if (bestChunk) {
results.push({
skillName,
similarity: bestChunk.similarity,
matchedChunk: {
content: bestChunk.content,
startLine: bestChunk.startLine,
},
});
}
}
results.sort((a, b) => b.similarity - a.similarity);
return results.slice(0, limit);
}
has(skillName: string): boolean {
return this.embeddings.has(skillName);
}
get(skillName: string): SkillEmbedding | undefined {
return this.embeddings.get(skillName);
}
async delete(skillName: string): Promise<void> {
await this.initialize();
this.embeddings.delete(skillName);
if (!this.db) return;
const db = this.db as {
prepare(sql: string): { run(...args: unknown[]): void };
};
if (this.usingSqliteVec) {
db.prepare(`DELETE FROM ${this.config.tableName}_meta WHERE skill_name = ?`).run(skillName);
db.prepare(`DELETE FROM ${this.config.tableName}_vec WHERE skill_name = ?`).run(skillName);
db.prepare(`
DELETE FROM ${this.config.tableName}_chunks_vec
WHERE chunk_id IN (
SELECT id FROM ${this.config.tableName}_chunks WHERE skill_name = ?
)
`).run(skillName);
} else {
db.prepare(`DELETE FROM ${this.config.tableName} WHERE skill_name = ?`).run(skillName);
}
db.prepare(`DELETE FROM ${this.config.tableName}_chunks WHERE skill_name = ?`).run(skillName);
}
async clear(): Promise<void> {
await this.initialize();
this.embeddings.clear();
if (!this.db) return;
const db = this.db as { exec(sql: string): void };
if (this.usingSqliteVec) {
db.exec(`DELETE FROM ${this.config.tableName}_meta`);
db.exec(`DELETE FROM ${this.config.tableName}_vec`);
db.exec(`DELETE FROM ${this.config.tableName}_chunks_vec`);
} else {
db.exec(`DELETE FROM ${this.config.tableName}`);
}
db.exec(`DELETE FROM ${this.config.tableName}_chunks`);
}
getStats(): EmbeddingServiceStats {
let indexSizeBytes = 0;
let totalChunks = 0;
for (const embedding of this.embeddings.values()) {
indexSizeBytes += embedding.vector.byteLength;
indexSizeBytes += embedding.textContent.length * 2;
if (embedding.chunks) {
totalChunks += embedding.chunks.length;
for (const chunk of embedding.chunks) {
indexSizeBytes += chunk.vector.byteLength;
indexSizeBytes += chunk.content.length * 2;
}
}
}
return {
totalSkillsIndexed: this.embeddings.size,
totalChunks,
indexSizeBytes,
lastIndexedAt: new Date().toISOString(),
modelName: 'nomic-embed-text-v1.5',
embeddingDimensions: this.config.dimensions,
};
}
isInitialized(): boolean {
return this.initialized;
}
isUsingSqliteVec(): boolean {
return this.usingSqliteVec;
}
getEmbeddingCount(): number {
return this.embeddings.size;
}
async close(): Promise<void> {
if (this.db && typeof (this.db as { close?: () => void }).close === 'function') {
(this.db as { close: () => void }).close();
}
this.db = null;
this.initialized = false;
}
}
export function createVectorStore(
config?: Partial<VectorStoreConfig>,
embeddingService?: EmbeddingService
): VectorStore {
return new VectorStore(config, embeddingService);
}
+1320
View File
File diff suppressed because it is too large Load Diff